diff --git a/.github/workflows/mvp-ci.yml b/.github/workflows/mvp-ci.yml index f7af955de..560dadb0e 100644 --- a/.github/workflows/mvp-ci.yml +++ b/.github/workflows/mvp-ci.yml @@ -20,7 +20,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 env: - # The full workspace links multiple DataFusion-heavy process test binaries. CARGO_PROFILE_DEV_DEBUG: "0" CARGO_PROFILE_TEST_DEBUG: "0" CARGO_INCREMENTAL: "0" @@ -33,9 +32,6 @@ jobs: - name: Install Rust components run: rustup component add rustfmt clippy - - name: Install protobuf compiler - run: sudo apt-get update && sudo apt-get install --yes protobuf-compiler - - name: Configure private Cargo dependencies env: ASAP_CI_REPO_TOKEN: ${{ secrets.ASAP_CI_REPO_TOKEN }} @@ -57,9 +53,7 @@ jobs: - name: Check workspace formatting working-directory: ASAPQuery-backend - # `--all` also follows this workspace's sibling path dependencies. - # Keep the gate scoped to every backend-owned workspace member. - run: cargo fmt -p control_plane -p data_plane -p asap_types -p asap_otel_proto -- --check + run: cargo fmt -p control_plane -p data_plane -- --check - name: Check workspace env: @@ -71,15 +65,10 @@ jobs: env: CARGO_NET_GIT_FETCH_WITH_CLI: "true" working-directory: ASAPQuery-backend - # Preserve the existing warning policy while expanding runtime coverage. run: cargo clippy --workspace --all-targets --locked -- -D warnings - name: Test workspace env: CARGO_NET_GIT_FETCH_WITH_CLI: "true" - # Give background flusher/sealer waits headroom on hosted runners. - ASAP_TEST_TIMEOUT_SCALE: "4" working-directory: ASAPQuery-backend - # Persistence tests wait on background flushers with bounded deadlines. - # Match scripts/e2e.sh to avoid competing flushers exhausting those waits. - run: cargo test --workspace --locked -- --test-threads=1 + run: cargo test --workspace --locked diff --git a/Cargo.lock b/Cargo.lock index 471b047ab..cac83d6d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,26 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "const-random", - "getrandom 0.3.4", - "once_cell", - "version_check", - "zerocopy", -] - [[package]] name = "aho-corasick" version = "1.1.5" @@ -31,27 +11,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" -dependencies = [ - "alloc-no-stdlib", -] - -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - [[package]] name = "android-tzdata" version = "0.1.1" @@ -67,12 +26,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - [[package]] name = "anstream" version = "1.0.0" @@ -130,323 +83,82 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] -name = "arc-swap" -version = "1.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" -dependencies = [ - "rustversion", -] - -[[package]] -name = "arrayvec" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" - -[[package]] -name = "arrow" -version = "53.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a3ec4fe573f9d1f59d99c085197ef669b00b088ba1d7bb75224732d9357a74" -dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-csv", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", -] - -[[package]] -name = "arrow-arith" -version = "53.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dcf19f07792d8c7f91086c67b574a79301e367029b17fcf63fb854332246a10" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "chrono", - "half", - "num", -] - -[[package]] -name = "arrow-array" -version = "53.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7845c32b41f7053e37a075b3c2f29c6f5ea1b3ca6e5df7a2d325ee6e1b4a63cf" -dependencies = [ - "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "chrono", - "chrono-tz", - "half", - "hashbrown 0.15.5", - "num", -] - -[[package]] -name = "arrow-buffer" -version = "53.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b5c681a99606f3316f2a99d9c8b6fa3aad0b1d34d8f6d7a1b471893940219d8" -dependencies = [ - "bytes", - "half", - "num", -] - -[[package]] -name = "arrow-cast" -version = "53.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365f8527d4f87b133eeb862f9b8093c009d41a210b8f101f91aa2392f61daac" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "atoi", - "base64 0.22.1", - "chrono", - "comfy-table", - "half", - "lexical-core", - "num", - "ryu", -] - -[[package]] -name = "arrow-csv" -version = "53.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30dac4d23ac769300349197b845e0fd18c7f9f15d260d4659ae6b5a9ca06f586" +name = "asap-executor" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=b517fbf256a17621d48fa3f8508c5d5ca1017bad#b517fbf256a17621d48fa3f8508c5d5ca1017bad" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "asap-types", + "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?rev=5f03ccbd798ed5fec62bdd839bcb331123cab369)", "chrono", - "csv", - "csv-core", - "lazy_static", - "lexical-core", + "futures", "regex", -] - -[[package]] -name = "arrow-data" -version = "53.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd962fc3bf7f60705b25bcaa8eb3318b2545aa1d528656525ebdd6a17a6cd6fb" -dependencies = [ - "arrow-buffer", - "arrow-schema", - "half", - "num", -] - -[[package]] -name = "arrow-ipc" -version = "53.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3527365b24372f9c948f16e53738eb098720eea2093ae73c7af04ac5e30a39b" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", - "flatbuffers", - "lz4_flex", -] - -[[package]] -name = "arrow-json" -version = "53.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdec0024749fc0d95e025c0b0266d78613727b3b3a5d4cf8ea47eb6d38afdd1" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", - "chrono", - "half", - "indexmap 2.14.2", - "lexical-core", - "num", "serde", "serde_json", + "thiserror 2.0.20", + "tracing", ] [[package]] -name = "arrow-ord" -version = "53.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79af2db0e62a508d34ddf4f76bfd6109b6ecc845257c9cba6f939653668f89ac" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "half", - "num", -] - -[[package]] -name = "arrow-row" -version = "53.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da30e9d10e9c52f09ea0cf15086d6d785c11ae8dcc3ea5f16d402221b6ac7735" -dependencies = [ - "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "half", -] - -[[package]] -name = "arrow-schema" -version = "53.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35b0f9c0c3582dd55db0f136d3b44bfa0189df07adcf7dc7f2f2e74db0f52eb8" - -[[package]] -name = "arrow-select" -version = "53.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92fc337f01635218493c23da81a364daf38c694b05fc20569c3193c11c561984" -dependencies = [ - "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "num", -] - -[[package]] -name = "arrow-string" -version = "53.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d596a9fc25dae556672d5069b090331aca8acb93cae426d8b7dcdf1c558fa0ce" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "memchr", - "num", - "regex", - "regex-syntax", -] - -[[package]] -name = "asap-aware-mapping" +name = "asap-frontend-common" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=4b0839ce1733aab8231eb90944c876063a4551f9#4b0839ce1733aab8231eb90944c876063a4551f9" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=b517fbf256a17621d48fa3f8508c5d5ca1017bad#b517fbf256a17621d48fa3f8508c5d5ca1017bad" dependencies = [ "asap-types", - "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib)", "serde", - "serde_json", "thiserror 2.0.20", ] [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=4b0839ce1733aab8231eb90944c876063a4551f9#4b0839ce1733aab8231eb90944c876063a4551f9" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=b517fbf256a17621d48fa3f8508c5d5ca1017bad#b517fbf256a17621d48fa3f8508c5d5ca1017bad" dependencies = [ + "asap-frontend-common", "asap-types", - "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser?rev=9fede7eecca923c9882fe256484d00d37f8706cb)", + "promql-parser", ] [[package]] -name = "asap-frontend-sql" +name = "asap-logical-optimizer" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=4b0839ce1733aab8231eb90944c876063a4551f9#4b0839ce1733aab8231eb90944c876063a4551f9" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=b517fbf256a17621d48fa3f8508c5d5ca1017bad#b517fbf256a17621d48fa3f8508c5d5ca1017bad" dependencies = [ - "asap-sql-function-catalog", "asap-types", - "datafusion", + "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib)", "serde_json", + "thiserror 2.0.20", ] [[package]] -name = "asap-physical-operators" +name = "asap-physical-optimizer" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=4b0839ce1733aab8231eb90944c876063a4551f9#4b0839ce1733aab8231eb90944c876063a4551f9" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=b517fbf256a17621d48fa3f8508c5d5ca1017bad#b517fbf256a17621d48fa3f8508c5d5ca1017bad" dependencies = [ "asap-types", - "asap_sketch_codec", - "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?rev=5f03ccbd798ed5fec62bdd839bcb331123cab369)", - "base64 0.21.7", - "bincode", - "futures", - "prost", - "rmp-serde", - "serde", - "serde_json", "thiserror 2.0.20", - "tracing", - "xxhash-rust", ] [[package]] -name = "asap-sql-function-catalog" -version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=4b0839ce1733aab8231eb90944c876063a4551f9#4b0839ce1733aab8231eb90944c876063a4551f9" - -[[package]] -name = "asap-types" +name = "asap-plan-selection" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=4b0839ce1733aab8231eb90944c876063a4551f9#4b0839ce1733aab8231eb90944c876063a4551f9" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=b517fbf256a17621d48fa3f8508c5d5ca1017bad#b517fbf256a17621d48fa3f8508c5d5ca1017bad" dependencies = [ + "asap-logical-optimizer", + "asap-physical-optimizer", + "asap-types", "serde", "serde_json", - "sha2", "thiserror 2.0.20", ] [[package]] -name = "asap_otel_proto" -version = "0.1.0" -dependencies = [ - "prost", - "prost-build", - "protoc-bin-vendored", - "tonic", - "tonic-build", -] - -[[package]] -name = "asap_sketch_codec" +name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=4b0839ce1733aab8231eb90944c876063a4551f9#4b0839ce1733aab8231eb90944c876063a4551f9" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=b517fbf256a17621d48fa3f8508c5d5ca1017bad#b517fbf256a17621d48fa3f8508c5d5ca1017bad" dependencies = [ - "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?rev=5f03ccbd798ed5fec62bdd839bcb331123cab369)", - "prost", + "serde", + "serde_json", + "thiserror 2.0.20", ] [[package]] @@ -463,7 +175,7 @@ dependencies = [ "serde-big-array", "serde_bytes", "smallvec", - "twox-hash 2.1.4", + "twox-hash", "xxhash-rust", ] @@ -480,70 +192,10 @@ dependencies = [ "serde-big-array", "serde_bytes", "smallvec", - "twox-hash 2.1.4", - "xxhash-rust", -] - -[[package]] -name = "asap_types" -version = "0.1.0" -dependencies = [ - "anyhow", - "asap-aware-mapping", - "asap-physical-operators", - "asap-types", - "base64 0.21.7", - "clap", - "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser)", - "serde", - "serde_json", - "serde_yaml", - "sha2", - "thiserror 1.0.69", - "tracing", + "twox-hash", "xxhash-rust", ] -[[package]] -name = "async-compression" -version = "0.4.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" -dependencies = [ - "bzip2 0.5.2", - "flate2", - "futures-core", - "futures-io", - "memchr", - "pin-project-lite", - "tokio", - "xz2", - "zstd", - "zstd-safe", -] - -[[package]] -name = "async-stream" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "async-trait" version = "0.1.92" @@ -555,15 +207,6 @@ dependencies = [ "syn 3.0.5", ] -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", -] - [[package]] name = "atomic-waker" version = "1.1.2" @@ -584,7 +227,6 @@ checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", "axum-core", - "base64 0.22.1", "bytes", "futures-util", "http", @@ -603,11 +245,9 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sha1", "sync_wrapper", "tokio", - "tokio-tungstenite 0.24.0", - "tower 0.5.3", + "tower", "tower-layer", "tower-service", "tracing", @@ -634,12 +274,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -655,129 +289,30 @@ dependencies = [ "serde", ] -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" -[[package]] -name = "blake2" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" -dependencies = [ - "digest", -] - -[[package]] -name = "blake3" -version = "1.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" -dependencies = [ - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", - "cpufeatures 0.3.1", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "brotli" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "4.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a334ef7c9e23abf0ce748e8cd309037da93e606ad52eb372e4ce327a0dcfbdfd" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - [[package]] name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "cactus" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbc26382d871df4b7442e3df10a9402bf3cf5e55cbd66f12be38861425f0564" -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "cc" version = "1.4.5" @@ -785,8 +320,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex", ] @@ -808,7 +341,7 @@ version = "0.13.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fe45e18904af7af10e4312df7c97251e98af98c70f42f1f2587aecfcbee56bf" dependencies = [ - "indexmap 2.14.2", + "indexmap", "lazy_static", "num-traits", "regex", @@ -823,7 +356,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures", "rand_core 0.10.1", ] @@ -837,51 +370,13 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", - "serde", "wasm-bindgen", "windows-targets", ] [[package]] -name = "chrono-tz" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" -dependencies = [ - "chrono", - "phf", -] - -[[package]] -name = "ciborium" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - -[[package]] -name = "clap" -version = "4.6.6" +name = "clap" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ @@ -925,75 +420,19 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" -[[package]] -name = "comfy-table" -version = "7.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" -dependencies = [ - "unicode-segmentation", - "unicode-width", -] - -[[package]] -name = "const-random" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" -dependencies = [ - "const-random-macro", -] - -[[package]] -name = "const-random-macro" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" -dependencies = [ - "getrandom 0.2.17", - "once_cell", - "tiny-keccak", -] - -[[package]] -name = "constant_time_eq" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" - [[package]] name = "control_plane" version = "0.1.0" dependencies = [ "anyhow", - "asap-aware-mapping", + "asap-executor", "asap-frontend-promql", - "asap-frontend-sql", - "asap-physical-operators", + "asap-plan-selection", "asap-types", - "asap_types", - "axum", - "bytes", - "chrono", - "futures-util", - "http-body-util", - "parking_lot", - "prometheus", - "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser)", - "prost", - "prost-build", - "reqwest", + "clap", "serde", "serde_json", - "serde_yaml", - "sha2", "thiserror 1.0.69", - "tokio", - "tokio-tungstenite 0.21.0", - "toml", - "tonic", - "tonic-build", - "tower 0.4.13", "tracing", "tracing-subscriber", ] @@ -1004,15 +443,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - [[package]] name = "cpufeatures" version = "0.3.1" @@ -1022,616 +452,26 @@ dependencies = [ "libc", ] -[[package]] -name = "crc32fast" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "criterion" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" -dependencies = [ - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "is-terminal", - "itertools 0.10.5", - "num-traits", - "once_cell", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast", - "itertools 0.10.5", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" - -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "csv" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" -dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde_core", -] - -[[package]] -name = "csv-core" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" -dependencies = [ - "memchr", -] - -[[package]] -name = "dashmap" -version = "5.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" -dependencies = [ - "cfg-if", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - -[[package]] -name = "dashmap" -version = "6.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" -dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - -[[package]] -name = "data-encoding" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" - [[package]] name = "data_plane" version = "0.1.0" dependencies = [ "anyhow", - "arc-swap", - "arrow", - "asap-aware-mapping", - "asap-frontend-promql", - "asap-physical-operators", "asap-types", - "asap_otel_proto", - "asap_sketch_codec", - "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?rev=5f03ccbd798ed5fec62bdd839bcb331123cab369)", - "asap_types", - "async-trait", "axum", - "base64 0.21.7", - "bincode", - "chrono", - "chrono-tz", "clap", "control_plane", - "crc32fast", - "criterion", - "dashmap 5.5.3", - "flate2", - "form_urlencoded", - "fs2", - "futures", - "hex", "http-body-util", - "lazy_static", - "memmap2", - "moka", - "prometheus", - "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser)", "prost", - "regex", "reqwest", - "rmp-serde", "serde", "serde_json", - "serde_yaml", - "sha2", "snap", - "tempfile", "thiserror 1.0.69", "tokio", - "tokio-stream", - "tonic", - "tower 0.5.3", + "tower", "tracing", - "tracing-appender", "tracing-subscriber", - "xxhash-rust", -] - -[[package]] -name = "datafusion" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbba0799cf6913b456ed07a94f0f3b6e12c62a5d88b10809e2284a0f2b915c05" -dependencies = [ - "ahash", - "arrow", - "arrow-array", - "arrow-ipc", - "arrow-schema", - "async-compression", - "async-trait", - "bytes", - "bzip2 0.4.4", - "chrono", - "dashmap 6.2.1", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", - "datafusion-functions-window", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-sql", - "flate2", - "futures", - "glob", - "half", - "hashbrown 0.14.5", - "indexmap 2.14.2", - "itertools 0.13.0", - "log", - "num_cpus", - "object_store", - "parking_lot", - "parquet", - "paste", - "pin-project-lite", - "rand 0.8.8", - "sqlparser", - "tempfile", - "tokio", - "tokio-util", - "url", - "uuid", - "xz2", - "zstd", -] - -[[package]] -name = "datafusion-catalog" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7493c5c2d40eec435b13d92e5703554f4efc7059451fcb8d3a79580ff0e45560" -dependencies = [ - "arrow-schema", - "async-trait", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-plan", - "parking_lot", -] - -[[package]] -name = "datafusion-common" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24953049ebbd6f8964f91f60aa3514e121b5e81e068e33b60e77815ab369b25c" -dependencies = [ - "ahash", - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-schema", - "chrono", - "half", - "hashbrown 0.14.5", - "indexmap 2.14.2", - "instant", - "libc", - "num_cpus", - "object_store", - "parquet", - "paste", - "sqlparser", - "tokio", -] - -[[package]] -name = "datafusion-common-runtime" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f06df4ef76872e11c924d3c814fd2a8dd09905ed2e2195f71c857d78abd19685" -dependencies = [ - "log", - "tokio", -] - -[[package]] -name = "datafusion-execution" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbdcb628d690f3ce5fea7de81642b514486d58ff9779a51f180a69a4eadb361" -dependencies = [ - "arrow", - "chrono", - "dashmap 6.2.1", - "datafusion-common", - "datafusion-expr", - "futures", - "hashbrown 0.14.5", - "log", - "object_store", - "parking_lot", - "rand 0.8.8", - "tempfile", - "url", -] - -[[package]] -name = "datafusion-expr" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8036495980e3131f706b7d33ab00b4492d73dc714e3cb74d11b50f9602a73246" -dependencies = [ - "ahash", - "arrow", - "arrow-array", - "arrow-buffer", - "chrono", - "datafusion-common", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", - "indexmap 2.14.2", - "paste", - "serde_json", - "sqlparser", - "strum", - "strum_macros", -] - -[[package]] -name = "datafusion-expr-common" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4da0f3cb4669f9523b403d6b5a0ec85023e0ab3bf0183afd1517475b3e64fdd2" -dependencies = [ - "arrow", - "datafusion-common", - "itertools 0.13.0", - "paste", -] - -[[package]] -name = "datafusion-functions" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52c4012648b34853e40a2c6bcaa8772f837831019b68aca384fb38436dba162" -dependencies = [ - "arrow", - "arrow-buffer", - "base64 0.22.1", - "blake2", - "blake3", - "chrono", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "hashbrown 0.14.5", - "hex", - "itertools 0.13.0", - "log", - "md-5", - "rand 0.8.8", - "regex", - "sha2", - "unicode-segmentation", - "uuid", -] - -[[package]] -name = "datafusion-functions-aggregate" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5b8bb624597ba28ed7446df4a9bd7c7a7bde7c578b6b527da3f47371d5f6741" -dependencies = [ - "ahash", - "arrow", - "arrow-schema", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "half", - "indexmap 2.14.2", - "log", - "paste", -] - -[[package]] -name = "datafusion-functions-aggregate-common" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb06208fc470bc8cf1ce2d9a1159d42db591f2c7264a8c1776b53ad8f675143" -dependencies = [ - "ahash", - "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", - "rand 0.8.8", -] - -[[package]] -name = "datafusion-functions-nested" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca25bbb87323716d05e54114666e942172ccca23c5a507e9c7851db6e965317" -dependencies = [ - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-ord", - "arrow-schema", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-physical-expr-common", - "itertools 0.13.0", - "log", - "paste", - "rand 0.8.8", -] - -[[package]] -name = "datafusion-functions-window" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ae23356c634e54c59f7c51acb7a5b9f6240ffb2cf997049a1a24a8a88598dbe" -dependencies = [ - "datafusion-common", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "log", - "paste", -] - -[[package]] -name = "datafusion-functions-window-common" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4b3d6ff7794acea026de36007077a06b18b89e4f9c3fea7f2215f9f7dd9059b" -dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", -] - -[[package]] -name = "datafusion-optimizer" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec6241eb80c595fa0e1a8a6b69686b5cf3bd5fdacb8319582a0943b0bd788aa" -dependencies = [ - "arrow", - "async-trait", - "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", - "hashbrown 0.14.5", - "indexmap 2.14.2", - "itertools 0.13.0", - "log", - "paste", - "regex-syntax", -] - -[[package]] -name = "datafusion-physical-expr" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3370357b8fc75ec38577700644e5d1b0bc78f38babab99c0b8bd26bafb3e4335" -dependencies = [ - "ahash", - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-ord", - "arrow-schema", - "arrow-string", - "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", - "half", - "hashbrown 0.14.5", - "indexmap 2.14.2", - "itertools 0.13.0", - "log", - "paste", - "petgraph 0.6.5", -] - -[[package]] -name = "datafusion-physical-expr-common" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b7734d94bf2fa6f6e570935b0ddddd8421179ce200065be97874e13d46a47b" -dependencies = [ - "ahash", - "arrow", - "datafusion-common", - "datafusion-expr-common", - "hashbrown 0.14.5", - "rand 0.8.8", -] - -[[package]] -name = "datafusion-physical-optimizer" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eee8c479522df21d7b395640dff88c5ed05361852dce6544d7c98e9dbcebffe" -dependencies = [ - "arrow", - "arrow-schema", - "datafusion-common", - "datafusion-execution", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-plan", - "itertools 0.13.0", -] - -[[package]] -name = "datafusion-physical-plan" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e1fc2e2c239d14e8556f2622b19a726bf6bc6962cc00c71fc52626274bee24" -dependencies = [ - "ahash", - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-ord", - "arrow-schema", - "async-trait", - "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "futures", - "half", - "hashbrown 0.14.5", - "indexmap 2.14.2", - "itertools 0.13.0", - "log", - "once_cell", - "parking_lot", - "pin-project-lite", - "rand 0.8.8", - "tokio", -] - -[[package]] -name = "datafusion-sql" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e3a4ed41dbee20a5d947a59ca035c225d67dc9cbe869c10f66dcdf25e7ce51" -dependencies = [ - "arrow", - "arrow-array", - "arrow-schema", - "datafusion-common", - "datafusion-expr", - "indexmap 2.14.2", - "log", - "regex", - "sqlparser", - "strum", ] [[package]] @@ -1640,17 +480,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", - "subtle", -] - [[package]] name = "displaydoc" version = "0.2.7" @@ -1684,12 +513,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "fastrand" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" - [[package]] name = "filetime" version = "0.2.29" @@ -1706,39 +529,6 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - -[[package]] -name = "flatbuffers" -version = "24.12.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" -dependencies = [ - "bitflags 1.3.2", - "rustc_version", -] - -[[package]] -name = "flate2" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" -dependencies = [ - "crc32fast", - "miniz_oxide", - "zlib-rs", -] - [[package]] name = "fnv" version = "1.0.7" @@ -1754,16 +544,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "futures" version = "0.3.34" @@ -1852,16 +632,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getopts" version = "0.2.24" @@ -1910,65 +680,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "glob" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" - -[[package]] -name = "h2" -version = "0.4.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap 2.14.2", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "cfg-if", - "crunchy", - "num-traits", - "zerocopy", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", - "allocator-api2", -] - -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" - [[package]] name = "hashbrown" version = "0.17.1" @@ -1981,18 +692,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - [[package]] name = "http" version = "1.5.0" @@ -2038,12 +737,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "humantime" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" - [[package]] name = "hyper" version = "1.11.1" @@ -2054,7 +747,6 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", "http", "http-body", "httparse", @@ -2082,26 +774,13 @@ dependencies = [ "webpki-roots", ] -[[package]] -name = "hyper-timeout" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" -dependencies = [ - "hyper", - "hyper-util", - "pin-project-lite", - "tokio", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-util", @@ -2242,18 +921,8 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", + "icu_normalizer", + "icu_properties", ] [[package]] @@ -2263,44 +932,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", - "hashbrown 0.17.1", -] - -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", + "hashbrown", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "ipnet" version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" -[[package]] -name = "is-terminal" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.59.0", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -2316,31 +956,12 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jobserver" -version = "0.1.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" -dependencies = [ - "getrandom 0.4.3", - "libc", -] - [[package]] name = "js-sys" version = "0.3.105" @@ -2358,87 +979,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "lexical-core" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" -dependencies = [ - "lexical-parse-float", - "lexical-parse-integer", - "lexical-util", - "lexical-write-float", - "lexical-write-integer", -] - -[[package]] -name = "lexical-parse-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" -dependencies = [ - "lexical-parse-integer", - "lexical-util", -] - -[[package]] -name = "lexical-parse-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" -dependencies = [ - "lexical-util", -] - -[[package]] -name = "lexical-util" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" - -[[package]] -name = "lexical-write-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" -dependencies = [ - "lexical-util", - "lexical-write-integer", -] - -[[package]] -name = "lexical-write-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" -dependencies = [ - "lexical-util", -] - [[package]] name = "libc" version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" -[[package]] -name = "libm" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" - -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - -[[package]] -name = "linux-raw-sys" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" - [[package]] name = "litemap" version = "0.8.3" @@ -2488,7 +1034,7 @@ dependencies = [ "cactus", "cfgrammar", "filetime", - "indexmap 2.14.2", + "indexmap", "lazy_static", "lrtable", "num-traits", @@ -2520,26 +1066,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" -[[package]] -name = "lz4_flex" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" -dependencies = [ - "twox-hash 2.1.4", -] - -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "matchers" version = "0.2.0" @@ -2555,47 +1081,18 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest", -] - [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" -[[package]] -name = "memmap2" -version = "0.9.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" -dependencies = [ - "libc", -] - [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "miniz_oxide" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" -dependencies = [ - "adler2", - "simd-adler32", -] - [[package]] name = "mio" version = "1.2.3" @@ -2607,29 +1104,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "moka" -version = "0.12.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" -dependencies = [ - "crossbeam-channel", - "crossbeam-epoch", - "crossbeam-utils", - "equivalent", - "parking_lot", - "portable-atomic", - "smallvec", - "tagptr", - "uuid", -] - -[[package]] -name = "multimap" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2639,75 +1113,12 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - -[[package]] -name = "num-bigint" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" -[[package]] -name = "num-integer" -version = "0.1.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -2715,17 +1126,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", -] - -[[package]] -name = "num_cpus" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" -dependencies = [ - "hermit-abi", - "libc", ] [[package]] @@ -2737,27 +1137,6 @@ dependencies = [ "libc", ] -[[package]] -name = "object_store" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf" -dependencies = [ - "async-trait", - "bytes", - "chrono", - "futures", - "humantime", - "itertools 0.13.0", - "parking_lot", - "percent-encoding", - "snafu", - "tokio", - "tracing", - "url", - "walkdir", -] - [[package]] name = "once_cell" version = "1.21.4" @@ -2770,21 +1149,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "oorandom" -version = "11.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "packedvec" version = "1.2.5" @@ -2815,160 +1179,20 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-link", -] - -[[package]] -name = "parquet" -version = "53.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f8cf58b29782a7add991f655ff42929e31a7859f5319e53db9e39a714cb113c" -dependencies = [ - "ahash", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", - "base64 0.22.1", - "brotli", - "bytes", - "chrono", - "flate2", - "futures", - "half", - "hashbrown 0.15.5", - "lz4_flex", - "num", - "num-bigint", - "object_store", - "paste", - "seq-macro", - "snap", - "thrift", - "tokio", - "twox-hash 1.6.3", - "zstd", - "zstd-sys", -] - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "petgraph" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" -dependencies = [ - "fixedbitset 0.4.2", - "indexmap 2.14.2", -] - -[[package]] -name = "petgraph" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" -dependencies = [ - "fixedbitset 0.5.7", - "indexmap 2.14.2", -] - -[[package]] -name = "phf" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_shared" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" -dependencies = [ - "siphasher", -] - -[[package]] -name = "pin-project" -version = "1.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "pkg-config" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" - -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", + "windows-link", ] [[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" +name = "percent-encoding" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] -name = "portable-atomic" -version = "1.15.0" +name = "pin-project-lite" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "potential_utf" @@ -2994,16 +1218,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn 2.0.119", -] - [[package]] name = "proc-macro2" version = "1.0.107" @@ -3013,46 +1227,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "procfs" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" -dependencies = [ - "bitflags 2.13.2", - "hex", - "lazy_static", - "procfs-core", - "rustix 0.38.44", -] - -[[package]] -name = "procfs-core" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" -dependencies = [ - "bitflags 2.13.2", - "hex", -] - -[[package]] -name = "prometheus" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" -dependencies = [ - "cfg-if", - "fnv", - "lazy_static", - "libc", - "memchr", - "parking_lot", - "procfs", - "protobuf", - "thiserror 1.0.69", -] - [[package]] name = "promql-parser" version = "0.10.0" @@ -3066,19 +1240,6 @@ dependencies = [ "regex", ] -[[package]] -name = "promql-parser" -version = "0.10.0" -source = "git+https://github.com/ProjectASAP/promql-parser#9fede7eecca923c9882fe256484d00d37f8706cb" -dependencies = [ - "cfgrammar", - "chrono", - "lazy_static", - "lrlex", - "lrpar", - "regex", -] - [[package]] name = "prost" version = "0.13.5" @@ -3089,26 +1250,6 @@ dependencies = [ "prost-derive", ] -[[package]] -name = "prost-build" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" -dependencies = [ - "heck", - "itertools 0.10.5", - "log", - "multimap", - "once_cell", - "petgraph 0.7.1", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn 2.0.119", - "tempfile", -] - [[package]] name = "prost-derive" version = "0.13.5" @@ -3116,91 +1257,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools", "proc-macro2", "quote", "syn 2.0.119", ] -[[package]] -name = "prost-types" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" -dependencies = [ - "prost", -] - -[[package]] -name = "protobuf" -version = "2.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" - -[[package]] -name = "protoc-bin-vendored" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa" -dependencies = [ - "protoc-bin-vendored-linux-aarch_64", - "protoc-bin-vendored-linux-ppcle_64", - "protoc-bin-vendored-linux-s390_64", - "protoc-bin-vendored-linux-x86_32", - "protoc-bin-vendored-linux-x86_64", - "protoc-bin-vendored-macos-aarch_64", - "protoc-bin-vendored-macos-x86_64", - "protoc-bin-vendored-win32", -] - -[[package]] -name = "protoc-bin-vendored-linux-aarch_64" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c" - -[[package]] -name = "protoc-bin-vendored-linux-ppcle_64" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c" - -[[package]] -name = "protoc-bin-vendored-linux-s390_64" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0" - -[[package]] -name = "protoc-bin-vendored-linux-x86_32" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5" - -[[package]] -name = "protoc-bin-vendored-linux-x86_64" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78" - -[[package]] -name = "protoc-bin-vendored-macos-aarch_64" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092" - -[[package]] -name = "protoc-bin-vendored-macos-x86_64" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756" - -[[package]] -name = "protoc-bin-vendored-win32" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" - [[package]] name = "quinn" version = "0.11.11" @@ -3278,24 +1340,13 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha 0.9.0", + "rand_chacha", "rand_core 0.9.5", ] @@ -3310,16 +1361,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - [[package]] name = "rand_chacha" version = "0.9.0" @@ -3330,15 +1371,6 @@ dependencies = [ "rand_core 0.9.5", ] -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] - [[package]] name = "rand_core" version = "0.9.5" @@ -3363,33 +1395,13 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.2", + "bitflags", ] [[package]] @@ -3427,7 +1439,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-core", "http", @@ -3449,7 +1461,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", - "tower 0.5.3", + "tower", "tower-http", "tower-service", "url", @@ -3498,41 +1510,6 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags 2.13.2", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustix" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" -dependencies = [ - "bitflags 2.13.2", - "errno", - "libc", - "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", -] - [[package]] name = "rustls" version = "0.23.44" @@ -3580,33 +1557,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - -[[package]] -name = "seq-macro" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" - [[package]] name = "serde" version = "1.0.229" @@ -3680,15 +1636,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "serde_spanned" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" -dependencies = [ - "serde", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3701,41 +1648,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap 2.14.2", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - -[[package]] -name = "sha1" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -3761,18 +1673,6 @@ dependencies = [ "libc", ] -[[package]] -name = "simd-adler32" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" - -[[package]] -name = "siphasher" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" - [[package]] name = "slab" version = "0.4.12" @@ -3785,27 +1685,6 @@ version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" -[[package]] -name = "snafu" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" -dependencies = [ - "snafu-derive", -] - -[[package]] -name = "snafu-derive" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "snap" version = "1.1.2" @@ -3844,27 +1723,6 @@ dependencies = [ "vob 3.0.6", ] -[[package]] -name = "sqlparser" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe11944a61da0da3f592e19a45ebe5ab92dc14a779907ff1f08fbb797bfefc7" -dependencies = [ - "log", - "sqlparser_derive", -] - -[[package]] -name = "sqlparser_derive" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -3875,47 +1733,19 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.119", -] +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] -name = "subtle" -version = "2.6.1" +name = "strsim" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] -name = "symlink" -version = "0.1.0" +name = "subtle" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -3959,25 +1789,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "tagptr" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" - -[[package]] -name = "tempfile" -version = "3.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" -dependencies = [ - "fastrand", - "getrandom 0.4.3", - "once_cell", - "rustix 1.1.4", - "windows-sys 0.59.0", -] - [[package]] name = "thiserror" version = "1.0.69" @@ -4027,17 +1838,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float", -] - [[package]] name = "time" version = "0.3.55" @@ -4070,15 +1870,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tinystr" version = "0.8.4" @@ -4089,16 +1880,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "tinyvec" version = "1.13.2" @@ -4152,161 +1933,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-stream" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.21.0", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.24.0", -] - -[[package]] -name = "tokio-util" -version = "0.7.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "libc", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "toml" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.22.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" -dependencies = [ - "indexmap 2.14.2", - "serde", - "serde_spanned", - "toml_datetime", - "toml_write", - "winnow", -] - -[[package]] -name = "toml_write" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" - -[[package]] -name = "tonic" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64 0.22.1", - "bytes", - "flate2", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-timeout", - "hyper-util", - "percent-encoding", - "pin-project", - "prost", - "socket2 0.5.10", - "tokio", - "tokio-stream", - "tower 0.4.13", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tonic-build" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "prost-types", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand 0.8.8", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "tower" version = "0.5.3" @@ -4329,13 +1955,13 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.13.2", + "bitflags", "bytes", "futures-util", "http", "http-body", "pin-project-lite", - "tower 0.5.3", + "tower", "tower-layer", "tower-service", "url", @@ -4365,19 +1991,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-appender" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" -dependencies = [ - "crossbeam-channel", - "symlink", - "thiserror 2.0.20", - "time", - "tracing-subscriber", -] - [[package]] name = "tracing-attributes" version = "0.1.31" @@ -4434,53 +2047,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "tungstenite" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http", - "httparse", - "log", - "rand 0.8.8", - "sha1", - "thiserror 1.0.69", - "url", - "utf-8", -] - -[[package]] -name = "tungstenite" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http", - "httparse", - "log", - "rand 0.8.8", - "sha1", - "thiserror 1.0.69", - "utf-8", -] - -[[package]] -name = "twox-hash" -version = "1.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" -dependencies = [ - "cfg-if", - "static_assertions", -] - [[package]] name = "twox-hash" version = "2.1.4" @@ -4490,36 +2056,18 @@ dependencies = [ "rand 0.10.2", ] -[[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" - [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-segmentation" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" - [[package]] name = "unicode-width" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "untrusted" version = "0.9.0" @@ -4538,12 +2086,6 @@ dependencies = [ "serde", ] -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -4556,17 +2098,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "1.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" -dependencies = [ - "getrandom 0.4.3", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "valuable" version = "0.1.1" @@ -4584,12 +2115,6 @@ dependencies = [ "time", ] -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - [[package]] name = "vob" version = "3.0.6" @@ -4610,16 +2135,6 @@ dependencies = [ "serde", ] -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - [[package]] name = "want" version = "0.3.1" @@ -4728,37 +2243,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-core" version = "0.62.2" @@ -4909,15 +2393,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "winnow" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" -dependencies = [ - "memchr", -] - [[package]] name = "wit-bindgen" version = "0.57.1" @@ -4936,15 +2411,6 @@ version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yoke" version = "0.8.3" @@ -5048,42 +2514,8 @@ dependencies = [ "syn 3.0.5", ] -[[package]] -name = "zlib-rs" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" - [[package]] name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.13+zstd.1.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/Cargo.toml b/Cargo.toml index 9040dbf29..b4a9cbb4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,44 +1,27 @@ [workspace] resolver = "2" -members = [ - "crates/asap_otel_proto", - "crates/asap_types", - "data_plane", - "control_plane", -] +members = ["control_plane", "data_plane"] [workspace.package] edition = "2021" version = "0.1.0" [workspace.dependencies] -# Keep Planner frontends, selection, and IR on the same immutable revision. -# Alias upstream asap-types because this workspace also defines asap_types. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "4b0839ce1733aab8231eb90944c876063a4551f9" } -asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "4b0839ce1733aab8231eb90944c876063a4551f9" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "4b0839ce1733aab8231eb90944c876063a4551f9" } -asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "4b0839ce1733aab8231eb90944c876063a4551f9" } +# Keep every Planner crate on the same immutable revision. Alias upstream +# asap-types as planner-types to name its origin at use sites. +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "b517fbf256a17621d48fa3f8508c5d5ca1017bad" } +asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "b517fbf256a17621d48fa3f8508c5d5ca1017bad" } +asap-plan-selection = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "b517fbf256a17621d48fa3f8508c5d5ca1017bad" } +asap-executor = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "b517fbf256a17621d48fa3f8508c5d5ca1017bad" } -# Shared external deps (used by 2+ crates) serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -serde_yaml = "0.9" thiserror = "1.0" anyhow = "1.0" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } clap = { version = "4.0", features = ["derive", "env"] } -chrono = { version = "0.4", features = ["serde"] } -# One parser source for backend parsing. Member crates inherit the current -# ProjectASAP default branch through `workspace = true`. -promql-parser = { git = "https://github.com/ProjectASAP/promql-parser" } tokio = { version = "1.0", features = ["full"] } -arc-swap = "1.7" +axum = "0.7" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } - -# Internal crates -asap-physical-operators = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "4b0839ce1733aab8231eb90944c876063a4551f9" } -asap_sketch_codec = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "4b0839ce1733aab8231eb90944c876063a4551f9" } -asap_types = { path = "crates/asap_types" } -asap_otel_proto = { path = "crates/asap_otel_proto" } -indexmap = { version = "2.0", features = ["serde"] } +control_plane = { path = "control_plane" } diff --git a/README.md b/README.md index f9391cbc1..d1ec53935 100644 --- a/README.md +++ b/README.md @@ -2,616 +2,64 @@ # ASAPQuery-backend -ASAPQuery-backend compiles and executes planned observability queries using -materialized exact accumulators and sketches, with configured exact backends -for unsupported or unavailable results. It contains the backend control plane, -summary storage, precompute runtime and query adapters. Query semantics and -post-ASAP candidate generation come from the external -[ASAPPlanner](https://github.com/ProjectASAP/ASAPPlanner) dependency. +ASAPQuery-backend is the deployment that executes +[ASAPPlanner](https://github.com/ProjectASAP/ASAPPlanner)'s plans for PromQL +workloads over Prometheus Remote Write data. -Start with the [E2E physical-DAG walkthrough](docs/evaluation/e2e-physical-dag.md) -for workload inputs, ERP evidence, selected plans, sample ingestion, storage -inspection and query execution. The [documentation index](docs/README.md) links -the detailed guides. This README describes the repository entry points; it does -not promise a universal latency improvement or a single accuracy bound across -summary families. - -## Planning, state and execution +Following the Planner's +[planning stages](https://github.com/ProjectASAP/ASAPPlanner/blob/main/docs/design_docs/proposals/planner-layering.md), +the Planner decides everything about a plan: which summaries to use, how to +share them, what to materialize and which plan is cheapest. The backend +supplies its capabilities, then executes the one selected `PhysicalASAPDAG` as +given. ```text -Workload + capabilities + cost/accuracy evidence - | - external ASAPPlanner - selected post-ASAP DAG - | - backend PhysicalCompiler - | - authoritative SummaryCatalog - + shared physical-plan publication - / \ - PrecomputePlan QueryPlan - producer/input subDAGs query/readout subDAGs - | | - SummaryStore <---- bound state reads - | | - physical instances result or exact fallback -``` - -The backend compiler consumes the selected Planner DAG and places supported -operations. It preserves semantic nodes and bindings rather than rediscovering -query intent from metric names. ERP evidence can influence eligible choices; -its presence alone does not prove selection or measured benefit. - -| Contract or component | Responsibility | -| --- | --- | -| `SummaryCatalog` | Authoritative generation and descriptors for data, summary semantics, population, windows and materialization identity. Sibling plans validate against the same catalog. | -| `PrecomputePlan` | Materializations, input schemas, producer bindings and installed precompute subDAGs. Window size, slide, layout and origin are explicit contracts. | -| `ProducerContract`, `CollectorPlan`, `TransmissionPlan` | Producer/materialization authorization, external collector work and state-frame encoding/sequence rules. They are shared producer/consumer contracts; backend-local Remote Write does not require ASAPCollector. | -| `QueryPlan` | Language-tagged query entries, exact dependencies, typed operations and materialization/readout bindings. Query subDAGs execute against the active installed snapshot. | -| SummaryStore (`SketchStore`) | Physical state, indexes, lifecycle and persistence. A `SummaryDefinitionId` identifies the planned materialization. SID identifies a physical stored series/lifetime; `SummaryInstanceId` identifies a concrete definition/window/group instance. Planner node IDs are not storage IDs. | -| Physical-plan publication | Publishes catalog, precompute, query, collector and transmission views together; invalid bindings reject installation rather than silently selecting replacement state. | - -See the [physical compiler](docs/developer_docs/control-plane/physical-compiler.md), -[catalog/runtime design](docs/developer_docs/query-engine/catalog-physical-plan-runtime.md) -and [maintenance protocol](docs/developer_docs/maintenance-replay.md). - -## Interfaces and support boundaries - -```mermaid -flowchart TB - subgraph planning["Planning and installed contracts"] - PL["External ASAPPlanner"] --> CP["Backend control plane"] - CP --> CAT["Authoritative SummaryCatalog"] - CP --> PP["PrecomputePlan"] - CP --> QP["QueryPlan"] - CP --> PC["Producer contracts / CollectorPlan"] - CP --> TP["TransmissionPlan"] - end - - subgraph ingest["Sample and state ingestion"] - RW["Prometheus Remote Write v1"] --> RX["RW receiver / bounded queue"] - RX --> PD["Precompute DAG"] - EP["External producers: modified OTLP / state"] --> PV["Producer / transmission validation"] - PV --> PD - PD --> SS["SummaryStore: physical series and window/group instances"] - end - - subgraph queries["Query adapters and execution"] - PQ["PromQL"] --> QD["PromQL query DAG"] - MQ["MetricsQL"] --> MA["MetricsQL adapter"] - SQL["ClickHouse SQL"] --> SD["Typed SQL query DAG"] - QD --> QR["Shared query runtime"] - MA --> QR - SD --> QR - QR -->|"bound state reads"| SS - QD -->|"unsupported / unavailable"| PE["Prometheus exact backend"] - MA -->|"unsupported / unavailable"| VE["VictoriaMetrics exact backend"] - SD -->|"unsupported / unavailable"| CE["ClickHouse exact backend"] - QR --> W["Warm: successful summary execution"] - QR --> H["Hybrid: summaries plus supported exact dependencies"] - PE --> F["Exact fallback result"] - VE --> F - CE --> F - CE -->|"supported exact SQL subtree"| SD - end - - subgraph completion["Independent completion authority"] - B["Typed watermark / barrier: installed producer and partition scope"] - B --> CC["Continuous coordinator: closure activation gated"] - CC -.->|"gated completion contract"| PD - end - - PP -.-> PD - QP -.-> QR - PC -.-> PV - TP -.-> PV - PC -.-> B - CAT -.-> SS - CAT -.-> QR +PlanningWorkload (JSON) DeploymentCapabilities + \ / + control_plane: ASAPPlanner stages 0-3 + | + plan.json: selected PhysicalASAPDAG + query per root + | + data_plane: Remote Write in, Prometheus query API out ``` -Dotted edges show installed contracts or explicitly gated paths. The barrier is -independent of Remote Write samples; this figure does not advertise a public -barrier endpoint or continuous/sliding activation. Warm and hybrid labels apply -only to successful execution with the corresponding actual summary reads; -external-only DAG execution is not acceleration. - -| Surface | Current path and boundary | -| --- | --- | -| Prometheus | `/api/v1/write` accepts Remote Write **v1** scalar samples. PromQL instant/range queries use installed plans with configured Prometheus exact fallback. The collector-free `asapquery` profile is a bounded compatibility profile, not every distributed/persistent deployment option. | -| VictoriaMetrics | A separate MetricsQL adapter and listener lower supported expressions through Planner and the shared runtime. Unsupported syntax or unavailable coverage retains VictoriaMetrics exact fallback. See [MetricsQL support](docs/developer_docs/query-engine/victoriametrics-metricsql-support.md). | -| ClickHouse | An optional SQL HTTP adapter uses the same catalog/publication with typed relational and mixed exact/summary DAG execution. Supported collection/scalar operations are explicit; arbitrary SQL, lambdas and counter queries are not implied. External-only DAG execution is not summary acceleration. See [SQL support](docs/developer_docs/query-engine/clickhouse-sql-support.md). | -| External producers | The distributed path accepts configured collector/state input, including modified OTLP. It is distinct from standard Prometheus Remote Write and uses the shared producer/transmission contracts. | - -### Query workload input - -Planning input and serving traffic are separate. Sending a PromQL request from -Grafana does not declare that it is a recurring dashboard query. Declare known -query classes through the control plane; send each evaluation to the data-plane -query endpoint. - -At startup, set `CONTROLLER_WORKLOADS` to a YAML registry containing the query -text and deployment hints: - -```yaml -- metric_name: http_requests_total - query_string: "sum by (region) (rate(http_requests_total[5m]))" - accuracy_sla: 0.99 - assign_to_role: agent - grouping_labels: [region] - repeat_every: 30s -``` - -`query_string` is the preferred source for metric, aggregation, filters, -grouping and range-window semantics. Optional registry fields include -`sketch_family_override`, `sample_p`, `distinct_keys_per_window`, `item_label`, -`monitor` and `repeat_every`. `accuracy_sla` is the legacy success fraction: -`1.0` requests exact results and `0.99` permits epsilon `0.01`. An unknown key -is rejected rather than ignored, and a registry file that exists but does not -parse fails startup instead of degrading to an empty registry. - -The startup registry and the legacy planning API carry the same evaluation -cadence, so a declaration is costed identically through either entry point. -`POST /api/v1/plan` (`CONTROLLER_ADDR`, default port `8080`): - -```bash -curl -X POST http://localhost:8080/api/v1/plan \ - -H 'Content-Type: application/json' \ - -d '{ - "query_string": "sum by (region) (rate(http_requests_total[5m]))", - "repeat_every": "30s", - "accuracy_sla": 0.99, - "latency_sla": "1s", - "workload": { - "series_count": 100000, - "samples_per_sec_per_series": 0.0667, - "bytes_per_raw_sample": 100, - "distinct_keys_per_window": null, - "data_distribution": "zipf", - "memory_budget_bytes": 268435456 - } - }' -``` - -`repeat_every` describes an expected cadence—for example, a dashboard panel -refreshed every 30 seconds—but does not schedule evaluations. Its current use is -narrow: the legacy deployment cost path uses it as the flush-period proxy for -batch-mode plans. Window-mode plans use their configured window duration, and -the canonical physical-plan request does not yet model general query recurrence. -Do not assume that changing `repeat_every` will change summary selection. - -The `workload` object describes the incoming data stream and may be omitted to -use conservative defaults. The API also accepts explicit `metric_name`, -`aggregations`, `time_window`, `group_by_labels` and `label_filters` instead of -a query string. +## Status -Once the matching physical plan is installed and its state is ready, clients -execute the query through the data plane's Prometheus-compatible API: - -```bash -curl -G http://localhost:8088/api/v1/query \ - --data-urlencode 'query=sum by (region) (rate(http_requests_total[5m]))' -``` - -Range evaluations use `/api/v1/query_range` with `query`, `start`, `end` and -`step`. Serving requests do not update `repeat_every` automatically. End-to-end -dashboard-frequency-aware planning still requires a typed recurrence field on -the canonical workload/publication path and an observer or explicit workload -registration to populate it. - -Remote Write `204` acknowledges atomic bounded queue admission, **not** durable -accumulator publication or a raw write-ahead log. Deduplication is in memory; -there is no durable raw-WAL replay guarantee. Stale markers are recognized and -excluded from numeric aggregation, not propagated as general lifecycle events. -Native histograms and exemplars are rejected in this v1 path. - -Remote Write does not carry an authoritative completion watermark. Installed -producer partition rosters and typed barriers define a separate completion -scope; roster membership alone does not provide authentication, durable progress -or continuous scheduling. Finite immutable maintenance has explicit source, -population, window and operator gates. Continuous/sliding activation must not be -inferred from available metadata or a successful ingest response. - -Fallback and execution provenance must reflect the path that actually produced -a successful result. Missing coverage and unsupported operations must not be -reported as warm success. Accuracy and benefit claims need the selected family's -contract and workload-specific evidence. Thanos/object-store integrations are -optional deployment paths, not the default architecture or a required quickstart. - -## Build and inspect: shared setup - -Run these steps in Bash on Linux. The Docker commands below use host networking; -other platforms need equivalent port/address configuration. Each backend runbook -has its own ports and artifacts and can be run independently after this setup. - -### 1. Check out the backend and its sibling dependencies - -```bash -mkdir -p asap-workspace -cd asap-workspace -git clone https://github.com/ProjectASAP/ASAPQuery-backend.git -git clone https://github.com/ProjectASAP/ASAPCollector.git -git clone https://github.com/ProjectASAP/asap_sketchlib.git -cd ASAPQuery-backend -# Follow this backend revision's CI dependency pin, rather than a copied SHA. -ASAP_SKETCH_REF="$(awk '/repository: ProjectASAP\/asap_sketchlib/{found=1;next} found && /^[[:space:]]*ref:/{print $2;exit}' .github/workflows/mvp-ci.yml)" -git -C ../asap_sketchlib checkout "${ASAP_SKETCH_REF:-main}" -git -C ../ASAPCollector checkout main -``` - -[MVP CI](.github/workflows/mvp-ci.yml) is the source for compatible dependency -checkouts; currently Collector uses its default branch. For reproducible runs, -record the exact revisions. ASAPPlanner is fetched at the revision pinned in -the Cargo manifests; do not substitute an unrelated local planner checkout. - -### 2. Check prerequisites and build - -Install Rust through rustup and install `protoc`, `curl`, `jq`, Python 3 and Git -using your system package manager. Docker is needed only for the real -Prometheus/Pushgateway demo below. Rust 1.98.0 is the toolchain used for these -local command checks. - -```bash -rustup toolchain install 1.98.0 --profile minimal --component rustfmt --component clippy -rustc +1.98.0 --version -protoc --version -curl --version -jq --version -python3 --version -# For Docker-backed demos only: -docker version - -cargo +1.98.0 fetch --locked -cargo +1.98.0 build --locked -p control_plane -p data_plane -cargo +1.98.0 build --locked -p control_plane \ - --example calibration_candidates --example compile_workload_artifact \ - --example compile_clickhouse_workload -target/debug/data_plane --help -mkdir -p target/readme-evidence -{ - git rev-parse HEAD - git -C ../ASAPCollector rev-parse HEAD - git -C ../asap_sketchlib rev-parse HEAD -} > target/readme-evidence/revisions.txt -``` - -The executable is `target/debug/data_plane`, or `target/release/data_plane` if -you build with `--release`. Access to the pinned Git dependencies is required. - -### 3. Export candidates and select a deployable plan by complete cost - -```bash -target/debug/examples/calibration_candidates \ - docs/examples/asapquery-compatibility-demo-snapshot.json \ - > target/readme-evidence/candidates.json -jq '.candidates[] | {candidate_index, unavailable_reason}' \ - target/readme-evidence/candidates.json - -target/debug/examples/compile_workload_artifact \ - "$ASAPQUERY_PLANNING_SNAPSHOT" \ - --dot target/readme-evidence/selected.dot \ - > target/readme-evidence/selected.json -jq '.cost_comparison' target/readme-evidence/selected.json -jq '.summary_catalog' target/readme-evidence/selected.json -jq '.precompute_plan | {materializations, executable_dags}' \ - target/readme-evidence/selected.json -jq '.query_plan.entries' target/readme-evidence/selected.json -jq '.precompute_plan.schemas[] | {materialization, schema_id}' \ - target/readme-evidence/selected.json -dot -Tsvg target/readme-evidence/selected.dot \ - -o target/readme-evidence/selected.svg -``` +The backend is being rebuilt on the Planner's physical DAG: -Candidate discovery accepts the checked-in unquoted templates. Deployment and -selected-plan inspection require `ASAPQUERY_PLANNING_SNAPSHOT` to point to a -snapshot with complete, valid workload cost evidence. Prepare that input using -the [cost evidence workflow](docs/examples/workload-cost-evidence.md). -There is one snapshot compiler: it compares complete executable alternatives, -including exact fallback. Materialization IDs are definitions, not physical SIDs. -For `--metricsql`, collect quotes for the MetricsQL frontend. +| Part | State | +|---|---| +| Control plane: plan a PromQL workload, write `plan.json` | done | +| Data plane: load and validate `plan.json`, accept Remote Write | done | +| Stored summaries (SDS) | next | +| Precompute engine: run ingestion-time nodes per pane | planned | +| Query engine: run query-time nodes over stored panes and raw data | planned | -## Prometheus runbook +Until the query engine lands, the data plane forwards every query to +Prometheus and marks the response `x-asap-execution: forwarded`. SQL and +ClickHouse are not supported yet. -### Start real Prometheus and Pushgateway, then the backend +## Running -The checked-in [Prometheus YAML](demos/asapquery/prometheus.yml) scrapes -Pushgateway at `127.0.0.1:19092` every second and includes: +```sh +# Plan #509 Example 1 with this backend's capabilities. +cargo run -p control_plane -- \ + --workload examples/planner-layering-example1.json --out plan.json -```yaml -remote_write: - - url: http://127.0.0.1:19091/api/v1/write - queue_config: - min_shards: 1 - max_shards: 1 - batch_send_deadline: 1s +# Serve Remote Write on /api/v1/write and the query API on /api/v1/query[_range]. +cargo run -p data_plane -- --plan plan.json --prometheus-url http://localhost:9090 ``` -Use unused ports 19090–19092 and container names below. Prometheus must be healthy -before the `asapquery` profile starts. Initial Remote Write retries while the -backend starts are expected. +`--capabilities caps.json` replaces the default capabilities, which are the +Planner executor's capabilities without state kept across query evaluations. -```bash -mkdir -p target/readme-evidence/prometheus -docker run -d --name asap-readme-pushgateway --network host \ - prom/pushgateway:v1.9.0 --web.listen-address=:19092 -docker run -d --name asap-readme-prometheus --network host \ - -v "$PWD/demos/asapquery/prometheus.yml:/etc/prometheus/prometheus.yml:ro" \ - prom/prometheus:v2.55.1 --config.file=/etc/prometheus/prometheus.yml \ - --storage.tsdb.path=/prometheus --web.listen-address=:19090 -for attempt in $(seq 1 60); do - curl -fsS http://127.0.0.1:19090/-/healthy && break - sleep 1 -done -curl -fsS http://127.0.0.1:19090/-/healthy +## Development -target/debug/data_plane --profile asapquery \ - --planning-snapshot "$ASAPQUERY_PLANNING_SNAPSHOT" \ - --prometheus-server http://127.0.0.1:19090 \ - --forward-unsupported-queries --http-port 19091 \ - --output-dir target/readme-evidence/prometheus/runtime \ - > target/readme-evidence/prometheus/backend.log 2>&1 & -echo $! > target/readme-evidence/prometheus/backend.pid -for attempt in $(seq 1 60); do - curl -fsS http://127.0.0.1:19091/api/v1/health && break - sleep 1 -done -curl -fsS http://127.0.0.1:19091/api/v1/health +```sh +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --locked -- -D warnings +cargo test --workspace --locked ``` -### Send samples and inspect execution - -Pushgateway accepts exposition text; Prometheus performs the real scrape and -Snappy/protobuf Remote Write encoding. Do not post this text to `/api/v1/write`. - -```bash -for sample in $(seq 1 15); do - printf '# TYPE asap_demo_gauge gauge\nasap_demo_gauge %s\n' "$sample" | - curl -fsS --data-binary @- http://127.0.0.1:19092/metrics/job/asapquery-demo - sleep 1 -done -curl -fsS http://127.0.0.1:19091/api/v1/physical-plan/status \ - | tee target/readme-evidence/prometheus/status.json | jq . -curl -fsS http://127.0.0.1:19091/api/v1/summary-inventory \ - | tee target/readme-evidence/prometheus/inventory.json | jq . -curl -sS -D target/readme-evidence/prometheus/query.headers --get \ - http://127.0.0.1:19091/api/v1/query \ - --data-urlencode 'query=sum(sum_over_time(asap_demo_gauge[5s]))' \ - > target/readme-evidence/prometheus/query.json -jq '{status, data, infos}' target/readme-evidence/prometheus/query.json -cat target/readme-evidence/prometheus/query.headers -``` - -Check actual response provenance and window coverage; a successful HTTP response -alone is not warm execution. For **a separately completed finite replay only**, -after every input has been delivered, the closure operation is: - -```bash -curl -fsS -X POST http://127.0.0.1:19091/api/v1/precompute/drain | jq . -``` - -Do not run that command against the live scraping setup above: it closes that -receiver generation and later writes are rejected. It is not a periodic flush -or an implicit continuous watermark. The real finite maintenance process test -is available separately: - -```bash -cargo +1.98.0 test --locked -p data_plane --test asapquery_compatibility_process_e2e \ - immutable_maintenance_process::complete_group_maintenance_is_automatic_and_durable \ - -- --exact --nocapture -``` - -### Validate and clean up - -The focused protocol matrix uses the real backend process with a **mock exact -upstream**, and removes temporary artifacts. The second command runs the -separate **real Prometheus/Pushgateway** demo and handles its own cleanup. -Stop the manual setup before the real demo because it uses the same ports. - -```bash -kill "$(cat target/readme-evidence/prometheus/backend.pid)" -docker rm -f asap-readme-prometheus asap-readme-pushgateway -cargo +1.98.0 test --locked -p data_plane --test asapquery_compatibility_process_e2e \ - collector_free_profile_serves_complete_matrix_and_falls_back_exactly -- --exact -export ASAPQUERY_PLANNING_SNAPSHOT=/absolute/path/priced-snapshot.json -./scripts/e2e.sh asapquery-demo -``` - -Keep `target/readme-evidence/prometheus/` and -`target/asapquery-demo-evidence/` for inspection; delete them yourself only after -you no longer need the logs, outputs and runtime state. - -## VictoriaMetrics / MetricsQL runbook - -This repository has a real backend-process MetricsQL fixture, but not a bundled -self-contained native VictoriaMetrics deployment demo. The fixture starts the -backend, installs an actual MetricsQL plan, sends Remote Write samples, checks -labeled results, and shuts down its child process. Its exact upstream is a -health-only test service, **not native VictoriaMetrics**. - -```bash -mkdir -p target/readme-evidence/victoriametrics -cargo +1.98.0 test --locked -p data_plane --test asapquery_compatibility_process_e2e \ - distinct_planning_process::distinct_range_uses_planner_selected_hll_and_source_labels \ - -- --exact --nocapture \ - > target/readme-evidence/victoriametrics/process.log 2>&1 -``` - -`DISTINCT_PLANNED` and `DISTINCT_INSTALLED` in the log expose selection and the -shared installed publication. The fixture checks the actual MetricsQL listener; -it is not a native differential or a performance benchmark. - -For an **existing native VM service** at port 8428, the following independently -starts an exact-fallback adapter. Load the intended dataset into that service -using its normal ingestion setup first. The empty bootstrap deliberately has -no accelerated bindings; this smoke request must not be called warm. - -```bash -printf 'aggregations: []\n' > target/readme-evidence/victoriametrics/bootstrap.yaml -target/debug/data_plane \ - --streaming-config target/readme-evidence/victoriametrics/bootstrap.yaml \ - --http-port 19080 --victoriametrics-http-port 19081 \ - --victoriametrics-url http://127.0.0.1:8428 \ - --output-dir target/readme-evidence/victoriametrics/runtime \ - > target/readme-evidence/victoriametrics/backend.log 2>&1 & -echo $! > target/readme-evidence/victoriametrics/backend.pid -for attempt in $(seq 1 60); do - curl -fsS http://127.0.0.1:19080/api/v1/health && break - sleep 1 -done -curl -fsS http://127.0.0.1:19080/api/v1/health -curl -sS -D target/readme-evidence/victoriametrics/query.headers --get \ - http://127.0.0.1:19081/api/v1/query \ - --data-urlencode 'query=default_rollup(asap_demo_gauge[5s])' \ - > target/readme-evidence/victoriametrics/query.json -jq . target/readme-evidence/victoriametrics/query.json -curl -fsS http://127.0.0.1:19080/api/v1/physical-plan/status | jq . -kill "$(cat target/readme-evidence/victoriametrics/backend.pid)" -``` - -To inspect supported MetricsQL planning independently: - -```bash -target/debug/examples/compile_workload_artifact \ - "$ASAPQUERY_PLANNING_SNAPSHOT" --metricsql \ - > target/readme-evidence/victoriametrics/selected.json -jq '.query_plan.entries' \ - target/readme-evidence/victoriametrics/selected.json -``` - -This does not install or ingest the artifact by itself. Use the fixture's normal -stage/activate path for an integrated example. Keep the logs, headers and JSON -under `target/readme-evidence/victoriametrics/`; see -[MetricsQL support](docs/developer_docs/query-engine/victoriametrics-metricsql-support.md) -for tenant paths and unsupported constructs. - -## ClickHouse / SQL runbook - -Use a **dedicated test ClickHouse server** at `http://127.0.0.1:8123`, with -credentials supplied through `CLICKHOUSE_USER`/`CLICKHOUSE_PASSWORD` if needed. -The real process fixture drops and recreates `default.telemetry` and -`default.divisors` at the start of each scenario, and leaves the final tables -on the dedicated server; do not point it at a production database. It loads samples, -compiles an actual mixed query, publishes the shared plans, queries the backend, -and cleans up its child backend. The native server remains yours to manage. - -```bash -mkdir -p target/readme-evidence/clickhouse -export CLICKHOUSE_URL=http://127.0.0.1:8123 -curl -fsS --user "${CLICKHOUSE_USER:-default}:${CLICKHOUSE_PASSWORD:-}" \ - "$CLICKHOUSE_URL/ping" -CLICKHOUSE_PLANNING_ARTIFACT="$PWD/target/readme-evidence/clickhouse/planning.json" \ - cargo +1.98.0 test --locked -p data_plane --test clickhouse_differential_e2e \ - compiled_publication_executes_mixed_dag_in_data_plane_process \ - -- --exact --nocapture \ - > target/readme-evidence/clickhouse/process.log 2>&1 -jq '{selection_trace, catalog: .publication.summary_catalog}' \ - target/readme-evidence/clickhouse/planning.json -``` - -The fixture runs SUM, COUNT and MAX scenarios. `planning.json` is overwritten -per scenario and retains the last publication; keep `process.log` for the complete -run. Without `CLICKHOUSE_URL` the real-server test skips; that is not a successful -native test. See [SQL support](docs/developer_docs/query-engine/clickhouse-sql-support.md) -for the typed backfill and format boundaries. - -### Compile a concrete SQL input without preset materializations - -The example reads `ClickHouseSqlAutomaticWorkload` JSON from stdin, not a -Prometheus snapshot. Reuse the current compiler envelope from the shared setup -and provide a typed table schema plus a fixed evaluation interval: - -```bash -python3 - <<'PY' -import json -from pathlib import Path -root = Path('target/readme-evidence') -envelope = json.loads((root / 'selected.json').read_text())['precompute_plan']['envelope'] -envelope['plan_id'] = 9001 -envelope['plan_version'] = 1 -workload = { - 'envelope': envelope, - 'tables': {'telemetry': { - 'columns': [ - {'name': 'timestamp_ms', 'dtype': 'timestamp', 'nullable': False}, - {'name': 'value', 'dtype': 'float64', 'nullable': False}, - {'name': 'metric', 'dtype': 'utf8', 'nullable': False}], - 'time_index': 0, 'unique_keys': [], 'closed': True}}, - 'accuracy': 'Exact', - 'queries': [{'sql': 'SELECT sum(value) FROM telemetry WHERE timestamp_ms >= 0 AND timestamp_ms < 2000', - 'start_ms': 0, 'end_ms': 2000, 'cumulative': True}]} -(root / 'clickhouse/workload.json').write_text(json.dumps(workload, indent=2)) -PY -target/debug/examples/compile_clickhouse_workload \ - < target/readme-evidence/clickhouse/workload.json \ - > target/readme-evidence/clickhouse/compiled.json -jq '{selection_trace, catalog: .install.summary_catalog, query: .install.query_plan}' \ - target/readme-evidence/clickhouse/compiled.json -``` - -The fixture above supplies real table data and exercises publication/execution; -this offline command alone does neither. For a standalone **exact-fallback** -listener against that native service: - -```bash -printf 'aggregations: []\n' > target/readme-evidence/clickhouse/bootstrap.yaml -target/debug/data_plane \ - --streaming-config target/readme-evidence/clickhouse/bootstrap.yaml \ - --http-port 19082 --clickhouse-http-port 19083 \ - --clickhouse-url "$CLICKHOUSE_URL" --clickhouse-database default \ - --output-dir target/readme-evidence/clickhouse/runtime \ - > target/readme-evidence/clickhouse/backend.log 2>&1 & -echo $! > target/readme-evidence/clickhouse/backend.pid -for attempt in $(seq 1 60); do - curl -fsS http://127.0.0.1:19082/api/v1/health && break - sleep 1 -done -curl -fsS http://127.0.0.1:19082/api/v1/health -curl -sS -D target/readme-evidence/clickhouse/query.headers --get \ - --user "${CLICKHOUSE_USER:-default}:${CLICKHOUSE_PASSWORD:-}" \ - http://127.0.0.1:19083/ --data-urlencode 'query=SELECT 1 FORMAT JSON' \ - > target/readme-evidence/clickhouse/query.json -jq . target/readme-evidence/clickhouse/query.json -curl -fsS http://127.0.0.1:19082/api/v1/physical-plan/status | jq . -kill "$(cat target/readme-evidence/clickhouse/backend.pid)" -``` - -For authenticated requests, set `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD` -for the curl commands above, or send `X-ClickHouse-User` and `X-ClickHouse-Key` -headers. The proxy forwards incoming authentication headers. Backend -`--clickhouse-user` and `--clickhouse-password` configure backfill access; they -do not automatically authenticate proxy fallback requests. The process fixture -uses its matching environment variables. Retain planning/selection artifacts, response headers, -JSON and process logs. A successful SQL response can be exact fallback or an -external-only DAG; only actual summary reads establish ASAP/hybrid execution. -The fixture's mixed-DAG assertions are stronger than a successful `SELECT 1`. - -For recorded datasets, see the [replay guide](docs/user_guide/o11y-replay.md) and -[execution calibration](tools/o11y-execution/CALIBRATION.md). For synthetic fake -metrics, Google and Alibaba query expressions and cross-engine measurements, see -the [dataset-specific accuracy evaluation](tools/shared-workload/ACCURACY_E2E.md). -Report correctness, -fallbacks, build/update cost and whole-deployment resources separately from -query latency. Manual examples leave evidence directories intact; remove them -only when no longer needed. PID-file cleanup commands apply only to the -still-running backend started by that manual run. The ClickHouse fixture leaves -its final tables on your dedicated native server; remove those test tables or -discard that test server separately after collecting evidence. - -## Repository map - -| Path | Contents | -| --- | --- | -| [control_plane/](control_plane/) | Planner adapters, physical compilation, placement/cost evaluation, publication and runtime configuration. | -| [crates/asap_types/](crates/asap_types/) | Shared catalog, SDS identities, plan, grouping, window and producer contracts. | -| [crates/asap_otel_proto/](crates/asap_otel_proto/) | Modified OTLP protobuf bindings for supported external state ingestion. | -| [data_plane/src/precompute_engine/](data_plane/src/precompute_engine/) | Ingest-time operators, coordination and immutable maintenance execution. | -| [data_plane/src/storage_engines/sketch_db/](data_plane/src/storage_engines/sketch_db/) | SummaryStore implementation, physical indexes, lifecycle, backfill and persistence. | -| [data_plane/src/query_engines/](data_plane/src/query_engines/) | Summary readout, shared DAG execution, protocol adapters and exact routing. | -| [data_plane/src/drivers/](data_plane/src/drivers/) | HTTP, ingestion and control-plane interfaces. | -| [data_plane/tests/](data_plane/tests/) | Runtime and process integration tests. | -| [demos/](demos/), [scripts/](scripts/), [tools/](tools/) | Runnable demonstrations, validation and workload/evaluation tooling. | -| [docs/](docs/README.md) | Architecture, operator guides, support boundaries and evaluation evidence. | - -## License - -MIT — see [LICENSE](LICENSE). +The Planner revision is pinned in `Cargo.toml`; see +[scripts/ci/README.md](scripts/ci/README.md). diff --git a/control_plane/.gitignore b/control_plane/.gitignore deleted file mode 100644 index 2f7896d1d..000000000 --- a/control_plane/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target/ diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 63b866090..8c08f7a90 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -1,53 +1,17 @@ [package] name = "control_plane" -version = "0.1.0" -edition = "2021" - -[lib] -name = "control_plane" -path = "src/lib.rs" - -[[bin]] -name = "control_plane" -path = "src/main.rs" +version.workspace = true +edition.workspace = true [dependencies] -tokio = { version = "1", features = ["full"] } -axum = { version = "0.7", features = ["ws"] } -futures-util = "0.3" -serde = { version = "1", features = ["derive"] } -serde_json = "1" -sha2 = "0.10" -serde_yaml = "0.9" -reqwest.workspace = true -anyhow = "1" -thiserror = "1" -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } -chrono = { version = "0.4", features = ["serde"] } -promql-parser.workspace = true -prost = "0.13" -bytes = "1" -parking_lot = "0.12" -prometheus = { version = "0.13", default-features = false, features = ["process"] } -tonic = { version = "0.12", features = ["gzip"] } -asap_types.workspace = true -asap-physical-operators.workspace = true - -# Planner's IR, replacement search, and frontends share the workspace pin. planner-types.workspace = true -asap-aware-mapping.workspace = true asap-frontend-promql.workspace = true -asap-frontend-sql.workspace = true - -[dev-dependencies] -tokio = { version = "1", features = ["full", "test-util"] } -tower = { version = "0.4", features = ["util"] } -axum = { version = "0.7", features = ["ws"] } -http-body-util = "0.1" -tokio-tungstenite = "0.21" - -[build-dependencies] -toml = "0.8" -prost-build = "0.13" -tonic-build = "0.12" +asap-plan-selection.workspace = true +asap-executor.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +anyhow.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +clap.workspace = true diff --git a/control_plane/Dockerfile b/control_plane/Dockerfile deleted file mode 100644 index 0ca9f984f..000000000 --- a/control_plane/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# Builds the control_plane (controller) binary → asap/control-plane:dev. -# -# Build from the backend checkout; Cargo fetches public Git dependencies -# at the revisions recorded in Cargo.lock. -# -# docker build -f control_plane/Dockerfile -t asap/control-plane:dev . -FROM rust:1.90-bookworm AS build -WORKDIR /src - -# protoc for control_plane/build.rs (prost-build/tonic-build compile -# proto/opamp.proto + proto/feedback.proto); git for git-dep fetches. -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - protobuf-compiler git ca-certificates && \ - rm -rf /var/lib/apt/lists/* - -COPY . ASAPQuery-backend - -RUN cd ASAPQuery-backend && cargo build --locked --release --bin control_plane - -# Runtime image: binary + CA certs. -FROM debian:bookworm-slim -RUN apt-get update && \ - apt-get install -y --no-install-recommends ca-certificates && \ - rm -rf /var/lib/apt/lists/* -COPY --from=build /src/ASAPQuery-backend/target/release/control_plane \ - /usr/local/bin/control_plane - -ENV RUST_LOG=info -# OpAMP ws 4320, controller gRPC 4321, controller HTTP 8080. -EXPOSE 4320 4321 8080 -ENTRYPOINT ["/usr/local/bin/control_plane"] diff --git a/control_plane/build.rs b/control_plane/build.rs deleted file mode 100644 index 7af15c0b3..000000000 --- a/control_plane/build.rs +++ /dev/null @@ -1,77 +0,0 @@ -use std::path::{Path, PathBuf}; -use std::process::Command; - -fn git_output(repo: &Path, args: &[&str]) -> Option { - let output = Command::new("git") - .arg("-C") - .arg(repo) - .args(args) - .output() - .ok()?; - if !output.status.success() { - return None; - } - String::from_utf8(output.stdout) - .ok() - .map(|value| value.trim().to_owned()) -} - -fn git_path(repo: &Path, name: &str) -> Option { - let path = PathBuf::from(git_output(repo, &["rev-parse", "--git-path", name])?); - let path = if path.is_absolute() { - path - } else { - repo.join(path) - }; - Some(path.canonicalize().unwrap_or(path)) -} - -fn main() -> Result<(), Box> { - let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR")?); - let repo = manifest_dir.parent().unwrap_or(&manifest_dir); - let lock_path = repo.join("Cargo.lock"); - println!("cargo:rerun-if-changed={}", lock_path.display()); - let lock: toml::Value = std::fs::read_to_string(&lock_path)?.parse()?; - let planner_source = lock - .get("package") - .and_then(toml::Value::as_array) - .into_iter() - .flatten() - .find(|package| package.get("name").and_then(toml::Value::as_str) == Some("asap-types")) - .and_then(|package| package.get("source")) - .and_then(toml::Value::as_str) - .ok_or("Cargo.lock has no resolved ASAPPlanner types source")?; - let planner_revision = planner_source - .rsplit_once('#') - .filter(|(_, revision)| { - revision.len() == 40 && revision.bytes().all(|b| b.is_ascii_hexdigit()) - }) - .map(|(_, revision)| revision) - .ok_or("ASAPPlanner source must resolve to a Git commit")?; - println!("cargo:rustc-env=ASAPPLANNER_REVISION={planner_revision}"); - let revision = std::env::var("ASAPQUERY_BACKEND_REVISION") - .ok() - .or_else(|| git_output(repo, &["rev-parse", "HEAD"])) - .unwrap_or_else(|| "unknown".to_owned()); - println!("cargo:rustc-env=ASAPQUERY_BACKEND_REVISION={revision}"); - println!("cargo:rerun-if-env-changed=ASAPQUERY_BACKEND_REVISION"); - if let Some(head) = git_path(repo, "HEAD") { - println!("cargo:rerun-if-changed={}", head.display()); - } - if let Some(symbolic_ref) = git_output(repo, &["symbolic-ref", "-q", "HEAD"]) { - if let Some(reference) = git_path(repo, &symbolic_ref) { - println!("cargo:rerun-if-changed={}", reference.display()); - } - } - prost_build::compile_protos(&["proto/opamp.proto"], &["proto/"])?; - // asap.runtime.v1.RuntimeSamples service — receives - // PushExporter batches from agents. Must stay in lockstep - // with `sketch-bench/sketch-runtime/proto/feedback.proto`. - tonic_build::configure() - .build_server(true) - // Keep the generated client available for black-box process E2E tests - // and for downstream agents that share this crate's wire contract. - .build_client(true) - .compile_protos(&["proto/feedback.proto"], &["proto/"])?; - Ok(()) -} diff --git a/control_plane/docs/candidate-physical-explain.md b/control_plane/docs/candidate-physical-explain.md deleted file mode 100644 index f6c0dddfe..000000000 --- a/control_plane/docs/candidate-physical-explain.md +++ /dev/null @@ -1,29 +0,0 @@ -# Candidate and physical alternative diagnostics - -The compiler preserves diagnostic associations from the original Planner search through physical quote comparison. These fields explain the existing decision; they do not alter candidate ranking, introduce a resource model, or enable provider-driven re-selection. - -`logical_selection` contains the original ranked groups, legal candidates, accuracy rejections, and selected roots. Group ordinals and `query_index` are local to that invocation. `target_id`, `candidate_id`, and `logical_root_id` use versioned semantic digests instead. Post-ASAP digests reuse Planner's canonical executable exporter, including operator payloads, schemas, guarantees, edge roles and execution states. They exclude assigned executable node IDs and incidental `Rc` sharing. The Planner revision is part of the digest domain. - -The trace's `roots` describe Planner selection. `committed_roots` and `deployment_overrides` describe any subsequent ERP exact fallback. Physical alternatives reference their own `logical_root_ids`; the added native alternative therefore does not masquerade as a selected summary candidate. Canonical values must round-trip without loss, so non-finite floats cannot alias through JSON null. No identity is fabricated when lossless canonical export is unavailable: the corresponding identity is null, and physical explanations include `identity_unavailable_reason`. - -`AlternativeCost` retains an `alternative_id`, concrete `physical_alternative_id` once binding succeeds, and status: - -* `bound`: manifest preparation succeeded; no quote decision yet. -* `selected` / `unselected`: a complete valid quote was considered by the existing selector. -* `bind_failed`: compilation or manifest construction failed; `unavailable_reason` is retained even without a plan ID. -* `evidence_missing` / `evidence_invalid`: no unambiguous matching quote, or malformed/incomplete component evidence. -* `rejected`: the provider reports that the implementation is unavailable. - -Physical identity combines the logical/mask alternative with existing materialization IDs, selected window implementation IDs and ingest mode. Activation time, plan version and quoted numeric prices are not semantic identities. Quotes still match the complete existing manifest, including its generation constraints; diagnostic IDs never replace quote validation or catalog identity. - -The read-only `/api/v1/physical-plan/cost-manifests` and MetricsQL equivalent retain their default manifest-array response. Add `"explain": true` to the existing request to receive `{ "manifests": [...], "alternatives": [...], "logical_selection": [...] }`. Failed alternatives remain alongside usable manifests. When none can bind or be completely priced, the error retains an `all_infeasible` report and every accumulated alternative rather than only a generic message. - -Snapshot compilation exposes the same logical trace on `CompiledPhysicalPlan`; `compile_workload_artifact` serializes it as `planner_selection_trace`. Compile-and-publish returns the trace without adding it to executable wire DTOs. SQL's existing selection trace gains the same semantic candidate/root identities. - -These are bounded explanations: they cover the actual Planner search and the existing physical materialization/exact inventory, not every possible placement or resource-constrained cluster assignment. Missing numeric measurements remain missing. The next provider integration must occur before logical commitment and reuse Planner's provider/resource contracts. - -## Realization providers - -The internal `RealizationProvider` boundary reuses the existing window implementation, physical plan, manifest and quote contracts. The current provider delegates to the established compiler and ThreeStage allocator/emitter. It preserves supplied implementation IDs and validates their evidence; it does not create algorithms from display names, change selected families, or infer panes from slides. - -Complete alternatives continue to report `AlternativeCost` identities, capability-bound quote availability, costs and rejection reasons. Missing or stale evidence is unavailable, not a zero-cost alternative. `try_split_typed` retains allocation and emission errors; the legacy ThreeStage compatibility wrapper logs the concrete error before its existing fallback. SingleStage/ZeroStage allocator support and continuous/sliding derived maintenance remain gated. Backend-local compilation remains available through its existing independent compiler path. diff --git a/control_plane/docs/design-erp-deployment.md b/control_plane/docs/design-erp-deployment.md deleted file mode 100644 index 89543a5cb..000000000 --- a/control_plane/docs/design-erp-deployment.md +++ /dev/null @@ -1,91 +0,0 @@ -# ERP deployment integration - -The compile-and-publish request and backend-local planning snapshot may carry -an optional `erp` object. It contains the versioned sketch-bench artifact, the -deployment's current distribution descriptor, workload operation counts, -resource weights, error metric, minimum trial count, accuracy mode, and runtime -capabilities. - -Planning is query-workload-aware: ASAPPlanner still owns legal families and the -materialization DAG, while the backend adapter translates the selected ERP -parameter JSON into its typed runtime `SketchParams`. - -Hybrid mode has three observable outcomes: - -1. exact distribution and implementation match, sufficient trials, acceptable - measured error, and supported parameters: use the least-cost ERP point; -2. ERP miss, invalid artifact, unparseable parameters, or distribution drift: - use ASAPPlanner's theoretical parameters when runtime capabilities accept - them; and -3. neither empirical nor theoretical state is deployable: preserve the - pre-ASAP subtree for exact execution. - -Empirical mode does not silently claim a theoretical fallback: an ERP miss -goes directly to exact execution. Selection and fallback reasons are emitted as -structured tracing events. ERP observations remain empirical and must not be -rendered as formal `(epsilon, delta)` guarantees. - -Without an observation, ERP matches the complete distribution JSON by equality. -With an observation, Planner first checks a matching empirical fingerprint and -then evaluates all admissible fitted families against benchmark shapes. Custom -datasets can use bounded shape matching when their exact fingerprint is absent. -A caller supplies a fresh observation with every new plan generation. - -## Validation and current scope - -This section is for developers validating the deployment contract. Workload -selection must use the empirical accuracy model as well as the ERP cost model; -otherwise the default theoretical bound rejects the smaller measured state. -The deployment currently admits empirical accuracy only for epsilon-only KLL -rank queries using `kll-percall` / `lib` and `max_rank_err`. Other metrics, -implementations, and explicit `(epsilon, delta)` requests take the mode's -theoretical/exact fallback. Empirical provenance carries an unknown failure -probability, never an invented confidence guarantee. - -The checked-in [measured fixture](../tests/fixtures/erp-kll-measured.json) -comes from sketch-bench revision `dedc2d4`: KLL k=32, 1,000 uniform samples, -seed 42, and measured maximum rank error 0.051. Its ten resource trials are -**not** ten independent accuracy trials (`accuracy_runs` is one). Resource -timings are machine-specific. This is a reproducible integration fixture, -not evidence of a distribution-independent error bound. - -Shape-aware mode consumes Planner's shared `ErpShapeObservation`, with multiple -family/parameter fits, goodness-of-fit, fit-quality confidence, cardinality and -observed events. Benchmark rows retain `ErpDataShape`. The bounded keyed observer -fits uniform and Zipf rank masses without collapsing them into one family. -Numeric-value distribution fitting is a separate observation problem: unique -numeric values alone do not reveal a normal or uniform value distribution. -Callers may supply other families through the same shared contract. Unsupported, -ambiguous and out-of-distribution observations use the configured fallback. - -Run from the backend repository: - -```sh -cargo test -p control_plane measured_erp_kll_parameters_survive_workload_selection -cargo test -p data_plane --test asapquery_compatibility_process_e2e erp_measured_kll_collector_to_query_oracle -cargo test -p data_plane --test all_sketches_process_oracle_e2e --test e2e_controller_plans_and_backend_serves --test asapquery_compatibility_process_e2e -``` - -The first regression checks k=32 survives workload selection and physical -compilation; descriptor drift, implementation mismatch, and explicit delta -requirements select theoretical sizing with a distinct materialization -identity; an unsupported runtime preserves the exact subtree. - -The process test installs a real physical plan, runs the Collector's KLL -precompute runtime over a separate deterministic 1,000-value input, sends its -compacted portable state through OTLP to the backend binary, and checks a warm -PromQL result against an exact rank oracle (error <= 0.06). It has no exact -fallback server. It covers the supported distributed-collector path, not an -ERP-enabled raw-ingest deployment or an online drift detector. The older -transport fixtures install explicit QueryPlan/SummaryCatalog bindings and -exercise actual frame validation; they do not substitute for planner E2E. - -To regenerate the benchmark fixture with the matching sketch-bench revision, -use a new report path for each run (benchmark reports append): - -```sh -approxbench sketchbench --variant kll-percall --library lib --config 'k=32' --dataset uniform --size 1000 --cardinality 1 --dtype f64 --runs 10 --warmup-runs 1 --operations insert,query,merge --metrics throughput,cpu,memory --report erp.raw.jsonl -approxbench sketchbench --variant kll-percall --library lib --config 'k=32' --dataset uniform --size 1000 --cardinality 1 --dtype f64 --runs 10 --warmup-runs 1 --operations query --metrics accuracy --report erp.raw.jsonl -approxbench flatten erp.raw.jsonl --output erp.flat.jsonl -approxbench erp erp.flat.jsonl --producer-version dedc2d4 --output erp.json -``` diff --git a/control_plane/docs/offline-sketch-evidence.md b/control_plane/docs/offline-sketch-evidence.md deleted file mode 100644 index a67df43b5..000000000 --- a/control_plane/docs/offline-sketch-evidence.md +++ /dev/null @@ -1,86 +0,0 @@ -# Offline sketch evidence replay - -Audience: developers evaluating planner integration without a deployed data plane. - -`ControlPlaneCostModel::with_offline_evidence` accepts the planner's validated -offline provider. Candidate ordering compares update CPU nanoseconds only when -every candidate has compatible evidence for the exact parameters returned by -this deployment's sizing policy. Missing, stale, incompatible, or ambiguous -measurements preserve the existing order. Offline errors never change formal -accuracy guarantees. Existing physical and lifecycle costs retain their units; -CPU nanoseconds are not added to legacy dimensionless costs. - -For explicit integer-key point-frequency queries, -`with_offline_frequency_comparison(evidence, request)` additionally compares -query-matched sketch measurements against an exact snapshot baseline. The -request supplies the observed mean-error budget, number of reads, retained -state count, horizon and CPU/memory weights. This is a fixed-snapshot offline -comparison: the caller asserts the recorded integer-key distribution and probe -population apply. The mean-error threshold applies to that recorded population, -not to each queried key or future live data. - -The backend restricts candidates to measured power-of-two CMS layouts before -comparison and supplies its own formal minimum parameters from the tighter -workload and query accuracy. Observed error may select a larger measured sketch, -but never weakens formal sizing. Missing evidence, an unacceptable error budget, -or an exact winner yields `PassThrough`, preserving exact execution. Unfiltered -legacy frequency totals and `count_over_time` never receive point-frequency -error acceptance. `offline_frequency_recommendation(payload)` exposes the same -decision and rejection reasons used by the binder. - -The typed frequency example reads real comparison artifacts without starting a -data plane, using the planner revision pinned in this backend: - -```bash -cargo run -p control_plane --example offline_frequency_plan -- \ - comparison-evidence.json comparison-request.json 7 0.01 -``` - -It binds a named integer-key source and reports the chosen parameters, exact -alternative, cost estimates and preserved point readout. It does not certify -that an existing Prometheus metric or deployed materialization uses that source. - -Run the PromQL replay using the pinned planner dependency: - -```bash -cargo run -p control_plane --example offline_planner_replay -- \ - o11y_bench_promql.txt planner-evidence.json context.json > control-plane-o11y.json -``` - -For development against unpublished planner changes, the optional local-checkout -wrapper supplies source patches and records the checkout revisions: - -```bash -python3 tools/run-offline-planner-replay.py \ - --planner /path/to/ASAPPlanner \ - --queries /path/to/ASAPPlanner/crates/frontend-promql/tests/observability/data/o11y_bench_promql.txt \ - --evidence /path/to/planner-evidence.json \ - --context /path/to/context.json \ - --output /tmp/control-plane-o11y.json -``` - -The wrapper supplies local Cargo source patches for all three planner crates, -preserving a single set of IR types. It also changes Cargo.lock; normal use of -the pinned dependency requires no source patches. The normal backend sibling -dependencies (ASAPCollector and asap_sketchlib) -must remain available at the paths in its workspace manifests. - -The replay calls the real control-plane parser and typed summary binder for -exact, default, and empirical modes. It records per-query rejection/fallback, -selected summary states, matching update/state evidence, provenance, and elapsed -planning time. The selected offline context is an explicit simulation assumption; -it is not an assertion that an o11y metric has that measured distribution. -Source scans beneath summaries count as raw subtrees, so they are not themselves -evidence that the whole query fell back. Root fallback is a separate field. -Bare selectors, sort roots and comparison/filter roots retain the complete -original query as `KeepPreAsap`. They bind successfully while preserving label -predicates, ordering and filtering. Executable query compilation marks these -roots `ExactFallback` and requests no summary materializations; successful -binding therefore does not mean they are served by the warm tier. - -This is binding coverage, not successful deployment compilation or execution. -No collectors or query servers start. Point-frequency benchmark errors are -exported as observations with `error_applies_to_current_query: false`. -Whole-plan resource savings remain null without matched raw/residual physical -operator evidence. Unsupported summary binary operators lower to explicit warm -tier fallback; they are not silently executed with different semantics. diff --git a/control_plane/examples/audit_clickhouse_corpus.rs b/control_plane/examples/audit_clickhouse_corpus.rs deleted file mode 100644 index d7488a927..000000000 --- a/control_plane/examples/audit_clickhouse_corpus.rs +++ /dev/null @@ -1,174 +0,0 @@ -#[path = "support/automatic_clickhouse_corpus.rs"] -mod automatic; -use asap_frontend_sql::SqlCatalog; -use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; -use control_plane::clickhouse::{ClickHouseSqlWorkload, ClickHouseSqlWorkloadEntry}; -use control_plane::physical::compiler::{ - PlanEnvelope, PrecomputePlan, BACKEND_COMPAT, PLANNER_REVISION, -}; -use planner_types::pre_asap::{Column, DataType, Schema}; -use planner_types::types::AccuracyTarget; -use serde::Deserialize; -use serde_json::json; -use std::{fs, path::PathBuf}; - -#[derive(Deserialize)] -struct Corpus { - queries: Vec, -} - -#[derive(Deserialize)] -struct Row { - id: String, - clickhouse_sql: Option, - operators: Vec, -} - -fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { - let mut materialization = PrecomputeMaterialization::new( - AggregationType::Max, - String::new(), - std::collections::HashMap::from([("variant".into(), json!(2))]), - KeyByLabelNames::new(vec!["labels".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - "raw_samples.value".into(), - None, - Some("raw_samples".into()), - Some("value".into()), - ); - materialization.pane_origin_ms = Some(0); - let envelope = PlanEnvelope { - plan_id: 27, - plan_version: 1, - generated_at_unix_ms: 1_788_891_296_000, - activation_unix_ms: 1_788_891_296_000, - expiry_unix_ms: None, - backend_compat: BACKEND_COMPAT.into(), - planner_revision: PLANNER_REVISION.into(), - capability_snapshot_id: "sql27-main-eval".into(), - }; - let mut precompute_plan = - PrecomputePlan::build_backend_local(envelope.clone(), vec![materialization.clone()]) - .unwrap(); - let mut transmission_plan = control_plane::physical::compiler::build_transmission_plan( - envelope, - &precompute_plan, - &Default::default(), - ) - .unwrap(); - let sds = asap_types::summary_catalog::SummaryCatalog::from_materializations( - 27, - 1, - &[materialization], - ) - .unwrap(); - let reference = sds.reference().unwrap(); - precompute_plan.bind_catalog(&sds).unwrap(); - transmission_plan.summary_catalog = Some(reference); - ClickHouseSqlWorkload { - summary_catalog: sds, - precompute_plan, - transmission_plan, - tables: std::collections::HashMap::from([("raw_samples".into(), schema.clone())]), - accuracy: AccuracyTarget::Epsilon(0.01), - queries: vec![ClickHouseSqlWorkloadEntry { - sql, - start_ms: 1_788_848_096_000, - end_ms: 1_788_891_296_000, - cumulative: false, - }], - } -} - -#[tokio::main] -async fn main() { - if std::env::args().nth(1).as_deref() == Some("--automatic") { - automatic::run().await; - return; - } - let path = std::env::args_os() - .nth(1) - .map(PathBuf::from) - .expect("usage: audit_clickhouse_corpus CORPUS.json"); - let corpus: Corpus = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); - let schema = Schema::with_time_index( - vec![ - Column::new("metric", DataType::Utf8, false), - Column::new("labels", DataType::Utf8, false), - Column::new("ts_ms", DataType::Timestamp, false), - Column::new("value", DataType::Float64, false), - ], - 2, - vec![], - ); - let catalog = SqlCatalog::new().with_table("raw_samples", schema.clone()); - let accuracy = AccuracyTarget::Epsilon(0.01); - let mut rows = Vec::new(); - for row in corpus.queries { - let Some(sql) = row.clickhouse_sql else { - rows.push(json!({ - "id": row.id, - "operators": row.operators, - "parser_planner": "failure", - "reason": "missing ClickHouse SQL mapping" - })); - continue; - }; - let sql = sql.replace("{eval_ms}", "1788891296000"); - match control_plane::clickhouse::plan_clickhouse_sql(&sql, &catalog, accuracy.clone()).await - { - Ok(planned) => { - let publication = control_plane::clickhouse::compile_clickhouse_workload( - &publication_inputs(&schema, sql.clone()), - ) - .await; - match publication { - Ok(publication) => rows.push(json!({ - "id": row.id, - "operators": row.operators, - "parser_planner": "pass", - "publication": "pass", - "classification": "publication_ready", - "query_plan_nodes": publication.query_plan.entries.values().next().map(|entry| &entry.nodes), - "canonical_sql": planned.canonical_sql, - "physical": format!("{:?}", planned.physical), - "sql": sql, - })), - Err(error) => rows.push(json!({ - "id": row.id, - "operators": row.operators, - "parser_planner": "pass", - "publication": "failure", - "classification": "failure", - "reason": error.to_string(), - "canonical_sql": planned.canonical_sql, - "physical": format!("{:?}", planned.physical), - "sql": sql, - })), - } - } - Err(error) => rows.push(json!({ - "id": row.id, - "operators": row.operators, - "parser_planner": "failure", - "reason": error.to_string(), - "sql": sql, - })), - } - } - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "baseline": env!("CARGO_PKG_VERSION"), - "query_count": rows.len(), - "queries": rows, - })) - .unwrap() - ); -} diff --git a/control_plane/examples/calibration_candidates.rs b/control_plane/examples/calibration_candidates.rs deleted file mode 100644 index 928dc1475..000000000 --- a/control_plane/examples/calibration_candidates.rs +++ /dev/null @@ -1,183 +0,0 @@ -//! Export every bindable candidate for isolated measurement, without selecting a winner. -use control_plane::physical::{ - compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}, - workload_cost, -}; -use planner_types::post_asap::{SummaryExpr, SummaryNode}; -use serde_json::{json, Value}; -use std::{collections::BTreeMap, rc::Rc}; - -// Planner IR does not implement Serialize. Preserve actual DAG identity and -// typed variant/edges; leaf metadata uses explicitly labelled Debug encoding. -fn planner_forest(queries: &[control_plane::physical::compiler::QueryCompilationInput]) -> Value { - fn visit( - node: &Rc, - seen: &mut BTreeMap, - nodes: &mut BTreeMap, - ) -> usize { - let pointer = Rc::as_ptr(node) as usize; - if let Some(id) = seen.get(&pointer) { - return *id; - } - let id = seen.len(); - seen.insert(pointer, id); - let (kind, children, detail): (&str, Vec<&Rc>, Value) = match &node.expr { - SummaryExpr::KeepPreAsap(expr) => ( - "KeepPreAsap", - vec![], - json!({"query_expr_debug":format!("{expr:#?}")}), - ), - SummaryExpr::BinaryOp { - lhs, - rhs, - operator, - timing, - } => ( - "BinaryOp", - vec![lhs, rhs], - json!({"operator_debug":format!("{operator:?}"),"timing_debug":format!("{timing:?}")}), - ), - - SummaryExpr::ValueOperation { - child, - operation, - timing, - } => ( - "ValueOperation", - vec![child], - json!({"operation_debug":format!("{operation:?}"),"timing_debug":format!("{timing:?}")}), - ), - SummaryExpr::SummaryAgg { - child, - family, - reduction, - grouping, - input, - .. - } => ( - "SummaryAgg", - vec![child], - json!({"family_debug":format!("{family:?}"),"reduction_debug":format!("{reduction:?}"),"grouping_debug":format!("{grouping:?}"),"input_debug":format!("{input:?}")}), - ), - SummaryExpr::SummaryJoin { - outer, - inner, - key, - family, - } => ( - "SummaryJoin", - vec![outer, inner], - json!({"key_debug":format!("{key:?}"),"family_debug":format!("{family:?}")}), - ), - SummaryExpr::RelationalJoin { - left, - right, - kind, - pred, - .. - } => ( - "RelationalJoin", - vec![left, right], - json!({"kind_debug":format!("{kind:?}"),"predicate_debug":format!("{pred:?}")}), - ), - SummaryExpr::SummarySubtract { left, right } => { - ("SummarySubtract", vec![left, right], json!({})) - } - SummaryExpr::SummaryDelete { summary_input, key } => ( - "SummaryDelete", - vec![summary_input], - json!({"key_debug":format!("{key:?}")}), - ), - SummaryExpr::SummaryEstimate { - summary_input, - query, - } => ( - "SummaryEstimate", - vec![summary_input], - json!({"query_debug":format!("{query:?}")}), - ), - SummaryExpr::SummaryMerge { children, .. } => { - ("SummaryMerge", children.iter().collect(), json!({})) - } - }; - let inputs: Vec<_> = children - .into_iter() - .map(|child| visit(child, seen, nodes)) - .collect(); - nodes.insert(id,json!({"kind":kind,"inputs":inputs,"detail":detail,"schema_debug":format!("{:?}",node.schema),"guarantee_debug":format!("{:?}",node.guarantee)})); - id - } - let mut seen = BTreeMap::new(); - let mut nodes = BTreeMap::new(); - let roots:Vec<_>=queries.iter().map(|q|json!({"query_id":q.query_id,"original_promql":q.query_string,"root":visit(&q.selected_plan_root,&mut seen,&mut nodes)})).collect(); - json!({"encoding":"structured_graph_with_debug_metadata_v1","scope":"actual candidate Planner post-ASAP input before physical binding; not reconstructed from installed nodes","roots":roots,"nodes":nodes}) -} - -fn main() -> Result<(), Box> { - let path = std::env::args() - .nth(1) - .ok_or("usage: calibration_candidates SNAPSHOT.json [--metricsql]")?; - let metricsql = std::env::args().skip(2).any(|arg| arg == "--metricsql"); - let snapshot: BackendLocalPlanningInput = serde_json::from_slice(&std::fs::read(path)?)?; - let (request, environment) = snapshot.into_physical_compilation_request()?; - let mut results = Vec::new(); - for (index, candidate) in workload_cost::enumerate_exact_and_materialized_candidates(request)? - .into_iter() - .enumerate() - { - let queries = candidate.queries.clone(); - let enabled_materialization_keys = candidate.enabled_materialization_keys.clone(); - let planner_selected_queries = planner_forest(&queries); - let compiled = if metricsql { - DeploymentPlanCompiler.compile_metricsql(candidate, environment.clone()) - } else { - DeploymentPlanCompiler.compile_promql(candidate, environment.clone()) - }; - let plan = match compiled { - Ok(plan) => plan, - Err(error) => { - results.push( - json!({"candidate_index": index, "materialization_policy": enabled_materialization_keys, "planner_selected_queries": planner_selected_queries, "unavailable_reason": error.to_string()}), - ); - continue; - } - }; - let manifest = match workload_cost::manifest(&plan, &queries) { - Ok(manifest) => manifest, - Err(error) => { - results.push( - json!({"candidate_index": index, "materialization_policy": enabled_materialization_keys, "planner_selected_queries": planner_selected_queries, "unavailable_reason": error.to_string()}), - ); - continue; - } - }; - results.push(json!({ - "candidate_index": index, - "materialization_policy": enabled_materialization_keys, - "planner_selected_queries": planner_selected_queries, - "manifest": manifest, - "lifecycle_estimates": plan.lifecycle_estimates, - "install_request": { - "summary_catalog": plan.summary_catalog, - "collector_plans": plan.collector_plans, - "precompute_plan": plan.precompute_plan, - "transmission_plan": plan.transmission_plan, - "query_plan": plan.query_plan, - "storage_routing": null, - "adaptation_evidence": [] - } - })); - } - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "purpose":"calibration_only", - "compiler_identity": { - "backend_revision": control_plane::physical::compiler::BACKEND_REVISION, - "planner_revision": control_plane::physical::compiler::PLANNER_REVISION - }, - "candidates":results - }))? - ); - Ok(()) -} diff --git a/control_plane/examples/compile_clickhouse_workload.rs b/control_plane/examples/compile_clickhouse_workload.rs deleted file mode 100644 index dccc30157..000000000 --- a/control_plane/examples/compile_clickhouse_workload.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! Compile a SQL workload through the real Planner without preset materializations. -use std::io::Read; - -#[tokio::main] -async fn main() -> Result<(), Box> { - let mut input = String::new(); - std::io::stdin().read_to_string(&mut input)?; - let workload: control_plane::clickhouse::ClickHouseSqlAutomaticWorkload = - serde_json::from_str(&input)?; - let (publication, selection_trace) = - control_plane::clickhouse::compile_automatic_clickhouse_workload(&workload).await?; - let install = publication.install_request(None, Vec::new())?; - serde_json::to_writer_pretty( - std::io::stdout(), - &serde_json::json!({ - "publication": publication, - "selection_trace": selection_trace, - "install": install, - }), - )?; - Ok(()) -} diff --git a/control_plane/examples/compile_workload_artifact.rs b/control_plane/examples/compile_workload_artifact.rs deleted file mode 100644 index 0513ea1ed..000000000 --- a/control_plane/examples/compile_workload_artifact.rs +++ /dev/null @@ -1,31 +0,0 @@ -//! Control-plane entry point: cost-select a workload and emit its atomic install request. -use control_plane::physical::compiler::BackendLocalPlanningInput; - -fn main() -> Result<(), Box> { - let mut args = std::env::args().skip(1); - let path = args - .next() - .ok_or("usage: compile_workload_artifact SNAPSHOT.json [--metricsql] [--dot OUTPUT.dot]")?; - let mut metricsql = false; - let mut dot_path = None; - while let Some(argument) = args.next() { - match argument.as_str() { - "--metricsql" => metricsql = true, - "--dot" => { - dot_path = Some(args.next().ok_or("--dot requires an output path")?); - } - _ => return Err(format!("unexpected argument `{argument}`").into()), - } - } - let snapshot: BackendLocalPlanningInput = serde_json::from_slice(&std::fs::read(path)?)?; - let plan = if metricsql { - snapshot.compile_metricsql()? - } else { - snapshot.compile_promql()? - }; - if let Some(path) = dot_path { - std::fs::write(path, control_plane::physical::plan_dot::render(&plan))?; - } - println!("{}", serde_json::to_string_pretty(&plan)?); - Ok(()) -} diff --git a/control_plane/examples/offline_frequency_plan.rs b/control_plane/examples/offline_frequency_plan.rs deleted file mode 100644 index 218571bff..000000000 --- a/control_plane/examples/offline_frequency_plan.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! Bind a typed fixed-snapshot point-frequency query using offline evidence. -use std::rc::Rc; - -use anyhow::{bail, Result}; -use asap_aware_mapping::empirical_comparison::{ - OfflineComparisonEvidence, OfflineComparisonRequest, -}; -use control_plane::{ - physical::post_asap::{bind_query_expr_with_cost_model, cost_model::ControlPlaneCostModel}, - planner_selection::frequency, - types::AccuracyTarget, -}; -use planner_types::pre_asap::{AggIntent, Column, DataType, QueryExpr, Reduction, Schema, Source}; -use serde_json::json; - -fn main() -> Result<()> { - let args: Vec<_> = std::env::args().skip(1).collect(); - if args.len() != 4 { - bail!("usage: offline_frequency_plan COMPARISON.json REQUEST.json INTEGER_KEY EPSILON"); - } - let evidence: OfflineComparisonEvidence = - serde_json::from_str(&std::fs::read_to_string(&args[0])?)?; - let request: OfflineComparisonRequest = - serde_json::from_str(&std::fs::read_to_string(&args[1])?)?; - let item: i64 = args[2].parse()?; - let epsilon: f64 = args[3].parse()?; - if !epsilon.is_finite() || epsilon <= 0.0 || epsilon >= 1.0 { - bail!("epsilon must be finite and strictly between zero and one"); - } - let accuracy = AccuracyTarget::Epsilon(epsilon); - let model = ControlPlaneCostModel::new(accuracy.clone()) - .with_offline_frequency_comparison(evidence, request.clone()); - let intent = frequency(accuracy, Some(("key".into(), item.to_string()))); - let AggIntent::Extension { payload, .. } = &intent else { - unreachable!() - }; - let recommendation = model.offline_frequency_recommendation(payload); - let query = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![intent], - output_names: vec![], - having: None, - child: Rc::new(QueryExpr::Scan { - source: Source::TimeSeries { - metric: "offline_integer_snapshot".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Column::new("ts", DataType::Timestamp, false), - Column::new("key", DataType::Int64, false), - Column::new("value", DataType::Float64, false), - ], - 0, - vec![], - ), - }), - }; - let bound = bind_query_expr_with_cost_model(&query, &model)?; - let root_raw_fallback = matches!(&bound, - control_plane::physical::post_asap::PhysicalExpr::Committed( - control_plane::physical::post_asap::PostAsapPlan::Summary(node) - ) if matches!(node.expr, planner_types::post_asap::SummaryExpr::KeepPreAsap(_))); - let (recommendation, unavailable_reason) = match recommendation { - Ok(value) => (Some(value), None), - Err(reason) => (None, Some(reason)), - }; - serde_json::to_writer_pretty( - std::io::stdout(), - &json!({ - "scope":"typed offline point-frequency recommendation and control-plane binding", - "request":request, "item":item, "formal_epsilon":epsilon, - "recommendation":recommendation,"unavailable_reason":unavailable_reason, - "bound_plan":format!("{bound:#?}"), "root_raw_fallback":root_raw_fallback, - "limitations":["The caller asserts the fixed-snapshot integer-key benchmark context", - "Observed mean error applies to the recorded offline probe population, not an individual key guarantee", - "No materializations deployed or data-plane execution performed"] - }), - )?; - Ok(()) -} diff --git a/control_plane/examples/offline_planner_replay.rs b/control_plane/examples/offline_planner_replay.rs deleted file mode 100644 index 78b6e1dd0..000000000 --- a/control_plane/examples/offline_planner_replay.rs +++ /dev/null @@ -1,160 +0,0 @@ -//! Offline control-plane binding, without starting collectors or query servers. -use std::{collections::HashSet, rc::Rc, time::Instant}; - -use anyhow::{bail, Context, Result}; -use asap_aware_mapping::empirical_cost::{ - EmpiricalEvidenceProvider, EvidenceArtifact, EvidenceContext, -}; -use control_plane::{ - physical::post_asap::{ - bind_query_expr_with_cost_model, cost_model::ControlPlaneCostModel, PhysicalExpr, - PostAsapPlan, - }, - query_parser::parse_query_expr_canonical, - types::AccuracyTarget, -}; -use planner_types::post_asap::{SummaryExpr, SummaryFamilyType, SummaryNode}; -use serde_json::{json, Value}; - -fn inspect( - node: &Rc, - model: &ControlPlaneCostModel, - seen: &mut HashSet, - states: &mut Vec, - raw: &mut usize, -) { - if !seen.insert(Rc::as_ptr(node) as usize) { - return; - } - match &node.expr { - SummaryExpr::KeepPreAsap(_) => *raw += 1, - SummaryExpr::SummaryAgg { family, child, .. } => { - if let SummaryFamilyType::Sketch(kind, _) = family { - let lookup = model - .offline_evidence() - .map(|provider| provider.lookup(kind.algorithm(), kind.params())); - let (measurement, reason) = match lookup { - Some(Ok(row)) => ( - Some(json!({ - "record_id": row.id, - "provenance": row.provenance, - "update_cpu_ns": row.metrics.resources.cpu.update_cpu_ns, - "retained_bytes": row.metrics.resources.retained_memory_bytes, - "offline_error_observation": row.error, - "error_applies_to_current_query": false, - })), - None, - ), - Some(Err(error)) => (None, Some(error.to_string())), - None => (None, Some("offline evidence not supplied".into())), - }; - states.push(json!({"algorithm":kind.algorithm(), "params":kind.params(), "measurement":measurement, "unavailable_reason":reason})); - } else { - states.push(json!({"exact_family":format!("{family:?}")})); - } - inspect(child, model, seen, states, raw); - } - SummaryExpr::SummaryEstimate { summary_input, .. } - | SummaryExpr::SummaryDelete { summary_input, .. } => { - inspect(summary_input, model, seen, states, raw) - } - SummaryExpr::ValueOperation { child, .. } => inspect(child, model, seen, states, raw), - SummaryExpr::SummaryMerge { children, .. } => { - for child in children { - inspect(child, model, seen, states, raw); - } - } - SummaryExpr::SummaryJoin { - outer: lhs, - inner: rhs, - .. - } - | SummaryExpr::SummarySubtract { - left: lhs, - right: rhs, - } - | SummaryExpr::RelationalJoin { - left: lhs, - right: rhs, - .. - } - | SummaryExpr::BinaryOp { lhs, rhs, .. } => { - inspect(lhs, model, seen, states, raw); - inspect(rhs, model, seen, states, raw); - } - } -} - -fn main() -> Result<()> { - let args: Vec<_> = std::env::args().skip(1).collect(); - if args.len() != 1 && args.len() != 3 { - bail!("usage: offline_planner_replay QUERIES.txt [EVIDENCE.json CONTEXT.json]"); - } - let corpus = std::fs::read_to_string(&args[0])?; - let evidence = if args.len() == 3 { - let artifact: EvidenceArtifact = serde_json::from_str(&std::fs::read_to_string(&args[1])?)?; - let context: EvidenceContext = serde_json::from_str(&std::fs::read_to_string(&args[2])?)?; - Some((artifact, context)) - } else { - None - }; - let mut rows = Vec::new(); - let mut modes = vec!["exact", "default"]; - if evidence.is_some() { - modes.push("empirical"); - } - for mode in modes { - let accuracy = if mode == "exact" { - AccuracyTarget::Exact - } else { - AccuracyTarget::Epsilon(0.01) - }; - let mut model = ControlPlaneCostModel::new(accuracy.clone()); - if mode == "empirical" { - let (artifact, context) = evidence.as_ref().context("missing evidence")?; - model = model.with_offline_evidence(EmpiricalEvidenceProvider::new( - artifact.clone(), - context.clone(), - )?); - } - for query in corpus - .lines() - .map(str::trim) - .filter(|q| !q.is_empty() && !q.starts_with('#')) - { - let start = Instant::now(); - let result = parse_query_expr_canonical(query, accuracy.clone()).and_then(|expr| { - bind_query_expr_with_cost_model(&expr, &model).map_err(Into::into) - }); - let elapsed_ns = start.elapsed().as_nanos(); - let result = match result { - Ok(PhysicalExpr::Committed(PostAsapPlan::Summary(node))) => { - let mut states = Vec::new(); - let mut raw = 0; - inspect(&node, &model, &mut HashSet::new(), &mut states, &mut raw); - json!({"status":"bound", "states":states, "raw_subtrees":raw, - "root_raw_fallback":matches!(node.expr, SummaryExpr::KeepPreAsap(_)), - "summary_plan":format!("{node:#?}")}) - } - Ok(plan) => json!({"status":"other_physical_plan", "plan":format!("{plan:?}")}), - Err(error) => json!({"status":"rejected", "reason":error.to_string()}), - }; - rows.push( - json!({"query":query,"mode":mode,"planning_elapsed_ns":elapsed_ns,"result":result}), - ); - } - } - serde_json::to_writer_pretty( - std::io::stdout(), - &json!({ - "schema_version":1, - "evaluation":"offline control-plane parser and typed summary binder", - "corpus_path":args[0], - "offline_context":evidence.as_ref().map(|(_, context)|context), - "limitations":["No deployed execution or measured end-to-end speedup", "Binding does not establish executable placement or complete physical cost", "Offline point-frequency errors do not establish current query guarantees", "raw_subtrees includes necessary source scans beneath summaries"], - "estimated_end_to_end_savings":null, - "rows":rows - }), - )?; - Ok(()) -} diff --git a/control_plane/examples/support/automatic_clickhouse_corpus.rs b/control_plane/examples/support/automatic_clickhouse_corpus.rs deleted file mode 100644 index 71cd2dfd3..000000000 --- a/control_plane/examples/support/automatic_clickhouse_corpus.rs +++ /dev/null @@ -1,139 +0,0 @@ -use control_plane::clickhouse::{ClickHouseSqlAutomaticWorkload, ClickHouseSqlWorkloadEntry}; -use control_plane::physical::compiler::{PlanEnvelope, BACKEND_COMPAT, PLANNER_REVISION}; -use planner_types::pre_asap::{Column, DataType, Schema}; -use planner_types::types::AccuracyTarget; -use serde::Deserialize; -use serde_json::json; -use std::{fs, path::PathBuf}; - -#[derive(Deserialize)] -struct Corpus { - queries: Vec, -} - -#[derive(Deserialize)] -struct Row { - id: String, - clickhouse_sql: Option, - operators: Vec, - promql: String, -} - -fn publication_inputs( - schema: &Schema, - sql: String, - window_ms: u64, -) -> ClickHouseSqlAutomaticWorkload { - ClickHouseSqlAutomaticWorkload { - envelope: PlanEnvelope { - plan_id: 27, - plan_version: 1, - generated_at_unix_ms: 1_788_891_296_000, - activation_unix_ms: 1_788_891_296_000, - expiry_unix_ms: None, - backend_compat: BACKEND_COMPAT.into(), - planner_revision: PLANNER_REVISION.into(), - capability_snapshot_id: "sql27-automatic-eval".into(), - }, - tables: std::collections::HashMap::from([("raw_samples".into(), schema.clone())]), - accuracy: AccuracyTarget::Epsilon(0.01), - queries: vec![ClickHouseSqlWorkloadEntry { - sql, - start_ms: 1_788_891_296_001 - window_ms, - end_ms: 1_788_891_296_001, - cumulative: false, - }], - } -} - -pub async fn run() { - let path = std::env::args_os() - .nth(2) - .map(PathBuf::from) - .expect("usage: audit_clickhouse_corpus --automatic CORPUS.json"); - let corpus: Corpus = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); - let schema = Schema::with_time_index( - vec![ - Column::new("metric", DataType::Utf8, false), - Column::new( - "labels", - DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Utf8), - value_nullable: false, - }, - false, - ), - Column::new("ts_ms", DataType::Int64, false), - Column::new("value", DataType::Float64, false), - ], - 2, - vec![], - ); - let mut rows = Vec::new(); - for row in corpus.queries { - let Some(sql) = row.clickhouse_sql else { - rows.push(json!({ - "id": row.id, - "operators": row.operators, - "parser_planner": "failure", - "reason": "missing ClickHouse SQL mapping" - })); - continue; - }; - let sql = sql.replace("{eval_ms}", "1788891296000"); - // Evaluation span only: SQL source boundaries are still validated by the compiler. - let window_ms = row - .promql - .split('[') - .skip(1) - .filter_map(|range| { - let digits = range.chars().take_while(char::is_ascii_digit).count(); - let number = range[..digits].parse::().ok()?; - let unit = range.as_bytes().get(digits)?; - let multiplier = match unit { - b's' => 1_000, - b'm' => 60_000, - b'h' => 3_600_000, - b'd' => 86_400_000, - _ => return None, - }; - Some(number * multiplier) - }) - .max() - .unwrap_or(300_000); - match control_plane::clickhouse::compile_automatic_clickhouse_workload(&publication_inputs( - &schema, - sql.clone(), - window_ms, - )) - .await - { - Ok((publication, traces)) => rows.push(json!({ - "id": row.id, "operators": row.operators, "sql": sql, - "publication": "pass", "execution": "not_run", - "selection_traces": traces, - "window_start_ms": 1_788_891_296_001u64 - window_ms, - "window_end_ms": 1_788_891_296_001u64, - "install": publication.install_request(None, Vec::new()).unwrap(), - "summary_catalog": publication.summary_catalog, - "precompute_plan": publication.precompute_plan, - "query_plan": publication.query_plan, - })), - Err(error) => rows.push(json!({ - "id": row.id, "operators": row.operators, "sql": sql, - "publication": "failure", "execution": "not_run", "reason": error.to_string(), - })), - } - } - - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "baseline": env!("CARGO_PKG_VERSION"), - "query_count": rows.len(), - "queries": rows, - })) - .unwrap() - ); -} diff --git a/control_plane/examples/workload_cost_manifest.rs b/control_plane/examples/workload_cost_manifest.rs deleted file mode 100644 index f01fd0574..000000000 --- a/control_plane/examples/workload_cost_manifest.rs +++ /dev/null @@ -1,28 +0,0 @@ -//! Emit pricing requirements; never fabricate quotes or publish a plan. -use control_plane::physical::{ - compiler::BackendLocalPlanningInput, compiler::DeploymentPlanCompiler, workload_cost, -}; - -fn main() -> Result<(), Box> { - let path = std::env::args() - .nth(1) - .ok_or("usage: workload_cost_manifest SNAPSHOT.json")?; - let snapshot: BackendLocalPlanningInput = - serde_json::from_str(&std::fs::read_to_string(path)?)?; - let (request, environment) = snapshot.into_physical_compilation_request()?; - let manifests = workload_cost::enumerate_exact_and_materialized_candidates(request)? - .into_iter() - .filter_map(|candidate| { - let queries = candidate.queries.clone(); - DeploymentPlanCompiler - .compile_promql(candidate, environment.clone()) - .and_then(|plan| workload_cost::manifest(&plan, &queries)) - .ok() - }) - .collect::>(); - if manifests.is_empty() { - return Err("no bindable workload cost manifests".into()); - } - println!("{}", serde_json::to_string_pretty(&manifests)?); - Ok(()) -} diff --git a/control_plane/proto/anyvalue.proto b/control_plane/proto/anyvalue.proto deleted file mode 100644 index 8ed973152..000000000 --- a/control_plane/proto/anyvalue.proto +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2019, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This file is copied and modified from https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/common/v1/common.proto -// Modifications: -// - Removal of unneeded InstrumentationLibrary and StringKeyValue messages. -// - Change of go_package to reference a package in this repo. -// - Removal of gogoproto usage. - -syntax = "proto3"; - -package opamp.proto; - -option go_package = "github.com/open-telemetry/opamp-go/protobufs"; - -// AnyValue is used to represent any type of attribute value. AnyValue may contain a -// primitive value such as a string or integer or it may contain an arbitrary nested -// object containing arrays, key-value lists and primitives. -message AnyValue { - // The value is one of the listed fields. It is valid for all values to be unspecified - // in which case this AnyValue is considered to be "null". - oneof value { - string string_value = 1; - bool bool_value = 2; - int64 int_value = 3; - double double_value = 4; - ArrayValue array_value = 5; - KeyValueList kvlist_value = 6; - bytes bytes_value = 7; - } -} - -// ArrayValue is a list of AnyValue messages. We need ArrayValue as a message -// since oneof in AnyValue does not allow repeated fields. -message ArrayValue { - // Array of values. The array may be empty (contain 0 elements). - repeated AnyValue values = 1; -} - -// KeyValueList is a list of KeyValue messages. We need KeyValueList as a message -// since `oneof` in AnyValue does not allow repeated fields. Everywhere else where we need -// a list of KeyValue messages (e.g. in Span) we use `repeated KeyValue` directly to -// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches -// are semantically equivalent. -message KeyValueList { - // A collection of key/value pairs of key-value pairs. The list may be empty (may - // contain 0 elements). - repeated KeyValue values = 1; -} - -// KeyValue is a key-value pair that is used to store Span attributes, Link -// attributes, etc. -message KeyValue { - string key = 1; - AnyValue value = 2; -} diff --git a/control_plane/proto/control_plane.proto b/control_plane/proto/control_plane.proto deleted file mode 100644 index de1bb002f..000000000 --- a/control_plane/proto/control_plane.proto +++ /dev/null @@ -1,115 +0,0 @@ -syntax = "proto3"; - -package control_plane.v1; - -option go_package = "github.com/ProjectASAP/control_plane/proto/control_planev1"; - -import "google/protobuf/duration.proto"; -import "google/protobuf/timestamp.proto"; - -// ── Enumerations ────────────────────────────────────────────────────────────── - -enum AggType { - AGG_TYPE_UNSPECIFIED = 0; - AGG_TYPE_QUANTILE = 1; - AGG_TYPE_CARDINALITY = 2; - AGG_TYPE_FREQUENCY = 3; -} - -enum SketchType { - SKETCH_TYPE_UNSPECIFIED = 0; - SKETCH_TYPE_DDSKETCH = 1; - SKETCH_TYPE_KLL = 2; - SKETCH_TYPE_HLL = 3; - SKETCH_TYPE_COUNT_SKETCH = 4; - SKETCH_TYPE_COUNT_MIN_SKETCH = 5; -} - -enum OutputMode { - OUTPUT_MODE_UNSPECIFIED = 0; - OUTPUT_MODE_RAW = 1; - OUTPUT_MODE_SKETCH = 2; -} - -enum ProcessorMode { - PROCESSOR_MODE_UNSPECIFIED = 0; - PROCESSOR_MODE_BATCH = 1; - PROCESSOR_MODE_WINDOW = 2; -} - -// ── Core types ──────────────────────────────────────────────────────────────── - -message QueryWorkload { - string metric_name = 1; - map label_filters = 2; - repeated string group_by_labels = 3; - repeated AggType aggregations = 4; - google.protobuf.Duration time_window = 5; - google.protobuf.Duration repeat_every = 6; - double accuracy_sla = 7; // relative error, e.g. 0.01 - google.protobuf.Duration latency_sla = 8; -} - -message SketchParams { - double relative_accuracy = 1; // DDSketch - int32 k = 2; // KLL - int32 precision = 3; // HLL - int32 rows = 4; // CountSketch / CountMinSketch - int32 cols = 5; - repeated double quantiles = 6; -} - -message AgentCollectorConfig { - OutputMode output_mode = 1; - SketchType sketch_type = 2; - SketchParams sketch_params = 3; - repeated string aggregate_by = 4; - repeated string label_matchers = 5; - google.protobuf.Duration window_duration = 6; - ProcessorMode mode = 7; - bool transmit_sketch = 8; - bool drop_original = 9; -} - -message GatewayCollectorConfig { - bool passthrough = 1; -} - -message BackendCollectorConfig { - SketchType merge_sketch_type = 1; - repeated string group_by = 2; -} - -message CollectionPlan { - AgentCollectorConfig agent_config = 1; - GatewayCollectorConfig gateway_config = 2; - BackendCollectorConfig backend_config = 3; - reserved 4; - reserved "precompute"; - google.protobuf.Timestamp valid_until = 5; -} - -// ── RPCs ────────────────────────────────────────────────────────────────────── - -message PlanRequest { - QueryWorkload workload = 1; -} - -message PlanResponse { - CollectionPlan plan = 1; - string status = 2; -} - -message PlanStatusRequest { - string metric_name = 1; -} - -message PlanStatusResponse { - CollectionPlan current_plan = 1; - string status = 2; -} - -service ControllerService { - rpc Plan(PlanRequest) returns (PlanResponse); - rpc GetPlanStatus(PlanStatusRequest) returns (PlanStatusResponse); -} diff --git a/control_plane/proto/feedback.proto b/control_plane/proto/feedback.proto deleted file mode 100644 index 235a46707..000000000 --- a/control_plane/proto/feedback.proto +++ /dev/null @@ -1,52 +0,0 @@ -// ASAP runtime-samples wire schema. -// -// Carries batches of `sketch-core::report::Record` from an -// agent's `sketch-runtime::GrpcExporter` into the control plane's -// real-time decision loop. The Record payload is forwarded as -// JSON so the control plane stays forward-compatible with new -// v1 schema fields without a proto rebump. -// -// Schema version is `sketch-core::report::SCHEMA_VERSION` (1 -// today). A bump implies both sides get recompiled. - -syntax = "proto3"; - -package asap.runtime.v1; - -// One record per emitted bench window. Mirrors -// `sketch-core::report::Record`: the `source` / `sketch` / -// `impl_name` tuple is the per-key routing handle on the -// control-plane side (→ `RuntimeSamplesStore`'s `SampleKey`); -// the rest of the record rides in `payload_json` so new -// fields on `Record` flow through without a proto change. -message RuntimeRecord { - string source = 1; - string sketch = 2; - string impl_name = 3; - uint32 schema_version = 4; - string payload_json = 5; -} - -// A batched push. `GrpcExporter`'s default batch is 64 records -// / 100 ms (whichever triggers first) — small enough to keep -// decision latency bounded, large enough to amortize gRPC -// per-call overhead. -message PushBatch { - repeated RuntimeRecord records = 1; -} - -message PushAck { - // Number of records the server accepted (could be less than - // records.size() if a few failed validation; log + continue - // on the client side in that case). - uint64 accepted = 1; -} - -service RuntimeSamples { - // Unary RPC per batch — simpler than bidi streaming, and the - // client already batches before pushing so HTTP/2 mux handles - // throughput without a per-message acks-on-stream dance. - // tonic negotiates gzip if `Accept-Encoding: gzip` is set on - // the server. - rpc Push(PushBatch) returns (PushAck); -} diff --git a/control_plane/proto/opamp.proto b/control_plane/proto/opamp.proto deleted file mode 100644 index 219118c8c..000000000 --- a/control_plane/proto/opamp.proto +++ /dev/null @@ -1,1079 +0,0 @@ -// Copyright 2021, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// OpAMP: Open Agent Management Protocol (https://github.com/open-telemetry/opamp-spec) - -syntax = "proto3"; - -package opamp.proto; - -import "anyvalue.proto"; - -option go_package = "github.com/open-telemetry/opamp-go/protobufs"; - -message AgentToServer { - // Globally unique identifier of the running instance of the Agent. SHOULD remain - // unchanged for the lifetime of the Agent process. - // MUST be 16 bytes long and SHOULD be generated using the UUID v7 spec. - bytes instance_uid = 1; - - // The sequence number is incremented by 1 for every AgentToServer sent - // by the Agent. This allows the Server to detect that it missed a message when - // it notices that the sequence_num is not exactly by 1 greater than the previously - // received one. - uint64 sequence_num = 2; - - // Data that describes the Agent, its type, where it runs, etc. - // May be omitted if nothing changed since last AgentToServer message. - AgentDescription agent_description = 3; - - // Bitmask of flags defined by AgentCapabilities enum. - // All bits that are not defined in AgentCapabilities enum MUST be set to 0 by - // the Agent. This allows extending the protocol and the AgentCapabilities enum - // in the future such that old Agents automatically report that they don't - // support the new capability. - // This field MUST be always set. - uint64 capabilities = 4; - - // The current health of the Agent and sub-components. The top-level ComponentHealth represents - // the health of the Agent overall. May be omitted if nothing changed since last AgentToServer - // message. - // Status: [Beta] - ComponentHealth health = 5; - - // The current effective configuration of the Agent. The effective configuration is - // the one that is currently used by the Agent. The effective configuration may be - // different from the remote configuration received from the Server earlier, e.g. - // because the Agent uses a local configuration instead (or in addition). - // - // This field SHOULD be unset if the effective config is unchanged since the last - // AgentToServer message. - EffectiveConfig effective_config = 6; - - // The status of the remote config that was previously received from the Server. - // This field SHOULD be unset if the remote config status is unchanged since the - // last AgentToServer message. - RemoteConfigStatus remote_config_status = 7; - - // The list of the Agent packages, including package statuses. This field SHOULD be - // unset if this information is unchanged since the last AgentToServer message for - // this Agent was sent in the stream. - // Status: [Beta] - PackageStatuses package_statuses = 8; - - // AgentDisconnect MUST be set in the last AgentToServer message sent from the - // Agent to the Server. - AgentDisconnect agent_disconnect = 9; - - // Bit flags as defined by AgentToServerFlags bit masks. - uint64 flags = 10; - - // A request to create connection settings. This field is set for flows where - // the Agent initiates the creation of connection settings. - // Status: [Development] - ConnectionSettingsRequest connection_settings_request = 11; - - // A message indicating custom capabilities supported by the Agent. - // Status: [Development] - CustomCapabilities custom_capabilities = 12; - - // A custom message sent from an Agent to the Server. - // Status: [Development] - CustomMessage custom_message = 13; - - // A message indicating the components that are available for configuration on the agent. - // Status: [Development] - AvailableComponents available_components = 14; - - // The status of the OfferedConnectionSettings that was previously received - // from the Server. This field SHOULD be unset if the offered connection - // settings status is unchanged since the last AgentToServer message. - // Status: [Development] - ConnectionSettingsStatus connection_settings_status = 15; -} - -enum AgentToServerFlags { - AgentToServerFlags_Unspecified = 0; - - // AgentToServerFlags is a bit mask. Values below define individual bits. - - // The Agent requests Server go generate a new instance_uid, which will - // be sent back in ServerToAgent message - AgentToServerFlags_RequestInstanceUid = 0x00000001; -} - -// AgentDisconnect is the last message sent from the Agent to the Server. The Server -// SHOULD forget the association of the Agent instance with the message stream. -// -// If the message stream is closed in the transport layer then the Server SHOULD -// forget association of all Agent instances that were previously established for -// this message stream using AgentConnect message, even if the corresponding -// AgentDisconnect message were not explicitly received from the Agent. -message AgentDisconnect { -} - -// ConnectionSettingsRequest is a request from the Agent to the Server to create -// and respond with an offer of connection settings for the Agent. -// Status: [Development] -message ConnectionSettingsRequest { - // Request for OpAMP connection settings. If this field is unset - // then the ConnectionSettingsRequest message is empty and is not actionable - // for the Server. - OpAMPConnectionSettingsRequest opamp = 1; - - // In the future we can add request fields for non-OpAMP connection types - // (own telemetry, other connections). -} - -// OpAMPConnectionSettingsRequest is a request for the Server to produce -// a OpAMPConnectionSettings in its response. -// Status: [Development] -message OpAMPConnectionSettingsRequest { - // A request to create a client certificate. This is used to initiate a - // Client Signing Request (CSR) flow. - // Required. - CertificateRequest certificate_request = 1; -} - -// Status: [Development] -message CertificateRequest { - // PEM-encoded Client Certificate Signing Request (CSR), signed by client's private key. - // The Server SHOULD validate the request and SHOULD respond with a - // OpAMPConnectionSettings where the certificate.cert contains the issued - // certificate. - bytes csr = 1; -} - -// AvailableComponents contains metadata relating to the components included -// within the agent. -// status: [Development] -message AvailableComponents { - // A map of a unique component ID to details about the component. - // This may be omitted from the message if the server has not - // explicitly requested it be sent by setting the ReportAvailableComponents - // flag in the previous ServerToAgent message. - map components = 1; - - // Agent-calculated hash of the components. - // This hash should be included in every AvailableComponents message. - bytes hash = 2; - } - -message ComponentDetails { - // Extra key/value pairs that may be used to describe the component. - // The key/value pairs are according to semantic conventions, see: - // https://opentelemetry.io/docs/specs/semconv/ - // - // For example, you may use the "code" semantic conventions to - // report the location of the code for a specific component: - // https://opentelemetry.io/docs/specs/semconv/attributes-registry/code/ - // - // Or you may use the "vcs" semantic conventions to report the - // repository the component may be a part of: - // https://opentelemetry.io/docs/specs/semconv/attributes-registry/vcs/ - repeated KeyValue metadata = 1; - - // A map of component ID to sub components details. It can nest as deeply as needed to - // describe the underlying system. - map sub_component_map = 2; -} - - -message ServerToAgent { - // Agent instance uid. MUST match the instance_uid field in AgentToServer message. - // Used for multiplexing messages from/to multiple agents using one message stream. - bytes instance_uid = 1; - - // error_response is set if the Server wants to indicate that something went wrong - // during processing of an AgentToServer message. If error_response is set then - // all other fields below must be unset and vice versa, if any of the fields below is - // set then error_response must be unset. - ServerErrorResponse error_response = 2; - - // remote_config field is set when the Server has a remote config offer for the Agent. - AgentRemoteConfig remote_config = 3; - - // This field is set when the Server wants the Agent to change one or more - // of its client connection settings (destination, headers, certificate, etc). - // Status: [Beta] - ConnectionSettingsOffers connection_settings = 4; - - // This field is set when the Server has packages to offer to the Agent. - // Status: [Beta] - PackagesAvailable packages_available = 5; - - // Bit flags as defined by ServerToAgentFlags bit masks. - uint64 flags = 6; - - // Bitmask of flags defined by ServerCapabilities enum. - // All bits that are not defined in ServerCapabilities enum MUST be set to 0 - // by the Server. This allows extending the protocol and the ServerCapabilities - // enum in the future such that old Servers automatically report that they - // don't support the new capability. - // This field MUST be set in the first ServerToAgent sent by the Server and MAY - // be omitted in subsequent ServerToAgent messages by setting it to - // UnspecifiedServerCapability value. - uint64 capabilities = 7; - - // Properties related to identification of the Agent, which can be overridden - // by the Server if needed. - AgentIdentification agent_identification = 8; - - // Allows the Server to instruct the Agent to perform a command, e.g. RESTART. This field should not be specified - // with fields other than instance_uid and capabilities. If specified, other fields will be ignored and the command - // will be performed. - // Status: [Beta] - ServerToAgentCommand command = 9; - - // A message indicating custom capabilities supported by the Server. - // Status: [Development] - CustomCapabilities custom_capabilities = 10; - - // A custom message sent from the Server to an Agent. - // Status: [Development] - CustomMessage custom_message = 11; -} - -enum ServerToAgentFlags { - ServerToAgentFlags_Unspecified = 0; - - // Flags is a bit mask. Values below define individual bits. - - // ReportFullState flag can be used by the Server if the Agent did not include the - // particular bit of information in the last status report (which is an allowed - // optimization) but the Server detects that it does not have it (e.g. was - // restarted and lost state). The detection happens using - // AgentToServer.sequence_num values. - // The Server asks the Agent to report full status. - ServerToAgentFlags_ReportFullState = 0x00000001; - - // ReportAvailableComponents flag can be used by the server if the Agent did - // not include the full AvailableComponents message, but only the hash. - // If this flag is specified, the agent will populate available_components.components - // with a full description of the agent's components. - // Status: [Development] - ServerToAgentFlags_ReportAvailableComponents = 0x00000002; -} - -enum ServerCapabilities { - // The capabilities field is unspecified. - ServerCapabilities_Unspecified = 0; - - // The Server can accept status reports. This bit MUST be set, since all Server - // MUST be able to accept status reports. - ServerCapabilities_AcceptsStatus = 0x00000001; - // The Server can offer remote configuration to the Agent. - ServerCapabilities_OffersRemoteConfig = 0x00000002; - // The Server can accept EffectiveConfig in AgentToServer. - ServerCapabilities_AcceptsEffectiveConfig = 0x00000004; - // The Server can offer Packages. - // Status: [Beta] - ServerCapabilities_OffersPackages = 0x00000008; - // The Server can accept Packages status. - // Status: [Beta] - ServerCapabilities_AcceptsPackagesStatus = 0x00000010; - // The Server can offer connection settings. - // Status: [Beta] - ServerCapabilities_OffersConnectionSettings = 0x00000020; - // The Server can accept ConnectionSettingsRequest and respond with an offer. - // Status: [Development] - ServerCapabilities_AcceptsConnectionSettingsRequest = 0x00000040; - - // Add new capabilities here, continuing with the least significant unused bit. -} - -// The OpAMPConnectionSettings message is a collection of fields which comprise an -// offer from the Server to the Agent to use the specified settings for OpAMP -// connection. -// Status: [Beta] -message OpAMPConnectionSettings { - // OpAMP Server URL This MUST be a WebSocket or HTTP URL and MUST be non-empty, for - // example: "wss://example.com:4318/v1/opamp" - string destination_endpoint = 1; - - // Optional headers to use when connecting. Typically used to set access tokens or - // other authorization headers. For HTTP-based protocols the Agent should - // set these in the request headers. - // For example: - // key="Authorization", Value="Basic YWxhZGRpbjpvcGVuc2VzYW1l". - Headers headers = 2; - - // The Agent should use the offered certificate to connect to the destination - // from now on. If the Agent is able to validate and connect using the offered - // certificate the Agent SHOULD forget any previous client certificates - // for this connection. - // This field is optional: if omitted the client SHOULD NOT use a client-side certificate. - // This field can be used to perform a client certificate revocation/rotation. - TLSCertificate certificate = 3; - - // The Agent MUST periodically send an AgentToServer message if the - // AgentCapabilities_ReportsHeartbeat capability is true. At a minimum the instance_uid - // field MUST be set. - // - // An HTTP Client MUST use the value as polling interval, if heartbeat_interval_seconds is non-zero. - // - // A heartbeat is used to keep the connection active and inform the server that the Agent - // is still alive and active. - // - // If this field has no value or is set to 0, the Agent should not send any heartbeats. - // Status: [Development] - uint64 heartbeat_interval_seconds = 4; - - // Optional connection specific TLS settings. - // Status: [Development] - TLSConnectionSettings tls = 5; - - // Optional connection specific proxy settings. - // Status: [Development] - ProxyConnectionSettings proxy = 6; -} - -// The TelemetryConnectionSettings message is a collection of fields which comprise an -// offer from the Server to the Agent to use the specified settings for a network -// connection to report own telemetry. -// Status: [Beta] -message TelemetryConnectionSettings { - // The value MUST be a full URL an OTLP/HTTP/Protobuf receiver with path. Schema - // SHOULD begin with "https://", for example "https://example.com:4318/v1/metrics" - // The Agent MAY refuse to send the telemetry if the URL begins with "http://". - string destination_endpoint = 1; - - // Optional headers to use when connecting. Typically used to set access tokens or - // other authorization headers. For HTTP-based protocols the Agent should - // set these in the request headers. - // For example: - // key="Authorization", Value="Basic YWxhZGRpbjpvcGVuc2VzYW1l". - Headers headers = 2; - - // The Agent should use the offered certificate to connect to the destination - // from now on. If the Agent is able to validate and connect using the offered - // certificate the Agent SHOULD forget any previous client certificates - // for this connection. - // This field is optional: if omitted the client SHOULD NOT use a client-side certificate. - // This field can be used to perform a client certificate revocation/rotation. - TLSCertificate certificate = 3; - - // Optional connection specific TLS settings. - // Status: [Development] - TLSConnectionSettings tls = 4; - - // Optional connection specific proxy settings. - // Status: [Development] - ProxyConnectionSettings proxy = 5; -} - -// The OtherConnectionSettings message is a collection of fields which comprise an -// offer from the Server to the Agent to use the specified settings for a network -// connection. It is not required that all fields in this message are specified. -// The Server may specify only some of the fields, in which case it means that -// the Server offers the Agent to change only those fields, while keeping the -// rest of the fields unchanged. -// -// For example the Server may send a ConnectionSettings message with only the -// certificate field set, while all other fields are unset. This means that -// the Server wants the Agent to use a new certificate and continue sending to -// the destination it is currently sending using the current header and other -// settings. -// -// For fields which reference other messages the field is considered unset -// when the reference is unset. -// -// For primitive field (string) we rely on the "flags" to describe that the -// field is not set (this is done to overcome the limitation of old protoc -// compilers don't generate methods that allow to check for the presence of -// the field. -// Status: [Beta] -message OtherConnectionSettings { - // A URL, host:port or some other destination specifier. - string destination_endpoint = 1; - - // Optional headers to use when connecting. Typically used to set access tokens or - // other authorization headers. For HTTP-based protocols the Agent should - // set these in the request headers. - // For example: - // key="Authorization", Value="Basic YWxhZGRpbjpvcGVuc2VzYW1l". - Headers headers = 2; - - // The Agent should use the offered certificate to connect to the destination - // from now on. If the Agent is able to validate and connect using the offered - // certificate the Agent SHOULD forget any previous client certificates - // for this connection. - // This field is optional: if omitted the client SHOULD NOT use a client-side certificate. - // This field can be used to perform a client certificate revocation/rotation. - TLSCertificate certificate = 3; - - // Other connection settings. These are Agent-specific and are up to the Agent - // interpret. - map other_settings = 4; - - // Optional connection specific TLS settings. - // Status: [Development] - TLSConnectionSettings tls = 5; - - // Optional connection specific proxy settings. - // Status: [Development] - ProxyConnectionSettings proxy = 6; -} - - -// TLSConnectionSettings are optional connection settings that can be passed to -// the client in order to specify TLS configuration. -// Status: [Development] -message TLSConnectionSettings { - // Provides CA cert contents as a string. - string ca_pem_contents = 1; - - // Load system CA pool alongside any passed CAs. - bool include_system_ca_certs_pool = 2; - - // skip certificate verification. - bool insecure_skip_verify = 3; - - // Miniumum accepted TLS version; default "1.2". - string min_version = 4; - - // Maxiumum accepted TLS version; default "". - string max_version = 5; - - // Explicit list of cipher suites. - repeated string cipher_suites = 6; -} - -// Status: [Development] -message ProxyConnectionSettings { - // A URL, host:port or some other destination specifier. - string url = 1; - - // Optional headers to send to proxies during CONNECT requests. - // These headers can be ignored for non-HTTP based proxies. - // For example: - // key="Authorization", Value="Basic YWxhZGRpbjpvcGVuc2VzYW1l". - Headers connect_headers = 2; -} - -// Status: [Beta] -message Headers { - repeated Header headers = 1; -} - -// Status: [Beta] -message Header { - string key = 1; - string value = 2; -} - -// Status: [Beta] -message TLSCertificate { - // The (cert,private_key) pair should be issued and signed by a Certificate - // Authority (CA) that the destination Server recognizes. - // - // It is highly recommended that the private key of the CA certificate is NOT - // stored on the destination Server otherwise compromising the Server will allow - // a malicious actor to issue valid Server certificates which will be automatically - // trusted by all agents and will allow the actor to trivially MITM Agent-to-Server - // traffic of all servers that use this CA certificate for their Server-side - // certificates. - // - // Alternatively the certificate may be self-signed, assuming the Server can - // verify the certificate. - - // PEM-encoded certificate. Required. - bytes cert = 1; - - // PEM-encoded private key of the certificate. Required. - bytes private_key = 2; - - // PEM-encoded certificate of the signing CA. - // Optional. MUST be specified if the certificate is CA-signed. - // Can be stored by TLS-terminating intermediary proxies in order to verify - // the connecting client's certificate in the future. - // It is not recommended that the Agent accepts this CA as an authority for - // any purposes. - bytes ca_cert = 3; -} - -// Status: [Beta] -message ConnectionSettingsOffers { - // Hash of all settings, including settings that may be omitted from this message - // because they are unchanged. - bytes hash = 1; - - // Settings to connect to the OpAMP Server. - // If this field is not set then the Agent should assume that the settings are - // unchanged and should continue using existing settings. - // The Agent MUST verify the offered connection settings by actually connecting - // before accepting the setting to ensure it does not loose access to the OpAMP - // Server due to invalid settings. - OpAMPConnectionSettings opamp = 2; - - // Settings to connect to an OTLP metrics backend to send Agent's own metrics to. - // If this field is not set then the Agent should assume that the settings - // are unchanged. - // - // Once accepted the Agent should periodically send to the specified destination - // its own metrics, i.e. metrics of the Agent process and any custom metrics that - // describe the Agent state. - // - // All attributes specified in the identifying_attributes field in AgentDescription - // message SHOULD be also specified in the Resource of the reported OTLP metrics. - // - // Attributes specified in the non_identifying_attributes field in - // AgentDescription message may be also specified in the Resource of the reported - // OTLP metrics, in which case they SHOULD have exactly the same values. - // - // Process metrics MUST follow the conventions for processes: - // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/semantic_conventions/process-metrics.md - TelemetryConnectionSettings own_metrics = 3; - - // Similar to own_metrics, but for traces. - TelemetryConnectionSettings own_traces = 4; - - // Similar to own_metrics, but for logs. - TelemetryConnectionSettings own_logs = 5; - - // Another set of connection settings, with a string name associated with each. - // How the Agent uses these is Agent-specific. Typically the name represents - // the name of the destination to connect to (as it is known to the Agent). - // If this field is not set then the Agent should assume that the other_connections - // settings are unchanged. - map other_connections = 6; -} - -// List of packages that the Server offers to the Agent. -// Status: [Beta] -message PackagesAvailable { - // Map of packages. Keys are package names, values are the packages available for download. - map packages = 1; - - // Aggregate hash of all remotely installed packages. The Agent SHOULD include this - // value in subsequent PackageStatuses messages. This in turn allows the management - // Server to identify that a different set of packages is available for the Agent - // and specify the available packages in the next ServerToAgent message. - // - // This field MUST be always set if the management Server supports packages - // of agents. - // - // The hash is calculated as an aggregate of all packages names and content. - bytes all_packages_hash = 2; -} - -// Each Agent is composed of one or more packages. A package has a name and -// content stored in a file. The content of the files, functionality -// provided by the packages, how they are stored and used by the Agent side is Agent -// type-specific and is outside the concerns of the OpAMP protocol. -// -// If the Agent does not have an installed package with the specified name then -// it SHOULD download it from the specified URL and install it. -// -// If the Agent already has an installed package with the specified name -// but with a different hash then the Agent SHOULD download and -// install the package again, since it is a different version of the same package. -// -// If the Agent has an installed package with the specified name and the same -// hash then the Agent does not need to do anything, it already -// has the right version of the package. -// Status: [Beta] -message PackageAvailable { - PackageType type = 1; - - // The package version that is available on the Server side. The Agent may for - // example use this information to avoid downloading a package that was previously - // already downloaded and failed to install. - string version = 2; - - // The downloadable file of the package. - DownloadableFile file = 3; - - // The hash of the package. SHOULD be calculated based on all other fields of the - // PackageAvailable message and content of the file of the package. The hash is - // used by the Agent to determine if the package it has is different from the - // package the Server is offering. - bytes hash = 4; -} - -// The type of the package, either an addon or a top-level package. -// Status: [Beta] -enum PackageType { - PackageType_TopLevel = 0; - PackageType_Addon = 1; -} - -// Status: [Beta] -message DownloadableFile { - // The URL from which the file can be downloaded using HTTP GET request. - // The Server at the specified URL SHOULD support range requests - // to allow for resuming downloads. - string download_url = 1; - - // The hash of the file content. Can be used by the Agent to verify that the file - // was downloaded correctly. - bytes content_hash = 2; - - // Optional signature of the file content. Can be used by the Agent to verify the - // authenticity of the downloaded file, for example can be the - // [detached GPG signature](https://www.gnupg.org/gph/en/manual/x135.html#AEN160). - // The exact signing and verification method is Agent specific. See - // https://github.com/open-telemetry/opamp-spec/blob/main/specification.md#code-signing - // for recommendations. - bytes signature = 3; - - // Optional headers to use when downloading a file. Typically used to set - // access tokens or other authorization headers. For HTTP-based protocols - // the Agent should set these in the request headers. - // For example: - // key="Authorization", Value="Basic YWxhZGRpbjpvcGVuc2VzYW1l". - // Status: [Development] - Headers headers = 4; -} - -message ServerErrorResponse { - ServerErrorResponseType type = 1; - - // Error message in the string form, typically human readable. - string error_message = 2; - - oneof Details { - // Additional information about retrying if type==UNAVAILABLE. - RetryInfo retry_info = 3; - } -} - -enum ServerErrorResponseType { - // Unknown error. Something went wrong, but it is not known what exactly. - // The Agent SHOULD NOT retry the message. - // The error_message field may contain a description of the problem. - ServerErrorResponseType_Unknown = 0; - - // The AgentToServer message was malformed. The Agent SHOULD NOT retry - // the message. - ServerErrorResponseType_BadRequest = 1; - - // The Server is overloaded and unable to process the request. The Agent - // should retry the message later. retry_info field may be optionally - // set with additional information about retrying. - ServerErrorResponseType_Unavailable = 2; -} - -message RetryInfo { - uint64 retry_after_nanoseconds = 1; -} - -// ServerToAgentCommand is sent from the Server to the Agent to request that the Agent -// perform a command. -// Status: [Beta] -message ServerToAgentCommand { - CommandType type = 1; -} - -// Status: [Beta] -enum CommandType { - // The Agent should restart. This request will be ignored if the Agent does not - // support restart. - CommandType_Restart = 0; -} - -//////////////////////////////////////////////////////////////////////////////////// -// Status reporting - -message AgentDescription { - // Attributes that identify the Agent. - // Keys/values are according to OpenTelemetry semantic conventions, see: - // https://github.com/open-telemetry/opentelemetry-specification/tree/main/specification/resource/semantic_conventions - // - // For standalone running Agents (such as OpenTelemetry Collector) the following - // attributes SHOULD be specified: - // - service.name should be set to a reverse FQDN that uniquely identifies the - // Agent type, e.g. "io.opentelemetry.collector" - // - service.namespace if it is used in the environment where the Agent runs. - // - service.version should be set to version number of the Agent build. - // - service.instance.id should be set. It may be set equal to the Agent's - // instance uid (equal to ServerToAgent.instance_uid field) or any other value - // that uniquely identifies the Agent in combination with other attributes. - // - any other attributes that are necessary for uniquely identifying the Agent's - // own telemetry. - // - // The Agent SHOULD also include these attributes in the Resource of its own - // telemetry. The combination of identifying attributes SHOULD be sufficient to - // uniquely identify the Agent's own telemetry in the destination system to which - // the Agent sends its own telemetry. - repeated KeyValue identifying_attributes = 1; - - // Attributes that do not necessarily identify the Agent but help describe - // where it runs. - // The following attributes SHOULD be included: - // - os.type, os.version - to describe where the Agent runs. - // - host.* to describe the host the Agent runs on. - // - cloud.* to describe the cloud where the host is located. - // - any other relevant Resource attributes that describe this Agent and the - // environment it runs in. - // - any user-defined attributes that the end user would like to associate - // with this Agent. - repeated KeyValue non_identifying_attributes = 2; - - // TODO: add ability to specify related entities (such as the Service the Agent is - // is responsible/associated with). -} - -enum AgentCapabilities { - // The capabilities field is unspecified. - AgentCapabilities_Unspecified = 0; - // The Agent can report status. This bit MUST be set, since all Agents MUST - // report status. - AgentCapabilities_ReportsStatus = 0x00000001; - // The Agent can accept remote configuration from the Server. - AgentCapabilities_AcceptsRemoteConfig = 0x00000002; - // The Agent will report EffectiveConfig in AgentToServer. - AgentCapabilities_ReportsEffectiveConfig = 0x00000004; - // The Agent can accept package offers. - // Status: [Beta] - AgentCapabilities_AcceptsPackages = 0x00000008; - // The Agent can report package status. - // Status: [Beta] - AgentCapabilities_ReportsPackageStatuses = 0x00000010; - // The Agent can report own trace to the destination specified by - // the Server via ConnectionSettingsOffers.own_traces field. - // Status: [Beta] - AgentCapabilities_ReportsOwnTraces = 0x00000020; - // The Agent can report own metrics to the destination specified by - // the Server via ConnectionSettingsOffers.own_metrics field. - // Status: [Beta] - AgentCapabilities_ReportsOwnMetrics = 0x00000040; - // The Agent can report own logs to the destination specified by - // the Server via ConnectionSettingsOffers.own_logs field. - // Status: [Beta] - AgentCapabilities_ReportsOwnLogs = 0x00000080; - // The can accept connections settings for OpAMP via - // ConnectionSettingsOffers.opamp field. - // Status: [Beta] - AgentCapabilities_AcceptsOpAMPConnectionSettings = 0x00000100; - // The can accept connections settings for other destinations via - // ConnectionSettingsOffers.other_connections field. - // Status: [Beta] - AgentCapabilities_AcceptsOtherConnectionSettings = 0x00000200; - // The Agent can accept restart requests. - // Status: [Beta] - AgentCapabilities_AcceptsRestartCommand = 0x00000400; - // The Agent will report Health via AgentToServer.health field. - AgentCapabilities_ReportsHealth = 0x00000800; - // The Agent will report RemoteConfig status via AgentToServer.remote_config_status field. - AgentCapabilities_ReportsRemoteConfig = 0x00001000; - // The Agent can report heartbeats. - // This is specified by the ServerToAgent.OpAMPConnectionSettings.heartbeat_interval_seconds field. - // If this capability is true, but the Server does not set a heartbeat_interval_seconds field, the - // Agent should use its own configured interval, which by default will be 30s. The Server may not - // know the configured interval and should not make assumptions about it. - // Status: [Development] - AgentCapabilities_ReportsHeartbeat = 0x00002000; - // The agent will report AvailableComponents via the AgentToServer.available_components field. - // Status: [Development] - AgentCapabilities_ReportsAvailableComponents = 0x00004000; - // The agent will report ConnectionSettingsOffers status via AgentToServer.connection_settings_status field. - // Status: [Development] - AgentCapabilities_ReportsConnectionSettingsStatus = 0x00008000; - // Add new capabilities here, continuing with the least significant unused bit. -} - -// The health of the Agent and sub-components -// Status: [Beta] -message ComponentHealth { - // Set to true if the component is up and healthy. - bool healthy = 1; - - // Timestamp since the component is up, i.e. when the component was started. - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. - // If the component is not running MUST be set to 0. - fixed64 start_time_unix_nano = 2; - - // Human-readable error message if the component is in erroneous state. SHOULD be set - // when healthy==false. - string last_error = 3; - - // Component status represented as a string. The status values are defined by agent-specific - // semantics and not at the protocol level. - string status = 4; - - // The time when the component status was observed. Value is UNIX Epoch time in - // nanoseconds since 00:00:00 UTC on 1 January 1970. - fixed64 status_time_unix_nano = 5; - - // A map to store more granular, sub-component health. It can nest as deeply as needed to - // describe the underlying system. - map component_health_map = 6; -} - -message EffectiveConfig { - // The effective config of the Agent. - AgentConfigMap config_map = 1; -} - -message RemoteConfigStatus { - // The hash of the remote config that was last received by this Agent in the - // AgentRemoteConfig.config_hash field. - // The Server SHOULD compare this hash with the config hash - // it has for the Agent and if the hashes are different the Server MUST include - // the remote_config field in the response in the ServerToAgent message. - bytes last_remote_config_hash = 1; - - RemoteConfigStatuses status = 2; - - // Optional error message if status==FAILED. - string error_message = 3; -} - -// Status: [Development] -message ConnectionSettingsStatus { - // The hash of the connection settings that was last recieved by this Agent - // in the connection_settings.hash field. The Server SHOULD compare this - // hash with the OfferedConnectionSettings hash it has for the Agent and if - // the hashes are different the Server MUST include the connection_settings - // field in the response in the ServerToAgent message. - bytes last_connection_settings_hash = 1; - - ConnectionSettingsStatuses status = 2; - - // Optional error message if status==FAILED. - string error_message = 3; -} - -// Status: [Development] -enum ConnectionSettingsStatuses { - // The value of status field is not set. - ConnectionSettingsStatuses_UNSET = 0; - - // ConnectionSettings were successfully applied by the Agent. - ConnectionSettingsStatuses_APPLIED = 1; - - // Agent is currently applying the ConnectionSettings that it received. - ConnectionSettingsStatuses_APPLYING = 2; - - // Agent tried to apply the ConnectionSettings it received earlier, but failed. - // See error_message for more details. - ConnectionSettingsStatuses_FAILED = 3; -} - -enum RemoteConfigStatuses { - // The value of status field is not set. - RemoteConfigStatuses_UNSET = 0; - - // Remote config was successfully applied by the Agent. - RemoteConfigStatuses_APPLIED = 1; - - // Agent is currently applying the remote config that it received earlier. - RemoteConfigStatuses_APPLYING = 2; - - // Agent tried to apply the config received earlier, but it failed. - // See error_message for more details. - RemoteConfigStatuses_FAILED = 3; -} - -// The PackageStatuses message describes the status of all packages that the Agent -// has or was offered. -// Status: [Beta] -message PackageStatuses { - // A map of PackageStatus messages, where the keys are package names. - // The key MUST match the name field of PackageStatus message. - map packages = 1; - - // The aggregate hash of all packages that this Agent previously received from the - // Server via PackagesAvailable message. - // - // The Server SHOULD compare this hash to the aggregate hash of all packages that - // it has for this Agent and if the hashes are different the Server SHOULD send - // an PackagesAvailable message to the Agent. - bytes server_provided_all_packages_hash = 2; - - // This field is set if the Agent encountered an error when processing the - // PackagesAvailable message and that error is not related to any particular single - // package. - // The field must be unset is there were no processing errors. - string error_message = 3; -} - -// The status of a single package. -// Status: [Beta] -message PackageStatus { - // Package name. MUST be always set and MUST match the key in the packages field - // of PackageStatuses message. - string name = 1; - - // The version of the package that the Agent has. - // MUST be set if the Agent has this package. - // MUST be empty if the Agent does not have this package. This may be the case - // for example if the package was offered by the Server but failed to install - // and the Agent did not have this package previously. - string agent_has_version = 2; - - // The hash of the package that the Agent has. - // MUST be set if the Agent has this package. - // MUST be empty if the Agent does not have this package. This may be the case for - // example if the package was offered by the Server but failed to install and the - // Agent did not have this package previously. - bytes agent_has_hash = 3; - - // The version of the package that the Server offered to the Agent. - // MUST be set if the installation of the package is initiated by an earlier offer - // from the Server to install this package. - // - // MUST be empty if the Agent has this package but it was installed locally and - // was not offered by the Server. - // - // Note that it is possible for both agent_has_version and server_offered_version - // fields to be set and to have different values. This is for example possible if - // the Agent already has a version of the package successfully installed, the Server - // offers a different version, but the Agent fails to install that version. - string server_offered_version = 4; - - // The hash of the package that the Server offered to the Agent. - // MUST be set if the installation of the package is initiated by an earlier - // offer from the Server to install this package. - // - // MUST be empty if the Agent has this package but it was installed locally and - // was not offered by the Server. - // - // Note that it is possible for both agent_has_hash and server_offered_hash - // fields to be set and to have different values. This is for example possible if - // the Agent already has a version of the package successfully installed, the - // Server offers a different version, but the Agent fails to install that version. - bytes server_offered_hash = 5; - - PackageStatusEnum status = 6; - - // Error message if the status is erroneous. - string error_message = 7; - - // Optional details that may be of interest to a user. - // Should only be set if status is Downloading. - // Status: [Development] - PackageDownloadDetails download_details = 8; -} - - -// Additional details that an agent can use to describe an in-progress package download. -// Status: [Development] -message PackageDownloadDetails { - // The package download progress as a percentage. - double download_percent = 1; - - // The current package download rate in bytes per second. - double download_bytes_per_second = 2; -} - -// The status of this package. -// Status: [Beta] -enum PackageStatusEnum { - // Package is successfully installed by the Agent. - // The error_message field MUST NOT be set. - PackageStatusEnum_Installed = 0; - - // Installation of this package has not yet started. - PackageStatusEnum_InstallPending = 1; - - // Agent is currently installing the package. - // server_offered_hash field MUST be set to indicate the version that the - // Agent is installing. The error_message field MUST NOT be set. - PackageStatusEnum_Installing = 2; - - // Agent tried to install the package but installation failed. - // server_offered_hash field MUST be set to indicate the version that the Agent - // tried to install. The error_message may also contain more details about - // the failure. - PackageStatusEnum_InstallFailed = 3; - - // Agent is currently downloading the package. - // server_offered_hash field MUST be set to indicate the version that the - // Agent is installing. The error_message field MUST NOT be set. - // Status: [Development] - PackageStatusEnum_Downloading = 4; -} - -// Properties related to identification of the Agent, which can be overridden -// by the Server if needed -message AgentIdentification { - // When new_instance_uid is set, Agent MUST update instance_uid - // to the value provided and use it for all further communication. - // MUST be 16 bytes long and SHOULD be generated using the UUID v7 spec. - bytes new_instance_uid = 1; -} - -///////////////////////////////////////////////////////////////////////////////////// -// Config messages -///////////////////////////////////////////////////////////////////////////////////// - -message AgentRemoteConfig { - // Agent config offered by the management Server to the Agent instance. SHOULD NOT be - // set if the config for this Agent has not changed since it was last requested (i.e. - // AgentConfigRequest.last_remote_config_hash field is equal to - // AgentConfigResponse.config_hash field). - AgentConfigMap config = 1; - - // Hash of "config". The Agent SHOULD include this value in subsequent - // RemoteConfigStatus messages in the last_remote_config_hash field. This in turn - // allows the management Server to identify that a new config is available for the Agent. - // - // This field MUST be always set if the management Server supports remote configuration - // of agents. - // - // Management Server must choose a hashing function that guarantees lack of hash - // collisions in practice. - bytes config_hash = 2; -} - -message AgentConfigMap { - // Map of configs. Keys are config file names or config section names. - // The configuration is assumed to be a collection of one or more named config files - // or sections. - // For agents that use a single config file or section the map SHOULD contain a single - // entry and the key may be an empty string. - map config_map = 1; -} - -message AgentConfigFile { - // Config file or section body. The content, format and encoding depends on the Agent - // type. The content_type field may optionally describe the MIME type of the body. - bytes body = 1; - - // Optional MIME Content-Type that describes what's in the body field, for - // example "text/yaml". - string content_type = 2; -} - -///////////////////////////////////////////////////////////////////////////////////// -// Custom messages -///////////////////////////////////////////////////////////////////////////////////// - -message CustomCapabilities { - // A list of custom capabilities that are supported. Each capability is a reverse FQDN - // with optional version information that uniquely identifies the custom capability - // and should match a capability specified in a supported CustomMessage. - // Status: [Development] - repeated string capabilities = 1; -} - -message CustomMessage { - // A reverse FQDN that uniquely identifies the capability and matches one of the - // capabilities in the CustomCapabilities message. - // Status: [Development] - string capability = 1; - - // Type of message within the capability. The capability defines the types of custom - // messages that are used to implement the capability. The type must only be unique - // within the capability. - // Status: [Development] - string type = 2; - - // Binary data of the message. The capability must specify the format of the contents - // of the data for each custom message type it defines. - // Status: [Development] - bytes data = 3; -} diff --git a/control_plane/sketch_capabilities.yml b/control_plane/sketch_capabilities.yml deleted file mode 100644 index c3daa3cba..000000000 --- a/control_plane/sketch_capabilities.yml +++ /dev/null @@ -1,66 +0,0 @@ -# Sketch capability profiles. -# -# These numbers describe the performance characteristics of each sketch -# implementation. The optimizer and physical planner use them to decide -# which sketch fits within a stage's budget. -# -# Source: initial values are estimates. Run benchmarks to calibrate: -# ./e2esdkbench --sketch-type=ddsketch --series=1000 --duration=60s -# Then update this file with measured values. -# -# Env var: CONTROLLER_SKETCH_CAPABILITIES=path/to/this/file.yml - -ddsketch: - insert_throughput: 10000000.0 # samples/sec (single core) - query_throughput: 50000000.0 # queries/sec (single core) - memory_bytes_per_series: 4096 # bytes - cpu_micros_per_insert: 0.1 # µs/sample - transmission_bytes: 4096 # bytes per flush - supported_intents: [quantile, extrema] - mergeable: true - supports_delta: true - supports_sliding_window: false - -kll: - insert_throughput: 5000000.0 - query_throughput: 20000000.0 - memory_bytes_per_series: 8192 - cpu_micros_per_insert: 0.2 - transmission_bytes: 8192 - supported_intents: [quantile, extrema] - mergeable: true - supports_delta: false # KLL uses random compaction, not additive - supports_sliding_window: false - -hll: - insert_throughput: 20000000.0 - query_throughput: 100000000.0 - memory_bytes_per_series: 16384 # 2^14 registers - cpu_micros_per_insert: 0.05 - transmission_bytes: 16384 - supported_intents: [cardinality] - mergeable: true - supports_delta: true - supports_sliding_window: false - -count_sketch: - insert_throughput: 8000000.0 - query_throughput: 10000000.0 - memory_bytes_per_series: 80000 # 2000 width × 5 depth × 8 bytes - cpu_micros_per_insert: 0.5 - transmission_bytes: 80000 - supported_intents: [frequency] - mergeable: true - supports_delta: true - supports_sliding_window: false - -count_min_sketch: - insert_throughput: 8000000.0 - query_throughput: 10000000.0 - memory_bytes_per_series: 80000 - cpu_micros_per_insert: 0.5 - transmission_bytes: 80000 - supported_intents: [frequency] - mergeable: true - supports_delta: true - supports_sliding_window: false diff --git a/control_plane/sketch_params_default.yml b/control_plane/sketch_params_default.yml deleted file mode 100644 index e978b433f..000000000 --- a/control_plane/sketch_params_default.yml +++ /dev/null @@ -1,36 +0,0 @@ -# Sketch parameter defaults for the control plane planner. -# -# All values here are the compiled-in defaults. Override any field to -# customise sketch sizing without recompiling. -# -# Env var: CONTROLLER_SKETCH_DEFAULTS=path/to/this/file.yaml - -# Quantile grid used when the query does not specify explicit φ values. -quantile_grid: [0.0, 0.25, 0.5, 0.75, 0.9, 0.99, 1.0] - -ddsketch: - # DDSketch relative accuracy (α). Lower = more buckets = higher memory. - relative_accuracy: 0.01 - -kll: - # Minimum k parameter (clamped from ⌈1/accuracy_sla⌉). - min_k: 32 - -hll: - # HLL precision for coarse SLA (accuracy > precision_threshold). - precision_coarse: 10 - # HLL precision for fine SLA (accuracy ≤ precision_threshold). - precision_fine: 14 - # SLA boundary between coarse and fine precision bands. - precision_threshold: 0.02 - -count_sketch: - # Relative error bound (ε ≈ 1/√cols for equivalent width). - epsilon: 0.022 - # Error probability (δ ≈ e^(−rows) for equivalent depth). - delta: 0.007 - -count_min_sketch: - rows: 5 - cols: 2048 - metric_name: "countsketch_partition" diff --git a/control_plane/src/accuracy.rs b/control_plane/src/accuracy.rs deleted file mode 100644 index 4f52b0d9d..000000000 --- a/control_plane/src/accuracy.rs +++ /dev/null @@ -1,150 +0,0 @@ -//! Adapt planner sketch parameters to ASAPPlanner accuracy guarantees. - -use crate::types::SketchParams; -pub use asap_types::accuracy::{AccuracyKind, AccuracyProfile}; -use planner_types::post_asap::SketchParams as PlannerParams; - -pub fn derive(params: &SketchParams) -> AccuracyProfile { - let normalized = match params { - SketchParams::CountMinSketch { rows, cols, .. } => PlannerParams::Cms { - depth: (*rows).max(1), - width: (*cols).max(1), - }, - SketchParams::CountSketch { epsilon, delta } => { - return AccuracyProfile { - epsilon: *epsilon, - delta: Some(*delta), - kind: AccuracyKind::AdditiveFrequency, - } - } - SketchParams::HLL { precision } => PlannerParams::Hll { - precision: u8::try_from(*precision).unwrap_or(14), - }, - SketchParams::KLL { k, .. } => PlannerParams::Kll { k: (*k).max(1) }, - SketchParams::DDSketch { - relative_accuracy, .. - } => PlannerParams::DDSketch { - alpha: *relative_accuracy, - }, - }; - AccuracyProfile::from_sketch_params(&normalized) - .expect("shared family has a numeric planner bound") -} - -/// Source adapter for planner configuration parameters. -pub trait PlannerAccuracyProfile { - fn derive(params: &SketchParams) -> Self; -} -impl PlannerAccuracyProfile for AccuracyProfile { - fn derive(params: &SketchParams) -> Self { - derive(params) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cms_bound_is_e_over_w() { - let p = derive(&SketchParams::CountMinSketch { - rows: 3, - cols: 1000, - metric_name: "m".into(), - }); - assert_eq!(p.kind, AccuracyKind::AdditiveFrequency); - assert!((p.epsilon - std::f64::consts::E / 1000.0).abs() < 1e-12); - assert!((p.delta.unwrap() - (-3.0_f64).exp()).abs() < 1e-12); - } - - #[test] - fn countsketch_passes_through_user_supplied_bounds() { - let p = derive(&SketchParams::CountSketch { - epsilon: 0.01, - delta: 0.001, - }); - assert_eq!(p.kind, AccuracyKind::AdditiveFrequency); - assert_eq!(p.epsilon, 0.01); - assert_eq!(p.delta, Some(0.001)); - } - - #[test] - fn hll_p14_matches_flajolet_bound() { - let p = derive(&SketchParams::HLL { precision: 14 }); - assert_eq!(p.kind, AccuracyKind::RelativeCardinality); - // 1.04 / √16384 = 0.008125 - assert!((p.epsilon - 0.008125).abs() < 1e-9); - } - - #[test] - fn kll_k200_matches_karnin_lang_liberty_bound() { - let p = derive(&SketchParams::KLL { - k: 200, - quantiles: vec![0.5, 0.95, 0.99], - }); - assert_eq!(p.kind, AccuracyKind::RankQuantile); - assert!((p.epsilon - 2.296 / 200.0_f64.powf(0.9723)).abs() < 1e-12); - assert!((p.delta.unwrap() - 0.01).abs() < 1e-12); - } - - #[test] - fn ddsketch_alpha_passes_through_verbatim() { - for alpha in [0.005, 0.01, 0.02, 0.05] { - let p = derive(&SketchParams::DDSketch { - relative_accuracy: alpha, - quantiles: vec![0.5, 0.99], - }); - assert_eq!(p.kind, AccuracyKind::RelativeQuantile); - assert_eq!(p.epsilon, alpha); - assert_eq!(p.delta, Some(0.0)); - } - } - - /// Keep the user-facing summary explicit about unknown HLL confidence. - #[test] - fn golden_parity_with_backend() { - // HLL precision=14: ε = 0.008125, kind = relative_cardinality - let p = derive(&SketchParams::HLL { precision: 14 }); - assert_eq!( - p.summary(), - "accuracy: ε=0.008125, δ=unknown, kind=relative_cardinality" - ); - - // KLL k=200: ε = 2.296 / 200^0.9723, kind = rank_quantile, δ = 0.01 - let p = derive(&SketchParams::KLL { - k: 200, - quantiles: vec![], - }); - let expected_eps = 2.296_f64 / 200.0_f64.powf(0.9723); - assert_eq!( - p.summary(), - format!("accuracy: ε={}, δ=0.01, kind=rank_quantile", expected_eps) - ); - - // DDSketch α=0.02: passes verbatim, δ=0, kind=relative_quantile - let p = derive(&SketchParams::DDSketch { - relative_accuracy: 0.02, - quantiles: vec![], - }); - assert_eq!(p.summary(), "accuracy: ε=0.02, δ=0, kind=relative_quantile"); - } - - #[test] - fn kind_serialises_to_snake_case() { - let p = derive(&SketchParams::HLL { precision: 14 }); - let json = serde_json::to_string(&p).unwrap(); - assert!(json.contains("\"kind\":\"relative_cardinality\"")); - } - - #[test] - fn round_trip_through_serde() { - let src = AccuracyProfile { - epsilon: 0.008125, - delta: None, - kind: AccuracyKind::RelativeCardinality, - }; - let json = serde_json::to_string(&src).unwrap(); - let back: AccuracyProfile = serde_json::from_str(&json).unwrap(); - assert_eq!(back, src); - } -} diff --git a/control_plane/src/backend_client.rs b/control_plane/src/backend_client.rs deleted file mode 100644 index dc356a00c..000000000 --- a/control_plane/src/backend_client.rs +++ /dev/null @@ -1,724 +0,0 @@ -//! Publish and activate complete physical-plan generations on the backend. - -use std::time::Duration; - -#[cfg(test)] -use anyhow::{Context, Result}; -use reqwest::Client; -#[cfg(test)] -use tracing::debug; -#[cfg(test)] -use tracing::warn; - -/// Classification of an HTTP push failure used by the retry layer in -/// [`crate::emit::backend_push`]. Transient errors are safe to retry -/// (the controller raced ahead of the backend's route bind, the host -/// hasn't finished accepting TCP yet, a transient 5xx during backend -/// startup, etc.); permanent errors indicate the call itself is wrong -/// (bad payload, 4xx other than 404) and retrying just amplifies the -/// log noise without improving the outcome. -/// -/// This type sits next to [`BackendClient`] because the classification -/// is a property of how the backend responded, not of how the caller -/// retries — keeping the classifier here lets every endpoint method -/// share the same transient/permanent definition. -#[derive(Debug)] -pub enum BackendPostError { - /// Worth another attempt after backoff: connection refused, - /// connect timeout, DNS resolution failure, 404 (route not yet - /// registered), or any 5xx. - Transient(anyhow::Error), - /// Will fail again the same way: 4xx other than 404 (bad payload, - /// auth, etc.). The retry layer surfaces these immediately. - Permanent(anyhow::Error), -} - -impl BackendPostError { - pub fn is_transient(&self) -> bool { - matches!(self, BackendPostError::Transient(_)) - } - - pub fn into_inner(self) -> anyhow::Error { - match self { - BackendPostError::Transient(e) | BackendPostError::Permanent(e) => e, - } - } -} - -impl std::fmt::Display for BackendPostError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - BackendPostError::Transient(e) => write!(f, "transient: {e}"), - BackendPostError::Permanent(e) => write!(f, "permanent: {e}"), - } - } -} - -impl std::error::Error for BackendPostError {} - -/// Inspect a `reqwest::Error` and decide whether the failure is worth -/// retrying. Connection-level failures (no TCP socket / DNS / connect -/// timeout) and request-side timeouts are transient — the backend is -/// likely still coming up. -fn classify_reqwest_error(err: reqwest::Error) -> BackendPostError { - if err.is_connect() || err.is_timeout() || err.is_request() { - BackendPostError::Transient(anyhow::Error::new(err)) - } else { - // body decode, redirect loops, etc. — these will not resolve - // themselves on retry. - BackendPostError::Permanent(anyhow::Error::new(err)) - } -} - -/// Map a non-2xx HTTP status to a [`BackendPostError`]. 404 and 5xx -/// are transient; every other 4xx is permanent. -fn classify_http_status(status: reqwest::StatusCode, body: String, what: &str) -> BackendPostError { - let err = anyhow::anyhow!("backend returned {} for {}: {}", status, what, body); - if status == reqwest::StatusCode::NOT_FOUND || status.is_server_error() { - BackendPostError::Transient(err) - } else { - BackendPostError::Permanent(err) - } -} - -/// Minimal HTTP client for ASAPQuery-backend's physical-plan endpoint. -/// Built once at control-plane startup from the -/// `CONTROL_PLANE_BACKEND_ENDPOINT` environment variable and shared -/// via `Arc` with the replanner. -#[derive(Debug, Clone)] -pub struct BackendClient { - endpoint: String, - http: Client, -} - -impl BackendClient { - /// Construct a client pointing at the backend's plan-push endpoint. - /// `endpoint` should be the full URL, e.g. - /// `http://backend.svc:8088/api/v1/physical-plan`. - /// - /// A 5-second timeout bounds the duration a slow or unreachable - /// backend can stall the replanner — consistent with the symmetric - /// 5-second timeout on ASAPQuery-backend's `HttpControlPlaneClient` - /// (the reverse direction in the same loop). - pub fn new(endpoint: impl Into) -> Self { - let http = Client::builder() - .timeout(Duration::from_secs(5)) - .build() - .unwrap_or_else(|_| Client::new()); - Self { - endpoint: endpoint.into(), - http, - } - } - - /// Construct with an explicit `reqwest::Client`. Used by tests that - /// need to inject a mock-server URL without reconfiguring the - /// timeout setup. - pub fn with_http(endpoint: impl Into, http: Client) -> Self { - Self { - endpoint: endpoint.into(), - http, - } - } - - pub fn endpoint(&self) -> &str { - &self.endpoint - } - - /// Test-only transport helper: POST the given YAML to the backend. Returns - /// `Ok(())` on any 2xx status, otherwise an error carrying the - /// status code and response body. The caller (typically - /// [`Replanner::replan_metric`]) logs the error and moves on — the - /// next replan cycle will retry with the latest plan. - #[cfg(test)] - pub async fn push_streaming_config(&self, yaml: String) -> Result<()> { - debug!( - endpoint = %self.endpoint, - yaml_bytes = yaml.len(), - "pushing streaming-config YAML to ASAPQuery-backend" - ); - let resp = self - .http - .post(&self.endpoint) - .header("content-type", "application/x-yaml") - .body(yaml) - .send() - .await - .context("failed to POST streaming-config to backend")?; - - let status = resp.status(); - if status.is_success() { - Ok(()) - } else { - let body = resp.text().await.unwrap_or_default(); - Err(anyhow::anyhow!( - "backend returned {} for streaming-config POST: {}", - status, - body - )) - } - } - - #[cfg(test)] - /// Post streaming configuration as `application/json`. The backend accepts - /// both JSON and YAML; a non-2xx response is an error for the caller to log. - pub async fn post_streaming_config_json(&self, json: String) -> Result<()> { - debug!( - endpoint = %self.endpoint, - json_bytes = json.len(), - "posting streaming-config JSON to ASAPQuery-backend" - ); - let resp = self - .http - .post(&self.endpoint) - .header("content-type", "application/json") - .body(json) - .send() - .await - .context("failed to POST streaming-config JSON to backend")?; - - let status = resp.status(); - if status.is_success() { - Ok(()) - } else { - let body = resp.text().await.unwrap_or_default(); - Err(anyhow::anyhow!( - "backend returned {} for streaming-config JSON POST: {}", - status, - body - )) - } - } - - #[cfg(test)] - /// Typed sibling of [`Self::post_streaming_config_json`] for the - /// retry layer. Returns the same `Ok(())` on 2xx, but on failure - /// classifies the underlying cause as [`BackendPostError::Transient`] - /// or [`BackendPostError::Permanent`] so the caller can decide - /// whether another attempt is worthwhile. - /// - /// Public-interface preservation: the existing - /// [`Self::post_streaming_config_json`] remains untouched — callers - /// that don't need retry semantics keep their `anyhow::Result` - /// shape. The retry layer in `emit::backend_push` uses this typed - /// variant. - #[cfg(test)] - pub async fn post_streaming_config_json_typed( - &self, - json: String, - ) -> std::result::Result<(), BackendPostError> { - debug!( - endpoint = %self.endpoint, - json_bytes = json.len(), - "posting streaming-config JSON to ASAPQuery-backend (typed)" - ); - let resp = self - .http - .post(&self.endpoint) - .header("content-type", "application/json") - .body(json) - .send() - .await - .map_err(classify_reqwest_error)?; - - let status = resp.status(); - if status.is_success() { - Ok(()) - } else { - let body = resp.text().await.unwrap_or_default(); - Err(classify_http_status( - status, - body, - "streaming-config JSON POST", - )) - } - } - - #[cfg(test)] - /// Post backend storage-routing JSON. Derive the URL by replacing the - /// `/api/v1/physical-plan` suffix with `/api/v1/storage_routing`; URLs - /// without that suffix are used verbatim. - pub async fn post_storage_routing_json(&self, json: String) -> Result<()> { - let url = derive_storage_routing_url(&self.endpoint); - debug!( - endpoint = %url, - json_bytes = json.len(), - "posting storage-routing JSON to ASAPQuery-backend" - ); - let resp = self - .http - .post(&url) - .header("content-type", "application/json") - .body(json) - .send() - .await - .context("failed to POST storage-routing JSON to backend")?; - - let status = resp.status(); - if status.is_success() { - Ok(()) - } else { - let body = resp.text().await.unwrap_or_default(); - Err(anyhow::anyhow!( - "backend returned {} for storage-routing JSON POST: {}", - status, - body - )) - } - } - - /// Publish one authoritative catalog generation and all plans that reference it. - pub async fn post_catalog_plan_typed( - &self, - publication: &crate::physical::publication::PhysicalPlanPublication, - storage_routing: Option, - adaptation_evidence: &[crate::physical::compiler::RuntimeAdaptationEvidence], - ) -> std::result::Result<(), BackendPostError> { - let body = publication - .install_request(storage_routing, adaptation_evidence.to_vec()) - .map_err(|error| BackendPostError::Permanent(anyhow::anyhow!(error)))?; - let response = self - .http - .post(derive_physical_plan_url(&self.endpoint)) - .json(&body) - .send() - .await - .map_err(classify_reqwest_error)?; - let status = response.status(); - if status.is_success() { - Ok(()) - } else { - let body = response.text().await.unwrap_or_default(); - Err(classify_http_status( - status, - body, - "catalog physical-plan POST", - )) - } - } - - pub async fn discard_staged_physical_plan( - &self, - plan_id: u64, - plan_version: u64, - ) -> std::result::Result<(), BackendPostError> { - let response = self - .http - .post(format!( - "{}/discard", - derive_physical_plan_url(&self.endpoint) - )) - .json(&serde_json::json!({"plan_id": plan_id, "plan_version": plan_version})) - .send() - .await - .map_err(classify_reqwest_error)?; - let status = response.status(); - if status.is_success() { - Ok(()) - } else { - Err(classify_http_status( - status, - response.text().await.unwrap_or_default(), - "PhysicalPlan discard POST", - )) - } - } - - pub async fn activate_physical_plan( - &self, - plan_id: u64, - plan_version: u64, - ) -> std::result::Result<(), BackendPostError> { - let url = format!("{}/activate", derive_physical_plan_url(&self.endpoint)); - let response = self - .http - .post(&url) - .json(&serde_json::json!({ - "plan_id": plan_id, - "plan_version": plan_version, - })) - .send() - .await - .map_err(classify_reqwest_error)?; - let status = response.status(); - if status.is_success() { - Ok(()) - } else { - let body = response.text().await.unwrap_or_default(); - Err(classify_http_status( - status, - body, - "PhysicalPlan activation POST", - )) - } - } -} - -fn derive_physical_plan_url(endpoint: &str) -> String { - endpoint.to_string() -} - -/// Map a physical-plan endpoint URL to the sibling storage-routing -/// endpoint by rewriting the trailing path component. URLs that don't -/// end with `/api/v1/physical-plan` (or `/api/v1/physical_plan` — -/// either spelling is supported) pass through unchanged so tests can -/// inject a mock-server URL directly. -#[cfg(test)] -fn derive_storage_routing_url(endpoint: &str) -> String { - const STREAMING_PATH_DASH: &str = "/api/v1/physical-plan"; - const STREAMING_PATH_UNDERSCORE: &str = "/api/v1/physical_plan"; - const ROUTING_PATH: &str = "/api/v1/storage_routing"; - if let Some(stripped) = endpoint.strip_suffix(STREAMING_PATH_DASH) { - return format!("{stripped}{ROUTING_PATH}"); - } - if let Some(stripped) = endpoint.strip_suffix(STREAMING_PATH_UNDERSCORE) { - return format!("{stripped}{ROUTING_PATH}"); - } - endpoint.to_string() -} - -#[cfg(test)] -/// Fire-and-forget convenience helper used by the replanner. Logs -/// errors at WARN and never propagates them — the replanner should -/// never fail an entire replan because the backend was temporarily -/// unreachable. -pub async fn push_or_log(client: &BackendClient, metric: &str, yaml: String) { - match client.push_streaming_config(yaml).await { - Ok(()) => { - debug!(metric, endpoint = %client.endpoint, "streaming-config push succeeded"); - } - Err(e) => { - warn!( - metric, - endpoint = %client.endpoint, - error = %e, - "streaming-config push to ASAPQuery-backend failed; \ - next replan cycle will retry" - ); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use axum::extract::State; - use axum::routing::post; - use axum::Router; - use std::sync::{Arc as StdArc, Mutex}; - - #[derive(Clone)] - struct SharedSink(StdArc>>); - - async fn start_mock_backend(sink: SharedSink, status: axum::http::StatusCode) -> String { - let app = Router::new() - .route( - "/api/v1/physical-plan", - post( - move |State(sink): State, body: axum::body::Bytes| async move { - let yaml = String::from_utf8_lossy(&body).to_string(); - sink.0.lock().unwrap().push(yaml); - status - }, - ), - ) - .with_state(sink); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve(listener, app).await.unwrap(); - }); - tokio::time::sleep(Duration::from_millis(50)).await; - format!("http://{addr}/api/v1/physical-plan") - } - - #[tokio::test] - async fn success_path_round_trips_yaml() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = start_mock_backend(sink.clone(), axum::http::StatusCode::OK).await; - - let client = BackendClient::new(url); - let yaml = "aggregations:\n - metric: cpu\n".to_string(); - client - .push_streaming_config(yaml.clone()) - .await - .expect("push ok"); - - let received = sink.0.lock().unwrap(); - assert_eq!(received.len(), 1); - assert_eq!(received[0], yaml); - } - - #[tokio::test] - async fn non_2xx_status_is_reported_as_error() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = - start_mock_backend(sink.clone(), axum::http::StatusCode::INTERNAL_SERVER_ERROR).await; - - let client = BackendClient::new(url); - let result = client.push_streaming_config("whatever".to_string()).await; - assert!(result.is_err(), "expected error on 500, got {result:?}"); - let msg = result.unwrap_err().to_string(); - assert!(msg.contains("500"), "error msg should mention 500: {msg}"); - } - - #[tokio::test] - async fn push_or_log_swallows_errors() { - // Point at an unreachable port so the request fails fast. - let client = BackendClient::new("http://127.0.0.1:1/api/v1/physical-plan"); - // Must not panic or propagate — fire-and-forget semantics. - push_or_log(&client, "cpu_usage", "content".to_string()).await; - } - - /// the JSON variant POSTs the body verbatim, returns - /// `Ok(())` on a 2xx, and surfaces non-2xx as `Err`. Mock backend - /// captures the body so we can verify it round-trips. - #[tokio::test] - async fn json_post_round_trips_body() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = start_mock_backend(sink.clone(), axum::http::StatusCode::OK).await; - - let client = BackendClient::new(url); - let json = r#"{"aggregations":[{"metric":"latency"}]}"#.to_string(); - client - .post_streaming_config_json(json.clone()) - .await - .expect("json post ok"); - - let received = sink.0.lock().unwrap(); - assert_eq!(received.len(), 1); - assert_eq!(received[0], json); - } - - /// non-2xx from the backend surfaces as an error so the - /// caller (handle_plan) can log + move on. - #[tokio::test] - async fn json_post_non_2xx_is_error() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = start_mock_backend(sink.clone(), axum::http::StatusCode::BAD_REQUEST).await; - - let client = BackendClient::new(url); - let result = client.post_streaming_config_json("{}".to_string()).await; - assert!(result.is_err(), "expected error on 400, got {result:?}"); - let msg = result.unwrap_err().to_string(); - assert!(msg.contains("400"), "error msg should mention 400: {msg}"); - } - - /// storage-routing-URL derivation rewrites the path - /// component when the configured endpoint ends in - /// `/api/v1/physical-plan`, leaving everything else untouched. - #[test] - fn storage_routing_url_rewrites_streaming_path() { - assert_eq!( - derive_storage_routing_url("http://backend:8088/api/v1/physical-plan"), - "http://backend:8088/api/v1/storage_routing" - ); - assert_eq!( - derive_storage_routing_url("http://backend:8088/api/v1/physical_plan"), - "http://backend:8088/api/v1/storage_routing" - ); - } - - #[test] - fn storage_routing_url_preserves_unknown_paths_for_tests() { - // Test escape hatch — a mock-server URL pointing directly at - // `/api/v1/storage_routing` already passes through unchanged. - assert_eq!( - derive_storage_routing_url("http://127.0.0.1:1/api/v1/storage_routing"), - "http://127.0.0.1:1/api/v1/storage_routing" - ); - // Unrelated path passes through too — no surprise rewriting. - assert_eq!(derive_storage_routing_url("http://x/foo"), "http://x/foo"); - } - - #[tokio::test] - async fn catalog_publication_posts_canonical_document_without_legacy_bytes() { - let snapshot: crate::physical::compiler::BackendLocalPlanningInput = serde_json::from_str( - include_str!("../../docs/examples/asapquery-planning-snapshot.json"), - ) - .unwrap(); - let publication = crate::physical::compiler::tests::quoted_snapshot( - snapshot, - crate::physical::compiler::QueryFrontend::PromQl, - ) - .compile_promql() - .unwrap() - .to_publication_artifact() - .unwrap(); - let hits: StdArc>> = StdArc::new(Mutex::new(Vec::new())); - let route_hits = hits.clone(); - let app = Router::new().route( - "/api/v1/physical-plan", - post(move |axum::Json(body): axum::Json| { - let hits = route_hits.clone(); - async move { - hits.lock().unwrap().push(body); - axum::http::StatusCode::OK - } - }), - ); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { - axum::serve(listener, app).await.unwrap(); - }); - let client = BackendClient::new(format!("http://{addr}/api/v1/physical-plan")); - client - .post_catalog_plan_typed(&publication, None, &[]) - .await - .unwrap(); - let bodies = hits.lock().unwrap(); - assert_eq!(bodies.len(), 1); - let envelope: crate::physical::publication::PhysicalPlanInstallRequest = - serde_json::from_value(bodies[0].clone()).unwrap(); - assert!(envelope.storage_routing.is_none()); - assert!(envelope.adaptation_evidence.is_empty()); - for field in [ - "summary_catalog", - "precompute_plan", - "collector_plans", - "transmission_plan", - "query_plan", - "storage_routing", - "adaptation_evidence", - ] { - assert!(bodies[0].get(field).is_some(), "missing {field}"); - } - server.abort(); - } - - /// full happy path. A mock backend hosts the storage - /// routing endpoint; the client POSTs the control-plane-emitted JSON - /// and the body round-trips verbatim. Mirrors `json_post_round_trips_body`. - async fn start_mock_routing_backend( - sink: SharedSink, - status: axum::http::StatusCode, - ) -> String { - let app = Router::new() - .route( - "/api/v1/storage_routing", - post( - move |State(sink): State, body: axum::body::Bytes| async move { - let json = String::from_utf8_lossy(&body).to_string(); - sink.0.lock().unwrap().push(json); - status - }, - ), - ) - .with_state(sink); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve(listener, app).await.unwrap(); - }); - tokio::time::sleep(Duration::from_millis(50)).await; - // Return the streaming-config URL — the client will rewrite - // the path before issuing the POST. - format!("http://{addr}/api/v1/physical-plan") - } - - #[tokio::test] - async fn storage_routing_post_round_trips_body_via_url_rewrite() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = start_mock_routing_backend(sink.clone(), axum::http::StatusCode::OK).await; - - let client = BackendClient::new(url); - let json = r#"{"default_engine":"sketch_store","metrics":[{"name":"x","targets":[]}]}"# - .to_string(); - client - .post_storage_routing_json(json.clone()) - .await - .expect("routing post ok"); - - let received = sink.0.lock().unwrap(); - assert_eq!(received.len(), 1); - assert_eq!(received[0], json); - } - - /// Typed-variant classification: 404 from the backend is - /// transient (route not yet registered during startup race). - #[tokio::test] - async fn typed_streaming_config_404_is_transient() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = start_mock_backend(sink.clone(), axum::http::StatusCode::NOT_FOUND).await; - - let client = BackendClient::new(url); - let err = client - .post_streaming_config_json_typed("{}".to_string()) - .await - .expect_err("404 should surface as Err"); - assert!(err.is_transient(), "404 must classify as transient: {err}"); - } - - /// Typed-variant classification: 500 is transient. - #[tokio::test] - async fn typed_streaming_config_500_is_transient() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = - start_mock_backend(sink.clone(), axum::http::StatusCode::INTERNAL_SERVER_ERROR).await; - - let client = BackendClient::new(url); - let err = client - .post_streaming_config_json_typed("{}".to_string()) - .await - .expect_err("500 should surface as Err"); - assert!(err.is_transient(), "500 must classify as transient: {err}"); - } - - /// Typed-variant classification: 400 (other 4xx) is permanent — - /// retrying won't fix a malformed payload. - #[tokio::test] - async fn typed_streaming_config_400_is_permanent() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = start_mock_backend(sink.clone(), axum::http::StatusCode::BAD_REQUEST).await; - - let client = BackendClient::new(url); - let err = client - .post_streaming_config_json_typed("{}".to_string()) - .await - .expect_err("400 should surface as Err"); - assert!(!err.is_transient(), "400 must classify as permanent: {err}"); - } - - /// Typed-variant classification: connection refused (no listener - /// on the target port) is transient — the backend may still be - /// binding. - #[tokio::test] - async fn typed_connection_refused_is_transient() { - // Port 1 on loopback is reserved and refuses connections. - let client = BackendClient::new("http://127.0.0.1:1/api/v1/physical-plan"); - let err = client - .post_streaming_config_json_typed("{}".to_string()) - .await - .expect_err("connection refused should surface as Err"); - assert!( - err.is_transient(), - "connection-refused must classify as transient: {err}" - ); - } - - /// Typed-variant happy path: 2xx returns Ok. - #[tokio::test] - async fn typed_streaming_config_success_is_ok() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = start_mock_backend(sink.clone(), axum::http::StatusCode::OK).await; - let client = BackendClient::new(url); - client - .post_streaming_config_json_typed("{}".to_string()) - .await - .expect("2xx must be Ok"); - } - - #[tokio::test] - async fn storage_routing_post_non_2xx_is_error() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = - start_mock_routing_backend(sink.clone(), axum::http::StatusCode::INTERNAL_SERVER_ERROR) - .await; - let client = BackendClient::new(url); - let result = client.post_storage_routing_json("{}".to_string()).await; - assert!(result.is_err(), "expected error on 500, got {result:?}"); - let msg = result.unwrap_err().to_string(); - assert!(msg.contains("500"), "error msg should mention 500: {msg}"); - } -} diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs deleted file mode 100644 index 6b5f8cd88..000000000 --- a/control_plane/src/clickhouse.rs +++ /dev/null @@ -1,1756 +0,0 @@ -//! ClickHouse SQL planning entry point. -//! -//! ASAPPlanner owns SQL parsing and canonicalization. This module only joins -//! that frontend to the same post-ASAP physical mapping used by PromQL. - -use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; -use planner_types::pre_asap::QueryExpr; -use planner_types::types::AccuracyTarget; -use planner_types::workload::SqlDialect; - -use crate::physical::compiler::{PrecomputePlan, TransmissionPlan}; -use crate::physical::post_asap::{cost_model::ControlPlaneCostModel, PhysicalExpr}; -use crate::query_plan::{ - ClickHousePlanningContext, FallbackPolicy, FixedEvaluationRange, InstantExecution, - MaterializationBinding, PhysicalGrouping, QueryLanguage, QueryPlan, QueryPlanEntry, -}; -use asap_types::summary_catalog::SummaryCatalog; -use serde::Deserialize; -use std::collections::HashMap; -use std::rc::Rc; - -#[derive(Debug, thiserror::Error)] -pub enum ClickHousePlanningError { - #[error("SQL lowering failed: {0}")] - Lower(String), - #[error("physical mapping failed: {0}")] - Bind(#[from] crate::planner_selection::SelectionError), -} - -pub struct ClickHousePlannedQuery { - pub canonical: QueryExpr, - pub canonical_sql: String, - pub physical: PhysicalExpr, - pub selection_trace: serde_json::Value, -} - -pub async fn plan_clickhouse_sql( - sql: &str, - catalog: &SqlCatalog, - accuracy: AccuracyTarget, -) -> Result { - let canonical = lower_sql_dialect(sql, catalog, SqlDialect::ClickhouseSQL, accuracy.clone()) - .await - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; - // SQL keeps relational parents such as Project and Filter above a - // summary-capable Aggregate. Use ASAPPlanner's recursive selector here; - // the PromQL deployment lowering retains its existing conservative rules. - let cost_model = ControlPlaneCostModel::new(accuracy.clone()); - let (selected, selection_trace) = - crate::planner_selection::select_workload_with_accuracy_model_and_trace( - vec![(0, Rc::new(canonical.clone()))], - accuracy, - &cost_model, - &asap_aware_mapping::NoAccuracyEvidence, - &asap_aware_mapping::DefaultAccuracyModel, - )?; - let selected = selected - .into_iter() - .next() - .map(|(_, node)| node) - .ok_or_else(|| { - crate::planner_selection::SelectionError::Workload( - "SQL workload search returned no root".into(), - ) - })?; - let physical = PhysicalExpr::committed(selected); - Ok(ClickHousePlannedQuery { - canonical_sql: canonical_sql_identity(&canonical), - canonical, - physical, - selection_trace, - }) -} - -#[cfg(test)] -pub async fn canonicalize_clickhouse_sql( - sql: &str, - catalog: &SqlCatalog, - accuracy: AccuracyTarget, -) -> Result { - let canonical = lower_sql_dialect(sql, catalog, SqlDialect::ClickhouseSQL, accuracy) - .await - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; - Ok(canonical_sql_identity(&canonical)) -} - -/// Resolve a request once, retaining the fixed identity for older publications. -pub async fn bind_clickhouse_sql( - sql: &str, - catalog: &SqlCatalog, - accuracy: AccuracyTarget, -) -> Result<(String, String, Option<(u64, u64)>), ClickHousePlanningError> { - let canonical = lower_sql_dialect(sql, catalog, SqlDialect::ClickhouseSQL, accuracy) - .await - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; - let fixed = format!("{canonical:?}"); - let binding = moving_window(&canonical); - Ok(match binding { - Some((identity, range)) => (fixed, identity, Some(range)), - None => (fixed.clone(), fixed, None), - }) -} - -/// Only the source bounds of a single-table aggregate are runtime parameters. -/// Expressions in projections, aggregate arguments and other predicates stay -/// in the identity, so changing them cannot reuse the installed computation. -pub fn canonical_sql_identity(canonical: &QueryExpr) -> String { - moving_window(canonical) - .map(|(identity, _)| identity) - .unwrap_or_else(|| format!("{canonical:?}")) -} - -fn moving_window(canonical: &QueryExpr) -> Option<(String, (u64, u64))> { - use planner_types::pre_asap::{CompareOpKind, ScalarValue, Source}; - let mut template = canonical.clone(); - let aggregate = match &mut template { - QueryExpr::Project { child, .. } => Rc::make_mut(child), - expr => expr, - }; - let QueryExpr::Aggregate { child, .. } = aggregate else { - return None; - }; - let QueryExpr::Scan { - source: Source::Table { .. }, - predicates, - schema, - } = Rc::make_mut(child) - else { - return None; - }; - let time = schema.time_index?; - fn bounds( - expr: &QueryExpr, - time: usize, - lower: &mut Option, - upper: &mut Option, - ) -> Option<()> { - match expr { - QueryExpr::BoolAnd(children) => { - for child in children { - bounds(child, time, lower, upper)?; - } - } - QueryExpr::Compare { left, op, right } if matches!(left.as_ref(), QueryExpr::Column(col) if *col == time) => - { - let value = constant_int64(right)?; - let (target, value) = match op { - CompareOpKind::Ge => (lower, value), - CompareOpKind::Gt => (lower, value.checked_add(1)?), - CompareOpKind::Lt => (upper, value), - CompareOpKind::Le => (upper, value.checked_add(1)?), - _ => return None, - }; - // Redundant bounds are deliberately not generalized. - if target.replace(value).is_some() { - return None; - } - } - _ => {} - } - Some(()) - } - let (mut lower, mut upper) = (None, None); - for predicate in predicates.iter() { - bounds(&predicate.0, time, &mut lower, &mut upper)?; - } - let (start, end) = (u64::try_from(lower?).ok()?, u64::try_from(upper?).ok()?); - if start >= end { - return None; - } - fn normalize(expr: &mut QueryExpr, time: usize, width: i64) { - match expr { - QueryExpr::BoolAnd(children) => { - for child in children { - normalize(child, time, width); - } - } - QueryExpr::Compare { left, op, right } if matches!(left.as_ref(), QueryExpr::Column(col) if *col == time) => - { - // `bounds` has checked the integer expressions and overflow. - // Preserve membership by converting both edges to [start,end). - let value = match op { - CompareOpKind::Ge | CompareOpKind::Gt => { - *op = CompareOpKind::Ge; - 0 - } - CompareOpKind::Lt | CompareOpKind::Le => { - *op = CompareOpKind::Lt; - width - } - _ => return, - }; - *right = Rc::new(QueryExpr::Literal(ScalarValue::Int64(value))); - } - _ => {} - } - } - for predicate in predicates { - normalize(Rc::make_mut(&mut predicate.0), time, (end - start) as i64); - } - Some((format!("moving-window-v1:{template:?}"), (start, end))) -} - -pub use asap_frontend_sql::SqlCatalog as ClickHouseSqlCatalog; - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ClickHouseSqlWorkload { - #[serde(rename = "sds")] - pub summary_catalog: SummaryCatalog, - pub precompute_plan: PrecomputePlan, - pub transmission_plan: TransmissionPlan, - pub tables: HashMap, - pub accuracy: AccuracyTarget, - pub queries: Vec, -} - -#[derive(Debug, Deserialize)] -pub struct ClickHouseSqlWorkloadEntry { - pub sql: String, - pub start_ms: u64, - pub end_ms: u64, - pub cumulative: bool, -} - -/// Workload input without predeclared summary families or materialization IDs. -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ClickHouseSqlAutomaticWorkload { - pub envelope: asap_types::precompute_plan::PlanEnvelope, - pub tables: HashMap, - pub accuracy: AccuracyTarget, - pub queries: Vec, -} - -pub async fn compile_automatic_clickhouse_workload( - request: &ClickHouseSqlAutomaticWorkload, -) -> Result< - ( - crate::physical::publication::PhysicalPlanPublication, - std::collections::BTreeMap, - ), - ClickHousePlanningError, -> { - let catalog = SqlCatalog { - tables: request.tables.clone(), - }; - let mut entries = std::collections::BTreeMap::new(); - let mut window_templates = std::collections::BTreeMap::>::new(); - let mut installed_dags = std::collections::BTreeMap::new(); - let mut selected_dags = std::collections::BTreeMap::new(); - let mut materializations = std::collections::BTreeMap::new(); - let mut selection_traces = std::collections::BTreeMap::new(); - for query in &request.queries { - validate_sql_evaluation(query)?; - // Selection runs once. Compilation installs only SummaryAgg nodes - // actually visited in this selected DAG, never a scripted family. - let mut planned = - plan_clickhouse_sql(&query.sql, &catalog, request.accuracy.clone()).await?; - let selection_trace = std::mem::take(&mut planned.selection_trace); - let template = planned.canonical_sql.clone(); - let (entry, installed) = compile_selected_sql(query, planned, |node, family| { - let config = materialize_selected_sql(node, family, query) - .map_err(crate::query_plan::QueryPlanError::Invalid)?; - let binding = MaterializationBinding { - full_window_slide_ms: matches!( - config.window_layout, - asap_types::WindowMaterializationLayout::FullWindow - ) - .then_some(config.slide_interval.saturating_mul(1_000)), - materialization: config.policy_fingerprint().into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_output( - config.policy_fingerprint().into(), - ), - output_grouping: PhysicalGrouping::Reduce(config.grouping_labels.names()), - window_ms: config.stored_window_ms(), - pane_origin_ms: config.pane_origin_ms, - readout_lookback_ms: Some(query.end_ms - query.start_ms), - item_labels: config.aggregated_labels.labels.clone(), - }; - materializations - .entry(config.policy_fingerprint()) - .or_insert(config); - Ok(binding) - })?; - index_sql_template(&mut window_templates, template, &entry); - selection_traces.insert(entry.canonical_query.clone(), selection_trace); - let key = QueryPlan::catalog_key(QueryLanguage::ClickHouseSql, &entry.canonical_query); - if entries.insert(key, entry).is_some() { - return Err(ClickHousePlanningError::Lower( - "duplicate canonical SQL query identity".into(), - )); - } - selected_dags.insert(query.sql.clone(), installed.document.clone()); - installed_dags.insert( - query.sql.clone(), - installed - .precompute_projection() - .map_err(ClickHousePlanningError::Lower)?, - ); - } - let configs: Vec<_> = materializations.into_values().collect(); - let sds = SummaryCatalog::from_materializations( - request.envelope.plan_id, - request.envelope.plan_version, - &configs, - ) - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; - let mut precompute = PrecomputePlan::build_backend_local(request.envelope.clone(), configs) - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; - precompute.executable_dags = installed_dags; - precompute - .bind_catalog(&sds) - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; - let mut transmission = crate::physical::compiler::build_transmission_plan( - request.envelope.clone(), - &precompute, - &std::collections::BTreeMap::new(), - ) - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; - transmission.summary_catalog = precompute.summary_catalog.clone(); - let mut publication = crate::physical::publication::PhysicalPlanPublication { - summary_catalog: sds, - precompute_plan: precompute, - collector_plans: Vec::new(), - transmission_plan: transmission, - query_plan: QueryPlan { - plan_id: request.envelope.plan_id, - plan_version: request.envelope.plan_version, - clickhouse_context: Some(ClickHousePlanningContext { - window_templates, - tables: request.tables.clone(), - accuracy: request.accuracy.clone(), - }), - selected_dags, - entries, - }, - }; - publication - .query_plan - .bind_catalog(&publication.summary_catalog) - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; - publication - .validate() - .map_err(ClickHousePlanningError::Lower)?; - Ok((publication, selection_traces)) -} - -fn materialize_selected_sql( - node: &planner_types::post_asap::SummaryNode, - family: &planner_types::post_asap::SummaryFamilyType, - query: &ClickHouseSqlWorkloadEntry, -) -> Result { - use crate::physical::backend_stage::{AggregationInput, BackendAggregation}; - use planner_types::{post_asap::SummaryExpr, pre_asap::Reduction}; - let SummaryExpr::SummaryAgg { - reduction: Reduction::Reduce(keys), - child, - .. - } = &node.expr - else { - return Err("SQL materialization requires a supported reduction".into()); - }; - if keys.is_without() { - return Err("SQL grouping exclusion requires a resolved projection".into()); - } - let SummaryExpr::KeepPreAsap(source) = &child.expr else { - return Err("SQL grouping requires a typed source subtree".into()); - }; - let source_schema = source.output_schema().map_err(|error| error.to_string())?; - let mut columns = Vec::new(); - for key in keys.keys() { - let mut column = source_schema - .columns - .get(*key) - .cloned() - .ok_or("SQL grouping column is absent from source schema")?; - column.table = None; - columns.push(column); - } - let grouping = asap_types::GroupingProjection::new(columns); - grouping.validate_table_group_codec()?; - let leaf = clickhouse_materialization_leaf_contract(node, query.start_ms, query.end_ms)?; - let ClickHouseMaterializationLeaf { - table, - value, - value_source_column, - window_secs: window, - population, - timestamp_column: timestamp, - } = leaf; - let window_secs = window.ok_or("SQL materialization requires a bounded window")?; - let aggregation = BackendAggregation { - aggregation_id: String::new(), - metric_name: format!("{table}.{}", value.column().unwrap_or("constant")), - family: family.clone(), - window_secs, - spatial_filter: String::new(), - grouping: grouping.names(), - item_label: None, - heap_update_mode: None, - aggregation_input: AggregationInput::Raw, - }; - let mut config = crate::physical::compiler::aggregation_config_for_materialization( - &aggregation, - asap_types::QueryLanguage::ClickHouseSql, - ) - .map_err(|error| error.to_string())?; - config.table_name = Some(table); - config.grouping_labels = grouping; - config.value_projection = Some(value); - config.table_timestamp_column = Some(timestamp); - config.table_population = Some(population); - config.value_source_column = value_source_column; - config.partitioning = Some(asap_types::sds::PopulationPartitioning::Grouped); - config.pane_origin_ms = Some( - i64::try_from(query.start_ms) - .map_err(|_| "SQL evaluation timestamp exceeds runtime range")?, - ); - config.num_aggregates_to_retain = Some(2); - Ok(config) -} - -pub async fn compile_clickhouse_workload( - request: &ClickHouseSqlWorkload, -) -> Result { - request - .precompute_plan - .validate_against_catalog(&request.summary_catalog) - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; - request - .transmission_plan - .validate(&request.precompute_plan) - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; - let catalog = SqlCatalog { - tables: request.tables.clone(), - }; - let mut entries = std::collections::BTreeMap::new(); - let mut window_templates = std::collections::BTreeMap::>::new(); - let mut installed_dags = std::collections::BTreeMap::new(); - let mut selected_dags = std::collections::BTreeMap::new(); - for query in &request.queries { - let planned = plan_clickhouse_sql(&query.sql, &catalog, request.accuracy.clone()).await?; - let template = planned.canonical_sql.clone(); - let (executable, installed) = compile_selected_sql(query, planned, |node, family| { - bind_selected_node(node, family, query, request) - })?; - index_sql_template(&mut window_templates, template, &executable); - selected_dags.insert(query.sql.clone(), installed.document.clone()); - installed_dags.insert( - query.sql.clone(), - installed - .precompute_projection() - .map_err(ClickHousePlanningError::Lower)?, - ); - let identity = - QueryPlan::catalog_key(QueryLanguage::ClickHouseSql, &executable.canonical_query); - if entries.insert(identity.clone(), executable).is_some() { - return Err(ClickHousePlanningError::Lower(format!( - "duplicate canonical SQL query identity `{identity}`" - ))); - } - } - let mut precompute_plan = request.precompute_plan.clone(); - precompute_plan.executable_dags = installed_dags; - let mut publication = crate::physical::publication::PhysicalPlanPublication { - summary_catalog: request.summary_catalog.clone(), - precompute_plan, - collector_plans: Vec::new(), - transmission_plan: request.transmission_plan.clone(), - query_plan: QueryPlan { - plan_id: request.summary_catalog.plan_id, - plan_version: request.summary_catalog.plan_version, - clickhouse_context: Some(ClickHousePlanningContext { - window_templates, - tables: request.tables.clone(), - accuracy: request.accuracy.clone(), - }), - selected_dags, - entries, - }, - }; - publication - .query_plan - .bind_catalog(&publication.summary_catalog) - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; - publication - .validate() - .map_err(ClickHousePlanningError::Lower)?; - Ok(publication) -} - -fn index_sql_template( - templates: &mut std::collections::BTreeMap>, - template: String, - entry: &QueryPlanEntry, -) { - // External subqueries still contain fixed SQL literals. - if template.starts_with("moving-window-v1:") - && !entry - .nodes - .values() - .any(|node| matches!(node, crate::query_plan::QueryPlanNode::ExternalExact { .. })) - { - templates - .entry(template) - .or_default() - .push(entry.canonical_query.clone()); - } -} - -fn compile_selected_sql( - query: &ClickHouseSqlWorkloadEntry, - planned: ClickHousePlannedQuery, - mut bind: F, -) -> Result< - ( - QueryPlanEntry, - crate::physical::executable_binding::InstalledPostAsapDag, - ), - ClickHousePlanningError, -> -where - F: FnMut( - &Rc, - &planner_types::post_asap::SummaryFamilyType, - ) -> Result, -{ - validate_sql_evaluation(query)?; - let PhysicalExpr::Committed(crate::physical::post_asap::PostAsapPlan::Summary(root)) = - planned.physical - else { - return Err(ClickHousePlanningError::Lower( - "SQL did not produce a summary DAG".into(), - )); - }; - let semantic = planner_types::post_asap::compile_executable_dag_with_node_ids(&root) - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; - let mut materialization_nodes = std::collections::BTreeMap::new(); - let mut query_nodes = std::collections::BTreeMap::new(); - let executable = crate::query_plan::compile_bound_relational_mapped( - query.sql.clone(), - format!("{:?}", planned.canonical), - &root, - FixedEvaluationRange { - start_ms: query.start_ms, - end_ms: query.end_ms, - cumulative: query.cumulative, - }, - InstantExecution { - lookback_ms: query.end_ms.saturating_sub(query.start_ms), - full_history: query.start_ms == 0, - cumulative_readout: query.cumulative, - }, - FallbackPolicy::ExactBackend, - |node, family| { - let binding = bind(node, family)?; - let id = semantic.node_ids.node_id(node).ok_or_else(|| { - crate::query_plan::QueryPlanError::Invalid( - "selected SQL node is absent from semantic DAG".into(), - ) - })?; - materialization_nodes.insert(id, binding.materialization); - Ok(binding) - }, - |node, query_node| { - if let Some(id) = semantic.node_ids.node_id(node) { - query_nodes.insert(id, query_node); - } - }, - ) - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; - let installed = crate::physical::executable_binding::install_selected_dag( - query.sql.clone(), - &semantic.dag, - executable.root, - |id| materialization_nodes.get(&id).copied(), - |id| query_nodes.get(&id).copied(), - ) - .map_err(ClickHousePlanningError::Lower)?; - crate::physical::executable_binding::validate_query_plan(&installed, &executable) - .map_err(ClickHousePlanningError::Lower)?; - if executable - .nodes - .values() - .any(|node| matches!(node, crate::query_plan::QueryPlanNode::ExactFallback { .. })) - { - return Err(ClickHousePlanningError::Lower( - "compiled SQL contains an unsupported operator; publication refused".into(), - )); - } - Ok((executable, installed)) -} - -fn validate_sql_evaluation( - query: &ClickHouseSqlWorkloadEntry, -) -> Result<(), ClickHousePlanningError> { - if query.start_ms >= query.end_ms || query.end_ms > i64::MAX as u64 { - return Err(ClickHousePlanningError::Lower( - "SQL evaluation requires start_ms < end_ms within signed Unix milliseconds".into(), - )); - } - Ok(()) -} - -fn bind_selected_node( - node: &planner_types::post_asap::SummaryNode, - family: &planner_types::post_asap::SummaryFamilyType, - query: &ClickHouseSqlWorkloadEntry, - request: &ClickHouseSqlWorkload, -) -> Result { - let ClickHouseMaterializationLeaf { - table: table_ref, - value: value_column, - window_secs: source_window, - population: spatial_filter, - timestamp_column, - .. - } = clickhouse_materialization_leaf_contract(node, query.start_ms, query.end_ms) - .map_err(crate::query_plan::QueryPlanError::Invalid)?; - let expected = family.clone(); - let selected = select_materialization( - &request.precompute_plan.materializations, - &table_ref, - &value_column, - &spatial_filter.canonical(), - &expected, - source_window.unwrap_or((query.end_ms.saturating_sub(query.start_ms)) / 1000), - )?; - if selected.table_timestamp_column.as_deref() != Some(timestamp_column.as_str()) { - return Err(crate::query_plan::QueryPlanError::Invalid( - "SQL timestamp projection differs from the installed materialization".into(), - )); - } - Ok(MaterializationBinding { - full_window_slide_ms: matches!( - selected.window_layout, - asap_types::WindowMaterializationLayout::FullWindow - ) - .then_some(selected.slide_interval.saturating_mul(1_000)), - materialization: selected.policy_fingerprint().into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_output( - selected.policy_fingerprint().into(), - ), - output_grouping: PhysicalGrouping::Reduce(selected.grouping_labels.names()), - window_ms: selected.stored_window_ms(), - pane_origin_ms: selected.pane_origin_ms, - readout_lookback_ms: source_window.map(|seconds| seconds.saturating_mul(1000)), - item_labels: selected.aggregated_labels.labels.clone(), - }) -} - -/// Evaluate only exact integer constant arithmetic at the installation boundary. -/// No SQL text rewrite, floating coercion, or runtime-column evaluation is allowed. -fn constant_int64(expr: &QueryExpr) -> Option { - use planner_types::pre_asap::{ArithmeticOpKind, ScalarValue}; - match expr { - QueryExpr::Literal(ScalarValue::Int64(value)) => Some(*value), - QueryExpr::Arithmetic { op, left, right } => { - let left = constant_int64(left)?; - let right = constant_int64(right)?; - match op { - ArithmeticOpKind::Add => left.checked_add(right), - ArithmeticOpKind::Sub => left.checked_sub(right), - ArithmeticOpKind::Mul => left.checked_mul(right), - _ => None, - } - } - _ => None, - } -} - -/// The table leaf a SQL summary materialization is admitted against. -#[derive(Debug)] -struct ClickHouseMaterializationLeaf { - table: String, - value: asap_types::sds::ValueProjectionIdentity, - /// Producer typing for a column projection: what the ingest path must know - /// to read the column safely (integer exactness, NULL skipping). `None` - /// for constant projections, which carry their own literal. - value_source_column: Option, - window_secs: Option, - population: asap_types::table_population::TablePopulation, - timestamp_column: String, -} - -fn clickhouse_materialization_leaf_contract( - node: &planner_types::post_asap::SummaryNode, - evaluation_start_ms: u64, - evaluation_end_ms: u64, -) -> Result { - use planner_types::{ - post_asap::SummaryExpr, - pre_asap::{CompareOpKind, QueryExpr, ScalarValue, Source}, - }; - let SummaryExpr::SummaryAgg { - child, - input, - family, - .. - } = &node.expr - else { - return Err("SQL materialization leaf is not a summary aggregate".into()); - }; - if input.item.is_some() { - return Err("SQL item/weight summary inputs require an explicit item projection".into()); - } - let SummaryExpr::KeepPreAsap(expr) = &child.expr else { - return Err("SQL materialization leaf has no tabular source".into()); - }; - fn table_scan( - expr: &QueryExpr, - ) -> Option<( - &str, - &[planner_types::pre_asap::Predicate], - &planner_types::pre_asap::Schema, - )> { - match expr { - QueryExpr::Scan { - source: Source::Table { table_ref }, - predicates, - schema, - } => Some((table_ref, predicates, schema)), - _ => None, - } - } - let (source, explicit_window) = match expr.as_ref() { - QueryExpr::TimeRange { child, range } - if range.as_millis() > 0 && range.as_millis() % 1_000 == 0 => - { - (child.as_ref(), Some(range.as_secs())) - } - source => (source, None), - }; - let Some((table_ref, predicates, schema)) = table_scan(source) else { - return Err("SQL materialization leaf has no table scan".into()); - }; - use asap_types::sds::ValueProjectionIdentity; - use planner_types::post_asap::SummaryInputExpr; - let mut value_source_column = None; - let value_projection = match &input.weight { - SummaryInputExpr::Column( - planner_types::pre_asap::ColumnRef::Wildcard - | planner_types::pre_asap::ColumnRef::SampleValue, - ) if matches!( - family, - planner_types::post_asap::SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Count, - _ - ) - ) => - { - ValueProjectionIdentity::Constant { - value: ScalarValue::Int64(1), - } - } - SummaryInputExpr::Column( - planner_types::pre_asap::ColumnRef::Named(name) - | planner_types::pre_asap::ColumnRef::Qualified { name, .. }, - ) => { - let column = schema - .columns - .iter() - .find(|column| column.name == *name) - .ok_or("SQL summary value projection is not a source column")?; - // Numeric source columns are admitted with their declared type and - // nullability, which the ingest reader honours: integers get an - // exactness guard on the way into f64 summary state, and NULL rows - // are skipped the way a SQL aggregate skips them. Non-numeric - // columns have no value semantics to summarise and stay refused. - if !matches!( - column.dtype, - planner_types::pre_asap::DataType::Float64 - | planner_types::pre_asap::DataType::Int64 - ) { - return Err(format!( - "SQL value readout requires a numeric source column; `{name}` is {:?}", - column.dtype - )); - } - let mut source_column = column.clone(); - source_column.table = None; - value_source_column = Some(source_column); - ValueProjectionIdentity::Column { name: name.clone() } - } - SummaryInputExpr::Constant(value) if value.is_finite() => { - let value = if value.fract() == 0.0 && value.abs() <= (1_u64 << 53) as f64 { - ScalarValue::Int64(*value as i64) - } else { - ScalarValue::Float64(*value) - }; - ValueProjectionIdentity::Constant { value } - } - _ => return Err("SQL summary requires a named column or finite numeric update".into()), - }; - - fn comparisons<'a>(expr: &'a QueryExpr, out: &mut Vec<&'a QueryExpr>) { - if let QueryExpr::BoolAnd(children) = expr { - for child in children { - comparisons(child, out); - } - } else { - out.push(expr); - } - } - let mut lower_ms = None; - let mut upper_ms = None; - let mut leaves = Vec::new(); - let mut population = asap_types::table_population::TablePopulation::default(); - for predicate in predicates { - comparisons(&predicate.0, &mut leaves); - } - for leaf in leaves { - let QueryExpr::Compare { left, op, right } = leaf else { - return Err("SQL materialization predicate is not a comparison".into()); - }; - let QueryExpr::Column(column) = left.as_ref() else { - return Err("SQL materialization predicate must reference a column".into()); - }; - let name = schema - .columns - .get(*column) - .map(|column| column.name.as_str()) - .ok_or_else(|| { - "SQL materialization predicate references an unknown column".to_string() - })?; - let folded = - constant_int64(right).map(|value| QueryExpr::Literal(ScalarValue::Int64(value))); - let right = folded.as_ref().unwrap_or(right.as_ref()); - match (name, op, right) { - ( - name, - CompareOpKind::Gt | CompareOpKind::Ge, - QueryExpr::Literal(ScalarValue::Int64(value)), - ) if schema - .time_index - .is_some_and(|index| schema.columns[index].name == name) => - { - let bound = if matches!(op, CompareOpKind::Gt) { - value - .checked_add(1) - .ok_or("SQL exclusive lower timestamp overflows")? - } else { - *value - }; - lower_ms = Some(lower_ms.map_or(bound, |previous: i64| previous.max(bound))); - } - ( - name, - CompareOpKind::Lt | CompareOpKind::Le, - QueryExpr::Literal(ScalarValue::Int64(value)), - ) if schema - .time_index - .is_some_and(|index| schema.columns[index].name == name) => - { - let bound = if matches!(op, CompareOpKind::Le) { - value - .checked_add(1) - .ok_or("SQL inclusive upper timestamp overflows")? - } else { - *value - }; - upper_ms = Some(upper_ms.map_or(bound, |previous: i64| previous.min(bound))); - } - (_, _, QueryExpr::Literal(value)) - if !schema - .time_index - .is_some_and(|index| schema.columns[index].name == name) => - { - population - .predicates - .push(asap_types::table_population::TableColumnPredicate { - column: name.into(), - operator: op.clone(), - value: value.clone(), - }); - } - _ => { - return Err(format!( - "SQL population predicate on {name} needs a canonical catalog filter" - )) - } - } - } - population.validate()?; - if let (Some(lower), Some(upper)) = (lower_ms, upper_ms) { - if u64::try_from(lower).ok() != Some(evaluation_start_ms) - || u64::try_from(upper).ok() != Some(evaluation_end_ms) - { - return Err( - "SQL source timestamp bounds differ from the fixed evaluation range".into(), - ); - } - } - let inferred_window = match (lower_ms, upper_ms) { - (Some(lower), Some(upper)) if upper > lower && (upper - lower) % 1_000 == 0 => { - Some((upper - lower) as u64 / 1_000) - } - (None, None) => None, - _ => { - return Err("SQL timestamp range must provide compatible lower and upper bounds".into()) - } - }; - let window_secs = explicit_window.or(inferred_window).ok_or_else(|| { - "SQL table summary requires a positive whole-second timestamp range".to_string() - })?; - Ok(ClickHouseMaterializationLeaf { - table: table_ref.to_owned(), - value: value_projection, - value_source_column, - window_secs: Some(window_secs), - population, - timestamp_column: schema - .time_index - .and_then(|index| schema.columns.get(index)) - .ok_or("SQL summary source has no timestamp projection")? - .name - .clone(), - }) -} - -fn select_materialization<'a>( - materializations: &'a [asap_types::PrecomputeMaterialization], - table_ref: &str, - value_projection: &asap_types::sds::ValueProjectionIdentity, - spatial_filter: &str, - expected: &planner_types::post_asap::SummaryFamilyType, - semantic_window_seconds: u64, -) -> Result<&'a asap_types::PrecomputeMaterialization, crate::query_plan::QueryPlanError> { - let mut matches = materializations.iter().filter(|candidate| { - candidate.table_name.as_deref() == Some(table_ref) - && candidate.effective_value_projection() == value_projection - && candidate.population_filter_canonical().ok().as_deref() == Some(spatial_filter) - && candidate - .accumulator_spec() - .ok() - .is_some_and(|spec| spec.family == *expected) - && semantic_window_seconds - .checked_mul(1000) - .is_some_and(|window| window % candidate.window_size.saturating_mul(1000) == 0) - }); - let selected = matches.next().ok_or_else(|| { - crate::query_plan::QueryPlanError::Invalid(format!( - "no precompute materialization matches {table_ref}/{value_projection:?}/{expected:?}" - )) - })?; - if matches.next().is_some() { - return Err(crate::query_plan::QueryPlanError::Invalid(format!( - "ambiguous precompute materializations match {table_ref}/{value_projection:?}/{expected:?}" - ))); - } - Ok(selected) -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - use planner_types::pre_asap::{Column, DataType, Schema}; - - // A dashboard refresh must reuse the installed identity without treating - // value thresholds or window length as runtime parameters. - #[tokio::test] - async fn moving_sql_window_reuses_identity() { - let catalog = SqlCatalog { - tables: HashMap::from([( - "telemetry".into(), - Schema { - columns: vec![ - Column::new("timestamp_ms", DataType::Int64, false), - Column::new("value", DataType::Float64, false), - ], - time_index: Some(0), - ..Default::default() - }, - )]), - }; - let identity = |start, end, threshold| { - format!( - "SELECT sum(value) FROM telemetry WHERE timestamp_ms >= {start} AND timestamp_ms < {end} AND value > {threshold}" - ) - }; - let first = - canonicalize_clickhouse_sql(&identity(1000, 3000, 5), &catalog, AccuracyTarget::Exact) - .await - .unwrap(); - let shifted = - canonicalize_clickhouse_sql(&identity(2000, 4000, 5), &catalog, AccuracyTarget::Exact) - .await - .unwrap(); - assert_eq!(first, shifted); - let wider = - canonicalize_clickhouse_sql(&identity(1000, 4000, 5), &catalog, AccuracyTarget::Exact) - .await - .unwrap(); - let threshold = - canonicalize_clickhouse_sql(&identity(2000, 4000, 6), &catalog, AccuracyTarget::Exact) - .await - .unwrap(); - assert_ne!(first, wider); - assert_ne!(first, threshold); - let (_, template, range) = - bind_clickhouse_sql(&identity(2000, 4000, 5), &catalog, AccuracyTarget::Exact) - .await - .unwrap(); - assert_eq!(template, first); - assert_eq!(range, Some((2000, 4000))); - - // Dashboard bounds may use arithmetic and an inclusive upper edge. - let (_, inclusive, range) = bind_clickhouse_sql( - "SELECT sum(value) FROM telemetry WHERE timestamp_ms > 3999 - 2000 AND timestamp_ms <= 3999 AND value > 5", - &catalog, - AccuracyTarget::Exact, - ).await.unwrap(); - assert_eq!(inclusive, first); - assert_eq!(range, Some((2000, 4000))); - - // Projection constants are query semantics, never time parameters. - let projected = |start, end| { - format!("SELECT sum(value), {start} AS window_start FROM telemetry WHERE timestamp_ms >= {start} AND timestamp_ms < {end}") - }; - assert_ne!( - canonicalize_clickhouse_sql(&projected(1000, 3000), &catalog, AccuracyTarget::Exact) - .await - .unwrap(), - canonicalize_clickhouse_sql(&projected(2000, 4000), &catalog, AccuracyTarget::Exact) - .await - .unwrap(), - ); - // Unsupported bound shapes and non-aggregate scans retain fixed keys. - for sql in [ - "SELECT sum(value) FROM telemetry", - "SELECT sum(value) FROM telemetry WHERE timestamp_ms >= 1000 AND timestamp_ms <= 9223372036854775807", - "SELECT value FROM telemetry WHERE timestamp_ms >= 1000 AND timestamp_ms < 3000", - ] { - let (fixed, template, range) = - bind_clickhouse_sql(sql, &catalog, AccuracyTarget::Exact) - .await - .unwrap(); - assert_eq!(fixed, template); - assert_eq!(range, None); - } - } - - fn materialization( - agg: AggregationType, - value_column: &str, - window: u64, - slide: u64, - parameter: (&str, serde_json::Value), - ) -> PrecomputeMaterialization { - let mut value = PrecomputeMaterialization::new( - agg, - String::new(), - std::collections::HashMap::from([(parameter.0.into(), parameter.1)]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - window, - slide, - WindowKind::Tumbling, - String::new(), - format!("telemetry.{value_column}"), - None, - Some("telemetry".into()), - Some(value_column.into()), - ); - value.pane_origin_ms = Some(0); - value.table_timestamp_column = Some("timestamp_ms".into()); - value - } - - /// A table leaf whose value column has the given producer typing. - fn typed_value_leaf( - dtype: planner_types::pre_asap::DataType, - nullable: bool, - ) -> planner_types::post_asap::SummaryNode { - use planner_types::post_asap::{ - SummaryExpr, SummaryInputExpr, SummaryNode, SummarySchema, SummaryUpdate, - }; - use planner_types::pre_asap::{ - Column, ColumnRef, CompareOpKind, DataType, Predicate, QueryExpr, Reduction, - ScalarValue, Schema, Source, - }; - let schema = Schema::with_time_index( - vec![ - Column::new("timestamp_ms", DataType::Timestamp, false), - Column::new("value", dtype, nullable), - ], - 0, - Vec::new(), - ); - let bound = |op: CompareOpKind, at: i64| { - Predicate(std::rc::Rc::new(QueryExpr::Compare { - left: std::rc::Rc::new(QueryExpr::Column(0)), - op, - right: std::rc::Rc::new(QueryExpr::Literal(ScalarValue::Int64(at))), - })) - }; - let scan = QueryExpr::Scan { - source: Source::Table { - table_ref: "telemetry".into(), - }, - predicates: vec![ - bound(CompareOpKind::Ge, 0), - bound(CompareOpKind::Lt, 60_000), - ], - schema, - }; - let family = materialization( - AggregationType::Sum, - "value", - 60, - 60, - ("variant", serde_json::json!(1)), - ) - .accumulator_spec() - .unwrap() - .family; - let summary_schema = SummarySchema { - fields: vec![], - time_index: None, - }; - SummaryNode { - expr: SummaryExpr::SummaryAgg { - child: std::rc::Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(std::rc::Rc::new(scan)), - schema: summary_schema.clone(), - guarantee: Default::default(), - }), - family, - input: SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::Named("value".into())), - weight_domain: Default::default(), - }, - reduction: Reduction::Reduce(vec![].into()), - grouping: Default::default(), - }, - schema: summary_schema, - guarantee: Default::default(), - } - } - - /// Numeric source columns are admitted with their producer typing, which - /// the ingest reader needs to read them safely. Before this the contract - /// took non-null `Float64` only, so an ordinary nullable or integer - /// ClickHouse column could not be automatically materialized at all. - #[test] - fn numeric_value_columns_are_admitted_with_their_producer_typing() { - use planner_types::pre_asap::DataType; - for (dtype, nullable) in [ - (DataType::Float64, false), - (DataType::Float64, true), - (DataType::Int64, false), - (DataType::Int64, true), - ] { - let leaf = clickhouse_materialization_leaf_contract( - &typed_value_leaf(dtype.clone(), nullable), - 0, - 60_000, - ) - .unwrap_or_else(|error| panic!("{dtype:?}/{nullable}: {error}")); - assert_eq!( - leaf.value, - asap_types::sds::ValueProjectionIdentity::Column { - name: "value".into() - } - ); - let column = leaf - .value_source_column - .expect("a column projection carries its producer typing"); - assert_eq!(column.dtype, dtype); - assert_eq!(column.nullable, nullable); - // Typing is a read concern, not an identity one: the same column - // is the same policy however it is declared. - assert_eq!(column.table, None); - } - } - - /// Non-numeric columns have no value semantics to summarise. Refusing them - /// protects the result; it is not a gap to be widened. - #[test] - fn non_numeric_value_columns_stay_refused() { - use planner_types::pre_asap::DataType; - for dtype in [DataType::Utf8, DataType::Bool, DataType::Timestamp] { - let error = clickhouse_materialization_leaf_contract( - &typed_value_leaf(dtype.clone(), false), - 0, - 60_000, - ) - .unwrap_err(); - assert!( - error.contains("numeric source column"), - "{dtype:?}: {error}" - ); - } - } - - #[test] - fn keyed_summary_input_is_not_replaced_by_its_unit_weight() { - use planner_types::post_asap::{ - SummaryExpr, SummaryInputExpr, SummaryNode, SummarySchema, SummaryUpdate, - }; - use planner_types::pre_asap::{ColumnRef, QueryExpr, Reduction, ScalarValue}; - let schema = SummarySchema { - fields: vec![], - time_index: None, - }; - let family = materialization( - AggregationType::Sum, - "value", - 60, - 60, - ("variant", serde_json::json!(1)), - ) - .accumulator_spec() - .unwrap() - .family; - let node = SummaryNode { - expr: SummaryExpr::SummaryAgg { - child: std::rc::Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(std::rc::Rc::new(QueryExpr::Literal( - ScalarValue::Int64(1), - ))), - schema: schema.clone(), - guarantee: Default::default(), - }), - family, - input: SummaryUpdate { - item: Some(SummaryInputExpr::Column(ColumnRef::Named("value".into()))), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: Default::default(), - }, - reduction: Reduction::Reduce(vec![].into()), - grouping: Default::default(), - }, - schema, - guarantee: Default::default(), - }; - assert!(clickhouse_materialization_leaf_contract(&node, 0, 60_000) - .unwrap_err() - .contains("explicit item projection")); - } - - #[test] - fn selected_nodes_bind_unique_family_parameters_source_and_window() { - let sum_60 = materialization( - AggregationType::Sum, - "requests", - 60, - 10, - ("variant", serde_json::json!(1)), - ); - let count_60 = materialization( - AggregationType::Max, - "requests", - 60, - 10, - ("variant", serde_json::json!(2)), - ); - let sum_300 = materialization( - AggregationType::Sum, - "requests", - 300, - 30, - ("variant", serde_json::json!(3)), - ); - let other = materialization( - AggregationType::Sum, - "latency", - 60, - 10, - ("variant", serde_json::json!(1)), - ); - let dd_2 = materialization( - AggregationType::DDSketch, - "requests", - 60, - 10, - ("relativeAccuracy", serde_json::json!(0.02)), - ); - let dd_5 = materialization( - AggregationType::DDSketch, - "requests", - 60, - 10, - ("relativeAccuracy", serde_json::json!(0.05)), - ); - let configs = vec![ - sum_60.clone(), - count_60.clone(), - sum_300, - other, - dd_2.clone(), - dd_5, - ]; - let sum_family = sum_60.accumulator_spec().unwrap().family; - let count_family = count_60.accumulator_spec().unwrap().family; - assert_eq!( - select_materialization( - &configs, - "telemetry", - &asap_types::sds::ValueProjectionIdentity::Column { - name: "requests".into() - }, - "", - &sum_family, - 60 - ) - .unwrap() - .policy_fingerprint(), - sum_60.policy_fingerprint() - ); - let dd_family = dd_2.accumulator_spec().unwrap().family; - assert_eq!( - select_materialization( - &configs, - "telemetry", - &asap_types::sds::ValueProjectionIdentity::Column { - name: "requests".into() - }, - "", - &dd_family, - 60 - ) - .unwrap() - .policy_fingerprint(), - dd_2.policy_fingerprint() - ); - assert_eq!( - select_materialization( - &configs, - "telemetry", - &asap_types::sds::ValueProjectionIdentity::Column { - name: "requests".into() - }, - "", - &count_family, - 60 - ) - .unwrap() - .policy_fingerprint(), - count_60.policy_fingerprint() - ); - assert!(select_materialization( - &configs, - "telemetry", - &asap_types::sds::ValueProjectionIdentity::Column { - name: "missing".into() - }, - "", - &sum_family, - 60 - ) - .is_err()); - let mut ambiguous = configs.clone(); - ambiguous.push(sum_60); - assert!(select_materialization( - &ambiguous, - "telemetry", - &asap_types::sds::ValueProjectionIdentity::Column { - name: "requests".into() - }, - "", - &sum_family, - 60 - ) - .unwrap_err() - .to_string() - .contains("ambiguous")); - } - - #[test] - fn constant_integer_boundaries_reject_overflow_and_dynamic_values() { - use planner_types::pre_asap::{ArithmeticOpKind, ScalarValue}; - let literal = |value| QueryExpr::Literal(ScalarValue::Int64(value)); - let subtract = |left, right| QueryExpr::Arithmetic { - op: ArithmeticOpKind::Sub, - left: std::rc::Rc::new(left), - right: std::rc::Rc::new(right), - }; - assert_eq!( - constant_int64(&subtract(literal(1_788_891_296_000), literal(43_200_000))), - Some(1_788_848_096_000) - ); - assert_eq!( - constant_int64(&subtract(literal(i64::MIN), literal(1))), - None - ); - assert_eq!( - constant_int64(&subtract(QueryExpr::Column(0), literal(1))), - None - ); - assert_eq!( - constant_int64(&QueryExpr::Literal(ScalarValue::Float64(1.0))), - None - ); - } - - #[test] - fn sql_evaluation_rejects_empty_reversed_and_unrepresentable_ranges() { - for (start_ms, end_ms) in [(2, 1), (1, 1), (0, u64::MAX)] { - assert!(validate_sql_evaluation(&ClickHouseSqlWorkloadEntry { - sql: "SELECT sum(value) FROM telemetry".into(), - start_ms, - end_ms, - cumulative: true, - }) - .is_err()); - } - } - - #[tokio::test] - async fn rejects_mixed_snapshots_and_preserves_local_sql_contracts() { - let config = materialization( - AggregationType::Sum, - "value", - 2, - 2, - ("variant", serde_json::json!(1)), - ); - let sds = - SummaryCatalog::from_materializations(71, 1, std::slice::from_ref(&config)).unwrap(); - let envelope = crate::physical::compiler::PlanEnvelope { - plan_id: 71, - plan_version: 1, - generated_at_unix_ms: 0, - activation_unix_ms: 0, - expiry_unix_ms: None, - backend_compat: crate::physical::compiler::BACKEND_COMPAT.into(), - planner_revision: crate::physical::compiler::PLANNER_REVISION.into(), - capability_snapshot_id: "clickhouse-mixed-compile-test".into(), - }; - let mut precompute = - PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); - precompute.bind_catalog(&sds).unwrap(); - let mut transmission = crate::physical::compiler::build_transmission_plan( - envelope, - &precompute, - &std::collections::BTreeMap::new(), - ) - .unwrap(); - transmission.summary_catalog = Some(sds.reference().unwrap()); - let timestamped = |time_name: &str, value_name: &str| { - Schema::with_time_index( - vec![ - Column::new(time_name, DataType::Timestamp, false), - Column::new(value_name, DataType::Float64, false), - ], - 0, - vec![], - ) - }; - let mut request = ClickHouseSqlWorkload { - summary_catalog: sds, - precompute_plan: precompute, - transmission_plan: transmission, - tables: HashMap::from([ - ("telemetry".into(), timestamped("timestamp_ms", "value")), - ( - "divisors".into(), - Schema::new(vec![ - Column::new("timestamp", DataType::Int64, false), - Column::new("divisor", DataType::Float64, false), - ]), - ), - ]), - accuracy: AccuracyTarget::Exact, - queries: vec![ClickHouseSqlWorkloadEntry { - sql: "SELECT sums.timestamp, sums.total / divisors.divisor AS ratio FROM (SELECT 2000 AS timestamp, sum(value) AS total FROM telemetry WHERE timestamp_ms >= 0 AND timestamp_ms < 2000) AS sums INNER JOIN divisors ON sums.timestamp = divisors.timestamp".into(), - start_ms: 0, - end_ms: 2_000, - cumulative: true, - }], - }; - let Err(error) = compile_clickhouse_workload(&request).await else { - panic!("local summary joined to an external table needs a common snapshot proof"); - }; - assert!( - error.to_string().contains("common snapshot proof"), - "{error}" - ); - request.queries[0].sql = - "SELECT sum(value) FROM telemetry WHERE timestamp_ms >= 0 AND timestamp_ms < 2000" - .into(); - let publication = compile_clickhouse_workload(&request).await.unwrap(); - // A simple installed aggregate publishes the same key as a refresh. - let simple_sql = - "SELECT sum(value) FROM telemetry WHERE timestamp_ms >= 0 AND timestamp_ms < 2000"; - let simple = compile_clickhouse_workload(&ClickHouseSqlWorkload { - summary_catalog: request.summary_catalog.clone(), - precompute_plan: request.precompute_plan.clone(), - transmission_plan: request.transmission_plan.clone(), - tables: request.tables.clone(), - accuracy: request.accuracy.clone(), - queries: vec![ClickHouseSqlWorkloadEntry { - sql: simple_sql.into(), - start_ms: 0, - end_ms: 2000, - cumulative: true, - }], - }) - .await - .unwrap(); - let shifted = canonicalize_clickhouse_sql( - "SELECT sum(value) FROM telemetry WHERE timestamp_ms >= 2000 AND timestamp_ms < 4000", - &SqlCatalog { - tables: request.tables.clone(), - }, - AccuracyTarget::Exact, - ) - .await - .unwrap(); - assert_eq!( - simple - .query_plan - .clickhouse_context - .as_ref() - .unwrap() - .window_templates[&shifted] - .len(), - 1 - ); - assert!(shifted.starts_with("moving-window-v1:")); - // Different windows remain distinct publications even with one template. - let windows = || { - vec![ - ClickHouseSqlWorkloadEntry { sql: simple_sql.into(), start_ms: 0, end_ms: 2000, cumulative: true }, - ClickHouseSqlWorkloadEntry { - sql: "SELECT sum(value) FROM telemetry WHERE timestamp_ms >= 2000 AND timestamp_ms < 4000".into(), - start_ms: 2000, end_ms: 4000, cumulative: true, - }, - ] - }; - let multiple = compile_clickhouse_workload(&ClickHouseSqlWorkload { - summary_catalog: request.summary_catalog.clone(), - precompute_plan: request.precompute_plan.clone(), - transmission_plan: request.transmission_plan.clone(), - tables: request.tables.clone(), - accuracy: request.accuracy.clone(), - queries: windows(), - }) - .await - .unwrap(); - assert_eq!(multiple.query_plan.entries.len(), 2); - let (multiple_auto, multiple_traces) = - compile_automatic_clickhouse_workload(&ClickHouseSqlAutomaticWorkload { - envelope: request.precompute_plan.envelope.clone(), - tables: request.tables.clone(), - accuracy: request.accuracy.clone(), - queries: windows(), - }) - .await - .unwrap(); - assert_eq!(multiple_auto.query_plan.entries.len(), 2); - for plan in [&multiple.query_plan, &multiple_auto.query_plan] { - let identities = &plan.clickhouse_context.as_ref().unwrap().window_templates[&shifted]; - assert_eq!(identities.len(), 2); - assert_ne!(identities[0], identities[1]); - for identity in identities { - assert!(plan.lookup_clickhouse(identity).is_ok()); - } - } - assert_eq!(multiple_traces.len(), 2); - assert_eq!(multiple_auto.precompute_plan.executable_dags.len(), 2); - multiple_auto.validate().unwrap(); - let bindings = multiple_auto - .query_plan - .entries - .values() - .map(|entry| entry.materialization_bindings()[0].materialization) - .collect::>(); - assert_eq!( - bindings.len(), - 2, - "different pane origins must retain distinct materializations" - ); - let mut invalid = multiple_auto.clone(); - invalid - .query_plan - .clickhouse_context - .as_mut() - .unwrap() - .window_templates - .values_mut() - .next() - .unwrap() - .push("missing concrete query".into()); - assert!(invalid.validate().is_err()); - let (automatic, traces) = - compile_automatic_clickhouse_workload(&ClickHouseSqlAutomaticWorkload { - envelope: request.precompute_plan.envelope.clone(), - tables: request.tables.clone(), - accuracy: request.accuracy.clone(), - queries: request - .queries - .iter() - .map(|query| ClickHouseSqlWorkloadEntry { - sql: query.sql.clone(), - start_ms: query.start_ms, - end_ms: query.end_ms, - cumulative: query.cumulative, - }) - .collect(), - }) - .await - .unwrap(); - assert_eq!(automatic.precompute_plan.materializations.len(), 1); - assert_eq!(traces.len(), 1); - assert!(traces - .values() - .any( - |trace| trace["groups"] - .as_array() - .unwrap() - .iter() - .any(|group| group["candidates"] - .as_array() - .unwrap() - .iter() - .any(|candidate| candidate["selected"] == true)) - )); - assert_eq!(automatic.precompute_plan.executable_dags.len(), 1); - automatic.validate().unwrap(); - let mut integer_source = ClickHouseSqlAutomaticWorkload { - envelope: request.precompute_plan.envelope.clone(), - tables: request.tables.clone(), - accuracy: request.accuracy.clone(), - queries: request - .queries - .iter() - .map(|query| ClickHouseSqlWorkloadEntry { - sql: query.sql.clone(), - start_ms: query.start_ms, - end_ms: query.end_ms, - cumulative: query.cumulative, - }) - .collect(), - }; - // An Int64 source is admitted and carries its declared type into the - // materialization, which is what lets the ingest reader widen the - // column explicitly and fail loudly on a value beyond the exact - // Float64 range instead of silently summarising a rounded one. - integer_source.tables.get_mut("telemetry").unwrap().columns[1].dtype = DataType::Int64; - let (integer_plan, _) = compile_automatic_clickhouse_workload(&integer_source) - .await - .expect("an Int64 source is admitted with its producer typing"); - assert_eq!( - integer_plan.precompute_plan.materializations[0] - .value_source_column - .as_ref() - .expect("column projections carry their producer typing") - .dtype, - DataType::Int64 - ); - integer_source.tables.get_mut("telemetry").unwrap().columns[1].dtype = DataType::Float64; - integer_source.queries[0].sql = integer_source.queries[0].sql.replace( - "FROM telemetry WHERE", - "FROM (SELECT timestamp_ms, value * 2 AS value FROM telemetry) doubled WHERE", - ); - assert!( - compile_automatic_clickhouse_workload(&integer_source) - .await - .is_err(), - "a producer projection must not be erased while binding its original table" - ); - let installed = publication - .precompute_plan - .executable_dags - .get(&request.queries[0].sql) - .unwrap(); - installed.validate().unwrap(); - assert_eq!(installed.binding.precompute_sinks.len(), 1); - assert_eq!( - installed.binding.nodes.len(), - installed.document.nodes.len() - ); - assert!(!installed.binding.nodes.values().any(|binding| matches!( - binding, - crate::physical::executable_binding::BackendNodeBinding::Query { .. } - ))); - assert!( - publication.query_plan.selected_dags[&request.queries[0].sql] - .nodes - .len() - > installed.document.nodes.len() - ); - let entry = publication.query_plan.entries.values().next().unwrap(); - assert!(!entry - .nodes - .values() - .any(|node| matches!(node, crate::query_plan::QueryPlanNode::ExternalExact { .. }))); - assert!(!entry.materialization_bindings().is_empty()); - let original = request.queries[0].sql.clone(); - let schema = request.tables.get_mut("telemetry").unwrap(); - schema.columns[schema.time_index.unwrap()].name = "other_timestamp".into(); - request.queries[0].sql = original.replace("timestamp_ms", "other_timestamp"); - assert!( - compile_clickhouse_workload(&request).await.is_err(), - "a summary cannot bind a different timestamp projection" - ); - let schema = request.tables.get_mut("telemetry").unwrap(); - schema.columns[schema.time_index.unwrap()].name = "timestamp_ms".into(); - request.queries[0].sql = original.replace("timestamp_ms < 2000", "timestamp_ms <= 1999"); - assert!(compile_clickhouse_workload(&request).await.is_ok()); - request.queries[0].sql = original.replace("timestamp_ms < 2000", "timestamp_ms <= 2000"); - assert!(compile_clickhouse_workload(&request).await.is_err()); - request.queries[0].sql = original - .replace("timestamp_ms >= 0", "timestamp_ms >= 1000") - .replace("timestamp_ms < 2000", "timestamp_ms < 3000"); - assert!(compile_clickhouse_workload(&request).await.is_err()); - - request - .tables - .get_mut("telemetry") - .unwrap() - .columns - .push(Column::new("metric", DataType::Utf8, false)); - request.queries[0].sql = original.replace( - "WHERE timestamp_ms", - "WHERE metric = 'requests' AND timestamp_ms", - ); - assert!( - compile_clickhouse_workload(&request).await.is_err(), - "an unfiltered summary cannot satisfy a filtered query" - ); - let mut config = request.precompute_plan.materializations[0].clone(); - config.table_population = Some(asap_types::table_population::TablePopulation { - predicates: vec![asap_types::table_population::TableColumnPredicate { - column: "metric".into(), - operator: planner_types::pre_asap::CompareOpKind::Eq, - value: planner_types::pre_asap::ScalarValue::Utf8("requests".into()), - }], - }); - request.summary_catalog = - SummaryCatalog::from_materializations(71, 1, &[config.clone()]).unwrap(); - let envelope = request.precompute_plan.envelope.clone(); - request.precompute_plan = - PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); - request - .precompute_plan - .bind_catalog(&request.summary_catalog) - .unwrap(); - request.transmission_plan = crate::physical::compiler::build_transmission_plan( - envelope, - &request.precompute_plan, - &std::collections::BTreeMap::new(), - ) - .unwrap(); - request.transmission_plan.summary_catalog = - Some(request.summary_catalog.reference().unwrap()); - assert!(compile_clickhouse_workload(&request).await.is_ok()); - } -} diff --git a/control_plane/src/emit/backend_wire.rs b/control_plane/src/emit/backend_wire.rs deleted file mode 100644 index ff294e8ce..000000000 --- a/control_plane/src/emit/backend_wire.rs +++ /dev/null @@ -1,344 +0,0 @@ -//! Backend wire construction for a compiled physical plan. -//! -//! Two documents, one classifier each: -//! -//! * the storage-routing table, which maps each metric's materialized summary -//! families to the query shapes the ASAP tier serves natively versus the -//! ones that belong to the archive; -//! * the aggregation and readout JSON the backend's `PrecomputeMaterialization` -//! parser consumes. -//! -//! `backend_plan::from_stage_config` reuses [`build_backend_aggregation_json`] -//! rather than re-deriving the field mapping, so `BackendPlan` materializations -//! and the JSON wire format share one `PolicyFingerprint` identity space. - -use planner_types::post_asap::{ExactKind, SketchAlgorithm, SketchParams, SummaryFamilyType}; -use serde_json::{json, Value as JsonValue}; - -use crate::physical::backend_stage::{AggregationInput, BackendAggregation}; - -// ── Window-size clamping (MVP blocker B4) ───────────────────────────────────── -// -// The controller derives `window_secs` from the workload's matrix-selector -// range (`metric[30s]` → 30s). Two bounds keep the emitted value sane: -// -// * Lower bound 5s — below this the sketch processor mints a new -// window before it has enough samples for the family's quality -// guarantees, and the per-flush cardinality on the sid catalog -// explodes (one (sid, window) row per few seconds). -// * Upper bound 60s — above this the user's query range no longer -// contains a closed sketch window, and replay queries return NoData -// while the warm tier still owns the metric. 60s is also the -// historical default the legacy single-pipeline emitter shipped with, -// so clamping here preserves backwards-compat for plans without an -// explicit range. -// -// `None` means "no `Window` node in the typed L5 — agent runs in batch -// mode, no window_duration in the YAML"; we pass that straight through. -// -// Centralised here so [`build_edge_processor_block`] (sketch processor -// `window_duration`) and the [`BackendAggregation`] consumer -// ([`emit_backend_streaming_config_json`]) clamp to the same bounds. The -// downstream backend's reducer keys windows by the emitted value, so -// the two MUST agree or replay-vs-warm answers go out of sync. - -/// Lower bound for [`clamp_window_secs`]. -pub const MIN_WINDOW_SECS: u64 = 5; - -/// Upper bound for [`clamp_window_secs`]. Matches the legacy default -/// the pre-B4 emitter shipped with. -pub const MAX_WINDOW_SECS: u64 = 60; - -/// Cardinality at which a per-series HLL is emitted DENSE rather than sparse -/// (ASAPCollector#472 follow-up to PR #358). -/// -/// The sketchlib-go in-memory sparse HLL base (`NewHLLWrapperSparse`) -/// auto-promotes to the dense register array once roughly this many registers -/// become non-zero (the sparse representation stops saving memory past that -/// point). A per-series HLL whose known distinct-key count -/// ([`crate::workload::WorkloadEntry::distinct_keys_per_window`]) is at or -/// above this crossover would promote almost immediately, so starting it sparse -/// only pays one-time promotion churn — we emit it dense instead. -/// -/// This is a HEURISTIC: distinct *keys* map to non-zero *registers* only -/// approximately (hash collisions mean registers < keys at high cardinality), -/// so the crossover is fuzzy. Being slightly off has NO correctness or accuracy -/// impact — the sparse base is lossless and serializes byte-identically to -/// dense for the same inputs; an over- or under-estimate at worst costs (or -/// saves) a single in-memory sparse→dense promotion. The value tracks the -/// in-memory promotion threshold (~4096 non-zero registers); the wire-crossover -/// constant the agent uses elsewhere is larger (~6000). -pub const DENSE_CROSSOVER: u64 = 4096; - -/// Clamp a derived window size to `[MIN_WINDOW_SECS, MAX_WINDOW_SECS]`. -/// `None` is preserved as `None` so callers can keep the -/// "no-window / batch-mode" branch distinguishable from a clamped value. -pub fn clamp_window_secs(w: Option) -> Option { - w.map(|s| s.clamp(MIN_WINDOW_SECS, MAX_WINDOW_SECS)) -} - -pub const DEFAULT_TENANT: &str = "default"; - -pub fn storage_routing_document( - tenant: &str, - metric_algorithms: &[(String, Vec)], -) -> JsonValue { - let metrics_json: Vec = metric_algorithms - .iter() - .map(|(metric_name, algorithms)| build_routing_entry(metric_name, algorithms)) - .collect(); - json!({ - "tenant": tenant, - "default_engine": "asap_query", - "metrics": metrics_json, - }) -} -// ── Internals ───────────────────────────────────────────────────────────────── - -/// Build the JSON `metrics:` entry for one metric — picks per-shape targets -/// from the summary families the plan landed at the backend. -/// -/// Returns a JSON object of shape: -/// ```text -/// { "name": , "targets": [, ...] } -/// ``` -/// where each `` is either `{ "engine": , "applies_to_query_shape": [...] }` -/// or `{ "engine": }` for the default slot. -fn build_routing_entry(metric_name: &str, algorithms: &[SketchAlgorithm]) -> JsonValue { - // Sketch-eligible shapes — the ASAP tier serves these natively - // because we planned a sketch for them. - let mut warm_shapes: Vec<&'static str> = Vec::new(); - let has_quantile_sketch = algorithms - .iter() - .any(|k| matches!(k, SketchAlgorithm::DDSketch | SketchAlgorithm::Kll)); - if has_quantile_sketch { - warm_shapes.push("quantile"); - warm_shapes.push("quantile_over_time"); - } - let has_hll = algorithms.iter().any(|k| matches!(k, SketchAlgorithm::Hll)); - if has_hll { - warm_shapes.push("count"); - } - // Heap-bearing frequency sketches also contribute their routing capability. - let has_count_sketch = algorithms.iter().any(|k| { - matches!( - k, - SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap - ) - }); - if has_count_sketch { - warm_shapes.push("topk"); - } - let has_cms = algorithms - .iter() - .any(|k| matches!(k, SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap)); - if has_cms { - // CMS's `Estimate` readout serves point-count / count queries. - // If HLL also planned, `count` is already in the list — push - // only when not already there (keep order stable). - if !warm_shapes.contains(&"count") { - warm_shapes.push("count"); - } - } - // Sketch-planned `rate / sum / avg / min / max` over the planned - // ranges — every sketch family the planner emits also tracks the - // range aggregation needed to answer these from the ASAP tier - // (the gateway merge processor produces a windowed accumulator). - if !algorithms.is_empty() { - warm_shapes.push("rate"); - warm_shapes.push("sum"); - warm_shapes.push("avg"); - warm_shapes.push("min"); - warm_shapes.push("max"); - } - - // One target: the ASAP tier, as the unfiltered default slot. #746 - // deleted the archive tier, so there is no second engine to claim the - // shapes the sketches cannot answer — the backend forwards those to its - // Prometheus fallback instead. We do NOT attach - // `applies_to_query_shape` to this slot: that would make it - // shape-specific and leave unanticipated shapes with no target at all. - let targets: Vec = vec![json!({ - "engine": "asap_query", - })]; - - // The warm-shape list is informational — surface it on a side - // field for operators / tests to spot-check what the controller - // decided the ASAP tier serves natively. The backend ignores - // unknown fields (`#[serde(default)]` on the parser side). - let mut entry = json!({ - "name": metric_name, - "targets": targets, - }); - if !warm_shapes.is_empty() { - entry["asap_tier_native_shapes"] = json!(warm_shapes); - } - entry -} - -pub(crate) fn build_backend_aggregation_json(agg: &BackendAggregation) -> JsonValue { - // Option B (post-PR-#287): when `agg_type_override` is set, use - // it as the wire `aggregationType` and emit an empty - // `parameters` object — bypasses the sketch_kind → backend type - // mapping for ExactAgg(Sum/Increase/Count) rows the Replanner - // synthesizes for non-sketch (Sum-shaped) workloads. The - // `sketch_kind` / `sketch_params` fields carry sentinel values - // in this case and are not emitted on the wire. - let (aggregation_type, mut parameters) = match &agg.family { - SummaryFamilyType::ExactAggregate(kind, _) => ( - match kind { - ExactKind::Sum => "Sum", - ExactKind::Count => "Count", - ExactKind::Min => "Min", - ExactKind::Max => "Max", - ExactKind::Increase => "Increase", - ExactKind::Rate => "Rate", - ExactKind::IRate => "IRate", - } - .to_string(), - json!({}), - ), - SummaryFamilyType::Sketch(kind, _) => ( - sketch_algorithm_to_backend_type(kind.algorithm()).to_string(), - sketch_params_to_json(kind.params()), - ), - other => panic!("backend emitter cannot encode summary family {other:?}"), - }; - // Carry the per-item dimension (e.g. "endpoint"/"service") into the - // policy parameters so the data-plane ingest can record it on the CMS - // sid and answer per-item estimate(key). Only set for item_label-mode - // frequency sketches; a subset content-match keeps policy resolution - // working for sketches that don't carry it. - if let Some(label) = &agg.item_label { - if let Some(obj) = parameters.as_object_mut() { - obj.insert("item_label".to_string(), JsonValue::String(label.clone())); - } - } - if let Some(mode) = agg.heap_update_mode { - if let Some(obj) = parameters.as_object_mut() { - obj.insert("weight_mode".into(), JsonValue::String(mode.into())); - if mode == "counter_delta" { - obj.insert("weight_scale".into(), json!(1_000_000)); - } - } - } - // PromQL range selectors are (start, end]. Encode the boundary convention - // in state identity so legacy half-open panes cannot satisfy this binding. - if matches!(agg.aggregation_input, AggregationInput::Raw) { - parameters["promql_right_closed"] = json!(true); - } - let aggregation_input = match agg.aggregation_input { - AggregationInput::SketchEnvelope => "sketch_envelope", - AggregationInput::Raw => "raw", - }; - // MVP blocker B4: clamp `windowSize` so the backend's reducer keys - // windows by the SAME size the agent's sketch processor uses. The - // backend's `streaming-config.window_size` must match the agent's - // `window_duration` exactly — drift here de-syncs the warm tier - // and replay queries return NoData (the backend's pre-compute - // engine looks for closed windows at the streaming-config size). - // `agg.window_secs` is u64 (not Option) here; passing through - // `clamp_window_secs(Some(_))` and unwrapping keeps the contract - // explicit. - let window_size = - clamp_window_secs(Some(agg.window_secs)).expect("clamp_window_secs preserves Some"); - json!({ - "aggregationType": aggregation_type, - "aggregationSubType": "", - "metric": agg.metric_name, - "labels": { - "grouping": agg.grouping, - "rollup": Vec::::new(), - "aggregated": agg.item_label.iter().cloned().collect::>(), - }, - "parameters": parameters, - "windowSize": window_size, - "windowType": "tumbling", - "spatialFilter": agg.spatial_filter, - "aggregationInput": aggregation_input, - }) -} - -/// Map a `SketchAlgorithm` to the backend's `AggregationType::Display` -/// string — the same mapping -/// [`crate::config::asapquery_backend::map_sketch_type_to_agg_type`] uses -/// (the strings must match `AggregationType::FromStr` in the backend's -/// `promql_utilities::query_logics::enums`). -/// -/// Heap-bearing is now identity, not a params flag (`SketchAlgorithm::CmsWithHeap` -/// / `CountSketchWithHeap`, set by `BindCountSketchOnTopK` — see -/// `physical::post_asap::rules::bind_cms_topk`), so this maps on `kind` alone; -/// `params` is unused but kept for call-site stability. This is what -/// lets the backend's `policy_capability` lookup return -/// `FrequencyTopk(*WithHeap)` for heap-bearing aggregations — required -/// for `topk(...)` queries to bind to the right sids. -fn sketch_algorithm_to_backend_type(kind: &SketchAlgorithm) -> &'static str { - match kind { - SketchAlgorithm::UnivMon => "UnivMon", - SketchAlgorithm::DDSketch => "DDSketch", - SketchAlgorithm::Kll => "DatasketchesKLL", - SketchAlgorithm::Hll => "HLL", - SketchAlgorithm::CountSketchWithHeap => "CountSketchWithHeap", - SketchAlgorithm::CountSketch => "CountSketch", - SketchAlgorithm::CmsWithHeap => "CountMinSketchWithHeap", - SketchAlgorithm::Cms => "CountMinSketch", - SketchAlgorithm::Kmv | SketchAlgorithm::Theta => unreachable!( - "sketch_algorithm_to_backend_type: unsupported SketchAlgorithm; \ - no Bind* rule in this repo produces one" - ), - } -} - -/// Serialize a `SketchParams` payload to a flat JSON object the backend -/// can read directly without round-tripping through the controller's -/// internally-tagged enum form. -fn sketch_params_to_json(p: &SketchParams) -> JsonValue { - match p { - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - } => json!({ - "heap_size": heap_size, "sketch_rows": sketch_rows, "sketch_cols": sketch_cols, "layers": layers, - }), - SketchParams::Kll { k } => json!({ "k": k }), - SketchParams::DDSketch { alpha } => json!({ "alpha": alpha }), - SketchParams::Hll { precision } => json!({ "precision": precision }), - SketchParams::Cms { width, depth } => json!({ "w": width, "d": depth }), - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - } => json!({ - "w": width, - "d": depth, - "with_heap": true, - "heap_size": heap_size, - }), - // CountSketch/CountSketchWithHeap: the old arm always emitted - // `with_heap` (from `CountSketchParams.with_heap: bool`); - // that boolean is now the kind identity itself. - SketchParams::CountSketch { width, depth } => { - json!({ "w": width, "d": depth, "with_heap": false }) - } - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - } => json!({ - "w": width, - "d": depth, - "with_heap": true, - "heap_size": heap_size, - }), - // Exact accumulators never reach here -- see - // `sketch_kind_to_backend_type`'s doc. - SketchParams::Kmv { .. } | SketchParams::Theta { .. } => { - unreachable!( - "sketch_params_to_json: non-sketch or unsupported SummaryParams; \ - no Bind* rule in this repo produces one" - ) - } - } -} diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs deleted file mode 100644 index a56536256..000000000 --- a/control_plane/src/emit/mod.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! Backend-facing emission for a compiled physical plan. -//! -//! * [`backend_wire`] builds the storage-routing table and the aggregation / -//! readout JSON the backend's `PrecomputeMaterialization` parser consumes. -//! * [`monitor`] carries the CDM monitor declarations. - -pub mod backend_wire; -pub mod monitor; - -use crate::physical::post_asap::deployment_expr::PostAsapPlan; -use crate::physical::post_asap::PhysicalExpr; -use planner_types::post_asap::{SketchAlgorithm, SummaryExpr, SummaryNode}; -use std::rc::Rc; - -/// Walk a `PhysicalExpr` tree and return the first `SketchAgg::sketch_type` -/// (or the `RawAtEdgeSketchAtBackend::family` Mode-2 equivalent). The -/// canonical shape produced by `bind_workload_typed` is -/// `SketchEstimate { child: SketchAgg { sketch_type, … } }`, so this is -/// effectively a one-level descent — but we walk recursively to stay -/// robust against future shape changes (e.g. Bind* rules wrapping -/// in `LetBinding` for fan-in shared sketches). -/// -/// Returns `None` only for trees that carry no sketch commitment -/// (`Logical`-only, unresolved `Ref`, raw Mode-3 archive). These map -/// onto the raw-passthrough default pipeline in the routing emitter, -/// which is correct. -pub fn extract_root_sketch_algorithm(expr: &PhysicalExpr) -> Option { - match expr { - PhysicalExpr::Committed(plan) => extract_from_plan(plan), - PhysicalExpr::RawAtEdgeSketchAtBackend { family, .. } => Some(family.clone()), - PhysicalExpr::RawAtEdgePrometheusArchive { .. } => None, - } -} - -fn extract_from_plan(plan: &PostAsapPlan) -> Option { - match plan { - PostAsapPlan::Summary(node) => extract_from_node(node), - PostAsapPlan::LetBinding { expr, child, .. } => { - extract_from_plan(expr).or_else(|| extract_from_plan(child)) - } - PostAsapPlan::Ref { .. } => None, - } -} - -fn extract_from_node(node: &Rc) -> Option { - match &node.expr { - // The `family` variant distinguishes exact accumulators from sketches. - SummaryExpr::SummaryAgg { - family: planner_types::post_asap::SummaryFamilyType::Sketch(kind, _), - .. - } => Some(kind.algorithm().clone()), - // An exact accumulator has no sketch family beneath it (its own - // child is always a plain `Logical` leaf) — same as the old - // `ExactAgg` case. - SummaryExpr::SummaryAgg { .. } => None, - SummaryExpr::SummaryEstimate { summary_input, .. } => extract_from_node(summary_input), - SummaryExpr::SummaryMerge { children, .. } => children.iter().find_map(extract_from_node), - SummaryExpr::ValueOperation { child, .. } => extract_from_node(child), - - // Not surfaced by any `Bind*` path yet (gated on rules that - // haven't landed — see `deployment_expr.rs`'s module docs). - SummaryExpr::BinaryOp { .. } - | SummaryExpr::RelationalJoin { .. } - | SummaryExpr::SummaryJoin { .. } - | SummaryExpr::SummarySubtract { .. } - | SummaryExpr::SummaryDelete { .. } - | SummaryExpr::KeepPreAsap(_) => None, - } -} diff --git a/control_plane/src/emit/monitor.rs b/control_plane/src/emit/monitor.rs deleted file mode 100644 index 59776f3a8..000000000 --- a/control_plane/src/emit/monitor.rs +++ /dev/null @@ -1,192 +0,0 @@ -//! Continuous-monitoring (CDM) auto-emission helpers. -//! -//! These render the two artifacts a monitored standing query needs, both keyed -//! by the SAME content-addressed `agg_id` the Go edge derives from the metric -//! name (so the edge's per-window sketch and the coordinator's monitor line up): -//! -//! 1. [`edge_threshold_block`] — the `threshold:` YAML mapping that goes into -//! the fused `asap_edge` processor's per-metric `metrics[]` entry (consumed -//! by `asapedgeprocessor.ThresholdConfig`). The edge derives the agg_id -//! from the metric name itself, so this block carries no agg_id. -//! 2. [`streaming_config_monitor_entry`] — the `monitors[]` JSON object for the -//! backend `InstalledPrecomputePlan` (consumed by `asap_types::MonitorSpec`), where -//! `agg_id` IS carried and MUST equal [`agg_id_for_metric`]. -//! -//! The functions are pure so they can be unit-tested and called from whichever -//! planner stage gains a monitor-intent slot. See -//! `ASAPCollector/docs/continuous-monitoring-tumbling-cost-analysis.md`. - -pub use asap_types::MonitorFunctional as Functional; -#[cfg(test)] -use serde_yaml::{Mapping, Value}; - -/// One monitored standing-query intent: "alert when the global Σ of `metric`'s -/// `functional` crosses `tau`". τ/ε/window are authoritative at the coordinator; -/// the edge copies are advisory. -#[derive(Debug, Clone)] -pub struct MonitorIntent { - pub metric: String, - pub functional: Functional, - /// CMS point-frequency key x (cms_point only); empty otherwise. - pub key: String, - /// Non-negative linear coefficients (linear_buckets only). - pub coeffs: Vec, - /// Coordinator MonitorService gRPC endpoint, e.g. "data-plane:4319". - pub coordinator_url: String, - pub tau: f64, - pub epsilon: f64, - pub window_ms: u64, -} - -/// Content-addressed aggregation id for a metric name. This MUST byte-match the -/// Go edge's `asapedgeprocessor.fnv64` (FNV-1a-64 with the exact constants used -/// there) so a controller-emitted `monitors[]` entry targets the same agg_id the -/// edge stamps on its per-window sketch. -pub fn agg_id_for_metric(metric: &str) -> u64 { - // NOTE: the edge uses the literal offset basis 1469598103934665603 and prime - // 1099511628211; replicate verbatim (do NOT "correct" to the textbook FNV - // basis — the two sides must agree, not match the spec). - let mut h: u64 = 1469598103934665603; - for b in metric.as_bytes() { - h ^= *b as u64; - h = h.wrapping_mul(1099511628211); - } - h -} - -#[cfg(test)] -/// Render the edge `threshold:` YAML mapping for the fused asap_edge processor's -/// per-metric entry. Omits `key`/`coeffs` when not applicable to the functional. -pub fn edge_threshold_block(intent: &MonitorIntent) -> Value { - let mut m = Mapping::new(); - m.insert("enabled".into(), Value::Bool(true)); - m.insert( - "functional".into(), - Value::String(intent.functional.as_str().to_string()), - ); - if intent.functional == Functional::CmsPoint && !intent.key.is_empty() { - m.insert("key".into(), Value::String(intent.key.clone())); - } - if intent.functional == Functional::LinearBuckets && !intent.coeffs.is_empty() { - m.insert( - "coeffs".into(), - Value::Sequence( - intent - .coeffs - .iter() - .map(|c| Value::Number((*c).into())) - .collect(), - ), - ); - } - m.insert( - "coordinator_url".into(), - Value::String(intent.coordinator_url.clone()), - ); - m.insert("tau".into(), Value::Number(intent.tau.into())); - m.insert("epsilon".into(), Value::Number(intent.epsilon.into())); - Value::Mapping(m) -} - -/// Render the backend `InstalledPrecomputePlan.monitors[]` JSON entry for this intent, -/// stamping the cross-language `agg_id`. -pub fn streaming_config_monitor_entry(intent: &MonitorIntent) -> serde_json::Value { - serde_json::json!({ - "agg_id": agg_id_for_metric(&intent.metric), - "functional": intent.functional.as_str(), - "key": intent.key, - "tau": intent.tau, - "epsilon": intent.epsilon, - "window_ms": intent.window_ms, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn agg_id_matches_go_fnv64() { - // Pinned against the Go edge's fnv64 (asapedgeprocessor/warm_sketch.go), - // computed directly from that implementation. - assert_eq!(agg_id_for_metric("bytes_sent"), 10036451356866642109); - assert_eq!( - agg_id_for_metric("http_requests_total"), - 16346598078036168951 - ); - assert_eq!(agg_id_for_metric("x"), 4953295298048672641); - } - - fn sum_intent() -> MonitorIntent { - MonitorIntent { - metric: "bytes_sent".into(), - functional: Functional::Sum, - key: String::new(), - coeffs: Vec::new(), - coordinator_url: "data-plane:4319".into(), - tau: 100.0, - epsilon: 0.05, - window_ms: 60_000, - } - } - - #[test] - fn edge_threshold_block_sum_shape() { - let v = edge_threshold_block(&sum_intent()); - let m = v.as_mapping().unwrap(); - assert_eq!(m.get(Value::from("enabled")).unwrap(), &Value::Bool(true)); - assert_eq!( - m.get(Value::from("functional")).unwrap().as_str().unwrap(), - "sum" - ); - assert_eq!( - m.get(Value::from("coordinator_url")) - .unwrap() - .as_str() - .unwrap(), - "data-plane:4319" - ); - // Sum carries no key/coeffs. - assert!(m.get(Value::from("key")).is_none()); - assert!(m.get(Value::from("coeffs")).is_none()); - } - - #[test] - fn edge_threshold_block_cms_point_carries_key() { - let mut intent = sum_intent(); - intent.functional = Functional::CmsPoint; - intent.key = "svc=checkout".into(); - let v = edge_threshold_block(&intent); - let m = v.as_mapping().unwrap(); - assert_eq!( - m.get(Value::from("functional")).unwrap().as_str().unwrap(), - "cms_point" - ); - assert_eq!( - m.get(Value::from("key")).unwrap().as_str().unwrap(), - "svc=checkout" - ); - } - - #[test] - fn streaming_entry_agg_id_matches_edge() { - let entry = streaming_config_monitor_entry(&sum_intent()); - assert_eq!( - entry["agg_id"].as_u64().unwrap(), - agg_id_for_metric("bytes_sent") - ); - assert_eq!(entry["tau"].as_f64().unwrap(), 100.0); - assert_eq!(entry["window_ms"].as_u64().unwrap(), 60_000); - } - - #[test] - fn streaming_entry_deserializes_as_monitor_spec() { - // The emitted JSON must round-trip into the backend's MonitorSpec. - let entry = streaming_config_monitor_entry(&sum_intent()); - let spec: asap_types::MonitorSpec = - serde_json::from_value(entry).expect("MonitorSpec deserialize"); - assert_eq!(spec.agg_id, agg_id_for_metric("bytes_sent")); - assert_eq!(spec.tau, 100.0); - assert_eq!(spec.window_ms, 60_000); - } -} diff --git a/control_plane/src/epsilon_alloc.rs b/control_plane/src/epsilon_alloc.rs deleted file mode 100644 index 7f322fabd..000000000 --- a/control_plane/src/epsilon_alloc.rs +++ /dev/null @@ -1,117 +0,0 @@ -//! Autonomous allocation, slice 3: derive the per-metric **knobs** — sampling -//! probability `p` and the CDM delta-transmission threshold `δ` — from the -//! target accuracy `ε`. -//! -//! Slice 2 ([`crate::query_planning`]) answered *which sketches* a query set -//! needs. This slice answers *how to run them*: it attaches `p` and `δ` to each -//! planned metric, completing `(ε, queries) → {sketches, p, δ}`. -//! -//! Two derivations, with an important asymmetry: -//! * **`p` is fully autonomous** from `(ε, rate)` via the unified ε-floor law -//! `p = 1/(1+ε²·rate)` — same law the data plane applies per-edge -//! ([`epsilon_sample_floor`]). `rate` is the per-window update count. -//! * **`δ` is only *sized* by ε.** The CDM threshold `τ` (the value of -//! interest the monitor alerts on) is a *user/query input*, not derivable -//! from ε. Given `τ` and the site count `k`, the even-split per-site slack -//! is `δ = ε·τ / k`. Metrics with no monitored threshold get no `δ`. -//! -//! [`epsilon_sample_floor`]: (mirrors `data_plane::monitor::sampling_alloc::epsilon_sample_floor`; -//! control_plane has no dependency on data_plane, so the canonical one-liner is -//! re-stated here — keep the two in sync.) - -/// Admission sampling probability under the unified ε-floor law, -/// `p = 1/(1 + ε²·rate)`. -/// -/// `rate` is the per-window update count for the metric (NOT a probability). -/// Degenerate inputs (`rate ≤ 0` or `ε ≤ 0`) mean "no sampling" → `1.0`. -/// Mirrors `data_plane::monitor::sampling_alloc::epsilon_sample_floor`. -pub fn derive_sample_p(epsilon: f64, rate: f64) -> f64 { - if rate <= 0.0 || epsilon <= 0.0 { - return 1.0; - } - 1.0 / (1.0 + epsilon * epsilon * rate) -} - -/// GOS budget split (design §7, Layer B) — **exact linear peel**. -/// -/// The staleness term `ε_st` is a *deterministic worst-case* bound, so it adds -/// **linearly** to the random error (Theorem 1): the composition is -/// `√(ε_sk² + ε_sa²) + ε_st ≤ ε_q`, NOT a three-way quadrature. This deriver -/// therefore peels `ε_st` off linearly first and splits only the *random* -/// residual between the sketch and sampling: -/// -/// 1. linear headroom above the sketch: `excess = ε_q − ε_sk` (0 if the sketch -/// already consumes the budget); -/// 2. give a fraction `t = w_comm/(w_edge+w_comm)` of it to staleness: -/// `ε_st = t·excess` (more comm-weight ⇒ looser thresholds ⇒ larger `ε_st`); -/// 3. the remaining *random* budget is `ε_rand = ε_q − ε_st`, split in -/// quadrature: `ε_sa = √(ε_rand² − ε_sk²)`. -/// -/// This satisfies `√(ε_sk² + ε_sa²) + ε_st = ε_q` **exactly**. Limits: -/// * `w_edge = 0` ⇒ `t=1` ⇒ `ε_st = ε_q−ε_sk`, `ε_rand=ε_sk`, `ε_sa=0` ⇒ -/// **no sampling** (`p=1`), cheap communication; -/// * `w_comm = 0` ⇒ `t=0` ⇒ `ε_st=0`, `ε_sa=√(ε_q²−ε_sk²)` ⇒ coarse sampling, -/// tight thresholds. -pub fn split_budget(eps_total: f64, eps_sketch: f64, w_edge: f64, w_comm: f64) -> (f64, f64) { - if eps_total <= eps_sketch { - return (0.0, 0.0); // the sketch already uses the whole ε budget - } - let excess = eps_total - eps_sketch; // linear headroom above the sketch - let t = if w_edge + w_comm <= 0.0 { - 0.0 - } else { - w_comm / (w_edge + w_comm) - }; - let eps_st = t * excess; - let eps_rand = eps_total - eps_st; // random budget after the linear peel - let eps_sa = (eps_rand * eps_rand - eps_sketch * eps_sketch) - .max(0.0) - .sqrt(); - (eps_sa, eps_st) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn sample_p_follows_epsilon_floor() { - // p = 1/(1+ε²·rate); ε=0.05, rate=1000 → 1/(1+2.5) = 0.2857… - let p = derive_sample_p(0.05, 1000.0); - assert!((p - 1.0 / 3.5).abs() < 1e-9, "got {p}"); - } - - #[test] - fn split_budget_linear_peel_and_limits() { - // Exact linear composition: √(ε_sk² + ε_sa²) + ε_st = ε_q. - let (eq, esk) = (0.1, 0.03); - let (sa, st) = split_budget(eq, esk, 1.0, 1.0); - assert!(((esk * esk + sa * sa).sqrt() + st - eq).abs() < 1e-12); - // No edge-CPU concern ⇒ no sampling (ε_sa=0), all headroom to staleness. - let (sa, st) = split_budget(eq, esk, 0.0, 1.0); - assert!(sa.abs() < 1e-12); - assert!((st - (eq - esk)).abs() < 1e-12); - // No comm concern ⇒ no staleness, ε_sa = √(ε_q²−ε_sk²). - let (sa, st) = split_budget(eq, esk, 1.0, 0.0); - assert_eq!(st, 0.0); - assert!((sa - (eq * eq - esk * esk).sqrt()).abs() < 1e-12); - // Higher edge weight ⇒ more of the budget to sampling (larger ε_sa). - let (sa_hi, _) = split_budget(eq, esk, 4.0, 1.0); - let (sa_lo, _) = split_budget(eq, esk, 1.0, 1.0); - assert!(sa_hi > sa_lo); - // Sketch consumes the whole budget ⇒ nothing left. - assert_eq!(split_budget(0.03, 0.05, 1.0, 1.0), (0.0, 0.0)); - } - - #[test] - fn sample_p_degenerate_inputs_disable_sampling() { - assert_eq!(derive_sample_p(0.0, 1000.0), 1.0); // ε=0 - assert_eq!(derive_sample_p(0.05, 0.0), 1.0); // rate=0 - } - - #[test] - fn sample_p_monotonic_decreasing_in_rate_and_epsilon() { - assert!(derive_sample_p(0.05, 100.0) > derive_sample_p(0.05, 10_000.0)); - assert!(derive_sample_p(0.01, 1000.0) > derive_sample_p(0.10, 1000.0)); - } -} diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index 99f2fa936..0483c8165 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -1,79 +1,134 @@ -//! Backend control-plane library and standalone service. +//! Backend control plane: asks ASAPPlanner for the workload's plan and +//! writes it for the data plane to install. //! -//! ASAPPlanner owns canonical semantic IR and legal logical selection. This -//! crate adapts workload/evidence inputs, compiles physical candidates, checks -//! provider quotes, and publishes one consistent catalog-backed generation. -//! Shared execution and installation contracts live in `asap_types`. -//! -//! `physical::compiler`, `physical::workload_cost`, and `clickhouse` are the -//! current compilation paths. Metric stage emission consumes the same canonical -//! workload model through a registration adapter. Public modules are integration APIs, -//! not an independent wire schema or a second semantic planner. +//! The Planner runs every planning stage, including plan selection. This +//! crate only supplies the deployment's capabilities and records which query +//! each root of the selected `PhysicalASAPDAG` answers. + +use std::path::Path; -#![allow( - clippy::collapsible_match, - clippy::doc_lazy_continuation, - clippy::doc_overindented_list_items, - clippy::field_reassign_with_default, - clippy::large_enum_variant, - clippy::map_identity, - clippy::result_large_err, - clippy::too_many_arguments, - clippy::type_complexity, - clippy::vec_init_then_push -)] +use asap_plan_selection::{plan_stages, PlanningModels}; +use planner_types::deployment::DeploymentCapabilities; +use planner_types::ir::physical_export::PhysicalASAPDAGDocument; +use planner_types::ir::schema_support::with_promql_series_identity; +use planner_types::ir::QueryRoot; +use planner_types::workload::{PlanningWorkload, QueryLanguage, RootDemand}; +use serde::{Deserialize, Serialize}; -pub mod accuracy; -pub mod backend_client; -pub mod clickhouse; -pub mod emit; -pub mod epsilon_alloc; -pub mod metrics_exposer; -pub mod opamp; -pub mod physical; -pub mod planner_selection; -pub mod query_parser; -pub mod query_plan; -pub mod runtime_samples; -pub mod types; -pub mod workload; +/// The plan the data plane installs: the Planner's selected physical DAG and +/// the query each of its roots answers. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct InstalledPlan { + /// PromQL text of each root of `dag`, in root order. + pub queries: Vec, + pub dag: PhysicalASAPDAGDocument, +} -/// Crate-wide test-only utilities for serialising access to process-global -/// state (environment variables). -/// -/// `cargo test` runs `#[test]`/`#[tokio::test]` functions on multiple -/// threads within a SINGLE process, and `std::env::set_var` / -/// `std::env::remove_var` mutate the one process-global environment table. -/// Those C `setenv`/`unsetenv` calls are not thread-safe — concurrent -/// invocations (even on different keys) are documented as unsound and can -/// corrupt the table — and any test that READS an env var via -/// `std::env::var` can transiently observe a value another test set -/// mid-emit. Both classes of failure are non-deterministic and surface as -/// flaky test results. -/// -/// Every unit test in this crate that reads OR writes a process-global env -/// var (`ASAP_EDGE_FUSED`, `ASAP_AGENT_MEMORY_LIMIT_MIB`, -/// `USE_TYPED_STAGE_SPLIT`, …) must serialise behind the SINGLE -/// [`env_lock()`] below — one shared lock so no two such tests ever run -/// concurrently, regardless of which module they live in. Tests that -/// mutate env should use [`EnvVarGuard`] so the prior value is restored on -/// drop (panic-safe), preventing state from leaking between tests. -#[cfg(test)] -pub(crate) mod test_support { - use std::rc::Rc; +#[derive(Debug, thiserror::Error)] +pub enum PlanError { + #[error("only PromQL workloads are supported, got {0:?}")] + UnsupportedLanguage(QueryLanguage), + #[error("query {0} lowers to a scalar, which the physical DAG cannot carry yet")] + ScalarRoot(usize), + #[error("PromQL lowering: {0}")] + Lowering(String), + #[error("planning: {0}")] + Planning(String), + #[error("invalid physical DAG: {0}")] + InvalidDag(String), + #[error("the DAG has {roots} roots for {queries} queries")] + RootCount { roots: usize, queries: usize }, + #[error("{path}: {message}")] + File { path: String, message: String }, +} - use planner_types::pre_asap::{CompareOpKind, Predicate, QueryExpr, ScalarValue, Schema}; +impl InstalledPlan { + pub fn validate(&self) -> Result<(), PlanError> { + self.dag + .validate() + .map_err(|e| PlanError::InvalidDag(e.to_string()))?; + let roots = self.dag.dag.roots.len(); + if roots != self.queries.len() { + return Err(PlanError::RootCount { + roots, + queries: self.queries.len(), + }); + } + Ok(()) + } + + /// Reads and validates a plan written by [`InstalledPlan::save`]. + pub fn load(path: &Path) -> Result { + let file_error = |message: String| PlanError::File { + path: path.display().to_string(), + message, + }; + let text = std::fs::read_to_string(path).map_err(|e| file_error(e.to_string()))?; + let plan: Self = serde_json::from_str(&text).map_err(|e| file_error(e.to_string()))?; + plan.validate()?; + Ok(plan) + } + + pub fn save(&self, path: &Path) -> Result<(), PlanError> { + let text = serde_json::to_string_pretty(self).expect("plans serialize"); + std::fs::write(path, text).map_err(|e| PlanError::File { + path: path.display().to_string(), + message: e.to_string(), + }) + } +} + +/// Plans `workload` with the Planner's #509 stage pipeline and returns the +/// selected plan. `capabilities` are what this deployment can execute; the +/// Planner rejects every candidate that needs more. +pub fn plan_workload( + workload: &PlanningWorkload, + capabilities: &DeploymentCapabilities, +) -> Result { + let language = &workload.query_workload.language; + if *language != QueryLanguage::PromQL { + return Err(PlanError::UnsupportedLanguage(language.clone())); + } + let roots = asap_frontend_promql::lower_promql_query_workload(workload, 0) + .map_err(|e| PlanError::Lowering(e.to_string()))? + .into_iter() + .enumerate() + .map(|(i, root)| match root { + // Rows carry each series' full identity: the representation + // per-series state needs at runtime. + QueryRoot::Operator(node) => with_promql_series_identity(&node) + .map(|node| (i, QueryRoot::Operator(node))) + .map_err(|e| PlanError::Lowering(e.to_string())), + QueryRoot::Scalar(_) => Err(PlanError::ScalarRoot(i)), + }) + .collect::, _>>()?; + let entries: Vec<_> = workload.query_workload.entries().collect(); + let demand: Vec = entries.iter().map(RootDemand::from).collect(); + let data = workload.data_workload.clone().unwrap_or_default(); + let models = PlanningModels::builtin().with_capabilities(capabilities); + let run = plan_stages(roots, &demand, &data, models, 0) + .map_err(|e| PlanError::Planning(e.to_string()))?; + let physical = run.plan.physical; + tracing::info!( + candidate = %physical.id, + label = %physical.label, + materialization = %physical.materialization, + "selected physical plan" + ); + let plan = InstalledPlan { + queries: entries.into_iter().map(|e| e.query.0).collect(), + dag: PhysicalASAPDAGDocument::new(physical.dag), + }; + plan.validate()?; + Ok(plan) +} - pub(crate) fn label_eq_predicate( - label: &str, - value: &str, - schema: &Schema, - ) -> Option { - let column = schema.column_id(label)?; - Some(Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(column)), - op: CompareOpKind::Eq, - right: Rc::new(QueryExpr::Literal(ScalarValue::Utf8(value.to_string()))), - }))) +/// What this backend's data plane can execute: the Planner executor it +/// runs, without state kept across query evaluations. +pub fn backend_capabilities() -> DeploymentCapabilities { + DeploymentCapabilities { + query_time_retention: false, + ..asap_executor::capabilities() } } diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index fae7acbb2..cb262967c 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -1,1130 +1,50 @@ -#![allow(clippy::doc_lazy_continuation, clippy::type_complexity)] - -use control_plane::backend_client; -use control_plane::clickhouse; -use control_plane::metrics_exposer; -use control_plane::opamp; -use control_plane::physical; -use control_plane::runtime_samples; -use control_plane::types; - -use axum::{ - extract::State, - http::StatusCode, - response::{IntoResponse, Response}, - routing::{get, post}, - Json, Router, -}; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use std::collections::HashMap; -use std::sync::Arc; -use std::time::Duration; -use tracing::info; - -use opamp::OpampServer; -use physical::deployment_cost::online as online_cost_model; -use physical::deployment_cost::online::{init_store as init_online_store, OnlineMetricsStore}; -use physical::deployment_cost::tco; - -// ── Shared state ────────────────────────────────────────────────────────────── - -#[derive(Clone)] -struct AppState { - opamp: Arc, - online_store: OnlineMetricsStore, - /// Bounded ring buffer for runtime-sample push batches from - /// agents' `sketch-runtime::PushExporter`. Read by decision - /// loops in the replanner. - runtime_samples: Arc, - /// The successfully activated typed catalog is authoritative for live ERP - /// input identity; incoming telemetry cannot supply its own descriptors. - active_summary_catalog: - Arc>>>, - /// Shared client for publishing physical plans from HTTP planning and replanning. - /// `None` when `CONTROLLER_BACKEND_ENDPOINT` is unset; pushes are then skipped. - backend_client: Option>, -} - -// ── Entry point ─────────────────────────────────────────────────────────────── - -#[tokio::main] -async fn main() { - tracing_subscriber::fmt::init(); - - let api_addr = std::env::var("CONTROLLER_ADDR").unwrap_or_else(|_| "0.0.0.0:8080".into()); - let opamp_addr = - std::env::var("CONTROLLER_OPAMP_ADDR").unwrap_or_else(|_| "0.0.0.0:4320".into()); - let backend_endpoint = std::env::var("CONTROLLER_BACKEND_ENDPOINT").ok(); - - // ── SP-5: Online EMA cost store ─────────────────────────────────────────── - let online_store = init_online_store(); - - // ── OpAMP server ────────────────────────────────────────────────────────── - // Collector plans are published on demand by the physical-plan path; the - // server itself carries no plan-push hooks. - let opamp_srv: Arc = Arc::new(OpampServer::new()); - - let backend_client_shared: Option> = - backend_endpoint.as_ref().map(|endpoint| { - info!( - endpoint = %endpoint, - "ASAPQuery-backend InstalledPrecomputePlan push enabled" - ); - Arc::new(backend_client::BackendClient::new(endpoint.clone())) - }); - if backend_client_shared.is_none() { - info!( - "ASAPQuery-backend physical-plan publication disabled \ - (set CONTROLLER_BACKEND_ENDPOINT= to enable)" - ); - } - - let runtime_samples_store = runtime_samples::RuntimeSamplesStore::new(1024); - let state = AppState { - opamp: Arc::clone(&opamp_srv), - online_store: Arc::clone(&online_store), - runtime_samples: Arc::clone(&runtime_samples_store), - active_summary_catalog: Arc::new(tokio::sync::Mutex::new(None)), - backend_client: backend_client_shared, - }; - - // ── OpAMP WebSocket listener ────────────────────────────────────────────── - let opamp_router = Router::new() - .route("/v1/opamp", get(OpampServer::ws_handler)) - .with_state(Arc::clone(&opamp_srv)); - - tokio::spawn(async move { - let listener = tokio::net::TcpListener::bind(&opamp_addr).await.unwrap(); - info!("OpAMP server listening on {opamp_addr}"); - axum::serve(listener, opamp_router).await.unwrap(); - }); - - // ── gRPC runtime-samples service (was HTTP+JSONL) ───────────────────────── - // Agents' `sketch-runtime::GrpcExporter` call - // `asap.runtime.v1.RuntimeSamples.Push` on this port. See - // commit message for the HTTP → gRPC pivot rationale. - let runtime_samples_state = Arc::clone(&state.runtime_samples); - let grpc_addr = std::env::var("CONTROLLER_GRPC_ADDR").unwrap_or_else(|_| "0.0.0.0:4321".into()); - let grpc_store = Arc::clone(&runtime_samples_state); - tokio::spawn(async move { - let addr: std::net::SocketAddr = grpc_addr.parse().expect("CONTROLLER_GRPC_ADDR"); - info!("runtime-samples gRPC server listening on {addr}"); - let svc = runtime_samples::RuntimeSamplesService::new(grpc_store).into_server(); - if let Err(e) = tonic::transport::Server::builder() - .add_service(svc) - .serve(addr) - .await - { - tracing::error!(error = %e, "runtime-samples gRPC server exited"); - } - }); - - // /metrics — same exposer as before, unchanged. Prom scrapes - // the control plane HTTP port; gRPC is the push side only. - let metrics_registry = metrics_exposer::MetricsRegistry::new(); - let metrics_state = metrics_exposer::MetricsState { - registry: Arc::clone(&metrics_registry), - store: Arc::clone(&runtime_samples_state), - stats: runtime_samples_state.stats_handle(), - }; - let metrics_router = Router::new() - .route( - "/metrics", - axum::routing::get(metrics_exposer::handle_metrics), - ) - .with_state(metrics_state); - - let app = Router::new() - .route( - "/api/v1/physical-plan/cost-manifests", - post(handle_workload_cost_manifests), - ) - .route( - "/api/v1/metricsql/physical-plan/cost-manifests", - post(handle_metricsql_workload_cost_manifests), - ) - .route( - "/api/v1/physical-plan/compile-and-publish", - post(handle_compile_and_publish_physical_plan), - ) - .route( - "/api/v1/metricsql/physical-plan/compile-and-publish", - post(handle_compile_and_publish_metricsql_physical_plan), - ) - .route( - "/api/v1/clickhouse-plan/compile-and-publish", - post(handle_compile_and_publish_clickhouse_plan), - ) - .route( - "/api/v1/clickhouse-plan/automatic/compile-and-publish", - post(handle_compile_and_publish_automatic_clickhouse_plan), - ) - .route("/api/v1/cost-model", get(handle_cost_model)) - .route("/api/v1/tco", post(handle_tco)) - .with_state(state) - .merge(metrics_router); - - let listener = tokio::net::TcpListener::bind(&api_addr).await.unwrap(); - info!("control plane API listening on {api_addr}"); - axum::serve(listener, app).await.unwrap(); -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct PhysicalPlanQueryRequest { - query_id: String, - query_string: String, - metric: String, - window_secs: u64, - #[serde(default)] - group_by: Vec, - accuracy: types::AccuracyTarget, - lifecycle: physical::compiler::SummaryLifecyclePlanningInputs, - window_cost_model: physical::compiler::WindowCostModel, - evaluation_phase_ms: u64, - #[serde(default)] - runtime_policy: physical::compiler::RuntimeRulePolicy, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct CompileAndPublishPhysicalPlanRequest { - /// Opt-in additive response for quote preparation; the default remains a manifest array. - #[serde(default)] - explain: bool, - #[serde(default = "default_physical_deployment_target")] - target: physical::compiler::PhysicalDeploymentTarget, - #[serde(default)] - workload_cost_evidence: Option, - queries: Vec, - data_workload: planner_types::workload::DataWorkload, - dataset_identity: planner_types::post_asap::LogicalDatasetIdentity, - #[serde(rename = "collector_ids")] - target_collector_ids: Vec, - capability_snapshot_id: String, - #[serde(default)] - evidence: HashMap, - #[serde(default)] - exact_composition_costs: - HashMap>, - #[serde(default)] - erp: Option, - #[serde(default)] - runtime_adaptation_evidence: Vec, - planner_revision: String, - max_evidence_age_ms: u64, - plan_version: u64, - activation_unix_ms: u64, - expiry_unix_ms: Option, - backend_compat: String, - #[serde(default = "default_physical_plan_timeout_ms")] - apply_timeout_ms: u64, -} - -fn default_physical_deployment_target() -> physical::compiler::PhysicalDeploymentTarget { - physical::compiler::PhysicalDeploymentTarget::DistributedCollectors -} - -fn default_physical_plan_timeout_ms() -> u64 { - 10_000 -} - -use physical::compiler::QueryFrontend; - -#[derive(Debug, Serialize)] -struct CompileAndPublishPhysicalPlanResponse { - cost_comparison: Option, - #[serde(rename = "logical_selection")] - planner_selection_trace: Vec, - plan_id: u64, - plan_version: u64, - status: &'static str, - generated_at_unix_ms: u64, - #[serde(rename = "collector_ids")] - target_collector_ids: Vec, - lifecycle_estimates: Vec, -} - -/// Compile one Planner IR decision into one catalog-backed physical plan and -/// install it atomically before publishing collector projections. -async fn handle_compile_and_publish_physical_plan( - State(st): State, - Json(request): Json, -) -> Response { - compile_and_publish_physical_plan(st, request, QueryFrontend::PromQl).await -} - -async fn handle_compile_and_publish_metricsql_physical_plan( - State(st): State, - Json(request): Json, -) -> Response { - compile_and_publish_physical_plan(st, request, QueryFrontend::MetricsQl).await -} - -async fn compile_and_publish_physical_plan( - st: AppState, - mut request: CompileAndPublishPhysicalPlanRequest, - frontend: QueryFrontend, -) -> Response { - // Serialize typed activations so an older response cannot overwrite the - // catalog recorded after a newer backend activation. - let mut active_catalog = st.active_summary_catalog.lock().await; - if let Some(erp) = &mut request.erp { - if let Err(error) = erp.hydrate_observed_shape(&st.runtime_samples) { - return (StatusCode::UNPROCESSABLE_ENTITY, error).into_response(); - } - let catalog = active_catalog.clone(); - erp.resolve_population_data_descriptor(catalog.as_deref()); - } - let (bundle, target_collector_ids, apply_timeout, adaptation_evidence, _) = - match compile_physical_plan_request(request, false, frontend) { - Ok((Some(bundle), ids, timeout, adaptation, manifests)) => { - (bundle, ids, timeout, adaptation, manifests) - } - Ok((None, ..)) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "publication requires a selected plan", - ) - .into_response() - } - Err(response) => return physical_compile_failure(response), - }; - - let Some(backend) = st.backend_client.as_ref() else { - return ( - StatusCode::SERVICE_UNAVAILABLE, - "CONTROLLER_BACKEND_ENDPOINT is required for physical-plan publication".to_string(), - ) - .into_response(); - }; - if let Err(error) = st - .opamp - .ensure_collector_plan_targets(&bundle.collector_plans, apply_timeout) - .await - { - return ( - StatusCode::BAD_GATEWAY, - format!("collector physical-plan preflight failed: {error}"), - ) - .into_response(); - } - let publication = match bundle.to_publication_artifact() { - Ok(publication) => publication, - Err(error) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("invalid catalog publication: {error}"), - ) - .into_response() - } - }; - if let Err(error) = backend - .post_catalog_plan_typed( - &publication, - Some(bundle.storage_routing.clone()), - &adaptation_evidence, - ) - .await - { - return ( - StatusCode::BAD_GATEWAY, - format!("backend rejected physical plan: {error}"), - ) - .into_response(); - } - if let Err(error) = st - .opamp - .publish_collector_plans(&bundle.collector_plans, apply_timeout) - .await - { - let cleanup = backend - .discard_staged_physical_plan(bundle.envelope.plan_id, bundle.envelope.plan_version) - .await; - return ( - StatusCode::BAD_GATEWAY, - format!("collector physical-plan publication failed: {error}; staged backend cleanup: {cleanup:?}"), - ) - .into_response(); - } - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64; - let activation_wait = bundle.envelope.activation_unix_ms.saturating_sub(now); - if activation_wait > apply_timeout.as_millis() as u64 { - return ( - StatusCode::GATEWAY_TIMEOUT, - "activation time exceeds apply_timeout_ms; backend remains staged".to_string(), - ) - .into_response(); - } - if activation_wait > 0 { - tokio::time::sleep(Duration::from_millis(activation_wait)).await; - } - if let Err(error) = backend - .activate_physical_plan(bundle.envelope.plan_id, bundle.envelope.plan_version) - .await - { - return ( - StatusCode::BAD_GATEWAY, - format!("backend physical-plan activation failed: {error}"), - ) - .into_response(); - } - - *active_catalog = Some(Arc::new(bundle.summary_catalog)); - - Json(CompileAndPublishPhysicalPlanResponse { - cost_comparison: bundle.cost_comparison, - planner_selection_trace: bundle.planner_selection_trace, - plan_id: bundle.envelope.plan_id, - plan_version: bundle.envelope.plan_version, - status: "active", - generated_at_unix_ms: bundle.envelope.generated_at_unix_ms, - target_collector_ids, - lifecycle_estimates: bundle.lifecycle_estimates, - }) - .into_response() -} - -async fn handle_compile_and_publish_clickhouse_plan( - State(state): State, - Json(request): Json, -) -> impl IntoResponse { - let publication = match clickhouse::compile_clickhouse_workload(&request).await { - Ok(publication) => publication, - Err(error) => return (StatusCode::BAD_REQUEST, error.to_string()).into_response(), - }; - publish_clickhouse_plan(&state, publication, None).await -} - -async fn handle_compile_and_publish_automatic_clickhouse_plan( - State(state): State, - Json(request): Json, -) -> impl IntoResponse { - let (publication, trace) = - match clickhouse::compile_automatic_clickhouse_workload(&request).await { - Ok(publication) => publication, - Err(error) => return (StatusCode::BAD_REQUEST, error.to_string()).into_response(), - }; - publish_clickhouse_plan(&state, publication, Some(serde_json::json!(trace))).await -} - -async fn publish_clickhouse_plan( - state: &AppState, - publication: physical::publication::PhysicalPlanPublication, - selection_trace: Option, -) -> axum::response::Response { - let mut active_catalog = state.active_summary_catalog.lock().await; - let plan_id = publication.summary_catalog.plan_id; - let plan_version = publication.summary_catalog.plan_version; - let Some(client) = state.backend_client.as_ref() else { - return ( - StatusCode::SERVICE_UNAVAILABLE, - "backend publication is not configured", - ) - .into_response(); - }; - if let Err(error) = client - .post_catalog_plan_typed(&publication, None, &[]) - .await - { - return (StatusCode::BAD_GATEWAY, error.to_string()).into_response(); - } - if let Err(error) = client.activate_physical_plan(plan_id, plan_version).await { - let _ = client - .discard_staged_physical_plan(plan_id, plan_version) - .await; - return (StatusCode::BAD_GATEWAY, error.to_string()).into_response(); - } - *active_catalog = Some(Arc::new(publication.summary_catalog)); - Json(serde_json::json!({ - "plan_id": plan_id, - "plan_version": plan_version, - "status": "active", - "selection_trace": selection_trace, - })) - .into_response() -} - -// Keep Planner's Rc-backed rewrite DAG outside the async handler's future. -// Only the Send-safe compiled bundle crosses an await point. -fn compile_physical_plan_request( - request: CompileAndPublishPhysicalPlanRequest, - manifests_only: bool, - frontend: QueryFrontend, -) -> Result< - ( - Option, - Vec, - Duration, - Vec, - ( - Vec, - Vec, - Vec, - ), - ), - (StatusCode, serde_json::Value), -> { - if request.queries.is_empty() - || (request.target == physical::compiler::PhysicalDeploymentTarget::DistributedCollectors - && request.target_collector_ids.is_empty()) - || (request.target == physical::compiler::PhysicalDeploymentTarget::BackendLocalRemoteWrite - && !request.target_collector_ids.is_empty()) - { - return Err(( - StatusCode::UNPROCESSABLE_ENTITY, - "queries must be non-empty; distributed deployment requires collectors and backend-local deployment requires none".to_string().into(), - )); - } - if request.max_evidence_age_ms == 0 || request.apply_timeout_ms == 0 { - return Err(( - StatusCode::UNPROCESSABLE_ENTITY, - "max_evidence_age_ms and apply_timeout_ms must be non-zero" - .to_string() - .into(), - )); - } - if request.plan_version == 0 - || request.activation_unix_ms == 0 - || request.backend_compat != control_plane::physical::compiler::BACKEND_COMPAT - || request - .expiry_unix_ms - .is_some_and(|expiry| expiry <= request.activation_unix_ms) - { - return Err(( - StatusCode::UNPROCESSABLE_ENTITY, - "plan_version, activation_unix_ms, backend_compat and expiry are invalid" - .to_string() - .into(), - )); - } - - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64; - for query in &request.queries { - if query.query_id.trim().is_empty() - || query.metric.trim().is_empty() - || query.window_secs == 0 - { - return Err(( - StatusCode::UNPROCESSABLE_ENTITY, - "query_id, metric, and window_secs must be non-empty/non-zero" - .to_string() - .into(), - )); - } - } - let workload_entries = request - .queries - .iter() - .map(|query| planner_types::workload::RepeatingEntry { - query: planner_types::workload::Query(query.query_string.clone()), - demand: planner_types::workload::RepeatedDemand::FixedIntervalAt { - interval: planner_types::workload::RepetitionInterval( - query.lifecycle.evaluation_interval_ms, - ), - evaluation_phase: planner_types::workload::TimestampMs(query.evaluation_phase_ms), - }, - requirements: planner_types::workload::QueryRequirements { - accuracy: planner_types::workload::AccuracyRequirement::Explicit( - query.accuracy.clone(), - ), - ..Default::default() - }, - predictability: planner_types::workload::Predictability::Predictable { known_at: None }, - time_selection: planner_types::workload::TimeSelection { - lookback: Some(planner_types::workload::DurationMs( - query.window_secs.saturating_mul(1_000), - )), - ..Default::default() - }, - }) - .collect(); - let query_workload = planner_types::workload::QueryWorkload { - language: planner_types::workload::QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(workload_entries), - }; - request - .data_workload - .validate() - .map_err(|error| (StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into()))?; - let lowered = match frontend { - QueryFrontend::PromQl => asap_frontend_promql::lower_promql_workload( - &planner_types::workload::PlanningWorkload { - query_workload: query_workload.clone(), - data_workload: Some(request.data_workload.clone()), - }, - now, - ) - .map_err(|error| (StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into()))?, - QueryFrontend::MetricsQl => request - .queries - .iter() - .map(|query| frontend.parse(&query.query_string, query.accuracy.clone())) - .collect::, _>>() - .map_err(|error| (StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into()))?, - }; - - let mut queries = Vec::with_capacity(request.queries.len()); - let mut canonical_roots = Vec::with_capacity(request.queries.len()); - let mut window_models = Vec::new(); - for (query, expr) in request.queries.into_iter().zip(lowered) { - let post_asap = match control_plane::planner_selection::keep_pre_asap(&expr) { - Ok(plan) => plan, - Err(error) => return Err((StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into())), - }; - canonical_roots.push(std::rc::Rc::new(expr)); - window_models.push(query.window_cost_model); - queries.push(physical::compiler::QueryCompilationInput { - query_id: query.query_id, - query_string: query.query_string, - selected_plan_root: post_asap, - legacy_query_source: planner_types::pre_asap::Source::TimeSeries { - metric: query.metric, - }, - query_lookback_seconds: query.window_secs, - group_by_labels: query.group_by, - accuracy_target: query.accuracy, - summary_lifecycle_inputs: query.lifecycle, - window_realization_candidates: Vec::new(), - materialization_runtime_policy: query.runtime_policy, - }); - } - - let planner_selection_trace = match physical::compiler::select_logical_roots_with_trace( - &mut queries, - canonical_roots.clone(), - &request.evidence, - &request.exact_composition_costs, - request.erp.as_ref(), - ) { - Ok(trace) => trace, - Err(error) => return Err((StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into())), - }; - - for (query, model) in queries.iter_mut().zip(window_models) { - physical::compiler::prepare_window_implementations(query, &model, request.target, 0) - .map_err(|error| (StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into()))?; - } - let compilation_request = physical::compiler::PhysicalCompilationRequest { - planner_selection_trace, - query_workload: Some(query_workload), - data_workload: Some(request.data_workload), - canonical_roots, - queries, - allow_mixed_summary_and_exact_execution: request.target - == physical::compiler::PhysicalDeploymentTarget::BackendLocalRemoteWrite, - enabled_materialization_keys: None, - topk_membership_evidence_by_query_id: request.evidence, - exact_composition_costs: request.exact_composition_costs, - erp: request.erp, - planner_revision: request.planner_revision, - scrape_interval_ms: None, - query_retention_margin_ms: 0, - retained_summary_memory_budget_bytes: None, - }; - let environment = physical::compiler::PhysicalDeploymentContext { - dataset_identity: request.dataset_identity, - target: request.target, - target_collector_ids: request.target_collector_ids.clone(), - capability_snapshot_id: request.capability_snapshot_id, - observed_at_unix_ms: now, - max_evidence_age_ms: request.max_evidence_age_ms, - plan_version: request.plan_version, - activation_unix_ms: request.activation_unix_ms, - expiry_unix_ms: request.expiry_unix_ms, - backend_compat: request.backend_compat, - }; - let candidates = physical::workload_cost::enumerate_exact_and_materialized_candidates( - compilation_request.clone(), - ) - .map_err(|error| (StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into()))?; - let planner_selection_trace = compilation_request.planner_selection_trace.clone(); - let (manifests, candidate_evaluations) = - physical::workload_cost::compile_candidates_for_pricing( - candidates.clone(), - environment.clone(), - frontend, - ); - let apply_timeout = Duration::from_millis(request.apply_timeout_ms); - // Quote preparation enumerates feasible bindings; it does not select the - // default warm candidate, which may be unavailable while exact is valid. - if manifests_only { - if manifests.is_empty() { - return Err(( - StatusCode::UNPROCESSABLE_ENTITY, - serde_json::json!({"status": "all_infeasible", "candidates": candidate_evaluations, - "logical_selection": compilation_request.planner_selection_trace}), - )); - } - return Ok(( - None, - request.target_collector_ids, - apply_timeout, - request.runtime_adaptation_evidence, - (manifests, candidate_evaluations, planner_selection_trace), - )); - } - let compiled = match request.workload_cost_evidence { - Some(evidence) => match frontend { - QueryFrontend::PromQl => physical::workload_cost::select_lowest_cost_candidate( - candidates, - environment, - &evidence, - ), - QueryFrontend::MetricsQl => { - physical::workload_cost::select_lowest_cost_metricsql_candidate( - candidates, - environment, - &evidence, - ) - } - }, - None => frontend.compile(compilation_request, environment), - }; - let bundle = match compiled { - Ok(bundle) => bundle, - Err(physical::compiler::CompileError::Candidates(report)) => { - return Err((StatusCode::UNPROCESSABLE_ENTITY, report)) - } - Err(error) => return Err((StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into())), - }; - Ok(( - Some(bundle), - request.target_collector_ids, - apply_timeout, - request.runtime_adaptation_evidence, - (manifests, candidate_evaluations, planner_selection_trace), - )) -} - -/// Read-only preparation: no OpAMP, staging, activation or data-plane writes. -async fn handle_workload_cost_manifests( - Json(request): Json, -) -> impl IntoResponse { - workload_cost_manifests(request, QueryFrontend::PromQl) -} - -async fn handle_metricsql_workload_cost_manifests( - Json(request): Json, -) -> impl IntoResponse { - workload_cost_manifests(request, QueryFrontend::MetricsQl) -} - -fn physical_compile_failure((status, report): (StatusCode, serde_json::Value)) -> Response { - match report { - serde_json::Value::String(message) => (status, message).into_response(), - report => (status, Json(report)).into_response(), - } -} - -fn workload_cost_manifests( - request: CompileAndPublishPhysicalPlanRequest, - frontend: QueryFrontend, -) -> Response { - if request.workload_cost_evidence.is_some() { - return ( - StatusCode::UNPROCESSABLE_ENTITY, - "omit quotes when requesting manifests", - ) - .into_response(); - } - let explain = request.explain; - match compile_physical_plan_request(request, true, frontend) { - Ok((_, _, _, _, (manifests, candidates, planner_selection_trace))) => { - if explain { - Json(serde_json::json!({"manifests": manifests, "candidates": candidates, "logical_selection": planner_selection_trace})) - .into_response() - } else { - Json(manifests).into_response() - } - } - Err(error) => physical_compile_failure(error), - } -} - -// ── Handlers ────────────────────────────────────────────────────────────────── - -/// Returns the current EMA cost model state — blended benchmark + observed costs -/// per sketch type. Useful for diagnosing whether the online cost model has -/// received sufficient observations to meaningfully influence plan selection. -async fn handle_cost_model(State(st): State) -> impl IntoResponse { - let table = online_cost_model::effective_table(&st.online_store); - let raw = st.online_store.try_read(); - - let entries: Vec = table - .iter() - .map(|(sketch_type, costs)| { - let observations = raw - .as_ref() - .ok() - .and_then(|m| m.get(sketch_type)) - .map(|o| o.observations) - .unwrap_or(0); - json!({ - "sketch_type": sketch_type.to_string(), - "bw_bytes_per_series_per_sec": costs.bytes_per_series_per_sec, - "cpu_micros_per_sample": costs.cpu_micros_per_sample, - "base_memory_bytes": costs.base_memory_bytes, - "relative_error": costs.relative_error_at_default, - "observations": observations, - }) - }) - .collect(); - - (StatusCode::OK, Json(json!({ "sketches": entries }))).into_response() -} - -// ── TCO endpoint ───────────────────────────────────────────────────────────── - -#[derive(serde::Deserialize)] -struct TcoRequest { - workload: tco::TcoWorkload, - pricing: Option, -} - -async fn handle_tco(Json(req): Json) -> impl IntoResponse { - let pricing = req.pricing.unwrap_or_default(); - let estimate = tco::estimate_tco(&req.workload, &pricing); - (StatusCode::OK, Json(estimate)) -} - -// ── Test helpers ────────────────────────────────────────────────────────────── - -/// Builds a minimal `AppState` + `Router` for integration tests. -/// No background tasks are started; OpAMP/scraper hold no real connections. -#[cfg(test)] -fn test_app() -> (AppState, axum::Router) { - test_app_with_backend(None) -} - -/// Build a test server with an optional mock backend URL. `None` disables -/// backend pushes; `Some(url)` exercises typed backend JSON delivery. -#[cfg(test)] -fn test_app_with_backend(backend_url: Option) -> (AppState, axum::Router) { - let state = AppState { - opamp: Arc::new(OpampServer::new()), - online_store: init_online_store(), - runtime_samples: runtime_samples::RuntimeSamplesStore::new(64), - active_summary_catalog: Arc::new(tokio::sync::Mutex::new(None)), - backend_client: backend_url.map(|u| Arc::new(backend_client::BackendClient::new(u))), - }; - let router = axum::Router::new() - .route("/api/v1/cost-model", axum::routing::get(handle_cost_model)) - .route("/api/v1/tco", axum::routing::post(handle_tco)) - .with_state(state.clone()); - (state, router) -} - -#[cfg(test)] -mod api_tests { - use super::*; - use axum::body::Body; - use axum::http::{Request, StatusCode}; - use http_body_util::BodyExt; - use tower::ServiceExt; - - // A missing warm implementation must not hide the executable exact quote. - #[tokio::test] - async fn cost_manifests_survive_unavailable_warm_candidate() { - let snapshot: physical::compiler::BackendLocalPlanningInput = serde_json::from_str( - include_str!("../../docs/examples/asapquery-planning-snapshot.json"), - ) - .unwrap(); - let (planning, _) = snapshot - .clone() - .into_physical_compilation_request() - .unwrap(); - let query = &planning.queries[0]; - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64; - let mut request_body = serde_json::json!({ - "data_workload": snapshot.data_workload, - "queries": [{ - "query_id": query.query_id, "query_string": query.query_string, - "metric": "m", "window_secs": 60, "accuracy": query.accuracy_target, - "lifecycle": query.summary_lifecycle_inputs, "evaluation_phase_ms": 0, "window_cost_model": snapshot.physical_inputs.window_cost_model - }], - "dataset_identity": snapshot.environment.dataset_identity, - "collector_ids": ["test"], "capability_snapshot_id": "test", - "planner_revision": physical::compiler::PLANNER_REVISION, - "max_evidence_age_ms": 60000, "plan_version": 1, - "activation_unix_ms": now, "backend_compat": control_plane::physical::compiler::BACKEND_COMPAT - }); - for explain in [false, true] { - request_body["explain"] = serde_json::json!(explain); - let request = serde_json::from_value(request_body.clone()).unwrap(); - let response = handle_workload_cost_manifests(Json(request)) - .await - .into_response(); - assert_eq!(response.status(), StatusCode::OK); - let manifests = body_json(response).await; - if explain { - assert_eq!(manifests["manifests"].as_array().unwrap().len(), 1); - let candidates = manifests["candidates"].as_array().unwrap(); - assert_eq!(candidates.len(), 2); - assert_eq!(candidates[0]["status"], "bind_failed"); - assert!(candidates[0]["unavailable_reason"].is_string()); - assert_eq!(candidates[1]["status"], "bound"); - assert!(candidates[1]["physical_candidate_id"].is_string()); - assert!(!manifests["logical_selection"] - .as_array() - .unwrap() - .is_empty()); - } else { - assert_eq!(manifests.as_array().unwrap().len(), 1); - } - } - request_body["planner_revision"] = serde_json::json!("unavailable-compiler"); - let response = - handle_workload_cost_manifests(Json(serde_json::from_value(request_body).unwrap())) - .await - .into_response(); - assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); - assert_eq!(response.headers()["content-type"], "application/json"); - let report = body_json(response).await; - assert_eq!(report["status"], "all_infeasible"); - assert_eq!(report["candidates"].as_array().unwrap().len(), 2); - assert!(report["candidates"] - .as_array() - .unwrap() - .iter() - .all(|item| item["status"] == "bind_failed")); - } - - #[test] - fn backend_local_typed_request_compiles_without_collectors() { - let snapshot: physical::compiler::BackendLocalPlanningInput = serde_json::from_str( - include_str!("../../docs/examples/asapquery-compatibility-demo-snapshot.json"), - ) - .unwrap(); - let (planning, _) = snapshot - .clone() - .into_physical_compilation_request() - .unwrap(); - let mut query = planning.queries[0].clone(); - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64; - query.summary_lifecycle_inputs.evidence_observed_at_unix_ms = now; - for implementation in &mut query.window_realization_candidates { - implementation.cost.observed_at_unix_ms = now; - } - let planner_types::pre_asap::Source::TimeSeries { metric } = &query.legacy_query_source - else { - panic!("expected time series fixture"); - }; - let value = serde_json::json!({ - "data_workload": snapshot.data_workload, - "target": "backend_local_remote_write", - "queries": [{ - "query_id": query.query_id, "query_string": query.query_string, - "metric": metric, "window_secs": query.query_lookback_seconds, "accuracy": query.accuracy_target, - "lifecycle": query.summary_lifecycle_inputs, "evaluation_phase_ms": 0, "window_cost_model": { "implementation_id": "test", "cost": query.window_realization_candidates[0].cost } - }], - "dataset_identity": snapshot.environment.dataset_identity, - "collector_ids": [], "capability_snapshot_id": "test", - "planner_revision": physical::compiler::PLANNER_REVISION, - "max_evidence_age_ms": 60000, "plan_version": 1, - "activation_unix_ms": now, "backend_compat": physical::compiler::BACKEND_COMPAT - }); - let request = serde_json::from_value(value.clone()).unwrap(); - let (plan, collectors, _, _, _) = - compile_physical_plan_request(request, false, QueryFrontend::PromQl).unwrap(); - let plan = plan.unwrap(); - assert!(collectors.is_empty()); - assert!(plan.collector_plans.is_empty()); - assert_eq!( - plan.precompute_plan.ingest.protocol, - physical::compiler::IngestProtocol::PrometheusRemoteWriteV1 - ); - assert!(!plan.precompute_plan.materializations.is_empty()); - // HTTP lowering uses the current planning clock, not a timeless compatibility parse. - let mut stale = value.clone(); - stale["data_workload"]["data_ingestion_interval"]["observed_at_ms"] = serde_json::json!(0); - stale["data_workload"]["data_ingestion_interval"]["valid_for_ms"] = serde_json::json!(0); - let error = compile_physical_plan_request( - serde_json::from_value(stale).unwrap(), - false, - QueryFrontend::PromQl, - ) - .expect_err("expired cadence must fail HTTP planning"); - assert_eq!(error.0, StatusCode::UNPROCESSABLE_ENTITY); - assert!(error - .1 - .to_string() - .contains("evidence is unavailable at planning time")); - let mut distributed = value; - distributed["target"] = serde_json::json!("distributed_collectors"); - assert!(compile_physical_plan_request( - serde_json::from_value(distributed).unwrap(), - false, - QueryFrontend::PromQl - ) - .is_err()); - } - - async fn body_json(resp: axum::response::Response) -> serde_json::Value { - let bytes = resp.into_body().collect().await.unwrap().to_bytes(); - serde_json::from_slice(&bytes).unwrap() - } - - // ── AppState.backend_client wiring ─────────────────────────────── - - /// Default-constructed AppState (no `CONTROLLER_BACKEND_ENDPOINT`) - /// must leave `backend_client` as `None` so the typed L5 backend - /// JSON push silently no-ops, matching the Phase B fire-and-forget - /// contract. - #[test] - fn app_state_backend_client_none_by_default() { - let (state, _router) = test_app(); - assert!( - state.backend_client.is_none(), - "backend_client should default to None when no endpoint is configured" - ); - } - - /// When constructed with a backend URL (the production path takes - /// it from `CONTROLLER_BACKEND_ENDPOINT`), the field is populated - /// and ready for the Phase C `handle_plan` push. - #[test] - fn app_state_backend_client_some_when_constructed_with_url() { - let (state, _router) = - test_app_with_backend(Some("http://127.0.0.1:1/api/v1/physical-plan".into())); - let bc = state.backend_client.expect("backend_client must be Some"); - assert_eq!(bc.endpoint(), "http://127.0.0.1:1/api/v1/physical-plan"); - } - - // ── POST /api/v1/plan ───────────────────────────────────────────────────── - - // ── GET /api/v1/plan/:metric ────────────────────────────────────────────── - - #[tokio::test] - async fn get_plan_not_found_returns_404() { - let (_, app) = test_app(); - let req = Request::builder() - .uri("/api/v1/plan/nonexistent") - .body(Body::empty()) - .unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - } - - // ── GET /api/v1/cost-model ──────────────────────────────────────────────── - - #[tokio::test] - async fn cost_model_returns_all_sketch_types() { - let (_, app) = test_app(); - let req = Request::builder() - .uri("/api/v1/cost-model") - .body(Body::empty()) - .unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = body_json(resp).await; - let sketches = body["sketches"].as_array().unwrap(); - assert!(sketches.len() >= 4, "expected at least 4 sketch types"); - for s in sketches { - assert!(s["sketch_type"].as_str().is_some()); - assert!(s["observations"].as_u64().is_some()); - } - } - - // ── POST /api/v1/tco ───────────────────────────────────────────────────── - - #[tokio::test] - async fn tco_returns_valid_estimate() { - let (_, app) = test_app(); - let body = serde_json::json!({ - "workload": { - "series_count": 100000, - "samples_per_sec": 1.0, - "bytes_per_sample": 100, - "scrape_interval_secs": 15, - "queries_per_sec": 1.0, - "query_window_secs": 300, - "retention_days": 30, - "sketch_compression_ratio": 0.05, - "delta_compression_ratio": 0.3 - } - }); - let req = Request::builder() - .method("POST") - .uri("/api/v1/tco") - .header("content-type", "application/json") - .body(Body::from(body.to_string())) - .unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = body_json(resp).await; - assert!(body["before"]["total_dollars"].as_f64().unwrap() > 0.0); - assert!(body["after"]["total_dollars"].as_f64().unwrap() > 0.0); - assert!(body["savings_percent"].as_f64().unwrap() > 0.0); - } - - // ── Integration: control plane ↔ collector wiring ───────────────────────── - - #[tokio::test] - async fn tco_with_custom_pricing() { - let (_, app) = test_app(); - let body = serde_json::json!({ - "workload": { - "series_count": 50000, - "samples_per_sec": 1.0, - "bytes_per_sample": 100, - "scrape_interval_secs": 15, - "queries_per_sec": 1.0, - "query_window_secs": 300, - "retention_days": 30, - "sketch_compression_ratio": 0.05, - "delta_compression_ratio": 0.3 - }, - "pricing": { - "grafana_per_1k_series_1dpm": 8.0, - "s3_storage_per_gb_month": 0.023, - "s3_put_per_1k": 0.005, - "s3_get_per_1k": 0.0004, - "s3_transfer_per_gb": 0.09, - "ec2_sketch_instance_per_hour": 0.384 - } - }); - let req = Request::builder() - .method("POST") - .uri("/api/v1/tco") - .header("content-type", "application/json") - .body(Body::from(body.to_string())) - .unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = body_json(resp).await; - // With higher Grafana pricing, before cost should be higher. - assert!(body["before"]["ingestion_dollars"].as_f64().unwrap() > 0.0); - assert!(body["monthly_savings_dollars"].as_f64().unwrap() > 0.0); - } - - // Bootstrap and plan-push must use the same typed emission pipeline. - - // ── MVP §46: planner ↔ 5-sketch emitter stitch (PR #339 ↔ PR #340) ───────── - // - // The acceptance contract: register the six contract metrics in the - // workload registry, hit the bootstrap GET endpoint, and verify the - // emitted YAML carries the 5-sketch routing-connector wire shape — - // every sketched metric routed to its family-specific pipeline, - // raw `http_requests_total` falling through to - // `metrics/raw_passthrough`. - // - // Without the stitch wired in `emit_bootstrap_typed`, the - // EdgeStageConfig.metric_to_family HashMap stays empty and the - // emitter falls back to single-pipeline DDSketch — none of the - // assertions below pass. +use std::path::PathBuf; + +use anyhow::Context; +use clap::Parser; +use control_plane::{backend_capabilities, plan_workload}; +use planner_types::deployment::DeploymentCapabilities; +use planner_types::workload::PlanningWorkload; + +/// Plan a workload with ASAPPlanner and write the selected plan. +#[derive(Parser)] +struct Args { + /// Planner `PlanningWorkload` (JSON). + #[arg(long)] + workload: PathBuf, + /// Planner `DeploymentCapabilities` (JSON); defaults to what this + /// backend's data plane executes. + #[arg(long)] + capabilities: Option, + /// Where to write the plan the data plane installs. + #[arg(long)] + out: PathBuf, +} + +fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .init(); + let args = Args::parse(); + let text = std::fs::read_to_string(&args.workload) + .with_context(|| format!("reading {}", args.workload.display()))?; + let workload: PlanningWorkload = serde_json::from_str(&text) + .with_context(|| format!("parsing {}", args.workload.display()))?; + let capabilities: DeploymentCapabilities = match &args.capabilities { + Some(path) => serde_json::from_str( + &std::fs::read_to_string(path) + .with_context(|| format!("reading {}", path.display()))?, + ) + .with_context(|| format!("parsing {}", path.display()))?, + None => backend_capabilities(), + }; + let plan = plan_workload(&workload, &capabilities)?; + plan.save(&args.out)?; + println!( + "wrote {} ({} queries, {} nodes)", + args.out.display(), + plan.queries.len(), + plan.dag.dag.nodes.len() + ); + Ok(()) } diff --git a/control_plane/src/metrics_exposer.rs b/control_plane/src/metrics_exposer.rs deleted file mode 100644 index 33721a132..000000000 --- a/control_plane/src/metrics_exposer.rs +++ /dev/null @@ -1,360 +0,0 @@ -//! `GET /metrics` — the control plane as a Prometheus exporter. -//! -//! The control plane already holds per-agent runtime samples in -//! [`RuntimeSamplesStore`](crate::runtime_samples::RuntimeSamplesStore) -//! (fed by the agents' `sketch-runtime` push pipeline). This -//! module renders that store's state on demand in Prometheus -//! exposition format so a bog-standard Prom `scrape_configs` -//! entry pointed at `control_plane:8080/metrics` picks up every -//! paper §6 signal without any extra agent-side plumbing. -//! -//! ## Rendering contract -//! -//! For each `(source, sketch, impl)` key in the store we emit -//! the **latest** sample's headline gauges. That's what a -//! scrape is — point-in-time, not history. Prom does retention; -//! we do fan-out-to-labels. -//! -//! Control-plane-wide counters (batches received, decode errors, -//! evictions) come from `RuntimeSamplesStats` and are pure -//! `Counter`s. -//! -//! ## What lives in the registry -//! -//! All gauges are labelled `{source, sketch, impl}`: -//! -//! | Metric | Type | Source field (in `bench`) | -//! |---|---|---| -//! | `asap_runtime_throughput_items_per_sec` | Gauge | `throughput_items_per_sec.mean` | -//! | `asap_runtime_latency_p50_ns` | Gauge | `latency_ns.p50` | -//! | `asap_runtime_latency_p99_ns` | Gauge | `latency_ns.p99` | -//! | `asap_runtime_memory_bytes` | Gauge | `memory_bytes` | -//! | `asap_runtime_last_seen_unix_seconds` | Gauge | record `timestamp` (freshness signal) | -//! -//! Control-plane-wide (no labels): -//! -//! | Metric | Type | -//! |---|---| -//! | `asap_runtime_samples_batches_received_total` | Counter | -//! | `asap_runtime_samples_records_stored_total` | Counter | -//! | `asap_runtime_samples_records_evicted_total` | Counter | -//! | `asap_runtime_samples_decode_errors_total` | Counter | - -use std::sync::atomic::Ordering; -use std::sync::Arc; - -use axum::extract::State; -use axum::http::{header, StatusCode}; -use axum::response::{IntoResponse, Response}; -use prometheus::{Encoder, GaugeVec, IntCounter, Opts, Registry, TextEncoder}; - -use crate::runtime_samples::RuntimeSamplesStats; -use crate::runtime_samples::RuntimeSamplesStore; - -const LABELS: &[&str] = &["source", "sketch", "impl"]; - -/// The Prometheus registry + pre-built metric handles. Built -/// once at startup; the `/metrics` handler pulls the latest -/// sample per key out of the store on each scrape and updates -/// the gauge vecs before encoding. -pub struct MetricsRegistry { - registry: Registry, - throughput: GaugeVec, - latency_p50: GaugeVec, - latency_p99: GaugeVec, - memory_bytes: GaugeVec, - last_seen_unix: GaugeVec, - // control-plane-wide counters, refreshed from atomic stats - batches_received: IntCounter, - records_stored: IntCounter, - records_evicted: IntCounter, - decode_errors: IntCounter, -} - -impl MetricsRegistry { - pub fn new() -> Arc { - let registry = Registry::new(); - - let throughput = GaugeVec::new( - Opts::new( - "asap_runtime_throughput_items_per_sec", - "Latest runtime throughput reported by each (source, sketch, impl)", - ), - LABELS, - ) - .expect("valid gauge vec"); - let latency_p50 = GaugeVec::new( - Opts::new( - "asap_runtime_latency_p50_ns", - "Latest p50 latency ns reported by each (source, sketch, impl)", - ), - LABELS, - ) - .unwrap(); - let latency_p99 = GaugeVec::new( - Opts::new( - "asap_runtime_latency_p99_ns", - "Latest p99 latency ns reported by each (source, sketch, impl)", - ), - LABELS, - ) - .unwrap(); - let memory_bytes = GaugeVec::new( - Opts::new( - "asap_runtime_memory_bytes", - "Latest memory footprint of each (source, sketch, impl)", - ), - LABELS, - ) - .unwrap(); - let last_seen_unix = GaugeVec::new( - Opts::new( - "asap_runtime_last_seen_unix_seconds", - "Unix timestamp of the latest sample received from each (source, sketch, impl). \ - Compare against `time() - N` to alert on a stale agent.", - ), - LABELS, - ) - .unwrap(); - - let batches_received = IntCounter::new( - "asap_runtime_samples_batches_received_total", - "Count of push batches accepted by /api/v1/runtime-samples", - ) - .unwrap(); - let records_stored = IntCounter::new( - "asap_runtime_samples_records_stored_total", - "Count of individual records stored in the ring buffer", - ) - .unwrap(); - let records_evicted = IntCounter::new( - "asap_runtime_samples_records_evicted_total", - "Count of records evicted by the per-key ring-buffer FIFO", - ) - .unwrap(); - let decode_errors = IntCounter::new( - "asap_runtime_samples_decode_errors_total", - "Count of decode / parse failures on incoming batches", - ) - .unwrap(); - - registry.register(Box::new(throughput.clone())).unwrap(); - registry.register(Box::new(latency_p50.clone())).unwrap(); - registry.register(Box::new(latency_p99.clone())).unwrap(); - registry.register(Box::new(memory_bytes.clone())).unwrap(); - registry.register(Box::new(last_seen_unix.clone())).unwrap(); - registry - .register(Box::new(batches_received.clone())) - .unwrap(); - registry.register(Box::new(records_stored.clone())).unwrap(); - registry - .register(Box::new(records_evicted.clone())) - .unwrap(); - registry.register(Box::new(decode_errors.clone())).unwrap(); - - Arc::new(Self { - registry, - throughput, - latency_p50, - latency_p99, - memory_bytes, - last_seen_unix, - batches_received, - records_stored, - records_evicted, - decode_errors, - }) - } - - /// Walk the store and push the latest sample per key into - /// the gauge vecs. Called at scrape time, not per-record — - /// cost scales with (# keys), not (# records). - fn refresh_gauges(&self, store: &RuntimeSamplesStore) { - for key in store.keys() { - let Some(rec) = store.latest(&key) else { - continue; - }; - let labels = &[ - key.source.as_str(), - key.sketch.as_str(), - key.impl_name.as_str(), - ]; - - // Pull the interesting numeric fields from the - // opaque JSON payload. Missing fields silently stay - // at their last value — an agent that dropped a - // metric briefly won't zap the gauge to 0. - let bench = rec.payload.get("bench"); - if let Some(v) = bench - .and_then(|b| b.get("throughput_items_per_sec")) - .and_then(|t| t.get("mean")) - .and_then(|m| m.as_f64()) - { - self.throughput.with_label_values(labels).set(v); - } - if let Some(v) = bench - .and_then(|b| b.get("latency_ns")) - .and_then(|l| l.get("p50")) - .and_then(|m| m.as_f64()) - { - self.latency_p50.with_label_values(labels).set(v); - } - if let Some(v) = bench - .and_then(|b| b.get("latency_ns")) - .and_then(|l| l.get("p99")) - .and_then(|m| m.as_f64()) - { - self.latency_p99.with_label_values(labels).set(v); - } - if let Some(v) = bench - .and_then(|b| b.get("memory_bytes")) - .and_then(|m| m.as_f64()) - { - self.memory_bytes.with_label_values(labels).set(v); - } - if let Some(ts) = rec.payload.get("timestamp").and_then(|t| t.as_str()) { - if let Ok(parsed) = chrono::DateTime::parse_from_rfc3339(ts) { - self.last_seen_unix - .with_label_values(labels) - .set(parsed.timestamp() as f64); - } - } - } - } - - /// Sync control-plane-wide counters from the store's atomic - /// stats. `IntCounter` only exposes `.inc()` — so we pump - /// in the delta since the last scrape rather than `.set()`. - fn refresh_counters(&self, stats: &RuntimeSamplesStats) { - fn delta(target: &IntCounter, snapshot: u64) { - let have = target.get(); - if snapshot > have { - target.inc_by(snapshot - have); - } - } - delta( - &self.batches_received, - stats.batches_received.load(Ordering::Relaxed), - ); - delta( - &self.records_stored, - stats.records_stored.load(Ordering::Relaxed), - ); - delta( - &self.records_evicted, - stats.records_evicted.load(Ordering::Relaxed), - ); - delta( - &self.decode_errors, - stats.decode_errors.load(Ordering::Relaxed), - ); - } -} - -#[derive(Clone)] -pub struct MetricsState { - pub registry: Arc, - pub store: Arc, - pub stats: Arc, -} - -pub async fn handle_metrics(State(state): State) -> Response { - state.registry.refresh_gauges(&state.store); - state.registry.refresh_counters(&state.stats); - - let metric_families = state.registry.registry.gather(); - let encoder = TextEncoder::new(); - let mut buf = Vec::with_capacity(1024); - if let Err(e) = encoder.encode(&metric_families, &mut buf) { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("prom encode failed: {e}"), - ) - .into_response(); - } - ( - StatusCode::OK, - [(header::CONTENT_TYPE, encoder.format_type())], - buf, - ) - .into_response() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::runtime_samples::RuntimeRecord; - use serde_json::json; - - fn seeded_store() -> (Arc, Arc) { - let store = RuntimeSamplesStore::new(16); - let stats = store.stats(); - for (source, sketch, impl_name, tp, p99, mem) in [ - ("dc-a", "cms", "oxide", 4.2e7, 120u64, 40960u64), - ("dc-a", "hll", "lib", 7.4e7, 21, 16384), - ] { - // Build a record whose payload mirrors the v1 bench - // shape — exactly what the store would have stored - // after receiving a PushExporter batch. - let payload = json!({ - "schema_version": 1, - "mode": "runtime", - "timestamp": "2026-04-21T19:00:00Z", - "bench": { - "throughput_items_per_sec": { "mean": tp, "stddev": 0.0 }, - "latency_ns": { "p50": 10, "p99": p99 }, - "memory_bytes": mem, - } - }); - let rec = RuntimeRecord { - source: source.into(), - sketch: sketch.into(), - impl_name: impl_name.into(), - schema_version: 1, - payload, - }; - // Inject via the public append-by-record shim we - // use in tests: the store's `append` is crate-public - // from `runtime_samples::tests` only, so go through - // the handler-side entry point with a JSON body in - // a real integration; here we lean on the trait. - // Shortcut: use store's pub `stats` + manual push - // by serialising + using the handler path. Simpler - // for this unit test: add a test-only helper below. - insert_for_test(&store, rec); - } - (store, stats) - } - - // Test-only shim — uses the `append_for_test` hook the - // `runtime_samples` module exposes under `cfg(test)`. - fn insert_for_test(store: &RuntimeSamplesStore, rec: RuntimeRecord) { - store.append_for_test(rec); - } - - #[tokio::test] - async fn metrics_scrape_reflects_latest_per_key_sample() { - let (store, stats) = seeded_store(); - let registry = MetricsRegistry::new(); - let metric_state = MetricsState { - registry: Arc::clone(®istry), - store: Arc::clone(&store), - stats: Arc::clone(&stats), - }; - let resp = handle_metrics(State(metric_state)).await; - let status = resp.status(); - assert_eq!(status, StatusCode::OK); - let body = axum::body::to_bytes(resp.into_body(), 64 * 1024) - .await - .unwrap(); - let text = String::from_utf8(body.to_vec()).unwrap(); - // Sanity: exposition format + at least one of our - // gauges should be present per key. - assert!(text.contains("# TYPE asap_runtime_throughput_items_per_sec gauge")); - assert!(text.contains( - "asap_runtime_throughput_items_per_sec{impl=\"oxide\",sketch=\"cms\",source=\"dc-a\"}" - )); - assert!(text - .contains("asap_runtime_latency_p99_ns{impl=\"lib\",sketch=\"hll\",source=\"dc-a\"}")); - } -} diff --git a/control_plane/src/opamp/mod.rs b/control_plane/src/opamp/mod.rs deleted file mode 100644 index 33759262a..000000000 --- a/control_plane/src/opamp/mod.rs +++ /dev/null @@ -1,1160 +0,0 @@ -/// OpAMP server implemented over WebSocket via axum. -/// -/// Speaks the standard OpAMP protobuf protocol (ServerToAgent / AgentToServer) -/// so the `opampextension` in OTel Collectors can connect directly. -/// -/// Each OTel collector connects as an "agent" identified by the `X-Agent-ID` -/// request header. The role (`agent` vs `backend`) is determined by the -/// optional `X-Agent-Role` header (defaults to `agent`). -/// -/// The server pushes `RemoteConfig` as an OpAMP `ServerToAgent.remote_config` -/// message (protobuf binary frame) and receives `AgentToServer` status reports. -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; -use std::time::Duration; - -use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::IntoResponse; -use futures_util::sink::SinkExt; -use futures_util::stream::StreamExt; -use prost::Message as ProstMessage; -use serde::{Deserialize, Serialize}; -use tokio::sync::{mpsc, Notify, RwLock}; -use tracing::{info, warn}; - -/// Generated OpAMP protobuf types (from proto/opamp.proto). -pub mod opamp_proto { - include!(concat!(env!("OUT_DIR"), "/opamp.proto.rs")); -} - -// ── Wire types ──────────────────────────────────────────────────────────────── - -/// Config message pushed to an agent collector. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RemoteConfig { - pub config_hash: String, - pub yaml: String, -} - -/// Status report sent back from an agent after applying a config. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AgentStatus { - pub agent_id: String, - pub config_hash: String, - pub healthy: bool, - pub error: Option, -} - -pub const COLLECTOR_PLAN_CAPABILITY: &str = "io.projectasap.collector-plan.v1"; -pub const COLLECTOR_PLAN_MESSAGE: &str = "collector_plan"; -pub const PLAN_STATUS_MESSAGE: &str = "plan_status"; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub enum CollectorPlanStatusKind { - Staged, - Applied, - Failed, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct CollectorPlanStatus { - pub plan_id: u64, - pub plan_version: u64, - pub status: CollectorPlanStatusKind, - #[serde(default)] - pub error: Option, -} - -#[derive(Debug, thiserror::Error)] -pub enum CollectorPlanPublishError { - #[error("collector {collector_id} did not advertise {COLLECTOR_PLAN_CAPABILITY}")] - CapabilityTimeout { collector_id: String }, - #[error("collector {collector_id} disconnected before plan publication")] - Disconnected { collector_id: String }, - #[error("compiled bundle contains duplicate collector target {collector_id}")] - DuplicateTarget { collector_id: String }, - #[error("collector {collector_id} plan serialization failed: {source}")] - Serialize { - collector_id: String, - #[source] - source: serde_json::Error, - }, - #[error( - "collector {collector_id} did not report plan {plan_id}/{plan_version} before timeout" - )] - StatusTimeout { - collector_id: String, - plan_id: u64, - plan_version: u64, - }, - #[error("collector {collector_id} rejected plan {plan_id}/{plan_version}: {error}")] - Rejected { - collector_id: String, - plan_id: u64, - plan_version: u64, - error: String, - }, -} - -// ── Role ────────────────────────────────────────────────────────────────────── - -/// Role of a connected OTel collector. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum AgentRole { - /// Edge / agent collector that produces sketches. - Agent, - /// Mid-tier gateway collector that forwards / merges sketches - /// between edge agents and the backend. Phase C (MVP v6) wires - /// this role through OpAMP so the typed L5 stage_split path can - /// push the gateway YAML directly via `push_to_role`. - Gateway, - /// Backend / aggregation collector that merges sketches. - Backend, -} - -impl AgentRole { - /// Parse the `X-Agent-Role` header value into an `AgentRole`. - /// Recognises `backend`, `gateway`, and `agent` (case-insensitive); - /// any other value (including the empty string) defaults to - /// `Agent` so legacy / mis-configured collectors keep working. - pub fn from_header(value: &str) -> Self { - match value.trim().to_lowercase().as_str() { - "backend" => AgentRole::Backend, - "gateway" => AgentRole::Gateway, - _ => AgentRole::Agent, - } - } -} - -// ── Server ──────────────────────────────────────────────────────────────────── - -#[derive(Debug, Clone)] -enum OutboundMessage { - RemoteConfig(RemoteConfig), - CollectorPlan(Vec), -} - -struct AgentConnection { - tx: mpsc::Sender, - role: AgentRole, - custom_capabilities: HashSet, -} - -type AgentMap = HashMap; -type PlanStatusMap = HashMap<(String, u64, u64), CollectorPlanStatus>; - -pub type OnConnectFn = Arc; -pub type OnDisconnectFn = Arc; - -pub struct OpampServer { - agents: Arc>, - plan_statuses: Arc>, - state_changed: Arc, - on_connect: Option, - on_disconnect: Option, -} - -impl Default for OpampServer { - fn default() -> Self { - Self { - agents: Arc::new(RwLock::new(HashMap::new())), - plan_statuses: Arc::new(RwLock::new(HashMap::new())), - state_changed: Arc::new(Notify::new()), - on_connect: None, - on_disconnect: None, - } - } -} - -impl Clone for OpampServer { - fn clone(&self) -> Self { - Self { - agents: Arc::clone(&self.agents), - plan_statuses: Arc::clone(&self.plan_statuses), - state_changed: Arc::clone(&self.state_changed), - on_connect: self.on_connect.clone(), - on_disconnect: self.on_disconnect.clone(), - } - } -} - -impl OpampServer { - pub fn new() -> Self { - Self::default() - } - - /// Register a callback invoked when an agent connects. - pub fn with_on_connect( - mut self, - f: impl Fn(String, AgentRole) + Send + Sync + 'static, - ) -> Self { - self.on_connect = Some(Arc::new(f)); - self - } - - /// Register a callback invoked when an agent disconnects. - pub fn with_on_disconnect(mut self, f: impl Fn(String) + Send + Sync + 'static) -> Self { - self.on_disconnect = Some(Arc::new(f)); - self - } - - /// axum handler: `GET /v1/opamp` — upgrades to WebSocket. - /// Agents must include `X-Agent-ID: ` in the upgrade request. - /// Optional `X-Agent-Role: agent|backend` (default: `agent`). - pub async fn ws_handler( - ws: WebSocketUpgrade, - headers: HeaderMap, - State(srv): State>, - ) -> impl IntoResponse { - let agent_id = headers - .get("X-Agent-ID") - .and_then(|v| v.to_str().ok()) - .unwrap_or("") - .to_string(); - - if agent_id.is_empty() { - return (StatusCode::BAD_REQUEST, "X-Agent-ID header required").into_response(); - } - - let role = headers - .get("X-Agent-Role") - .and_then(|v| v.to_str().ok()) - .map(AgentRole::from_header) - .unwrap_or(AgentRole::Agent); - - ws.on_upgrade(move |socket| handle_socket(socket, agent_id, role, srv)) - .into_response() - } - - /// Pushes a config to a specific agent. Returns false if not connected. - pub async fn push(&self, agent_id: &str, cfg: RemoteConfig) -> bool { - let tx = self - .agents - .read() - .await - .get(agent_id) - .map(|connection| connection.tx.clone()); - match tx { - Some(tx) => tx.send(OutboundMessage::RemoteConfig(cfg)).await.is_ok(), - None => false, - } - } - - /// Broadcasts a config only to agents matching the given role. - pub async fn push_to_role(&self, role: AgentRole, cfg: RemoteConfig) { - let ids: Vec = self - .agents - .read() - .await - .iter() - .filter(|(_, connection)| connection.role == role) - .map(|(id, _)| id.clone()) - .collect(); - for id in ids { - self.push(&id, cfg.clone()).await; - } - } - - /// Returns the IDs of currently connected agents (all roles). - pub async fn connected_agents(&self) -> Vec { - self.agents.read().await.keys().cloned().collect() - } - - /// Returns a map of agent_id → role for all connected agents. - pub async fn connected_agents_with_roles(&self) -> HashMap { - self.agents - .read() - .await - .iter() - .map(|(id, connection)| (id.clone(), connection.role.clone())) - .collect() - } - - /// Publish all per-target physical plans and require an exact semantic - /// STAGED report from every Collector. Activation is synchronized later at - /// the envelope timestamp; config hashes are not accepted as evidence. - pub async fn publish_collector_plans( - &self, - plans: &[crate::physical::compiler::CollectorPlan], - timeout: Duration, - ) -> Result, CollectorPlanPublishError> { - self.ensure_collector_plan_targets(plans, timeout).await?; - - for plan in plans { - let body = serde_json::to_vec(plan).map_err(|source| { - CollectorPlanPublishError::Serialize { - collector_id: plan.collector_id.clone(), - source, - } - })?; - self.plan_statuses.write().await.remove(&( - plan.collector_id.clone(), - plan.envelope.plan_id, - plan.envelope.plan_version, - )); - let sent = self.send_collector_plan(&plan.collector_id, body).await; - if !sent { - return Err(CollectorPlanPublishError::Disconnected { - collector_id: plan.collector_id.clone(), - }); - } - } - - let mut reports = Vec::with_capacity(plans.len()); - for plan in plans { - let report = self - .wait_for_plan_status( - &plan.collector_id, - plan.envelope.plan_id, - plan.envelope.plan_version, - timeout, - ) - .await - .ok_or_else(|| CollectorPlanPublishError::StatusTimeout { - collector_id: plan.collector_id.clone(), - plan_id: plan.envelope.plan_id, - plan_version: plan.envelope.plan_version, - })?; - if report.status != CollectorPlanStatusKind::Staged { - return Err(CollectorPlanPublishError::Rejected { - collector_id: plan.collector_id.clone(), - plan_id: report.plan_id, - plan_version: report.plan_version, - error: report.error.clone().unwrap_or_else(|| "unspecified".into()), - }); - } - reports.push(report); - } - Ok(reports) - } - - /// Validate target uniqueness and wait for every target to advertise the - /// physical-plan capability. The orchestrator calls this before changing - /// backend state, then publication repeats it to close disconnect races. - pub async fn ensure_collector_plan_targets( - &self, - plans: &[crate::physical::compiler::CollectorPlan], - timeout: Duration, - ) -> Result<(), CollectorPlanPublishError> { - let mut targets = HashSet::with_capacity(plans.len()); - for plan in plans { - if !targets.insert(plan.collector_id.as_str()) { - return Err(CollectorPlanPublishError::DuplicateTarget { - collector_id: plan.collector_id.clone(), - }); - } - } - for plan in plans { - if !self - .wait_for_capability(&plan.collector_id, COLLECTOR_PLAN_CAPABILITY, timeout) - .await - { - return Err(CollectorPlanPublishError::CapabilityTimeout { - collector_id: plan.collector_id.clone(), - }); - } - } - - Ok(()) - } - - async fn send_collector_plan(&self, agent_id: &str, body: Vec) -> bool { - let agents = self.agents.read().await; - let Some(connection) = agents.get(agent_id) else { - return false; - }; - if !connection - .custom_capabilities - .contains(COLLECTOR_PLAN_CAPABILITY) - { - return false; - } - let tx = connection.tx.clone(); - drop(agents); - tx.send(OutboundMessage::CollectorPlan(body)).await.is_ok() - } - - async fn wait_for_capability( - &self, - agent_id: &str, - capability: &str, - timeout: Duration, - ) -> bool { - let deadline = tokio::time::Instant::now() + timeout; - loop { - let changed = self.state_changed.notified(); - if self - .agents - .read() - .await - .get(agent_id) - .is_some_and(|agent| agent.custom_capabilities.contains(capability)) - { - return true; - } - let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); - if remaining.is_zero() || tokio::time::timeout(remaining, changed).await.is_err() { - return false; - } - } - } - - async fn wait_for_plan_status( - &self, - agent_id: &str, - plan_id: u64, - plan_version: u64, - timeout: Duration, - ) -> Option { - let key = (agent_id.to_string(), plan_id, plan_version); - let deadline = tokio::time::Instant::now() + timeout; - loop { - let changed = self.state_changed.notified(); - if let Some(status) = self.plan_statuses.read().await.get(&key).cloned() { - return Some(status); - } - let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); - if remaining.is_zero() || tokio::time::timeout(remaining, changed).await.is_err() { - return None; - } - } - } -} - -async fn handle_socket( - socket: WebSocket, - agent_id: String, - role: AgentRole, - srv: Arc, -) { - let (tx, mut rx) = mpsc::channel::(16); - srv.agents.write().await.insert( - agent_id.clone(), - AgentConnection { - tx, - role: role.clone(), - custom_capabilities: HashSet::new(), - }, - ); - srv.state_changed.notify_waiters(); - info!(agent = %agent_id, ?role, "agent connected"); - - if let Some(cb) = &srv.on_connect { - cb(agent_id.clone(), role); - } - - let (mut ws_tx, mut ws_rx) = socket.split(); - - // Forward channel messages → WebSocket as standard OpAMP protobuf. - // - // OpAMP WS wire format prepends each binary frame with a varint - // header (`uint64(0)` today). See `opamp-go/internal/wsmessage.go`. - // Without the header, the agent's `DecodeWSMessage` falls back to - // "old format" and decodes successfully — which is why pushes - // worked even before this fix. We add the header for spec - // conformance so the agent never has to take the legacy path. - let writer_id = agent_id.clone(); - let write_task = tokio::spawn(async move { - while let Some(message) = rx.recv().await { - let (server_to_agent, description) = match message { - OutboundMessage::RemoteConfig(cfg) => ( - encode_remote_config(&cfg), - format!("remote config hash={}", cfg.config_hash), - ), - OutboundMessage::CollectorPlan(body) => ( - encode_collector_plan(body), - "typed CollectorPlan".to_string(), - ), - }; - let mut payload = Vec::new(); - if server_to_agent.encode(&mut payload).is_err() { - warn!(agent = %writer_id, "failed to encode OpAMP protobuf"); - continue; - } - // Prepend the wsMsgHeader varint (zero byte today; the - // varint is `0u64`, which encodes to a single 0x00). - let mut buf = Vec::with_capacity(1 + payload.len()); - buf.push(0u8); - buf.extend_from_slice(&payload); - if ws_tx.send(Message::Binary(buf)).await.is_err() { - break; - } - info!(agent = %writer_id, message = %description, "message pushed (OpAMP protobuf)"); - } - }); - - // Receive AgentToServer protobuf messages. - // - // Strip the OpAMP wire-format header before decoding. Per - // `opamp-go/internal/wsmessage.go::DecodeWSMessage`, the spec - // header is a varint-encoded `uint64(0)` and is detected by a - // leading 0 byte. Older clients send raw protobuf with no header - // — in that case the first byte is the protobuf field tag and is - // never zero (a tag-0 wire type is illegal), so the - // "first-byte-is-zero" check is unambiguous. - while let Some(Ok(msg)) = ws_rx.next().await { - match msg { - Message::Binary(data) => { - let payload: &[u8] = if !data.is_empty() && data[0] == 0 { - // Spec format. Decode the varint header (always - // 0 today) and skip it. - match prost::encoding::decode_varint(&mut &data[..]) { - Ok(_hdr) => { - // Recompute consumed bytes = varint length. - // For the canonical zero header this is 1 - // byte; for any future non-zero header - // it's `n` bytes from the unsigned LEB128 - // encoding. - let mut tmp: &[u8] = data.as_ref(); - let _ = prost::encoding::decode_varint(&mut tmp); - let consumed = data.len() - tmp.len(); - &data[consumed..] - } - Err(_) => &data[..], - } - } else { - &data[..] - }; - match opamp_proto::AgentToServer::decode(payload) { - Ok(ats) => { - info!(agent = %agent_id, "received AgentToServer (OpAMP protobuf)"); - if let Some(capabilities) = &ats.custom_capabilities { - if let Some(connection) = srv.agents.write().await.get_mut(&agent_id) { - connection.custom_capabilities = - capabilities.capabilities.iter().cloned().collect(); - } - srv.state_changed.notify_waiters(); - } - if let Some(message) = &ats.custom_message { - if message.capability == COLLECTOR_PLAN_CAPABILITY - && message.r#type == PLAN_STATUS_MESSAGE - { - match serde_json::from_slice::(&message.data) { - Ok(status) => { - srv.plan_statuses.write().await.insert( - (agent_id.clone(), status.plan_id, status.plan_version), - status, - ); - srv.state_changed.notify_waiters(); - } - Err(error) => warn!( - agent = %agent_id, - %error, - "invalid typed CollectorPlan status" - ), - } - } - } - // Log effective config if reported. Triggered by - // the `ReportFullState` flag on our outgoing - // ServerToAgent (see `encode_remote_config`). - if let Some(ec) = &ats.effective_config { - if let Some(cm) = &ec.config_map { - for (name, file) in &cm.config_map { - let body_preview = String::from_utf8_lossy( - &file.body[..file.body.len().min(160)], - ); - info!( - agent = %agent_id, - config_name = %name, - bytes = file.body.len(), - preview = %body_preview.replace('\n', " ⏎ "), - "agent reported effective config", - ); - } - } - } - // Log remote-config apply state — this is the - // signal that "the agent received our pushed - // RemoteConfig, attempted to apply it, and ended - // up in {Applied | Failed | Applying}". - // RemoteConfigStatuses enum values: - // 0 = Unset - // 1 = Applied - // 2 = Applying - // 3 = Failed - if let Some(rcs) = &ats.remote_config_status { - let status_str = match rcs.status { - 0 => "Unset", - 1 => "Applied", - 2 => "Applying", - 3 => "Failed", - n => { - // Future spec-defined values fall through - // here; surface the raw int rather than - // claim a meaning. - return_unknown_status(n) - } - }; - let last_hash_hex = rcs - .last_remote_config_hash - .iter() - .map(|b| format!("{:02x}", b)) - .collect::(); - if rcs.status == 3 { - warn!( - agent = %agent_id, - status = status_str, - last_hash = %last_hash_hex, - error = %rcs.error_message, - "agent reported remote-config status", - ); - } else { - info!( - agent = %agent_id, - status = status_str, - last_hash = %last_hash_hex, - "agent reported remote-config status", - ); - } - } - // Log health if reported. - if let Some(health) = &ats.health { - info!(agent = %agent_id, healthy = health.healthy, "agent health"); - } - } - Err(e) => { - warn!(agent = %agent_id, error = %e, "failed to decode AgentToServer") - } - } - } - // Also accept JSON for backward compatibility. - Message::Text(text) => match serde_json::from_str::(&text) { - Ok(s) => { - info!(agent = %s.agent_id, healthy = s.healthy, "agent status (legacy JSON)") - } - Err(_) => warn!(agent = %agent_id, "unexpected text message"), - }, - Message::Close(_) => break, - _ => {} - } - } - - write_task.abort(); - srv.agents.write().await.remove(&agent_id); - srv.state_changed.notify_waiters(); - info!(agent = %agent_id, "agent disconnected"); - - if let Some(cb) = &srv.on_disconnect { - cb(agent_id); - } -} - -/// Encode a `RemoteConfig` as a standard OpAMP `ServerToAgent` protobuf message. -/// -/// The YAML config body is wrapped in: -/// ServerToAgent.remote_config.config.config_map[""].body = yaml_bytes -/// -/// Format an unknown `RemoteConfigStatuses` int as a stable string -/// for logs. Pulled out into a helper to keep the match arm above -/// borrow-checker-friendly (returning a `&'static str`). -fn return_unknown_status(n: i32) -> &'static str { - // Leak the formatted int into a `'static str` only if needed. - // For diagnostic logs we accept the cost of a Box::leak per - // unrecognised value since this is an "out-of-spec status" - // signal that should be rare. Avoids reworking the surrounding - // match into String. - Box::leak(format!("Unknown({})", n).into_boxed_str()) -} - -/// This is the standard OpAMP way to push collector configuration. -/// The opampextension in the OTel Collector decodes this and applies the config. -/// -/// Two protocol bits the control plane sets per spec: -/// -/// 1. `flags = ReportFullState` (`0x01`) asks the agent's next -/// `AgentToServer` to include the full status block — -/// `effective_config` (the YAML the agent ended up running) -/// and `remote_config_status` (Applied / Failed / Applying). -/// Without this, the agent is allowed to elide both fields as -/// an optimization once the control plane has acknowledged a -/// given sequence_num, and we lose visibility into whether the -/// push actually took. -/// -/// 2. `capabilities` advertises what the control plane can accept -/// back. `AcceptsStatus` is mandatory; `OffersRemoteConfig` -/// must be set whenever we send `remote_config`; -/// `AcceptsEffectiveConfig` tells the agent it's worth -/// populating the field (some agents skip it if the server -/// didn't claim it could parse it). -fn encode_remote_config(cfg: &RemoteConfig) -> opamp_proto::ServerToAgent { - let config_file = opamp_proto::AgentConfigFile { - body: cfg.yaml.as_bytes().to_vec(), - content_type: "text/yaml".to_string(), - }; - - let mut config_map = HashMap::new(); - config_map.insert(String::new(), config_file); // empty key = single config file - - let agent_config_map = opamp_proto::AgentConfigMap { config_map }; - - let remote_config = opamp_proto::AgentRemoteConfig { - config: Some(agent_config_map), - config_hash: cfg.config_hash.as_bytes().to_vec(), - }; - - // ServerToAgentFlags_ReportFullState = 0x01. - const FLAG_REPORT_FULL_STATE: u64 = 0x0000_0001; - // ServerCapabilities bitmask: - // AcceptsStatus = 0x01 - // OffersRemoteConfig = 0x02 - // AcceptsEffectiveConfig = 0x04 - const CAPS_DEFAULT: u64 = 0x01 | 0x02 | 0x04; - - opamp_proto::ServerToAgent { - remote_config: Some(remote_config), - flags: FLAG_REPORT_FULL_STATE, - capabilities: CAPS_DEFAULT, - ..Default::default() - } -} - -fn encode_collector_plan(body: Vec) -> opamp_proto::ServerToAgent { - opamp_proto::ServerToAgent { - capabilities: 0x01, - custom_capabilities: Some(opamp_proto::CustomCapabilities { - capabilities: vec![COLLECTOR_PLAN_CAPABILITY.into()], - }), - custom_message: Some(opamp_proto::CustomMessage { - capability: COLLECTOR_PLAN_CAPABILITY.into(), - r#type: COLLECTOR_PLAN_MESSAGE.into(), - data: body, - }), - ..Default::default() - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use axum::{routing::get, Router}; - - #[tokio::test] - async fn no_agents_push_returns_false() { - let srv = Arc::new(OpampServer::new()); - let sent = srv - .push( - "unknown", - RemoteConfig { - config_hash: "h".into(), - yaml: "y".into(), - }, - ) - .await; - assert!(!sent); - } - - #[tokio::test] - async fn connected_agents_empty_initially() { - let srv = Arc::new(OpampServer::new()); - assert!(srv.connected_agents().await.is_empty()); - } - - #[test] - fn role_from_header() { - assert_eq!(AgentRole::from_header("backend"), AgentRole::Backend); - assert_eq!(AgentRole::from_header("agent"), AgentRole::Agent); - assert_eq!(AgentRole::from_header(""), AgentRole::Agent); - assert_eq!(AgentRole::from_header("BACKEND"), AgentRole::Backend); - } - - /// `Gateway` is a recognised role and round-trips through - /// the OpAMP `X-Agent-Role` header parser. This locks in the wire - /// vocabulary that the typed L5 stage_split path relies on when it - /// calls `push_to_role(AgentRole::Gateway, ...)` and expects to - /// reach gateway-role collectors only. - #[test] - fn role_from_header_recognises_gateway() { - assert_eq!(AgentRole::from_header("gateway"), AgentRole::Gateway); - assert_eq!(AgentRole::from_header("Gateway"), AgentRole::Gateway); - assert_eq!(AgentRole::from_header("GATEWAY"), AgentRole::Gateway); - // Round-trip through serde lowercase rename. - let s = serde_json::to_string(&AgentRole::Gateway).unwrap(); - assert_eq!(s, "\"gateway\""); - let back: AgentRole = serde_json::from_str(&s).unwrap(); - assert_eq!(back, AgentRole::Gateway); - // Still distinct from the other two roles. - assert_ne!(AgentRole::Gateway, AgentRole::Agent); - assert_ne!(AgentRole::Gateway, AgentRole::Backend); - } - - /// a gateway-role client connecting via WebSocket appears - /// in `connected_agents_with_roles` tagged as `Gateway`. Together - /// with the from_header test above this proves the role plumbs - /// through the connect path that `push_to_role` selects on. - #[tokio::test] - async fn gateway_role_round_trips_through_connection() { - let (srv, addr) = start_server().await; - let _gateway_ws = connect_ws_client(addr, "gw-1", "gateway").await; - - // Wait for server-side registration to complete. - for _ in 0..20 { - tokio::time::sleep(std::time::Duration::from_millis(25)).await; - let map = srv.connected_agents_with_roles().await; - if map.get("gw-1") == Some(&AgentRole::Gateway) { - return; - } - } - let map = srv.connected_agents_with_roles().await; - panic!("gateway role never registered; map = {:?}", map); - } - - /// Helper: start a real OpAMP server on a random port, return the server - /// Arc and the bound address. - async fn start_server() -> (Arc, std::net::SocketAddr) { - let srv = Arc::new(OpampServer::new()); - let app = Router::new() - .route("/v1/opamp", get(OpampServer::ws_handler)) - .with_state(Arc::clone(&srv)); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); - (srv, addr) - } - - /// Connect a WebSocket client with a given agent-id and role. - async fn connect_ws_client( - addr: std::net::SocketAddr, - agent_id: &str, - role: &str, - ) -> tokio_tungstenite::WebSocketStream> - { - use tokio_tungstenite::tungstenite::http::Request; - let req = Request::builder() - .uri(format!("ws://{addr}/v1/opamp")) - .header("Host", addr.to_string()) - .header("X-Agent-ID", agent_id) - .header("X-Agent-Role", role) - .header("Upgrade", "websocket") - .header("Connection", "Upgrade") - .header("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ==") - .header("Sec-WebSocket-Version", "13") - .body(()) - .unwrap(); - let (ws, _) = tokio_tungstenite::connect_async(req).await.unwrap(); - ws - } - - fn decode_server_to_agent_frame(data: &[u8]) -> opamp_proto::ServerToAgent { - let payload = if !data.is_empty() && data[0] == 0 { - &data[1..] - } else { - data - }; - opamp_proto::ServerToAgent::decode(payload).expect("valid protobuf") - } - - /// `push_to_role` delivers a RemoteConfig to a connected agent-role client. - #[tokio::test] - async fn push_to_role_delivers_yaml_to_agent_role() { - let (srv, addr) = start_server().await; - let mut agent_ws = connect_ws_client(addr, "agent-1", "agent").await; - - // Allow the server to register the connection. - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - let yaml_payload = "ddsketch:\n mode: window\n"; - srv.push_to_role( - AgentRole::Agent, - RemoteConfig { - config_hash: "hash-1".into(), - yaml: yaml_payload.to_string(), - }, - ) - .await; - - let msg = tokio::time::timeout( - std::time::Duration::from_secs(2), - futures_util::StreamExt::next(&mut agent_ws), - ) - .await - .expect("timed out waiting for message") - .unwrap() - .unwrap(); - - // Decode standard OpAMP protobuf binary frame. - let data = msg.into_data(); - let sta = decode_server_to_agent_frame(data.as_ref()); - let rc = sta.remote_config.expect("should have remote_config"); - let config = rc.config.expect("should have config"); - let file = config - .config_map - .get("") - .expect("should have empty-key entry"); - let yaml = String::from_utf8(file.body.clone()).unwrap(); - assert_eq!(yaml, yaml_payload, "delivered yaml must match"); - assert_eq!( - String::from_utf8(rc.config_hash).unwrap(), - "hash-1", - "delivered hash must match" - ); - } - - /// Phase C integration test: gateway YAML emitted from the typed L5 - /// stage_split path is queued onto the gateway-role connection and - /// not onto agent-role / backend-role connections. We don't decode - /// the protobuf (an unrelated decode-tag-zero issue affects sibling - /// tests today); we only assert *delivery routing* — the gateway - /// client receives a non-empty binary frame within the timeout, the - /// other roles receive nothing. - #[tokio::test] - async fn push_to_role_gateway_routes_only_to_gateway_role() { - use futures_util::StreamExt; - let (srv, addr) = start_server().await; - let mut agent_ws = connect_ws_client(addr, "agent-1", "agent").await; - let mut gateway_ws = connect_ws_client(addr, "gateway-1", "gateway").await; - let mut backend_ws = connect_ws_client(addr, "backend-1", "backend").await; - - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - // Mirror the call site that handle_plan now exercises: - // emit_gateway_yaml(...) → push_to_role(Gateway, ...). - // We use a stand-in YAML payload here; the emitter has its own - // tests in stage_config.rs. - let yaml = "extensions:\n opamp: {}\n".to_string(); - srv.push_to_role( - AgentRole::Gateway, - RemoteConfig { - config_hash: "hash-gw".into(), - yaml, - }, - ) - .await; - - // Gateway must receive exactly one frame. - let msg = tokio::time::timeout(std::time::Duration::from_secs(2), gateway_ws.next()) - .await - .expect("gateway timed out") - .unwrap() - .unwrap(); - let bytes = msg.into_data(); - assert!(!bytes.is_empty(), "gateway must receive a non-empty frame"); - - // Other roles must receive nothing within a short window. - let agent_result = - tokio::time::timeout(std::time::Duration::from_millis(200), agent_ws.next()).await; - assert!( - agent_result.is_err(), - "agent-role client must not receive gateway-role push" - ); - let backend_result = - tokio::time::timeout(std::time::Duration::from_millis(200), backend_ws.next()).await; - assert!( - backend_result.is_err(), - "backend-role client must not receive gateway-role push" - ); - } - - /// `push_to_role(Agent)` must not deliver to a backend-role client. - #[tokio::test] - async fn push_to_agent_role_does_not_reach_backend_role() { - use futures_util::StreamExt; - let (srv, addr) = start_server().await; - let mut agent_ws = connect_ws_client(addr, "agent-1", "agent").await; - let mut backend_ws = connect_ws_client(addr, "backend-1", "backend").await; - - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - srv.push_to_role( - AgentRole::Agent, - RemoteConfig { - config_hash: "hash-agent".into(), - yaml: "ddsketch:\n mode: batch\n".to_string(), - }, - ) - .await; - - // Agent must receive the message. - let msg = tokio::time::timeout(std::time::Duration::from_secs(2), agent_ws.next()) - .await - .expect("agent timed out") - .unwrap() - .unwrap(); - let data = msg.into_data(); - let sta = decode_server_to_agent_frame(data.as_ref()); - let rc = sta.remote_config.expect("should have remote_config"); - assert_eq!(String::from_utf8(rc.config_hash).unwrap(), "hash-agent"); - - // Backend must receive nothing within a short window. - let backend_result = - tokio::time::timeout(std::time::Duration::from_millis(200), backend_ws.next()).await; - assert!( - backend_result.is_err(), - "backend-role client must not receive agent-role push" - ); - } - - fn test_collector_plan( - collector_id: &str, - plan_id: u64, - ) -> crate::physical::compiler::CollectorPlan { - crate::physical::compiler::CollectorPlan { - summary_catalog: None, - collector_id: collector_id.into(), - envelope: crate::physical::compiler::PlanEnvelope { - plan_id, - plan_version: 1, - generated_at_unix_ms: 1, - activation_unix_ms: 1, - expiry_unix_ms: None, - backend_compat: "asap-query-backend.v1".into(), - planner_revision: crate::physical::compiler::PLANNER_REVISION.into(), - capability_snapshot_id: "caps-1".into(), - }, - materializations: vec![crate::physical::compiler::CollectorMaterialization { - query_id: "q".into(), - materialization: asap_types::PolicyFingerprint(1).into(), - metric: "requests".into(), - algorithm: "hll".into(), - parameters: serde_json::json!({"precision": 14}), - group_by: vec!["service".into()], - window_secs: 60, - abstract_window_framework: - planner_types::post_asap::SummaryWindowFramework::Tumbling, - window_realization_id: "collector-tumbling-v1".into(), - slide_secs: 60, - pane_origin_ms: Some(0), - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }, - evidence_source: None, - lifecycle: crate::physical::compiler::CollectorLifecycle { - kind: "continuously_maintained".into(), - maintenance_mode: "incremental".into(), - evaluation_schedule: "per_update".into(), - output_representation: "summary_state".into(), - }, - }], - transmission_rules: vec![crate::physical::compiler::TransmissionRule { - materialization: asap_types::PolicyFingerprint(1).into(), - producer_id: collector_id.into(), - schema_id: "asap-query-backend.v1:summary-state:v1:1".into(), - mode: crate::physical::compiler::TransmissionMode::Full, - encoding: crate::physical::compiler::StateEncoding::SketchlibProtobufV1, - emit_every_ms: 60_000, - full_checkpoint_every_ms: None, - destination_ref: "asapquery-backend".into(), - runtime_policy: crate::physical::compiler::RuntimeRulePolicy::default(), - }], - } - } - - async fn send_agent_message( - ws: &mut tokio_tungstenite::WebSocketStream< - tokio_tungstenite::MaybeTlsStream, - >, - message: opamp_proto::AgentToServer, - ) { - use futures_util::SinkExt; - let mut protobuf = Vec::new(); - message.encode(&mut protobuf).unwrap(); - let mut frame = vec![0]; - frame.extend(protobuf); - ws.send(tokio_tungstenite::tungstenite::Message::Binary(frame)) - .await - .unwrap(); - } - - #[tokio::test] - async fn typed_plan_publication_waits_for_capability_and_exact_staged_status() { - let (srv, addr) = start_server().await; - let mut agent_ws = connect_ws_client(addr, "edge-a", "agent").await; - - send_agent_message( - &mut agent_ws, - opamp_proto::AgentToServer { - custom_capabilities: Some(opamp_proto::CustomCapabilities { - capabilities: vec![COLLECTOR_PLAN_CAPABILITY.into()], - }), - ..Default::default() - }, - ) - .await; - - let publisher = Arc::clone(&srv); - let plan = test_collector_plan("edge-a", 42); - let publish = tokio::spawn(async move { - publisher - .publish_collector_plans(&[plan], Duration::from_secs(2)) - .await - }); - - let frame = tokio::time::timeout(Duration::from_secs(2), agent_ws.next()) - .await - .expect("typed plan timed out") - .unwrap() - .unwrap(); - let message = decode_server_to_agent_frame(&frame.into_data()); - let custom = message.custom_message.expect("custom message"); - assert_eq!(custom.capability, COLLECTOR_PLAN_CAPABILITY); - assert_eq!(custom.r#type, COLLECTOR_PLAN_MESSAGE); - let wire_plan: crate::physical::compiler::CollectorPlan = - serde_json::from_slice(&custom.data).unwrap(); - assert_eq!(wire_plan.collector_id, "edge-a"); - assert_eq!(wire_plan.envelope.plan_id, 42); - - let status = serde_json::to_vec(&CollectorPlanStatus { - plan_id: 42, - plan_version: 1, - status: CollectorPlanStatusKind::Staged, - error: None, - }) - .unwrap(); - send_agent_message( - &mut agent_ws, - opamp_proto::AgentToServer { - custom_message: Some(opamp_proto::CustomMessage { - capability: COLLECTOR_PLAN_CAPABILITY.into(), - r#type: PLAN_STATUS_MESSAGE.into(), - data: status, - }), - ..Default::default() - }, - ) - .await; - - let reports = publish.await.unwrap().unwrap(); - assert_eq!(reports.len(), 1); - assert_eq!(reports[0].plan_id, 42); - assert_eq!(reports[0].plan_version, 1); - assert_eq!(reports[0].status, CollectorPlanStatusKind::Staged); - } - - #[tokio::test] - async fn typed_plan_publication_fails_without_advertised_capability() { - let (srv, addr) = start_server().await; - let _agent_ws = connect_ws_client(addr, "edge-a", "agent").await; - let error = srv - .publish_collector_plans( - &[test_collector_plan("edge-a", 42)], - Duration::from_millis(30), - ) - .await - .unwrap_err(); - assert!(matches!( - error, - CollectorPlanPublishError::CapabilityTimeout { collector_id } - if collector_id == "edge-a" - )); - } - - #[tokio::test] - async fn typed_plan_publication_rejects_duplicate_targets_before_sending() { - let srv = OpampServer::new(); - let plan = test_collector_plan("edge-a", 42); - let error = srv - .publish_collector_plans(&[plan.clone(), plan], Duration::from_millis(1)) - .await - .unwrap_err(); - assert!(matches!( - error, - CollectorPlanPublishError::DuplicateTarget { collector_id } - if collector_id == "edge-a" - )); - } -} diff --git a/control_plane/src/physical/backend_stage.rs b/control_plane/src/physical/backend_stage.rs deleted file mode 100644 index bc60cf45a..000000000 --- a/control_plane/src/physical/backend_stage.rs +++ /dev/null @@ -1,87 +0,0 @@ -//! Backend-facing projection of one planning cycle. -//! -//! These types are the input to [`crate::backend_plan::from_stage_config`] and -//! to `emit::backend_wire`'s backend JSON builders. `DeploymentPlanCompiler` builds -//! them directly from the summaries ASAPPlanner selected. -//! -//! They are deliberately not `Serialize`/`Deserialize`: `SummaryFamilyType` -//! and `SketchQuery` have no serde impls upstream, and the wire payload is -//! produced by `emit::backend_wire::build_backend_aggregation_json`, a -//! hand-written JSON builder reading these fields, never a whole-struct -//! serialize. `backend_plan::from_stage_config` reuses that same builder so -//! the two wire formats share one `PolicyFingerprint` identity space. - -use planner_types::post_asap::{SketchQuery, SummaryFamilyType}; -use serde::{Deserialize, Serialize}; - -/// Everything the backend must materialize and serve for one planning cycle. -#[derive(Debug, Clone)] -pub struct BackendStageConfig { - /// One entry per materialization the backend maintains. - pub aggregations: Vec, - /// One readout per summary-estimate node — what the backend returns to - /// the query evaluator. - pub readouts: Vec, -} - -/// One summary the backend must accept and maintain. -/// -/// `aggregation_id` is internal plumbing: it threads a selected summary to -/// its readout while compiling. It is not emitted on the wire — the backend -/// content-addresses identity via `PolicyFingerprint`, derived from -/// `metric_name`, the summary family, grouping labels and `spatial_filter`. -#[derive(Debug, Clone, PartialEq)] -pub struct BackendAggregation { - /// Internal-only id (see struct doc). Not on the wire. - pub aggregation_id: String, - /// Source metric the aggregation runs over. Required by the backend's - /// `PrecomputeMaterialization` parser. - pub metric_name: String, - /// Planner-owned committed summary identity. Sketch entries carry a - /// validated `SketchKind` (category + algorithm + params); exact entries - /// carry the matching `ExactKind`/`ExactParams` pair. - pub family: SummaryFamilyType, - /// Window size in seconds. The backend's parser rejects a zero window. - pub window_secs: u64, - /// Spatial filter (comma-joined `k=v` pairs). Empty when none applies. - pub spatial_filter: String, - /// Group-by label names — keys in `labels.grouping` on the backend side, - /// where the precompute engine keys per-aggregation state by the - /// projected attribute set. - pub grouping: Vec, - /// Per-item dimension (the data-point attribute name, e.g. `endpoint`) - /// for an item_label-mode frequency sketch. Emitted into the - /// aggregation's `parameters["item_label"]` so data-plane ingest records - /// it on the sid and can answer per-item `estimate(key)`. - pub item_label: Option, - /// Runtime accumulator mode derived from the summary's input weight, - /// never from the TopK readout. `None` retains the value-update default. - pub heap_update_mode: Option<&'static str>, - /// What wire shape the backend ingests for this aggregation. - pub aggregation_input: AggregationInput, -} - -/// What wire shape the backend ingests for an aggregation: whether it builds -/// the summary from raw samples or accepts pre-built state from upstream. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AggregationInput { - /// The backend receives summary-state envelopes. - #[default] - SketchEnvelope, - /// The backend receives raw OTLP samples and builds the summary at - /// ingest. - Raw, -} - -/// One readout entry — what the backend's query evaluator asks for. -/// -/// Not `PartialEq`/`Serialize`/`Deserialize`: `op: SketchQuery` has none of -/// those upstream. -#[derive(Debug, Clone)] -pub struct BackendReadout { - /// Aggregation this readout reads from. - pub aggregation_id: String, - /// Readout op (mirror of `SummaryExpr::SummaryEstimate::query`). - pub op: SketchQuery, -} diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs deleted file mode 100644 index d90199c2a..000000000 --- a/control_plane/src/physical/compiler.rs +++ /dev/null @@ -1,8151 +0,0 @@ -//! Backend-owned physical compilation over ASAPPlanner's selected post-ASAP IR. -//! -//! Planner owns semantic candidates and guarantees. This module owns the -//! deployment decision: evidence freshness, target capabilities, windows, the -//! Collector execution projection, SummaryCatalog, and executable plans. - -use std::collections::{BTreeMap, BTreeSet, HashMap}; -use std::rc::Rc; - -use asap_aware_mapping::cost_model::Cost; -#[cfg(test)] -use asap_aware_mapping::DefaultAccuracyModel; -use asap_aware_mapping::{ - plan_summary_maintenance_lifecycles, AccuracyEvidenceProvider, CostRate, Horizon, - PropagationStats, SummaryMaintenanceCapabilities, SummaryMaintenanceLifecycleCapabilities, - SummaryMaintenanceLifecycleCostInputs, WorkloadDemand, -}; -use planner_types::post_asap::{ - CompositionOperator, EvaluationSchedule, ExecutableDagCompilation, OutputRepresentation, - PostAsapNodeId, SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, - SummaryMaintenanceLifecycle, SummaryMaintenanceMode, SummaryNode, SummaryWindowFramework, -}; -use planner_types::pre_asap::QueryExpr; -use planner_types::workload::{ - AccuracyRequirement, DataArrival, DataWorkload, DurationMs, Evidence, EvidenceSource, - Predictability, Query, QueryLanguage, QueryRecurrence, QueryRequirements, QueryTimeScope, - QueryWorkload, Rate, RepeatedDemand, RepeatingEntry, RepetitionInterval, TimeSelection, -}; -use serde::{de::Error as _, Deserialize, Deserializer, Serialize}; -use serde_json::{json, Value}; -use thiserror::Error; - -use crate::physical::backend_stage::{AggregationInput, BackendAggregation}; -use crate::physical::post_asap::cost_model::{ControlPlaneCostModel, ExactCompositionCostEvidence}; -use crate::query_plan::{ - canonical_promql, FallbackPolicy, InstantExecution, MaterializationBinding, PhysicalGrouping, - QueryPlan, QueryPlanEntry, -}; -use crate::types::AccuracyTarget; -use planner_types::pre_asap::Source; - -mod windows; -pub(super) use windows::gcd; -pub use windows::{prepare_window_implementations, WindowCostModel}; - -pub const PLANNER_REVISION: &str = env!("ASAPPLANNER_REVISION"); -pub const BACKEND_REVISION: &str = env!("ASAPQUERY_BACKEND_REVISION"); -pub use asap_types::precompute_plan::BACKEND_COMPAT; -/// Matches the data plane's default persistence memory limit. A backend-local -/// summary candidate must fit its complete retained state inside this budget. -pub const DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES: u64 = 2 * 1024 * 1024 * 1024; - -fn default_retained_summary_memory_budget_bytes() -> u64 { - DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES -} - -fn is_default_retained_summary_memory_budget_bytes(value: &u64) -> bool { - *value == DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES -} - -#[derive(Debug, Clone)] -pub struct QueryCompilationInput { - pub query_id: String, - /// Catalog expression used only to build the stable QueryPlan identity. - /// The selected implementation comes from `selected_plan_root`, never this text. - pub query_string: String, - /// Planner-selected post-ASAP DAG. The physical compiler must not - /// re-select a summary family from pre-ASAP input. - pub selected_plan_root: Rc, - /// Legacy catalog source retained for request compatibility. Physical - /// materialization sources are derived from each post-ASAP SummaryAgg; - /// it also remains part of the cost manifest workload identity. - pub legacy_query_source: Source, - pub query_lookback_seconds: u64, - /// Label names are deployment metadata because Planner's canonical IR - /// currently carries positional column IDs at this boundary. - pub group_by_labels: Vec, - pub accuracy_target: AccuracyTarget, - pub summary_lifecycle_inputs: SummaryLifecyclePlanningInputs, - /// Executor-feasible concrete realizations offered to Planner for its - /// abstract window-framework decision. The compiler retains physical - /// identities and exposes only framework + complete weighted cost to - /// Planner. An empty or stale set fails closed. - pub window_realization_candidates: Vec, - /// Physical runtime policy selected for this Planner materialization. - /// It is validated against the selected summary family during physical - /// compilation and becomes part of the immutable plan generation. - pub materialization_runtime_policy: RuntimeRulePolicy, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct WindowRealizationCostQuote { - pub model_version: String, - pub workload_fingerprint: String, - pub observed_at_unix_ms: u64, - pub valid_for_ms: u64, - pub horizon_seconds: f64, - /// Measured build plus incremental-update CPU over `horizon_seconds`. - pub cpu_cost: f64, - pub peak_memory_bytes: u64, - pub network_bytes: u64, - pub storage_bytes: u64, - pub source_scan_bytes: u64, - /// Dimensionally calibrated scalar passed to Planner for comparison. The - /// evidence producer must price update CPU, query-time merges at the - /// workload's read frequency, retained memory, storage, source scans, and - /// network traffic. Keeping the components alongside this quote makes the - /// selected tradeoff auditable without teaching Planner backend units. - pub weighted_cost: f64, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct WindowRealizationCandidate { - /// Backend-owned identity; never copied into Planner IR. - #[serde(rename = "implementation_id")] - pub realization_id: String, - pub framework: SummaryWindowFramework, - pub window_secs: u64, - pub slide_secs: u64, - pub layout: asap_types::WindowMaterializationLayout, - pub cost: WindowRealizationCostQuote, - /// Only compiler-generated quotes may be repriced after changing their layout. - /// Serialized input always becomes provider evidence. - #[serde(skip)] - pub derived: bool, - /// This offer was generated for a restricted derived-maintenance cohort, - /// whose cadence cannot satisfy ordinary raw-state consumers of the same W. - #[serde(skip)] - pub cohort_only: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct LifecycleUnitCosts { - pub build: f64, - pub maintenance_per_update: f64, - pub read: f64, - pub retention_per_second: f64, - pub retirement: f64, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct SummaryLifecyclePlanningInputs { - pub evaluation_interval_ms: u32, - pub ingestion_rate_per_second: f64, - pub evidence_observed_at_unix_ms: u64, - pub evidence_valid_for_ms: u64, - pub horizon_seconds: f64, - pub costs: LifecycleUnitCosts, -} - -#[derive(Debug, Clone, Default)] -pub struct PhysicalCompilationRequest { - /// Diagnostic projections of the original Planner search; never consumed by selection. - pub planner_selection_trace: Vec, - /// Enable a composable DAG with SummaryStore materializations and Prometheus exact subtrees. - pub allow_mixed_summary_and_exact_execution: bool, - /// Enabled optional candidate keys: None enables all eligible keys; an - /// explicitly empty set enables none. These are not catalog definition IDs. - pub enabled_materialization_keys: Option>, - /// Original dashboard demand, in the same order as queries. None is legacy input. - pub query_workload: Option, - /// Source evidence supplied independently from query demand. - pub data_workload: Option, - /// Workload-lowered roots retained across physical alternative enumeration. - pub canonical_roots: Vec>, - pub queries: Vec, - pub topk_membership_evidence_by_query_id: HashMap, - /// Fresh measured costs for Planner exact/summary composition sites, - /// scoped to query IDs just like accuracy evidence. - pub exact_composition_costs: HashMap>, - /// Optional Error–Resource Profile inputs and distribution-conditioned sizing. Hybrid - /// mode falls back to theoretical sizing and then exact execution. - pub erp: Option, - pub planner_revision: String, - /// Source cadence used to bound current-series input lag. - pub scrape_interval_ms: Option, - /// How far behind the newest ingested sample an admitted query may be - /// evaluated. This extends physical retention only; it never changes the - /// PromQL range selector used for readout. - pub query_retention_margin_ms: u64, - /// Maximum aggregate encoded footprint of every retained summary pane. - /// `None` uses the backend's default persistence-memory limit. - pub retained_summary_memory_budget_bytes: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct TopKMembershipEvidence { - pub selected_lower_bound: f64, - pub excluded_upper_bound: f64, - pub interval_failure_probability: f64, - pub observed_at_unix_ms: u64, - pub source: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct PhysicalDeploymentContext { - /// Semantic dataset served by this deployment's input channel; never an endpoint. - pub dataset_identity: planner_types::post_asap::LogicalDatasetIdentity, - pub target: PhysicalDeploymentTarget, - #[serde(rename = "collector_ids")] - pub target_collector_ids: Vec, - pub capability_snapshot_id: String, - pub observed_at_unix_ms: u64, - pub max_evidence_age_ms: u64, - pub plan_version: u64, - pub activation_unix_ms: u64, - pub expiry_unix_ms: Option, - pub backend_compat: String, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum PhysicalDeploymentTarget { - DistributedCollectors, - BackendLocalRemoteWrite, -} - -/// Startup and candidate-discovery input for backend-local planning. -/// Version 3 requires an explicit logical dataset identity; it is the sole supported schema; deployment always requires quotes. -/// The one workload-snapshot schema this backend accepts. -/// -/// Every producer of a snapshot has to agree with this number, and they are -/// written in different languages, so the number cannot live in each of them. -/// It lives here, and `snapshot_version_is_declared_once` fails if a shipped -/// example or a tool in `tools/` disagrees. Bumping the schema is then a single -/// edit plus whatever that test reports, instead of a literal that some -/// producers follow and others quietly do not. -pub const WORKLOAD_SNAPSHOT_VERSION: u32 = 3; - -/// Query/data semantics use ASAPPlanner's canonical workload types directly; -/// this wrapper adds only backend-owned implementation evidence and lifecycle -/// identity required to choose a concrete physical realization. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct BackendLocalPlanningInput { - /// Must equal [`WORKLOAD_SNAPSHOT_VERSION`]. - #[serde(rename = "snapshot_version")] - pub schema_version: u32, - /// May be absent during candidate discovery, never during deployment. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub workload_cost_evidence: Option, - #[serde(deserialize_with = "deserialize_snapshot_query_workload")] - pub query_workload: QueryWorkload, - pub data_workload: DataWorkload, - #[serde(rename = "implementation")] - pub physical_inputs: BackendLocalPhysicalInputs, - pub environment: PhysicalDeploymentContext, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct BackendLocalPhysicalInputs { - pub lifecycle_costs: LifecycleUnitCosts, - pub evidence_observed_at_unix_ms: u64, - pub evidence_valid_for_ms: u64, - pub horizon_seconds: f64, - pub window_cost_model: WindowCostModel, - /// Prometheus source scrape cadence. Rangeless instant-vector expressions - /// use this as their derived readout window. This backend-local planning - /// default is distinct from Prometheus instant-selector lookback delta. - pub scrape_interval_ms: u64, - #[serde(default, skip_serializing_if = "u64_is_zero")] - #[serde(rename = "query_staleness_margin_ms")] - pub query_retention_margin_ms: u64, - /// Admission budget for all retained panes and estimated partitions. - /// Missing legacy snapshots inherit the backend default. - #[serde( - default = "default_retained_summary_memory_budget_bytes", - skip_serializing_if = "is_default_retained_summary_memory_budget_bytes" - )] - #[serde(rename = "max_retained_summary_bytes")] - pub retained_summary_memory_budget_bytes: u64, - /// Certificates keyed by exact registered PromQL; converted to root IDs - /// before workload selection so one query cannot borrow another's evidence. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub topk_evidence: HashMap, - /// Measured exact/summary composition profiles keyed by registered - /// PromQL. Missing rows keep the corresponding Planner site opaque. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub exact_composition_costs: HashMap>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub erp: Option, -} - -/// Snapshot lookback is derived from PromQL and the declared scrape interval; -/// accepting it here would silently restore the competing legacy input. -fn deserialize_snapshot_query_workload<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let value = Value::deserialize(deserializer)?; - let has_legacy_lookback = value - .get("repeating_queries") - .and_then(Value::as_array) - .is_some_and(|entries| { - entries.iter().any(|entry| { - entry - .get("time_selection") - .and_then(Value::as_object) - .and_then(|selection| selection.get("lookback")) - .is_some_and(|lookback| !lookback.is_null()) - }) - }); - if has_legacy_lookback { - return Err(D::Error::custom( - "query_workload.repeating_queries[].time_selection.lookback is not supported; it is derived from PromQL", - )); - } - serde_json::from_value(value).map_err(D::Error::custom) -} - -fn u64_is_zero(value: &u64) -> bool { - *value == 0 -} - -pub use asap_types::precompute_plan::{ - ExactStateKind, IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, - PrecomputePlanError, ProducerContract, StateEncoding, StateFamilyContract, StateSchemaContract, - StateWindowContract, TimestampUnit, -}; - -pub use asap_types::producer_plan::{ - AdaptiveF64Bounds, AdaptiveU64Bounds, CollectorLifecycle, CollectorMaterialization, - CollectorPlan, DeltaPolicy, FrameIdentityContract, GosPolicy, GosThresholdMode, - RuntimeAdaptationEvidence, RuntimeAdaptationPolicy, RuntimeRulePolicy, SamplingEstimator, - SamplingPolicy, SequenceScope, SummaryFrameIdentity, SummaryFrameKind, TransmissionMode, - TransmissionPlan, TransmissionPlanError, TransmissionRule, -}; - -#[cfg(test)] -/// Build the fixed physical knob from the controller's canonical -/// epsilon-floor allocator. Degenerate budgets/rates disable sampling. -pub fn sampling_policy_from_accuracy_budget( - epsilon_sampling: f64, - updates_per_window: f64, - estimator: SamplingEstimator, -) -> SamplingPolicy { - if !epsilon_sampling.is_finite() - || epsilon_sampling <= 0.0 - || !updates_per_window.is_finite() - || updates_per_window <= 0.0 - { - return SamplingPolicy::Disabled; - } - let probability = crate::epsilon_alloc::derive_sample_p(epsilon_sampling, updates_per_window); - if probability >= 1.0 { - SamplingPolicy::Disabled - } else { - SamplingPolicy::Fixed { - probability, - estimator, - } - } -} - -#[cfg(test)] -/// Allocate the deterministic staleness share with the same linear-peel -/// composition used by `epsilon_alloc`. `None` means the selected sketch -/// already consumes the budget or communication has no allocated weight. -pub fn gos_policy_from_accuracy_budget( - epsilon_total: f64, - epsilon_sketch: f64, - sites: u32, - edge_cpu_weight: f64, - communication_weight: f64, - threshold_mode: GosThresholdMode, -) -> Option { - if !epsilon_total.is_finite() - || !epsilon_sketch.is_finite() - || !(0.0..=1.0).contains(&epsilon_total) - || epsilon_sketch < 0.0 - || !edge_cpu_weight.is_finite() - || edge_cpu_weight < 0.0 - || !communication_weight.is_finite() - || communication_weight <= 0.0 - { - return None; - } - let (_, epsilon_staleness) = crate::epsilon_alloc::split_budget( - epsilon_total, - epsilon_sketch, - edge_cpu_weight, - communication_weight, - ); - (epsilon_staleness.is_finite() && epsilon_staleness > 0.0).then_some(GosPolicy { - epsilon_staleness, - sites: sites.max(1), - threshold_mode, - }) -} - -pub fn build_transmission_plan( - envelope: PlanEnvelope, - precompute: &PrecomputePlan, - runtime_policies: &BTreeMap, -) -> Result { - if envelope != precompute.envelope { - return Err(TransmissionPlanError::EnvelopeMismatch); - } - let schemas: BTreeMap<_, _> = precompute - .schemas - .iter() - .map(|schema| (schema.materialization, schema)) - .collect(); - let rules = precompute - .producers - .iter() - .map(|producer| { - let schema = schemas - .get(&producer.materialization) - .expect("validated PrecomputePlan schema binding"); - let materialization = precompute - .materializations - .iter() - .find(|m| m.policy_fingerprint() == producer.materialization.fingerprint()) - .expect("validated PrecomputePlan materialization binding"); - let runtime_policy = runtime_policies - .get(&producer.materialization.fingerprint()) - .cloned() - .unwrap_or_default(); - let mode = if runtime_policy.delta.is_some() { - TransmissionMode::Delta - } else { - TransmissionMode::Full - }; - let emit_every_ms = materialization.window_size.saturating_mul(1_000); - TransmissionRule { - materialization: producer.materialization, - producer_id: producer.producer_id.clone(), - schema_id: producer.schema_id.clone(), - mode, - encoding: schema.encodings[0].clone(), - emit_every_ms, - full_checkpoint_every_ms: (mode == TransmissionMode::Delta) - .then(|| emit_every_ms.saturating_mul(10)), - destination_ref: "asapquery-backend".into(), - runtime_policy, - } - }) - .collect(); - let plan = TransmissionPlan { - summary_catalog: precompute.summary_catalog.clone(), - envelope, - frame_identity: FrameIdentityContract { - identity_version: 1, - sequence_scope: SequenceScope::MaterializationSeriesProducerEpoch, - require_checkpoint_for_full: true, - require_base_checkpoint_for_delta: true, - }, - rules, - }; - plan.validate(precompute)?; - Ok(plan) -} - -/// Complete physical projection of one post-ASAP planning decision. -/// All three child plans share the same envelope and are compiled together. -/// Complete selected physical plan, serializable for developer inspection. -/// -/// The serialized form is an inspection artifact emitted by -/// `compile_workload_artifact`; it is not an input accepted by the compiler. -#[derive(Debug, Clone, Serialize)] -pub struct CompiledPhysicalPlan { - pub envelope: PlanEnvelope, - pub summary_catalog: super::summary_catalog::SummaryCatalog, - pub collector_plans: Vec, - pub precompute_plan: PrecomputePlan, - pub transmission_plan: TransmissionPlan, - pub query_plan: QueryPlan, - /// Backend storage-routing table for the metrics this plan materializes. - /// Published alongside the plan so the query-shape → engine split always - /// describes the generation that is actually installed; without it the - /// backend falls back to its default engine and archive-shape queries miss. - pub storage_routing: serde_json::Value, - /// Lifecycle component only, not a complete physical-plan comparison. - pub lifecycle_estimates: Vec, - pub cost_comparison: Option, - pub planner_selection_trace: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct MaterializationLifecycleEstimate { - pub materialization: asap_types::sds::StoredOutputId, - pub consumer_query_ids: Vec, - #[serde(rename = "window_implementation_id")] - pub window_realization_id: String, - pub horizon_seconds: f64, - pub expected_reads: f64, - pub expected_updates: f64, - pub lifecycle_cost: f64, -} - -#[derive(Debug, Error)] -pub enum CompileError { - #[error("invalid backend-local workload snapshot: {0}")] - Snapshot(String), - #[error("no feasible completely costed alternative: {0}")] - Candidates(serde_json::Value), - #[error("planner revision mismatch: request={request}, compiler={compiler}")] - PlannerRevision { - request: String, - compiler: &'static str, - }, - #[error("query {query_id}: {reason}")] - Query { query_id: String, reason: String }, - #[error("query {query_id}: TopK evidence is stale or invalid: {reason}")] - InvalidEvidence { query_id: String, reason: String }, - #[error("query {query_id}: lifecycle planning failed: {reason}")] - Lifecycle { query_id: String, reason: String }, - #[error("failed to construct a physical materialization: {0}")] - Materialization(#[from] anyhow::Error), - #[error("failed to construct QueryPlan: {0}")] - QueryPlan(#[from] crate::query_plan::QueryPlanError), -} - -struct QueryEvidence<'a>(Option<&'a TopKMembershipEvidence>); - -impl AccuracyEvidenceProvider for QueryEvidence<'_> { - fn propagation_stats( - &self, - op: &CompositionOperator, - _family: &SummaryFamilyType, - _query: Option<&SketchQuery>, - ) -> PropagationStats { - match (op, self.0) { - (CompositionOperator::TopKSelection, Some(e)) => PropagationStats { - topk_selected_lower_bound: Some(e.selected_lower_bound), - topk_excluded_upper_bound: Some(e.excluded_upper_bound), - topk_interval_failure_probability: Some(e.interval_failure_probability), - ..Default::default() - }, - _ => PropagationStats::default(), - } - } -} - -#[derive(Debug, Default)] -pub struct DeploymentPlanCompiler; - -impl BackendLocalPlanningInput { - /// Invoke the pinned Planner from canonical startup workloads and compile - /// one backend-local CompiledPhysicalPlan. No CollectorPlan is produced and no - /// precompiled serving artifact is accepted at this boundary. - pub fn compile_promql(self) -> Result { - self.compile_frontend(QueryFrontend::PromQl) - } - - /// Use the shared parser subset with MetricsQL serving and exact routing. - pub fn compile_metricsql(self) -> Result { - self.compile_frontend(QueryFrontend::MetricsQl) - } - - fn compile_frontend( - self, - frontend: QueryFrontend, - ) -> Result { - let evidence = self.workload_cost_evidence.clone().ok_or_else(|| { - CompileError::Snapshot( - "deployment requires complete workload cost evidence; export candidates and price them before compiling".into(), - ) - })?; - let (request, environment) = self.into_physical_compilation_request()?; - let candidates = - super::workload_cost::enumerate_exact_and_materialized_candidates(request)?; - if frontend == QueryFrontend::MetricsQl { - super::workload_cost::select_lowest_cost_metricsql_candidate( - candidates, - environment, - &evidence, - ) - } else { - super::workload_cost::select_lowest_cost_candidate(candidates, environment, &evidence) - } - } - - /// Build Planner-authorized candidates for evidence collection without publishing. - pub fn into_physical_compilation_request( - self, - ) -> Result<(PhysicalCompilationRequest, PhysicalDeploymentContext), CompileError> { - if self.schema_version != WORKLOAD_SNAPSHOT_VERSION { - return Err(CompileError::Snapshot(format!( - "unsupported workload snapshot version {}; only version {} is supported", - self.schema_version, WORKLOAD_SNAPSHOT_VERSION - ))); - } - if self.environment.target != PhysicalDeploymentTarget::BackendLocalRemoteWrite { - return Err(CompileError::Snapshot( - "compatibility workload requires backend_local_remote_write target".into(), - )); - } - let workload = self.query_workload; - let mut data_workload = self.data_workload.clone(); - if data_workload.data_ingestion_interval.value.is_none() - && self.physical_inputs.scrape_interval_ms > 0 - { - data_workload.data_ingestion_interval = Evidence { - value: Some(DurationMs(self.physical_inputs.scrape_interval_ms)), - source: EvidenceSource::Declared, - observed_at_ms: None, - valid_for_ms: None, - }; - } - data_workload - .validate() - .map_err(|error| CompileError::Snapshot(error.to_string()))?; - workload - .validate() - .map_err(|error| CompileError::Snapshot(error.to_string()))?; - if workload.language != QueryLanguage::PromQL { - return Err(CompileError::Snapshot( - "ASAPQuery compatibility profile accepts PromQL workloads only".into(), - )); - } - let ingestion_rate = data_workload - .ingestion_rate - .value_at(self.environment.observed_at_unix_ms) - .copied() - .ok_or_else(|| { - CompileError::Snapshot("DataWorkload requires fresh ingestion_rate evidence".into()) - })?; - let entries = workload.entries().collect::>(); - if entries.is_empty() { - return Err(CompileError::Snapshot( - "QueryWorkload must contain at least one query".into(), - )); - } - let canonical_queries = asap_frontend_promql::lower_promql_workload( - &planner_types::workload::PlanningWorkload { - query_workload: workload.clone(), - data_workload: Some(data_workload.clone()), - }, - self.environment.observed_at_unix_ms, - ) - .map_err(|error| CompileError::Snapshot(error.to_string()))?; - let mut queries = Vec::with_capacity(entries.len()); - let mut canonical_roots = Vec::with_capacity(entries.len()); - let mut topk_evidence_by_id = HashMap::new(); - for (index, (entry, parsed)) in entries.into_iter().zip(canonical_queries).enumerate() { - let evaluation_interval_ms = match entry.recurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval, .. - }) => interval.0, - _ => { - return Err(CompileError::Snapshot(format!( - "query {index} must use phase-bearing fixed-interval repeated demand in the MVP profile" - ))) - } - }; - if self.physical_inputs.scrape_interval_ms == 0 - || !self - .physical_inputs - .scrape_interval_ms - .is_multiple_of(1_000) - { - return Err(CompileError::Snapshot( - "scrape_interval_ms must be a positive whole number of seconds".into(), - )); - } - let accuracy = entry.requirements.accuracy.target(); - let query_string = entry.query.0; - let lookback_ms = - query_history_window_ms(&parsed, self.physical_inputs.scrape_interval_ms) - .map_err(|error| CompileError::Snapshot(format!("query {index}: {error}")))?; - let metadata = crate::query_parser::qe_to_parsed_query(&parsed); - let source_metrics = super::workload_cost::exact_source_metrics(&parsed)?; - let source_hint = source_metrics.iter().next().cloned().ok_or_else(|| { - CompileError::Snapshot(format!("query {index} has no named time-series source")) - })?; - let lifecycle = SummaryLifecyclePlanningInputs { - evaluation_interval_ms, - ingestion_rate_per_second: ingestion_rate.0, - evidence_observed_at_unix_ms: self.physical_inputs.evidence_observed_at_unix_ms, - evidence_valid_for_ms: self.physical_inputs.evidence_valid_for_ms, - horizon_seconds: self.physical_inputs.horizon_seconds, - costs: self.physical_inputs.lifecycle_costs.clone(), - }; - let post_asap = crate::planner_selection::keep_pre_asap(&parsed) - .map_err(|error| CompileError::Snapshot(format!("query {index}: {error}")))?; - canonical_roots.push(Rc::new(parsed)); - let query_id = format!("compat-query-{index}"); - if let Some(evidence) = self.physical_inputs.topk_evidence.get(&query_string) { - topk_evidence_by_id.insert(query_id.clone(), evidence.clone()); - } - queries.push(QueryCompilationInput { - query_id, - query_string: query_string.clone(), - selected_plan_root: post_asap, - legacy_query_source: Source::TimeSeries { - metric: source_hint, - }, - query_lookback_seconds: lookback_ms / 1_000, - group_by_labels: metadata.group_by_labels, - accuracy_target: accuracy, - summary_lifecycle_inputs: lifecycle, - window_realization_candidates: Vec::new(), - materialization_runtime_policy: RuntimeRulePolicy::default(), - }); - } - let mut exact_costs_by_id = HashMap::new(); - for (index, entry) in workload.entries().enumerate() { - if let Some(rows) = self - .physical_inputs - .exact_composition_costs - .get(&entry.query.0) - { - for row in rows { - row.validate( - self.environment.observed_at_unix_ms, - self.environment.max_evidence_age_ms, - ) - .map_err(|reason| { - CompileError::Snapshot(format!( - "query {index}: invalid exact-composition evidence: {reason}" - )) - })?; - } - exact_costs_by_id.insert(format!("compat-query-{index}"), rows.clone()); - } - } - let planner_selection_trace = select_logical_roots_with_trace( - &mut queries, - canonical_roots.clone(), - &topk_evidence_by_id, - &exact_costs_by_id, - self.physical_inputs.erp.as_ref(), - )?; - for query in &mut queries { - prepare_window_implementations( - query, - &self.physical_inputs.window_cost_model, - self.environment.target, - self.physical_inputs.query_retention_margin_ms, - )?; - } - // Composable lowering defers unsafe leaves to query time individually; retain Planner siblings. - Ok(( - PhysicalCompilationRequest { - planner_selection_trace, - allow_mixed_summary_and_exact_execution: true, - enabled_materialization_keys: None, - query_workload: Some(workload), - data_workload: Some(data_workload), - canonical_roots, - queries, - topk_membership_evidence_by_query_id: topk_evidence_by_id, - exact_composition_costs: exact_costs_by_id, - erp: self.physical_inputs.erp, - planner_revision: PLANNER_REVISION.into(), - scrape_interval_ms: Some(self.physical_inputs.scrape_interval_ms), - query_retention_margin_ms: self.physical_inputs.query_retention_margin_ms, - retained_summary_memory_budget_bytes: Some( - self.physical_inputs.retained_summary_memory_budget_bytes, - ), - }, - self.environment, - )) - } -} - -/// Admit per-entity raw state only when the installed partition contract and -/// scalar input evaluator preserve source rows. Composite updates still require -/// an executable maintenance evaluator; a partition flag cannot authorize them. -fn has_unsafe_raw_entity_leaf( - node: &Rc, - selected: &[Rc], - pooling: bool, -) -> bool { - use planner_types::post_asap::ExactKind; - use planner_types::pre_asap::Reduction; - match &node.expr { - SummaryExpr::SummaryAgg { - child, - reduction, - family, - .. - } => { - if selected.iter().any(|selected| Rc::ptr_eq(selected, node)) { - let preserves_series_state = matches!( - family, - SummaryFamilyType::ExactAggregate( - ExactKind::Increase | ExactKind::Rate | ExactKind::Min | ExactKind::Max, - _ - ) - ); - let scalar_series_input = matches!(&node.expr, - SummaryExpr::SummaryAgg { input, family, .. } - if !matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::UnivMon)) - && asap_types::accumulator_spec::is_scalar_sample_value(input)); - let frequency_series_input = matches!(&node.expr, - SummaryExpr::SummaryAgg { input, family: SummaryFamilyType::Sketch(kind, _), .. } - if matches!(kind.algorithm(), SketchAlgorithm::Hll | SketchAlgorithm::UnivMon) - && asap_types::accumulator_spec::is_unit_sample_frequency(input)); - return matches!(reduction, Reduction::PerEntity) - && !pooling - && !(preserves_series_state || scalar_series_input || frequency_series_input); - } - let additive_reduction = matches!(reduction, Reduction::Reduce(_)) - && matches!(family, SummaryFamilyType::ExactAggregate(ExactKind::Sum, _)) - && matches!( - &child.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate( - ExactKind::Sum | ExactKind::Count, - _ - ), - .. - } - ); - has_unsafe_raw_entity_leaf(child, selected, additive_reduction) - } - SummaryExpr::BinaryOp { lhs, rhs, .. } => { - has_unsafe_raw_entity_leaf(lhs, selected, false) - || has_unsafe_raw_entity_leaf(rhs, selected, false) - } - SummaryExpr::SummaryEstimate { summary_input, .. } => { - has_unsafe_raw_entity_leaf(summary_input, selected, false) - } - SummaryExpr::SummaryMerge { children, .. } => children - .iter() - .any(|child| has_unsafe_raw_entity_leaf(child, selected, false)), - _ => false, - } -} - -fn original_root( - query: &QueryCompilationInput, - index: usize, - roots: &[Rc], -) -> Result { - if let Some(root) = roots.get(index) { - return Ok(root.as_ref().clone()); - } - crate::query_parser::parse_query_expr_canonical( - &query.query_string, - query.accuracy_target.clone(), - ) - .map_err(|error| CompileError::Query { - query_id: query.query_id.clone(), - reason: error.to_string(), - }) -} - -/// Preserve native execution for raw states that cannot preserve source semantics. -fn preserve_native_unsafe_raw_roots( - queries: &mut [QueryCompilationInput], - canonical_roots: &[Rc], -) -> Result<(), CompileError> { - for (index, query) in queries.iter_mut().enumerate() { - let selected = collect_selected_materializations(&query.selected_plan_root, false) - .map_err(|reason| CompileError::Query { - query_id: query.query_id.clone(), - reason, - })?; - if selected.is_empty() { - continue; - } - let selected_nodes = selected - .iter() - .map(|state| Rc::clone(&state.node)) - .collect::>(); - let unsafe_entities = - has_unsafe_raw_entity_leaf(&query.selected_plan_root, &selected_nodes, false); - if unsafe_entities { - let parsed = original_root(query, index, canonical_roots)?; - query.selected_plan_root = - crate::planner_selection::keep_pre_asap(&parsed).map_err(|error| { - CompileError::Query { - query_id: query.query_id.clone(), - reason: error.to_string(), - } - })?; - } - } - Ok(()) -} - -/// Canonicalize Planner candidates that have no backend-maintained state at -/// the physical compiler boundary. Their selected post-ASAP shape may be -/// intentionally unsupported (and therefore invalid as an executable -/// maintenance DAG), but the original query remains a valid exact plan. -fn preserve_invalid_exact_fallback_roots( - queries: &mut [QueryCompilationInput], - canonical_roots: &[Rc], - composable: bool, -) -> Result<(), CompileError> { - for (index, query) in queries.iter_mut().enumerate() { - let selected = collect_selected_materializations(&query.selected_plan_root, composable) - .map_err(|reason| CompileError::Query { - query_id: query.query_id.clone(), - reason, - })?; - let invalid_executable = - selected.is_empty() && validate_executable_subdag(&query.selected_plan_root).is_err(); - if invalid_executable - && !matches!(query.selected_plan_root.expr, SummaryExpr::KeepPreAsap(_)) - { - let parsed = original_root(query, index, canonical_roots)?; - query.selected_plan_root = - crate::planner_selection::keep_pre_asap(&parsed).map_err(|error| { - CompileError::Query { - query_id: query.query_id.clone(), - reason: error.to_string(), - } - })?; - } - } - Ok(()) -} - -/// A MetricsQL query whose only selected states are Prometheus-specific -/// counter readouts has no backend materialization to bind. Keep the original -/// query as one native exact root. Mixed queries retain their other selected -/// summaries and let query-time lowering cut only the counter branches. -fn preserve_metricsql_counter_only_roots( - queries: &mut [QueryCompilationInput], - canonical_roots: &[Rc], - composable: bool, -) -> Result<(), CompileError> { - for (index, query) in queries.iter_mut().enumerate() { - let selected = collect_selected_materializations(&query.selected_plan_root, composable) - .map_err(|reason| CompileError::Query { - query_id: query.query_id.clone(), - reason, - })?; - if selected.is_empty() - || !selected.iter().all(|state| { - matches!( - state.family, - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Rate - | planner_types::post_asap::ExactKind::Increase, - _ - ) - ) - }) - { - continue; - } - let parsed = original_root(query, index, canonical_roots)?; - query.selected_plan_root = - crate::planner_selection::keep_pre_asap(&parsed).map_err(|error| { - CompileError::Query { - query_id: query.query_id.clone(), - reason: error.to_string(), - } - })?; - } - Ok(()) -} - -impl DeploymentPlanCompiler { - pub fn compile_promql( - &self, - request: PhysicalCompilationRequest, - environment: PhysicalDeploymentContext, - ) -> Result { - self.compile_for_frontend(request, environment, QueryFrontend::PromQl) - } - - pub fn compile_metricsql( - &self, - request: PhysicalCompilationRequest, - environment: PhysicalDeploymentContext, - ) -> Result { - self.compile_for_frontend(request, environment, QueryFrontend::MetricsQl) - } - - pub fn compile_for_frontend( - &self, - mut request: PhysicalCompilationRequest, - environment: PhysicalDeploymentContext, - frontend: QueryFrontend, - ) -> Result { - environment - .dataset_identity - .validate() - .map_err(CompileError::Snapshot)?; - if let Some(data) = &request.data_workload { - data.validate() - .map_err(|error| CompileError::Snapshot(error.to_string()))?; - } - if super::maintained_population::supported(&request) - && (frontend != QueryFrontend::PromQl - || environment.target != PhysicalDeploymentTarget::BackendLocalRemoteWrite) - { - return Err(CompileError::Snapshot( - "current-series maintenance requires backend-local PromQL deployment".into(), - )); - } - if request.allow_mixed_summary_and_exact_execution - && environment.target != PhysicalDeploymentTarget::BackendLocalRemoteWrite - { - return Err(CompileError::Snapshot( - "hybrid execution requires backend-local deployment".into(), - )); - } - if request.planner_revision != PLANNER_REVISION { - return Err(CompileError::PlannerRevision { - request: request.planner_revision, - compiler: PLANNER_REVISION, - }); - } - if environment.target == PhysicalDeploymentTarget::BackendLocalRemoteWrite - && !environment.target_collector_ids.is_empty() - { - return Err(CompileError::Query { - query_id: "deployment-target".into(), - reason: "backend-local target cannot declare Collector producers".into(), - }); - } - let query_ids = request - .queries - .iter() - .map(|query| query.query_id.as_str()) - .collect::>(); - if query_ids.len() != request.queries.len() { - return Err(CompileError::Snapshot( - "planning query IDs must be unique within a plan generation".into(), - )); - } - - if let Some(workload) = &request.query_workload { - let entries = workload.entries().collect::>(); - if entries.len() != request.queries.len() - || entries - .iter() - .zip(&request.queries) - .any(|(entry, query)| entry.query.0 != query.query_string) - { - return Err(CompileError::Snapshot("original workload and planning queries must have identical order and query text".into())); - } - } - - if frontend == QueryFrontend::MetricsQl { - preserve_metricsql_counter_only_roots( - &mut request.queries, - &request.canonical_roots, - request.allow_mixed_summary_and_exact_execution, - )?; - } - preserve_invalid_exact_fallback_roots( - &mut request.queries, - &request.canonical_roots, - request.allow_mixed_summary_and_exact_execution, - )?; - - if environment.target == PhysicalDeploymentTarget::BackendLocalRemoteWrite - && !request.allow_mixed_summary_and_exact_execution - { - preserve_native_unsafe_raw_roots(&mut request.queries, &request.canonical_roots)?; - } - - let roots = request - .queries - .iter() - .enumerate() - .map(|(id, query)| (id, Rc::clone(&query.selected_plan_root))) - .collect(); - for (id, root) in planner_types::post_asap::share_common_summary_subtrees(roots) { - request.queries[id].selected_plan_root = root; - } - let mut compiled_materializations = Vec::with_capacity(request.queries.len()); - let mut collector_materializations = Vec::with_capacity(request.queries.len()); - let mut plan_materializations = Vec::with_capacity(request.queries.len()); - let mut shared_materializations = BTreeMap::new(); - let mut runtime_policies = BTreeMap::new(); - // Binding is established while compiling the physical - // materializations, then consumed by QueryPlan lowering. The key is - // the planner DAG node identity across the workload; serving never scans - // downstream components to rediscover this decision. - let mut executable_dags = vec![None::; request.queries.len()]; - let mut node_bindings = - BTreeMap::<(usize, PostAsapNodeId), asap_types::PolicyFingerprint>::new(); - let mut query_node_bindings = - BTreeMap::<(usize, PostAsapNodeId), crate::query_plan::QueryNodeId>::new(); - let consumers = materialization_consumers( - &request.queries, - environment.target, - request.allow_mixed_summary_and_exact_execution, - )?; - let mut lifecycle_estimates = - BTreeMap::::new(); - // Per-metric sketch families this cycle materializes. The storage-routing - // classifier turns them into the warm/archive shape split, so routing is - // derived from the same decisions that produced the materializations - // rather than from a separately maintained table. - let mut routed_algorithms = - BTreeMap::>::new(); - - for (query_index, query) in request.queries.iter().enumerate() { - let evidence = request - .topk_membership_evidence_by_query_id - .get(&query.query_id); - if let Some(e) = evidence { - validate_evidence(&query.query_id, e, &environment)?; - } - let node = query.selected_plan_root.clone(); - reject_uncertified_readouts(&query.query_id, &node)?; - let selected = collect_selected_materializations( - &node, - request.allow_mixed_summary_and_exact_execution, - ) - .map_err(|reason| CompileError::Query { - query_id: query.query_id.clone(), - reason, - })?; - let selected = selected - .into_iter() - .filter(|state| { - // Counter readout implements Prometheus extrapolation. Native - // MetricsQL includes boundary samples differently, so retain - // these leaves as external exact dependencies until its - // counter semantics have a dedicated implementation. - !(frontend == QueryFrontend::MetricsQl && matches!(state.family, - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Rate - | planner_types::post_asap::ExactKind::Increase, _))) - && (!request.allow_mixed_summary_and_exact_execution - || state.window_secs.is_none_or(|window| { - query - .window_realization_candidates - .iter() - .any(|candidate| candidate.window_secs == window) - })) - && (!matches!( - state.family, - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Max, - _ - ) - ) || crate::query_plan::query_time::selected_range_max_materialization( - &query.query_string, - &state.node, - ) - .ok() - .flatten() - .is_some()) - && request.enabled_materialization_keys.as_ref().is_none_or(|policy| { - let key = crate::query_plan::query_time::selected_counter_materialization( - &query.query_string, - &state.node, - ) - .ok() - .flatten() - .or_else(|| { - crate::query_plan::query_time::selected_range_max_materialization( - &query.query_string, - &state.node, - ) - .ok() - .flatten() - }); - // Masks enumerate counter/max choices only. Other selected - // summaries remain required by this physical alternative. - key.is_none_or(|key| policy.contains(&key)) - }) - }) - .collect::>(); - // An exact native fallback has no maintained state and must not - // depend on evidence for unused window/state implementations. - if selected.is_empty() - && super::maintained_population::operator(&request, query)?.is_none() - { - continue; - } - let executable = planner_types::post_asap::compile_executable_dag_with_node_ids( - &query.selected_plan_root, - ) - .map_err(|error| CompileError::Query { - query_id: query.query_id.clone(), - reason: format!("invalid executable subDAG: {error}"), - })?; - executable_dags[query_index] = Some(executable); - if selected.is_empty() { - continue; - } - validate_lifecycle_input(&query.query_id, &query.summary_lifecycle_inputs)?; - if environment.target == PhysicalDeploymentTarget::DistributedCollectors - && selected.iter().any(|state| { - matches!( - state.family, - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Count, - _ - ) - ) - }) - { - return Err(CompileError::Query { - query_id: query.query_id.clone(), - reason: "observation-count readout requires the backend precompute producer" - .into(), - }); - } - match &query.legacy_query_source { - Source::TimeSeries { .. } => {} - Source::Table { .. } => { - return Err(CompileError::Query { - query_id: query.query_id.clone(), - reason: "MVP physical compiler supports time-series sources only".into(), - }) - } - } - // Choose the population protocol before any source fingerprint or - // derived frontier binding is created. Only the actual selected - // global maintenance program and its raw inputs opt into it. - let mut canonical_nodes = std::collections::HashSet::new(); - if environment.target == PhysicalDeploymentTarget::BackendLocalRemoteWrite { - for state in &selected { - if matches!(&state.node.expr, SummaryExpr::SummaryAgg { - reduction: planner_types::pre_asap::Reduction::Reduce(keys), .. - } if keys.is_empty()) - { - if let Some(sources) = immutable_materialization_sources(&state.node) { - canonical_nodes.insert(Rc::as_ptr(&state.node) as usize); - canonical_nodes - .extend(sources.iter().map(|source| Rc::as_ptr(source) as usize)); - } - } - } - } - let cohort_nodes = windows::cohort_nodes(&selected); - for (ordinal, selected) in selected.into_iter().enumerate() { - let mut branch_query = query.clone(); - branch_query.query_lookback_seconds = - selected.window_secs.unwrap_or(query.query_lookback_seconds); - branch_query.group_by_labels = selected - .group_by - .clone() - .unwrap_or_else(|| query.group_by_labels.clone()); - branch_query - .window_realization_candidates - .retain(|candidate| { - candidate.window_secs == branch_query.query_lookback_seconds - && if cohort_nodes.contains(&(Rc::as_ptr(&selected.node) as usize)) { - windows::is_full_cohort(candidate) - } else { - !candidate.cohort_only - } - }); - let query = &branch_query; - let lifecycle_costs = SummaryMaintenanceLifecycleCostInputs { - build_cost: Some(Cost(query.summary_lifecycle_inputs.costs.build)), - maintenance_cost_per_update: Some(Cost( - query.summary_lifecycle_inputs.costs.maintenance_per_update, - )), - summary_read_cost: Some(Cost(query.summary_lifecycle_inputs.costs.read)), - retention_cost_rate: Some(CostRate( - query.summary_lifecycle_inputs.costs.retention_per_second, - )), - retirement_cost: Some(Cost(query.summary_lifecycle_inputs.costs.retirement)), - }; - let window_costs = super::realization::RealizationProvider::windows( - &super::realization::ExistingRealizations, - query, - &environment, - )?; - let model = ControlPlaneCostModel::new(query.accuracy_target.clone()) - .with_summary_maintenance( - lifecycle_costs, - SummaryMaintenanceCapabilities { - incremental_update: true, - merge: true, - delete: environment.target - == PhysicalDeploymentTarget::BackendLocalRemoteWrite, - }, - ) - .with_window_implementation_costs(window_costs); - let metric = selected.metric.clone(); - let aggregation_id = format!("{}:{ordinal}:{}", query.query_id, metric); - let physical_family = selected.family.clone(); - let physical_algorithm = match &physical_family { - SummaryFamilyType::ExactAggregate(kind, _) => { - format!("{kind:?}").to_ascii_lowercase() - } - _ => selected.algorithm.clone(), - }; - let mut aggregation = physical_aggregation( - query, - &selected, - aggregation_id.clone(), - environment.target, - ); - if let SummaryFamilyType::Sketch(kind, _) = &aggregation.family { - routed_algorithms - .entry(aggregation.metric_name.clone()) - .or_default() - .push(kind.algorithm().clone()); - } - let precompute_materialization = - scoped_materialization(&aggregation, &selected.node)?; - let materialization = precompute_materialization.policy_fingerprint(); - let consumer_indices = consumers[&materialization] - .iter() - .copied() - .filter(|&index| { - let Some(workload) = &request.query_workload else { - return true; - }; - let other = &request.queries[index]; - if other.summary_lifecycle_inputs.evaluation_interval_ms - != query.summary_lifecycle_inputs.evaluation_interval_ms - { - return false; - } - let phases = workload - .entries() - .map(|entry| match entry.recurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - evaluation_phase, - .. - }) => Some(evaluation_phase.0), - _ => None, - }) - .collect::>(); - match (phases[query_index], phases[index]) { - (Some(a), Some(b)) => { - let cadence = u64::from( - query.summary_lifecycle_inputs.evaluation_interval_ms, - ); - let window = query.query_lookback_seconds.saturating_mul(1_000); - a % cadence == b % cadence && a % window == b % window - } - _ => index == query_index, - } - }) - .collect::>(); - let state_consumers = consumer_indices - .iter() - .map(|index| &request.queries[*index]) - .collect::>(); - let planner_selection = select_lifecycle( - query, - &selected.node, - &model, - &environment, - &state_consumers, - request.query_workload.as_ref().map(|workload| { - ( - workload, - request.data_workload.as_ref(), - consumer_indices.clone(), - ) - }), - )?; - let window_implementation = query.window_realization_candidates.iter() - .find(|candidate| candidate.realization_id == planner_selection.window_realization_id - && candidate.framework == planner_selection.window_framework) - .ok_or_else(|| CompileError::Lifecycle { - query_id: query.query_id.clone(), - reason: "Planner selected a window framework without a retained concrete implementation".into(), - })?; - // The logical consumer group is identified above. The installed state - // identity includes the actual pane width selected by Planner. - // Preserve semantic window and evaluation cadence independently - // from the selected storage representation. - aggregation.window_secs = window_implementation.window_secs; - let mut runtime_materialization = - scoped_materialization(&aggregation, &selected.node)?; - if canonical_nodes.contains(&(Rc::as_ptr(&selected.node) as usize)) { - runtime_materialization.population_key_encoding = - asap_types::PopulationKeyEncoding::CanonicalLabelsV1; - } - runtime_materialization.window_size = window_implementation.window_secs; - runtime_materialization.slide_interval = window_implementation.slide_secs; - runtime_materialization.window_type = - if window_implementation.slide_secs == window_implementation.window_secs { - asap_types::WindowKind::Tumbling - } else { - asap_types::WindowKind::Sliding - }; - runtime_materialization.window_layout = window_implementation.layout.clone(); - let pane_width_ms = match &window_implementation.layout { - asap_types::WindowMaterializationLayout::FullWindow => { - window_implementation.slide_secs - } - layout => layout.base_pane_secs(), - } - .saturating_mul(1_000); - runtime_materialization.pane_origin_ms = shared_pane_origin_ms( - request.query_workload.as_ref(), - [query_index], - pane_width_ms, - ) - .map_err(|reason| CompileError::Query { - query_id: query.query_id.clone(), - reason, - })?; - if matches!( - window_implementation.layout, - asap_types::WindowMaterializationLayout::FullWindow - ) { - runtime_materialization.pane_origin_ms = - runtime_materialization.pane_origin_ms.map(|end_phase| { - (i128::from(end_phase) - - i128::from(window_implementation.window_secs) * 1_000) - .rem_euclid(i128::from(pane_width_ms)) - as i64 - }); - } - if let Some(source) = immutable_materialization_sources(&selected.node) { - if environment.target != PhysicalDeploymentTarget::BackendLocalRemoteWrite { - return Err(CompileError::Query { - query_id: query.query_id.clone(), - reason: "immutable derived state requires backend-local execution" - .into(), - }); - } - let compiled = executable_dags[query_index].as_ref().expect("compiled DAG"); - let mut frontiers = BTreeMap::new(); - let mut source_configs = Vec::new(); - for source in source { - let source_node = compiled.node_ids.node_id(&source).ok_or_else(|| { - CompileError::Query { - query_id: query.query_id.clone(), - reason: "derived source absent from selected DAG".into(), - } - })?; - let source_id = node_bindings - .get(&(query_index, source_node)) - .copied() - .ok_or_else(|| CompileError::Query { - query_id: query.query_id.clone(), - reason: "derived source was not installed before consumer".into(), - })?; - frontiers.insert(source_node, source_id.into()); - let config = compiled_materializations - .iter() - .find(|config: &&asap_types::PrecomputeMaterialization| { - config.policy_fingerprint() == source_id - }) - .ok_or_else(|| CompileError::Query { - query_id: query.query_id.clone(), - reason: "derived source config missing".into(), - })?; - if !source_configs.iter().any( - |existing: &&asap_types::PrecomputeMaterialization| { - existing.policy_fingerprint() == source_id - }, - ) { - source_configs.push(config); - } - } - asap_types::precompute_plan::validated_source_window_cohort( - &runtime_materialization, - &source_configs, - ) - .map_err(|error| CompileError::Query { - query_id: query.query_id.clone(), - reason: error.to_string(), - })?; - if runtime_materialization.window_size != runtime_materialization.slide_interval - { - return Err(CompileError::Query { query_id: query.query_id.clone(), - reason: "immutable scalar composition runtime requires nonoverlapping windows".into() }); - } - let SummaryExpr::SummaryAgg { child, .. } = &selected.node.expr else { - unreachable!() - }; - let input_node = compiled - .node_ids - .node_id(child) - .expect("selected input node"); - let document = asap_types::executable_plan::OwnedPostAsapDag::from_executable( - query.query_id.clone(), - &compiled.dag, - ) - .map_err(|reason| CompileError::Query { - query_id: query.query_id.clone(), - reason, - })?; - runtime_materialization.derived_input = Some( - asap_types::derived_input::DerivedInputIdentity::from_dag( - &document, input_node, &frontiers, - ) - .map_err(|reason| CompileError::Query { - query_id: query.query_id.clone(), - reason, - })?, - ); - } - let compiled_dag = executable_dags[query_index].as_ref().expect("selected DAG"); - let semantic_root = - compiled_dag - .node_ids - .node_id(&selected.node) - .ok_or_else(|| CompileError::Query { - query_id: query.query_id.clone(), - reason: "persisted semantic root is absent".into(), - })?; - runtime_materialization.semantic_fragment = Some( - asap_types::semantic_fragment::SemanticFragment::from_stored_output_in_dataset( - &compiled_dag.dag, - semantic_root, - environment.dataset_identity.clone(), - ) - .map_err(|reason| CompileError::Query { - query_id: query.query_id.clone(), - reason, - })?, - ); - let materialization = runtime_materialization.policy_fingerprint(); - let consumer_query_ids = state_consumers - .iter() - .map(|query| query.query_id.clone()) - .collect::>(); - // Lifecycle demand was priced before choosing pane width. Two - // distinct logical cohorts can now collide on one physical - // fingerprint; retaining either quote would omit consumers. - if lifecycle_estimates - .get(&materialization) - .is_some_and(|existing| existing.consumer_query_ids != consumer_query_ids) - { - return Err(CompileError::Lifecycle { - query_id: query.query_id.clone(), - reason: "selected pane coalesces distinct logical consumer cohorts; joint physical-pane lifecycle evidence is required".into(), - }); - } - lifecycle_estimates - .entry(materialization) - .or_insert_with(|| MaterializationLifecycleEstimate { - materialization: materialization.into(), - consumer_query_ids, - window_realization_id: window_implementation.realization_id.clone(), - horizon_seconds: query.summary_lifecycle_inputs.horizon_seconds, - expected_reads: planner_selection.expected_reads, - expected_updates: planner_selection.expected_updates, - lifecycle_cost: planner_selection.lifecycle_cost, - }); - let node_id = executable_dags[query_index] - .as_ref() - .expect("selected query has a compiled executable DAG") - .node_ids - .node_id(&selected.node) - .ok_or_else(|| CompileError::Query { - query_id: query.query_id.clone(), - reason: "selected materialization is absent from the compiled Planner DAG" - .into(), - })?; - let binding_key = (query_index, node_id); - if let Some(existing) = node_bindings.insert(binding_key, materialization) { - if existing != materialization { - return Err(CompileError::Query { - query_id: query.query_id.clone(), - reason: "one post-ASAP node resolved to conflicting materializations" - .into(), - }); - } - } - if let Some(existing) = runtime_policies.insert( - materialization, - query.materialization_runtime_policy.clone(), - ) { - if existing != query.materialization_runtime_policy { - return Err(CompileError::Query { - query_id: query.query_id.clone(), - reason: "queries sharing one materialization specify different runtime policies" - .into(), - }); - } - } - let physical_group_by = aggregation.grouping.clone(); - compiled_materializations.push(runtime_materialization.clone()); - let collector_materialization = CollectorMaterialization { - query_id: format!("state-{}", materialization.0), - materialization: materialization.into(), - metric: metric.clone(), - algorithm: physical_algorithm, - parameters: selected.parameters, - group_by: physical_group_by, - window_secs: runtime_materialization.window_size, - abstract_window_framework: planner_selection.window_framework.clone(), - window_realization_id: window_implementation.realization_id.clone(), - slide_secs: runtime_materialization.slide_interval, - pane_origin_ms: runtime_materialization.pane_origin_ms, - window_layout: window_implementation.layout.clone(), - evidence_source: evidence.map(|e| e.source.clone()), - lifecycle: planner_selection.lifecycle.clone(), - }; - // The store fingerprint identifies state, not its concrete - // deployment. Consumers may share it only when their selected - // implementations agree; otherwise publication would install - // conflicting producers for the same state identity. - let mut shared_contract = collector_materialization.clone(); - // Keep every consumer in the existing plan identity even - // though the executable producer declaration is deduplicated. - plan_materializations.push(collector_materialization.clone()); - shared_contract.query_id.clear(); - if let Some(existing) = shared_materializations.get(&materialization) { - if existing != &shared_contract { - return Err(CompileError::Query { - query_id: query.query_id.clone(), - reason: format!( - "queries sharing one materialization specify conflicting deployment contracts: existing={existing:?}, requested={shared_contract:?}" - ), - }); - } - } else { - shared_materializations.insert(materialization, shared_contract); - if runtime_materialization.derived_input.is_none() { - collector_materializations.push(collector_materialization); - } - } - } - } - - if environment.target == PhysicalDeploymentTarget::BackendLocalRemoteWrite { - super::pane_reuse::share_additive_panes( - &request, - &mut compiled_materializations, - &mut collector_materializations, - &mut plan_materializations, - &mut node_bindings, - &mut runtime_policies, - &mut lifecycle_estimates, - ); - } - - let plan_id = if request.allow_mixed_summary_and_exact_execution - || super::maintained_population::supported(&request) - { - use std::hash::{Hash, Hasher}; - let mut hash = std::collections::hash_map::DefaultHasher::new(); - stable_workload_plan_id(&plan_materializations, &request.queries).hash(&mut hash); - "typed-local-residual-v3-counter-index".hash(&mut hash); - super::maintained_population::supported(&request).hash(&mut hash); - request.enabled_materialization_keys.hash(&mut hash); - for query in &request.queries { - format!("{:?}", query.selected_plan_root).hash(&mut hash); - } - hash.finish() - } else { - stable_workload_plan_id(&plan_materializations, &request.queries) - }; - let envelope = PlanEnvelope { - plan_id, - plan_version: environment.plan_version, - generated_at_unix_ms: environment.observed_at_unix_ms, - activation_unix_ms: environment.activation_unix_ms, - expiry_unix_ms: environment.expiry_unix_ms, - backend_compat: environment.backend_compat.clone(), - planner_revision: PLANNER_REVISION.into(), - capability_snapshot_id: environment.capability_snapshot_id, - }; - let producer_ids = match environment.target { - PhysicalDeploymentTarget::DistributedCollectors => { - environment.target_collector_ids.clone() - } - PhysicalDeploymentTarget::BackendLocalRemoteWrite => Vec::new(), - }; - // Several queries/readouts may intentionally share one maintained - // summary. PrecomputePlan is keyed by physical identity, not query ID. - let mut materializations_by_fingerprint = BTreeMap::new(); - for materialization in compiled_materializations { - if materializations_by_fingerprint - .get(&materialization.policy_fingerprint()) - .is_some_and(|old: &asap_types::PrecomputeMaterialization| { - old.semantic_fragment != materialization.semantic_fragment - }) - { - return Err(CompileError::Query { - query_id: "shared-output".into(), - reason: "one deployed output cannot have different semantic definitions".into(), - }); - } - materializations_by_fingerprint - .entry(materialization.policy_fingerprint()) - .or_insert(materialization); - } - let mut materializations: Vec = - materializations_by_fingerprint.into_values().collect(); - let materialization_fingerprints: BTreeSet<_> = materializations - .iter() - .map(asap_types::PrecomputeMaterialization::policy_fingerprint) - .collect(); - let mut query_entries = BTreeMap::new(); - for (query_index, query) in request.queries.iter().enumerate() { - // VictoriaMetrics acceleration accepts only the shared - // PromQL-compatible parser contract. MetricsQL-only syntax fails - // closed here and remains eligible for exact VM execution. - let canonical = - canonical_promql(&query.query_string).map_err(|error| CompileError::Query { - query_id: query.query_id.clone(), - reason: format!("promql-compatible identity: {error}"), - })?; - let binding = |node: &Rc, node_family: &SummaryFamilyType| -> Result { - if immutable_materialization_sources(node).is_none() { - summary_agg_metric(node).ok_or_else(|| { - crate::query_plan::QueryPlanError::Invalid( - "materialized node has no unique time-series source".into(), - ) - })?; - } - let node_id = executable_dags[query_index] - .as_ref() - .ok_or_else(|| { - crate::query_plan::QueryPlanError::Invalid( - "materialized query has no compiled executable DAG".into(), - ) - })? - .node_ids - .node_id(node) - .ok_or_else(|| { - crate::query_plan::QueryPlanError::Invalid( - "query materialization is absent from the compiled Planner DAG".into(), - ) - })?; - let fingerprint = node_bindings - .get(&(query_index, node_id)) - .copied() - .ok_or_else(|| { - crate::query_plan::QueryPlanError::Invalid(format!( - "post-ASAP node has no compiled physical binding for {node_family:?}" - )) - })?; - let materialization = materializations - .iter() - .find(|candidate| candidate.policy_fingerprint() == fingerprint) - .ok_or_else(|| { - crate::query_plan::QueryPlanError::Invalid(format!( - "compiled binding {} is absent from PrecomputePlan", - fingerprint.0 - )) - })?; - let stored_interval_ms = materializations - .iter() - .find(|candidate| candidate.policy_fingerprint() == fingerprint) - .map(asap_types::PrecomputeMaterialization::stored_window_ms) - .ok_or_else(|| { - crate::query_plan::QueryPlanError::Invalid(format!( - "compiled binding {} has no precompute materialization", - fingerprint.0 - )) - })?; - let (_, source_window, _) = selected_input_contract(node) - .map_err(crate::query_plan::QueryPlanError::Invalid)?; - let materialization_family = materialization.accumulator_spec() - .map_err(|error| crate::query_plan::QueryPlanError::Invalid(error.to_string()))? - .family; - let window_ms = materialization.window_size.saturating_mul(1_000); - if materialization_family != *node_family - || window_ms == 0 - || source_window.unwrap_or(query.query_lookback_seconds).saturating_mul(1_000) - % window_ms != 0 - { - return Err(crate::query_plan::QueryPlanError::Invalid(format!( - "compiled binding {} disagrees with post-ASAP/deployment semantics", - fingerprint.0 - ))); - } - Ok(MaterializationBinding { - full_window_slide_ms: matches!(materialization.window_layout, asap_types::WindowMaterializationLayout::FullWindow) - .then_some(materialization.slide_interval.saturating_mul(1_000)), - readout_lookback_ms: source_window.map(|seconds| seconds.saturating_mul(1_000)), - materialization: fingerprint.into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_output(fingerprint.into()), - output_grouping: PhysicalGrouping::Reduce( - materialization.grouping_labels.names(), - ), - item_labels: materialization.aggregated_labels.labels.clone(), - window_ms: stored_interval_ms, - pane_origin_ms: materialization.pane_origin_ms, - }) - }; - let instant = InstantExecution { - lookback_ms: query.query_lookback_seconds.saturating_mul(1_000), - full_history: false, - cumulative_readout: true, - }; - // A whole-query native fallback need not be expressible in the local - // query-time algebra (for example an ERP-rejected entropy readout). - // Retain its native boundary without discarding other workload roots. - let native_root = request.allow_mixed_summary_and_exact_execution - && if let SummaryExpr::KeepPreAsap(expr) = &query.selected_plan_root.expr { - let original = original_root(query, query_index, &request.canonical_roots)?; - expr.as_ref() == &original - && crate::query_plan::query_time::compile_logical( - query.query_id.clone(), - canonical.clone(), - instant, - FallbackPolicy::ExactBackend, - ) - .is_err() - } else { - false - }; - let mut entry = if let Some(operator) = - super::maintained_population::operator(&request, query)? - { - let root = crate::query_plan::QueryNodeId(0); - let compiled = executable_dags[query_index] - .as_ref() - .expect("compiled Planner DAG"); - query_node_bindings.insert((query_index, compiled.dag.root), root); - Ok(crate::query_plan::QueryPlanEntry { - language: crate::query_plan::QueryLanguage::PromQl, - query_id: query.query_id.clone(), - canonical_query: canonical.clone(), - fixed_evaluation: None, - root, - nodes: BTreeMap::from([( - root, - crate::query_plan::QueryPlanNode::Logical { - operator, - inputs: vec![], - }, - )]), - instant, - fallback: FallbackPolicy::ExactBackend, - }) - } else if !request.allow_mixed_summary_and_exact_execution - && executable_dags[query_index].is_none() - && !collect_selected_materializations( - &query.selected_plan_root, - request.allow_mixed_summary_and_exact_execution, - ) - .map_err(|reason| CompileError::Query { - query_id: query.query_id.clone(), - reason, - })? - .is_empty() - { - // The selected state has no supported physical implementation. - // Keep native semantics instead of lowering an unbound summary. - let root = crate::query_plan::QueryNodeId(0); - Ok(crate::query_plan::QueryPlanEntry { - language: crate::query_plan::QueryLanguage::PromQl, - query_id: query.query_id.clone(), - canonical_query: canonical.clone(), - fixed_evaluation: None, - root, - nodes: BTreeMap::from([( - root, - crate::query_plan::QueryPlanNode::ExactFallback { - reason: "selected summary has no supported physical implementation" - .into(), - }, - )]), - instant, - fallback: FallbackPolicy::ExactBackend, - }) - } else if request.allow_mixed_summary_and_exact_execution && !native_root { - crate::query_plan::compile_bound_composable_mapped( - query.query_id.clone(), - canonical.clone(), - &query.selected_plan_root, - instant, - FallbackPolicy::ExactBackend, - binding, - |node, query_node| { - if let Some(post_asap_node) = executable_dags[query_index] - .as_ref() - .and_then(|compiled| compiled.node_ids.node_id(node)) - { - query_node_bindings.insert((query_index, post_asap_node), query_node); - } - }, - ) - } else { - crate::query_plan::compile_bound_mapped( - query.query_id.clone(), - canonical.clone(), - &query.selected_plan_root, - instant, - FallbackPolicy::ExactBackend, - binding, - |node, query_node| { - if let Some(post_asap_node) = executable_dags[query_index] - .as_ref() - .and_then(|compiled| compiled.node_ids.node_id(node)) - { - query_node_bindings.insert((query_index, post_asap_node), query_node); - } - }, - ) - }?; - if request.allow_mixed_summary_and_exact_execution { - // Any Planner-selected leaf without a physical summary binding - // is an exact subtree boundary. Deployed plans never retain a - // backend-local range index leaf. - crate::query_plan::query_time::finalize_query_time_nodes(&mut entry)?; - } - if frontend == QueryFrontend::MetricsQl { - entry.language = crate::query_plan::QueryLanguage::MetricsQl; - } - let catalog_key = QueryPlan::catalog_key(entry.language, &canonical); - if query_entries.insert(catalog_key, entry).is_some() { - return Err(CompileError::Query { - query_id: query.query_id.clone(), - reason: format!("duplicate canonical query identity `{canonical}`"), - }); - } - } - let mut query_plan = QueryPlan { - plan_id, - plan_version: envelope.plan_version, - clickhouse_context: None, - selected_dags: BTreeMap::new(), - entries: query_entries, - }; - let mut installed_dags = BTreeMap::new(); - for (query_index, compiled) in executable_dags.iter().enumerate() { - let Some(compiled) = compiled else { continue }; - let query_id = request.queries[query_index].query_id.clone(); - let query_plan_sink = query_plan - .entries - .values() - .find(|entry| entry.query_id == query_id) - .expect("compiled query entry exists") - .root; - let installed = super::executable_binding::install_selected_dag( - query_id.clone(), - &compiled.dag, - query_plan_sink, - |id| { - node_bindings - .get(&(query_index, id)) - .copied() - .map(Into::into) - }, - |id| query_node_bindings.get(&(query_index, id)).copied(), - ) - .map_err(|reason| CompileError::Query { - query_id: query_id.clone(), - reason, - })?; - query_plan - .selected_dags - .insert(query_id.clone(), installed.document.clone()); - installed_dags.insert(query_id, installed); - } - for materialization in &mut materializations { - let fingerprint = materialization.policy_fingerprint(); - let max_lookback_ms = query_plan - .entries - .values() - .flat_map(QueryPlanEntry::materialization_bindings) - .filter(|binding| binding.materialization.fingerprint() == fingerprint) - .filter_map(|binding| binding.readout_lookback_ms) - .max(); - if let Some(lookback_ms) = max_lookback_ms { - materialization.num_aggregates_to_retain = Some(retained_state_count( - lookback_ms, - request.query_retention_margin_ms, - materialization.slide_interval.saturating_mul(1_000), - &materialization.window_layout, - )); - } - } - validate_retained_summary_footprint( - &materializations, - request - .data_workload - .as_ref() - .and_then(|data| { - data.input_cardinality - .value_at(environment.observed_at_unix_ms) - }) - .copied(), - request - .retained_summary_memory_budget_bytes - .unwrap_or(DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES), - )?; - query_plan.validate(&materialization_fingerprints)?; - for (query_id, installed) in &installed_dags { - let entry = query_plan - .entries - .values() - .find(|entry| &entry.query_id == query_id) - .ok_or_else(|| CompileError::Query { - query_id: query_id.clone(), - reason: "installed post-ASAP DAG has no query-plan entry".into(), - })?; - super::executable_binding::validate_query_plan(installed, entry).map_err(|reason| { - CompileError::Query { - query_id: query_id.clone(), - reason, - } - })?; - } - // QueryPlan already owns executable readout nodes. Persist only - // maintenance ancestors under PrecomputePlan. - installed_dags = installed_dags - .into_iter() - .filter(|(_, installed)| !installed.binding.precompute_sinks.is_empty()) - .map(|(query_id, installed)| { - installed - .precompute_projection() - .map(|projected| (query_id.clone(), projected)) - .map_err(|reason| CompileError::Query { query_id, reason }) - }) - .collect::>()?; - let mut precompute_plan = match environment.target { - PhysicalDeploymentTarget::DistributedCollectors => { - PrecomputePlan::build(envelope.clone(), materializations, &producer_ids).and_then( - |mut plan| { - plan.executable_dags = installed_dags; - plan.validate()?; - Ok(plan) - }, - ) - } - PhysicalDeploymentTarget::BackendLocalRemoteWrite => { - PrecomputePlan::build_backend_local_with_dags( - envelope.clone(), - materializations, - installed_dags, - ) - } - } - .map_err(|error| CompileError::Query { - query_id: "precompute-plan".into(), - reason: error.to_string(), - })?; - precompute_plan.ingest.dataset_identity = Some(environment.dataset_identity.clone()); - let mut transmission_plan = crate::physical::compiler::build_transmission_plan( - envelope.clone(), - &precompute_plan, - &runtime_policies, - ) - .map_err(|error| CompileError::Query { - query_id: "transmission-plan".into(), - reason: error.to_string(), - })?; - let mut collector_plans = producer_ids - .into_iter() - .map(|collector_id| CollectorPlan { - summary_catalog: transmission_plan.summary_catalog.clone(), - transmission_rules: transmission_plan - .rules - .iter() - .filter(|rule| rule.producer_id == collector_id) - .cloned() - .collect(), - collector_id, - envelope: envelope.clone(), - materializations: collector_materializations.clone(), - }) - .collect::>(); - let summary_catalog = super::summary_catalog::SummaryCatalog::from_materializations( - envelope.plan_id, - envelope.plan_version, - &precompute_plan.materializations, - ) - .map_err(|error| CompileError::Query { - query_id: "summary-catalog".into(), - reason: error.to_string(), - })?; - precompute_plan - .bind_catalog(&summary_catalog) - .map_err(|error| CompileError::Query { - query_id: "precompute-catalog".into(), - reason: error.to_string(), - })?; - transmission_plan.summary_catalog = precompute_plan.summary_catalog.clone(); - for collector in &mut collector_plans { - collector.summary_catalog = precompute_plan.summary_catalog.clone(); - } - transmission_plan - .validate_against_catalog(&summary_catalog) - .map_err(|error| CompileError::Query { - query_id: "summary-catalog".into(), - reason: error.to_string(), - })?; - for collector in &collector_plans { - collector - .validate_against_catalog(&summary_catalog) - .map_err(|error| CompileError::Query { - query_id: "summary-catalog".into(), - reason: error.to_string(), - })?; - } - query_plan.bind_catalog(&summary_catalog)?; - let storage_routing = crate::emit::backend_wire::storage_routing_document( - crate::emit::backend_wire::DEFAULT_TENANT, - &routed_algorithms.into_iter().collect::>(), - ); - Ok(CompiledPhysicalPlan { - envelope, - summary_catalog, - collector_plans, - precompute_plan, - transmission_plan, - query_plan, - storage_routing, - lifecycle_estimates: lifecycle_estimates.into_values().collect(), - cost_comparison: None, - planner_selection_trace: request.planner_selection_trace, - }) - } -} - -fn summary_agg_metric(node: &SummaryNode) -> Option { - fn walk(node: &SummaryNode, metrics: &mut BTreeSet) { - match &node.expr { - SummaryExpr::KeepPreAsap(expr) => { - let parsed = crate::query_parser::qe_to_parsed_query(expr); - if !parsed.metric_name.is_empty() { - metrics.insert(parsed.metric_name); - } - } - SummaryExpr::SummaryAgg { child, .. } => walk(child, metrics), - - SummaryExpr::ValueOperation { child, .. } => walk(child, metrics), - SummaryExpr::SummaryEstimate { summary_input, .. } => walk(summary_input, metrics), - SummaryExpr::SummaryMerge { children, .. } => { - for child in children { - walk(child, metrics); - } - } - SummaryExpr::SummaryJoin { - outer: left, - inner: right, - .. - } - | SummaryExpr::RelationalJoin { left, right, .. } - | SummaryExpr::SummarySubtract { left, right } - | SummaryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } => { - walk(left, metrics); - walk(right, metrics); - } - SummaryExpr::SummaryDelete { summary_input, .. } => walk(summary_input, metrics), - } - } - let mut metrics = BTreeSet::new(); - walk(node, &mut metrics); - (metrics.len() == 1) - .then(|| metrics.into_iter().next()) - .flatten() -} - -/// Shared selection boundary for canonical startup and compile-and-publish. -/// Certificate-bearing roots stay isolated: equal certificate values do not -/// establish that the certificate's source scope covers another query. -pub fn select_logical_roots_for_queries( - queries: &mut [QueryCompilationInput], - roots: Vec>, - evidence: &HashMap, - exact_costs: &HashMap>, -) -> Result<(), CompileError> { - select_logical_roots_with_error_resource_profiles(queries, roots, evidence, exact_costs, None) -} - -fn observed_population_matches_root( - policy: &super::erp::ErpPlanningInput, - root: &QueryExpr, -) -> bool { - use asap_types::sds::{PopulationPartitioning, ValueProjectionIdentity}; - use planner_types::pre_asap::{AggIntent, Reduction}; - let (Some(data), Some(observed)) = ( - &policy.resolved_data_descriptor, - &policy.observed_populations, - ) else { - return false; - }; - let QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures, - having: None, - child, - .. - } = root - else { - return false; - }; - if measures.is_empty() - || !measures.iter().all(|intent| { - matches!(intent, AggIntent::Cardinality { cols, .. } if cols.is_empty()) - || matches!( - intent, - AggIntent::FrequencyL2 { col: None, .. } - | AggIntent::FrequencyEntropy { col: None, .. } - ) - }) - { - return false; - } - let Ok((metric, Some(window), filter)) = raw_time_series_input_contract(child, false) else { - return false; - }; - data.time_series_metric() == Some(metric.as_str()) - && data.population_filter_canonical == filter - && data.value_projection == ValueProjectionIdentity::SampleValue - && data.partitioning == Some(PopulationPartitioning::PerEntity) - && data.group_by_keys.is_empty() - && data.observation_semantics == asap_types::sds::TIMESTAMPED_OBSERVATION_SEMANTICS - && observed.window_end_ms.checked_sub(observed.window_start_ms) - == i64::try_from(window.saturating_mul(1000)).ok() -} - -pub fn select_logical_roots_with_error_resource_profiles( - queries: &mut [QueryCompilationInput], - roots: Vec>, - evidence: &HashMap, - exact_costs: &HashMap>, - erp: Option<&super::erp::ErpPlanningInput>, -) -> Result<(), CompileError> { - select_logical_roots_with_trace(queries, roots, evidence, exact_costs, erp).map(|_| ()) -} - -pub fn select_logical_roots_with_trace( - queries: &mut [QueryCompilationInput], - roots: Vec>, - evidence: &HashMap, - exact_costs: &HashMap>, - erp: Option<&super::erp::ErpPlanningInput>, -) -> Result, CompileError> { - let mut traces = Vec::new(); - if roots.len() != queries.len() { - return Err(CompileError::Snapshot( - "canonical root/query mapping is incomplete".into(), - )); - } - let mut cohorts: Vec<(AccuracyTarget, Option, Vec<(usize, Rc)>)> = - Vec::new(); - let original_roots = roots.clone(); - for (index, root) in roots.into_iter().enumerate() { - let accuracy = &queries[index].accuracy_target; - let certificate_scope = (evidence.contains_key(&queries[index].query_id) - || exact_costs.contains_key(&queries[index].query_id)) - .then(|| queries[index].query_id.clone()); - if let Some((_, _, roots)) = cohorts - .iter_mut() - .find(|(target, scope, _)| target == accuracy && scope == &certificate_scope) - { - roots.push((index, root)); - } else { - cohorts.push((accuracy.clone(), certificate_scope, vec![(index, root)])); - } - } - for (accuracy, scope, roots) in cohorts { - // ERP v1 has no calibrated failure probability. Preserve explicit - // confidence requirements through theoretical/exact fallback. - let scoped_erp = erp.map(|policy| { - let mut policy = policy.clone(); - if !matches!(accuracy, AccuracyTarget::Epsilon(_)) { - policy.artifact.records.clear(); - } - if policy.observed_populations.is_some() - && !roots - .iter() - .all(|(_, root)| observed_population_matches_root(&policy, root)) - { - if let Some(observed) = &mut policy.observed_populations { - observed.invalid_reason = - Some("candidate input differs from observed catalog data semantics".into()); - observed.populations.clear(); - } - } - // A benchmark of a different KLL implementation is not evidence - // for the collector's sketchlib KLL, even with the same k. - policy.artifact.records.retain(|row| { - (row.sketch == "kll-percall" && row.implementation == "lib") - || (row.sketch == "hll" - && row.implementation == "asap-sketchlib-hll-regular-v1") - || (row.sketch == "univmon" - && row.implementation == "asap-sketchlib-univmon-standard-v1") - }); - policy - }); - let erp = scoped_erp.as_ref(); - let mut model = ControlPlaneCostModel::new(accuracy.clone()).with_exact_composition_costs( - scope - .as_ref() - .and_then(|id| exact_costs.get(id)) - .cloned() - .unwrap_or_default(), - ); - if let Some(erp) = erp { - model = model.with_erp(erp.clone()); - } - let certificate = scope.as_ref().and_then(|id| evidence.get(id)); - let accuracy_model = super::erp::ErpAccuracyModel { - policy: erp, - max_error: match accuracy { - AccuracyTarget::Epsilon(e) | AccuracyTarget::EpsilonDelta { epsilon: e, .. } => e, - AccuracyTarget::Exact => 0.0, - }, - }; - let (selected, mut trace) = - crate::planner_selection::select_workload_with_accuracy_model_and_trace( - roots, - accuracy, - &model, - &QueryEvidence(certificate), - &accuracy_model, - ) - .map_err(|error| CompileError::Snapshot(error.to_string()))?; - trace["deployment_overrides"] = serde_json::json!([]); - let selected_indices = selected.iter().map(|(index, _)| *index).collect::>(); - for (index, node) in selected { - if erp.is_some_and(|policy| { - requires_exact_erp_fallback(&node, &queries[index].accuracy_target, policy) - }) { - if let Some(values) = trace["deployment_overrides"].as_array_mut() { - values.push( - serde_json::json!({"query_index": index, "status": "rejected", - "reason": "ERP requires exact fallback"}), - ); - } - queries[index].selected_plan_root = - crate::planner_selection::keep_pre_asap(&original_roots[index]) - .map_err(|error| CompileError::Snapshot(error.to_string()))?; - } else { - queries[index].selected_plan_root = node; - } - } - trace["committed_roots"] = serde_json::json!(selected_indices.into_iter().map(|index| - serde_json::json!({"query_index": index, - "logical_root_id": crate::planner_selection::explained_root_id(&queries[index].selected_plan_root, &queries[index].accuracy_target) - })).collect::>()); - traces.push(trace); - } - Ok(traces) -} - -fn requires_exact_erp_fallback( - node: &SummaryNode, - accuracy: &AccuracyTarget, - erp: &super::erp::ErpPlanningInput, -) -> bool { - fn walk( - node: &SummaryNode, - out: &mut Vec<( - SketchAlgorithm, - SketchParams, - Option, - )>, - ) { - match &node.expr { - SummaryExpr::SummaryAgg { child, .. } => { - walk(child, out); - } - SummaryExpr::SummaryEstimate { - summary_input, - query, - .. - } => { - for field in &summary_input.schema.fields { - if node.guarantee.as_ref().is_some_and(|g| g.is_exact()) { - continue; - } - let SummaryFamilyType::Sketch(kind, _) = &field.dtype else { - continue; - }; - out.push(( - kind.algorithm().clone(), - kind.params().clone(), - super::erp::ReadoutEvidence::for_query(kind.algorithm(), query), - )); - } - walk(summary_input, out); - } - SummaryExpr::SummaryDelete { summary_input, .. } - | SummaryExpr::ValueOperation { - child: summary_input, - .. - } => walk(summary_input, out), - SummaryExpr::SummaryMerge { children, .. } => { - children.iter().for_each(|child| walk(child, out)) - } - SummaryExpr::SummaryJoin { outer, inner, .. } => { - walk(outer, out); - walk(inner, out); - } - SummaryExpr::SummarySubtract { left, right } - | SummaryExpr::RelationalJoin { left, right, .. } - | SummaryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } => { - walk(left, out); - walk(right, out); - } - - SummaryExpr::KeepPreAsap(_) => {} - } - } - let max_error = match accuracy { - AccuracyTarget::Epsilon(value) | AccuracyTarget::EpsilonDelta { epsilon: value, .. } => { - *value - } - AccuracyTarget::Exact => return false, - }; - let mut sketches = Vec::new(); - walk(node, &mut sketches); - sketches.into_iter().any(|(algorithm, params, readout)| { - let decision = match readout { - Some(readout) => erp.select_readout(algorithm, readout, max_error, params), - None => erp.select(algorithm, max_error, params), - }; - matches!( - decision, - super::erp::ErpParameterDecision::ExactFallback { .. } - ) - }) -} - -#[cfg(test)] -/// Planner-adapter selection step used before physical compilation. Keeping -/// this separate makes the ownership boundary explicit: callers supply the -/// selected post-ASAP DAG to [`DeploymentPlanCompiler::compile`]. -pub fn select_post_asap( - expr: &QueryExpr, - accuracy: AccuracyTarget, - lifecycle: &SummaryLifecyclePlanningInputs, - evidence: Option<&TopKMembershipEvidence>, -) -> Result, crate::planner_selection::SelectionError> { - let model = ControlPlaneCostModel::new(accuracy).with_summary_maintenance( - SummaryMaintenanceLifecycleCostInputs { - build_cost: Some(Cost(lifecycle.costs.build)), - maintenance_cost_per_update: Some(Cost(lifecycle.costs.maintenance_per_update)), - summary_read_cost: Some(Cost(lifecycle.costs.read)), - retention_cost_rate: Some(CostRate(lifecycle.costs.retention_per_second)), - retirement_cost: Some(Cost(lifecycle.costs.retirement)), - }, - SummaryMaintenanceCapabilities { - incremental_update: true, - merge: true, - delete: false, - }, - ); - // These fixtures exercise collector heap transport. Collector fixtures do - // not offer backend-only temporal value readouts as a physical capability. - struct CollectorFixtureModel(ControlPlaneCostModel); - impl asap_aware_mapping::CostModel for CollectorFixtureModel { - fn rank_candidates( - &self, - intent: &planner_types::pre_asap::AggIntent, - candidates: &[planner_types::post_asap::SketchAlgorithm], - ) -> Vec { - self.0.rank_candidates(intent, candidates) - } - fn size_params( - &self, - kind: planner_types::post_asap::SketchAlgorithm, - intent: &planner_types::pre_asap::AggIntent, - eps: f64, - delta: f64, - ) -> planner_types::post_asap::SketchParams { - self.0.size_params(kind, intent, eps, delta) - } - fn candidate_cost( - &self, - candidate: &asap_aware_mapping::ReplacementSubDAG, - target: &asap_aware_mapping::TargetSubDAG<'_>, - ) -> Option { - self.0.candidate_cost(candidate, target) - } - fn summary_support_evidence(&self, summary: &SummaryNode) -> Option { - if matches!( - summary.expr, - SummaryExpr::ValueOperation { - operation: planner_types::post_asap::ValueOperation::Limit { .. }, - .. - } - ) { - return Some(false); - } - self.0.summary_support_evidence(summary) - } - } - crate::planner_selection::select_query_with_models( - expr, - &CollectorFixtureModel(model), - &DefaultAccuracyModel, - &QueryEvidence(evidence), - ) -} - -fn validate_evidence( - query_id: &str, - evidence: &TopKMembershipEvidence, - env: &PhysicalDeploymentContext, -) -> Result<(), CompileError> { - let age = env - .observed_at_unix_ms - .saturating_sub(evidence.observed_at_unix_ms); - let valid = evidence.observed_at_unix_ms <= env.observed_at_unix_ms - && evidence.selected_lower_bound.is_finite() - && evidence.excluded_upper_bound.is_finite() - && evidence.selected_lower_bound > evidence.excluded_upper_bound - && (0.0..=1.0).contains(&evidence.interval_failure_probability) - && !evidence.source.trim().is_empty() - && age <= env.max_evidence_age_ms; - if valid { - Ok(()) - } else { - Err(CompileError::InvalidEvidence { - query_id: query_id.into(), - reason: format!("margin/failure/source invalid or age {age}ms exceeds policy"), - }) - } -} - -fn validate_lifecycle_input( - query_id: &str, - input: &SummaryLifecyclePlanningInputs, -) -> Result<(), CompileError> { - let costs = [ - input.costs.build, - input.costs.maintenance_per_update, - input.costs.read, - input.costs.retention_per_second, - input.costs.retirement, - ]; - if input.evaluation_interval_ms == 0 - || input.evidence_valid_for_ms == 0 - || !input.ingestion_rate_per_second.is_finite() - || input.ingestion_rate_per_second < 0.0 - || !input.horizon_seconds.is_finite() - || input.horizon_seconds <= 0.0 - || costs.iter().any(|cost| !cost.is_finite() || *cost < 0.0) - { - return Err(CompileError::Lifecycle { - query_id: query_id.into(), - reason: "rates, horizon, validity, intervals, and costs must be finite and non-negative (interval/horizon/validity non-zero)".into(), - }); - } - Ok(()) -} - -/// Price one derived layout from the snapshot's own lifecycle unit costs. -/// -/// `WindowRealizationCostQuote` is normally measured evidence, and its -/// `weighted_cost` doc puts pricing update CPU, query-time merges, retained -/// memory, storage, scans and network on the evidence producer. When a -/// snapshot prices no candidate, the control plane becomes that producer for -/// the derived shapes — and it does so without inventing a single magnitude. -/// Every unit cost below is supplied evidence (`LifecycleUnitCosts`, from -/// `implementation.lifecycle_costs`); every multiplier is a structural count -/// that follows from the layout's definition. Nothing here is a measurement. -/// -/// Over `horizon_seconds`, for window `W`, slide `S` and the layout's own -/// shape: -/// -/// - **states created**: a `Pane { P }` seals a state every `P`, a -/// `FullWindow` every `S`. Each one is built once and retired once, so -/// `build` and `retirement` are charged per created state. -/// - **update fanout**: this is the layout's whole point (`worker.rs`'s -/// `stores_full_windows` branch). A pane takes each sample exactly once; a -/// full window takes it into every overlapping window that contains it, -/// averaging `W / S` under the supplied stationary ingestion-rate model. -/// The maximum simultaneously open count is separately charged as residency. -/// - **finalizations per read**: the mirror image. A full window is read -/// whole; `W / P` panes are composed into one answer. Charged at the -/// query's own evaluation cadence. -/// - **retention**: `retained_state_count`, the same function that fills -/// `num_aggregates_to_retain`, so the quote and the plan cannot disagree. -/// Open worker accumulators are charged separately from published states; -/// updating a state and keeping it resident are different resources. -/// -/// The byte fields are left exactly as the snapshot supplied them: state size -/// needs sketch parameters that do not exist yet at this point, and guessing -/// them would be the fabrication this function otherwise avoids. Only -/// `cpu_cost` and `weighted_cost` are derived, and only those two are read — -/// by `validate_window_implementations` and by the `min_by` that ranks -/// candidates. `model_version` records that the quote is derived. -pub(super) fn derived_window_cost( - template: &WindowRealizationCostQuote, - lifecycle: &SummaryLifecyclePlanningInputs, - window_secs: u64, - slide_secs: u64, - layout: &asap_types::WindowMaterializationLayout, - staleness_margin_ms: u64, -) -> WindowRealizationCostQuote { - let costs = &lifecycle.costs; - let horizon = lifecycle.horizon_seconds.max(0.0); - let window = window_secs.max(1) as f64; - let slide = slide_secs.max(1) as f64; - let (seal_interval, update_fanout, finalizations_per_read) = match layout { - asap_types::WindowMaterializationLayout::FullWindow => (slide, window / slide, 1.0), - asap_types::WindowMaterializationLayout::Pane { pane_secs } => { - let pane = (*pane_secs).max(1) as f64; - (pane, 1.0, (window / pane).ceil()) - } - asap_types::WindowMaterializationLayout::HierarchicalRollup { base_pane_secs, .. } => { - let pane = (*base_pane_secs).max(1) as f64; - (pane, 1.0, (window / pane).ceil()) - } - }; - let states_created = horizon / seal_interval; - let updates = lifecycle.ingestion_rate_per_second.max(0.0) * horizon * update_fanout; - let evaluation_secs = (f64::from(lifecycle.evaluation_interval_ms) / 1_000.0).max(1.0); - let reads = horizon / evaluation_secs; - let retained = retained_state_count( - window_secs.saturating_mul(1_000), - staleness_margin_ms, - slide_secs.saturating_mul(1_000), - layout, - ) as f64; - // Store retention does not include worker accumulators that are still open. - let active = match layout { - asap_types::WindowMaterializationLayout::FullWindow => (window / slide).ceil(), - _ => 1.0, - }; - - let build = costs.build * states_created; - let maintenance = costs.maintenance_per_update * updates; - let read = costs.read * reads * finalizations_per_read; - let retention = costs.retention_per_second * horizon * (retained + active); - let retirement = costs.retirement * states_created; - let cpu_cost = build + maintenance; - let weighted_cost = cpu_cost + read + retention + retirement; - - WindowRealizationCostQuote { - model_version: format!( - "{}+derived-window-layout-v1", - template - .model_version - .trim_end_matches("+derived-window-layout-v1") - ), - cpu_cost, - weighted_cost, - ..template.clone() - } -} - -/// The furthest point in the past that evaluating `expr` can read, in ms. -/// -/// A range selector and a subquery each evaluate their child over prior -/// history; a positive offset moves that history further back. The result is a -/// lower-bound horizon, not a claim that the entire interval is read. For -/// example, `a[1m] offset 1h` selects `(t - 61m, t - 60m]`. -fn query_history_window_ms(expr: &QueryExpr, scrape_interval_ms: u64) -> Result { - fn duration_ms(duration: std::time::Duration) -> Result { - if duration.subsec_nanos() != 0 { - return Err(CompileError::Snapshot( - "PromQL ranges must be a whole number of seconds in the backend-local profile" - .into(), - )); - } - u64::try_from(duration.as_millis()) - .map_err(|_| CompileError::Snapshot("PromQL range exceeds supported history".into())) - } - - fn add(left: u64, right: u64) -> Result { - left.checked_add(right).ok_or_else(|| { - CompileError::Snapshot("PromQL history exceeds supported duration".into()) - }) - } - - fn visit( - expr: &QueryExpr, - scrape_interval_ms: u64, - in_range: bool, - ) -> Result { - match expr { - // A subquery evaluates instant expressions at earlier timestamps; - // those leaves still need their own default readout window. - QueryExpr::PromqlSubquery { range, child, .. } => add( - duration_ms(*range)?, - visit(child, scrape_interval_ms, false)?, - ), - QueryExpr::TimeRange { range, child } => add( - duration_ms(*range)?, - visit(child, scrape_interval_ms, true)?, - ), - QueryExpr::TimeShift { shift, child } => { - if !shift.offset_ms.unsigned_abs().is_multiple_of(1_000) { - return Err(CompileError::Snapshot( - "PromQL offsets must be a whole number of seconds in the backend-local profile".into(), - )); - } - add( - shift.offset_ms.max(0) as u64, - visit(child, scrape_interval_ms, in_range)?, - ) - } - QueryExpr::PromqlScalarBridge(child) - | QueryExpr::PromqlVectorFromScalar(child) - | QueryExpr::PromqlScalarFromVector(child) - | QueryExpr::PromqlRelabel { child, .. } - | QueryExpr::PromqlSeriesSample { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Project { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } => visit(child, scrape_interval_ms, in_range), - QueryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } - | QueryExpr::Join { left, right, .. } - | QueryExpr::SetOp { left, right, .. } => Ok(visit( - left, - scrape_interval_ms, - in_range, - )? - .max(visit(right, scrape_interval_ms, in_range)?)), - QueryExpr::Concat { children, .. } => children.iter().try_fold(0, |history, child| { - Ok(history.max(visit(child, scrape_interval_ms, in_range)?)) - }), - QueryExpr::Scan { .. } if !in_range => Ok(scrape_interval_ms), - _ => Ok(0), - } - } - - visit(expr, scrape_interval_ms, false) -} - -/// Every distinct range-selector window in `expr`, as seconds. -/// -/// A single query can carry several. `sum(sum_over_time(a[1m])) / sum(sum_over_time(b[5m]))` -/// has two, and each one becomes its own materialization with its own window. -/// These are materialization contracts. Enclosing subquery ranges and offsets -/// affect a query's furthest lookback but do not change a leaf contract. -#[cfg(test)] -fn range_selector_windows_secs(expr: &QueryExpr) -> BTreeSet { - fn visit(expr: &QueryExpr, windows: &mut BTreeSet) { - if let QueryExpr::TimeRange { range, .. } = expr { - let secs = range.as_secs(); - if secs != 0 { - windows.insert(secs); - } - } - match expr { - QueryExpr::PromqlScalarBridge(child) - | QueryExpr::PromqlVectorFromScalar(child) - | QueryExpr::PromqlScalarFromVector(child) - | QueryExpr::PromqlRelabel { child, .. } - | QueryExpr::PromqlSeriesSample { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Project { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - | QueryExpr::TimeRange { child, .. } - | QueryExpr::TimeShift { child, .. } => visit(child, windows), - QueryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } - | QueryExpr::Join { left, right, .. } - | QueryExpr::SetOp { left, right, .. } => { - visit(left, windows); - visit(right, windows); - } - _ => {} - } - } - let mut windows = BTreeSet::new(); - visit(expr, &mut windows); - windows -} - -/// Derive and price one implementation per supported layout for each range. -/// Derived maintenance cohorts are restricted to their supported full windows; -/// raw additive pane producers may subsequently be shared by Planner. -#[cfg(test)] -fn derived_window_candidates( - implementation_id: &str, - expr: &QueryExpr, - lookback_ms: u64, - evaluation_interval_ms: u32, - cost: WindowRealizationCostQuote, - lifecycle: &SummaryLifecyclePlanningInputs, - staleness_margin_ms: u64, -) -> Vec { - let mut windows = range_selector_windows_secs(expr); - if windows.is_empty() { - windows.insert(lookback_ms / 1_000); - } - let mut lifecycle = lifecycle.clone(); - lifecycle.evaluation_interval_ms = evaluation_interval_ms; - windows - .into_iter() - .flat_map(|window| { - windows::derive( - &WindowCostModel { - implementation_id: implementation_id.into(), - cost: cost.clone(), - quotes: Vec::new(), - }, - &lifecycle, - window, - false, - PhysicalDeploymentTarget::BackendLocalRemoteWrite, - staleness_margin_ms, - ) - }) - .collect() -} - -pub(super) fn validate_window_implementations( - query: &QueryCompilationInput, - environment: &PhysicalDeploymentContext, -) -> Result, CompileError> { - let mut ids = BTreeSet::new(); - let mut candidates = Vec::new(); - for candidate in &query.window_realization_candidates { - let evidence = &candidate.cost; - let age = environment - .observed_at_unix_ms - .saturating_sub(evidence.observed_at_unix_ms); - let valid = evidence.observed_at_unix_ms <= environment.observed_at_unix_ms - && !candidate.realization_id.trim().is_empty() - && ids.insert(candidate.realization_id.clone()) - && !evidence.model_version.trim().is_empty() - && !evidence.workload_fingerprint.trim().is_empty() - && evidence.valid_for_ms != 0 - && age <= environment.max_evidence_age_ms.min(evidence.valid_for_ms) - && evidence.horizon_seconds.is_finite() - && (evidence.horizon_seconds - query.summary_lifecycle_inputs.horizon_seconds).abs() - <= f64::EPSILON - && evidence.cpu_cost.is_finite() - && evidence.cpu_cost >= 0.0 - && evidence.weighted_cost.is_finite() - && evidence.weighted_cost >= 0.0 - && candidate.window_secs == query.query_lookback_seconds - && candidate - .layout - .validate(candidate.window_secs, candidate.slide_secs) - .is_ok() - && windows::supported(candidate, environment.target); - if !valid { - return Err(CompileError::Lifecycle { - query_id: query.query_id.clone(), - reason: format!( - "window implementation `{}` has incomplete, stale, incompatible, or duplicate physical evidence", - candidate.realization_id - ), - }); - } - candidates.push(( - candidate.realization_id.clone(), - candidate.framework.clone(), - Cost(evidence.weighted_cost), - )); - } - if candidates.is_empty() { - return Err(CompileError::Lifecycle { - query_id: query.query_id.clone(), - reason: "no complete executor-feasible window implementation evidence".into(), - }); - } - Ok(candidates) -} - -pub(super) fn retained_state_count( - lookback_ms: u64, - staleness_margin_ms: u64, - slide_ms: u64, - layout: &asap_types::WindowMaterializationLayout, -) -> u64 { - match layout { - asap_types::WindowMaterializationLayout::FullWindow => staleness_margin_ms - .div_ceil(slide_ms.max(1)) - .saturating_add(1), - asap_types::WindowMaterializationLayout::Pane { pane_secs } => lookback_ms - .saturating_add(staleness_margin_ms) - .div_ceil(pane_secs.saturating_mul(1_000).max(1)) - .saturating_add(1), - asap_types::WindowMaterializationLayout::HierarchicalRollup { base_pane_secs, .. } => { - lookback_ms - .saturating_add(staleness_margin_ms) - .div_ceil(base_pane_secs.saturating_mul(1_000).max(1)) - .saturating_add(1) - } - } -} - -/// Estimate the encoded bytes retained by one physical state. Sketch matrix -/// cells use two words here, covering the counter plus observed serialization -/// overhead. Heap and exact-state estimates include container slack. -fn retained_state_bytes(materialization: &asap_types::PrecomputeMaterialization) -> u128 { - estimated_state_bytes( - &materialization.aggregation_type, - &materialization.parameters, - ) -} - -pub(super) fn estimated_state_bytes( - aggregation: &asap_types::AggregationType, - parameters: &HashMap, -) -> u128 { - use asap_types::AggregationType as A; - - let parameter = |names: &[&str], fallback: u64| { - names - .iter() - .find_map(|name| parameters.get(*name).and_then(Value::as_u64)) - .unwrap_or(fallback) as u128 - }; - match aggregation { - A::CountMinSketch | A::CountSketch => { - parameter(&["width", "w", "col_num", "col"], 1) - * parameter(&["depth", "d", "row_num", "row"], 1) - * 16 - } - A::CountMinSketchWithHeap | A::CountSketchWithHeap => { - parameter(&["width", "w", "col_num", "col"], 1) - * parameter(&["depth", "d", "row_num", "row"], 1) - * 16 - + parameter(&["heap_size"], 1) * 256 - } - A::DatasketchesKLL => parameter(&["k"], 200) * 32, - A::UnivMon => { - parameter(&["layers"], 4) - * (parameter(&["sketch_rows"], 5) * parameter(&["sketch_cols"], 1024) * 16 - + parameter(&["heap_size"], 32) * 256) - } - A::HydraKLL => parameter(&["k"], 200) * parameter(&["col", "cols"], 1) * 32, - A::HLL => 1u128 << parameter(&["precision", "p"], 14).min(24), - A::DDSketch => 64 * 1024, - A::Sum - | A::Count - | A::Increase - | A::Rate - | A::Min - | A::Max - | A::SingleSubpopulation - | A::MultipleSubpopulation => 256, - } -} - -fn retained_partition_count( - materialization: &asap_types::PrecomputeMaterialization, - input_cardinality: Option, -) -> u128 { - use asap_types::AggregationType as A; - - // Reset-aware and min/max state remains source-series scoped even with an - // empty output grouping. Grouped states have at most one partition per - // input series. Other empty groupings are the Reduce([]) global singleton. - if materialization.partitioning == Some(asap_types::sds::PopulationPartitioning::PerEntity) - || matches!( - materialization.aggregation_type, - A::Increase | A::Rate | A::Min | A::Max - ) - || !materialization.grouping_labels.names().is_empty() - { - u128::from(input_cardinality.unwrap_or(1).max(1)) - } else { - 1 - } -} - -fn validate_retained_summary_footprint( - materializations: &[asap_types::PrecomputeMaterialization], - input_cardinality: Option, - budget_bytes: u64, -) -> Result<(), CompileError> { - let estimated_bytes = materializations - .iter() - .fold(0u128, |total, materialization| { - total.saturating_add( - retained_state_bytes(materialization) - .saturating_mul(u128::from( - materialization.num_aggregates_to_retain.unwrap_or(1), - )) - .saturating_mul(retained_partition_count(materialization, input_cardinality)), - ) - }); - if estimated_bytes > u128::from(budget_bytes) { - return Err(CompileError::Query { - query_id: "retained-summary-footprint".into(), - reason: format!( - "estimated retained summary footprint {estimated_bytes} bytes exceeds budget {budget_bytes} bytes across panes and partitions" - ), - }); - } - Ok(()) -} - -struct PlannerPhysicalSelection { - window_realization_id: String, - lifecycle: CollectorLifecycle, - window_framework: SummaryWindowFramework, - expected_reads: f64, - expected_updates: f64, - lifecycle_cost: f64, -} - -fn select_lifecycle( - query: &QueryCompilationInput, - node: &SummaryNode, - model: &ControlPlaneCostModel, - environment: &PhysicalDeploymentContext, - consumers: &[&QueryCompilationInput], - original_workload: Option<(&QueryWorkload, Option<&DataWorkload>, Vec)>, -) -> Result { - // Current lifecycle evidence is per producer with one unit read cost. - // Conflicting source/rate/horizon/cost snapshots cannot be averaged into - // invented evidence. Only recurrence may differ between consumers. - let mut common = query.summary_lifecycle_inputs.clone(); - common.evaluation_interval_ms = 0; - for consumer in consumers { - let mut input = consumer.summary_lifecycle_inputs.clone(); - input.evaluation_interval_ms = 0; - if input != common { - return Err(CompileError::Lifecycle { - query_id: consumer.query_id.clone(), - reason: "shared producer consumers have conflicting lifecycle evidence".into(), - }); - } - } - let workload = QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some( - consumers - .iter() - .map(|query| RepeatingEntry { - query: Query(query.query_id.clone()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval( - query.summary_lifecycle_inputs.evaluation_interval_ms, - )), - requirements: QueryRequirements { - accuracy: AccuracyRequirement::Explicit(query.accuracy_target.clone()), - ..QueryRequirements::default() - }, - predictability: Predictability::Predictable { known_at: None }, - time_selection: TimeSelection { - // Collector windows are retired as whole states. They do not - // claim deletion support for moving-window retractions. - scope: QueryTimeScope::Unknown, - lookback: Some(DurationMs( - raw_materialization_input_contract(node) - .ok() - .and_then(|(_, window, _)| window) - .unwrap_or(query.query_lookback_seconds) - .saturating_mul(1_000), - )), - as_of: None, - }, - }) - .collect(), - ), - }; - let fallback_data = DataWorkload { - arrival: DataArrival::ContinuouslyIngesting, - // This workload is an internal lifecycle projection, not a - // plan-ready query input. Use the compatibility profile cadence - // when there is no original workload to carry source evidence. - data_ingestion_interval: Evidence { - value: Some(DurationMs(1_000)), - source: EvidenceSource::Declared, - observed_at_ms: None, - valid_for_ms: None, - }, - ingestion_rate: Evidence { - value: Some(Rate( - query.summary_lifecycle_inputs.ingestion_rate_per_second, - )), - source: EvidenceSource::Observed, - observed_at_ms: Some(query.summary_lifecycle_inputs.evidence_observed_at_unix_ms), - valid_for_ms: Some(query.summary_lifecycle_inputs.evidence_valid_for_ms), - }, - ..DataWorkload::default() - }; - let (workload, data, indices) = match original_workload { - Some((original, data, indices)) => (original, data.unwrap_or(&fallback_data), indices), - None => (&workload, &fallback_data, (0..consumers.len()).collect()), - }; - let plan = plan_summary_maintenance_lifecycles( - Rc::new(node.clone()), - WorkloadDemand::new_with_data(workload, data, &indices), - environment.observed_at_unix_ms, - Some(Horizon(query.summary_lifecycle_inputs.horizon_seconds)), - SummaryMaintenanceLifecycleCapabilities { - supports_ephemeral: false, - supports_prepared: false, - supports_shared: false, - supports_continuously_maintained: true, - }, - model, - ) - .map_err(|error| CompileError::Lifecycle { - query_id: query.query_id.clone(), - reason: error.to_string(), - })?; - let guarantee = plan - .deployments - .first() - .and_then(|deployment| deployment.summary_maintenance_lifecycle_guarantee.as_ref()) - .ok_or_else(|| CompileError::Lifecycle { - query_id: query.query_id.clone(), - reason: "latest ASAPPlanner selected no executable Collector lifecycle".into(), - })?; - let window_framework = plan - .deployments - .first() - .and_then(|deployment| deployment.selected_window_framework.clone()) - .ok_or_else(|| CompileError::Lifecycle { - query_id: query.query_id.clone(), - reason: "latest ASAPPlanner selected no window framework from the supplied physical evidence".into(), - })?; - Ok(PlannerPhysicalSelection { - window_realization_id: plan - .selected_window_implementation_id - .clone() - .ok_or_else(|| CompileError::Lifecycle { - query_id: query.query_id.clone(), - reason: "Planner returned no concrete window implementation identity".into(), - })?, - expected_reads: plan.expected_reads.ok_or_else(|| CompileError::Lifecycle { - query_id: query.query_id.clone(), - reason: "missing joint read demand".into(), - })?, - expected_updates: plan - .update_rate - .ok_or_else(|| CompileError::Lifecycle { - query_id: query.query_id.clone(), - reason: "missing source update demand".into(), - })? - .0 - * query.summary_lifecycle_inputs.horizon_seconds, - lifecycle_cost: plan.deployments[0] - .alternatives - .iter() - .find(|alternative| { - alternative.rejection.is_none() - && alternative.summary_maintenance_lifecycle - == guarantee.summary_maintenance_lifecycle - }) - .and_then(|alternative| alternative.total_cost) - .ok_or_else(|| CompileError::Lifecycle { - query_id: query.query_id.clone(), - reason: "missing selected lifecycle cost".into(), - })? - .0, - lifecycle: CollectorLifecycle { - kind: match guarantee.summary_maintenance_lifecycle { - SummaryMaintenanceLifecycle::Ephemeral => "ephemeral", - SummaryMaintenanceLifecycle::Prepared { .. } => "prepared", - SummaryMaintenanceLifecycle::Shared { .. } => "shared", - SummaryMaintenanceLifecycle::ContinuouslyMaintained => "continuously_maintained", - } - .into(), - maintenance_mode: match guarantee.summary_maintenance_mode { - SummaryMaintenanceMode::DirectBuild => "direct_build", - SummaryMaintenanceMode::Incremental => "incremental", - } - .into(), - evaluation_schedule: match guarantee.evaluation_schedule { - EvaluationSchedule::OneShot => "one_shot", - EvaluationSchedule::PerUpdate => "per_update", - EvaluationSchedule::OnRead => "on_read", - } - .into(), - output_representation: match guarantee.output_representation { - OutputRepresentation::PlainRows => "plain_rows", - OutputRepresentation::SummaryState => "summary_state", - OutputRepresentation::FinalizedValue => "finalized_value", - } - .into(), - }, - window_framework, - }) -} - -/// A warm producer may consume only a source whose semantics its precompute accumulator -/// implements. Predicates and shifted ranges remain executable query-time nodes. -pub(crate) fn raw_materialization_input_contract( - node: &SummaryNode, -) -> Result<(String, Option, String), String> { - let SummaryExpr::SummaryAgg { child, .. } = &node.expr else { - return Err("materialization requires a SummaryAgg leaf".into()); - }; - let SummaryExpr::KeepPreAsap(expr) = &child.expr else { - return Err("materialization input is not a raw source".into()); - }; - raw_time_series_input_contract( - expr, - matches!( - &node.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(..), - .. - } - ), - ) -} - -pub fn raw_time_series_input_contract( - expr: &QueryExpr, - exact: bool, -) -> Result<(String, Option, String), String> { - use planner_types::pre_asap::{CompareOpKind, ScalarValue}; - let (source, window_secs) = match expr { - QueryExpr::TimeRange { child, range } => { - if range.as_millis() == 0 || range.as_millis() % 1000 != 0 { - return Err("warm producer requires a positive whole-second range".into()); - } - (child.as_ref(), Some(range.as_secs())) - } - QueryExpr::Scan { .. } if exact => { - return Err( - "instantaneous sample selection is not a temporal accumulator readout".into(), - ); - } - source => (source, None), - }; - match source { - QueryExpr::Scan { - source: Source::TimeSeries { metric }, - predicates, - schema, - } if !metric.is_empty() => { - let mut matchers = Vec::with_capacity(predicates.len()); - for predicate in predicates { - let QueryExpr::Compare { left, op, right } = predicate.0.as_ref() else { - return Err("materialization filter is not a label comparison".into()); - }; - let (QueryExpr::Column(column), QueryExpr::Literal(ScalarValue::Utf8(value))) = - (left.as_ref(), right.as_ref()) - else { - return Err("materialization filter must compare a label with a string".into()); - }; - let label = schema - .columns - .get(*column) - .map(|field| field.name.as_str()) - .ok_or_else(|| { - "materialization filter references an unknown column".to_string() - })?; - let operator = match op { - CompareOpKind::Eq => "=", - CompareOpKind::Ne => "!=", - CompareOpKind::Regex => "=~", - CompareOpKind::NotRegex => "!~", - _ => return Err("materialization filter uses a non-PromQL comparison".into()), - }; - let encoded = serde_json::to_string(value).map_err(|error| error.to_string())?; - matchers.push(format!("{label}{operator}{encoded}")); - } - matchers.sort(); - let spatial_filter = if matchers.is_empty() { - String::new() - } else { - format!("{{{}}}", matchers.join(",")) - }; - Ok((metric.clone(), window_secs, spatial_filter)) - } - _ => { - Err("source predicates or temporal modifiers require a Prometheus exact subtree".into()) - } - } -} - -/// Immutable inputs may combine explicit exact accumulator readouts using -/// maintenance-time arithmetic. Population and window closure are checked by -/// the installed runtime, not inferred from the presence of this syntax. -fn immutable_materialization_sources(node: &SummaryNode) -> Option>> { - use planner_types::post_asap::{ExactKind, ExecutionTiming, SummaryInputExpr}; - let SummaryExpr::SummaryAgg { - child, - input, - family: SummaryFamilyType::Sketch(..), - .. - } = &node.expr - else { - // Existing exact spatial reductions execute over read-time values. - // They must not acquire a new durable maintenance dependency merely - // because Planner made the exact accumulator boundary explicit. - return None; - }; - if input.item.is_some() - || !matches!( - input.weight, - SummaryInputExpr::Column(planner_types::pre_asap::ColumnRef::SampleValue) - ) - { - return None; - } - fn collect(node: &Rc, sources: &mut Vec>) -> Option<()> { - match &node.expr { - SummaryExpr::BinaryOp { - lhs, - rhs, - operator, - timing: ExecutionTiming::IngestionTime, - } if operator.vector_match.is_none() - && matches!( - operator.kind, - planner_types::pre_asap::BinaryOpKind::Arithmetic(_) - ) => - { - collect(lhs, sources)?; - collect(rhs, sources)?; - } - SummaryExpr::ValueOperation { - child: source, - operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - timing: ExecutionTiming::IngestionTime, - } if matches!(&source.expr, - SummaryExpr::SummaryAgg { family: SummaryFamilyType::ExactAggregate(ExactKind::Sum | ExactKind::Count, _), child, .. } - if matches!(child.expr, SummaryExpr::KeepPreAsap(_))) => - { - if !sources.iter().any(|old| Rc::ptr_eq(old, source)) { - sources.push(source.clone()); - } - } - _ => return None, - } - Some(()) - } - let mut sources = Vec::new(); - collect(child, &mut sources)?; - Some(sources) -} - -fn selected_input_contract(node: &SummaryNode) -> Result<(String, Option, String), String> { - if let Some(source) = immutable_materialization_sources(node) { - raw_materialization_input_contract(&source[0]) - } else { - raw_materialization_input_contract(node) - } -} - -struct SelectedMaterialization { - node: Rc, - metric: String, - window_secs: Option, - spatial_filter: String, - group_by: Option>, - item_label: Option, - family: SummaryFamilyType, - algorithm: String, - parameters: Value, -} - -fn validate_executable_subdag(node: &Rc) -> Result<(), String> { - let executable = planner_types::post_asap::compile_executable_dag(node) - .map_err(|error| format!("invalid executable subDAG: {error}"))?; - if let Some(edge) = executable.edges.iter().find(|edge| { - edge.grouping == planner_types::post_asap::GroupingEdgeCompatibility::Incompatible - }) { - return Err(format!( - "executable subDAG contains incompatible grouping edge {} -> {} ({:?})", - edge.producer.0, edge.consumer.0, edge.role - )); - } - Ok(()) -} - -fn reject_uncertified_readouts(query_id: &str, root: &Rc) -> Result<(), CompileError> { - let dag = planner_types::post_asap::compile_executable_dag(root).map_err(|error| { - CompileError::Query { - query_id: query_id.into(), - reason: format!("invalid executable subDAG: {error}"), - } - })?; - for node in &dag.nodes { - if matches!( - node.payload, - planner_types::post_asap::ExecutableOperatorPayload::SummaryEstimate { .. } - ) && node - .guarantee - .as_ref() - .is_none_or(planner_types::post_asap::ResultGuarantee::has_unknown) - { - return Err(CompileError::Query { - query_id: query_id.into(), - reason: "selected summary readout has no certified accuracy guarantee; provide scoped evidence or use exact execution".into(), - }); - } - } - Ok(()) -} - -fn physical_aggregation( - query: &QueryCompilationInput, - selected: &SelectedMaterialization, - aggregation_id: String, - target: PhysicalDeploymentTarget, -) -> BackendAggregation { - BackendAggregation { - aggregation_id, - metric_name: selected.metric.clone(), - family: selected.family.clone(), - window_secs: selected.window_secs.unwrap_or(query.query_lookback_seconds), - spatial_filter: selected.spatial_filter.clone(), - grouping: selected - .group_by - .clone() - .unwrap_or_else(|| query.group_by_labels.clone()), - item_label: selected.item_label.clone(), - heap_update_mode: selected.parameters.get("weight_mode").and_then(|mode| { - match mode.as_str() { - Some("count") => Some("count"), - Some("value") => Some("value"), - Some("counter_delta") => Some("counter_delta"), - _ => None, - } - }), - aggregation_input: match target { - PhysicalDeploymentTarget::DistributedCollectors => AggregationInput::SketchEnvelope, - PhysicalDeploymentTarget::BackendLocalRemoteWrite => AggregationInput::Raw, - }, - } -} - -/// Convert the typed physical aggregation into the runtime's canonical -/// content-addressed materialization contract. This is the one conversion -/// shared by the physical compiler and the compatibility replanner; it does -/// not create a second registry or wire plan. -fn scoped_materialization( - aggregation: &BackendAggregation, - node: &SummaryNode, -) -> anyhow::Result { - let mut config = - aggregation_config_for_materialization(aggregation, asap_types::QueryLanguage::PromQl)?; - if !matches!(aggregation.aggregation_input, AggregationInput::Raw) { - return Ok(config); - } - let SummaryExpr::SummaryAgg { reduction, .. } = &node.expr else { - anyhow::bail!("materialization lacks SummaryAgg partition contract"); - }; - config.partitioning = Some(match reduction { - planner_types::pre_asap::Reduction::PerEntity => { - asap_types::sds::PopulationPartitioning::PerEntity - } - planner_types::pre_asap::Reduction::Reduce(_) => { - asap_types::sds::PopulationPartitioning::Grouped - } - }); - Ok(config) -} - -pub(crate) fn aggregation_config_for_materialization( - aggregation: &BackendAggregation, - language: asap_types::QueryLanguage, -) -> anyhow::Result { - use anyhow::Context as _; - let mut json = crate::emit::backend_wire::build_backend_aggregation_json(aggregation); - // The selected physical duration is authoritative. The legacy edge emitter's - // 5..60 second clamp must not silently change a backend materialization. - json["windowSize"] = serde_json::json!(aggregation.window_secs); - if language == asap_types::QueryLanguage::ClickHouseSql { - // Raw input does not imply PromQL's (start,end] convention. SQL bounds - // are normalized and bound explicitly by the SQL compiler. - json["parameters"] - .as_object_mut() - .expect("emitter parameters are an object") - .remove("promql_right_closed"); - } - let yaml = serde_yaml::to_value(json).context("convert physical aggregation fields")?; - asap_types::PrecomputeMaterialization::from_yaml_data( - &yaml, - None, - asap_types::QueryLanguage::PromQl, - ) - .context("build materialization from physical aggregation") -} - -fn materialization_consumers( - queries: &[QueryCompilationInput], - target: PhysicalDeploymentTarget, - composable: bool, -) -> Result>, CompileError> { - let mut consumers = BTreeMap::<_, BTreeSet<_>>::new(); - // Until logical lifecycle costing carries a derived-program identity, - // never combine unrelated programs under the legacy raw-state key. - let mut cohort_programs = - BTreeMap::>::new(); - for (index, query) in queries.iter().enumerate() { - let states = collect_selected_materializations(&query.selected_plan_root, composable) - .map_err(|reason| CompileError::Query { - query_id: query.query_id.clone(), - reason, - })?; - for state in states { - if composable - && state.window_secs.is_some_and(|window| { - !query - .window_realization_candidates - .iter() - .any(|candidate| candidate.window_secs == window) - }) - { - continue; - } - let config = scoped_materialization( - &physical_aggregation(query, &state, query.query_id.clone(), target), - &state.node, - )?; - let program = - immutable_materialization_sources(&state.node).map(|_| Rc::as_ptr(&state.node)); - if let Some(previous) = cohort_programs.insert(config.policy_fingerprint(), program) { - if previous != program && (previous.is_some() || program.is_some()) { - return Err(CompileError::Query { - query_id: query.query_id.clone(), - reason: - "distinct immutable programs require separate lifecycle cost cohorts" - .into(), - }); - } - } - consumers - .entry(config.policy_fingerprint()) - .or_default() - .insert(index); - } - } - Ok(consumers) -} - -fn shared_pane_origin_ms( - workload: Option<&QueryWorkload>, - consumer_indices: impl IntoIterator, - pane_width_ms: u64, -) -> Result, String> { - if pane_width_ms == 0 { - return Err("materialized pane width is zero".into()); - } - let pane_width = i64::try_from(pane_width_ms) - .map_err(|_| "materialized pane width exceeds runtime timestamp range".to_string())?; - let Some(workload) = workload else { - // Compatibility-only PhysicalCompilationRequest callers do not claim a certified - // phase. The read path rejects this binding and uses exact fallback. - return Ok(None); - }; - let entries = workload.entries().collect::>(); - let mut selected_phase = None; - for index in consumer_indices { - let entry = entries - .get(index) - .ok_or_else(|| format!("consumer index {index} is absent from QueryWorkload"))?; - let QueryRecurrence::Repeated(demand @ RepeatedDemand::FixedIntervalAt { .. }) = - &entry.recurrence - else { - return Err(format!( - "shared pane-only summary consumer {index} has unknown evaluation phase" - )); - }; - let layout = planner_types::post_asap::plan_pane_phase(demand, pane_width_ms) - .map_err(|error| format!("invalid pane phase for consumer {index}: {error:?}"))?; - let origin = layout.pane_origin_ms.ok_or_else(|| { - format!("shared pane-only summary consumer {index} has unknown evaluation phase") - })?; - let phase = origin.rem_euclid(pane_width); - if selected_phase.is_some_and(|selected| selected != phase) { - return Err(format!( - "shared pane-only summary consumers have conflicting phases {selected_phase:?} and {phase} for pane width {pane_width_ms}ms" - )); - } - selected_phase = Some(phase); - } - selected_phase - .ok_or_else(|| "materialized summary has no workload consumers".to_string()) - .map(Some) -} - -/// Collect every executable materialization leaf in the selected post-ASAP -/// graph. Readout context flows through merge nodes, so a graph such as -/// `Estimate(Merge(Agg(a), Agg(b)))` creates two physical bindings while the -/// serialized QueryPlan retains the merge edges. Unsupported operators are -/// intentionally not traversed: QueryPlan lowers them to an explicit exact -/// fallback node and no unused warm state is provisioned. -/// Admit per-entity raw state only when the installed partition contract and -/// scalar input evaluator preserve source rows. Composite updates still require -/// an executable maintenance evaluator; a partition flag cannot authorize them. -fn collect_selected_materializations( - node: &Rc, - composable: bool, -) -> Result, String> { - fn walk( - node: &Rc, - readout: Option<&SketchQuery>, - composable: bool, - inherited_grouping: Option>, - selected: &mut Vec, - ) -> Result<(), String> { - let grouping = if composable { - if let SummaryExpr::SummaryAgg { - reduction, child, .. - } = &node.expr - { - if let Some(keys) = reduction.group_keys() { - Some( - keys.keys() - .iter() - .map(|id| { - child - .schema - .fields - .get(*id) - .map(|field| field.name.clone()) - .ok_or_else(|| { - format!("unresolved producer grouping column {id}") - }) - }) - .collect::, _>>()?, - ) - } else { - inherited_grouping - } - } else { - inherited_grouping - } - } else { - None - }; - if let Some(source) = immutable_materialization_sources(node) { - for source in source { - walk(&source, None, composable, None, selected)?; - } - } - match &node.expr { - SummaryExpr::RelationalJoin { - left: values, - right: candidates, - kind: planner_types::pre_asap::JoinKind::Semi, - pruning: Some(_), - .. - } => { - walk(candidates, readout, composable, grouping.clone(), selected)?; - // Explicit external authoritative values do not need duplicate local state. - if !composable { - walk(values, readout, composable, grouping.clone(), selected)?; - } - } - SummaryExpr::ValueOperation { child, .. } => { - walk(child, readout, composable, grouping.clone(), selected)?; - } - SummaryExpr::RelationalJoin { left, right, .. } => { - walk(left, readout, composable, grouping.clone(), selected)?; - walk(right, readout, composable, grouping.clone(), selected)?; - } - SummaryExpr::BinaryOp { lhs, rhs, .. } - if composable || crate::query_plan::exact_value_executable(node) => - { - walk(lhs, readout, composable, grouping.clone(), selected)?; - walk(rhs, readout, composable, grouping.clone(), selected)?; - } - SummaryExpr::SummaryAgg { - child, - family: - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Sum, _), - .. - } if !matches!(child.expr, SummaryExpr::KeepPreAsap(_)) - && ((composable - && crate::query_plan::exact_accumulator_value_source(child).is_some()) - || crate::query_plan::exact_value_executable(node)) => - { - walk(child, readout, composable, grouping.clone(), selected)?; - } - SummaryExpr::SummaryAgg { child, .. } - if !matches!(child.expr, SummaryExpr::KeepPreAsap(_)) - && immutable_materialization_sources(node).is_none() => {} - SummaryExpr::SummaryAgg { - family: - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Count, _), - .. - } if !crate::query_plan::exact_value_executable(node) => {} - SummaryExpr::SummaryEstimate { - summary_input, - query, - } => walk( - summary_input, - Some(query), - composable, - grouping.clone(), - selected, - )?, - SummaryExpr::SummaryMerge { children, .. } => { - for child in children { - walk(child, readout, composable, grouping.clone(), selected)?; - } - } - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::Sketch(kind, _), - input, - .. - } => { - // Planner can model these families, but no backend state - // implementation exists. Leave an exact boundary unbound. - if matches!( - kind.algorithm(), - SketchAlgorithm::Kmv | SketchAlgorithm::Theta - ) { - return Ok(()); - } - if let Some(readout) = readout { - let mut parameters = sketch_params_json(kind.params()); - let mut item_label = None; - if matches!(readout, SketchQuery::TopK { .. }) { - use planner_types::post_asap::SummaryInputExpr; - item_label = match &input.item { - Some(SummaryInputExpr::Column( - planner_types::pre_asap::ColumnRef::Named(label), - )) => Some(label.clone()), - Some(SummaryInputExpr::Column( - planner_types::pre_asap::ColumnRef::Qualified { name, .. }, - )) => Some(name.clone()), - // Legacy/direct TopK plans did not carry an item - // projection. Keep their generic item-key behavior; - // typed Planner plans name the inner aggregate - // identity explicitly through SummaryUpdate.item. - _ => None, - }; - let mode = match &input.weight { - SummaryInputExpr::Constant(value) if *value == 1.0 => "count", - SummaryInputExpr::Column( - planner_types::pre_asap::ColumnRef::SampleValue, - ) => "value", - _ => return Err("unsupported TopK SummaryUpdate weight".into()), - }; - parameters["weight_mode"] = mode.into(); - } - let (metric, window_secs, spatial_filter) = match selected_input_contract(node) - { - Ok(contract) => contract, - Err(_) if composable => return Ok(()), - Err(error) => return Err(error), - }; - selected.push(SelectedMaterialization { - node: Rc::clone(node), - metric, - window_secs, - spatial_filter, - group_by: grouping.clone(), - item_label, - family: SummaryFamilyType::Sketch( - kind.clone(), - planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, - ), - algorithm: format!("{:?}", kind.algorithm()).to_ascii_lowercase(), - parameters, - }); - } - } - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(kind, params), - .. - } => { - let (metric, window_secs, spatial_filter) = match selected_input_contract(node) { - Ok(contract) => contract, - Err(_) if composable => return Ok(()), - Err(error) => return Err(error), - }; - selected.push(SelectedMaterialization { - node: Rc::clone(node), - metric, - window_secs, - spatial_filter, - group_by: grouping.clone(), - item_label: None, - family: SummaryFamilyType::ExactAggregate(kind.clone(), params.clone()), - algorithm: format!("{kind:?}").to_ascii_lowercase(), - parameters: Value::Object(Default::default()), - }); - } - SummaryExpr::BinaryOp { .. } - | SummaryExpr::KeepPreAsap(_) - | SummaryExpr::SummaryAgg { .. } - | SummaryExpr::SummaryJoin { .. } - | SummaryExpr::SummarySubtract { .. } - | SummaryExpr::SummaryDelete { .. } => {} - } - Ok(()) - } - - let mut selected = Vec::new(); - walk(node, None, composable, None, &mut selected)?; - if composable { - selected - .retain(|state| !has_unsafe_raw_entity_leaf(node, &[Rc::clone(&state.node)], false)); - } - if !selected.is_empty() { - validate_executable_subdag(node)?; - } - Ok(selected) -} - -pub(super) fn sketch_params_json(params: &planner_types::post_asap::SketchParams) -> Value { - use planner_types::post_asap::SketchParams as P; - match params { - P::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - } => json!({"heap_size": heap_size, "sketch_rows": sketch_rows, - "sketch_cols": sketch_cols, "layers": layers}), - P::Kll { k } => json!({"k": k}), - P::Cms { width, depth } => json!({"width": width, "depth": depth}), - P::Hll { precision } => json!({"precision": precision}), - P::DDSketch { alpha } => json!({"alpha": alpha}), - P::CmsWithHeap { - width, - depth, - heap_size, - } => json!({"width": width, "depth": depth, "heap_size": heap_size}), - P::Kmv { k } | P::Theta { k } => json!({"k": k}), - P::CountSketch { width, depth } => json!({"width": width, "depth": depth}), - P::CountSketchWithHeap { - width, - depth, - heap_size, - } => json!({"width": width, "depth": depth, "heap_size": heap_size}), - } -} - -fn stable_plan_id(materializations: &[CollectorMaterialization]) -> u64 { - use std::hash::{Hash, Hasher}; - let bytes = serde_json::to_vec(materializations).unwrap_or_default(); - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - bytes.hash(&mut hasher); - hasher.finish() -} - -fn stable_workload_plan_id( - materializations: &[CollectorMaterialization], - queries: &[QueryCompilationInput], -) -> u64 { - use std::hash::{Hash, Hasher}; - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - stable_plan_id(materializations).hash(&mut hasher); - for query in queries { - query.query_id.hash(&mut hasher); - query.query_string.hash(&mut hasher); - } - hasher.finish() -} - -/// Parser/executor frontend for the time-series physical compiler. SQL has its -/// own compilation input and must not silently enter this path. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum QueryFrontend { - PromQl, - MetricsQl, -} -impl QueryFrontend { - pub fn parse( - self, - query: &str, - accuracy: crate::types::AccuracyTarget, - ) -> Result { - crate::query_parser::parse_query_expr_canonical(query, accuracy).map_err(|error| match self - { - Self::PromQl => format!("frontend.promql: {error}"), - Self::MetricsQl => format!("victoriametrics.promql_subset: {error}"), - }) - } - pub fn compile( - self, - request: PhysicalCompilationRequest, - environment: PhysicalDeploymentContext, - ) -> Result { - DeploymentPlanCompiler.compile_for_frontend(request, environment, self) - } -} -#[cfg(test)] -pub(crate) mod tests { - use super::*; - - /// Multiple current-value quantiles and TopK limits share one maintained population. - #[test] - fn current_series_quantiles_and_topk_have_a_shared_executable_candidate() { - let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - snapshot.schema_version = 3; - snapshot.data_workload.data_ingestion_interval.value = Some(DurationMs(1_000)); - let template = snapshot.query_workload.repeating_queries.as_ref().unwrap()[0].clone(); - let queries = [ - "quantile by (job) (0.5, a)", - "quantile by (job) (0.9, a)", - "quantile by (job) (0.95, a)", - "quantile by (job) (0.99, a)", - "topk by (job) (1, a)", - "topk by (job) (5, a)", - ]; - snapshot.query_workload.repeating_queries = Some( - queries - .iter() - .map(|q| { - let mut entry = template.clone(); - entry.query = planner_types::workload::Query((*q).into()); - entry - }) - .collect(), - ); - let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let plans: Vec<_> = - super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) - .unwrap() - .into_iter() - .filter_map(|r| { - DeploymentPlanCompiler - .compile_promql(r, environment.clone()) - .ok() - }) - .collect(); - let plan = plans - .iter() - .find(|plan| { - plan.query_plan.entries.values().all(|entry| { - entry.nodes.values().any(|node| { - matches!(node, crate::query_plan::QueryPlanNode::Logical { - operator: crate::query_plan::query_time::QueryTimeOperator::CurrentSeries { .. }, .. - }) - }) - }) - }) - .expect("no shared current-series candidate"); - let mut populations = BTreeSet::new(); - for entry in plan.query_plan.entries.values() { - for node in entry.nodes.values() { - if let crate::query_plan::QueryPlanNode::Logical { - operator: - crate::query_plan::query_time::QueryTimeOperator::CurrentSeries { - population, - .. - }, - .. - } = node - { - // A longer scrape cadence cannot extend the declared selector horizon. - assert_eq!(population.max_input_lag_ms, 1_000); - assert_eq!(population.lookback_ms, 1_000); - assert_eq!(population.max_k, 5); - assert!(population.quantiles); - populations.insert(population.key()); - } - } - } - assert_eq!(populations.len(), 1); - let installed = serde_json::to_string(&plan.precompute_plan.executable_dags).unwrap(); - assert!( - !installed.contains("MaintainPopulation"), - "query-only population readout must not be executable maintenance" - ); - } - - // Physical lowering follows selected Planner IR, independent of catalog text. - #[test] - fn current_series_lowering_uses_selected_ir_not_query_text() { - let mut request = request("ir", "quantile(0.5, a)"); - let root = Rc::new( - crate::query_parser::parse_query_expr_canonical( - "quantile(0.5, a)", - AccuracyTarget::Exact, - ) - .unwrap(), - ); - let strategy = asap_aware_mapping::maintained_population::MaintainedPopulationStrategy::new( - std::slice::from_ref(&root), - ); - request.queries[0].selected_plan_root = strategy.candidate(&root).unwrap(); - let before = super::super::maintained_population::operator(&request, &request.queries[0]) - .unwrap() - .unwrap(); - request.queries[0].query_string = "quantile(0.99, b)".into(); - let after = super::super::maintained_population::operator(&request, &request.queries[0]) - .unwrap() - .unwrap(); - assert_eq!(before, after); - request.queries[0].selected_plan_root = - crate::planner_selection::keep_pre_asap(&root).unwrap(); - assert!( - super::super::maintained_population::operator(&request, &request.queries[0]) - .unwrap() - .is_none() - ); - } - - // A valid table-row DAG cannot be served by remote-write latest-series state. - #[test] - fn maintained_table_population_requires_a_compatible_executor() { - let mut request = request("sql", "quantile(0.5, a)"); - let mut root = crate::query_parser::parse_query_expr_canonical( - "quantile(0.5, a)", - AccuracyTarget::Exact, - ) - .unwrap(); - let planner_types::pre_asap::QueryExpr::Aggregate { child, .. } = &mut root else { - unreachable!() - }; - // SQL rows have no PromQL instant-selector time range. - if let QueryExpr::TimeRange { child: scan, .. } = child.as_ref() { - *child = Rc::clone(scan); - } - let planner_types::pre_asap::QueryExpr::Scan { source, schema, .. } = Rc::make_mut(child) - else { - unreachable!() - }; - *source = planner_types::pre_asap::Source::Table { - table_ref: "samples".into(), - }; - schema.closed = true; - let root = Rc::new(root); - let rule = asap_aware_mapping::maintained_population::MaintainedPopulationStrategy::new( - std::slice::from_ref(&root), - ); - let candidate = rule.candidate(&root).unwrap(); - planner_types::post_asap::compile_executable_dag(&candidate).unwrap(); - assert!(!super::super::maintained_population::supported_node( - &candidate - )); - request.queries[0].selected_plan_root = candidate; - let error = super::super::maintained_population::operator(&request, &request.queries[0]) - .unwrap_err(); - assert!(error.to_string().contains("row-update executor"), "{error}"); - } - - // Compiler preserves the Planner's conditional-average execution guard. - #[test] - fn temporal_average_lowers_with_finite_division_guard() { - let mut environment = environment(10_000); - environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.target_collector_ids.clear(); - let request = request("average", "avg_over_time(a[1m])"); - let plan = DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap(); - assert!( - plan.query_plan - .entries - .values() - .flat_map(|e| e.nodes.values()) - .any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::Logical { - operator: asap_types::query_plan::query_time::QueryTimeOperator::Binary { - operation: - asap_types::query_plan::query_time::BinaryOperation::FiniteDiv, - .. - }, - .. - } - )), - "{plan:#?}" - ); - } - - // The Planner's minimum state lowers without reconstructing direction from text. - #[test] - fn minimum_retains_its_typed_direction() { - let mut env = environment(10_000); - env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - env.target_collector_ids.clear(); - let mut input = request("minimum", "min_over_time(data[1m])"); - input.allow_mixed_summary_and_exact_execution = true; - let plan = DeploymentPlanCompiler.compile_promql(input, env).unwrap(); - assert_eq!(plan.precompute_plan.materializations.len(), 1); - assert_eq!( - plan.precompute_plan.materializations[0].aggregation_type, - asap_types::AggregationType::Min - ); - assert!(plan - .query_plan - .entries - .values() - .all(|entry| entry.nodes.values().any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ExactReadout { - readout: crate::query_plan::ExactReadout::Min, - .. - } - )))); - } - - // Every metric the plan materializes must carry a routing entry, or the - // backend falls through to its default engine and archive-shape queries - // miss. Publication passes this document straight to the data plane. - #[test] - fn compiled_plan_routes_every_materialized_metric() { - let plan = quoted_snapshot(planning_snapshot(), QueryFrontend::PromQl) - .compile_promql() - .unwrap(); - let routing = &plan.storage_routing; - assert_eq!(routing["default_engine"], "asap_query"); - let routed: std::collections::BTreeSet = routing["metrics"] - .as_array() - .expect("routing document carries a metrics array") - .iter() - .map(|entry| entry["name"].as_str().unwrap().to_string()) - .collect(); - assert!( - !routed.is_empty(), - "a plan with materializations emitted an empty routing table: {routing}" - ); - // Archive-shape claims are what the default engine cannot serve; a - // routing entry without them would silently widen the warm tier. - for entry in routing["metrics"].as_array().unwrap() { - let engines: Vec<&str> = entry["targets"] - .as_array() - .unwrap() - .iter() - .map(|t| t["engine"].as_str().unwrap()) - .collect(); - assert!( - engines.contains(&"asap_query"), - "metric {} lost its warm target: {entry}", - entry["name"] - ); - } - } - - // Complete deployment quotes must preserve one producer with two window readouts. - #[test] - fn complete_cost_selection_preserves_shared_sum_panes() { - let mut snapshot = planning_snapshot(); - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query("sum_over_time(a[1m]) / sum_over_time(a[10m])".into()); - entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let plan = quoted_snapshot(snapshot, crate::physical::compiler::QueryFrontend::PromQl) - .compile_promql() - .unwrap(); - assert!(plan.cost_comparison.is_some()); - assert_eq!(plan.precompute_plan.materializations.len(), 1); - let bindings = plan - .query_plan - .entries - .values() - .next() - .unwrap() - .materialization_bindings(); - assert_eq!(bindings[0].materialization, bindings[1].materialization); - assert_eq!( - bindings - .iter() - .filter_map(|b| b.readout_lookback_ms) - .collect::>(), - BTreeSet::from([60_000, 600_000]) - ); - } - - // Synthetic quotes exercise deployment selection in tests, never production defaults. - pub(crate) fn quoted_snapshot( - mut snapshot: BackendLocalPlanningInput, - frontend: QueryFrontend, - ) -> BackendLocalPlanningInput { - use super::super::workload_cost::{ - enumerate_exact_and_materialized_candidates, manifest, WorkloadCostEvidence, - WorkloadQuote, - }; - let (request, environment) = snapshot - .clone() - .into_physical_compilation_request() - .unwrap(); - let quotes = enumerate_exact_and_materialized_candidates(request) - .unwrap() - .into_iter() - .enumerate() - .filter_map(|(index, candidate)| { - let plan = if frontend == QueryFrontend::MetricsQl { - DeploymentPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) - } else { - DeploymentPlanCompiler.compile_promql(candidate.clone(), environment.clone()) - } - .ok()?; - let manifest = manifest(&plan, &candidate.queries).unwrap(); - Some(WorkloadQuote { - unit_costs: manifest - .components - .keys() - .map(|key| (key.clone(), if index == 0 { 1.0 } else { 1e12 })) - .collect(), - manifest, - executable: true, - }) - }) - .collect(); - snapshot.workload_cost_evidence = Some(WorkloadCostEvidence { - backend_revision: BACKEND_REVISION.into(), - planner_revision: PLANNER_REVISION.into(), - data_snapshot_id: "compiler-unit-fixture".into(), - model_version: "test-only-unit-costs".into(), - observed_at_unix_ms: environment.observed_at_unix_ms, - valid_for_ms: environment.max_evidence_age_ms, - quotes, - }); - snapshot - } - - // Issue workloads must expose executable maintained candidates under the schema-2 API. - #[test] - fn issue_701_702_temporal_workloads_have_warm_candidates() { - for text in [ - "avg_over_time(data[5m])", - "min_over_time(data[5m])", - "quantile_over_time(0.9,data[5m])", - ] { - let mut snapshot = planning_snapshot(); - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query(text.into()); - if !text.contains("quantile") { - entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - } - let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let candidates = - super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) - .unwrap(); - let mut reasons = vec![]; - assert!( - candidates.into_iter().any(|candidate| { - match DeploymentPlanCompiler.compile_promql(candidate, environment.clone()) { - Ok(plan) => { - !plan.precompute_plan.materializations.is_empty() - && plan - .query_plan - .entries - .values() - .all(|entry| !entry.materialization_bindings().is_empty()) - } - Err(error) => { - reasons.push(error.to_string()); - false - } - } - }), - "no warm candidate for {text}: {reasons:?}" - ); - } - } - - // Quantile rank error does not certify relative error of a quotient. - #[test] - fn uncertified_quantile_ratios_retain_exact_execution() { - for text in [ - "quantile_over_time(0.9,data[5m])/quantile_over_time(0.5,data[5m])", - "avg_over_time(data[5m])/quantile_over_time(0.5,data[5m])", - ] { - let mut snapshot = planning_snapshot(); - snapshot.query_workload.repeating_queries.as_mut().unwrap()[0].query = - Query(text.into()); - let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let plan = DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap(); - assert!(plan.precompute_plan.materializations.is_empty(), "{text}"); - assert!( - plan.query_plan - .entries - .values() - .all(|entry| entry.nodes.values().any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ExactFallback { .. } - ))), - "{text}" - ); - } - } - - /// Optional counter masks must retain the workload's mandatory sketch bindings. - #[test] - fn costed_mixed_workload_retains_sketches_and_counter_readouts() { - let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" - )) - .unwrap(); - let plan = quoted_snapshot(snapshot, crate::physical::compiler::QueryFrontend::PromQl) - .compile_promql() - .unwrap(); - assert!(!plan.precompute_plan.materializations.is_empty()); - for entry in plan.query_plan.entries.values() { - assert!(!entry.materialization_bindings().is_empty(), "{entry:#?}"); - } - } - - /// A schema marker cannot opt into a legacy deployment policy. - #[test] - fn only_current_snapshot_schema_is_accepted() { - let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - for version in [0, 1, 2, 4] { - let mut old = snapshot.clone(); - old.schema_version = version; - assert!(old - .clone() - .into_physical_compilation_request() - .unwrap_err() - .to_string() - .contains("only version 3")); - assert!(old.compile_promql().is_err()); - } - assert!(snapshot.into_physical_compilation_request().is_ok()); - } - - #[test] - fn installed_partition_must_match_the_bound_dag_reduction() { - let mut env = environment(10_000); - env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - env.target_collector_ids.clear(); - let mut plan = DeploymentPlanCompiler - .compile_promql(request("scope", "sum_over_time(m[1m])"), env) - .unwrap(); - let installed = plan - .precompute_plan - .executable_dags - .values_mut() - .next() - .unwrap(); - let mut dag = installed.document.decode().unwrap(); - let node = dag - .nodes - .iter_mut() - .find(|node| { - matches!( - node.payload, - planner_types::post_asap::ExecutableOperatorPayload::SummaryAgg { .. } - ) - }) - .unwrap(); - if let planner_types::post_asap::ExecutableOperatorPayload::SummaryAgg { - reduction, .. - } = &mut node.payload - { - *reduction = planner_types::pre_asap::Reduction::by(vec![]); - } - installed.document = asap_types::executable_plan::OwnedPostAsapDag::from_executable( - installed.document.query_id.clone(), - &dag, - ) - .unwrap(); - installed.document.schema_version = - asap_types::executable_plan::PRECOMPUTE_DAG_SCHEMA_VERSION; - assert!(plan - .precompute_plan - .validate() - .unwrap_err() - .to_string() - .contains("partition")); - } - - #[test] - fn raw_per_entity_state_carries_explicit_isolation() { - for query in [ - "sum_over_time(m[1m])", - "quantile_over_time(0.99, m[1m])", - // Planner's certified sum/count decomposition, including its division guard. - "avg_over_time(m[1m])", - ] { - let mut environment = environment(10_000); - environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.target_collector_ids.clear(); - let plan = DeploymentPlanCompiler - .compile_promql(request("per-entity", query), environment) - .unwrap(); - assert!( - plan.precompute_plan - .materializations - .iter() - .all(|state| state.partitioning - == Some(asap_types::sds::PopulationPartitioning::PerEntity)), - "{query} pooled source entities" - ); - } - for query in [ - "sum(sum_over_time(m[1m]))", - "sum by (job) (sum_over_time(m[1m]))", - ] { - let mut environment = environment(10_000); - environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.target_collector_ids.clear(); - let plan = DeploymentPlanCompiler - .compile_promql(request("reduced", query), environment) - .unwrap(); - assert!( - !plan.precompute_plan.materializations.is_empty(), - "{query} lost safe additive state" - ); - } - } - - #[test] - fn backend_local_materializes_counter_with_per_series_state() { - let request = request("counter", "sum(rate(m[1m]))"); - let mut environment = environment(10_000); - environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.target_collector_ids.clear(); - let plan = DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap(); - assert_eq!(plan.precompute_plan.materializations.len(), 1); - assert_eq!( - plan.precompute_plan.materializations[0].num_aggregates_to_retain, - Some(2), - "the explicitly selected one-minute pane plus a boundary pane is retained" - ); - assert!(plan - .query_plan - .entries - .values() - .all(|entry| entry.nodes.values().any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ExactReadout { - readout: crate::query_plan::ExactReadout::Rate, - .. - } - )))); - } - - #[test] - fn raw_counter_artifact_is_valid_for_backend_precompute() { - let plan = DeploymentPlanCompiler - .compile_promql(request("counter", "rate(m[1m])"), environment(10_000)) - .unwrap(); - plan.precompute_plan.validate().unwrap(); - let catalog = plan.summary_catalog.clone(); - let mut raw = plan.precompute_plan; - raw.ingest.protocol = IngestProtocol::PrometheusRemoteWriteV1; - raw.ingest.endpoint_path = "/api/v1/write".into(); - raw.ingest.timestamp_unit = TimestampUnit::UnixMilliseconds; - raw.ingest.require_plan_identity = false; - raw.ingest.require_stored_output_identity = false; - raw.ingest.require_registered_producer = false; - raw.producers.clear(); - raw.bind_catalog(&catalog).unwrap(); - raw.validate().unwrap(); - } - - // Counter fallback does not disable an independent safe summary in the same workload. - #[test] - fn counter_fallback_preserves_other_workload_summaries() { - let mut workload = request("counter", "sum(rate(m[1m]))"); - workload - .queries - .extend(request("gauge", "sum(sum_over_time(g[1m]))").queries); - let mut environment = environment(10_000); - environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.target_collector_ids.clear(); - let plan = DeploymentPlanCompiler - .compile_promql(workload, environment) - .unwrap(); - assert_eq!(plan.precompute_plan.materializations.len(), 2); - assert_eq!( - plan.query_plan - .lookup("sum(rate(m[1m]))") - .unwrap() - .materialization_bindings() - .len(), - 1 - ); - assert_eq!( - plan.query_plan - .lookup("sum(sum_over_time(g[1m]))") - .unwrap() - .materialization_bindings() - .len(), - 1 - ); - } - - // Capability normalization precedes candidate enumeration, avoiding duplicate exact quotes. - #[test] - fn counter_only_snapshot_has_distinct_local_and_native_cost_alternatives() { - let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query("rate(m[1m])".into()); - entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (request, _) = snapshot.into_physical_compilation_request().unwrap(); - assert_eq!( - super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) - .unwrap() - .len(), - 3 - ); - } - - // Count and value rankings must configure different state update contracts. - #[test] - fn temporal_topk_binds_planner_update_weight() { - for (query, mode) in [ - ("topk(1, sum_over_time(m[1m]))", "value"), - ("topk(1, count_over_time(m[1m]))", "count"), - ] { - let evidence = TopKMembershipEvidence { - selected_lower_bound: 101.0, - excluded_upper_bound: 100.0, - interval_failure_probability: 0.001, - observed_at_unix_ms: 9500, - source: "unit-fixture".into(), - }; - let request = request_with_evidence("topk", query, Some(evidence)).unwrap(); - let plan = DeploymentPlanCompiler - .compile_promql(request, environment(10000)) - .unwrap(); - assert_eq!(plan.precompute_plan.materializations.len(), 1, "{query}"); - assert_eq!( - plan.precompute_plan.materializations[0].parameters["weight_mode"], mode, - "{query}" - ); - } - } - - // The explicit rate-value frontier cannot be rebound as raw counter deltas. - #[test] - fn unsupported_rate_heap_uses_explicit_exact_route() { - let query = "topk(2, sum by (job) (rate(m[1m])))"; - let evidence = TopKMembershipEvidence { - selected_lower_bound: 101.0, - excluded_upper_bound: 100.0, - interval_failure_probability: 0.001, - observed_at_unix_ms: 9_500, - source: "unit-fixture".into(), - }; - let request = request_with_evidence("topk-rate", query, Some(evidence)).unwrap(); - let plan = DeploymentPlanCompiler - .compile_promql(request, environment(10_000)) - .unwrap(); - let entry = plan.query_plan.lookup(query).unwrap(); - assert!(matches!( - &entry.nodes[&entry.root], - crate::query_plan::QueryPlanNode::ExactFallback { .. } - )); - assert_eq!( - entry.fallback, - crate::query_plan::FallbackPolicy::ExactBackend - ); - assert_eq!(entry.canonical_query, query); - assert!(plan.precompute_plan.materializations.is_empty()); - } - - // Native exact values remain explicit; unsupported heaps publish no stored state. - #[test] - fn hybrid_rate_topk_preserves_the_original_exact_subquery() { - let query = "topk(2, sum by (job) (rate(m[1m])))"; - let evidence = TopKMembershipEvidence { - selected_lower_bound: 101.0, - excluded_upper_bound: 100.0, - interval_failure_probability: 0.001, - observed_at_unix_ms: 9_500, - source: "unit-fixture".into(), - }; - let mut request = request_with_evidence("topk-rate", query, Some(evidence)).unwrap(); - request.allow_mixed_summary_and_exact_execution = true; - let mut environment = environment(10_000); - environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.target_collector_ids.clear(); - - let plan = DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap(); - let entry = plan.query_plan.lookup(query).unwrap(); - use crate::query_plan::{query_time::QueryTimeOperator, QueryPlanNode}; - assert!(matches!( - &entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { n: 2, .. }, - .. - } - )); - assert_eq!( - entry - .nodes - .values() - .filter(|node| matches!(node, QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { query }, .. - } if query == "sum by (job) (rate(m[1m]))")) - .count(), - 1 - ); - assert!(entry.materialization_bindings().is_empty()); - assert!(plan.precompute_plan.materializations.is_empty()); - let artifact = plan.to_publication_artifact().unwrap(); - artifact.validate().unwrap(); - let encoded = serde_json::to_value(&artifact).unwrap(); - assert!(!encoded.to_string().contains("counter_delta")); - } - - #[test] - fn retained_footprint_rejects_eval_sized_cms_across_all_panes() { - let evidence = TopKMembershipEvidence { - selected_lower_bound: 101.0, - excluded_upper_bound: 100.0, - interval_failure_probability: 0.001, - observed_at_unix_ms: 9_500, - source: "unit-fixture".into(), - }; - let plan = DeploymentPlanCompiler - .compile_promql( - request_with_evidence( - "topk-rate", - "topk(2, count_over_time(m[1m]))", - Some(evidence), - ) - .unwrap(), - environment(10_000), - ) - .unwrap(); - let mut materializations = plan.precompute_plan.materializations; - let cms = materializations - .iter_mut() - .find(|materialization| { - materialization.aggregation_type - == asap_types::AggregationType::CountMinSketchWithHeap - }) - .unwrap(); - cms.parameters.remove("width"); - cms.parameters.remove("depth"); - cms.parameters.insert("w".into(), json!(524_288)); - cms.parameters.insert("d".into(), json!(7)); - cms.num_aggregates_to_retain = Some(80); - - let error = validate_retained_summary_footprint( - &materializations, - Some(5), - DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES, - ) - .unwrap_err(); - assert!(error.to_string().contains("exceeds budget")); - - let cms = materializations - .iter_mut() - .find(|materialization| { - materialization.aggregation_type - == asap_types::AggregationType::CountMinSketchWithHeap - }) - .unwrap(); - cms.parameters.insert("w".into(), json!(4_096)); - validate_retained_summary_footprint( - &materializations, - Some(5), - DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES, - ) - .unwrap(); - - let mut request = request("bounded", "sum(sum_over_time(m[1m]))"); - request.retained_summary_memory_budget_bytes = Some(1); - let error = DeploymentPlanCompiler - .compile_promql(request, environment(10_000)) - .unwrap_err(); - assert!(error.to_string().contains("retained summary footprint")); - } - - #[test] - fn snapshot_uses_retained_memory_default_and_canonical_override() { - let source = include_str!("../../../docs/examples/asapquery-planning-snapshot.json"); - let snapshot: BackendLocalPlanningInput = serde_json::from_str(source).unwrap(); - assert_eq!( - snapshot - .physical_inputs - .retained_summary_memory_budget_bytes, - DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES - ); - - let mut value: Value = serde_json::from_str(source).unwrap(); - value["implementation"]["max_retained_summary_bytes"] = json!(123_456); - let snapshot: BackendLocalPlanningInput = serde_json::from_value(value).unwrap(); - assert_eq!( - snapshot - .physical_inputs - .retained_summary_memory_budget_bytes, - 123_456 - ); - } - - /// Dataset changes alter persisted meaning; relocating the same input does not. - #[test] - fn dataset_identity_survives_binding_and_rejects_wrong_input() { - let compile = |env| { - DeploymentPlanCompiler - .compile_promql(request("q", "sum_over_time(m[1m])"), env) - .unwrap() - }; - let first = compile(environment(10_000)); - assert!(!first.summary_catalog.definitions.is_empty()); - let mut other = environment(10_000); - other.dataset_identity.namespace = "other-tenant".into(); - let second = compile(other); - assert_ne!( - first.summary_catalog.definitions.keys().collect::>(), - second - .summary_catalog - .definitions - .keys() - .collect::>() - ); - let mut relocated = environment(10_000); - relocated.target_collector_ids = vec!["relocated-source".into()]; - let relocated = compile(relocated); - assert_eq!( - first.summary_catalog.definitions, - relocated.summary_catalog.definitions - ); - let mut forged = first.precompute_plan.clone(); - forged.ingest.dataset_identity.as_mut().unwrap().namespace = "other-tenant".into(); - assert!(forged - .validate() - .unwrap_err() - .to_string() - .contains("dataset")); - forged.ingest.dataset_identity = None; - assert!(forged.validate().is_err()); - } - - fn environment(now: u64) -> PhysicalDeploymentContext { - PhysicalDeploymentContext { - dataset_identity: planner_types::post_asap::LogicalDatasetIdentity { - namespace: "test".into(), - dataset: "metrics".into(), - }, - target: PhysicalDeploymentTarget::DistributedCollectors, - target_collector_ids: vec!["edge-a".into(), "edge-b".into()], - capability_snapshot_id: "caps-7".into(), - observed_at_unix_ms: now, - max_evidence_age_ms: 60_000, - plan_version: 1, - activation_unix_ms: now, - expiry_unix_ms: None, - backend_compat: "asap-query-backend.v1".into(), - } - } - - fn request_with_evidence( - query_id: &str, - promql: &str, - evidence: Option, - ) -> Result { - let accuracy = AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }; - let parsed = crate::query_parser::parse_query_expr_canonical(promql, accuracy.clone()) - .expect("canonical query"); - let lifecycle = SummaryLifecyclePlanningInputs { - evaluation_interval_ms: 10_000, - ingestion_rate_per_second: 100.0, - evidence_observed_at_unix_ms: 9_500, - evidence_valid_for_ms: 60_000, - horizon_seconds: 300.0, - costs: LifecycleUnitCosts { - build: 10.0, - maintenance_per_update: 0.001, - read: 0.1, - retention_per_second: 0.001, - retirement: 1.0, - }, - }; - let post_asap = select_post_asap(&parsed, accuracy.clone(), &lifecycle, evidence.as_ref())?; - let mut evidence_by_query = HashMap::new(); - if let Some(evidence) = evidence { - evidence_by_query.insert(query_id.to_string(), evidence); - } - Ok(PhysicalCompilationRequest { - planner_selection_trace: Vec::new(), - canonical_roots: Vec::new(), - allow_mixed_summary_and_exact_execution: false, - enabled_materialization_keys: None, - query_workload: None, - data_workload: None, - queries: vec![QueryCompilationInput { - query_id: query_id.into(), - query_string: promql.into(), - selected_plan_root: post_asap, - legacy_query_source: Source::TimeSeries { metric: "m".into() }, - query_lookback_seconds: 60, - group_by_labels: vec![], - accuracy_target: accuracy, - summary_lifecycle_inputs: lifecycle, - window_realization_candidates: vec![WindowRealizationCandidate { - derived: false, - cohort_only: false, - realization_id: "collector-tumbling-v1".into(), - framework: SummaryWindowFramework::Tumbling, - window_secs: 60, - slide_secs: 60, - layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }, - cost: WindowRealizationCostQuote { - model_version: "test-cost-v1".into(), - workload_fingerprint: "test-workload".into(), - observed_at_unix_ms: 9_500, - valid_for_ms: 60_000, - horizon_seconds: 300.0, - cpu_cost: 1.0, - peak_memory_bytes: 1_024, - network_bytes: 512, - storage_bytes: 512, - source_scan_bytes: 0, - weighted_cost: 1.0, - }, - }], - materialization_runtime_policy: RuntimeRulePolicy::default(), - }], - topk_membership_evidence_by_query_id: evidence_by_query, - erp: None, - exact_composition_costs: HashMap::new(), - planner_revision: PLANNER_REVISION.into(), - scrape_interval_ms: None, - query_retention_margin_ms: 0, - retained_summary_memory_budget_bytes: None, - }) - } - - fn request(query_id: &str, promql: &str) -> PhysicalCompilationRequest { - request_with_evidence(query_id, promql, None).expect("post-ASAP selection") - } - - #[test] - fn unshared_immutable_nodes_do_not_share_legacy_raw_cost_cohort() { - let sum = request("sum", "quantile(0.9, sum_over_time(m[1m]))"); - let count = request("other", "quantile(0.5, sum_over_time(m[1m]))"); - let queries = vec![sum.queries[0].clone(), count.queries[0].clone()]; - let error = materialization_consumers( - &queries, - PhysicalDeploymentTarget::BackendLocalRemoteWrite, - true, - ) - .unwrap_err(); - assert!( - error - .to_string() - .contains("separate lifecycle cost cohorts"), - "{error}" - ); - } - - #[test] - fn immutable_nested_summary_keeps_actual_source_and_derived_bindings() { - let mut workload = request("nested", "quantile(0.9, sum_over_time(m[1m]))"); - workload.allow_mixed_summary_and_exact_execution = true; - let states = - collect_selected_materializations(&workload.queries[0].selected_plan_root, true) - .unwrap(); - assert_eq!(states.len(), 2, "source and consumer must both be selected"); - assert!(immutable_materialization_sources(&states[0].node).is_none()); - assert!(immutable_materialization_sources(&states[1].node).is_some()); - let mut deployment = environment(10_000); - deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.target_collector_ids.clear(); - let plan = DeploymentPlanCompiler - .compile_promql(workload, deployment) - .unwrap(); - assert_eq!(plan.precompute_plan.materializations.len(), 2); - let derived = plan - .precompute_plan - .materializations - .iter() - .find(|m| m.derived_input.is_some()) - .unwrap(); - let source = plan - .precompute_plan - .materializations - .iter() - .find(|m| m.derived_input.is_none()) - .unwrap(); - assert_eq!( - derived.derived_input.as_ref().unwrap().inputs, - BTreeSet::from([source.policy_fingerprint().into()]) - ); - let entry = plan.query_plan.entries.values().next().unwrap(); - assert!(entry.nodes.values().any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ReadMaterialization { .. } - ))); - assert!(!entry - .nodes - .values() - .any(|node| matches!(node, crate::query_plan::QueryPlanNode::ExactFallback { .. }))); - } - #[test] - fn immutable_two_sources_keep_actual_frontiers_and_bindings() { - let mut workload = request( - "nested", - "quantile(0.9, sum_over_time(m[1m]) + sum_over_time(n[1m]))", - ); - workload.allow_mixed_summary_and_exact_execution = true; - let states = - collect_selected_materializations(&workload.queries[0].selected_plan_root, true) - .unwrap(); - assert_eq!(states.len(), 3, "source and consumer must both be selected"); - assert!(immutable_materialization_sources(&states[0].node).is_none()); - assert!(immutable_materialization_sources(&states[2].node).is_some()); - let mut deployment = environment(10_000); - deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.target_collector_ids.clear(); - let plan = DeploymentPlanCompiler - .compile_promql(workload, deployment) - .unwrap(); - assert_eq!(plan.precompute_plan.materializations.len(), 3); - assert!(plan - .precompute_plan - .materializations - .iter() - .all(|config| config.population_key_encoding - == asap_types::PopulationKeyEncoding::CanonicalLabelsV1)); - let mut mixed = plan.precompute_plan.clone(); - mixed - .materializations - .iter_mut() - .find(|config| config.derived_input.is_none()) - .unwrap() - .population_key_encoding = asap_types::PopulationKeyEncoding::LegacyDelimited; - assert!( - mixed.validate().is_err(), - "canonical target cannot use legacy source identity" - ); - let derived = plan - .precompute_plan - .materializations - .iter() - .find(|m| m.derived_input.is_some()) - .unwrap(); - let sources = plan - .precompute_plan - .materializations - .iter() - .filter(|m| m.derived_input.is_none()) - .map(|m| m.policy_fingerprint().into()) - .collect::>(); - assert_eq!(derived.derived_input.as_ref().unwrap().inputs, sources); - assert_eq!(sources.len(), 2); - let entry = plan.query_plan.entries.values().next().unwrap(); - assert!(entry.nodes.values().any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ReadMaterialization { .. } - ))); - assert!(!entry - .nodes - .values() - .any(|node| matches!(node, crate::query_plan::QueryPlanNode::ExactFallback { .. }))); - } - - /// An externally supplied unknown guarantee must not bypass global selection. - #[test] - fn supplied_uncertified_readout_is_rejected() { - use asap_aware_mapping::{ - Replacement, ReplacementStrategy, SketchAlgorithmStrategy, TargetSubDAG, - }; - let mut workload = request("unknown", "distinct_over_time(m[1m])"); - let root = Rc::new( - crate::query_parser::parse_query_expr_canonical( - "distinct_over_time(m[1m])", - AccuracyTarget::Epsilon(0.05), - ) - .unwrap(), - ); - let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(0.05)); - workload.queries[0].selected_plan_root = SketchAlgorithmStrategy::new(&model) - .replacements(&TargetSubDAG::new(&root)) - .into_iter() - .find_map(|candidate| { - if !candidate.has_missing_accuracy_evidence() { - return None; - } - match candidate.replacement { - Replacement::Summary(node) => Some(node), - _ => None, - } - }) - .expect("unknown HLL candidate stays inspectable"); - let result = DeploymentPlanCompiler.compile_metricsql(workload, environment(10_000)); - assert!( - matches!(result, Err(CompileError::Query { reason, .. }) if reason.contains("no certified accuracy guarantee")) - ); - } - - /// HLL without a confidence proof remains exact under canonical selection. - #[test] - fn uncertified_distinct_range_remains_exact() { - let mut deployment = environment(10_000); - deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.target_collector_ids.clear(); - let mut workload = request("distinct", "distinct_over_time(m{job=\"api\"}[1m])"); - let query = &mut workload.queries[0]; - // HLL's modeled RSE does not certify a failure probability. - query.accuracy_target = AccuracyTarget::Epsilon(0.05); - let parsed = crate::query_parser::parse_query_expr_canonical( - &query.query_string, - query.accuracy_target.clone(), - ) - .unwrap(); - query.selected_plan_root = select_post_asap( - &parsed, - query.accuracy_target.clone(), - &query.summary_lifecycle_inputs, - None, - ) - .unwrap(); - let plan = DeploymentPlanCompiler - .compile_metricsql(workload, deployment) - .unwrap(); - assert!(plan.precompute_plan.materializations.is_empty()); - assert!(matches!( - plan.query_plan - .entries - .values() - .next() - .unwrap() - .nodes - .values() - .next() - .unwrap(), - crate::query_plan::QueryPlanNode::ExactFallback { .. } - )); - } - - /// Mixed execution does not authorize an uncertified cardinality sketch. - #[test] - fn uncertified_cardinality_is_not_materialized() { - let mut deployment = environment(10_000); - deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.target_collector_ids.clear(); - let mut workload = request("confidence", "distinct_over_time(m[1m])"); - workload.allow_mixed_summary_and_exact_execution = true; - let result = DeploymentPlanCompiler.compile_metricsql(workload, deployment); - assert!(result.unwrap().precompute_plan.materializations.is_empty()); - } - - #[test] - fn snapshot_metricsql_entry_uses_the_shared_serving_language_contract() { - let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" - )) - .unwrap(); - let plan = quoted_snapshot( - snapshot, - crate::physical::compiler::QueryFrontend::MetricsQl, - ) - .compile_metricsql() - .unwrap(); - assert!(!plan.query_plan.entries.is_empty()); - assert!(plan - .query_plan - .entries - .values() - .all(|entry| entry.language == crate::query_plan::QueryLanguage::MetricsQl)); - } - - #[test] - fn duplicate_query_ids_cannot_overwrite_installed_dag_documents() { - let mut workload = request("shared-id", "max_over_time(a[1m])"); - let mut second = workload.queries[0].clone(); - second.query_string = "max_over_time(b[1m])".into(); - workload.queries.push(second); - - let error = DeploymentPlanCompiler - .compile_promql(workload, environment(10_000)) - .unwrap_err(); - assert!(matches!( - error, - CompileError::Snapshot(reason) - if reason == "planning query IDs must be unique within a plan generation" - )); - } - - #[test] - fn metricsql_counter_readouts_remain_external_exact() { - for text in [ - "rate(counter_probe{case=\"reset\"}[5s])", - "increase(counter_probe{case=\"reset\"}[5s])", - ] { - let mut workload = request("counter", text); - workload.allow_mixed_summary_and_exact_execution = true; - let mut deployment = environment(10_000); - deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.target_collector_ids.clear(); - let plan = DeploymentPlanCompiler - .compile_metricsql(workload, deployment) - .unwrap(); - assert!(plan.precompute_plan.materializations.is_empty()); - let entry = plan.query_plan.entries.values().next().unwrap(); - assert!(entry.materialization_bindings().is_empty()); - assert_eq!(entry.language, crate::query_plan::QueryLanguage::MetricsQl); - } - } - - #[test] - fn metricsql_counter_gate_rejects_mixed_input_snapshots() { - let mut workload = request("mixed", "max_over_time(m[1m]) + rate(m[1m])"); - workload.allow_mixed_summary_and_exact_execution = true; - let mut deployment = environment(10_000); - deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.target_collector_ids.clear(); - let Err(error) = DeploymentPlanCompiler.compile_metricsql(workload, deployment) else { - panic!("mixed local/external snapshots must fail deployment binding"); - }; - assert!( - error.to_string().contains("common snapshot proof"), - "{error}" - ); - } - - #[test] - fn metricsql_compilation_publishes_a_language_tagged_query_entry() { - let query = "mad_over_time(m[1m])"; - let mut workload = request("vm-q", "last_over_time(m[1m])"); - let accuracy = workload.queries[0].accuracy_target.clone(); - let canonical = - crate::query_parser::parse_query_expr_canonical(query, accuracy.clone()).unwrap(); - workload.queries[0].query_string = query.into(); - workload.queries[0].selected_plan_root = - crate::planner_selection::keep_pre_asap(&canonical).unwrap(); - let plan = DeploymentPlanCompiler - .compile_metricsql(workload, environment(10_000)) - .unwrap(); - let identity = canonical_promql(query).unwrap(); - let entry = plan - .query_plan - .lookup_canonical(crate::query_plan::QueryLanguage::MetricsQl, &identity) - .unwrap(); - assert_eq!(entry.query_id, "vm-q"); - assert_eq!(entry.language, crate::query_plan::QueryLanguage::MetricsQl); - } - - #[test] - fn hybrid_erp_capability_miss_preserves_exact_subtree() { - let mut workload = request("q", "quantile_over_time(0.9, m[1m])"); - let roots = vec![Rc::new( - crate::query_parser::parse_query_expr_canonical( - &workload.queries[0].query_string, - workload.queries[0].accuracy_target.clone(), - ) - .unwrap(), - )]; - let erp = super::super::erp::ErpPlanningInput { - artifact: asap_aware_mapping::erp::ErpArtifact { - schema_version: asap_aware_mapping::erp::ERP_SCHEMA_VERSION, - producer_version: "bench-rev".into(), - records: vec![], - }, - distribution: serde_json::json!({"synthetic":{"kind":"zipf"}}), - implementation: None, - error_metric: "relative_error".into(), - min_trials: 10, - expected_updates: 1000.0, - expected_queries: 10.0, - expected_merges: 0.0, - retention_seconds: 60.0, - cpu_weight: 1.0, - byte_second_weight: 1e-9, - mode: super::super::erp::ErpAccuracyMode::Hybrid, - observed_shape: None, - observed_populations: None, - resolved_data_descriptor: None, - observed_shape_source: None, - shape_match: None, - runtime: super::super::erp::ErpRuntimeCapabilities { - allowed_algorithms: vec![SketchAlgorithm::Hll], - max_memory_bytes: None, - }, - }; - select_logical_roots_with_error_resource_profiles( - &mut workload.queries, - roots, - &workload.topk_membership_evidence_by_query_id, - &workload.exact_composition_costs, - Some(&erp), - ) - .unwrap(); - assert!(matches!( - workload.queries[0].selected_plan_root.expr, - SummaryExpr::KeepPreAsap(_) - )); - } - - // Frozen sketch-bench output exercises the same wire schema on every CI run. - #[test] - fn measured_erp_without_failure_probability_uses_exact_fallback() { - use super::super::erp::{ - ErpAccuracyMode, ErpParameterDecision, ErpPlanningInput, ErpRuntimeCapabilities, - }; - - let artifact: asap_aware_mapping::erp::ErpArtifact = - serde_json::from_str(include_str!("../../tests/fixtures/erp-kll-measured.json")) - .unwrap(); - let row = artifact - .records - .first() - .expect("nonempty measured artifact"); - let erp = ErpPlanningInput { - distribution: row.distribution.clone(), - implementation: Some("lib".into()), - artifact, - error_metric: "max_rank_err".into(), - min_trials: 10, - expected_updates: 1000.0, - expected_queries: 10.0, - expected_merges: 0.0, - retention_seconds: 60.0, - cpu_weight: 1.0, - byte_second_weight: 1e-9, - mode: ErpAccuracyMode::Hybrid, - observed_shape: None, - observed_populations: None, - resolved_data_descriptor: None, - observed_shape_source: None, - shape_match: None, - runtime: ErpRuntimeCapabilities { - allowed_algorithms: vec![SketchAlgorithm::Kll], - max_memory_bytes: None, - }, - }; - assert!( - matches!( - erp.select(SketchAlgorithm::Kll, 0.06, SketchParams::Kll { k: 269 }), - ErpParameterDecision::Empirical { - params: SketchParams::Kll { k: 32 }, - .. - } - ), - "the measured row must pass adapter selection before testing the planner" - ); - - let mut workload = request("q", "quantile_over_time(0.9, m[1m])"); - workload.queries[0].accuracy_target = AccuracyTarget::Epsilon(0.06); - let root = Rc::new( - crate::query_parser::parse_query_expr_canonical( - &workload.queries[0].query_string, - workload.queries[0].accuracy_target.clone(), - ) - .unwrap(), - ); - select_logical_roots_with_error_resource_profiles( - &mut workload.queries, - vec![root], - &workload.topk_membership_evidence_by_query_id, - &workload.exact_composition_costs, - Some(&erp), - ) - .unwrap(); - - assert!(matches!( - workload.queries[0].selected_plan_root.expr, - SummaryExpr::KeepPreAsap(_) - )); - let plan = DeploymentPlanCompiler - .compile_promql(workload, environment(10000)) - .unwrap(); - assert!(plan.precompute_plan.materializations.is_empty()); - } - - fn measured_exact_composition_rows( - expr: &QueryExpr, - observed_at_unix_ms: u64, - ) -> Vec { - fn visit( - expr: &QueryExpr, - observed_at_unix_ms: u64, - rows: &mut Vec, - ) { - use planner_types::post_asap::ExactOperation; - match expr { - QueryExpr::Aggregate { - reduction, - measures, - output_names, - having, - child, - } => { - rows.push(ExactCompositionCostEvidence { - target: serde_json::to_value(expr).unwrap(), - operation: serde_json::to_value(ExactOperation::Aggregate { - reduction: reduction.clone(), - measures: measures.clone(), - output_names: output_names.clone(), - having: having.clone(), - }) - .unwrap(), - placement: - crate::physical::post_asap::cost_model::ExactOperationPlacement::Read, - expected_input_rows: 100.0, - expected_output_rows: 10.0, - exact_cpu_ns_per_row: 2.0, - summary_maintenance_cpu_ns_per_update: 3.0, - summary_read_cpu_ns: 20.0, - update_rate_per_second: 100.0, - evaluation_rate_per_second: 0.1, - raw_recompute_cpu_ns: 100_000.0, - observed_peak_memory_bytes: 4096, - data_snapshot_id: "topk-planning-fixture".into(), - model_version: "measured-test-v1".into(), - observed_at_unix_ms, - valid_for_ms: 60_000, - }); - visit(child, observed_at_unix_ms, rows); - } - QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::TimeRange { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Project { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - | QueryExpr::TimeShift { child, .. } - | QueryExpr::PromqlScalarBridge(child) - | QueryExpr::PromqlVectorFromScalar(child) - | QueryExpr::PromqlScalarFromVector(child) - | QueryExpr::PromqlRelabel { child, .. } - | QueryExpr::PromqlSeriesSample { child, .. } - | QueryExpr::Dedup { child, .. } => visit(child, observed_at_unix_ms, rows), - _ => {} - } - } - let mut rows = Vec::new(); - visit(expr, observed_at_unix_ms, &mut rows); - rows - } - - #[test] - fn sum_rate_uses_summary_child_only_with_measured_composition_costs() { - let promql = "sum by (job) (rate(m[1m]))"; - let mut with_evidence = request("topk-rate", promql); - with_evidence.allow_mixed_summary_and_exact_execution = true; - let root = Rc::new( - crate::query_parser::parse_query_expr_canonical( - promql, - with_evidence.queries[0].accuracy_target.clone(), - ) - .unwrap(), - ); - with_evidence.exact_composition_costs.insert( - "topk-rate".into(), - measured_exact_composition_rows(&root, 9_500), - ); - select_logical_roots_for_queries( - &mut with_evidence.queries, - vec![root], - &with_evidence.topk_membership_evidence_by_query_id, - &with_evidence.exact_composition_costs, - ) - .unwrap(); - let mut backend = environment(10_000); - backend.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - backend.target_collector_ids.clear(); - let plan = DeploymentPlanCompiler - .compile_promql(with_evidence, backend) - .unwrap(); - assert!( - !plan.summary_catalog.outputs.is_empty(), - "measured exact-composition evidence must expose the rate child as a SummaryStore binding" - ); - } - - #[test] - fn sum_rate_without_composition_costs_does_not_invent_a_composition_cost() { - let promql = "sum by (job) (rate(m[1m]))"; - let mut unavailable = request("topk-rate", promql); - unavailable.allow_mixed_summary_and_exact_execution = true; - let root = Rc::new( - crate::query_parser::parse_query_expr_canonical( - promql, - unavailable.queries[0].accuracy_target.clone(), - ) - .unwrap(), - ); - select_logical_roots_for_queries( - &mut unavailable.queries, - vec![root], - &unavailable.topk_membership_evidence_by_query_id, - &unavailable.exact_composition_costs, - ) - .unwrap(); - let mut backend = environment(10_000); - backend.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - backend.target_collector_ids.clear(); - let plan = DeploymentPlanCompiler - .compile_promql(unavailable, backend) - .unwrap(); - // Planner 54f can realize this particular shape directly as an exact - // counter readout plus a query-time reduce; it does not require an - // ExactComposition candidate. The absence of evidence must therefore - // leave that direct legal path intact rather than inventing a composed - // cost or forcing an exact fallback. - assert!(!plan.summary_catalog.outputs.is_empty()); - let entry = plan - .query_plan - .entries - .values() - .find(|entry| entry.query_id == "topk-rate") - .unwrap(); - assert!(entry - .nodes - .values() - .any(|node| matches!(node, crate::query_plan::QueryPlanNode::ExactReadout { .. }))); - } - - fn phase_workload(phases: &[u64]) -> QueryWorkload { - QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some( - phases - .iter() - .enumerate() - .map(|(index, phase)| RepeatingEntry { - query: Query(format!("q{index}")), - demand: RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(10_000), - evaluation_phase: planner_types::workload::TimestampMs(*phase), - }, - requirements: QueryRequirements::default(), - predictability: Predictability::Predictable { known_at: None }, - time_selection: TimeSelection::default(), - }) - .collect(), - ), - } - } - - #[test] - fn shared_summary_requires_one_consumer_phase() { - let aligned = phase_workload(&[7_000, 67_000]); - assert_eq!( - shared_pane_origin_ms(Some(&aligned), [0, 1], 60_000).unwrap(), - Some(7_000) - ); - - let conflicting = phase_workload(&[7_000, 8_000]); - assert!(shared_pane_origin_ms(Some(&conflicting), [0, 1], 60_000) - .unwrap_err() - .contains("conflicting phases")); - - let mut unknown = phase_workload(&[7_000]); - unknown.repeating_queries.as_mut().unwrap()[0].demand = - RepeatedDemand::FixedInterval(RepetitionInterval(10_000)); - assert!(shared_pane_origin_ms(Some(&unknown), [0], 60_000) - .unwrap_err() - .contains("unknown evaluation phase")); - } - - #[test] - fn query_staleness_extends_retention_without_changing_readout_lookback() { - let panes = asap_types::WindowMaterializationLayout::Pane { pane_secs: 30 }; - assert_eq!( - retained_state_count(6 * 60 * 60_000, 19 * 60_000, 30_000, &panes), - 759 - ); - assert_eq!( - retained_state_count(6 * 60 * 60_000, 0, 30_000, &panes), - 721 - ); - assert_eq!( - retained_state_count( - 6 * 60 * 60_000, - 19 * 60_000, - 30_000, - &asap_types::WindowMaterializationLayout::FullWindow, - ), - 39 - ); - } - - // Both production adapters preserve canonical root identity and select the - // whole evidence-free cohort, rather than independently binding roots. - #[test] - fn shared_selection_adapter_preserves_query_mapping() { - let mut workload = request("q90", "quantile_over_time(0.9, m[1m])"); - workload - .queries - .extend(request("q99", "quantile_over_time(0.99, m[1m])").queries); - let roots = workload - .queries - .iter() - .map(|query| { - Rc::new( - crate::query_parser::parse_query_expr_canonical( - &query.query_string, - query.accuracy_target.clone(), - ) - .unwrap(), - ) - }) - .collect(); - select_logical_roots_for_queries( - &mut workload.queries, - roots, - &workload.topk_membership_evidence_by_query_id, - &workload.exact_composition_costs, - ) - .unwrap(); - let bundle = DeploymentPlanCompiler - .compile_promql(workload, environment(10000)) - .unwrap(); - assert_eq!(bundle.query_plan.entries.len(), 2); - assert_eq!(bundle.collector_plans[0].materializations.len(), 1); - assert_eq!( - bundle - .query_plan - .entries - .values() - .map(|entry| entry.query_id.as_str()) - .collect::>(), - BTreeSet::from(["q90", "q99"]) - ); - } - - // A broken input mapping must be rejected, never silently drop a root. - #[test] - fn shared_selection_rejects_incomplete_root_mapping() { - let mut workload = request("q", "quantile_over_time(0.9, m[1m])"); - assert!(select_logical_roots_for_queries( - &mut workload.queries, - vec![], - &workload.topk_membership_evidence_by_query_id, - &workload.exact_composition_costs, - ) - .is_err()); - } - - // Adding another readout adds recurring reads, not another update stream. - #[test] - fn joint_lifecycle_charges_shared_updates_once() { - let baseline = DeploymentPlanCompiler - .compile_promql( - request("q90", "quantile_over_time(0.9, m[1m])"), - environment(10000), - ) - .unwrap(); - let mut workload = request("q90", "quantile_over_time(0.9, m[1m])"); - let mut second = request("q99", "quantile_over_time(0.99, m[1m])") - .queries - .remove(0); - second.summary_lifecycle_inputs.evaluation_interval_ms = 20000; - workload.queries.push(second); - let shared = DeploymentPlanCompiler - .compile_promql(workload, environment(10000)) - .unwrap(); - assert_eq!(shared.lifecycle_estimates.len(), 1); - let estimate = &shared.lifecycle_estimates[0]; - assert_eq!(estimate.consumer_query_ids, vec!["q90", "q99"]); - assert_eq!(estimate.expected_reads, 45.0); - assert_eq!(estimate.expected_updates, 30000.0); - assert!((estimate.lifecycle_cost - 45.8).abs() < 1e-9); - assert_eq!( - estimate.expected_updates, - baseline.lifecycle_estimates[0].expected_updates - ); - assert!( - (estimate.lifecycle_cost - baseline.lifecycle_estimates[0].lifecycle_cost - 1.5).abs() - < 1e-9 - ); - } - - // Unknown joint provenance cannot be replaced by whichever root came first. - #[test] - fn shared_lifecycle_rejects_conflicting_source_evidence() { - let mut workload = request("q90", "quantile_over_time(0.9, m[1m])"); - let mut second = request("q99", "quantile_over_time(0.99, m[1m])") - .queries - .remove(0); - second.summary_lifecycle_inputs.ingestion_rate_per_second = 200.0; - workload.queries.push(second); - assert!(matches!( - DeploymentPlanCompiler.compile_promql(workload, environment(10000)), - Err(CompileError::Lifecycle { .. }) - )); - } - - #[test] - fn shared_materialization_is_emitted_once_for_every_runtime() { - for target in [ - PhysicalDeploymentTarget::DistributedCollectors, - PhysicalDeploymentTarget::BackendLocalRemoteWrite, - ] { - let (first, second) = if target == PhysicalDeploymentTarget::BackendLocalRemoteWrite { - ("sum(rate(m[1m]))", "sum by (job) (rate(m[1m]))") - } else { - ( - "quantile_over_time(0.90, m[1m])", - "quantile_over_time(0.99, m[1m])", - ) - }; - let mut workload = request("first", first); - workload.queries.extend(request("second", second).queries); - let mut env = environment(10_000); - env.target = target; - if target == PhysicalDeploymentTarget::BackendLocalRemoteWrite { - env.target_collector_ids.clear(); - } - let bundle = DeploymentPlanCompiler - .compile_promql(workload, env) - .expect("shared compile"); - assert_eq!(bundle.query_plan.entries.len(), 2); - assert_eq!(bundle.summary_catalog.outputs.len(), 1); - assert_eq!(bundle.precompute_plan.materializations.len(), 1); - assert_eq!(bundle.precompute_plan.schemas.len(), 1); - let bindings = bundle - .query_plan - .entries - .values() - .map(|entry| entry.materialization_bindings()[0].materialization) - .collect::>(); - assert_eq!(bindings.len(), 1); - for collector in &bundle.collector_plans { - assert_eq!(collector.materializations.len(), 1); - assert!(bindings.contains(&collector.materializations[0].materialization)); - } - } - } - - #[test] - fn adding_shared_consumer_changes_plan_identity() { - let workload = request("q90", "quantile_over_time(0.90, m[1m])"); - let one = DeploymentPlanCompiler - .compile_promql(workload, environment(10_000)) - .unwrap(); - let mut workload = request("q90", "quantile_over_time(0.90, m[1m])"); - workload - .queries - .extend(request("q99", "quantile_over_time(0.99, m[1m])").queries); - let two = DeploymentPlanCompiler - .compile_promql(workload, environment(10_000)) - .unwrap(); - assert_ne!(one.envelope.plan_id, two.envelope.plan_id); - assert_eq!(two.collector_plans[0].materializations.len(), 1); - } - - #[test] - fn different_sources_do_not_share_materializations() { - let mut workload = request("qm", "quantile_over_time(0.90, m[1m])"); - let mut other = request("qn", "quantile_over_time(0.90, n[1m])"); - other.queries[0].legacy_query_source = Source::TimeSeries { metric: "n".into() }; - workload.queries.extend(other.queries); - let bundle = DeploymentPlanCompiler - .compile_promql(workload, environment(10_000)) - .unwrap(); - assert_eq!(bundle.summary_catalog.outputs.len(), 2); - assert_eq!(bundle.precompute_plan.materializations.len(), 2); - for collector in &bundle.collector_plans { - assert_eq!(collector.materializations.len(), 2); - } - } - - #[test] - fn rate_and_increase_keep_planner_families_distinct() { - let mut workload = request("rate", "rate(m[1m])"); - workload - .queries - .extend(request("increase", "increase(m[1m])").queries); - let bundle = DeploymentPlanCompiler - .compile_promql(workload, environment(10_000)) - .unwrap(); - assert_eq!(bundle.query_plan.entries.len(), 2); - assert_eq!(bundle.precompute_plan.materializations.len(), 2); - for collector in &bundle.collector_plans { - let algorithms: std::collections::BTreeSet<_> = collector - .materializations - .iter() - .map(|materialization| materialization.algorithm.as_str()) - .collect(); - assert_eq!(algorithms, ["increase", "rate"].into()); - } - } - - #[test] - fn shared_materialization_rejects_conflicting_deployment_contracts() { - let mut workload = request("q90", "quantile_over_time(0.90, m[1m])"); - let mut other = request("q99", "quantile_over_time(0.99, m[1m])"); - other.queries[0].window_realization_candidates[0].realization_id = - "another-implementation".into(); - workload.queries.extend(other.queries); - let error = DeploymentPlanCompiler - .compile_promql(workload, environment(10_000)) - .expect_err("conflicting shared state must fail before publication"); - assert!(error - .to_string() - .contains("conflicting deployment contracts")); - } - - #[test] - fn exact_dashboard_preserves_distinct_sum_and_count_producers() { - let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let entries = snapshot.query_workload.repeating_queries.as_mut().unwrap(); - entries[0].query = Query("sum by (service) (sum_over_time(m[1m]))".into()); - entries[0].requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let mut mean = entries[0].clone(); - mean.query = Query( - "sum by (service) (sum_over_time(m[1m])) / sum by (service) (count_over_time(m[1m]))" - .into(), - ); - entries.push(mean); - let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let bundle = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - assert_eq!(bundle.precompute_plan.materializations.len(), 2); - assert_eq!(bundle.query_plan.entries.len(), 2); - for entry in bundle.query_plan.entries.values() { - assert!( - !entry.nodes.values().any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ExactFallback { .. } - )), - "{entry:?}" - ); - assert!(!entry.materialization_bindings().is_empty()); - } - assert!(bundle - .query_plan - .entries - .values() - .any(|entry| entry.nodes.values().any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::Logical { - operator: crate::query_plan::query_time::QueryTimeOperator::Binary { .. }, - .. - } - )))); - } - - #[test] - fn invalid_unselected_candidate_is_canonicalized_to_exact_fallback() { - let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = - Query("sum by (service) (sum_over_time(m[1m]) / count_over_time(m[1m]))".into()); - entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (mut request, _) = snapshot.into_physical_compilation_request().unwrap(); - // Exercise the invalid externally supplied graph, independent of the - // global selector (which now already chooses an exact fallback). - use asap_aware_mapping::ReplacementStrategy; - let candidates = asap_aware_mapping::SketchAlgorithmStrategy::new( - &ControlPlaneCostModel::new(AccuracyTarget::Exact), - ) - .replacements(&asap_aware_mapping::TargetSubDAG::new( - &request.canonical_roots[0], - )); - request.queries[0].selected_plan_root = candidates - .into_iter() - .find_map(|candidate| match candidate.replacement { - asap_aware_mapping::Replacement::Summary(node) - if !matches!(node.expr, SummaryExpr::KeepPreAsap(_)) => - { - Some(node) - } - _ => None, - }) - .expect("invalid summary fixture"); - assert!(!matches!( - request.queries[0].selected_plan_root.expr, - SummaryExpr::KeepPreAsap(_) - )); - - preserve_invalid_exact_fallback_roots(&mut request.queries, &request.canonical_roots, true) - .unwrap(); - - assert!(matches!( - request.queries[0].selected_plan_root.expr, - SummaryExpr::KeepPreAsap(_) - )); - } - - #[test] - fn selected_maintenance_dependency_still_requires_a_valid_executable_dag() { - let mut request = request("invalid-dependency", "sum_over_time(m[1m])"); - let selected_root = request.queries[0].selected_plan_root.clone(); - let selected = match &selected_root.expr { - SummaryExpr::SummaryEstimate { summary_input, .. } => summary_input.clone(), - SummaryExpr::SummaryAgg { .. } => selected_root.clone(), - SummaryExpr::ValueOperation { - child, - operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - .. - } => child.clone(), - _ => panic!("expected maintained aggregate fixture"), - }; - request.queries[0].selected_plan_root = Rc::new(SummaryNode { - expr: SummaryExpr::BinaryOp { - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - lhs: selected.clone(), - rhs: selected.clone(), - operator: planner_types::post_asap::BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: planner_types::pre_asap::BinaryOpKind::Arithmetic( - planner_types::pre_asap::ArithmeticOpKind::Add, - ), - vector_match: None, - }, - }, - schema: selected.schema.clone(), - guarantee: None, - }); - request.allow_mixed_summary_and_exact_execution = true; - let mut environment = environment(10_000); - environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.target_collector_ids.clear(); - - let error = DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap_err(); - - assert!(error.to_string().contains("invalid executable subDAG")); - } - - // Grouping must not move through non-additive arithmetic during physical - // packing: SUM(instance SUM / instance COUNT) is not pooled SUM / COUNT. - #[test] - fn non_additive_entity_reduction_does_not_bind_pooled_state() { - for query in [ - "sum by (service) (sum_over_time(m[1m]) / count_over_time(m[1m]))", - "sum by (service) (sum_over_time(m[1m])) / sum by (region) (count_over_time(m[1m]))", - "sum(m) / sum_over_time(m[1m])", - "sum_over_time(m[1m]) / count_over_time(m[5m])", - "sum_over_time(m[1m] offset 1h) / count_over_time(m[1m] offset 1h)", - ] { - let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let entries = snapshot.query_workload.repeating_queries.as_mut().unwrap(); - entries[0].query = Query(query.into()); - entries[0].requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (mut request, environment) = snapshot.into_physical_compilation_request().unwrap(); - request.allow_mixed_summary_and_exact_execution = false; - let bundle = DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap(); - assert!(bundle.precompute_plan.materializations.is_empty()); - assert!(bundle.query_plan.entries.values().all(|entry| matches!( - entry.nodes[&entry.root], - crate::query_plan::QueryPlanNode::ExactFallback { .. } - ))); - } - } - - // Local composition must evaluate per-series division before the outer sum. - #[test] - fn composable_non_additive_rollup_does_not_pool_raw_producers() { - let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = - Query("sum by (service) (sum_over_time(m[1m]) / count_over_time(m[1m]))".into()); - entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let candidates = - super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) - .unwrap(); - match DeploymentPlanCompiler.compile_promql(candidates[0].clone(), environment.clone()) { - Ok(plan) => assert!(plan.precompute_plan.materializations.is_empty()), - Err(error) => assert!(error - .to_string() - .contains("semantically identical original subtree witness")), - } - let native = DeploymentPlanCompiler - .compile_promql(candidates.last().unwrap().clone(), environment) - .unwrap(); - assert!(native.precompute_plan.materializations.is_empty()); - } - - #[test] - fn composable_per_entity_window_installs_isolated_state() { - use crate::query_plan::QueryPlanNode; - let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query("sum_over_time(m[1m])".into()); - entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - assert_eq!(plan.precompute_plan.materializations.len(), 1); - assert_eq!( - plan.precompute_plan.materializations[0].partitioning, - Some(asap_types::sds::PopulationPartitioning::PerEntity) - ); - let entry = plan.query_plan.lookup("sum_over_time(m[1m])").unwrap(); - assert!(entry - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::ReadMaterialization { .. }))); - } - - // Each operand retains its source and semantic range; a smaller shared pane - // remains valid for both readouts. - #[test] - fn composable_binary_binds_independent_source_windows() { - let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query("sum(sum_over_time(a[1m])) / sum(sum_over_time(b[5m]))".into()); - entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - // The derivation now covers both range selectors, so this no longer - // needs a hand-supplied 5m candidate to keep `b` from falling back. - let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - let bindings = plan - .query_plan - .entries - .values() - .next() - .unwrap() - .materialization_bindings(); - let actual = bindings - .iter() - .map(|binding| { - let identity = &plan.summary_catalog.outputs[&binding.materialization]; - let data = &plan.summary_catalog.data_descriptors[&identity.data_descriptor_id]; - ( - data.time_series_metric().unwrap(), - binding.window_ms, - binding.readout_lookback_ms, - ) - }) - .collect::>(); - // `window_ms` is the stored pane width, `readout_lookback_ms` the - // semantic range. Each operand keeps its own range -- 1m for `a`, 5m - // for `b` -- while both store 10s panes, because the snapshot - // evaluates every 10s and the derivation now covers both selectors. - assert_eq!( - actual, - BTreeSet::from([("a", 10_000, Some(60_000)), ("b", 10_000, Some(300_000)),]) - ); - assert_eq!(plan.precompute_plan.materializations.len(), 2); - } - - // Derived programs and their raw inputs must keep a runtime-supported cohort. - #[test] - fn derived_window_regression_nested_snapshot() { - for rate in [0.0, 100.0] { - let mut value: Value = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - value["query_workload"]["repeating_queries"][0]["query"] = - json!("quantile(0.9, sum_over_time(m[1m]))"); - value["data_workload"]["ingestion_rate"]["value"] = json!(rate); - let snapshot: BackendLocalPlanningInput = serde_json::from_value(value).unwrap(); - let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - assert!(plan - .precompute_plan - .materializations - .iter() - .any(|m| m.derived_input.is_some())); - assert!(plan - .precompute_plan - .materializations - .iter() - .all(|m| m.window_size == m.slide_interval)); - } - } - - // A derived cohort must not impose its maintenance cadence on an unrelated raw leaf at the same W. - #[test] - fn raw_leaf_keeps_its_cadence_beside_a_same_window_derived_cohort() { - let mut snapshot = planning_snapshot(); - let model = snapshot.physical_inputs.window_cost_model.clone(); - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query("quantile(0.9, sum_over_time(m[1m]))".into()); - entry.demand = RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(45_000), - evaluation_phase: planner_types::workload::TimestampMs(0), - }; - let mut right = snapshot.clone(); - right.query_workload.repeating_queries.as_mut().unwrap()[0].query = - Query("sum_over_time(n[1m])".into()); - let (mut request, env) = snapshot.into_physical_compilation_request().unwrap(); - let (right, _) = right.into_physical_compilation_request().unwrap(); - let left = request.queries[0].selected_plan_root.clone(); - let right = right.queries[0].selected_plan_root.clone(); - request.queries[0].selected_plan_root = Rc::new(SummaryNode { - expr: SummaryExpr::BinaryOp { - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - lhs: left.clone(), - rhs: right, - operator: planner_types::post_asap::BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: planner_types::pre_asap::BinaryOpKind::Arithmetic( - planner_types::pre_asap::ArithmeticOpKind::Add, - ), - vector_match: None, - }, - }, - schema: left.schema.clone(), - guarantee: None, - }); - let text = "quantile(0.9, sum_over_time(m[1m])) + sum_over_time(n[1m])"; - request.queries[0].query_string = text.into(); - request - .query_workload - .as_mut() - .unwrap() - .repeating_queries - .as_mut() - .unwrap()[0] - .query = Query(text.into()); - prepare_window_implementations(&mut request.queries[0], &model, env.target, 0).unwrap(); - request.queries[0] - .window_realization_candidates - .retain(|candidate| { - matches!( - candidate.layout, - asap_types::WindowMaterializationLayout::Pane { .. } - ) - }); - let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - assert!(plan - .precompute_plan - .materializations - .iter() - .any(|m| m.derived_input.is_some())); - let raw = plan - .precompute_plan - .materializations - .iter() - .find(|m| m.metric == "n") - .unwrap(); - assert_eq!(raw.slide_interval, 45); - assert_eq!(raw.window_layout.base_pane_secs(), 15); - } - - // A full-window producer keeps overlapping accumulators alive even before publication. - #[test] - fn derived_window_regression_resident_cost() { - let template = planning_snapshot().physical_inputs.window_cost_model.cost; - let mut lifecycle = planning_lifecycle(); - lifecycle.costs = LifecycleUnitCosts { - build: 0.0, - maintenance_per_update: 0.0, - read: 0.0, - retention_per_second: 1.0, - retirement: 0.0, - }; - let full = derived_window_cost( - &template, - &lifecycle, - 300, - 30, - &asap_types::WindowMaterializationLayout::FullWindow, - 0, - ); - assert!(full.weighted_cost >= lifecycle.horizon_seconds * 11.0); - } - - // Temporal SUM readouts share raw state only for identical source populations. - #[test] - fn derived_window_regression_shared_sum_panes() { - for interval in [10_000, 60_000] { - for (rhs, expected_states) in [("a", 1), ("b", 2), ("a{job=\"x\"}", 2)] { - let mut snapshot = planning_snapshot(); - let query = format!("sum_over_time(a[1m]) / sum_over_time({rhs}[10m])"); - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query(query); - entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - entry.demand = RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(interval), - evaluation_phase: planner_types::workload::TimestampMs(0), - }; - let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - assert_eq!(plan.precompute_plan.materializations.len(), expected_states); - let entry = plan.query_plan.entries.values().next().unwrap(); - let bindings = entry.materialization_bindings(); - assert_eq!( - bindings - .iter() - .filter_map(|b| b.readout_lookback_ms) - .collect::>(), - BTreeSet::from([60_000, 600_000]) - ); - if rhs == "a" { - assert_eq!(bindings[0].materialization, bindings[1].materialization); - assert_eq!( - plan.precompute_plan.materializations[0].num_aggregates_to_retain, - Some(600_000 / u64::from(interval) + 1) - ); - } - } - } - } - - // Distinct workload entries share one producer and retain both lifecycle consumers. - #[test] - fn shared_panes_preserve_workload_consumers_and_phase() { - for phase in [0, 5_000] { - let mut snapshot = planning_snapshot(); - let entries = snapshot.query_workload.repeating_queries.as_mut().unwrap(); - entries[0].query = Query("sum_over_time(a[1m])".into()); - entries[0].requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let mut second = entries[0].clone(); - second.query = Query("sum_over_time(a[10m])".into()); - second.demand = RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(10_000), - evaluation_phase: planner_types::workload::TimestampMs(phase), - }; - entries.push(second); - let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - assert_eq!( - plan.precompute_plan.materializations.len(), - if phase == 0 { 1 } else { 2 } - ); - if phase == 0 { - assert_eq!(plan.lifecycle_estimates.len(), 1); - let estimate = &plan.lifecycle_estimates[0]; - assert_eq!(estimate.consumer_query_ids.len(), 2); - assert_eq!(estimate.expected_reads, 60.0); - assert_eq!(estimate.expected_updates, 30_000.0); - } - } - } - - // Overflow must fail candidate validation, never turn an expensive layout into a free one. - #[test] - fn derived_cost_overflow_is_rejected() { - let snapshot = planning_snapshot(); - let (mut request, env) = snapshot.into_physical_compilation_request().unwrap(); - let query = &mut request.queries[0]; - let mut lifecycle = query.summary_lifecycle_inputs.clone(); - lifecycle.costs.build = f64::MAX; - let candidate = &mut query.window_realization_candidates[0]; - candidate.cost = derived_window_cost( - &candidate.cost, - &lifecycle, - candidate.window_secs, - candidate.slide_secs, - &candidate.layout, - 0, - ); - assert!(validate_window_implementations(query, &env).is_err()); - } - - // Serialized derived quotes become authoritative when a deployment supplies them explicitly. - #[test] - fn explicit_window_quotes_are_not_repriced_for_sharing() { - let mut snapshot = planning_snapshot(); - let query = "sum_over_time(a[1m]) / sum_over_time(a[10m])"; - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query(query.into()); - entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (derived, _) = snapshot - .clone() - .into_physical_compilation_request() - .unwrap(); - snapshot.physical_inputs.window_cost_model.quotes = - derived.queries[0].window_realization_candidates.clone(); - let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - assert!(request.queries[0] - .window_realization_candidates - .iter() - .any(|c| !c.derived)); - let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - assert_eq!(plan.precompute_plan.materializations.len(), 2); - } - - fn planning_lifecycle() -> SummaryLifecyclePlanningInputs { - planning_snapshot() - .into_physical_compilation_request() - .unwrap() - .0 - .queries[0] - .summary_lifecycle_inputs - .clone() - } - - fn planning_snapshot() -> BackendLocalPlanningInput { - serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap() - } - - fn derived_query_window_secs(query: &str) -> u64 { - let mut snapshot = planning_snapshot(); - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query(query.into()); - entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - snapshot - .into_physical_compilation_request() - .unwrap() - .0 - .queries[0] - .query_lookback_seconds - } - - fn set_data_ingestion_interval( - snapshot: &mut BackendLocalPlanningInput, - interval_ms: Option, - ) { - let evidence = Evidence { - value: interval_ms.map(DurationMs), - source: EvidenceSource::Declared, - observed_at_ms: None, - valid_for_ms: None, - }; - snapshot.data_workload.data_ingestion_interval = evidence; - } - - // Plan-ready lowering uses workload cadence, not the backend-local migration field. - #[test] - fn instant_aggregate_uses_data_ingestion_interval() { - let mut snapshot = planning_snapshot(); - snapshot.query_workload.repeating_queries.as_mut().unwrap()[0].query = - Query("sum(data)".into()); - set_data_ingestion_interval(&mut snapshot, Some(1_000)); - - let (request, _) = snapshot.into_physical_compilation_request().unwrap(); - assert_eq!(request.queries[0].query_lookback_seconds, 1); - } - - // Snapshot lowering must use the environment clock for cadence freshness. - #[test] - fn snapshot_rejects_unavailable_cadence_evidence() { - for (observed, valid_for) in [ - (Some(0), Some(0)), - (Some(u64::MAX), None), - (None, Some(100)), - ] { - let mut snapshot = planning_snapshot(); - snapshot - .data_workload - .data_ingestion_interval - .observed_at_ms = observed; - snapshot.data_workload.data_ingestion_interval.valid_for_ms = valid_for; - let error = snapshot.into_physical_compilation_request().unwrap_err(); - assert!( - error - .to_string() - .contains("evidence is unavailable at planning time"), - "{error}" - ); - } - } - - // Legacy snapshots migrate their explicit scrape cadence into DataWorkload. - #[test] - fn missing_data_ingestion_interval_migrates_scrape_interval() { - let mut snapshot = planning_snapshot(); - snapshot.query_workload.repeating_queries.as_mut().unwrap()[0].query = - Query("sum(data)".into()); - set_data_ingestion_interval(&mut snapshot, None); - - let (request, _) = snapshot.into_physical_compilation_request().unwrap(); - assert_eq!( - request.data_workload.unwrap().data_ingestion_interval.value, - Some(DurationMs(5_000)) - ); - assert_eq!(request.queries[0].query_lookback_seconds, 5); - } - - // An explicitly invalid interval must not be replaced by the migration value. - #[test] - fn zero_data_ingestion_interval_is_rejected() { - let mut snapshot = planning_snapshot(); - set_data_ingestion_interval(&mut snapshot, Some(0)); - - let error = snapshot - .into_physical_compilation_request() - .expect_err("zero interval must fail") - .to_string(); - assert!(error.contains("data_ingestion_interval must be greater than zero")); - } - - // Every concatenated histogram result contributes its source history. - #[test] - fn derived_history_visits_concat_branches() { - assert_eq!(derived_query_window_secs( - "histogram_quantiles(rate(request_duration_seconds_bucket[5m]), \"quantile\", 0.5, 0.9)" - ), 300); - } - - // A short ranged sibling must not suppress a raw selector's default window. - #[test] - fn derived_history_applies_cadence_at_each_rangeless_source() { - assert_eq!( - derived_query_window_secs("sum(a) + sum(sum_over_time(b[1s]))"), - 5 - ); - assert_eq!(derived_query_window_secs("sum(a offset 1h)"), 3605); - assert_eq!(derived_query_window_secs("sum_over_time(a[1s])"), 1); - } - - // The seconds-based backend must fail explicitly instead of shrinking history. - #[test] - fn derived_history_rejects_fractional_seconds() { - for query in [ - "sum_over_time(a[1500ms])", - "sum_over_time(a[500ms])", - "sum_over_time(a[1500ms] offset 500ms)", - "sum_over_time(a[1s]) + sum(sum_over_time(b[500ms]))", - "avg_over_time((sum(a))[1500ms:])", - "sum(a offset 500ms)", - ] { - let mut snapshot = planning_snapshot(); - snapshot.query_workload.repeating_queries.as_mut().unwrap()[0].query = - Query(query.into()); - let error = snapshot - .into_physical_compilation_request() - .expect_err(query) - .to_string(); - assert!( - error.contains("whole number of seconds"), - "{query}: {error}" - ); - } - } - - #[test] - fn snapshot_rejects_manual_lookback() { - let mut wire = serde_json::to_value(planning_snapshot()).unwrap(); - wire["query_workload"]["repeating_queries"][0]["time_selection"]["lookback"] = - 60_000.into(); - let error = serde_json::from_value::(wire) - .expect_err("manual lookback must not be accepted") - .to_string(); - assert!(error.contains("lookback is not supported"), "{error}"); - } - - #[test] - fn subquery_range_derives_the_query_window() { - assert_eq!( - derived_query_window_secs("avg_over_time((sum(a))[6h:])"), - 6 * 60 * 60 + 5 - ); - } - - #[test] - fn offset_extends_the_query_furthest_lookback() { - // Evaluated at t, `a[1m] offset 1h` selects `(t - 61m, t - 60m]`. - assert_eq!( - derived_query_window_secs("sum_over_time(a[1m] offset 1h)"), - 61 * 60 - ); - } - - // A workload evaluated more often than its window is wide must advance its - // state at that cadence. Planning it as one lookback-wide tumbling window - // answers with results that only change once per window. - #[test] - fn derived_window_candidate_follows_the_evaluation_cadence() { - let cost = planning_snapshot().physical_inputs.window_cost_model.cost; - let expr = crate::query_parser::parse_query_expr_canonical( - "quantile_over_time(0.5, data[5m])", - AccuracyTarget::Exact, - ) - .unwrap(); - let derived = - derived_window_candidates("id", &expr, 300_000, 30_000, cost, &planning_lifecycle(), 0); - // A sliding shape has two legal layouts, so both are offered and the - // cost model picks between them. - assert_eq!( - derived - .iter() - .map(|c| (c.realization_id.as_str(), c.layout.clone())) - .collect::>(), - vec![ - ( - "id-300s-slide-30s-pane-30s", - asap_types::WindowMaterializationLayout::Pane { pane_secs: 30 } - ), - ( - "id-300s-slide-30s-full-window", - asap_types::WindowMaterializationLayout::FullWindow - ), - ] - ); - let candidate = &derived[0]; - assert_eq!(candidate.framework, SummaryWindowFramework::Sliding); - assert_eq!((candidate.window_secs, candidate.slide_secs), (300, 30)); - assert_eq!( - candidate.layout, - asap_types::WindowMaterializationLayout::Pane { pane_secs: 30 } - ); - } - - // The layout choice is a write-amplification-versus-read-amplification - // trade, and the derived quote has to price it in the right direction: a - // full window takes every sample into all ten overlapping windows but is - // read whole, panes take each sample once but compose ten per read. Which - // wins depends on how hard the source is pushing, so pin the crossover, - // not the magnitudes. - #[test] - fn derived_window_layout_prices_write_against_read_amplification() { - let cost = planning_snapshot().physical_inputs.window_cost_model.cost; - let expr = crate::query_parser::parse_query_expr_canonical( - "quantile_over_time(0.5, data[5m])", - AccuracyTarget::Exact, - ) - .unwrap(); - let quote = |rate: f64| { - let mut lifecycle = planning_lifecycle(); - lifecycle.ingestion_rate_per_second = rate; - let derived = derived_window_candidates( - "id", - &expr, - 300_000, - 30_000, - cost.clone(), - &lifecycle, - 0, - ); - let weighted = |layout: &asap_types::WindowMaterializationLayout| { - derived - .iter() - .find(|c| c.layout == *layout) - .expect("both layouts offered") - .cost - .weighted_cost - }; - ( - weighted(&asap_types::WindowMaterializationLayout::Pane { pane_secs: 30 }), - weighted(&asap_types::WindowMaterializationLayout::FullWindow), - ) - }; - let (idle_pane, idle_full) = quote(0.0); - assert!( - idle_full < idle_pane, - "with no arriving data the fanout is free and the merges are not: \ - pane {idle_pane} full {idle_full}" - ); - let (busy_pane, busy_full) = quote(100.0); - assert!( - busy_pane < busy_full, - "under load the tenfold update fanout dominates: pane {busy_pane} full {busy_full}" - ); - } - - // A tumbling shape pairs only with `Pane` in the validator's - // framework/layout table, so there is no alternative to price against it. - #[test] - fn tumbling_shapes_have_no_layout_alternative_to_rank() { - let cost = planning_snapshot().physical_inputs.window_cost_model.cost; - let expr = crate::query_parser::parse_query_expr_canonical( - "quantile_over_time(0.5, data[5m])", - AccuracyTarget::Exact, - ) - .unwrap(); - let derived = derived_window_candidates( - "id", - &expr, - 300_000, - 300_000, - cost, - &planning_lifecycle(), - 0, - ); - assert_eq!(derived.len(), 1); - assert_eq!(derived[0].realization_id, "id-300s-slide-300s-pane-300s"); - assert_eq!(derived[0].framework, SummaryWindowFramework::Tumbling); - } - - // Every shape this function can emit must survive the validator, or a bad - // derivation would reach a plan instead of a compile error. - #[test] - fn derived_window_candidate_shapes_are_accepted_by_validation() { - let snapshot = planning_snapshot(); - let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let cost = planning_snapshot().physical_inputs.window_cost_model.cost; - for (lookback_ms, evaluation_ms) in [ - (300_000, 30_000), - (300_000, 300_000), - (300_000, 45_000), - (60_000, 90_000), - ] { - let mut query = request.queries[0].clone(); - query.query_lookback_seconds = lookback_ms / 1_000; - let expr = crate::query_parser::parse_query_expr_canonical( - &format!("quantile_over_time(0.5, data[{}s])", lookback_ms / 1_000), - AccuracyTarget::Exact, - ) - .unwrap(); - query.window_realization_candidates = derived_window_candidates( - "derived", - &expr, - lookback_ms, - evaluation_ms, - cost.clone(), - &planning_lifecycle(), - 0, - ); - validate_window_implementations(&query, &environment).unwrap_or_else(|error| { - panic!("lookback {lookback_ms} cadence {evaluation_ms}: {error:?}") - }); - } - } - - // Compilation keeps the demand grid while publishing independently sized panes or full windows. - #[test] - fn scheduled_window_layouts_preserve_cadence_phase_and_readout() { - for (evaluation, pane) in [(20, 20), (45, 15), (60, 60), (120, 60), (90, 30)] { - for phase in [0, 5_000] { - for full in [false, true] { - if full && evaluation == 60 { - continue; - } - let mut snapshot = planning_snapshot(); - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query("sum_over_time(a[1m])".into()); - entry.requirements.accuracy = - AccuracyRequirement::Explicit(AccuracyTarget::Exact); - entry.demand = RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(evaluation * 1_000), - evaluation_phase: planner_types::workload::TimestampMs(phase), - }; - let (mut request, env) = snapshot.into_physical_compilation_request().unwrap(); - request.queries[0] - .window_realization_candidates - .retain(|c| { - matches!( - c.layout, - asap_types::WindowMaterializationLayout::FullWindow - ) == full - }); - let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - let config = &plan.precompute_plan.materializations[0]; - assert_eq!(config.slide_interval, u64::from(evaluation)); - assert_eq!(config.window_size, 60); - assert_eq!( - config.stored_window_ms(), - if full { 60_000 } else { pane * 1_000 } - ); - let entry = plan.query_plan.entries.values().next().unwrap(); - let binding = entry.materialization_bindings()[0]; - for tick in 3..6 { - let end = phase + u64::from(evaluation) * 1_000 * tick; - assert!( - binding.covers_range(end - 60_000, end), - "{evaluation} {phase} {full}" - ); - assert!(!binding.covers_range(end - 60_000 + 1, end + 1)); - } - } - } - } - } - - // Smaller common panes are selected only when saved maintenance exceeds extra merge work. - #[test] - fn different_cadences_share_common_panes_only_when_cheaper() { - for read_cost in [0.0, 1_000_000.0] { - let mut snapshot = planning_snapshot(); - snapshot.physical_inputs.lifecycle_costs.read = read_cost; - snapshot - .physical_inputs - .lifecycle_costs - .maintenance_per_update = 1.0; - let entries = snapshot.query_workload.repeating_queries.as_mut().unwrap(); - entries[0].query = Query("sum_over_time(a[1m])".into()); - entries[0].requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - entries[0].demand = RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(20_000), - evaluation_phase: planner_types::workload::TimestampMs(0), - }; - let mut second = entries[0].clone(); - second.query = Query("sum_over_time(a[90s])".into()); - second.demand = RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(30_000), - evaluation_phase: planner_types::workload::TimestampMs(0), - }; - entries.push(second); - let (mut request, env) = snapshot.into_physical_compilation_request().unwrap(); - for query in &mut request.queries { - query.window_realization_candidates.retain(|c| { - matches!( - c.layout, - asap_types::WindowMaterializationLayout::Pane { .. } - ) - }); - } - let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - assert_eq!( - plan.precompute_plan.materializations.len(), - if read_cost == 0.0 { 1 } else { 2 } - ); - if read_cost == 0.0 { - assert_eq!( - plan.precompute_plan.materializations[0].stored_window_ms(), - 10_000 - ); - assert_eq!(plan.lifecycle_estimates[0].expected_reads, 25.0); - assert_eq!(plan.lifecycle_estimates[0].expected_updates, 30_000.0); - } - } - } - - // An incompatible third consumer cannot disable sharing between the first two. - #[test] - fn sharing_keeps_profitable_subsets_with_other_phases_or_finer_cadences() { - for (third_phase, third_interval) in [(5_000, 20_000), (0, 1_000)] { - let mut snapshot = planning_snapshot(); - snapshot - .physical_inputs - .lifecycle_costs - .maintenance_per_update = 1.0; - snapshot.physical_inputs.lifecycle_costs.read = - if third_interval == 1_000 { 100.0 } else { 0.0 }; - let entries = snapshot.query_workload.repeating_queries.as_mut().unwrap(); - let template = entries[0].clone(); - entries.clear(); - for (window, phase, interval) in [ - (60, 0, 20_000), - (120, 0, 20_000), - (180, third_phase, third_interval), - ] { - let mut entry = template.clone(); - entry.query = Query(format!("sum_over_time(a[{window}s])")); - entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - entry.demand = RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(interval), - evaluation_phase: planner_types::workload::TimestampMs(phase), - }; - entries.push(entry); - } - let (mut request, env) = snapshot.into_physical_compilation_request().unwrap(); - for query in &mut request.queries { - query.window_realization_candidates.retain(|c| { - matches!( - c.layout, - asap_types::WindowMaterializationLayout::Pane { .. } - ) - }); - } - let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - assert_eq!(plan.precompute_plan.materializations.len(), 2); - assert!(plan - .lifecycle_estimates - .iter() - .any(|e| e.consumer_query_ids.len() == 2)); - } - } - - // A subsecond cadence cannot be rounded into a different supported query schedule. - #[test] - fn fractional_cadence_uses_native_fallback_without_truncation() { - let mut snapshot = planning_snapshot(); - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query("sum_over_time(a[1m])".into()); - entry.demand = RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(1_500), - evaluation_phase: planner_types::workload::TimestampMs(0), - }; - let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - assert!(request.queries[0].window_realization_candidates.is_empty()); - let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - assert!(plan.precompute_plan.materializations.is_empty()); - } - - // Every supported cadence has panes that exactly tile each scheduled range. - #[test] - fn derived_window_candidates_cover_dividing_nondividing_and_sparse_cadences() { - let expr = crate::query_parser::parse_query_expr_canonical( - "sum_over_time(data[1m])", - AccuracyTarget::Exact, - ) - .unwrap(); - for (evaluation_secs, pane_secs) in [(20, 20), (45, 15), (60, 60), (120, 60), (90, 30)] { - let candidates = derived_window_candidates( - "id", - &expr, - 60_000, - evaluation_secs * 1_000, - planning_snapshot().physical_inputs.window_cost_model.cost, - &planning_lifecycle(), - 0, - ); - let pane = candidates - .iter() - .find(|c| { - matches!( - c.layout, - asap_types::WindowMaterializationLayout::Pane { .. } - ) - }) - .unwrap(); - assert_eq!(pane.slide_secs, u64::from(evaluation_secs)); - assert_eq!( - pane.layout, - asap_types::WindowMaterializationLayout::Pane { pane_secs } - ); - assert!(pane - .layout - .validate(pane.window_secs, pane.slide_secs) - .is_ok()); - } - } - - // A measured quote replaces only the matching shape, leaving other legal layouts available. - #[test] - fn measured_window_quote_preserves_shape_and_price() { - let mut snapshot = planning_snapshot(); - let (derived, _) = snapshot - .clone() - .into_physical_compilation_request() - .unwrap(); - let mut quote = derived.queries[0].window_realization_candidates[0].clone(); - quote.realization_id = "measured-pane".into(); - quote.cost.weighted_cost = 8.0; - quote.derived = false; - snapshot - .physical_inputs - .window_cost_model - .quotes - .push(quote.clone()); - let (request, _) = snapshot.into_physical_compilation_request().unwrap(); - assert!(request.queries[0] - .window_realization_candidates - .contains("e)); - assert!(request.queries[0] - .window_realization_candidates - .iter() - .any(|candidate| candidate.derived)); - } - - // End to end: the retained-state count is derived from the pane width, so - // fixing the shape fixes it too. Six 10s panes cover the 1m lookback, plus - // the one still being filled. - #[test] - fn retained_state_count_follows_the_derived_pane_width() { - let snapshot = planning_snapshot(); - let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let plan = DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap(); - assert_eq!( - plan.precompute_plan.materializations[0].num_aggregates_to_retain, - Some(7) - ); - } - - // The reported case: two 5m-lookback quantiles evaluated every 30s. The - // whole chain has to land — sliding framework, 30s panes, and the retained - // count that falls out of the pane width — or the answer only changes once - // every five minutes. - #[test] - fn five_minute_lookback_evaluated_every_thirty_seconds_slides_by_thirty() { - let mut snapshot = planning_snapshot(); - { - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query("quantile_over_time(0.5, data[5m])".into()); - entry.demand = RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(30_000), - evaluation_phase: planner_types::workload::TimestampMs(0), - }; - } - let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let plan = DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap(); - let materialization = &plan.precompute_plan.materializations[0]; - assert_eq!( - ( - materialization.window_size, - materialization.slide_interval, - materialization.window_type, - materialization.window_layout.clone(), - materialization.num_aggregates_to_retain, - ), - ( - 300, - 30, - asap_types::WindowKind::Sliding, - asap_types::WindowMaterializationLayout::Pane { pane_secs: 30 }, - // Ten 30s panes cover the 5m lookback, plus the one still filling. - Some(11), - ) - ); - } - - // A filtered operand gets a summary over its own filtered population, - // with each operand keeping its own range. - #[test] - fn composable_binary_summarizes_each_prometheus_filtered_operand() { - use crate::query_plan::{query_time::QueryTimeOperator, QueryPlanNode}; - let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = - Query("sum(sum_over_time(a[1m])) / sum(sum_over_time(b{job!=\"x\"}[5m]))".into()); - entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - let query = plan.query_plan.entries.values().next().unwrap(); - let bindings = query.materialization_bindings(); - // Both operands now hold a summary. The filtered denominator is no - // longer query-time only: its 5m range has a candidate, so it gets - // its own summary over the filtered population rather than exact - // execution. Nothing about the filter forced query-time execution -- the - // missing 5m window candidate did, and this test previously pinned - // that artifact as intended behavior. - let bound = bindings - .iter() - .map(|binding| { - let identity = &plan.summary_catalog.outputs[&binding.materialization]; - let data = &plan.summary_catalog.data_descriptors[&identity.data_descriptor_id]; - ( - data.time_series_metric().unwrap(), - data.population_filter_canonical.clone(), - binding.readout_lookback_ms, - ) - }) - .collect::>(); - assert_eq!( - bound, - BTreeSet::from([ - ("a", String::new(), Some(60_000)), - ("b", "{job!=\"x\"}".to_string(), Some(300_000)), - ]) - ); - assert!(!query - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. }))); - assert!(!query.nodes.values().any(|node| matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::Scan { .. }, - .. - } - ))); - assert_eq!(plan.precompute_plan.materializations.len(), 2); - } - - #[test] - fn warm_leaf_contract_preserves_prometheus_matchers_and_rejects_offsets() { - for query in [ - "sum_over_time(m{job=\"a\"}[1m])", - "sum_over_time(m{job!=\"a\"}[1m])", - "sum_over_time(m{job=~\"a.*\"}[1m])", - "sum_over_time(m{job!~\"a.*\"}[1m])", - ] { - let request = request("scope", query); - let selected = - collect_selected_materializations(&request.queries[0].selected_plan_root, false); - let selected = selected.unwrap(); - assert_eq!(selected.len(), 1, "{query}"); - assert!(!selected[0].spatial_filter.is_empty(), "{query}"); - } - let request = request("scope", "sum_over_time(m[1m] offset 1h)"); - let selected = - collect_selected_materializations(&request.queries[0].selected_plan_root, false); - assert!(selected.is_err() || selected.unwrap().is_empty()); - } - - // Unsupported producer filters/multiple sources retain executable native fallback. - #[test] - fn snapshot_accepts_filtered_and_multisource_exact_fallback() { - for query in [ - "sum(rate(http_requests_total{job=\"order-service\"}[5m]))", - "sum(rate(a[5m])) / sum(rate(b[5m]))", - "sum(avg_over_time(m{job=~\".+\"}[6h]))", - "sum(rate(m[5m] offset 1h))", - ] { - let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query(query.into()); - entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (mut request, environment) = snapshot.into_physical_compilation_request().unwrap(); - request = - super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) - .unwrap() - .pop() - .unwrap(); - let bundle = DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap(); - assert!(bundle.precompute_plan.materializations.is_empty()); - assert!(bundle.query_plan.entries.values().all(|entry| matches!( - entry.nodes[&entry.root], - crate::query_plan::QueryPlanNode::ExactFallback { .. } - ))); - } - } - - // Projections reference one immutable snapshot and reject drift or foreign state. - #[test] - fn catalog_projection_rejects_missing_stale_and_foreign_references() { - let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let bundle = quoted_snapshot(snapshot, crate::physical::compiler::QueryFrontend::PromQl) - .compile_promql() - .unwrap(); - let catalog = &bundle.summary_catalog; - let mut transmission = bundle.transmission_plan.clone(); - transmission.validate_against_catalog(catalog).unwrap(); - assert_eq!( - transmission.summary_catalog.as_ref(), - Some(&catalog.reference().unwrap()) - ); - transmission - .summary_catalog - .as_mut() - .unwrap() - .snapshot_sha256 - .push('0'); - assert!(transmission.validate_against_catalog(catalog).is_err()); - assert!(transmission.validate(&bundle.precompute_plan).is_err()); - transmission.summary_catalog = None; - assert!(transmission.validate_against_catalog(catalog).is_err()); - let mut collector = CollectorPlan { - summary_catalog: Some(catalog.reference().unwrap()), - collector_id: "collector-test".into(), - envelope: bundle.envelope.clone(), - materializations: vec![], - transmission_rules: vec![], - }; - collector.validate_against_catalog(catalog).unwrap(); - collector.envelope.plan_version += 1; - assert!(collector.validate_against_catalog(catalog).is_err()); - let mut unknown_materialization = bundle.transmission_plan.clone(); - unknown_materialization.rules.push(TransmissionRule { - materialization: asap_types::PolicyFingerprint(u64::MAX).into(), - producer_id: "foreign".into(), - schema_id: "foreign".into(), - mode: TransmissionMode::Full, - encoding: StateEncoding::ExactAccumulatorV1, - emit_every_ms: 60_000, - full_checkpoint_every_ms: None, - destination_ref: "backend".into(), - runtime_policy: Default::default(), - }); - assert!(unknown_materialization - .validate_against_catalog(catalog) - .is_err()); - let encoded = serde_json::to_value(&collector).unwrap(); - assert!(encoded["summary_catalog"] - .get("summary_descriptors") - .is_none()); - for actual in &bundle.collector_plans { - actual.validate_against_catalog(catalog).unwrap(); - assert_eq!( - actual.summary_catalog, - bundle.transmission_plan.summary_catalog - ); - } - } - - #[test] - fn canonical_snapshot_preserves_shared_bindings_after_serialization() { - // Two different registered readouts survive publication with one state. - let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let entries = snapshot.query_workload.repeating_queries.as_mut().unwrap(); - entries[0].query = Query("sum(sum_over_time(m[1m]))".into()); - let mut second = entries[0].clone(); - second.query = Query("sum(sum_over_time(m[1m])) * 2".into()); - entries.push(second); - let bundle = quoted_snapshot(snapshot, crate::physical::compiler::QueryFrontend::PromQl) - .compile_promql() - .unwrap(); - assert_eq!(bundle.query_plan.entries.len(), 2); - assert_eq!(bundle.precompute_plan.materializations.len(), 1); - let query_plan: QueryPlan = - serde_json::from_slice(&serde_json::to_vec(&bundle.query_plan).unwrap()).unwrap(); - let bindings = query_plan - .entries - .values() - .flat_map(|entry| entry.materialization_bindings()) - .map(|binding| binding.materialization.fingerprint()) - .collect::>(); - assert_eq!(bindings.len(), 1); - query_plan.validate(&bindings).unwrap(); - let stored_outputs = query_plan - .entries - .values() - .flat_map(|entry| entry.materialization_bindings()) - .map(|binding| binding.stored_output_reference.clone()) - .collect::>(); - assert_eq!(stored_outputs.len(), 2); - assert_eq!(stored_outputs[0], stored_outputs[1]); - assert_eq!( - stored_outputs[0], - bundle.precompute_plan.schemas[0].stored_output_reference - ); - asap_types::plan_publication::validate_stored_output_references( - &bundle.precompute_plan, - &query_plan, - ) - .unwrap(); - // Output identity is independent of definition identity, but changing - // only the writer must still invalidate every unchanged reader binding. - let mut rebound_writer = bundle.precompute_plan.clone(); - rebound_writer.schemas[0] - .stored_output_reference - .stored_output_id = asap_types::sds::StoredOutputId(123); - assert!(rebound_writer - .validate_against_catalog(&bundle.summary_catalog) - .is_err()); - assert!( - asap_types::plan_publication::validate_stored_output_references( - &rebound_writer, - &query_plan, - ) - .is_err() - ); - } - - #[test] - fn precompute_catalog_validates_without_backend_projection() { - let bundle = DeploymentPlanCompiler - .compile_promql( - request("catalog", "quantile_over_time(0.99, m[1m])"), - environment(10_000), - ) - .unwrap(); - let original = &bundle.precompute_plan; - let catalog = &bundle.summary_catalog; - let roundtrip: PrecomputePlan = - serde_json::from_slice(&serde_json::to_vec(original).unwrap()).unwrap(); - roundtrip.validate_against_catalog(catalog).unwrap(); - let mut legacy = serde_json::to_value(original).unwrap(); - for schema in legacy["schemas"].as_array_mut().unwrap() { - schema.as_object_mut().unwrap().remove("value_projection"); - schema["value_column"] = serde_json::json!("SampleValue"); - } - assert!(serde_json::from_value::(legacy).is_err()); - let reject = - |mutated: PrecomputePlan| assert!(mutated.validate_against_catalog(catalog).is_err()); - let mut bad = original.clone(); - bad.schemas[0].source = planner_types::pre_asap::Source::TimeSeries { - metric: "other".into(), - }; - reject(bad); - let mut bad = original.clone(); - bad.schemas[0].value_projection = asap_types::sds::ValueProjectionIdentity::Column { - name: "other".into(), - }; - reject(bad); - let mut bad = original.clone(); - bad.schemas[0].group_by.push("other".into()); - reject(bad); - let mut bad = original.clone(); - bad.schemas[0].window.size_ms += 1; - reject(bad); - let mut bad = original.clone(); - bad.materializations[0].num_aggregates_to_retain = Some(0); - reject(bad); - let mut bad = original.clone(); - bad.summary_catalog - .as_mut() - .unwrap() - .snapshot_sha256 - .push('0'); - reject(bad); - let mut corrupt_catalog = catalog.clone(); - corrupt_catalog.data_descriptors.clear(); - assert!(original.validate_against_catalog(&corrupt_catalog).is_err()); - let mut legacy = original.clone(); - legacy.summary_catalog = None; - legacy.validate().unwrap(); - assert!(legacy.validate_against_catalog(catalog).is_err()); - } - - #[test] - fn publication_is_catalog_authoritative_and_round_trips() { - let bundle = DeploymentPlanCompiler - .compile_promql( - request("publication", "quantile_over_time(0.99, m[1m])"), - environment(10_000), - ) - .unwrap(); - let publication = bundle.to_publication_artifact().unwrap(); - let json = serde_json::to_value(&publication).unwrap(); - let mut decoded: super::super::publication::PhysicalPlanPublication = - serde_json::from_value(json).unwrap(); - decoded.validate().unwrap(); - decoded.collector_plans.clear(); - assert!(decoded.validate().is_err()); - let mut decoded = publication.clone(); - decoded.summary_catalog.plan_version += 1; - assert!(decoded.validate().is_err()); - let mut decoded = publication; - decoded - .collector_plans - .push(decoded.collector_plans[0].clone()); - assert!(decoded.validate().is_err()); - } - - #[test] - fn compiles_one_decision_into_matching_collector_and_backend_views() { - let bundle = DeploymentPlanCompiler - .compile_promql( - request("q-quantile", "quantile_over_time(0.99, m[1m])"), - environment(10_000), - ) - .expect("compile"); - assert_eq!(bundle.collector_plans.len(), 2); - bundle.summary_catalog.validate().expect("catalog contract"); - assert_eq!(bundle.summary_catalog.plan_id, bundle.envelope.plan_id); - assert_eq!( - bundle.summary_catalog.plan_version, - bundle.envelope.plan_version - ); - assert_eq!( - bundle - .summary_catalog - .outputs - .keys() - .cloned() - .collect::>(), - bundle - .precompute_plan - .materializations - .iter() - .map(|config| config.policy_fingerprint().into()) - .collect::>() - ); - assert_eq!(bundle.precompute_plan.envelope, bundle.envelope); - assert_eq!(bundle.precompute_plan.materializations.len(), 1); - assert_eq!(bundle.precompute_plan.schemas.len(), 1); - assert_eq!(bundle.precompute_plan.producers.len(), 2); - assert_eq!(bundle.precompute_plan.ingest.endpoint_path, "/v1/metrics"); - bundle.precompute_plan.validate().expect("runtime contract"); - bundle - .transmission_plan - .validate(&bundle.precompute_plan) - .expect("transmission contract"); - assert_eq!(bundle.transmission_plan.rules.len(), 2); - let rule = &bundle.transmission_plan.rules[0]; - let frame = SummaryFrameIdentity { - identity_version: 1, - plan_id: bundle.envelope.plan_id, - plan_version: bundle.envelope.plan_version, - backend_compat: bundle.envelope.backend_compat.clone(), - materialization: rule.materialization, - series_identity: "service=checkout,zone=a".into(), - schema_id: rule.schema_id.clone(), - producer_id: rule.producer_id.clone(), - producer_epoch: "boot-1".into(), - window_start_unix_nano: 1, - window_end_unix_nano: 2, - sequence: 1, - kind: SummaryFrameKind::Full, - encoding: rule.encoding.clone(), - checkpoint_id: Some("checkpoint-1".into()), - base_checkpoint_id: None, - }; - bundle - .transmission_plan - .validate_frame(&frame) - .expect("matching frame identity"); - let mut wrong_version = frame; - wrong_version.plan_version += 1; - assert!(matches!( - bundle.transmission_plan.validate_frame(&wrong_version), - Err(TransmissionPlanError::InvalidFrame(_)) - )); - assert_eq!(bundle.summary_catalog.outputs.len(), 1); - assert_eq!( - bundle - .query_plan - .entries - .values() - .flat_map(QueryPlanEntry::materialization_bindings) - .count(), - 1 - ); - assert_eq!(bundle.query_plan.plan_id, bundle.envelope.plan_id); - let entry = bundle - .query_plan - .lookup("quantile_over_time( 0.99, m[1m] )") - .expect("canonical QueryPlan lookup"); - assert_eq!(entry.query_id, "q-quantile"); - assert_eq!( - entry - .nodes - .values() - .filter(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ReadMaterialization { .. } - )) - .count(), - 1 - ); - entry - .validate( - &bundle - .precompute_plan - .materializations - .iter() - .map(asap_types::PrecomputeMaterialization::policy_fingerprint) - .collect(), - ) - .expect("executable physical DAG"); - let wire = serde_json::to_vec(&bundle.query_plan).expect("serialize QueryPlan"); - let decoded: QueryPlan = serde_json::from_slice(&wire).expect("deserialize QueryPlan"); - assert_eq!(decoded, bundle.query_plan); - for plan in &bundle.collector_plans { - assert_eq!(plan.envelope, bundle.envelope); - assert_eq!(plan.materializations[0].metric, "m"); - assert_eq!(plan.materializations[0].window_secs, 60); - assert_eq!(plan.transmission_rules.len(), 1); - assert_eq!( - plan.materializations[0].abstract_window_framework, - SummaryWindowFramework::Tumbling - ); - assert_eq!( - plan.materializations[0].window_realization_id, - "collector-tumbling-v1" - ); - assert_eq!(plan.materializations[0].slide_secs, 60); - assert_eq!( - plan.materializations[0].lifecycle, - CollectorLifecycle { - kind: "continuously_maintained".into(), - maintenance_mode: "incremental".into(), - evaluation_schedule: "per_update".into(), - output_representation: "summary_state".into(), - } - ); - assert!(matches!( - plan.materializations[0].algorithm.as_str(), - "ddsketch" | "kll" - )); - } - } - - #[test] - fn backend_local_hll_and_envelope_ingest_are_supported() { - let bundle = DeploymentPlanCompiler - .compile_promql( - request("q", "quantile_over_time(0.99, m[1m])"), - environment(10_000), - ) - .unwrap(); - let mut materializations = bundle.precompute_plan.materializations; - materializations[0].aggregation_type = asap_types::AggregationType::HLL; - materializations[0].parameters.clear(); - let mut envelope_plan = PrecomputePlan::build( - bundle.envelope.clone(), - materializations.clone(), - &["collector".into()], - ) - .unwrap(); - envelope_plan.ingest = IngestContract { - dataset_identity: envelope_plan.ingest.dataset_identity.clone(), - protocol: IngestProtocol::PrometheusRemoteWriteV1, - endpoint_path: "/api/v1/write".into(), - timestamp_unit: TimestampUnit::UnixMilliseconds, - require_plan_identity: false, - require_stored_output_identity: false, - require_registered_producer: false, - }; - envelope_plan.producers.clear(); - envelope_plan.validate().unwrap(); - PrecomputePlan::build_backend_local(bundle.envelope, materializations).unwrap(); - } - - #[test] - fn backend_local_precompute_contract_has_no_collector_producers() { - let bundle = DeploymentPlanCompiler - .compile_promql( - request("q-quantile", "quantile_over_time(0.99, m[1m])"), - environment(10_000), - ) - .expect("compile"); - let plan = PrecomputePlan::build_backend_local( - bundle.envelope.clone(), - bundle.precompute_plan.materializations.clone(), - ) - .expect("backend-local contract"); - - assert_eq!( - plan.ingest.protocol, - IngestProtocol::PrometheusRemoteWriteV1 - ); - assert_eq!(plan.ingest.endpoint_path, "/api/v1/write"); - assert_eq!(plan.ingest.timestamp_unit, TimestampUnit::UnixMilliseconds); - assert!(plan.producers.is_empty()); - plan.validate().expect("valid backend-local projection"); - crate::physical::compiler::build_transmission_plan( - bundle.envelope, - &plan, - &BTreeMap::new(), - ) - .expect("empty backend-local transmission contract") - .validate(&plan) - .expect("valid empty transmission plan"); - } - - #[test] - fn canonical_workload_snapshot_invokes_backend_local_planning() { - let data_workload = DataWorkload { - arrival: DataArrival::ContinuouslyIngesting, - ingestion_rate: Evidence { - value: Some(Rate(100.0)), - source: EvidenceSource::Declared, - observed_at_ms: None, - valid_for_ms: None, - }, - ..DataWorkload::default() - }; - let query_workload = QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![RepeatingEntry { - query: Query("sum(sum_over_time(m[1m]))".into()), - demand: RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(10_000), - evaluation_phase: planner_types::workload::TimestampMs(7_000), - }, - requirements: QueryRequirements { - accuracy: AccuracyRequirement::Explicit(AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }), - ..QueryRequirements::default() - }, - predictability: Predictability::Predictable { known_at: None }, - time_selection: TimeSelection { - scope: QueryTimeScope::RealTime, - lookback: None, - as_of: None, - }, - }]), - }; - let mut environment = environment(10_000); - environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.target_collector_ids.clear(); - let template = request("template", "sum(sum_over_time(m[1m]))") - .queries - .remove(0); - let snapshot = BackendLocalPlanningInput { - schema_version: 3, - workload_cost_evidence: None, - query_workload, - data_workload, - physical_inputs: BackendLocalPhysicalInputs { - lifecycle_costs: template.summary_lifecycle_inputs.costs, - evidence_observed_at_unix_ms: 9_500, - evidence_valid_for_ms: 60_000, - horizon_seconds: 300.0, - window_cost_model: WindowCostModel { - implementation_id: "backend-tumbling-v1".into(), - cost: template.window_realization_candidates[0].cost.clone(), - quotes: Vec::new(), - }, - scrape_interval_ms: 1_000, - query_retention_margin_ms: 0, - retained_summary_memory_budget_bytes: DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES, - topk_evidence: HashMap::new(), - exact_composition_costs: HashMap::new(), - erp: None, - }, - environment, - }; - assert_eq!( - snapshot - .clone() - .into_physical_compilation_request() - .unwrap() - .0 - .query_workload - .as_ref(), - Some(&snapshot.query_workload) - ); - let first = quoted_snapshot( - snapshot.clone(), - crate::physical::compiler::QueryFrontend::PromQl, - ) - .compile_promql() - .expect("first deterministic plan"); - let second = quoted_snapshot( - snapshot.clone(), - crate::physical::compiler::QueryFrontend::PromQl, - ) - .compile_promql() - .expect("second deterministic plan"); - assert_eq!(first.envelope, second.envelope); - assert_eq!(first.summary_catalog, second.summary_catalog); - assert_eq!(first.query_plan, second.query_plan); - assert_eq!(first.transmission_plan, second.transmission_plan); - assert_eq!( - serde_json::to_value(&first.precompute_plan).unwrap(), - serde_json::to_value(&second.precompute_plan).unwrap() - ); - - let encoded = serde_json::to_vec(&snapshot).expect("serialize startup snapshot"); - let decoded: BackendLocalPlanningInput = - serde_json::from_slice(&encoded).expect("deserialize startup snapshot"); - let bundle = quoted_snapshot(decoded, crate::physical::compiler::QueryFrontend::PromQl) - .compile_promql() - .expect("canonical startup planning"); - - assert!(bundle.collector_plans.is_empty()); - assert!(bundle.transmission_plan.rules.is_empty()); - assert_eq!( - bundle.precompute_plan.ingest.protocol, - IngestProtocol::PrometheusRemoteWriteV1 - ); - assert!(bundle.precompute_plan.producers.is_empty()); - assert_eq!(bundle.query_plan.entries.len(), 1); - assert_eq!(bundle.envelope.planner_revision, PLANNER_REVISION); - let config = &bundle.precompute_plan.materializations[0]; - assert_eq!(config.pane_origin_ms, Some(7_000)); - assert_eq!( - bundle.precompute_plan.schemas[0].window.pane_origin_ms, - Some(7_000) - ); - assert!(bundle.summary_catalog.outputs.contains_key( - &asap_types::sds::StoredOutputId::from(config.policy_fingerprint()) - )); - assert_eq!( - bundle - .query_plan - .entries - .values() - .next() - .unwrap() - .materialization_bindings()[0] - .pane_origin_ms, - Some(7_000) - ); - } - - #[test] - fn backend_local_compiler_materializes_declared_exact_promql_cases() { - for (query_id, promql, expected_readout) in [ - ( - "q-rate", - "rate(m[1m])", - crate::query_plan::ExactReadout::Rate, - ), - ( - "q-increase", - "increase(m[1m])", - crate::query_plan::ExactReadout::Increase, - ), - ( - "q-sum", - "sum(sum_over_time(m[1m]))", - crate::query_plan::ExactReadout::Sum, - ), - ] { - let mut deployment = environment(10_000); - deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.target_collector_ids.clear(); - let compiled = - DeploymentPlanCompiler.compile_promql(request(query_id, promql), deployment); - let plan = compiled.unwrap_or_else(|error| panic!("{promql} must compile: {error}")); - assert_eq!(plan.summary_catalog.outputs.len(), 1, "{promql}"); - assert_eq!(plan.query_plan.entries.len(), 1, "{promql}"); - assert!(plan.collector_plans.is_empty(), "{promql}"); - let entry = plan.query_plan.entries.values().next().unwrap(); - assert!(matches!( - entry.nodes.values().find(|node| matches!(node, crate::query_plan::QueryPlanNode::ExactReadout { .. })), - Some(crate::query_plan::QueryPlanNode::ExactReadout { readout, .. }) - if *readout == expected_readout - )); - if expected_readout == crate::query_plan::ExactReadout::Rate { - let materialization = plan.precompute_plan.materializations.first().unwrap(); - assert_eq!( - materialization.accumulator_spec().unwrap().family, - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Rate, - planner_types::post_asap::ExactParams::Rate, - ) - ); - } - } - } - - #[test] - fn checked_in_per_entity_snapshot_preserves_native_alternative() { - let source = include_str!("../../../docs/examples/asapquery-planning-snapshot.json"); - let snapshot: BackendLocalPlanningInput = - serde_json::from_str(source).expect("strict canonical workload fixture"); - let encoded = serde_json::to_value(&snapshot).expect("canonical snapshot value"); - let fixture: serde_json::Value = serde_json::from_str(source).expect("fixture JSON"); - assert_eq!(encoded, fixture); - - assert!( - snapshot.clone().compile_promql().is_err(), - "discovery fixtures must be priced before deployment" - ); - let (local, env) = snapshot - .clone() - .into_physical_compilation_request() - .unwrap(); - let isolated = DeploymentPlanCompiler.compile_promql(local, env).unwrap(); - assert!(!isolated.precompute_plan.materializations.is_empty()); - assert!(isolated - .precompute_plan - .materializations - .iter() - .all(|state| state.partitioning.is_some())); - let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let native = - crate::physical::workload_cost::enumerate_exact_and_materialized_candidates(request) - .unwrap() - .pop() - .unwrap(); - let plan = DeploymentPlanCompiler - .compile_promql(native, environment) - .expect("native fixture compiles"); - assert!(plan.precompute_plan.materializations.is_empty()); - assert!(plan.collector_plans.is_empty()); - assert!(plan.transmission_plan.rules.is_empty()); - assert_eq!( - plan.precompute_plan.ingest.protocol, - IngestProtocol::PrometheusRemoteWriteV1 - ); - assert_eq!(plan.query_plan.entries.len(), 1); - } - - #[test] - fn compatibility_demo_preserves_complete_native_query_matrix() { - let source = - include_str!("../../../docs/examples/asapquery-compatibility-demo-snapshot.json"); - let snapshot: BackendLocalPlanningInput = - serde_json::from_str(source).expect("strict compatibility demo fixture"); - assert!( - snapshot.clone().compile_promql().is_err(), - "discovery fixtures must be priced before deployment" - ); - let (local, env) = snapshot - .clone() - .into_physical_compilation_request() - .unwrap(); - let isolated = DeploymentPlanCompiler.compile_promql(local, env).unwrap(); - assert!(!isolated.precompute_plan.materializations.is_empty()); - assert!(isolated - .precompute_plan - .materializations - .iter() - .all(|state| state.partitioning.is_some())); - let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let native = - crate::physical::workload_cost::enumerate_exact_and_materialized_candidates(request) - .unwrap() - .pop() - .unwrap(); - let plan = DeploymentPlanCompiler - .compile_promql(native, environment) - .expect("native demo compiles"); - - assert!(plan.collector_plans.is_empty()); - assert!(plan.transmission_plan.rules.is_empty()); - assert_eq!(plan.query_plan.entries.len(), 6); - assert!(plan.precompute_plan.materializations.is_empty()); - for query in [ - "rate(asap_demo_counter_total[5s])", - "increase(asap_demo_counter_total[5s])", - "sum(sum_over_time(asap_demo_gauge[5s]))", - "quantile_over_time(0.5, asap_demo_latency_ms[5s])", - "topk(1, sum_over_time(asap_demo_gauge[5s]))", - "topk(1, count_over_time(asap_demo_gauge[5s]))", - ] { - assert!(plan.query_plan.lookup(query).is_ok(), "missing {query}"); - } - } - - #[test] - fn multiple_readouts_share_one_precompute_materialization() { - let mut compilation_request = request("q-p90", "quantile_over_time(0.90, m[1m])"); - for (id, q) in [("q-p50", 0.5), ("q-p95", 0.95), ("q-p99", 0.99)] { - compilation_request.queries.push( - request(id, &format!("quantile_over_time({q}, m[1m])")) - .queries - .remove(0), - ); - } - let bundle = DeploymentPlanCompiler - .compile_promql(compilation_request, environment(10_000)) - .unwrap(); - - assert_eq!(bundle.query_plan.entries.len(), 4); - assert_eq!(bundle.summary_catalog.outputs.len(), 1); - assert_eq!(bundle.precompute_plan.materializations.len(), 1); - assert_eq!(bundle.precompute_plan.schemas.len(), 1); - assert_eq!(bundle.precompute_plan.producers.len(), 2); - } - - #[test] - fn compiles_every_materialization_leaf_in_a_merge_dag() { - let mut compilation_request = request("q-merge", "quantile_over_time(0.90, m[1m])"); - let right_request = request("q-right", "quantile_over_time(0.90, n[1m])"); - let left_root = compilation_request.queries[0].selected_plan_root.clone(); - let right_root = right_request.queries[0].selected_plan_root.clone(); - let (left, query) = match &left_root.expr { - SummaryExpr::SummaryEstimate { - summary_input, - query, - } => (summary_input.clone(), query.clone()), - other => panic!("expected selected estimate, got {other:?}"), - }; - let right = match &right_root.expr { - SummaryExpr::SummaryEstimate { summary_input, .. } => summary_input.clone(), - other => panic!("expected selected estimate, got {other:?}"), - }; - let merge = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryMerge { - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - children: vec![left.clone(), right.clone()], - }, - schema: left.schema.clone(), - guarantee: None, - }); - compilation_request.queries[0].selected_plan_root = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryEstimate { - summary_input: merge, - query, - }, - schema: left_root.schema.clone(), - guarantee: left_root.guarantee.clone(), - }); - - let bundle = DeploymentPlanCompiler - .compile_promql(compilation_request, environment(10_000)) - .expect("compile merged post-ASAP DAG"); - assert_eq!(bundle.summary_catalog.outputs.len(), 2); - assert_eq!(bundle.precompute_plan.materializations.len(), 2); - assert_eq!( - bundle - .query_plan - .entries - .values() - .flat_map(QueryPlanEntry::materialization_bindings) - .count(), - 2 - ); - let entry = bundle.query_plan.entries.values().next().unwrap(); - assert_eq!( - entry - .nodes - .values() - .filter(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ReadMaterialization { .. } - )) - .count(), - 2 - ); - assert!(entry.nodes.values().any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::SummaryMerge { inputs } if inputs.len() == 2 - ))); - let bound_metrics = entry - .nodes - .values() - .filter_map(|node| match node { - crate::query_plan::QueryPlanNode::ReadMaterialization { binding } => Some( - bundle.summary_catalog.data_descriptors[&bundle.summary_catalog.outputs - [&binding.materialization] - .data_descriptor_id] - .time_series_metric() - .unwrap(), - ), - _ => None, - }) - .collect::>(); - assert_eq!(bound_metrics, BTreeSet::from(["m", "n"])); - } - - #[test] - fn precompute_schema_must_match_materialization_semantics() { - let bundle = DeploymentPlanCompiler - .compile_promql( - request("q-quantile", "quantile_over_time(0.99, m[1m])"), - environment(10_000), - ) - .unwrap(); - let mut plan = bundle.precompute_plan.clone(); - plan.schemas[0].group_by.push("invented".into()); - assert!(matches!( - plan.validate(), - Err(PrecomputePlanError::InvalidSchema { .. }) - )); - } - - // Concrete candidates with the same framework survive selection; changing - // their quoted costs changes installed state, not the query's lookback. - #[test] - fn pane_sizes_are_selected_by_cost_without_changing_semantic_window() { - for (small_cost, expected_pane_secs, expected_id) in - [(0.1, 10, "small"), (10.0, 60, "large")] - { - let mut request = request("q", "sum(sum_over_time(m[1m]))"); - let query = &mut request.queries[0]; - let mut small = query.window_realization_candidates[0].clone(); - small.realization_id = "small".into(); - small.layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 10 }; - small.cost.weighted_cost = small_cost; - query.window_realization_candidates[0].realization_id = "large".into(); - query.window_realization_candidates.push(small); - let mut env = environment(10_000); - env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - env.target_collector_ids.clear(); - let bundle = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - let materialization = bundle.precompute_plan.materializations.first().unwrap(); - assert_eq!(materialization.window_size, 60); - assert_eq!( - materialization.window_layout.base_pane_secs(), - expected_pane_secs - ); - let entry = bundle - .query_plan - .lookup("sum(sum_over_time(m[1m]))") - .unwrap(); - assert_eq!(entry.instant.lookback_ms, 60_000); - assert_eq!( - entry.materialization_bindings()[0].window_ms, - expected_pane_secs * 1000 - ); - assert_eq!( - bundle.lifecycle_estimates[0].window_realization_id, - expected_id - ); - } - } - - #[test] - fn sliding_layout_cost_selects_query_or_update_optimized_state() { - for (full_cost, expected_id, full_selected) in [ - (0.1, "full-window", true), - (100.0, "mergeable-panes", false), - ] { - let mut request = request("q", "sum(sum_over_time(m[1m]))"); - let query = &mut request.queries[0]; - let mut panes = query.window_realization_candidates[0].clone(); - panes.realization_id = "mergeable-panes".into(); - panes.framework = SummaryWindowFramework::Sliding; - panes.slide_secs = 10; - panes.layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 10 }; - panes.cost.weighted_cost = 1.0; - panes.cost.cpu_cost = 0.2; - panes.cost.storage_bytes = 1_024; - - let mut full = panes.clone(); - full.realization_id = "full-window".into(); - full.layout = asap_types::WindowMaterializationLayout::FullWindow; - full.cost.weighted_cost = full_cost; - // Full windows spend more update CPU and retained bytes, while - // avoiding query-time pane merges. The provider's weighted quote - // includes the workload's measured read frequency and cardinality. - full.cost.cpu_cost = 20.0; - full.cost.storage_bytes = 64 * 1_024; - query.window_realization_candidates = vec![panes, full]; - - let mut env = environment(10_000); - env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - env.target_collector_ids.clear(); - let bundle = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - assert_eq!( - bundle.lifecycle_estimates[0].window_realization_id, - expected_id - ); - assert_eq!( - matches!( - bundle.precompute_plan.materializations[0].window_layout, - asap_types::WindowMaterializationLayout::FullWindow - ), - full_selected - ); - // Publication validates stored extent independently of slide cadence. - // In particular the full-window candidate stores 60s at a 10s slide. - bundle - .to_publication_artifact() - .unwrap() - .validate() - .unwrap(); - assert_eq!(bundle.precompute_plan.materializations[0].window_size, 60); - assert_eq!( - bundle.precompute_plan.materializations[0].slide_interval, - 10 - ); - } - } - - // Distinct semantic windows remain distinct definitions even when they use - // an equal base-pane width and implementation provider. - #[test] - fn selected_panes_reject_unpriced_cross_cohort_coalescing() { - let mut workload = request("q20", "sum(sum_over_time(m[20s]))"); - let mut second = request("q40", "sum(sum_over_time(m[40s]))") - .queries - .remove(0); - workload.queries[0].query_lookback_seconds = 20; - workload.queries[0].window_realization_candidates[0].window_secs = 20; - second.query_lookback_seconds = 40; - second.summary_lifecycle_inputs.evaluation_interval_ms = 20_000; - second.window_realization_candidates[0].window_secs = 40; - workload.queries.push(second); - for query in &mut workload.queries { - query.window_realization_candidates[0].framework = SummaryWindowFramework::Sliding; - query.window_realization_candidates[0].slide_secs = 10; - query.window_realization_candidates[0].layout = - asap_types::WindowMaterializationLayout::Pane { pane_secs: 10 }; - query.window_realization_candidates[0].realization_id = "shared-ten-second-pane".into(); - } - let mut env = environment(10_000); - env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - env.target_collector_ids.clear(); - let bundle = DeploymentPlanCompiler - .compile_promql(workload, env) - .unwrap(); - assert_eq!(bundle.precompute_plan.materializations.len(), 2); - } - - // Non-divisor panes cannot reconstruct a lookback from whole states. - #[test] - fn tumbling_sizes_reject_non_divisors() { - let mut request = request("q", "sum(sum_over_time(m[1m]))"); - request.queries[0].window_realization_candidates[0].layout = - asap_types::WindowMaterializationLayout::Pane { pane_secs: 7 }; - let mut env = environment(10_000); - env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - assert!(DeploymentPlanCompiler.compile_promql(request, env).is_err()); - } - - #[test] - fn missing_window_implementation_evidence_fails_closed() { - let mut request = request("q-window", "quantile_over_time(0.99, m[1m])"); - request.queries[0].window_realization_candidates.clear(); - let error = DeploymentPlanCompiler - .compile_promql(request, environment(10_000)) - .expect_err("Planner must not receive a zero-cost invented window"); - assert!(matches!(error, CompileError::Lifecycle { .. })); - } - - #[test] - fn precompute_plan_rejects_schema_or_producer_drift() { - let bundle = DeploymentPlanCompiler - .compile_promql( - request("q-quantile", "quantile_over_time(0.99, m[1m])"), - environment(10_000), - ) - .expect("compile"); - - let mut schema_drift = bundle.precompute_plan.clone(); - schema_drift.schemas[0].schema_version = 0; - assert!(matches!( - schema_drift.validate(), - Err(PrecomputePlanError::InvalidSchema { .. }) - )); - - let mut producer_drift = bundle.precompute_plan; - producer_drift.producers[0].schema_id = "wrong".into(); - assert!(matches!( - producer_drift.validate(), - Err(PrecomputePlanError::InvalidProducer { .. }) - )); - } - - #[test] - fn precompute_plan_rejects_empty_or_duplicate_schema_ids() { - let bundle = DeploymentPlanCompiler - .compile_promql( - request("q-quantile", "quantile_over_time(0.99, m[1m])"), - environment(10_000), - ) - .expect("compile schema"); - - let mut empty = bundle.precompute_plan.clone(); - empty.schemas[0].schema_id.clear(); - assert!(matches!( - empty.validate(), - Err(PrecomputePlanError::InvalidSchema { .. }) - )); - - let mut duplicate = bundle.precompute_plan; - duplicate.schemas.push(duplicate.schemas[0].clone()); - assert!(matches!( - duplicate.validate(), - Err(PrecomputePlanError::InvalidSchema { .. }) - )); - } - - #[test] - fn topk_fails_closed_without_membership_evidence() { - let request = request_with_evidence("q-topk", "topk(5, m)", None).unwrap(); - assert!(request.queries[0] - .selected_plan_root - .guarantee - .as_ref() - .unwrap() - .is_exact()); - } - - #[test] - fn stale_topk_evidence_is_rejected_before_planner_selection() { - let request = request_with_evidence( - "q-topk", - "topk(5, count_over_time(m[1m]))", - Some(TopKMembershipEvidence { - selected_lower_bound: 101.0, - excluded_upper_bound: 100.0, - interval_failure_probability: 0.005, - observed_at_unix_ms: 1, - source: "runtime-margin-monitor".into(), - }), - ) - .expect("selection accepts evidence before freshness validation"); - let error = DeploymentPlanCompiler - .compile_promql(request, environment(100_000)) - .expect_err("stale certificate must fail"); - assert!(matches!(error, CompileError::InvalidEvidence { .. })); - } - - #[test] - fn future_topk_and_window_evidence_are_rejected() { - let topk = request_with_evidence( - "q-topk", - "topk(5, count_over_time(m[1m]))", - Some(TopKMembershipEvidence { - selected_lower_bound: 101.0, - excluded_upper_bound: 100.0, - interval_failure_probability: 0.005, - observed_at_unix_ms: 10_001, - source: "runtime-margin-monitor".into(), - }), - ) - .expect("selection occurs before deployment-time freshness validation"); - assert!(matches!( - DeploymentPlanCompiler.compile_promql(topk, environment(10_000)), - Err(CompileError::InvalidEvidence { .. }) - )); - - let mut window = request("q-window", "quantile_over_time(0.99, m[1m])"); - window.queries[0].window_realization_candidates[0] - .cost - .observed_at_unix_ms = 10_001; - assert!(matches!( - DeploymentPlanCompiler.compile_promql(window, environment(10_000)), - Err(CompileError::Lifecycle { .. }) - )); - } - - #[test] - fn fresh_topk_evidence_enables_physical_compilation() { - let request = request_with_evidence( - "q-topk", - "topk(5, count_over_time(m[1m]))", - Some(TopKMembershipEvidence { - selected_lower_bound: 101.0, - excluded_upper_bound: 100.0, - interval_failure_probability: 0.005, - observed_at_unix_ms: 9_500, - source: "runtime-margin-monitor".into(), - }), - ) - .expect("selection accepts valid evidence"); - let bundle = DeploymentPlanCompiler - .compile_promql(request, environment(10_000)) - .expect("certified TopK compiles"); - assert_eq!(bundle.summary_catalog.outputs.len(), 1); - assert_eq!( - bundle.collector_plans[0].materializations[0] - .evidence_source - .as_deref(), - Some("runtime-margin-monitor") - ); - } - - #[test] - fn planner_revision_is_part_of_the_compile_contract() { - let mut request = request("q", "quantile_over_time(0.9, m[1m])"); - request.planner_revision = "different".into(); - assert!(matches!( - DeploymentPlanCompiler.compile_promql(request, environment(10_000)), - Err(CompileError::PlannerRevision { .. }) - )); - } - - #[test] - fn stale_lifecycle_evidence_fails_closed() { - let mut request = request("q", "quantile_over_time(0.9, m[1m])"); - request.queries[0] - .summary_lifecycle_inputs - .evidence_observed_at_unix_ms = 1; - request.queries[0] - .summary_lifecycle_inputs - .evidence_valid_for_ms = 10; - assert!(matches!( - DeploymentPlanCompiler.compile_promql(request, environment(10_000)), - Err(CompileError::Lifecycle { .. }) - )); - } - - #[test] - fn runtime_policy_uses_canonical_sampling_and_gos_allocators() { - let sampling = sampling_policy_from_accuracy_budget( - 0.05, - 1_000.0, - SamplingEstimator::GeometricAdmission, - ); - let SamplingPolicy::Fixed { probability, .. } = sampling else { - panic!("positive budget and rate must enable sampling"); - }; - assert!((probability - 1.0 / 3.5).abs() < 1e-9); - - let gos = - gos_policy_from_accuracy_budget(0.1, 0.03, 4, 1.0, 1.0, GosThresholdMode::Isotropic) - .expect("positive communication allocation"); - assert!((gos.epsilon_staleness - 0.035).abs() < 1e-12); - assert_eq!(gos.sites, 4); - } - - #[test] - fn runtime_policy_encoding_is_checked() { - let bundle = DeploymentPlanCompiler - .compile_promql( - request("q", "quantile_over_time(0.99, m[1m])"), - environment(10_000), - ) - .expect("compile"); - let mut invalid_encoding = bundle.transmission_plan.clone(); - invalid_encoding.rules[0].encoding = StateEncoding::ExactAccumulatorV1; - assert!(matches!( - invalid_encoding.validate(&bundle.precompute_plan), - Err(TransmissionPlanError::InvalidRule(_)) - )); - } - - #[test] - fn compiler_emits_delta_rule_and_accepts_periodic_full_checkpoint() { - let mut request = request_with_evidence( - "q-topk", - "topk(5, count_over_time(m[1m]))", - Some(TopKMembershipEvidence { - selected_lower_bound: 101.0, - excluded_upper_bound: 100.0, - interval_failure_probability: 0.005, - observed_at_unix_ms: 9_500, - source: "runtime-margin-monitor".into(), - }), - ) - .expect("frequency selection"); - request.queries[0].materialization_runtime_policy.delta = Some(DeltaPolicy { - absolute_threshold: 0.0, - gos: None, - }); - let bundle = DeploymentPlanCompiler - .compile_promql(request, environment(10_000)) - .expect("delta-capable physical plan"); - let rule = &bundle.transmission_plan.rules[0]; - assert_eq!(rule.mode, TransmissionMode::Delta); - assert!(rule.full_checkpoint_every_ms.is_some()); - - for (kind, sequence) in [(SummaryFrameKind::Full, 1), (SummaryFrameKind::Delta, 2)] { - let frame = SummaryFrameIdentity { - identity_version: 1, - plan_id: bundle.envelope.plan_id, - plan_version: bundle.envelope.plan_version, - backend_compat: bundle.envelope.backend_compat.clone(), - materialization: rule.materialization, - series_identity: "service=checkout,zone=a".into(), - schema_id: rule.schema_id.clone(), - producer_id: rule.producer_id.clone(), - producer_epoch: "boot-1".into(), - window_start_unix_nano: 1, - window_end_unix_nano: 2, - sequence, - encoding: rule.encoding.clone(), - checkpoint_id: (kind == SummaryFrameKind::Full).then(|| "cp-1".into()), - base_checkpoint_id: (kind == SummaryFrameKind::Delta).then(|| "cp-1".into()), - kind, - }; - bundle - .transmission_plan - .validate_frame(&frame) - .expect("delta rule accepts its deltas and recovery full frames"); - } - } - - #[test] - fn runtime_adaptation_requires_fresh_exact_evidence_and_successor_version() { - let bundle = DeploymentPlanCompiler - .compile_promql( - request("q", "quantile_over_time(0.99, m[1m])"), - environment(10_000), - ) - .expect("compile"); - let mut current = bundle.transmission_plan; - let rule = &mut current.rules[0]; - rule.runtime_policy.adaptation = RuntimeAdaptationPolicy { - enabled: true, - not_before_unix_ms: 10_500, - max_evidence_age_ms: 5_000, - min_evidence_samples: 100, - sample_probability: None, - emit_every_ms: Some(AdaptiveU64Bounds { - min: 30_000, - max: 120_000, - max_step: 10_000, - }), - delta_threshold: None, - gos_epsilon_staleness: None, - }; - let mut successor = current.clone(); - successor.envelope.plan_version += 1; - successor.envelope.generated_at_unix_ms = 11_000; - successor.envelope.activation_unix_ms = 11_000; - successor.rules[0].emit_every_ms += 5_000; - let evidence = RuntimeAdaptationEvidence { - plan_id: current.envelope.plan_id, - plan_version: current.envelope.plan_version, - materialization: current.rules[0].materialization, - producer_id: current.rules[0].producer_id.clone(), - schema_id: current.rules[0].schema_id.clone(), - producer_version: "collector.v1".into(), - observed_at_unix_ms: 10_900, - sample_count: 100, - }; - current - .authorize_successor(&successor, std::slice::from_ref(&evidence), 11_000) - .expect("bounded change with exact fresh evidence"); - - let mut oversized = successor.clone(); - oversized.rules[0].emit_every_ms += 20_000; - assert!(matches!( - current.authorize_successor(&oversized, std::slice::from_ref(&evidence), 11_000), - Err(TransmissionPlanError::AdaptationOutOfBounds { .. }) - )); - let stale = RuntimeAdaptationEvidence { - observed_at_unix_ms: 1, - ..evidence.clone() - }; - assert!(matches!( - current.authorize_successor(&successor, &[stale], 11_000), - Err(TransmissionPlanError::InvalidAdaptationEvidence(_)) - )); - let future = RuntimeAdaptationEvidence { - observed_at_unix_ms: 11_001, - ..evidence - }; - assert!(matches!( - current.authorize_successor(&successor, &[future], 11_000), - Err(TransmissionPlanError::InvalidAdaptationEvidence(_)) - )); - let mut different_plan_id = successor.clone(); - different_plan_id.envelope.plan_id += 1; - assert!(matches!( - current.authorize_successor(&different_plan_id, &[], 11_000), - Err(TransmissionPlanError::InvalidSuccessor(_)) - )); - let mut in_place = successor; - in_place.envelope.plan_version = current.envelope.plan_version; - assert!(matches!( - current.authorize_successor(&in_place, &[], 11_000), - Err(TransmissionPlanError::InvalidSuccessor(_)) - )); - } -} diff --git a/control_plane/src/physical/compiler/windows.rs b/control_plane/src/physical/compiler/windows.rs deleted file mode 100644 index 072ee7c2e..000000000 --- a/control_plane/src/physical/compiler/windows.rs +++ /dev/null @@ -1,377 +0,0 @@ -//! Generate feasible layouts from selected state requirements before pricing them. -use super::*; -use asap_types::WindowMaterializationLayout; - -/// One input contract for snapshot and HTTP window planning. Quotes apply only -/// to their exact shape and workload; absent quotes use the lifecycle unit costs. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct WindowCostModel { - pub implementation_id: String, - pub cost: WindowRealizationCostQuote, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub quotes: Vec, -} - -pub(in crate::physical) fn gcd(mut a: u64, mut b: u64) -> u64 { - while b != 0 { - (a, b) = (b, a % b); - } - a -} - -pub(super) fn is_full_cohort(candidate: &WindowRealizationCandidate) -> bool { - candidate.slide_secs == candidate.window_secs - && candidate.layout - == WindowMaterializationLayout::Pane { - pane_secs: candidate.window_secs, - } -} - -pub(super) fn cohort_nodes(states: &[SelectedMaterialization]) -> BTreeSet { - let mut nodes = BTreeSet::new(); - for state in states { - if let Some(sources) = immutable_materialization_sources(&state.node) { - nodes.insert(Rc::as_ptr(&state.node) as usize); - nodes.extend(sources.iter().map(|source| Rc::as_ptr(source) as usize)); - } - } - nodes -} - -pub(super) fn supported( - candidate: &WindowRealizationCandidate, - target: PhysicalDeploymentTarget, -) -> bool { - candidate - .layout - .validate(candidate.window_secs, candidate.slide_secs) - .is_ok() - && match (&candidate.framework, &candidate.layout) { - ( - SummaryWindowFramework::Tumbling | SummaryWindowFramework::Sliding, - WindowMaterializationLayout::Pane { pane_secs }, - ) => { - target == PhysicalDeploymentTarget::BackendLocalRemoteWrite - || *pane_secs == candidate.window_secs - } - (SummaryWindowFramework::Sliding, WindowMaterializationLayout::FullWindow) => { - // Collector schedulers currently support overlapping or tumbling windows only. - target == PhysicalDeploymentTarget::BackendLocalRemoteWrite - || candidate.slide_secs <= candidate.window_secs - } - _ => false, - } -} - -pub(super) fn derive( - model: &WindowCostModel, - lifecycle: &SummaryLifecyclePlanningInputs, - window_secs: u64, - full_cohort: bool, - target: PhysicalDeploymentTarget, - staleness_margin_ms: u64, -) -> Vec { - let evaluation_ms = u64::from(lifecycle.evaluation_interval_ms); - // Runtime layouts have second precision. Never truncate a fractional cadence. - if window_secs == 0 || evaluation_ms == 0 || evaluation_ms % 1_000 != 0 { - return Vec::new(); - } - let evaluation_secs = evaluation_ms / 1_000; - let pane_secs = if full_cohort { - window_secs - } else { - gcd(window_secs, evaluation_secs) - }; - let slide_secs = if full_cohort { - window_secs - } else { - evaluation_secs - }; - let mut layouts = vec![WindowMaterializationLayout::Pane { pane_secs }]; - if !full_cohort && evaluation_secs != window_secs { - layouts.push(WindowMaterializationLayout::FullWindow); - } - layouts - .into_iter() - .filter_map(|layout| { - let suffix = match layout { - WindowMaterializationLayout::Pane { pane_secs } => format!("pane-{pane_secs}s"), - _ => "full-window".into(), - }; - let candidate = WindowRealizationCandidate { - realization_id: format!( - "{}-{window_secs}s-slide-{slide_secs}s-{suffix}", - model.implementation_id - ), - framework: if slide_secs == window_secs { - SummaryWindowFramework::Tumbling - } else { - SummaryWindowFramework::Sliding - }, - window_secs, - slide_secs, - cost: derived_window_cost( - &model.cost, - lifecycle, - window_secs, - slide_secs, - &layout, - staleness_margin_ms, - ), - layout, - derived: true, - cohort_only: full_cohort && slide_secs != evaluation_secs, - }; - supported(&candidate, target).then_some(candidate) - }) - .collect() -} - -pub fn prepare_window_implementations( - query: &mut QueryCompilationInput, - model: &WindowCostModel, - target: PhysicalDeploymentTarget, - staleness_margin_ms: u64, -) -> Result<(), CompileError> { - if model.implementation_id.trim().is_empty() { - return Err(CompileError::Lifecycle { - query_id: query.query_id.clone(), - reason: "window cost model requires an implementation identity".into(), - }); - } - let mut model = model.clone(); - let fingerprint = canonical_promql(&query.query_string).map_err(CompileError::QueryPlan)?; - model.cost.workload_fingerprint = fingerprint.clone(); - model.cost.horizon_seconds = query.summary_lifecycle_inputs.horizon_seconds; - let states = collect_selected_materializations( - &query.selected_plan_root, - target == PhysicalDeploymentTarget::BackendLocalRemoteWrite, - ) - .map_err(|reason| CompileError::Query { - query_id: query.query_id.clone(), - reason, - })?; - let cohorts = cohort_nodes(&states); - let requirements = states - .iter() - .map(|state| { - ( - state.window_secs.unwrap_or(query.query_lookback_seconds), - cohorts.contains(&(Rc::as_ptr(&state.node) as usize)), - ) - }) - .collect::>(); - let mut candidates = requirements - .iter() - .flat_map(|&(window, cohort)| { - derive( - &model, - &query.summary_lifecycle_inputs, - window, - cohort, - target, - staleness_margin_ms, - ) - }) - .collect::>(); - let mut quote_shapes = BTreeSet::new(); - for quote in model - .quotes - .iter() - .filter(|q| q.cost.workload_fingerprint == fingerprint) - { - let shape = ( - quote.window_secs, - quote.slide_secs, - serde_json::to_string("e.layout).unwrap(), - ); - if !quote_shapes.insert(shape) { - return Err(CompileError::Lifecycle { - query_id: query.query_id.clone(), - reason: "duplicate measured window layout".into(), - }); - } - let applicable = requirements.iter().any(|&(window, cohort)| { - quote.window_secs == window - && if cohort { - is_full_cohort(quote) - } else { - quote.slide_secs.saturating_mul(1_000) - == u64::from(query.summary_lifecycle_inputs.evaluation_interval_ms) - } - }); - if !applicable || !supported(quote, target) { - return Err(CompileError::Lifecycle { - query_id: query.query_id.clone(), - reason: format!( - "window quote `{}` does not match an executable state layout", - quote.realization_id - ), - }); - } - candidates.retain(|candidate| { - !(candidate.window_secs == quote.window_secs - && candidate.slide_secs == quote.slide_secs - && candidate.layout == quote.layout) - }); - let mut quote = quote.clone(); - quote.derived = false; - quote.cohort_only = quote.slide_secs.saturating_mul(1_000) - != u64::from(query.summary_lifecycle_inputs.evaluation_interval_ms); - candidates.push(quote); - } - let mut unique = BTreeMap::new(); - for candidate in &candidates { - if let Some(previous) = unique.insert(&candidate.realization_id, candidate) { - if previous != candidate { - return Err(CompileError::Lifecycle { - query_id: query.query_id.clone(), - reason: "window implementation identity describes different offers".into(), - }); - } - } - } - let mut ids = BTreeSet::new(); - candidates.retain(|candidate| ids.insert(candidate.realization_id.clone())); - query.window_realization_candidates = candidates; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn snapshot() -> BackendLocalPlanningInput { - serde_json::from_str(include_str!( - "../../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap() - } - - // Target constraints eliminate infeasible offers before lifecycle selection. - #[test] - fn collector_generation_excludes_partial_panes_and_sparse_full_windows() { - let snapshot = snapshot(); - let model = snapshot.physical_inputs.window_cost_model.clone(); - let (request, _) = snapshot.into_physical_compilation_request().unwrap(); - let mut lifecycle = request.queries[0].summary_lifecycle_inputs.clone(); - for (interval, expected_count) in [ - (20_000, 1), - (45_000, 1), - (60_000, 1), - (90_000, 0), - (120_000, 1), - ] { - lifecycle.evaluation_interval_ms = interval; - let candidates = derive( - &model, - &lifecycle, - 60, - false, - PhysicalDeploymentTarget::DistributedCollectors, - 0, - ); - assert_eq!(candidates.len(), expected_count, "{interval}"); - assert!(candidates.iter().all(|candidate| supported( - candidate, - PhysicalDeploymentTarget::DistributedCollectors - ))); - } - } - - // A quote cannot authorize a pane that cuts scheduled ranges or an unimplemented rollup. - #[test] - fn quotes_cannot_bypass_layout_or_runtime_constraints() { - let snapshot = snapshot(); - let mut model = snapshot.physical_inputs.window_cost_model.clone(); - let (request, _) = snapshot.into_physical_compilation_request().unwrap(); - let mut query = request.queries[0].clone(); - query.summary_lifecycle_inputs.evaluation_interval_ms = 20_000; - prepare_window_implementations( - &mut query, - &model, - PhysicalDeploymentTarget::BackendLocalRemoteWrite, - 0, - ) - .unwrap(); - let mut quote = query.window_realization_candidates[0].clone(); - quote.layout = WindowMaterializationLayout::Pane { pane_secs: 30 }; - model.quotes = vec![quote.clone()]; - assert!(prepare_window_implementations( - &mut query, - &model, - PhysicalDeploymentTarget::BackendLocalRemoteWrite, - 0 - ) - .is_err()); - quote.framework = SummaryWindowFramework::ExponentialHistogram; - quote.layout = WindowMaterializationLayout::HierarchicalRollup { - base_pane_secs: 10, - levels_secs: vec![30], - }; - assert!(quote - .layout - .validate(quote.window_secs, quote.slide_secs) - .is_ok()); - assert!(!supported( - "e, - PhysicalDeploymentTarget::BackendLocalRemoteWrite - )); - model.quotes = vec![quote]; - let error = prepare_window_implementations( - &mut query, - &model, - PhysicalDeploymentTarget::BackendLocalRemoteWrite, - 0, - ) - .unwrap_err(); - assert!(matches!( - error, - CompileError::Lifecycle { reason, .. } - if reason.contains("does not match an executable state layout") - )); - } - - // Conflicting identities and repeated shape evidence must fail rather than silently lose an offer. - #[test] - fn conflicting_quote_ids_and_duplicate_shapes_are_rejected() { - let snapshot = snapshot(); - let mut model = snapshot.physical_inputs.window_cost_model.clone(); - let (request, _) = snapshot.into_physical_compilation_request().unwrap(); - let mut query = request.queries[0].clone(); - let mut quote = query.window_realization_candidates[0].clone(); - quote.realization_id = query.window_realization_candidates[1] - .realization_id - .clone(); - model.quotes = vec![quote.clone()]; - assert!(prepare_window_implementations( - &mut query, - &model, - PhysicalDeploymentTarget::BackendLocalRemoteWrite, - 0 - ) - .is_err()); - quote.realization_id = "duplicate-shape".into(); - model.quotes.push(quote); - assert!(prepare_window_implementations( - &mut query, - &model, - PhysicalDeploymentTarget::BackendLocalRemoteWrite, - 0 - ) - .is_err()); - } - - // External serialization never grants permission to reinterpret measured prices. - #[test] - fn serialized_generated_quote_loses_compiler_provenance() { - let (request, _) = snapshot().into_physical_compilation_request().unwrap(); - let candidate = &request.queries[0].window_realization_candidates[0]; - assert!(candidate.derived); - let value = serde_json::to_value(candidate).unwrap(); - assert!(value.get("derived").is_none()); - let decoded: WindowRealizationCandidate = serde_json::from_value(value).unwrap(); - assert!(!decoded.derived); - } -} diff --git a/control_plane/src/physical/deployment_cost/mod.rs b/control_plane/src/physical/deployment_cost/mod.rs deleted file mode 100644 index 0c3a8d7cb..000000000 --- a/control_plane/src/physical/deployment_cost/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Deployment cost inputs that survive outside the planner. -//! -//! * [`online`] — EMA-smoothed per-sketch cost observations fed by agent -//! runtime samples, exposed through `GET /api/v1/cost-model`. -//! * [`tco`] — standalone cloud total-cost estimator behind `POST /api/v1/tco`. -//! * [`wire`] — wire-cost table the post-ASAP cost model prices against. - -pub mod online; -pub mod tco; -pub mod wire; diff --git a/control_plane/src/physical/deployment_cost/online.rs b/control_plane/src/physical/deployment_cost/online.rs deleted file mode 100644 index ee711358c..000000000 --- a/control_plane/src/physical/deployment_cost/online.rs +++ /dev/null @@ -1,282 +0,0 @@ -//! SP-5 online profiling — EMA-adjusted physical cost constants. -//! -//! The static benchmark table in [`cost_model`] seeds initial estimates. -//! As [`monitor::Scraper`] collects actual observations from running agents, -//! this module blends them in via an Exponential Moving Average (EMA) so the -//! planner's cost estimates converge toward real-world behaviour. -//! -//! # Blending weight -//! -//! After `MIN_OBS` observations the online weight reaches `MAX_ONLINE_WEIGHT` -//! (70 %). Below that the benchmark retains the majority share so a single -//! noisy scrape cannot destabilise the planner. -//! -//! ```text -//! online_weight = min(observations / MIN_OBS, 1.0) × MAX_ONLINE_WEIGHT -//! effective = online_weight × observed + (1 − online_weight) × benchmark -//! ``` - -use std::collections::HashMap; -use std::sync::Arc; - -use tokio::sync::RwLock; - -use crate::types::SketchType; - -// ── Benchmark-derived cost table ───────────────────────────────────────────── -// -// Source: e2e benchmark results (2026-03-15), 1000 series x 1000 Hz row. -// Units: bandwidth bytes/series/sec, CPU us/sample, memory bytes/sketch. -// These are the priors the EMA smooths towards until enough runtime samples -// arrive from the agents. - -#[derive(Debug, Clone, Copy)] -pub struct SketchCosts { - pub bytes_per_series_per_sec: f64, - pub cpu_micros_per_sample: f64, - pub base_memory_bytes: f64, - pub relative_error_at_default: f64, -} - -/// Public accessor used by `apply_delta_decision` to retrieve the cost table. -pub fn benchmark_table_pub() -> HashMap { - benchmark_table() -} - -fn benchmark_table() -> HashMap { - [ - ( - SketchType::DDSketch, - SketchCosts { - bytes_per_series_per_sec: 120.0, - cpu_micros_per_sample: 0.8, - base_memory_bytes: 4_096.0, - relative_error_at_default: 0.01, - }, - ), - ( - SketchType::KLL, - SketchCosts { - bytes_per_series_per_sec: 80.0, - cpu_micros_per_sample: 0.5, - base_memory_bytes: 2_048.0, - relative_error_at_default: 0.02, - }, - ), - ( - SketchType::HLL, - SketchCosts { - bytes_per_series_per_sec: 40.0, - cpu_micros_per_sample: 0.3, - base_memory_bytes: 16_384.0, // precision=14 → 16 KB - relative_error_at_default: 0.008, - }, - ), - ( - SketchType::CountSketch, - SketchCosts { - bytes_per_series_per_sec: 200.0, - cpu_micros_per_sample: 1.2, - base_memory_bytes: 40_960.0, - relative_error_at_default: 0.01, - }, - ), - ( - SketchType::CountMinSketch, - SketchCosts { - bytes_per_series_per_sec: 200.0, - cpu_micros_per_sample: 1.0, - base_memory_bytes: 40_960.0, - relative_error_at_default: 0.01, - }, - ), - ] - .into() -} - -// ── Tuning constants ────────────────────────────────────────────────────────── - -/// EMA smoothing factor α. Smaller → slower adaptation, more stable. -const EMA_ALPHA: f64 = 0.15; - -/// Minimum number of observations before online data gets majority weight. -const MIN_OBS: f64 = 5.0; - -/// Maximum fraction of weight given to online (observed) data. -const MAX_ONLINE_WEIGHT: f64 = 0.70; - -// ── Types ───────────────────────────────────────────────────────────────────── - -/// EMA-smoothed cost estimates for one sketch type. -#[derive(Debug, Clone)] -pub struct OnlineSketchCosts { - /// EMA of observed bytes/series/sec transmitted by agents using this sketch. - pub bw_bytes_per_series_per_sec: f64, - /// EMA of observed CPU µs/sample at the agent collector. - pub cpu_micros_per_sample: f64, - /// Number of observations incorporated so far. - pub observations: u64, -} - -impl OnlineSketchCosts { - fn from_benchmark(base: &SketchCosts) -> Self { - Self { - bw_bytes_per_series_per_sec: base.bytes_per_series_per_sec, - cpu_micros_per_sample: base.cpu_micros_per_sample, - observations: 0, - } - } - - /// Incorporate a new observation. - pub fn update(&mut self, observed_bw: f64, observed_cpu: f64) { - self.bw_bytes_per_series_per_sec = - EMA_ALPHA * observed_bw + (1.0 - EMA_ALPHA) * self.bw_bytes_per_series_per_sec; - self.cpu_micros_per_sample = - EMA_ALPHA * observed_cpu + (1.0 - EMA_ALPHA) * self.cpu_micros_per_sample; - self.observations += 1; - } - - /// Returns a `SketchCosts` that blends the EMA observation with `base`. - /// The online weight grows with observation count, capping at 70 %. - pub fn effective_costs(&self, base: &SketchCosts) -> SketchCosts { - let online_w = (self.observations as f64 / MIN_OBS).min(1.0) * MAX_ONLINE_WEIGHT; - let bench_w = 1.0 - online_w; - SketchCosts { - bytes_per_series_per_sec: online_w * self.bw_bytes_per_series_per_sec - + bench_w * base.bytes_per_series_per_sec, - cpu_micros_per_sample: online_w * self.cpu_micros_per_sample - + bench_w * base.cpu_micros_per_sample, - base_memory_bytes: base.base_memory_bytes, - relative_error_at_default: base.relative_error_at_default, - } - } -} - -// ── Store ───────────────────────────────────────────────────────────────────── - -/// Thread-safe map of sketch type → online EMA costs. -/// Shared between the `Scraper`'s `on_metrics` callback and `DeploymentCostPlanner`. -pub type OnlineMetricsStore = Arc>>; - -/// Initialise the store from the static benchmark table. -pub fn init_store() -> OnlineMetricsStore { - let map = benchmark_table_pub() - .iter() - .map(|(st, c)| (st.clone(), OnlineSketchCosts::from_benchmark(c))) - .collect(); - Arc::new(RwLock::new(map)) -} - -/// Update the EMA for `sketch_type` with a new bandwidth/cpu observation. -/// If `sketch_type` is not yet in the store it is inserted from the benchmark. -pub async fn update( - store: &OnlineMetricsStore, - sketch_type: &SketchType, - observed_bw: f64, - observed_cpu: f64, -) { - let mut map = store.write().await; - let entry = map.entry(sketch_type.clone()).or_insert_with(|| { - let base = benchmark_table_pub(); - let costs = base.get(sketch_type).cloned().unwrap_or(SketchCosts { - bytes_per_series_per_sec: 200.0, - cpu_micros_per_sample: 1.0, - base_memory_bytes: 4096.0, - relative_error_at_default: 0.01, - }); - OnlineSketchCosts::from_benchmark(&costs) - }); - entry.update(observed_bw, observed_cpu); -} - -/// Returns an effective cost table that blends online EMA data with benchmarks. -/// Uses `try_read()` so callers in hot paths never block. -pub fn effective_table(store: &OnlineMetricsStore) -> HashMap { - let base = benchmark_table_pub(); - match store.try_read() { - Ok(map) => base - .iter() - .map(|(st, b)| { - let eff = map - .get(st) - .map(|o| o.effective_costs(b)) - .unwrap_or_else(|| *b); - (st.clone(), eff) - }) - .collect(), - Err(_) => base, // fall back to benchmark if lock is contended - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::types::SketchType; - - fn base() -> SketchCosts { - SketchCosts { - bytes_per_series_per_sec: 100.0, - cpu_micros_per_sample: 1.0, - base_memory_bytes: 4096.0, - relative_error_at_default: 0.01, - } - } - - #[test] - fn no_observations_returns_benchmark() { - let o = OnlineSketchCosts::from_benchmark(&base()); - let eff = o.effective_costs(&base()); - assert_eq!(eff.bytes_per_series_per_sec, 100.0); - } - - #[test] - fn after_min_obs_online_dominates() { - let mut o = OnlineSketchCosts::from_benchmark(&base()); - for _ in 0..10 { - o.update(200.0, 2.0); // observed = 2× benchmark - } - let eff = o.effective_costs(&base()); - // Online weight = 1.0 × 0.70 = 0.70 - // effective = 0.70 × online_ema + 0.30 × benchmark - // online_ema after 10 EMA steps from 100→200: converging toward 200 - assert!( - eff.bytes_per_series_per_sec > 100.0, - "effective bw should exceed benchmark after high observations: {}", - eff.bytes_per_series_per_sec - ); - } - - #[test] - fn single_observation_has_low_weight() { - let mut o = OnlineSketchCosts::from_benchmark(&base()); - o.update(1000.0, 10.0); // extreme spike - let eff = o.effective_costs(&base()); - // online_weight = (1/5).min(1.0) × 0.70 = 0.14 - // should only move a little from benchmark - assert!( - eff.bytes_per_series_per_sec < 200.0, - "single spike should not dominate: {}", - eff.bytes_per_series_per_sec - ); - } - - #[tokio::test] - async fn store_update_and_effective_table() { - let store = init_store(); - update(&store, &SketchType::DDSketch, 200.0, 2.0).await; - let table = effective_table(&store); - let eff = table.get(&SketchType::DDSketch).unwrap(); - assert!(eff.bytes_per_series_per_sec > 0.0); - } - - #[tokio::test] - async fn effective_table_falls_back_on_benchmark() { - // Even without any updates the store returns sensible values. - let store = init_store(); - let table = effective_table(&store); - assert!(table.contains_key(&SketchType::HLL)); - assert!(table.contains_key(&SketchType::KLL)); - } -} diff --git a/control_plane/src/physical/deployment_cost/tco.rs b/control_plane/src/physical/deployment_cost/tco.rs deleted file mode 100644 index 5a7469236..000000000 --- a/control_plane/src/physical/deployment_cost/tco.rs +++ /dev/null @@ -1,436 +0,0 @@ -//! Physical deployment total-cost-of-ownership estimator. - -use serde::{Deserialize, Serialize}; - -/// Cloud pricing configuration (loaded from YAML or defaults). -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct CloudPricing { - /// Grafana Cloud: $/1000 active series at 1 DPM (data point per minute). - pub grafana_per_1k_series_1dpm: f64, - /// AWS S3: $/GB stored per month. - pub s3_storage_per_gb_month: f64, - /// AWS S3: $/1000 PUT requests. - pub s3_put_per_1k: f64, - /// AWS S3: $/1000 GET requests. - pub s3_get_per_1k: f64, - /// AWS S3: $/GB data transfer out. - pub s3_transfer_per_gb: f64, - /// EC2 compute cost for sketch processing ($/hour for the instance type). - pub ec2_sketch_instance_per_hour: f64, -} - -impl Default for CloudPricing { - fn default() -> Self { - Self { - grafana_per_1k_series_1dpm: 6.50, - s3_storage_per_gb_month: 0.023, - s3_put_per_1k: 0.005, - s3_get_per_1k: 0.0004, - s3_transfer_per_gb: 0.09, - ec2_sketch_instance_per_hour: 0.384, // c6i.xlarge - } - } -} - -/// Workload parameters for TCO estimation. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct TcoWorkload { - /// Number of active time series. - pub series_count: u64, - /// Samples per second per series. - pub samples_per_sec: f64, - /// Bytes per raw sample (metric name + labels + value + timestamp). - pub bytes_per_sample: u64, - /// Scrape interval in seconds. - pub scrape_interval_secs: u64, - /// Average query rate (queries per second). - pub queries_per_sec: f64, - /// Average query window in seconds. - pub query_window_secs: u64, - /// Retention period in days. - pub retention_days: u64, - /// Sketch compression ratio (sketch bytes / raw bytes). Typically 0.03-0.10. - pub sketch_compression_ratio: f64, - /// Delta compression ratio (delta bytes / full sketch bytes). Typically 0.2-0.5. - pub delta_compression_ratio: f64, -} - -impl Default for TcoWorkload { - fn default() -> Self { - Self { - series_count: 100_000, - samples_per_sec: 1.0, - bytes_per_sample: 100, - scrape_interval_secs: 15, - queries_per_sec: 1.0, - query_window_secs: 300, - retention_days: 30, - sketch_compression_ratio: 0.05, - delta_compression_ratio: 0.3, - } - } -} - -/// TCO estimation result. -#[derive(Debug, Clone, Serialize)] -pub struct TcoEstimate { - /// Monthly cost breakdown for traditional pipeline. - pub before: TcoBefore, - /// Monthly cost breakdown for sketch pipeline. - pub after: TcoAfter, - /// Absolute monthly savings in dollars. - pub monthly_savings_dollars: f64, - /// Savings as a percentage of the before cost. - pub savings_percent: f64, -} - -#[derive(Debug, Clone, Serialize)] -pub struct TcoBefore { - pub ingestion_dollars: f64, - pub query_dollars: f64, - pub storage_dollars: f64, - pub total_dollars: f64, -} - -#[derive(Debug, Clone, Serialize)] -pub struct TcoAfter { - pub sketch_ingestion_dollars: f64, - pub s3_ingestion_dollars: f64, - pub sketch_query_dollars: f64, - pub s3_query_dollars: f64, - pub sketch_storage_dollars: f64, - pub s3_storage_dollars: f64, - pub compute_dollars: f64, - pub total_dollars: f64, -} - -/// Seconds in a 30-day month. -const SECS_PER_MONTH: f64 = 86_400.0 * 30.0; -/// Hours in a 30-day month. -const HOURS_PER_MONTH: f64 = 24.0 * 30.0; -/// Bytes per gigabyte. -const BYTES_PER_GB: f64 = 1_000_000_000.0; -/// Grafana bundled storage cost approximation ($/GB/month for active series). -const GRAFANA_STORAGE_PER_GB_MONTH: f64 = 0.10; -/// Approximate Grafana query cost per 1000 queries (bundled in series pricing). -const GRAFANA_QUERY_COST_PER_1K: f64 = 0.10; -/// Series capacity per EC2 instance for sketch processing. -const SERIES_PER_INSTANCE: f64 = 100_000.0; - -/// Compute a TCO estimate comparing traditional TSDB vs sketch-based pipeline. -pub fn estimate_tco(workload: &TcoWorkload, pricing: &CloudPricing) -> TcoEstimate { - let before = compute_before(workload, pricing); - let after = compute_after(workload, pricing); - - let savings = before.total_dollars - after.total_dollars; - let savings_pct = if before.total_dollars > 0.0 { - savings / before.total_dollars * 100.0 - } else { - 0.0 - }; - - TcoEstimate { - before, - after, - monthly_savings_dollars: savings, - savings_percent: savings_pct, - } -} - -fn compute_before(w: &TcoWorkload, p: &CloudPricing) -> TcoBefore { - // Data points per minute multiplier: if scrape_interval is 15s, that's 4 DPM. - let dpm_multiplier = if w.scrape_interval_secs > 0 { - 60.0 / w.scrape_interval_secs as f64 - } else { - 1.0 - }; - - // Ingestion: Grafana charges per 1000 active series at 1 DPM. - let ingestion = - (w.series_count as f64 / 1000.0) * p.grafana_per_1k_series_1dpm * dpm_multiplier; - - // Storage: raw bytes over retention period. - let samples_per_day = w.samples_per_sec * 86_400.0; - let total_bytes = w.bytes_per_sample as f64 - * samples_per_day - * w.series_count as f64 - * w.retention_days as f64; - let storage = (total_bytes / BYTES_PER_GB) * GRAFANA_STORAGE_PER_GB_MONTH; - - // Query: approximate cost based on query rate. - let total_queries_per_month = w.queries_per_sec * SECS_PER_MONTH; - let query = (total_queries_per_month / 1000.0) * GRAFANA_QUERY_COST_PER_1K; - - let total = ingestion + storage + query; - TcoBefore { - ingestion_dollars: ingestion, - query_dollars: query, - storage_dollars: storage, - total_dollars: total, - } -} - -fn compute_after(w: &TcoWorkload, p: &CloudPricing) -> TcoAfter { - let series = w.series_count as f64; - - // Sketch bytes per second per series after both compression stages. - let sketch_bps = w.samples_per_sec - * w.bytes_per_sample as f64 - * w.sketch_compression_ratio - * w.delta_compression_ratio; - - // -- Sketch ingestion: transfer cost for sketch data -- - let sketch_bytes_per_month = series * sketch_bps * SECS_PER_MONTH; - let sketch_ingestion = (sketch_bytes_per_month / BYTES_PER_GB) * p.s3_transfer_per_gb; - - // -- S3 ingestion: sketch-compressed backup (storage + PUT costs) -- - // In the sketch pipeline, S3 stores compressed sketches, not raw data. - let sketch_gb_per_month = sketch_bytes_per_month / BYTES_PER_GB; - // PUTs are batched: one PUT per flush interval (scrape_interval) containing - // all series in a single object, not one PUT per series. - // No PUTs needed if there are no series to back up. - let puts_per_month = if w.scrape_interval_secs > 0 && w.series_count > 0 { - SECS_PER_MONTH / w.scrape_interval_secs as f64 - } else { - 0.0 - }; - let s3_ingestion = sketch_gb_per_month * p.s3_storage_per_gb_month - + (puts_per_month / 1000.0) * p.s3_put_per_1k; - - // -- Sketch query: near-zero (local sketch eval) -- - let sketch_query = 0.0; - - // -- S3 query: only for ad-hoc queries (assume 1% of total) -- - let total_queries_per_month = w.queries_per_sec * SECS_PER_MONTH; - let s3_query = 0.01 * (total_queries_per_month / 1000.0) * p.s3_get_per_1k; - - // -- Compute: EC2 for sketch processing -- - let num_instances = (series / SERIES_PER_INSTANCE).ceil().max(1.0); - let compute = p.ec2_sketch_instance_per_hour * HOURS_PER_MONTH * num_instances; - - // -- Sketch storage: in-memory sketches (small, included in compute) -- - // Approximate: sketch data retained for query_window only, not full retention. - let sketch_mem_bytes = series * sketch_bps * w.query_window_secs as f64; - let sketch_storage = (sketch_mem_bytes / BYTES_PER_GB) * p.s3_storage_per_gb_month; - - // -- S3 storage: sketch-compressed data over retention -- - let s3_storage_bytes = sketch_bytes_per_month * w.retention_days as f64 / 30.0; - let s3_storage = (s3_storage_bytes / BYTES_PER_GB) * p.s3_storage_per_gb_month; - - let total = sketch_ingestion - + s3_ingestion - + sketch_query - + s3_query - + sketch_storage - + s3_storage - + compute; - - TcoAfter { - sketch_ingestion_dollars: sketch_ingestion, - s3_ingestion_dollars: s3_ingestion, - sketch_query_dollars: sketch_query, - s3_query_dollars: s3_query, - sketch_storage_dollars: sketch_storage, - s3_storage_dollars: s3_storage, - compute_dollars: compute, - total_dollars: total, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn default_pricing_is_reasonable() { - let p = CloudPricing::default(); - assert!((p.grafana_per_1k_series_1dpm - 6.50).abs() < f64::EPSILON); - assert!((p.s3_storage_per_gb_month - 0.023).abs() < f64::EPSILON); - assert!((p.s3_put_per_1k - 0.005).abs() < f64::EPSILON); - assert!((p.s3_get_per_1k - 0.0004).abs() < f64::EPSILON); - assert!((p.s3_transfer_per_gb - 0.09).abs() < f64::EPSILON); - assert!((p.ec2_sketch_instance_per_hour - 0.384).abs() < f64::EPSILON); - } - - #[test] - fn test_100k_series_15s_scrape() { - let workload = TcoWorkload { - series_count: 100_000, - samples_per_sec: 1.0, - bytes_per_sample: 100, - scrape_interval_secs: 15, - queries_per_sec: 1.0, - query_window_secs: 300, - retention_days: 30, - sketch_compression_ratio: 0.05, - delta_compression_ratio: 0.3, - }; - let pricing = CloudPricing::default(); - let est = estimate_tco(&workload, &pricing); - - assert!( - est.before.total_dollars > 0.0, - "before total should be positive" - ); - assert!( - est.after.total_dollars > 0.0, - "after total should be positive" - ); - assert!( - est.savings_percent > 50.0, - "expected >50% savings for 100K series, got {:.1}%", - est.savings_percent - ); - assert!(est.monthly_savings_dollars > 0.0, "should save money"); - } - - #[test] - fn test_1m_series_high_cardinality() { - let workload = TcoWorkload { - series_count: 1_000_000, - samples_per_sec: 1.0, - bytes_per_sample: 150, - scrape_interval_secs: 15, - queries_per_sec: 5.0, - query_window_secs: 600, - retention_days: 90, - sketch_compression_ratio: 0.04, - delta_compression_ratio: 0.25, - }; - let pricing = CloudPricing::default(); - let est = estimate_tco(&workload, &pricing); - - assert!(est.before.total_dollars > est.after.total_dollars); - assert!( - est.savings_percent > 50.0, - "expected >50% savings at 1M series, got {:.1}%", - est.savings_percent - ); - // At 1M series we need ceil(1M/100K) = 10 instances. - let expected_compute = pricing.ec2_sketch_instance_per_hour * HOURS_PER_MONTH * 10.0; - assert!( - (est.after.compute_dollars - expected_compute).abs() < 0.01, - "compute should be ~${:.2}, got ${:.2}", - expected_compute, - est.after.compute_dollars - ); - } - - #[test] - fn zero_series_returns_zero() { - let workload = TcoWorkload { - series_count: 0, - ..Default::default() - }; - let pricing = CloudPricing::default(); - let est = estimate_tco(&workload, &pricing); - - // Before costs should be zero (no series). - assert!( - est.before.ingestion_dollars.abs() < f64::EPSILON, - "ingestion should be 0" - ); - assert!( - est.before.storage_dollars.abs() < f64::EPSILON, - "storage should be 0" - ); - // After still has a minimum 1-instance compute cost. - assert!( - est.after.compute_dollars > 0.0, - "compute has a 1-instance minimum" - ); - // But sketch/s3 ingestion should be zero. - assert!( - est.after.sketch_ingestion_dollars.abs() < f64::EPSILON, - "sketch ingestion should be 0 for 0 series" - ); - assert!( - est.after.s3_ingestion_dollars.abs() < f64::EPSILON, - "s3 ingestion should be 0 for 0 series" - ); - } - - #[test] - fn sketch_compression_ratio_affects_result() { - let pricing = CloudPricing::default(); - - let low_ratio = TcoWorkload { - series_count: 100_000, - sketch_compression_ratio: 0.03, - ..Default::default() - }; - let high_ratio = TcoWorkload { - series_count: 100_000, - sketch_compression_ratio: 0.10, - ..Default::default() - }; - - let est_low = estimate_tco(&low_ratio, &pricing); - let est_high = estimate_tco(&high_ratio, &pricing); - - // Higher compression ratio means more sketch bytes, so higher after cost. - assert!( - est_high.after.sketch_ingestion_dollars > est_low.after.sketch_ingestion_dollars, - "higher sketch ratio should cost more: {:.4} vs {:.4}", - est_high.after.sketch_ingestion_dollars, - est_low.after.sketch_ingestion_dollars - ); - // Before cost should be the same (independent of sketch ratio). - assert!( - (est_high.before.total_dollars - est_low.before.total_dollars).abs() < f64::EPSILON, - "before cost should not change with sketch ratio" - ); - } - - #[test] - fn delta_compression_ratio_affects_result() { - let pricing = CloudPricing::default(); - - let low_delta = TcoWorkload { - series_count: 100_000, - delta_compression_ratio: 0.2, - ..Default::default() - }; - let high_delta = TcoWorkload { - series_count: 100_000, - delta_compression_ratio: 0.5, - ..Default::default() - }; - - let est_low = estimate_tco(&low_delta, &pricing); - let est_high = estimate_tco(&high_delta, &pricing); - - assert!( - est_high.after.sketch_ingestion_dollars > est_low.after.sketch_ingestion_dollars, - "higher delta ratio should increase sketch ingestion cost" - ); - } - - #[test] - fn retention_days_affects_storage() { - let pricing = CloudPricing::default(); - - let short = TcoWorkload { - retention_days: 7, - ..Default::default() - }; - let long = TcoWorkload { - retention_days: 90, - ..Default::default() - }; - - let est_short = estimate_tco(&short, &pricing); - let est_long = estimate_tco(&long, &pricing); - - assert!( - est_long.before.storage_dollars > est_short.before.storage_dollars, - "longer retention should cost more storage" - ); - assert!( - est_long.after.s3_storage_dollars > est_short.after.s3_storage_dollars, - "longer retention should cost more S3 storage" - ); - } -} diff --git a/control_plane/src/physical/deployment_cost/wire.rs b/control_plane/src/physical/deployment_cost/wire.rs deleted file mode 100644 index 7eb7bf3f3..000000000 --- a/control_plane/src/physical/deployment_cost/wire.rs +++ /dev/null @@ -1,105 +0,0 @@ -//! Per-family OTLP wire footprints used to rank candidate sketch families. -//! -//! Sizes assume delta encoding where supported. KLL uses full state because its -//! randomized compaction is not additively mergeable. Defaults are conservative -//! estimates for observability workloads, rather than measured telemetry. - -use planner_types::post_asap::SketchAlgorithm; - -// ── Wire-cost table ────────────────────────────────────────────────────────── - -/// Per-window wire footprint of one sketch family. -/// -/// `state_bytes` is the per-flush payload after delta encoding (where -/// supported); `envelope_bytes` is the OTLP `SketchEnvelope` overhead -/// (gRPC framing + resource attributes + scope info). -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct SketchWireCost { - /// Per-flush sketch state bytes (delta-encoded where supported). - pub state_bytes: u64, - /// OTLP envelope bytes per flush — gRPC framing + resource attrs + scope. - pub envelope_bytes: u64, -} - -impl SketchWireCost { - /// Total wire bytes per flush (state + envelope). - pub const fn per_flush(&self) -> u64 { - self.state_bytes + self.envelope_bytes - } -} - -/// Phase ε.1 wire-cost table — one entry per sketch family. -/// -/// All values are delta-encoded where a delta variant exists. KLL has no -/// delta variant per `Implementation.tex`; uses full state. Numbers are -/// conservative for typical observability workloads; a future iteration -/// can refine via observed agent telemetry. -#[derive(Debug, Clone, Copy)] -pub struct WireCostTable { - /// DDSketch with sparse bucket-map delta encoding. - pub ddsketch_delta: SketchWireCost, - /// KLL — no delta variant, ships full compactor hierarchy. - pub kll_full: SketchWireCost, - /// HLL register-array delta (~60% of 16 KB full). - pub hll_delta: SketchWireCost, - /// Count-Min row-by-row delta (~70% of 6 KB full). - pub count_min_delta: SketchWireCost, - /// Count-Sketch sparse-cell delta (~60% of 400 KB full). - pub count_sketch_delta: SketchWireCost, -} - -impl WireCostTable { - /// The default Phase ε.1 cost table — see module docs for the source - /// rationale. - pub const fn default_phase_eps_1() -> Self { - Self { - ddsketch_delta: SketchWireCost { - state_bytes: 600, - envelope_bytes: 200, - }, - kll_full: SketchWireCost { - state_bytes: 3_000, - envelope_bytes: 200, - }, - hll_delta: SketchWireCost { - state_bytes: 10_000, - envelope_bytes: 200, - }, - count_min_delta: SketchWireCost { - state_bytes: 4_000, - envelope_bytes: 200, - }, - count_sketch_delta: SketchWireCost { - state_bytes: 250_000, - envelope_bytes: 200, - }, - } - } - - /// Look up per-flush wire cost. Heap-bearing sketches reuse their bare family - /// cost; heap bytes are not priced separately. KMV and Theta reuse the HLL - /// estimate because no independent measurement is available. - pub const fn for_algorithm(&self, algorithm: &SketchAlgorithm) -> SketchWireCost { - match algorithm { - // No collector wire implementation is available for this family. - SketchAlgorithm::UnivMon => SketchWireCost { - state_bytes: u64::MAX, - envelope_bytes: 0, - }, - SketchAlgorithm::DDSketch => self.ddsketch_delta, - SketchAlgorithm::Kll => self.kll_full, - SketchAlgorithm::Hll => self.hll_delta, - SketchAlgorithm::Kmv | SketchAlgorithm::Theta => self.hll_delta, - SketchAlgorithm::Cms => self.count_min_delta, - SketchAlgorithm::CmsWithHeap => self.count_min_delta, - SketchAlgorithm::CountSketch => self.count_sketch_delta, - SketchAlgorithm::CountSketchWithHeap => self.count_sketch_delta, - } - } -} - -impl Default for WireCostTable { - fn default() -> Self { - Self::default_phase_eps_1() - } -} diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs deleted file mode 100644 index 72dfb3e2a..000000000 --- a/control_plane/src/physical/erp.rs +++ /dev/null @@ -1,1698 +0,0 @@ -//! Deployment adapter for ASAPPlanner Error–Resource Profiles. - -use asap_aware_mapping::erp::{ - AccuracyMode, ErpArtifact, ErpMultiFitSelectionRequest, ErpSelectionRequest, ErpShapeFit, - ErpShapeObservation, -}; -use planner_types::post_asap::{SketchAlgorithm, SketchParams}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use std::collections::BTreeMap; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ErpObservedShape { - pub observation: ErpShapeObservation, - /// Largest interval rate divided by the median non-zero interval rate. - pub burst_ratio: f64, -} - -/// Bounded online observer fed by the same keys that enter a summary. It keeps -/// exact sampled frequencies up to a configured cap and fails closed beyond it -/// instead of silently under-reporting cardinality. -#[derive(Debug)] -pub struct ErpShapeObserver { - inner: asap_types::erp_observation::BoundedFrequencyObserver, -} -impl ErpShapeObserver { - pub fn observed_key_count(&self) -> usize { - self.inner.observed_key_count() - } - pub fn new(max_keys: usize) -> Result { - Self::with_limits(max_keys, 256) - } - pub fn with_limits(max_keys: usize, max_intervals: usize) -> Result { - Ok(Self { - inner: asap_types::erp_observation::BoundedFrequencyObserver::new( - max_keys, - max_intervals, - )?, - }) - } - pub fn observe(&mut self, key: &str, interval: usize) -> Result<(), &'static str> { - if key.len() > 4096 { - self.inner.invalidate(); - return Err("ERP shape observer key size exceeded"); - } - self.inner.observe(key.to_owned(), interval) - } - pub fn snapshot(&self) -> Option { - ErpObservedShape::from_empirical(&self.inner.snapshot()?) - } -} -impl ErpObservedShape { - pub fn from_empirical( - empirical: &asap_types::erp_observation::EmpiricalFrequencyObservation, - ) -> Option { - let events = empirical.event_count()?; - let counts = &empirical.sorted_counts; - let exponent = fit_zipf_exponent(counts); - let fits = [("uniform", 0.0), ("zipf", exponent)] - .into_iter() - // Zipf(0) is exactly uniform; duplicate models are not ambiguity. - .filter(|(family, _)| *family != "zipf" || counts.first() != counts.last()) - .map(|(family, slope)| { - let expected: Vec<_> = (1..=counts.len()) - .map(|rank| (rank as f64).powf(-slope)) - .collect(); - let total: f64 = expected.iter().sum(); - let distance = counts - .iter() - .zip(expected) - .map(|(count, expected)| { - (*count as f64 / events as f64 - expected / total).abs() - }) - .sum::() - / 2.0; - ErpShapeFit { - family: family.into(), - parameters: if family == "zipf" { - BTreeMap::from([("exponent".into(), slope)]) - } else { - BTreeMap::new() - }, - goodness_of_fit: distance, - // A fit-quality score, not an estimator tail-probability claim. - confidence: (1.0 - distance) * (1.0 - 1.0 / (events as f64).sqrt()), - } - }) - .collect(); - let mut nonzero: Vec<_> = empirical - .interval_counts - .iter() - .copied() - .filter(|count| *count > 0) - .collect(); - nonzero.sort_unstable(); - let median = nonzero.get(nonzero.len() / 2).copied().unwrap_or(1); - let peak = nonzero.last().copied().unwrap_or(median); - Some(ErpObservedShape { - observation: ErpShapeObservation { - cardinality: counts.len() as u64, - observed_events: events, - fits, - empirical_fingerprint: None, - }, - burst_ratio: peak as f64 / median as f64, - }) - } -} - -fn fit_zipf_exponent(descending_counts: &[u64]) -> f64 { - if descending_counts.len() < 2 { - return 0.0; - } - let points: Vec<_> = descending_counts - .iter() - .enumerate() - .filter(|(_, count)| **count > 0) - .map(|(index, count)| (((index + 1) as f64).ln(), (*count as f64).ln())) - .collect(); - let mean_x = points.iter().map(|(x, _)| x).sum::() / points.len() as f64; - let mean_y = points.iter().map(|(_, y)| y).sum::() / points.len() as f64; - let covariance = points - .iter() - .map(|(x, y)| (x - mean_x) * (y - mean_y)) - .sum::(); - let variance = points - .iter() - .map(|(x, _)| (x - mean_x).powi(2)) - .sum::(); - if variance == 0.0 { - 0.0 - } else { - (-covariance / variance).max(0.0) - } -} - -/// The measurement units are part of the readout contract, not a property of -/// the shared sketch state. These keys must be supplied by the benchmark. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum ReadoutEvidence { - QuantileRank, - DistinctRelative, - FrequencyL2Relative, - EntropyAbsoluteBits, -} - -impl ReadoutEvidence { - pub(crate) fn for_intent( - algorithm: &SketchAlgorithm, - intent: &planner_types::pre_asap::AggIntent, - ) -> Option { - use planner_types::pre_asap::AggIntent; - match (algorithm, intent) { - (SketchAlgorithm::Kll, AggIntent::Quantile { .. }) => Some(Self::QuantileRank), - (SketchAlgorithm::Hll | SketchAlgorithm::UnivMon, AggIntent::Cardinality { .. }) => { - Some(Self::DistinctRelative) - } - (SketchAlgorithm::UnivMon, AggIntent::FrequencyL2 { .. }) => { - Some(Self::FrequencyL2Relative) - } - (SketchAlgorithm::UnivMon, AggIntent::FrequencyEntropy { .. }) => { - Some(Self::EntropyAbsoluteBits) - } - _ => None, - } - } - - pub(crate) fn for_query( - algorithm: &SketchAlgorithm, - query: &planner_types::post_asap::SketchQuery, - ) -> Option { - use planner_types::post_asap::SketchQuery; - match (algorithm, query) { - (SketchAlgorithm::Kll, SketchQuery::Quantile { .. }) => Some(Self::QuantileRank), - (SketchAlgorithm::Hll | SketchAlgorithm::UnivMon, SketchQuery::Cardinality) => { - Some(Self::DistinctRelative) - } - (SketchAlgorithm::UnivMon, SketchQuery::FrequencyL2) => Some(Self::FrequencyL2Relative), - (SketchAlgorithm::UnivMon, SketchQuery::FrequencyEntropy) => { - Some(Self::EntropyAbsoluteBits) - } - _ => None, - } - } - - fn metric_key(self) -> &'static str { - match self { - Self::QuantileRank => "max_rank_err", - Self::DistinctRelative => "max_cardinality_relative_error", - Self::FrequencyL2Relative => "max_frequency_l2_relative_error", - Self::EntropyAbsoluteBits => "max_frequency_entropy_absolute_bits_error", - } - } - - fn metric(self) -> planner_types::post_asap::ErrorMetric { - use planner_types::post_asap::ErrorMetric; - match self { - Self::QuantileRank => ErrorMetric::Rank, - Self::DistinctRelative => ErrorMetric::Cardinality, - Self::FrequencyL2Relative => ErrorMetric::RelativeValue, - Self::EntropyAbsoluteBits => ErrorMetric::AbsoluteValue, - } - } -} - -/// ERP v1 measures error magnitudes, not tail probabilities. Only an explicit -/// epsilon-only request may use these observations as its accuracy contract. -pub(crate) struct ErpAccuracyModel<'a> { - pub policy: Option<&'a ErpPlanningInput>, - pub max_error: f64, -} - -impl asap_aware_mapping::AccuracyModel for ErpAccuracyModel<'_> { - fn exact_operation_rule( - &self, - operation: &planner_types::post_asap::ExactOperation, - ) -> Option { - asap_aware_mapping::DefaultAccuracyModel.exact_operation_rule(operation) - } - fn local_guarantee( - &self, - family: &planner_types::post_asap::SummaryFamilyType, - query: &planner_types::post_asap::SketchQuery, - ) -> Option { - use planner_types::post_asap::*; - let theoretical = asap_aware_mapping::DefaultAccuracyModel.local_guarantee(family, query); - let (Some(policy), SummaryFamilyType::Sketch(kind, _)) = (self.policy, family) else { - return theoretical; - }; - let Some(readout) = ReadoutEvidence::for_query(kind.algorithm(), query) else { - // In particular, UnivMon's total unit count is exact without - // empirical error evidence. No unsupported readout gets a bound. - if theoretical.as_ref().is_some_and(ResultGuarantee::is_exact) { - return theoretical; - } - return match policy.select( - kind.algorithm().clone(), - self.max_error, - kind.params().clone(), - ) { - ErpParameterDecision::ExactFallback { .. } => None, - _ => theoretical, - }; - }; - match policy.evidence_for_readout( - kind.algorithm().clone(), - readout, - self.max_error, - kind.params(), - ) { - ErpParameterDecision::ExactFallback { .. } => None, - ErpParameterDecision::TheoreticalFallback { .. } => theoretical, - ErpParameterDecision::Empirical { - params, - record_id, - observed_error, - .. - } => { - if ¶ms != kind.params() { - return None; - } - Some(ResultGuarantee { - metric: readout.metric(), - bound: BoundExpr::Constant { - value: observed_error, - }, - failure_probability: ProbabilityExpr::Unknown { - statistic: "erp_v1_has_no_failure_probability_evidence".into(), - }, - provenance: vec![GuaranteeSource::SketchReadout { - algorithm: format!("{:?}", kind.algorithm()), - contract: format!( - "erp_v1_empirical:{}:{}:{}", - policy.artifact.producer_version, - record_id, - readout.metric_key() - ), - params: serde_json::to_value(¶ms).ok()?, - query: format!("{query:?}"), - }], - }) - } - } - } - - fn propagate( - &self, - op: &planner_types::post_asap::CompositionOperator, - inputs: &[planner_types::post_asap::ResultGuarantee], - local: Option<&planner_types::post_asap::ResultGuarantee>, - stats: &asap_aware_mapping::PropagationStats, - ) -> Result - { - asap_aware_mapping::DefaultAccuracyModel.propagate(op, inputs, local, stats) - } - - fn satisfies( - &self, - guarantee: &planner_types::post_asap::ResultGuarantee, - target: &crate::types::AccuracyTarget, - ) -> bool { - asap_aware_mapping::DefaultAccuracyModel.satisfies(guarantee, target) - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum ErpAccuracyMode { - Empirical, - Hybrid, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ErpPlanningInput { - pub artifact: ErpArtifact, - /// Current deployment distribution descriptor. ERP v1 requires exact - /// equality with the benchmark descriptor, so drift fails closed. - pub distribution: Value, - pub implementation: Option, - pub error_metric: String, - pub min_trials: u32, - pub expected_updates: f64, - pub expected_queries: f64, - pub expected_merges: f64, - pub retention_seconds: f64, - pub cpu_weight: f64, - pub byte_second_weight: f64, - pub mode: ErpAccuracyMode, - /// Runtime-observed shape. If present, exact descriptor equality is - /// replaced by bounded nearest-profile matching. - #[serde(default)] - pub observed_shape: Option, - #[serde(default)] - pub observed_populations: - Option>, - #[serde(skip)] - pub resolved_data_descriptor: Option>, - /// Runtime-samples ring key from which the backend resolves the freshest - /// `erp_observed_shape` payload before compiling a plan. - #[serde(default)] - pub observed_shape_source: Option, - #[serde(default)] - pub shape_match: Option, - #[serde(default)] - pub runtime: ErpRuntimeCapabilities, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct ErpObservedShapeSource { - pub source: String, - pub sketch: String, - pub implementation: String, - /// Required for catalog-scoped online evidence; absent only for legacy - /// offline runtime records that contain a single shape. - #[serde(default)] - pub population_scope: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct ErpPopulationObservationScope { - pub catalog_generation: asap_types::sds::CatalogGeneration, - pub stored_output_id: asap_types::sds::StoredOutputId, - pub input_semantics: asap_types::erp_observation::ErpObservationInputSemantics, - pub freshness: asap_types::erp_observation::ErpObservationFreshness, -} - -impl ErpPlanningInput { - /// Resolve evidence against the control plane's accepted catalog, never - /// descriptors supplied by the observation producer. - pub fn resolve_population_data_descriptor( - &mut self, - catalog: Option<&asap_types::summary_catalog::SummaryCatalog>, - ) { - self.resolved_data_descriptor = None; - let Some(populations) = self.observed_populations.as_mut() else { - return; - }; - if populations.invalid_reason.is_some() { - return; - } - let resolved = (|| -> Result<_, String> { - let catalog = catalog.ok_or("no active authoritative catalog for ERP evidence")?; - let generation = catalog.reference().map_err(|error| error.to_string())?; - if generation != populations.catalog_generation { - return Err("ERP evidence catalog differs from the active catalog".into()); - } - let materialization = catalog - .outputs - .get(&populations.stored_output_id) - .ok_or("ERP evidence summary is absent from the active catalog")?; - let summary = catalog - .summary_descriptors - .get(&materialization.summary_descriptor_id) - .ok_or("ERP summary descriptor is absent from the active catalog")?; - let actual_semantics = match &summary.operator { - asap_types::sds::SummaryOperator::Configured { - aggregation_type: asap_types::AggregationType::HLL, - .. - } => Some( - asap_types::erp_observation::ErpObservationInputSemantics::ScalarSampleValue, - ), - asap_types::sds::SummaryOperator::Configured { - aggregation_type: asap_types::AggregationType::UnivMon, - .. - } => Some( - asap_types::erp_observation::ErpObservationInputSemantics::UnitSampleFrequency, - ), - _ => None, - }; - if actual_semantics != Some(populations.input_semantics) { - return Err( - "ERP observation semantics differ from the installed summary operator".into(), - ); - } - let data = catalog - .data_descriptors - .get(&materialization.data_descriptor_id) - .ok_or("ERP evidence data descriptor is absent from the active catalog")?; - Ok(std::sync::Arc::new(data.clone())) - })(); - match resolved { - Ok(data) => self.resolved_data_descriptor = Some(data), - Err(reason) => { - populations.invalid_reason = Some(reason); - populations.populations.clear(); - } - } - } - - pub fn hydrate_observed_shape( - &mut self, - samples: &crate::runtime_samples::RuntimeSamplesStore, - ) -> Result<(), String> { - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_err(|error| format!("cannot timestamp ERP observation: {error}"))? - .as_millis() - .try_into() - .map_err(|_| "ERP observation timestamp overflows milliseconds")?; - self.hydrate_observed_shape_at(samples, now_ms) - } - - pub fn hydrate_observed_shape_at( - &mut self, - samples: &crate::runtime_samples::RuntimeSamplesStore, - now_ms: u64, - ) -> Result<(), String> { - let Some(source) = &self.observed_shape_source else { - return Ok(()); - }; - // A reused planning request must resolve new evidence on every compile. - // Never keep an old inline or previously hydrated fit after drift. - self.observed_shape = None; - self.observed_populations = None; - let key = crate::runtime_samples::SampleKey { - source: source.source.clone(), - sketch: source.sketch.clone(), - impl_name: source.implementation.clone(), - }; - let record = samples.latest(&key); - if let Some(scope) = &source.population_scope { - let resolved = (|| -> Result<_, String> { - let record = record - .as_ref() - .ok_or("no live ERP population observation")?; - let value = record - .payload - .get("erp_population_observations") - .ok_or("latest runtime sample has no erp_population_observations")?; - let observed: asap_types::erp_observation::ErpPopulationObservations< - asap_types::erp_observation::EmpiricalFrequencyObservation, - > = serde_json::from_value(value.clone()) - .map_err(|error| format!("invalid ERP population observations: {error}"))?; - observed.validate_identity_and_freshness( - &scope.catalog_generation, - scope.stored_output_id, - now_ms, - scope.freshness, - )?; - if observed.input_semantics != scope.input_semantics { - return Err("ERP observation input semantics do not match the summary".into()); - } - observed - .try_map_shapes(|empirical| ErpObservedShape::from_empirical(&empirical)) - .ok_or_else(|| "invalid bounded empirical frequency observations".to_owned()) - })(); - // An explicit invalid envelope prevents selection from borrowing an - // older fit or falling back to the artifact's distribution identity. - // The normal ERP -> theoretical -> exact policy handles this miss. - self.observed_populations = Some(resolved.unwrap_or_else(|reason| { - asap_types::erp_observation::ErpPopulationObservations { - schema_version: 1, - catalog_generation: scope.catalog_generation.clone(), - stored_output_id: scope.stored_output_id, - observed_at_unix_ms: now_ms, - window_start_ms: 0, - window_end_ms: 0, - input_semantics: scope.input_semantics, - invalid_reason: Some(reason), - populations: Vec::new(), - } - })); - return Ok(()); - } - let record = record.ok_or_else(|| { - format!( - "no runtime shape sample for {}/{}/{}", - source.source, source.sketch, source.implementation - ) - })?; - let value = record.payload.get("erp_observed_shape").ok_or_else(|| { - format!( - "latest runtime sample for {}/{}/{} has no erp_observed_shape", - source.source, source.sketch, source.implementation - ) - })?; - let observed: ErpObservedShape = serde_json::from_value(value.clone()) - .map_err(|error| format!("invalid erp_observed_shape: {error}"))?; - self.observed_shape = Some(observed.observation); - Ok(()) - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ErpShapeMatchPolicy { - pub minimum_benchmark_events: u64, - pub max_log2_cardinality_distance: f64, - pub max_parameter_distance: f64, - pub max_goodness_of_fit: f64, - pub minimum_confidence: f64, - pub minimum_confidence_margin: f64, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ErpRuntimeCapabilities { - /// Empty means the normal backend catalog is authoritative. A non-empty - /// list restricts ERP/theoretical materialization to these algorithms. - #[serde(default)] - pub allowed_algorithms: Vec, - pub max_memory_bytes: Option, -} - -#[derive(Debug, Clone, PartialEq)] -pub enum ErpParameterDecision { - Empirical { - params: SketchParams, - record_id: String, - observed_error: f64, - estimated_cost: f64, - }, - TheoreticalFallback { - params: SketchParams, - reason: String, - }, - ExactFallback { - reason: String, - }, -} - -impl ErpParameterDecision { - pub fn params(&self) -> Option<&SketchParams> { - match self { - Self::Empirical { params, .. } | Self::TheoreticalFallback { params, .. } => { - Some(params) - } - Self::ExactFallback { .. } => None, - } - } -} - -impl ErpPlanningInput { - pub fn select( - &self, - algorithm: SketchAlgorithm, - max_error: f64, - theoretical: SketchParams, - ) -> ErpParameterDecision { - self.select_metric(algorithm, &self.error_metric, max_error, theoretical, None) - } - - pub(crate) fn select_readout( - &self, - algorithm: SketchAlgorithm, - readout: ReadoutEvidence, - max_error: f64, - theoretical: SketchParams, - ) -> ErpParameterDecision { - self.select_metric( - algorithm, - readout.metric_key(), - max_error, - theoretical, - None, - ) - } - - /// Validate an existing state's readout independently of which new state - /// would be cheapest to allocate for this one consumer. - fn evidence_for_readout( - &self, - algorithm: SketchAlgorithm, - readout: ReadoutEvidence, - max_error: f64, - actual: &SketchParams, - ) -> ErpParameterDecision { - self.select_metric( - algorithm, - readout.metric_key(), - max_error, - actual.clone(), - Some(actual), - ) - } - - fn select_metric( - &self, - algorithm: SketchAlgorithm, - error_metric: &str, - max_error: f64, - theoretical: SketchParams, - required_params: Option<&SketchParams>, - ) -> ErpParameterDecision { - if let Some(populations) = &self.observed_populations { - // Every installed partition must support the same configuration. - // Never aggregate their frequency maps into a fictitious population. - let mut best: Option = None; - if populations.invalid_reason.is_none() && !populations.populations.is_empty() { - for row in &self.artifact.records { - if !sketch_name_matches(&row.sketch, &algorithm) { - continue; - } - let Some(params) = parse_params(&algorithm, &row.parameters) else { - continue; - }; - if required_params.is_some_and(|required| required != ¶ms) { - continue; - } - let mut policy = self.clone(); - policy.observed_populations = None; - policy.mode = ErpAccuracyMode::Empirical; - let total_events: f64 = populations - .populations - .iter() - .map(|p| p.shape.observation.observed_events as f64) - .sum(); - let mut errors: f64 = 0.0; - let mut cost = 0.0; - let mut evidence = Vec::new(); - let valid = populations.populations.iter().all(|population| { - policy.observed_shape = Some(population.shape.observation.clone()); - // expected_updates is the definition-wide demand; distribute - // it across partitions instead of charging it once per series. - policy.expected_updates = self.expected_updates - * population.shape.observation.observed_events as f64 - / total_events; - - match policy.select_metric( - algorithm.clone(), - error_metric, - max_error, - theoretical.clone(), - Some(¶ms), - ) { - ErpParameterDecision::Empirical { - record_id, - observed_error, - estimated_cost, - .. - } => { - errors = errors.max(observed_error); - cost += estimated_cost; - evidence.push(record_id); - true - } - _ => false, - } - }); - if valid && cost.is_finite() && best.as_ref().is_none_or(|previous| { - matches!(previous, ErpParameterDecision::Empirical { estimated_cost, .. } if cost < *estimated_cost) - }) { - best = Some(ErpParameterDecision::Empirical { - params, record_id: serde_json::to_string(&evidence).expect("string IDs serialize"), - observed_error: errors, estimated_cost: cost, - }); - } - } - } - return best.unwrap_or_else(|| { - let reason = - "ERP has no configuration valid for every observed population".to_owned(); - if self.mode == ErpAccuracyMode::Hybrid - && self.runtime.supports(&algorithm, &theoretical, None) - { - ErpParameterDecision::TheoreticalFallback { - params: theoretical, - reason, - } - } else { - ErpParameterDecision::ExactFallback { reason } - } - }); - } - // Runtime admissibility belongs before ranking: an unusable cheap - // profile must not hide a more expensive executable alternative. - let mut artifact = self.artifact.clone(); - artifact.records.retain(|row| { - sketch_name_matches(&row.sketch, &algorithm) - && parse_params(&algorithm, &row.parameters).is_some_and(|params| { - required_params.is_none_or(|required| required == ¶ms) - && self.runtime.supports( - &algorithm, - ¶ms, - Some(row.resources.memory_bytes), - ) - }) - }); - let allowed_sketches = artifact - .records - .iter() - .map(|row| row.sketch.clone()) - .collect(); - let request = ErpSelectionRequest { - distribution: self.distribution.clone(), - implementation: self.implementation.clone(), - allowed_sketches, - error_metric: error_metric.to_owned(), - max_error, - min_trials: self.min_trials, - expected_updates: self.expected_updates, - expected_queries: self.expected_queries, - expected_merges: self.expected_merges, - retention_seconds: self.retention_seconds, - cpu_weight: self.cpu_weight, - byte_second_weight: self.byte_second_weight, - mode: match self.mode { - ErpAccuracyMode::Empirical => AccuracyMode::Empirical, - ErpAccuracyMode::Hybrid => AccuracyMode::Hybrid, - }, - }; - let empirical = if request.allowed_sketches.is_empty() { - Err(asap_aware_mapping::erp::ErpError::NoApplicableConfiguration) - } else { - let selected = match (&self.observed_shape, self.shape_match) { - (Some(observed), Some(policy)) => { - artifact.select_multi_fit(&ErpMultiFitSelectionRequest { - selection: request.clone(), - observed: observed.clone(), - minimum_benchmark_events: policy.minimum_benchmark_events, - max_log2_cardinality_distance: policy.max_log2_cardinality_distance, - max_parameter_distance: policy.max_parameter_distance, - max_goodness_of_fit: policy.max_goodness_of_fit, - minimum_confidence: policy.minimum_confidence, - minimum_confidence_margin: policy.minimum_confidence_margin, - }) - } - (None, None) => artifact.select(&request), - _ => Err(asap_aware_mapping::erp::ErpError::Invalid( - "observed_shape and shape_match must be supplied together", - )), - }; - selected.and_then(|selected| { - parse_params(&algorithm, &selected.record.parameters) - .filter(|params| { - self.runtime.supports( - &algorithm, - params, - Some(selected.record.resources.memory_bytes), - ) - }) - .map(|params| (selected, params)) - .ok_or(asap_aware_mapping::erp::ErpError::NoApplicableConfiguration) - }) - }; - match empirical { - Ok((selected, params)) => ErpParameterDecision::Empirical { - params, - record_id: selected.record.id.clone(), - observed_error: selected.observed_error, - estimated_cost: selected.estimated_cost, - }, - Err(error) if self.mode == ErpAccuracyMode::Hybrid => { - if self.runtime.supports(&algorithm, &theoretical, None) { - ErpParameterDecision::TheoreticalFallback { - params: theoretical, - reason: error.to_string(), - } - } else { - ErpParameterDecision::ExactFallback { - reason: format!( - "ERP unavailable ({error}); theoretical {algorithm:?} is unsupported" - ), - } - } - } - Err(error) => ErpParameterDecision::ExactFallback { - reason: format!("empirical ERP selection failed: {error}"), - }, - } - } -} - -impl ErpRuntimeCapabilities { - fn supports( - &self, - algorithm: &SketchAlgorithm, - params: &SketchParams, - measured_memory: Option, - ) -> bool { - if !self.allowed_algorithms.is_empty() && !self.allowed_algorithms.contains(algorithm) { - return false; - } - if let Some(limit) = self.max_memory_bytes { - let Some(measured) = measured_memory else { - // A theoretical configuration has no measured byte size in - // ERP v1. Do not claim it satisfies a deployment byte cap. - return false; - }; - if !limit.is_finite() || limit < 0.0 || measured > limit { - return false; - } - } - valid_runtime_params(algorithm, params) - } -} - -fn normalized(value: &str) -> String { - value - .chars() - .filter(|ch| ch.is_ascii_alphanumeric()) - .flat_map(char::to_lowercase) - .collect() -} - -fn sketch_name_matches(name: &str, algorithm: &SketchAlgorithm) -> bool { - let name = normalized(name); - match algorithm { - SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap => { - name.starts_with("cms") || name.starts_with("countmin") - } - SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap => { - name.starts_with("countsketch") - } - SketchAlgorithm::Hll => name.starts_with("hll") || name.starts_with("hyperloglog"), - SketchAlgorithm::Kll => name.starts_with("kll"), - SketchAlgorithm::DDSketch => name.starts_with("ddsketch"), - SketchAlgorithm::UnivMon => name == "univmon", - _ => false, - } -} - -fn number(parameters: &Value, names: &[&str]) -> Option { - names - .iter() - .find_map(|name| parameters.get(name).and_then(Value::as_f64)) - .or_else(|| { - parameters.get("params").and_then(|params| { - names - .iter() - .find_map(|name| params.get(name).and_then(Value::as_f64)) - }) - }) -} - -fn u32_param(parameters: &Value, names: &[&str]) -> Option { - let value = number(parameters, names)?; - (value.is_finite() && value.fract() == 0.0 && value >= 0.0 && value <= u32::MAX as f64) - .then_some(value as u32) -} - -fn parse_params(algorithm: &SketchAlgorithm, parameters: &Value) -> Option { - let width = || u32_param(parameters, &["width", "cols"]); - let depth = || u32_param(parameters, &["depth", "rows"]); - Some(match algorithm { - SketchAlgorithm::Cms => SketchParams::Cms { - width: width()?, - depth: depth()?, - }, - SketchAlgorithm::CmsWithHeap => SketchParams::CmsWithHeap { - width: width()?, - depth: depth()?, - heap_size: u32_param(parameters, &["heap_size", "k"])?, - }, - SketchAlgorithm::CountSketch => SketchParams::CountSketch { - width: width()?, - depth: depth()?, - }, - SketchAlgorithm::CountSketchWithHeap => SketchParams::CountSketchWithHeap { - width: width()?, - depth: depth()?, - heap_size: u32_param(parameters, &["heap_size", "k"])?, - }, - SketchAlgorithm::Hll => { - let precision = u32_param(parameters, &["precision", "lg_k", "p"])?; - SketchParams::Hll { - precision: u8::try_from(precision).ok()?, - } - } - SketchAlgorithm::Kll => SketchParams::Kll { - k: u32_param(parameters, &["k"])?, - }, - SketchAlgorithm::UnivMon => SketchParams::UnivMon { - heap_size: u32_param(parameters, &["heap_size"])?, - sketch_rows: u32_param(parameters, &["sketch_rows"])?, - sketch_cols: u32_param(parameters, &["sketch_cols"])?, - layers: u8::try_from(u32_param(parameters, &["layers"])?).ok()?, - }, - SketchAlgorithm::DDSketch => SketchParams::DDSketch { - alpha: number(parameters, &["alpha", "relative_accuracy"])?, - }, - _ => return None, - }) -} - -fn valid_runtime_params(algorithm: &SketchAlgorithm, params: &SketchParams) -> bool { - match (algorithm, params) { - (SketchAlgorithm::Cms, SketchParams::Cms { width, depth }) - | (SketchAlgorithm::CountSketch, SketchParams::CountSketch { width, depth }) => { - *width >= 2 && width.is_power_of_two() && *depth >= 1 - } - ( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - }, - ) - | ( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - }, - ) => *width >= 2 && width.is_power_of_two() && *depth >= 1 && *heap_size >= 1, - ( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - }, - ) => { - *heap_size > 0 - && *sketch_cols > 0 - && (1..=20).contains(sketch_rows) - && (1..=64).contains(layers) - && sketch_rows - .checked_mul(*sketch_cols) - .and_then(|n| n.checked_mul(u32::from(*layers))) - .is_some() - } - (SketchAlgorithm::Hll, SketchParams::Hll { precision }) => (4..=18).contains(precision), - (SketchAlgorithm::Kll, SketchParams::Kll { k }) => (8..=65_535).contains(k), - (SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }) => { - alpha.is_finite() && (0.0..1.0).contains(alpha) - } - _ => false, - } -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use asap_aware_mapping::erp::{ErpRecord, ErpResourceProfile, ERP_SCHEMA_VERSION}; - - use super::*; - - /// Match the collector and portable-state decoder's supported k range. - #[test] - fn kll_runtime_limits_reject_unusable_profiles() { - for k in [0, 2, 7, 65_536] { - assert!(!valid_runtime_params( - &SketchAlgorithm::Kll, - &SketchParams::Kll { k } - )); - } - for k in [8, 32, 65_535] { - assert!(valid_runtime_params( - &SketchAlgorithm::Kll, - &SketchParams::Kll { k } - )); - } - } - - fn input(mode: ErpAccuracyMode) -> ErpPlanningInput { - ErpPlanningInput { - artifact: ErpArtifact { - schema_version: ERP_SCHEMA_VERSION, - producer_version: "bench-rev".into(), - records: vec![ErpRecord { - id: "cms-512".into(), - sketch: "cms-fastpath-vector2d".into(), - implementation: "oxide".into(), - parameters: serde_json::json!({"rows": 3, "cols": 512}), - distribution: serde_json::json!({"synthetic":{"kind":"zipf","s":1.1}}), - trials: 20, - error_metrics: BTreeMap::from([("relative_error".into(), 0.009)]), - resources: ErpResourceProfile { - memory_bytes: 12_288.0, - update_cpu_seconds: 1e-7, - merge_cpu_seconds: 1e-5, - query_cpu_seconds: 1e-6, - }, - }], - }, - distribution: serde_json::json!({"synthetic":{"kind":"zipf","s":1.1}}), - implementation: Some("oxide".into()), - error_metric: "relative_error".into(), - min_trials: 10, - expected_updates: 1_000.0, - expected_queries: 100.0, - expected_merges: 0.0, - retention_seconds: 60.0, - cpu_weight: 1.0, - byte_second_weight: 1e-9, - mode, - observed_shape: None, - observed_populations: None, - resolved_data_descriptor: None, - observed_shape_source: None, - shape_match: None, - runtime: ErpRuntimeCapabilities::default(), - } - } - - /// Structural evidence fixture, not measured calibration data. - #[test] - fn existing_univmon_readout_uses_its_own_evidence_despite_cheaper_sibling() { - use asap_aware_mapping::AccuracyModel; - use planner_types::post_asap::*; - let mut policy = input(ErpAccuracyMode::Empirical); - let small = &mut policy.artifact.records[0]; - small.sketch = "univmon".into(); - small.parameters = - serde_json::json!({"heap_size":32,"sketch_rows":5,"sketch_cols":128,"layers":4}); - small.error_metrics = - BTreeMap::from([("max_frequency_entropy_absolute_bits_error".into(), 0.1)]); - let mut large = small.clone(); - large.id = "larger-shared-state".into(); - large.parameters["heap_size"] = 128.into(); - large.resources.memory_bytes *= 2.0; - large - .error_metrics - .insert("max_frequency_entropy_absolute_bits_error".into(), 0.02); - policy.artifact.records.push(large); - let small_params = SketchParams::UnivMon { - heap_size: 32, - sketch_rows: 5, - sketch_cols: 128, - layers: 4, - }; - let actual = SketchParams::UnivMon { - heap_size: 128, - sketch_rows: 5, - sketch_cols: 128, - layers: 4, - }; - assert_eq!( - policy - .select_readout( - SketchAlgorithm::UnivMon, - ReadoutEvidence::EntropyAbsoluteBits, - 0.2, - actual.clone() - ) - .params(), - Some(&small_params) - ); - let family = SummaryFamilyType::Sketch( - SketchKind::new(SketchAlgorithm::UnivMon, actual), - GroupingStrategy::PerSubpopulationInstance, - ); - let model = ErpAccuracyModel { - policy: Some(&policy), - max_error: 0.2, - }; - let guarantee = model - .local_guarantee(&family, &SketchQuery::FrequencyEntropy) - .unwrap(); - assert_eq!(guarantee.bound, BoundExpr::Constant { value: 0.02 }); - assert!(matches!( - guarantee.failure_probability, - ProbabilityExpr::Unknown { .. } - )); - policy.artifact.records[1].error_metrics.clear(); - assert!(ErpAccuracyModel { - policy: Some(&policy), - max_error: 0.2 - } - .local_guarantee(&family, &SketchQuery::FrequencyEntropy) - .is_none()); - } - - #[test] - fn hll_cardinality_uses_measured_relative_error_not_rse() { - use asap_aware_mapping::AccuracyModel; - use planner_types::post_asap::*; - let mut policy = input(ErpAccuracyMode::Hybrid); - let row = &mut policy.artifact.records[0]; - row.sketch = "hll".into(); - row.parameters = serde_json::json!({"precision":12}); - row.error_metrics = BTreeMap::from([("max_cardinality_relative_error".into(), 0.04)]); - let family = SummaryFamilyType::Sketch( - SketchKind::new(SketchAlgorithm::Hll, SketchParams::Hll { precision: 12 }), - GroupingStrategy::PerSubpopulationInstance, - ); - let model = ErpAccuracyModel { - policy: Some(&policy), - max_error: 0.05, - }; - let guarantee = model - .local_guarantee(&family, &SketchQuery::Cardinality) - .unwrap(); - assert_eq!(guarantee.metric, ErrorMetric::Cardinality); - assert_eq!(guarantee.bound, BoundExpr::Constant { value: 0.04 }); - assert!(matches!( - guarantee.failure_probability, - ProbabilityExpr::Unknown { .. } - )); - assert_eq!( - ReadoutEvidence::for_query(&SketchAlgorithm::Hll, &SketchQuery::FrequencyEntropy), - None - ); - } - - /// Contract fixture only; the process test measures real sketch errors. - #[test] - fn readout_evidence_keeps_units_and_missing_metrics_fail_closed() { - use asap_aware_mapping::AccuracyModel; - use planner_types::post_asap::*; - let mut policy = input(ErpAccuracyMode::Hybrid); - let row = &mut policy.artifact.records[0]; - row.sketch = "univmon".into(); - row.parameters = - serde_json::json!({"heap_size": 32, "sketch_rows": 5, "sketch_cols": 128, "layers": 4}); - row.error_metrics = BTreeMap::from([ - ("max_cardinality_relative_error".into(), 0.03), - ("max_frequency_l2_relative_error".into(), 0.02), - ("max_frequency_entropy_absolute_bits_error".into(), 0.1), - ]); - let family = SummaryFamilyType::Sketch( - SketchKind::new( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 32, - sketch_rows: 5, - sketch_cols: 128, - layers: 4, - }, - ), - GroupingStrategy::PerSubpopulationInstance, - ); - let model = ErpAccuracyModel { - policy: Some(&policy), - max_error: 0.2, - }; - for (query, metric, bound) in [ - (SketchQuery::Cardinality, ErrorMetric::Cardinality, 0.03), - (SketchQuery::FrequencyL2, ErrorMetric::RelativeValue, 0.02), - ( - SketchQuery::FrequencyEntropy, - ErrorMetric::AbsoluteValue, - 0.1, - ), - ] { - let guarantee = model.local_guarantee(&family, &query).unwrap(); - assert_eq!(guarantee.metric, metric); - assert_eq!(guarantee.bound, BoundExpr::Constant { value: bound }); - assert!(matches!( - guarantee.failure_probability, - ProbabilityExpr::Unknown { .. } - )); - assert!(!model.satisfies( - &guarantee, - &crate::types::AccuracyTarget::EpsilonDelta { - epsilon: 0.2, - delta: 0.01 - } - )); - } - policy.artifact.records[0] - .error_metrics - .remove("max_frequency_entropy_absolute_bits_error"); - let model = ErpAccuracyModel { - policy: Some(&policy), - max_error: 0.2, - }; - assert!(model - .local_guarantee(&family, &SketchQuery::FrequencyEntropy) - .is_none()); - assert!(model - .local_guarantee(&family, &SketchQuery::FrequencyL2) - .is_some()); - } - - /// A matching benchmark context may reduce CMS state below theory. - #[test] - fn matching_profile_selects_empirical_parameters() { - let decision = input(ErpAccuracyMode::Hybrid).select( - SketchAlgorithm::Cms, - 0.01, - SketchParams::Cms { - width: 4096, - depth: 5, - }, - ); - assert!(matches!( - decision, - ErpParameterDecision::Empirical { - params: SketchParams::Cms { - width: 512, - depth: 3 - }, - .. - } - )); - } - - #[test] - fn unusable_cheapest_profile_does_not_hide_executable_alternative() { - let mut policy = input(ErpAccuracyMode::Hybrid); - let mut unusable = policy.artifact.records[0].clone(); - unusable.id = "invalid-cheapest".into(); - unusable.parameters = serde_json::json!({"rows": 0, "cols": 0}); - unusable.resources.memory_bytes = 1.0; - policy.artifact.records.insert(0, unusable); - assert!(matches!(policy.select(SketchAlgorithm::Cms, 0.01, - SketchParams::Cms { width: 4096, depth: 5 }), - ErpParameterDecision::Empirical { record_id, .. } if record_id == "cms-512")); - } - - /// Distribution drift in Hybrid mode preserves the analytical fallback. - #[test] - fn hybrid_drift_falls_back_to_theoretical_parameters() { - let mut policy = input(ErpAccuracyMode::Hybrid); - policy.distribution = serde_json::json!({"synthetic":{"kind":"uniform"}}); - let theory = SketchParams::Cms { - width: 4096, - depth: 5, - }; - assert!(matches!( - policy.select(SketchAlgorithm::Cms, 0.01, theory.clone()), - ErpParameterDecision::TheoreticalFallback { params, .. } if params == theory - )); - } - - /// When neither empirical nor theoretical state is deployable, Hybrid - /// explicitly requests exact execution. - #[test] - fn hybrid_capability_miss_falls_back_to_exact() { - let mut policy = input(ErpAccuracyMode::Hybrid); - policy.distribution = serde_json::json!({"synthetic":{"kind":"uniform"}}); - policy.runtime.allowed_algorithms = vec![SketchAlgorithm::Hll]; - assert!(matches!( - policy.select( - SketchAlgorithm::Cms, - 0.01, - SketchParams::Cms { - width: 4096, - depth: 5, - } - ), - ErpParameterDecision::ExactFallback { .. } - )); - } - - #[test] - fn observer_detects_skew_cardinality_and_burst() { - let mut observer = ErpShapeObserver::new(10).unwrap(); - for _ in 0..100 { - observer.observe("hot", 5).unwrap(); - } - for key in ["a", "b", "c"] { - observer.observe(key, 0).unwrap(); - } - for interval in 1..5 { - for _ in 0..3 { - observer.observe("a", interval).unwrap(); - } - } - let observed = observer.snapshot().unwrap(); - assert_eq!(observed.observation.cardinality, 4); - assert_eq!(observed.observation.fits.len(), 2); - assert!( - observed - .observation - .fits - .iter() - .find(|fit| fit.family == "zipf") - .unwrap() - .parameters["exponent"] - > 1.0 - ); - assert!(observed.burst_ratio > 30.0); - } - - #[test] - fn planning_input_hydrates_shape_from_runtime_feedback() { - let samples = crate::runtime_samples::RuntimeSamplesStore::new(4); - samples.append_for_test(crate::runtime_samples::RuntimeRecord { - source: "edge-a".into(), - sketch: "cms".into(), - impl_name: "oxide".into(), - schema_version: 1, - payload: serde_json::json!({ - "erp_observed_shape": { - "observation": { - "cardinality": 1000, - "observed_events": 500000, - "fits": [{"family": "zipf", "parameters": {"exponent": 1.1}, "goodness_of_fit": 0.01, "confidence": 0.99}] - }, - "burst_ratio": 2.5 - } - }), - }); - let mut policy = input(ErpAccuracyMode::Hybrid); - policy.observed_shape_source = Some(ErpObservedShapeSource { - source: "edge-a".into(), - sketch: "cms".into(), - implementation: "oxide".into(), - population_scope: None, - }); - policy.hydrate_observed_shape(&samples).unwrap(); - assert_eq!(policy.observed_shape.unwrap().cardinality, 1000); - } - - fn online_population_fixture() -> ( - ErpPlanningInput, - asap_types::erp_observation::ErpPopulationObservations, - ) { - use asap_types::erp_observation::*; - let generation = asap_types::sds::CatalogGeneration { - schema_version: 1, - plan_id: 7, - plan_version: 3, - snapshot_sha256: "test-catalog".into(), - }; - let definition = asap_types::PolicyFingerprint(7).into(); - let mut observer = ErpShapeObserver::new(4).unwrap(); - for key in ["a", "a", "b", "b"] { - observer.observe(key, 0).unwrap(); - } - let observed = ErpPopulationObservations { - schema_version: 1, - catalog_generation: generation.clone(), - stored_output_id: definition, - observed_at_unix_ms: 1_000, - window_start_ms: 0, - window_end_ms: 1_000, - input_semantics: ErpObservationInputSemantics::UnitSampleFrequency, - invalid_reason: None, - populations: vec![ErpPopulationObservation { - population_id: asap_types::sds::SummaryInstanceId::new("partition-a").unwrap(), - shape: observer.snapshot().unwrap(), - }], - }; - let mut policy = input(ErpAccuracyMode::Hybrid); - policy.observed_shape_source = Some(ErpObservedShapeSource { - source: "backend-a".into(), - sketch: "univmon".into(), - implementation: "asap_sketchlib".into(), - population_scope: Some(ErpPopulationObservationScope { - catalog_generation: generation, - stored_output_id: definition, - input_semantics: ErpObservationInputSemantics::UnitSampleFrequency, - freshness: ErpObservationFreshness { - max_age_ms: 100, - max_future_skew_ms: 5, - }, - }), - }); - (policy, observed) - } - - fn publish_population_fixture( - samples: &crate::runtime_samples::RuntimeSamplesStore, - mut payload: serde_json::Value, - ) { - if let Some(populations) = payload - .pointer_mut("/erp_population_observations/populations") - .and_then(Value::as_array_mut) - { - for population in populations { - if let Some(cardinality) = population - .pointer("/shape/observation/cardinality") - .and_then(Value::as_u64) - { - population["shape"] = serde_json::json!({"sorted_counts":vec![2;cardinality as usize],"interval_counts":[cardinality*2]}); - } - } - } - samples.append_for_test(crate::runtime_samples::RuntimeRecord { - source: "backend-a".into(), - sketch: "univmon".into(), - impl_name: "asap_sketchlib".into(), - schema_version: 1, - payload, - }); - } - - /// New live evidence replaces hydrated fits; expiry invalidates them even - /// when a caller reuses the same planning request. - #[test] - fn online_hydration_refreshes_and_expires_population_evidence() { - let (mut policy, mut observed) = online_population_fixture(); - let samples = crate::runtime_samples::RuntimeSamplesStore::new(4); - publish_population_fixture( - &samples, - serde_json::json!({"erp_population_observations": observed}), - ); - policy.hydrate_observed_shape_at(&samples, 1_000).unwrap(); - assert_eq!( - policy.observed_populations.as_ref().unwrap().populations[0] - .shape - .observation - .cardinality, - 2 - ); - observed.populations[0].shape.observation.cardinality = 3; - observed.observed_at_unix_ms = 1_010; - publish_population_fixture( - &samples, - serde_json::json!({"erp_population_observations": observed}), - ); - policy.hydrate_observed_shape_at(&samples, 1_010).unwrap(); - assert_eq!( - policy.observed_populations.as_ref().unwrap().populations[0] - .shape - .observation - .cardinality, - 3 - ); - policy.hydrate_observed_shape_at(&samples, 1_111).unwrap(); - let invalid = policy.observed_populations.as_ref().unwrap(); - assert!(invalid.populations.is_empty()); - assert!(invalid.invalid_reason.as_deref().unwrap().contains("stale")); - assert!(policy.observed_shape.is_none()); - } - - /// Only the activated catalog may supply a candidate's data contract. - #[test] - fn online_population_descriptor_requires_authoritative_catalog() { - let mut fixture: Value = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" - )) - .unwrap(); - let mut query = fixture["query_workload"]["repeating_queries"][3].clone(); - query["query"] = "quantile_over_time(0.9,asap_demo_latency_ms[5s])".into(); - query["requirements"]["accuracy"] = serde_json::json!({"explicit":{"Epsilon":0.05}}); - fixture["query_workload"]["repeating_queries"] = serde_json::json!([query]); - let snapshot: crate::physical::compiler::BackendLocalPlanningInput = - serde_json::from_value(fixture).unwrap(); - let plan = crate::physical::compiler::tests::quoted_snapshot( - snapshot, - crate::physical::compiler::QueryFrontend::PromQl, - ) - .compile_promql() - .unwrap(); - // Catalog resolution is independent of Planner selection. Build an - // HLL catalog fixture from a compiled source identity; production - // selection cannot deploy HLL without a known confidence guarantee. - let mut materialization = plan.precompute_plan.materializations[0].clone(); - materialization.aggregation_type = asap_types::AggregationType::HLL; - materialization.parameters = - serde_json::from_value(serde_json::json!({"precision": 14})).unwrap(); - let catalog = asap_types::summary_catalog::SummaryCatalog::from_materializations( - plan.summary_catalog.plan_id, - plan.summary_catalog.plan_version, - &[materialization], - ) - .unwrap(); - let (mut policy, mut observed) = online_population_fixture(); - observed.catalog_generation = catalog.reference().unwrap(); - observed.stored_output_id = *catalog.outputs.keys().next().unwrap(); - observed.input_semantics = - asap_types::erp_observation::ErpObservationInputSemantics::ScalarSampleValue; - policy.observed_populations = Some(observed.clone()); - policy.resolve_population_data_descriptor(Some(&catalog)); - let expected = &catalog.outputs[&observed.stored_output_id].data_descriptor_id; - assert_eq!( - &policy.resolved_data_descriptor.as_ref().unwrap().id, - expected - ); - policy.resolve_population_data_descriptor(None); - assert!(policy.resolved_data_descriptor.is_none()); - assert!(policy - .observed_populations - .as_ref() - .unwrap() - .invalid_reason - .is_some()); - assert!(policy - .observed_populations - .as_ref() - .unwrap() - .populations - .is_empty()); - } - - /// Missing, corrupt, foreign and overflow evidence must never recover a - /// previously accepted fit or the artifact's legacy distribution match. - #[test] - fn online_hydration_invalidates_unusable_latest_evidence() { - let (template, observed) = online_population_fixture(); - let mut foreign = observed.clone(); - foreign.catalog_generation.plan_version += 1; - let mut wrong_input = observed.clone(); - wrong_input.input_semantics = - asap_types::erp_observation::ErpObservationInputSemantics::ScalarSampleValue; - let mut overflow = observed.clone(); - overflow.invalid_reason = Some("population limit".into()); - let mut future = observed.clone(); - future.observed_at_unix_ms = 1_006; - for payload in [ - None, - Some(serde_json::json!({})), - Some(serde_json::json!({"erp_population_observations": foreign})), - Some(serde_json::json!({"erp_population_observations": wrong_input})), - Some(serde_json::json!({"erp_population_observations": overflow})), - Some(serde_json::json!({"erp_population_observations": future})), - ] { - let mut policy = template.clone(); - policy.observed_shape = Some(observed.populations[0].shape.observation.clone()); - let samples = crate::runtime_samples::RuntimeSamplesStore::new(4); - if let Some(payload) = payload { - publish_population_fixture(&samples, payload); - } - policy.hydrate_observed_shape_at(&samples, 1_000).unwrap(); - let invalid = policy.observed_populations.as_ref().unwrap(); - assert!(invalid.invalid_reason.is_some()); - assert!(invalid.populations.is_empty()); - assert!(policy.observed_shape.is_none()); - assert!(matches!( - policy.select( - SketchAlgorithm::Hll, - 0.05, - SketchParams::Hll { precision: 12 } - ), - ErpParameterDecision::TheoreticalFallback { .. } - )); - } - } - - #[test] - fn observer_fails_loud_at_cardinality_cap() { - let mut observer = ErpShapeObserver::new(1).unwrap(); - observer.observe("a", 0).unwrap(); - assert_eq!( - observer.observe("b", 0), - Err("ERP observation key or interval budget exceeded") - ); - assert!(observer.snapshot().is_none()); - assert!(observer.observe("a", 0).is_err()); - } - - #[test] - fn sparse_intervals_and_overflow_cannot_publish_partial_observations() { - let mut observer = ErpShapeObserver::with_limits(2, 2).unwrap(); - observer.observe("a", usize::MAX).unwrap(); - observer.observe("a", 0).unwrap(); - assert_eq!(observer.inner.snapshot().unwrap().interval_counts.len(), 2); - assert!(observer.observe("a", 1).is_err()); - assert!(observer.snapshot().is_none()); - assert!(ErpObservedShape::from_empirical( - &asap_types::erp_observation::EmpiricalFrequencyObservation { - sorted_counts: vec![u64::MAX, 1], - interval_counts: vec![u64::MAX, 1], - } - ) - .is_none()); - } - - #[test] - fn uniform_observation_matches_without_degenerate_zipf_ambiguity() { - let mut observer = ErpShapeObserver::new(4).unwrap(); - for _ in 0..1000 { - for key in ["a", "b", "c", "d"] { - observer.observe(key, 0).unwrap(); - } - } - let observed = observer.snapshot().unwrap().observation; - assert_eq!(observed.fits.len(), 1); - assert_eq!(observed.fits[0].family, "uniform"); - let mut policy = input(ErpAccuracyMode::Hybrid); - policy.artifact.records[0].distribution = serde_json::json!({"erp_shape": { - "cardinality": 4, "family": "uniform", "parameters": {}, - "benchmark_events": 4000 - }}); - policy.observed_shape = Some(observed); - policy.shape_match = Some(ErpShapeMatchPolicy { - minimum_benchmark_events: 1000, - max_log2_cardinality_distance: 1.0, - max_parameter_distance: 0.1, - max_goodness_of_fit: 0.1, - minimum_confidence: 0.9, - minimum_confidence_margin: 0.05, - }); - assert!(matches!( - policy.select( - SketchAlgorithm::Cms, - 0.01, - SketchParams::Cms { - width: 4096, - depth: 5 - } - ), - ErpParameterDecision::Empirical { .. } - )); - } - - #[test] - fn nearest_profile_miss_keeps_hybrid_fallback() { - let mut policy = input(ErpAccuracyMode::Hybrid); - policy.observed_shape = Some(ErpShapeObservation { - cardinality: 1_000_000, - observed_events: 10_000, - fits: vec![ErpShapeFit { - family: "zipf".into(), - parameters: BTreeMap::from([("exponent".into(), 2.0)]), - goodness_of_fit: 0.01, - confidence: 0.99, - }], - empirical_fingerprint: None, - }); - policy.shape_match = Some(ErpShapeMatchPolicy { - minimum_benchmark_events: 1_000, - max_log2_cardinality_distance: 1.0, - max_parameter_distance: 0.2, - max_goodness_of_fit: 0.2, - minimum_confidence: 0.5, - minimum_confidence_margin: 0.05, - }); - let theory = SketchParams::Cms { - width: 4096, - depth: 5, - }; - assert!(matches!( - policy.select(SketchAlgorithm::Cms, 0.01, theory.clone()), - ErpParameterDecision::TheoreticalFallback { params, .. } if params == theory - )); - } - - #[test] - fn custom_dataset_can_match_an_evidenced_shape_without_same_identity() { - let mut policy = input(ErpAccuracyMode::Hybrid); - policy.distribution = serde_json::json!({ - "workload": {"external": {"dataset": "customer-a"}} - }); - policy.artifact.records[0].distribution = serde_json::json!({ - "workload": {"external": {"dataset": "customer-b"}}, - "erp_shape": { - "cardinality": 1000, - "family": "zipf", - "parameters": {"exponent": 1.1}, - "benchmark_events": 100000 - } - }); - policy.observed_shape = Some(ErpShapeObservation { - cardinality: 1000, - observed_events: 100000, - fits: vec![ErpShapeFit { - family: "zipf".into(), - parameters: BTreeMap::from([("exponent".into(), 1.1)]), - goodness_of_fit: 0.01, - confidence: 0.99, - }], - empirical_fingerprint: None, - }); - policy.shape_match = Some(ErpShapeMatchPolicy { - minimum_benchmark_events: 1000, - max_log2_cardinality_distance: 1.0, - max_parameter_distance: 0.2, - max_goodness_of_fit: 0.2, - minimum_confidence: 0.5, - minimum_confidence_margin: 0.05, - }); - assert!(matches!( - policy.select( - SketchAlgorithm::Cms, - 0.01, - SketchParams::Cms { - width: 4096, - depth: 5, - } - ), - ErpParameterDecision::Empirical { .. } - )); - } -} diff --git a/control_plane/src/physical/executable_binding.rs b/control_plane/src/physical/executable_binding.rs deleted file mode 100644 index 93056ad3a..000000000 --- a/control_plane/src/physical/executable_binding.rs +++ /dev/null @@ -1,91 +0,0 @@ -//! Compiler checks relating the shared executable contract to QueryPlan. - -pub use asap_types::executable_plan::*; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum OperatorExecution { - Ingestion, - Query, -} - -/// Every physical operator uses its node's placement; payload kind does not -/// restrict execution phase. Runtime capability is checked separately. -fn operator_execution(node: &planner_types::post_asap::ExecutableDagNode) -> OperatorExecution { - match node.output_state.timing { - planner_types::post_asap::ExecutionTiming::IngestionTime => OperatorExecution::Ingestion, - planner_types::post_asap::ExecutionTiming::QueryTime => OperatorExecution::Query, - } -} - -/// Assign backend phases to a selected semantic DAG without changing its nodes. -pub fn install_selected_dag( - query_id: String, - dag: &planner_types::post_asap::ExecutableDag, - query_plan_sink: QueryNodeId, - materialization: impl Fn( - planner_types::post_asap::PostAsapNodeId, - ) -> Option, - query_node: impl Fn(planner_types::post_asap::PostAsapNodeId) -> Option, -) -> Result { - let mut nodes = std::collections::BTreeMap::new(); - let mut precompute_sinks = Vec::new(); - for node in &dag.nodes { - if let planner_types::post_asap::ExecutableOperatorPayload::SummaryAgg { - family, - input, - grouping, - .. - } = &node.payload - { - asap_physical_operators::capability::validate_summary_kernel(family, input, grouping) - .map_err(|reason| format!("post-ASAP node {:?}: {reason}", node.id))?; - } - let execution = operator_execution(node); - let binding = if let Some(stored_output) = materialization(node.id) { - precompute_sinks.push(node.id); - BackendNodeBinding::Materialization { stored_output } - } else if execution == OperatorExecution::Ingestion { - BackendNodeBinding::MaintenanceInput - } else { - query_node(node.id).map_or(BackendNodeBinding::QueryInput, |query_node| { - BackendNodeBinding::Query { query_node } - }) - }; - nodes.insert(node.id, binding); - } - precompute_sinks.sort(); - let installed = InstalledPostAsapDag { - document: OwnedPostAsapDag::from_executable(query_id, dag)?, - binding: BackendExecutableBinding { - nodes, - query_sink: dag.root, - query_plan_sink, - precompute_sinks, - }, - }; - installed.binding.validate(&installed.document.decode()?)?; - Ok(installed) -} - -pub fn validate_query_plan( - installed: &InstalledPostAsapDag, - query: &crate::query_plan::QueryPlanEntry, -) -> Result<(), String> { - if query.query_id != installed.document.query_id { - return Err("post-ASAP document and query plan identity disagree".into()); - } - if query.root != installed.binding.query_plan_sink { - return Err("backend query-plan sink disagrees with installed query root".into()); - } - for placement in installed.binding.nodes.values() { - if let BackendNodeBinding::Query { query_node } = placement { - if !query.nodes.contains_key(query_node) { - return Err(format!( - "backend binding references missing query node {}", - query_node.0 - )); - } - } - } - Ok(()) -} diff --git a/control_plane/src/physical/maintained_population.rs b/control_plane/src/physical/maintained_population.rs deleted file mode 100644 index abe0d16f7..000000000 --- a/control_plane/src/physical/maintained_population.rs +++ /dev/null @@ -1,111 +0,0 @@ -//! Lower typed population operators according to executor membership capabilities. -use super::compiler::{CompileError, PhysicalCompilationRequest, QueryCompilationInput}; -use asap_types::query_plan::{ - current_series::{SeriesPopulation, SeriesReadout}, - query_time::{Grouping, LabelMatch, LabelMatcher, QueryTimeOperator}, -}; -use planner_types::post_asap::{ - maintained_population::*, SummaryExpr, SummaryNode, ValueOperation, -}; - -fn selected(node: &SummaryNode) -> Option<(&MaintainedPopulation, &PopulationReadout)> { - let SummaryExpr::ValueOperation { - child, - operation: ValueOperation::ReadPopulation { readout }, - .. - } = &node.expr - else { - return None; - }; - let SummaryExpr::ValueOperation { - operation: ValueOperation::MaintainPopulation { population }, - .. - } = &child.expr - else { - return None; - }; - Some((population, readout)) -} - -pub(super) fn supported_node(node: &SummaryNode) -> bool { - selected(node).is_some_and(|(population, _)| { - matches!(population.input, PopulationInput::CurrentSeries(_)) - }) -} - -pub(super) fn supported(request: &PhysicalCompilationRequest) -> bool { - request - .queries - .iter() - .any(|q| supported_node(&q.selected_plan_root)) -} - -pub(super) fn operator( - request: &PhysicalCompilationRequest, - query: &QueryCompilationInput, -) -> Result, CompileError> { - let Some((spec, readout)) = selected(&query.selected_plan_root) else { - return Ok(None); - }; - let PopulationInput::CurrentSeries(input) = &spec.input else { - return Err(CompileError::Query { query_id: query.query_id.clone(), reason: "maintained table-row populations require a row-update executor; remote-write current-series state is incompatible".into() }); - }; - let populations: std::collections::BTreeSet<_> = request - .queries - .iter() - .filter_map(|q| { - selected(&q.selected_plan_root) - .map(|(p, _)| serde_json::to_string(p).expect("typed population serializes")) - }) - .collect(); - let max_bytes = request - .retained_summary_memory_budget_bytes - .unwrap_or(64 * 1024 * 1024) - .min(1_073_741_824) - / populations.len().max(1) as u64; - let population = SeriesPopulation { - metric: input.metric.clone(), - matchers: input - .matchers - .iter() - .map(|m| LabelMatcher { - name: m.label.clone(), - value: m.value.clone(), - operation: match m.operation { - CurrentSeriesMatch::Equal => LabelMatch::Equal, - CurrentSeriesMatch::NotEqual => LabelMatch::NotEqual, - CurrentSeriesMatch::Regex => LabelMatch::Regex, - CurrentSeriesMatch::NotRegex => LabelMatch::NotRegex, - }, - }) - .collect(), - grouping: Grouping { - labels: input.grouping.clone(), - without: input.without, - }, - lookback_ms: input.lookback_ms, - history_retention_ms: request.query_retention_margin_ms, - max_k: spec.max_k as u64, - quantiles: spec.quantiles, - max_bytes, - max_series: 100_000.min((max_bytes / 1024) as usize), - max_input_lag_ms: request - .scrape_interval_ms - .unwrap_or(60_000) - .saturating_add(request.query_retention_margin_ms) - .max(1) - .min(input.lookback_ms), - }; - population.validate()?; - let readout = match readout { - PopulationReadout::Quantile { q } => SeriesReadout::Quantile { q: *q }, - PopulationReadout::TopK { k } => SeriesReadout::TopK { k: *k as u64 }, - PopulationReadout::Sum => SeriesReadout::Sum, - PopulationReadout::Count => SeriesReadout::Count, - PopulationReadout::Average => SeriesReadout::Average, - }; - Ok(Some(QueryTimeOperator::CurrentSeries { - population, - readout, - })) -} diff --git a/control_plane/src/physical/mod.rs b/control_plane/src/physical/mod.rs deleted file mode 100644 index fb61d2c43..000000000 --- a/control_plane/src/physical/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Physical compilation, deployment costs, sketch capabilities, and stage emission. - -pub mod backend_stage; -pub mod compiler; -pub mod deployment_cost; -pub mod erp; -pub mod executable_binding; -mod pane_reuse; -pub mod post_asap; -pub(crate) mod realization; -pub mod runtime_capability; -pub mod sketch_catalog; -pub mod summary_catalog; -pub mod workload_cost; - -pub mod plan_dot; -pub mod publication; - -pub(crate) mod maintained_population; diff --git a/control_plane/src/physical/pane_reuse.rs b/control_plane/src/physical/pane_reuse.rs deleted file mode 100644 index e1e29a4dc..000000000 --- a/control_plane/src/physical/pane_reuse.rs +++ /dev/null @@ -1,338 +0,0 @@ -use super::compiler::{ - derived_window_cost, gcd, retained_state_count, CollectorMaterialization, - MaterializationLifecycleEstimate, PhysicalCompilationRequest, RuntimeRulePolicy, -}; -use asap_types::WindowMaterializationLayout; -use planner_types::post_asap::{PostAsapNodeId, SummaryWindowFramework}; -use std::collections::{BTreeMap, BTreeSet}; - -/// Select and install cheaper shared panes without changing logical readouts. -/// Only raw additive states are eligible; derived cohorts retain their full-window identity. -#[allow(clippy::too_many_arguments)] -pub(super) fn share_additive_panes( - request: &PhysicalCompilationRequest, - materializations: &mut [asap_types::PrecomputeMaterialization], - producers: &mut Vec, - plan_producers: &mut [CollectorMaterialization], - bindings: &mut BTreeMap<(usize, PostAsapNodeId), asap_types::PolicyFingerprint>, - policies: &mut BTreeMap, - estimates: &mut BTreeMap, -) { - use asap_types::{AggregationType, WindowKind}; - let derived_sources = materializations - .iter() - .filter_map(|m| m.derived_input.as_ref()) - .flat_map(|d| d.inputs.iter().map(|id| id.fingerprint())) - .collect::>(); - let mut seen = BTreeSet::new(); - let mut physical = Vec::new(); - let mut groups: Vec> = Vec::new(); - let mut keys = Vec::new(); - let mut member_consumers = Vec::new(); - for m in materializations.iter() { - let old = m.policy_fingerprint(); - if !seen.insert(old) - || m.derived_input.is_some() - || derived_sources.contains(&old) - || !matches!(m.aggregation_type, AggregationType::Sum) - { - continue; - } - let WindowMaterializationLayout::Pane { pane_secs } = m.window_layout else { - continue; - }; - if pane_secs == 0 || m.pane_origin_ms.is_none() { - continue; - } - let consumers = bindings - .iter() - .filter(|(_, id)| **id == old) - .map(|((query, _), _)| *query) - .collect::>(); - let Some(&first) = consumers.first() else { - continue; - }; - let query = &request.queries[first]; - let Some(estimate) = estimates.get(&old) else { - continue; - }; - // Provenance belongs to the selected layout, not to its request entry point. - if consumers.iter().any(|index| { - !request.queries[*index] - .window_realization_candidates - .iter() - .any(|candidate| { - candidate.realization_id == estimate.window_realization_id && candidate.derived - }) - }) { - continue; - } - let Some(policy) = policies.get(&old) else { - continue; - }; - let mut lifecycle = query.summary_lifecycle_inputs.clone(); - lifecycle.evaluation_interval_ms = 0; - if consumers.iter().any(|index| { - let mut other = request.queries[*index].summary_lifecycle_inputs.clone(); - other.evaluation_interval_ms = 0; - other != lifecycle || request.queries[*index].accuracy_target != query.accuracy_target - }) { - continue; - } - // Normalize only layout dimensions. Source, projection, grouping, state, - // policy and unit-cost evidence must still agree before considering reuse. - let mut canonical = m.clone(); - canonical.window_size = 1; - canonical.slide_interval = 1; - canonical.window_type = WindowKind::Tumbling; - canonical.window_layout = WindowMaterializationLayout::Pane { pane_secs: 1 }; - canonical.pane_origin_ms = Some(0); - let key = ( - canonical.policy_fingerprint(), - serde_json::to_string(&lifecycle).unwrap(), - serde_json::to_string(policy).unwrap(), - serde_json::to_string(&query.accuracy_target).unwrap(), - ); - let index = physical.len(); - if let Some(group) = groups.iter_mut().find(|group| keys[group[0]] == key) { - group.push(index); - } else { - groups.push(vec![index]); - } - keys.push(key); - member_consumers.push(consumers); - physical.push((old, m.clone())); - } - let groups = select_shared_groups(request, &physical, &member_consumers, estimates, groups); - for group in &groups { - for &index in &group.members { - let canonical = &mut physical[index].1; - canonical.window_size = group.pane_secs; - canonical.slide_interval = group.pane_secs; - canonical.window_type = WindowKind::Tumbling; - canonical.window_layout = WindowMaterializationLayout::Pane { - pane_secs: group.pane_secs, - }; - canonical.pane_origin_ms = Some(group.origin); - } - } - let mut target_counts = BTreeMap::new(); - for group in &groups { - *target_counts - .entry(physical[group.members[0]].1.policy_fingerprint()) - .or_insert(0) += 1; - } - let mut replacements = BTreeMap::new(); - for group in groups { - let mut canonical = physical[group.members[0]].1.clone(); - canonical.num_aggregates_to_retain = Some(retained_state_count( - group.lookback_ms, - request.query_retention_margin_ms, - canonical.slide_interval * 1000, - &canonical.window_layout, - )); - let new = canonical.policy_fingerprint(); - let members = group - .members - .iter() - .map(|index| physical[*index].0) - .collect::>(); - // A physical ID cannot hide a second policy/evidence cohort or an - // independently installed producer that was not offered for sharing. - if target_counts[&new] != 1 - || materializations.iter().any(|m| { - let id = m.policy_fingerprint(); - id == new && !members.contains(&id) - }) - { - continue; - } - let mut combined = estimates[&physical[group.members[0]].0].clone(); - combined.materialization = new.into(); - combined.consumer_query_ids.clear(); - combined.expected_reads = 0.0; - combined.expected_updates = 0.0; - combined.lifecycle_cost = group.cost; - combined.window_realization_id = format!("shared-pane-{}", new.0); - for index in group.members { - let old = physical[index].0; - if let Some(estimate) = estimates.remove(&old) { - combined - .consumer_query_ids - .extend(estimate.consumer_query_ids); - combined.expected_reads += estimate.expected_reads; - combined.expected_updates = - combined.expected_updates.max(estimate.expected_updates); - } - replacements.insert(old, canonical.clone()); - } - combined.consumer_query_ids.sort(); - combined.consumer_query_ids.dedup(); - estimates.insert(new, combined); - } - for m in materializations { - if let Some(canonical) = replacements.get(&m.policy_fingerprint()) { - *m = canonical.clone(); - } - } - for id in bindings.values_mut() { - if let Some(canonical) = replacements.get(id) { - *id = canonical.policy_fingerprint(); - } - } - for (old, canonical) in &replacements { - if let Some(policy) = policies.remove(old) { - policies.insert(canonical.policy_fingerprint(), policy); - } - } - for producer in producers.iter_mut().chain(plan_producers.iter_mut()) { - if let Some(canonical) = replacements.get(&producer.materialization.fingerprint()) { - let new = canonical.policy_fingerprint(); - producer.materialization = new.into(); - producer.query_id = format!("state-{}", new.0); - producer.window_secs = canonical.window_size; - producer.slide_secs = canonical.slide_interval; - producer.window_layout = canonical.window_layout.clone(); - producer.pane_origin_ms = canonical.pane_origin_ms; - producer.abstract_window_framework = SummaryWindowFramework::Tumbling; - producer.window_realization_id = estimates[&new].window_realization_id.clone(); - } - } - let mut seen = BTreeSet::new(); - producers.retain(|producer| seen.insert(producer.materialization)); -} - -struct SharedGroup { - members: Vec, - lookback_ms: u64, - cost: f64, - savings: f64, - pane_secs: u64, - origin: i64, -} - -fn select_shared_groups( - request: &PhysicalCompilationRequest, - physical: &[( - asap_types::PolicyFingerprint, - asap_types::PrecomputeMaterialization, - )], - member_consumers: &[BTreeSet], - estimates: &BTreeMap, - groups: Vec>, -) -> Vec { - let price = |members: Vec| { - if members.len() < 2 { - return None; - } - let pane_secs = members - .iter() - .map(|&i| physical[i].1.window_layout.base_pane_secs()) - .reduce(gcd) - .unwrap(); - let origin = physical[members[0]] - .1 - .pane_origin_ms - .unwrap() - .rem_euclid((pane_secs * 1_000) as i64); - if members.iter().any(|&i| { - physical[i] - .1 - .pane_origin_ms - .unwrap() - .rem_euclid((pane_secs * 1_000) as i64) - != origin - }) { - return None; - } - let mut independent = 0.0; - let mut producer = 0.0_f64; - let mut reads = 0.0; - let mut lookback_ms = 0; - for &index in &members { - let (old, m) = &physical[index]; - let consumers = &member_consumers[index]; - let query = &request.queries[*consumers.first().unwrap()]; - let selected_id = &estimates[old].window_realization_id; - let template = &query - .window_realization_candidates - .iter() - .find(|candidate| &candidate.realization_id == selected_id) - .unwrap() - .cost; - let mut maintenance = query.summary_lifecycle_inputs.clone(); - maintenance.costs.read = 0.0; - independent += derived_window_cost( - template, - &maintenance, - m.window_size, - m.slide_interval, - &m.window_layout, - request.query_retention_margin_ms, - ) - .weighted_cost; - producer = producer.max( - derived_window_cost( - template, - &maintenance, - m.window_size, - m.slide_interval, - &WindowMaterializationLayout::Pane { pane_secs }, - request.query_retention_margin_ms, - ) - .weighted_cost, - ); - for &consumer in consumers { - let lifecycle = &request.queries[consumer].summary_lifecycle_inputs; - let unit_reads = lifecycle.costs.read * lifecycle.horizon_seconds - / (f64::from(lifecycle.evaluation_interval_ms) / 1_000.0); - independent += - unit_reads * (m.window_size / m.window_layout.base_pane_secs()) as f64; - reads += unit_reads * (m.window_size / pane_secs) as f64; - } - lookback_ms = lookback_ms.max(m.window_size.saturating_mul(1_000)); - } - let cost = producer + reads; - if !independent.is_finite() || !cost.is_finite() || cost >= independent { - return None; - } - Some(SharedGroup { - members, - lookback_ms, - cost, - savings: independent - cost, - pane_secs, - origin, - }) - }; - let mut selected_groups = Vec::new(); - for mut remaining in groups { - while remaining.len() >= 2 { - if let Some(group) = price(remaining.clone()) { - selected_groups.push(group); - break; - } - // An incompatible phase or expensive fine-pane consumer must not - // prevent the remaining consumers from sharing profitable state. - let mut best: Option = None; - for (position, &left) in remaining.iter().enumerate() { - for &right in &remaining[position + 1..] { - if let Some(group) = price(vec![left, right]) { - if best - .as_ref() - .is_none_or(|previous| group.savings > previous.savings) - { - best = Some(group); - } - } - } - } - let Some(group) = best else { - break; - }; - remaining.retain(|index| !group.members.contains(index)); - selected_groups.push(group); - } - } - selected_groups -} diff --git a/control_plane/src/physical/plan_dot.rs b/control_plane/src/physical/plan_dot.rs deleted file mode 100644 index 22dc9ed99..000000000 --- a/control_plane/src/physical/plan_dot.rs +++ /dev/null @@ -1,252 +0,0 @@ -//! Graphviz inspection rendering for one selected physical plan. -//! -//! This is intentionally a developer-facing view: JSON remains the complete -//! representation, while DOT keeps labels compact enough to follow execution. - -use super::compiler::CompiledPhysicalPlan; -use crate::query_plan::QueryPlanNode; -use asap_types::query_plan::query_time::QueryTimeOperator; - -/// Render the selected precompute and query execution DAGs as deterministic DOT. -pub fn render(plan: &CompiledPhysicalPlan) -> String { - let mut dot = String::from( - "digraph compiled_physical_plan {\n rankdir=LR;\n node [shape=box, fontname=Helvetica];\n", - ); - - dot.push_str(" subgraph cluster_precompute {\n label=\"PrecomputePlan\";\n"); - for materialization in &plan.precompute_plan.materializations { - let id = materialization.policy_fingerprint(); - let node = materialization_node(id.as_u64()); - let label = format!( - "materialization\n{}\nmetric={}\nwindow_type={}\nwindow={}s / {}s\nlayout={:?}", - id, - materialization.metric, - materialization.window_type, - materialization.window_size, - materialization.slide_interval, - materialization.window_layout, - ); - emit_node(&mut dot, &node, &label, "shape=component"); - } - for (dag_index, (query_id, installed)) in - plan.precompute_plan.executable_dags.iter().enumerate() - { - dot.push_str(&format!( - " subgraph cluster_precompute_dag_{dag_index} {{\n label=\"{}\";\n", - escape(query_id) - )); - for node in &installed.document.nodes { - let id = precompute_node(dag_index, node.id.0); - let binding = installed - .binding - .nodes - .get(&node.id) - .map(|binding| format!("\n{:?}", binding)) - .unwrap_or_default(); - emit_node( - &mut dot, - &id, - &format!( - "{} #{}{}", - node.payload - .get("kind") - .and_then(serde_json::Value::as_str) - .unwrap_or("unknown"), - node.id.0, - binding - ), - "", - ); - } - for edge in &installed.document.edges { - dot.push_str(&format!( - " {} -> {} [label=\"{:?}\"];\n", - precompute_node(dag_index, edge.producer.0), - precompute_node(dag_index, edge.consumer.0), - edge.role - )); - } - dot.push_str(" }\n"); - } - dot.push_str(" }\n"); - - for (query_index, (query_key, entry)) in plan.query_plan.entries.iter().enumerate() { - dot.push_str(&format!( - " subgraph cluster_query_{query_index} {{\n label=\"QueryPlan: {}\";\n", - escape(query_key) - )); - for (id, node) in &entry.nodes { - let attributes = if *id == entry.root { - "shape=doubleoctagon" - } else { - "" - }; - emit_node( - &mut dot, - &query_node(query_index, id.0), - &query_node_label(node), - attributes, - ); - if let QueryPlanNode::ReadMaterialization { binding } = node { - dot.push_str(&format!( - " {} -> {} [style=dashed, color=gray40, label=\"reads\"];\n", - materialization_node(binding.materialization.as_u64()), - query_node(query_index, id.0), - )); - } - } - for (id, node) in &entry.nodes { - for input in node.inputs() { - dot.push_str(&format!( - " {} -> {};\n", - query_node(query_index, input.0), - query_node(query_index, id.0), - )); - } - } - dot.push_str(" }\n"); - } - dot.push_str("}\n"); - dot -} - -fn materialization_node(id: u64) -> String { - format!("materialization_{id:016x}") -} - -fn precompute_node(dag: usize, id: u32) -> String { - format!("precompute_{dag}_{id}") -} - -fn query_node(query: usize, id: u64) -> String { - format!("query_{query}_{id}") -} - -fn emit_node(dot: &mut String, id: &str, label: &str, attributes: &str) { - dot.push_str(&format!(" {id} [label=\"{}\"", escape(label))); - if !attributes.is_empty() { - dot.push_str(&format!(", {attributes}")); - } - dot.push_str("];\n"); -} - -fn escape(value: &str) -> String { - value - .replace('\\', "\\\\") - .replace('"', "\\\"") - .replace('\n', "\\n") -} - -fn query_node_label(node: &QueryPlanNode) -> String { - match node { - QueryPlanNode::PhysicalFragment { dag, .. } => { - asap_physical_operators::physical_planner::CompiledPhysicalDag::decode(dag) - .map(|plan| { - format!( - "Physical\n{}", - plan.operator_name(plan.roots()[0]).unwrap_or("Input") - ) - }) - .unwrap_or_else(|_| "Invalid physical DAG".into()) - } - QueryPlanNode::RelationalJoin { .. } => "RelationalJoin".into(), - QueryPlanNode::Relational { .. } => "Relational".into(), - QueryPlanNode::Logical { operator, .. } => { - format!("Logical\n{}", query_time_label(operator)) - } - QueryPlanNode::Scalar { value } => format!("Scalar\n{value}"), - QueryPlanNode::Binary { operator, .. } => format!("Binary\n{operator:?}"), - QueryPlanNode::ReduceSum { .. } => "ReduceSum".into(), - QueryPlanNode::ReadMaterialization { binding } => format!( - "ReadMaterialization\n{}\nwindow={}ms\nlookback={:?}", - binding.materialization.fingerprint(), - binding.window_ms, - binding.readout_lookback_ms - ), - QueryPlanNode::SummaryEstimate { query, .. } => format!("SummaryEstimate\n{query:?}"), - QueryPlanNode::ExactReadout { readout, .. } => format!("ExactReadout\n{readout:?}"), - QueryPlanNode::SummaryMerge { .. } => "SummaryMerge".into(), - QueryPlanNode::ExternalExact { .. } => "ExternalExact".into(), - QueryPlanNode::ExactFallback { reason } => format!("ExactFallback\n{reason}"), - } -} - -fn query_time_label(operator: &QueryTimeOperator) -> &'static str { - match operator { - QueryTimeOperator::CurrentSeries { .. } => "CurrentSeries", - QueryTimeOperator::ExactSubquery { .. } => "ExactSubquery", - QueryTimeOperator::CandidateExactSubquery { .. } => "CandidateExactSubquery", - QueryTimeOperator::Scan { .. } => "Scan", - QueryTimeOperator::UnaryNegate => "UnaryNegate", - QueryTimeOperator::VectorToScalar => "VectorToScalar", - QueryTimeOperator::Aggregate { .. } => "Aggregate", - QueryTimeOperator::Limit { .. } => "Limit", - QueryTimeOperator::Binary { .. } => "Binary", - QueryTimeOperator::Temporal { .. } => "Temporal", - QueryTimeOperator::Sort { .. } => "Sort", - QueryTimeOperator::HistogramQuantile => "HistogramQuantile", - QueryTimeOperator::Subquery { .. } => "Subquery", - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::physical::compiler::{BackendLocalPlanningInput, QueryFrontend}; - - fn fixture() -> CompiledPhysicalPlan { - let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - crate::physical::compiler::tests::quoted_snapshot(snapshot, QueryFrontend::PromQl) - .compile_promql() - .unwrap() - } - - #[test] - fn render_links_query_reads_to_precompute_materializations() { - let dot = render(&fixture()); - assert!(dot.contains("cluster_precompute"), "{dot}"); - assert!(dot.contains("cluster_query_"), "{dot}"); - assert!(dot.contains("style=dashed"), "{dot}"); - assert!(dot.contains("ReadMaterialization"), "{dot}"); - assert!(dot.contains("window_type="), "{dot}"); - assert!(dot.contains("layout="), "{dot}"); - } - - #[test] - fn render_uses_graphviz_line_breaks_in_labels() { - let dot = render(&fixture()); - assert!( - dot.contains("materialization\\npolicy_fp:"), - "expected a Graphviz line break, not a literal backslash-n: {dot}" - ); - assert!( - !dot.contains("materialization\\\\npolicy_fp:"), - "label double-escaped its Graphviz line break: {dot}" - ); - } - - #[test] - fn compiled_physical_plan_serializes_each_projection() { - let artifact = serde_json::to_value(fixture()).unwrap(); - for field in [ - "envelope", - "summary_catalog", - "collector_plans", - "precompute_plan", - "transmission_plan", - "query_plan", - "storage_routing", - "lifecycle_estimates", - "cost_comparison", - "planner_selection_trace", - ] { - assert!( - artifact.get(field).is_some(), - "missing `{field}`: {artifact}" - ); - } - } -} diff --git a/control_plane/src/physical/post_asap/cost_model.rs b/control_plane/src/physical/post_asap/cost_model.rs deleted file mode 100644 index 9610ed0a1..000000000 --- a/control_plane/src/physical/post_asap/cost_model.rs +++ /dev/null @@ -1,1122 +0,0 @@ -//! Deployment family selection and parameter sizing for ASAPPlanner binding. -//! -//! `rank_candidates` chooses a family; `size_params` sets its parameters. -//! Frequency extensions use `realize_extension` and `readout_extension`. -//! Exact TopK falls back to logical execution upstream because it has no -//! exact accumulator realization; these cost-model hooks cannot bind it. - -#![allow(dead_code)] - -use asap_aware_mapping::cost_model::{Cost, CostedSummaryDeployment}; -use asap_aware_mapping::empirical_comparison::{ - recommend_offline, OfflineComparisonEvidence, OfflineComparisonRequest, OfflineRecommendation, - SketchConfiguration, -}; -use asap_aware_mapping::empirical_cost::EmpiricalEvidenceProvider; -use asap_aware_mapping::{ - CompleteSummaryCandidateEstimate, CostModel, CostProvenance, EvaluationRate, - ExactCompositionCostInputs, ExactCompositionCostRequest, Horizon, OperationPlacement, - Realization, Replacement, ReplacementSubDAG, SummaryMaintenanceCapabilities, - SummaryMaintenanceLifecycleCostInputs, TargetSubDAG, ValueOperationCapabilities, -}; -use planner_types::post_asap::{ - ExecutableOperatorPayload, GroupingStrategy, SketchAlgorithm, SketchParams, SketchQuery, - SummaryFamilyType, SummaryWindowFramework, -}; -use planner_types::pre_asap::expr_ir::ColumnRef; - -use crate::physical::deployment_cost::wire::WireCostTable; -use crate::physical::erp::{ErpParameterDecision, ErpPlanningInput}; -use crate::planner_selection::FREQUENCY_EXT_KIND; -use crate::types::AccuracyTarget; -use planner_types::pre_asap::AggIntent; -use serde::{Deserialize, Serialize}; - -/// A local state-footprint estimate, never a complete deployment quote. -#[derive(Debug, Serialize)] -pub struct CandidateCostEstimate { - pub value: f64, - pub unit: &'static str, - pub model: &'static str, - pub source: &'static str, - pub erp_record_ids: Vec, -} - -fn analytical_state_bytes(family: &SummaryFamilyType) -> Option { - use asap_types::AggregationType as A; - use planner_types::post_asap::ExactKind; - let (aggregation, params) = match family { - SummaryFamilyType::ExactAggregate(kind, _) => ( - match kind { - ExactKind::Sum | ExactKind::Count => A::Sum, - ExactKind::Min => A::Min, - ExactKind::Max => A::Max, - ExactKind::Increase | ExactKind::Rate | ExactKind::IRate => A::Increase, - }, - std::collections::HashMap::new(), - ), - SummaryFamilyType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance) => { - let aggregation = match kind.algorithm() { - SketchAlgorithm::DDSketch => A::DDSketch, - SketchAlgorithm::Kll => A::DatasketchesKLL, - SketchAlgorithm::Hll => A::HLL, - SketchAlgorithm::Cms => A::CountMinSketch, - SketchAlgorithm::CountSketch => A::CountSketch, - SketchAlgorithm::CmsWithHeap => A::CountMinSketchWithHeap, - SketchAlgorithm::CountSketchWithHeap => A::CountSketchWithHeap, - SketchAlgorithm::UnivMon => A::UnivMon, - SketchAlgorithm::Kmv | SketchAlgorithm::Theta => return None, - }; - let params = super::super::compiler::sketch_params_json(kind.params()) - .as_object()? - .iter() - .map(|(key, value)| (key.clone(), value.clone())) - .collect(); - (aggregation, params) - } - // A shared grid needs its own population/layout model; an independent - // state's measurement is not a measurement of that grid. - _ => return None, - }; - Some(super::super::compiler::estimated_state_bytes(&aggregation, ¶ms) as f64) -} - -/// One measured execution profile for an exact operator composed with a -/// maintained summary. Values use CPU nanoseconds so every term in Planner's -/// recurring-cost formula has the same physical unit. Peak memory is retained -/// as measured resource evidence and reported separately; it is deliberately -/// not converted into CPU cost by an invented weight. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ExactCompositionCostEvidence { - /// Exact pre-ASAP target serialized with the pinned Planner revision. - pub target: serde_json::Value, - /// Exact operation serialized with the pinned Planner revision. - pub operation: serde_json::Value, - pub placement: ExactOperationPlacement, - pub expected_input_rows: f64, - pub expected_output_rows: f64, - pub exact_cpu_ns_per_row: f64, - pub summary_maintenance_cpu_ns_per_update: f64, - pub summary_read_cpu_ns: f64, - pub update_rate_per_second: f64, - pub evaluation_rate_per_second: f64, - pub raw_recompute_cpu_ns: f64, - pub observed_peak_memory_bytes: u64, - pub data_snapshot_id: String, - pub model_version: String, - pub observed_at_unix_ms: u64, - pub valid_for_ms: u64, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum ExactOperationPlacement { - Read, - Maintenance, -} - -impl ExactCompositionCostEvidence { - pub fn validate(&self, now_unix_ms: u64, max_age_ms: u64) -> Result<(), String> { - let positive = [ - self.exact_cpu_ns_per_row, - self.summary_maintenance_cpu_ns_per_update, - self.summary_read_cpu_ns, - self.update_rate_per_second, - self.evaluation_rate_per_second, - self.raw_recompute_cpu_ns, - ]; - let rows = [self.expected_input_rows, self.expected_output_rows]; - if positive.iter().any(|v| !v.is_finite() || *v <= 0.0) - || rows.iter().any(|v| !v.is_finite() || *v < 0.0) - || self.observed_peak_memory_bytes == 0 - || self.data_snapshot_id.trim().is_empty() - || self.model_version.trim().is_empty() - || self.valid_for_ms == 0 - || self.observed_at_unix_ms > now_unix_ms - || now_unix_ms - self.observed_at_unix_ms > self.valid_for_ms.min(max_age_ms) - { - return Err( - "missing, non-physical, future, or stale exact-composition evidence".into(), - ); - } - Ok(()) - } - - fn matches(&self, request: &ExactCompositionCostRequest<'_>) -> bool { - let placement = match request.composition.placement { - OperationPlacement::Read => ExactOperationPlacement::Read, - OperationPlacement::Maintenance => ExactOperationPlacement::Maintenance, - }; - self.placement == placement - && serde_json::to_value(request.target).ok().as_ref() == Some(&self.target) - && serde_json::to_value(&request.composition.op).ok().as_ref() == Some(&self.operation) - } -} - -/// See module docs. -pub struct ControlPlaneCostModel { - /// Workload-level accuracy policy — combined (tighter-of) with each - /// intent's own accuracy field, matching every `bind_*.rs` rule's old - /// `accuracy: &AccuracyTarget` parameter. - pub workload_accuracy: AccuracyTarget, - lifecycle_costs: SummaryMaintenanceLifecycleCostInputs, - summary_maintenance: SummaryMaintenanceCapabilities, - window_framework_costs: Vec<(Option, SummaryWindowFramework, Cost)>, - offline_evidence: Option, - offline_frequency_comparison: Option<(OfflineComparisonEvidence, OfflineComparisonRequest)>, - exact_composition_costs: Vec, - erp: Option, -} - -impl ControlPlaneCostModel { - pub fn candidate_cost_estimate( - &self, - candidate: &ReplacementSubDAG, - ) -> Option { - if let Replacement::Rewrite(root) = &candidate.replacement { - // Price a semantic rewrite through its executable summary candidates, - // in the same retained-state units as a direct summary candidate. - // Unknown realizations remain uncosted rather than receiving zero. - use asap_aware_mapping::{ReplacementStrategy, SketchAlgorithmStrategy, TargetSubDAG}; - return SketchAlgorithmStrategy::new(self) - .replacements(&TargetSubDAG::new(root)) - .iter() - .filter(|candidate| matches!(candidate.replacement, Replacement::Summary(_))) - .filter_map(|candidate| self.candidate_cost_estimate(candidate)) - .min_by(|a, b| a.value.total_cmp(&b.value)); - } - let Replacement::Summary(root) = &candidate.replacement else { - // Exact compositions have a separate measured rate model. - return None; - }; - let dag = planner_types::post_asap::compile_executable_dag(root).ok()?; - let mut value = 0.0; - let mut states = 0; - for node in &dag.nodes { - let family = match &node.payload { - ExecutableOperatorPayload::SummaryAgg { family, .. } - | ExecutableOperatorPayload::SummaryJoin { family, .. } => family, - _ => continue, - }; - states += 1; - value += analytical_state_bytes(family)?; - } - if states == 0 || !value.is_finite() { - return None; - } - Some(CandidateCostEstimate { - value, - unit: "bytes_per_state_partition", - model: "backend_state_footprint_v1", - source: "analytical", - erp_record_ids: vec![], - }) - } - - pub fn new(workload_accuracy: AccuracyTarget) -> Self { - Self { - workload_accuracy, - lifecycle_costs: SummaryMaintenanceLifecycleCostInputs::default(), - summary_maintenance: SummaryMaintenanceCapabilities::default(), - window_framework_costs: Vec::new(), - offline_evidence: None, - offline_frequency_comparison: None, - exact_composition_costs: Vec::new(), - erp: None, - } - } - - pub fn with_exact_composition_costs( - mut self, - costs: Vec, - ) -> Self { - self.exact_composition_costs = costs; - self - } - - pub fn with_erp(mut self, erp: ErpPlanningInput) -> Self { - self.erp = Some(erp); - self - } - - pub fn erp_parameter_decision( - &self, - algorithm: SketchAlgorithm, - max_error: f64, - theoretical: SketchParams, - ) -> Option { - self.erp - .as_ref() - .map(|erp| erp.select(algorithm, max_error, theoretical)) - } - - /// Use offline update CPU evidence for algorithm ordering. Physical costs - /// and accuracy guarantees retain their deployment-specific contracts. - pub fn with_offline_evidence(mut self, evidence: EmpiricalEvidenceProvider) -> Self { - self.offline_evidence = Some(evidence); - self - } - - pub fn offline_evidence(&self) -> Option<&EmpiricalEvidenceProvider> { - self.offline_evidence.as_ref() - } - - /// Opt into a fixed-snapshot frequency comparison. The caller asserts that - /// the explicit point queries use the supplied integer-key distribution and - /// offline probe population. The observed mean is an acceptance criterion - /// over that population, not a per-key or real-time error guarantee. - pub fn with_offline_frequency_comparison( - mut self, - evidence: OfflineComparisonEvidence, - request: OfflineComparisonRequest, - ) -> Self { - self.offline_frequency_comparison = Some((evidence, request)); - self - } - - /// Explain the same recommendation used by the frequency extension binder. - /// Exact selection and any unavailable comparison preserve exact execution. - pub fn offline_frequency_recommendation( - &self, - payload: &serde_json::Value, - ) -> Result { - let (evidence, request) = self - .offline_frequency_comparison - .as_ref() - .ok_or("offline frequency comparison is not configured")?; - if payload - .get("item_label") - .and_then(|v| v.as_str()) - .is_none_or(str::is_empty) - || payload - .get("item_value") - .and_then(|v| v.as_str()) - .and_then(|v| v.parse::().ok()) - .is_none() - { - return Err( - "offline point-frequency comparison requires an explicit integer item readout" - .into(), - ); - } - let accuracy: AccuracyTarget = serde_json::from_value( - payload - .get("accuracy") - .cloned() - .ok_or("missing frequency accuracy")?, - ) - .map_err(|error| error.to_string())?; - let (eps, delta) = self - .combined_eps_delta(&accuracy) - .ok_or("exact accuracy requires exact execution")?; - if !eps.is_finite() - || eps <= 0.0 - || eps >= 1.0 - || !delta.is_finite() - || delta <= 0.0 - || delta >= 1.0 - { - return Err("invalid frequency accuracy budget".into()); - } - let (width, depth) = Self::cms_width_depth(eps, delta); - let width = width - .checked_next_power_of_two() - .ok_or("frequency width overflows")?; - let mut request = request.clone(); - // Only CMS is supported by the backend's frequency capability table. - // Caller-supplied minima cannot weaken workload/intent requirements. - request.formal_minimums = Some(vec![SketchConfiguration { - algorithm: SketchAlgorithm::Cms, - params: SketchParams::Cms { width, depth }, - }]); - evidence - .sketch_evidence - .validate() - .map_err(|error| error.to_string())?; - // Exclude layouts the backend cannot instantiate before selecting the - // winner, so a cheap unsupported width cannot hide a legal measured one. - let mut evidence = evidence.clone(); - evidence.sketch_evidence.records.retain(|row| { - row.algorithm == SketchAlgorithm::Cms - && matches!(&row.params, SketchParams::Cms { width, .. } if width.is_power_of_two()) - }); - evidence.query_bindings.retain(|binding| { - evidence - .sketch_evidence - .records - .iter() - .any(|row| row.id == binding.record_id) - }); - recommend_offline(&evidence, &request) - } - - fn rank_with_offline_evidence( - &self, - intent: &AggIntent, - defaults: Vec, - ) -> Vec { - let Some(provider) = &self.offline_evidence else { - return defaults; - }; - let accuracy = intent_accuracy(intent); - if matches!(accuracy, AccuracyTarget::Exact) { - return defaults; - } - let (eps, delta) = asap_aware_mapping::replacement::accuracy_budget(&accuracy); - // Compare the parameters this deployment will actually bind, not the - // planner's default sizing or another benchmark configuration. - let costs: Option> = defaults - .iter() - .map(|algorithm| { - let params = self.size_params(algorithm.clone(), intent, eps, delta); - let row = provider.lookup(algorithm, ¶ms).ok()?; - Some(( - algorithm.clone(), - row.metrics.resources.cpu.update_cpu_ns.as_ref()?.value, - )) - }) - .collect(); - let Some(mut costs) = costs else { - return defaults; - }; - costs.sort_by(|left, right| left.1.total_cmp(&right.1)); - costs.into_iter().map(|(algorithm, _)| algorithm).collect() - } - - /// Keep concrete physical identities in Planner's complete-candidate estimate, - /// including distinct pane sizes using the same abstract window framework. - pub fn with_window_implementation_costs( - mut self, - costs: Vec<(String, SummaryWindowFramework, Cost)>, - ) -> Self { - self.window_framework_costs = costs - .into_iter() - .map(|(id, framework, cost)| (Some(id), framework, cost)) - .collect(); - self - } - - pub fn with_summary_maintenance( - mut self, - lifecycle_costs: SummaryMaintenanceLifecycleCostInputs, - capabilities: SummaryMaintenanceCapabilities, - ) -> Self { - self.lifecycle_costs = lifecycle_costs; - self.summary_maintenance = capabilities; - self - } - - /// The tighter (lower) of the workload policy and an intent's own - /// accuracy target, as `(eps, delta)`. `None` when either side is - /// `Exact` — mirrors `bind_kll_quantile.rs` / `bind_ddsketch_quantile.rs` - /// / `bind_hll_cardinality.rs` / `bind_cms_count.rs`'s identical - /// `match (accuracy, &intent_accuracy) {...}` block. (`TopK` has its - /// own combination rule — an `Exact` side there picks the *other* - /// side's budget rather than bailing — see [`Self::topk_eps_delta`].) - fn combined_eps_delta(&self, intent_accuracy: &AccuracyTarget) -> Option<(f64, f64)> { - match (&self.workload_accuracy, intent_accuracy) { - (AccuracyTarget::Exact, _) | (_, AccuracyTarget::Exact) => None, - (AccuracyTarget::Epsilon(a), AccuracyTarget::Epsilon(b)) => Some((a.min(*b), 0.01)), - (AccuracyTarget::Epsilon(a), AccuracyTarget::EpsilonDelta { epsilon, delta }) - | (AccuracyTarget::EpsilonDelta { epsilon, delta }, AccuracyTarget::Epsilon(a)) => { - Some((a.min(*epsilon), *delta)) - } - ( - AccuracyTarget::EpsilonDelta { - epsilon: a, - delta: da, - }, - AccuracyTarget::EpsilonDelta { - epsilon: b, - delta: db, - }, - ) => Some((a.min(*b), da.min(*db))), - } - } - - /// `TopK`'s own `(eps, delta)` combination — verbatim port of - /// `bind_cms_topk.rs`'s `bind` match. Unlike - /// [`Self::combined_eps_delta`], an `Exact` side does not bail: it - /// picks the *other* side's budget (falling back to the catalog - /// default `(0.01, 0.01)` only when both sides are `Exact`). Public - /// (within the crate) because [`crate::physical::post_asap::lower`]'s - /// `TopK { accuracy: Exact }` pre-pass needs the same combination. - pub(crate) fn topk_eps_delta(&self, intent_accuracy: &AccuracyTarget) -> (f64, f64) { - match (&self.workload_accuracy, intent_accuracy) { - (AccuracyTarget::Exact, AccuracyTarget::Exact) => (0.01, 0.01), - (AccuracyTarget::Exact, other) | (other, AccuracyTarget::Exact) => match other { - AccuracyTarget::Epsilon(a) => (*a, 0.01), - AccuracyTarget::EpsilonDelta { epsilon, delta } => (*epsilon, *delta), - AccuracyTarget::Exact => (0.01, 0.01), - }, - (AccuracyTarget::Epsilon(a), AccuracyTarget::Epsilon(b)) => (a.min(*b), 0.01), - (AccuracyTarget::Epsilon(a), AccuracyTarget::EpsilonDelta { epsilon, delta }) - | (AccuracyTarget::EpsilonDelta { epsilon, delta }, AccuracyTarget::Epsilon(a)) => { - (a.min(*epsilon), *delta) - } - ( - AccuracyTarget::EpsilonDelta { - epsilon: a, - delta: da, - }, - AccuracyTarget::EpsilonDelta { - epsilon: b, - delta: db, - }, - ) => (a.min(*b), da.min(*db)), - } - } - - /// Recall-tier-filtered, wire-cost-ordered top-k family candidates. - /// Verbatim port of `bind_cms_topk.rs`'s `TopkRecallTier` + - /// `candidate_families` + `cheapest_family`. Public (within the - /// crate) for the same reason as [`Self::topk_eps_delta`]. - pub(crate) fn topk_family_order( - &self, - intent_accuracy: &AccuracyTarget, - ) -> Vec { - let tight = matches!(self.workload_accuracy, AccuracyTarget::Exact) - || matches!(intent_accuracy, AccuracyTarget::Exact); - let allowed: &[SketchAlgorithm] = if tight { - &[SketchAlgorithm::CountSketchWithHeap] - } else { - &[ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap, - ] - }; - let table = WireCostTable::default(); - let mut ranked: Vec = allowed.to_vec(); - ranked.sort_by_key(|algorithm| table.for_algorithm(algorithm).per_flush()); - ranked - } - - /// `(width, depth)` for a CMS-family sketch under `(eps, delta)`. - /// Verbatim: `w = ⌈e/eps⌉` clamped to `≥2`, `d = ⌈ln(1/delta)⌉` - /// clamped to `≥1`. - fn cms_width_depth(eps: f64, delta: f64) -> (u32, u32) { - let w = (std::f64::consts::E / eps).ceil().max(2.0) as u32; - let d = (1.0 / delta).ln().ceil().max(1.0) as u32; - (w, d) - } -} - -/// The workload-level accuracy target combined with an intent's own — -/// pulls the intent's `accuracy` field out of whichever variant carries -/// one. `_` covers every non-approximate-capable variant, unreachable in -/// practice (`rank_candidates`/`size_params` are only ever called for -/// `Quantile`/`Cardinality`/`Count`/`TopK` — the shapes -/// `replacement::SketchAlgorithmStrategy::replacements` handles). -fn intent_accuracy(intent: &AggIntent) -> AccuracyTarget { - match intent { - AggIntent::Quantile { accuracy, .. } - | AggIntent::Cardinality { accuracy, .. } - | AggIntent::TopK { accuracy, .. } - | AggIntent::FrequencyL2 { accuracy, .. } - | AggIntent::FrequencyEntropy { accuracy, .. } => accuracy.clone(), - AggIntent::Count { accuracy } => accuracy.clone(), - _ => AccuracyTarget::Exact, - } -} - -impl CostModel for ControlPlaneCostModel { - fn summary_support_evidence( - &self, - summary: &planner_types::post_asap::SummaryNode, - ) -> Option { - use planner_types::post_asap::{NonNegativeWeightProof, WeightDomain}; - let dag = - planner_types::post_asap::compile_executable_dag(&std::rc::Rc::new(summary.clone())) - .ok()?; - // Counter-weighted heaps now consume explicit rate values. The raw - // ingestion adapter cannot bind that frontier as counter deltas. - let requires_rate_values = dag.nodes.iter().any(|node| matches!(&node.payload, - ExecutableOperatorPayload::SummaryAgg { input, family: SummaryFamilyType::Sketch(kind, _), .. } - if matches!(kind.algorithm(), SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap) - && matches!(input.weight_domain, WeightDomain::NonNegative { proof: NonNegativeWeightProof::ResetAwareCounterDerivative }))); - requires_rate_values.then_some(false) - } - - fn candidate_cost( - &self, - candidate: &ReplacementSubDAG, - _target: &TargetSubDAG<'_>, - ) -> Option { - self.candidate_cost_estimate(candidate) - .map(|estimate| Cost(estimate.value)) - } - - fn value_operation_capabilities(&self) -> ValueOperationCapabilities { - ValueOperationCapabilities { - query_time: true, - ingestion_time: false, - } - } - - fn exact_composition_cost_inputs( - &self, - request: &ExactCompositionCostRequest<'_>, - ) -> ExactCompositionCostInputs { - let provenance = || CostProvenance { - model: "ASAPQuery measured exact composition".into(), - version: "unavailable".into(), - }; - let Some(row) = self - .exact_composition_costs - .iter() - .find(|row| row.matches(request)) - else { - return ExactCompositionCostInputs::unknown(provenance()); - }; - ExactCompositionCostInputs { - exact_cost_per_row: Some(row.exact_cpu_ns_per_row), - expected_input_rows: Some(row.expected_input_rows), - expected_output_rows: Some(row.expected_output_rows), - summary_maintenance_cost_per_update: Some(row.summary_maintenance_cpu_ns_per_update), - summary_read_cost: Some(row.summary_read_cpu_ns), - update_rate: Some(row.update_rate_per_second), - evaluation_rate: Some(EvaluationRate( - row.evaluation_rate_per_second * request.effective_consumer_count as f64, - )), - raw_recompute_cost: Some(row.raw_recompute_cpu_ns), - unit: asap_aware_mapping::CostUnit::CostUnitsPerSecond, - provenance: CostProvenance { - model: format!( - "ASAPQuery measured exact composition ({})", - row.data_snapshot_id - ), - version: row.model_version.clone(), - }, - } - } - - fn summary_maintenance_lifecycle_cost_inputs( - &self, - _summary: &planner_types::post_asap::SummaryNode, - ) -> SummaryMaintenanceLifecycleCostInputs { - self.lifecycle_costs.clone() - } - - fn summary_maintenance_capabilities( - &self, - _summary: &planner_types::post_asap::SummaryNode, - ) -> SummaryMaintenanceCapabilities { - self.summary_maintenance - } - - fn complete_summary_candidate_estimate( - &self, - _root: &planner_types::post_asap::SummaryNode, - _target: Option<&planner_types::pre_asap::QueryExpr>, - deployments: &[CostedSummaryDeployment<'_>], - _horizon: Option, - _expected_reads: Option, - _required_accuracy: &[AccuracyTarget], - ) -> Option { - // One compiler candidate describes the complete concrete realization - // of this query DAG. The current executor exposes one anchored window - // framework across all reachable summary states; represent that full - // per-state assignment explicitly rather than relying on traversal - // order or leaving any state uncosted. - if deployments.is_empty() || self.window_framework_costs.is_empty() { - return None; - } - let lifecycle_cost: f64 = deployments - .iter() - .map(|deployment| deployment.selected_cost.0) - .sum(); - self.window_framework_costs - .iter() - // GOS/error propagation is introduced by the later adaptation - // slice. Until then, do not claim an approximate exponential - // histogram window is exact. - .filter(|(_, framework, _)| { - !matches!(framework, SummaryWindowFramework::ExponentialHistogram) - }) - .filter(|(_, _, cost)| cost.0.is_finite() && cost.0 >= 0.0) - .min_by(|left, right| { - left.2 - .0 - .total_cmp(&right.2 .0) - .then_with(|| left.0.cmp(&right.0)) - }) - .map( - |(id, framework, physical_cost)| CompleteSummaryCandidateEstimate { - physical_plan_id: id.clone(), - cost: Cost(lifecycle_cost + physical_cost.0), - window_frameworks: vec![Some(framework.clone()); deployments.len()], - window_accuracy_guarantee: Some( - planner_types::post_asap::ResultGuarantee::exact( - "backend exact window implementation", - ), - ), - }, - ) - } - - fn rank_candidates( - &self, - intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - let defaults = match intent { - // bind_ddsketch_quantile (priority 6) always wins the old - // dispatcher's tie-break over bind_kll_quantile (priority 5) - // whenever both can bind (see bind_kll_quantile.rs's - // `priority()` doc — no eps-dependent split actually exists - // between the two rules today, both read the same combined - // eps). Pure static reorder: DDSketch before Kll. - AggIntent::Quantile { .. } => { - let mut v = candidates.to_vec(); - if let Some(pos) = v.iter().position(|k| *k == SketchAlgorithm::DDSketch) { - let dd = v.remove(pos); - v.insert(0, dd); - } - v - } - AggIntent::TopK { .. } => { - let preferred = self.topk_family_order(&intent_accuracy(intent)); - // Distinct grouping implementations can use the same algorithm. - // Ranking must preserve the candidate multiset, not deduplicate it. - let mut ranked = candidates.to_vec(); - ranked.sort_by_key(|kind| { - preferred - .iter() - .position(|candidate| candidate == kind) - .unwrap_or(preferred.len()) - }); - ranked - } - // Cardinality → Hll, Count → Cms: control_plane only ever - // binds one family for each; asap-plan's static order already - // puts it first (`summary_candidates`), nothing to reorder. - _ => candidates.to_vec(), - }; - self.rank_with_offline_evidence(intent, defaults) - } - - fn size_params( - &self, - kind: SketchAlgorithm, - intent: &AggIntent, - eps: f64, - delta: f64, - ) -> SketchParams { - let (max_error, theoretical) = match intent { - AggIntent::TopK { k, .. } => { - let (eps, delta) = self.topk_eps_delta(&intent_accuracy(intent)); - let (w, d) = Self::cms_width_depth(eps, delta); - // CountSketch/CMS columns MUST be a power of two: the - // agent (asapedgeprocessor config_validate) rejects - // non-pow2 cols. Round up — this only tightens the - // additive bound (ε ≤ e/w). - let w = w.next_power_of_two(); - let heap_size = *k as u32; - let params = match kind { - SketchAlgorithm::CmsWithHeap => SketchParams::CmsWithHeap { - width: w, - depth: d, - heap_size, - }, - _ => SketchParams::CountSketchWithHeap { - width: w, - depth: d, - heap_size, - }, - }; - (eps, params) - } - _ => { - let Some((eps, delta)) = self.combined_eps_delta(&intent_accuracy(intent)) else { - // Either side Exact: unreachable in practice for - // Quantile/Cardinality/Count (an Exact intent never - // reaches the replacement strategies upstream — see - // `realizations_for_intent`'s `Exact => exact_realization` - // arm), kept as a safe fallback rather than a panic. - return asap_aware_mapping::DefaultCostModel - .size_params(kind, intent, eps, delta); - }; - let params = match kind.clone() { - SketchAlgorithm::Kll => SketchParams::Kll { - k: kll_k_for_eps(eps), - }, - SketchAlgorithm::DDSketch if (0.0..1.0).contains(&eps) => { - SketchParams::DDSketch { alpha: eps } - } - SketchAlgorithm::Hll => SketchParams::Hll { - precision: hll_precision_for_eps(eps), - }, - SketchAlgorithm::Cms => { - let (w, d) = Self::cms_width_depth(eps, delta); - SketchParams::Cms { width: w, depth: d } - } - other => { - asap_aware_mapping::DefaultCostModel.size_params(other, intent, eps, delta) - } - }; - (eps, params) - } - }; - let decision = match ( - &self.erp, - super::super::erp::ReadoutEvidence::for_intent(&kind, intent), - ) { - (Some(policy), Some(readout)) => { - Some(policy.select_readout(kind, readout, max_error, theoretical.clone())) - } - _ => self.erp_parameter_decision(kind, max_error, theoretical.clone()), - }; - match decision { - Some(ErpParameterDecision::Empirical { - params, - record_id, - observed_error, - estimated_cost, - }) => { - tracing::info!(erp_record_id = %record_id, observed_error, estimated_cost, "selected empirical ERP sketch parameters"); - params - } - Some(ErpParameterDecision::TheoreticalFallback { params, reason }) => { - tracing::warn!(reason = %reason, "ERP miss or drift; using theoretical sizing"); - params - } - Some(ErpParameterDecision::ExactFallback { reason }) => { - tracing::warn!(reason = %reason, "ERP and theoretical sizing unavailable; exact fallback required"); - theoretical - } - None => theoretical, - } - } - - /// Realize control_plane's `Frequency` (`ext_kind: "frequency"`) - /// point-query intent (ASAPController#150). `SketchAlgorithm::Cms` (heap- - /// less — a point lookup needs no heap, unlike `TopK`) matches - /// `capability_matching::pick_family`'s own `Frequency → Cms` mapping - /// (and its `is_valid_pair` truth table, which declares `(Cms, - /// Frequency)` valid and has no `(CountSketch, Frequency)` entry) — - /// the newer, tested, currently-authoritative source of truth for this - /// choice, not the older `sketch_catalog::sketch_type_for_op`'s - /// `SketchType::CountSketch` (a genuinely different sketch algorithm - /// under a same-ish name — `SketchType` has separate `CountSketch` - /// and `CountMinSketch` variants; that mapping predates - /// `capability_matching` and disagrees with it). Sized the same - /// `e/eps` width / `ln(1/delta)` depth way every other CMS-family kind - /// here is. `PassThrough` for any other `ext_kind` (none exist yet) - /// or an unparseable/`Exact` accuracy, matching every other - /// approximate-capable intent's `Exact ⇒ no sketch form` policy. - fn realize_extension(&self, ext_kind: &str, payload: &serde_json::Value) -> Realization { - if ext_kind != FREQUENCY_EXT_KIND { - return Realization::PassThrough; - } - if self.offline_frequency_comparison.is_some() { - return self - .offline_frequency_recommendation(payload) - .ok() - .and_then(|recommendation| recommendation.selected_sketch().cloned()) - .map(|configuration| { - Realization::Sketch(planner_types::post_asap::SketchKind::new( - configuration.algorithm, - configuration.params, - )) - }) - .unwrap_or(Realization::PassThrough); - } - let Some(accuracy) = payload - .get("accuracy") - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - else { - return Realization::PassThrough; - }; - let Some((eps, delta)) = self.combined_eps_delta(&accuracy) else { - return Realization::PassThrough; - }; - let (width, depth) = Self::cms_width_depth(eps, delta); - Realization::Sketch(planner_types::post_asap::SketchKind::new( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: width.next_power_of_two(), - depth, - }, - )) - } - - /// Build the `SketchQuery` readout for `Frequency`. `item_label`/ - /// `item_value` are populated by `intent_algebra::agg_intent::frequency` - /// once the filter value is threaded through (ASAPQuery-backend Phase - /// 3 — not yet); until then `payload` never has them, so this - /// correctly falls back to the bare bucket total - /// (`key: SampleValue, value: None`) — the same answer a `Frequency` - /// intent with no item filter should give either way. - fn readout_extension( - &self, - ext_kind: &str, - payload: &serde_json::Value, - _col: &ColumnRef, - ) -> SketchQuery { - debug_assert_eq!( - ext_kind, FREQUENCY_EXT_KIND, - "readout_extension called for an ext_kind realize_extension never realizes as Sketch" - ); - let item_label = payload.get("item_label").and_then(|v| v.as_str()); - let item_value = payload - .get("item_value") - .and_then(|v| v.as_str()) - .map(str::to_string); - match item_label { - Some(label) => SketchQuery::PointCount { - key: ColumnRef::Named(label.to_string()), - value: item_value, - }, - None => SketchQuery::PointCount { - key: ColumnRef::SampleValue, - value: None, - }, - } - } -} - -/// A `CostModel` that forces a single family for whichever intent it's -/// asked to rank, delegating parameter sizing to an inner -/// [`ControlPlaneCostModel`]. Used by `physical::workload_planner::bind_workload_typed`, -/// which already has a definitive family pick from the capability matrix -/// (or a `sketch_type_override`) and just needs the matching binding, not -/// a fresh selection decision. -pub struct ForcedFamilyCostModel { - inner: ControlPlaneCostModel, - forced: SketchAlgorithm, -} - -impl ForcedFamilyCostModel { - pub fn new(workload_accuracy: AccuracyTarget, forced: SketchAlgorithm) -> Self { - Self { - inner: ControlPlaneCostModel::new(workload_accuracy), - forced, - } - } -} - -impl CostModel for ForcedFamilyCostModel { - fn summary_support_evidence( - &self, - summary: &planner_types::post_asap::SummaryNode, - ) -> Option { - self.inner.summary_support_evidence(summary) - } - - fn candidate_cost( - &self, - candidate: &ReplacementSubDAG, - target: &TargetSubDAG<'_>, - ) -> Option { - self.inner.candidate_cost(candidate, target) - } - - fn rank_candidates( - &self, - intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - let mut ranked = self.inner.rank_candidates(intent, candidates); - if let Some(pos) = ranked.iter().position(|kind| kind == &self.forced) { - let forced = ranked.remove(pos); - ranked.insert(0, forced); - } - ranked - } - - fn size_params( - &self, - kind: SketchAlgorithm, - intent: &AggIntent, - eps: f64, - delta: f64, - ) -> SketchParams { - // Latest Planner validates the selected candidate against its own - // accuracy algebra. Reuse its sizing formula for an explicitly - // forced family so the override changes only algorithm preference, - // never weakens the requested guarantee. - asap_aware_mapping::DefaultCostModel.size_params(kind, intent, eps, delta) - } - - // `realize_extension`/`readout_extension` delegate to `inner` rather - // than falling back to the trait's default `PassThrough` — otherwise - // `bind_workload_typed`'s `Frequency` contract row (which binds via - // `ForcedFamilyCostModel`, already knowing its family pick from the - // capability matrix) would still decline pending #150 even after - // `ControlPlaneCostModel` itself learned to realize it. - fn realize_extension(&self, ext_kind: &str, payload: &serde_json::Value) -> Realization { - self.inner.realize_extension(ext_kind, payload) - } - - fn readout_extension( - &self, - ext_kind: &str, - payload: &serde_json::Value, - col: &ColumnRef, - ) -> SketchQuery { - self.inner.readout_extension(ext_kind, payload, col) - } -} - -/// Map an ε rank-error budget to a KLL stream-size `k`. Verbatim port of -/// `bind_kll_quantile.rs::kll_k_for_eps` — power-of-two rungs (200, 400, -/// 800, 2048, 8192) so the in-tree `algebra::directory` continues to -/// recognise the parameter. -fn kll_k_for_eps(eps: f64) -> u32 { - if eps <= 0.0 { - return 8192; - } - if eps >= 0.01 { - 200 - } else if eps >= 0.005 { - 400 - } else if eps >= 0.0025 { - 800 - } else if eps >= 0.001 { - 2048 - } else { - 8192 - } -} - -/// Map an ε standard-error budget to the HLL `precision`. Verbatim port -/// of `bind_hll_cardinality.rs::hll_precision_for_eps`. -fn hll_precision_for_eps(eps: f64) -> u8 { - if eps <= 0.0 { - return 16; - } - if eps >= 0.03 { - 10 - } else if eps >= 0.015 { - 12 - } else if eps >= 0.008 { - 14 - } else { - 16 - } -} - -#[cfg(test)] -mod tests { - use super::*; - use planner_types::pre_asap::{default_cardinality, default_quantile}; - - fn eps(e: f64) -> AccuracyTarget { - AccuracyTarget::Epsilon(e) - } - - #[test] - fn quantile_always_prefers_ddsketch_over_kll() { - let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(0.1)); - let ranked = model.rank_candidates( - &default_quantile(0.99), - &[SketchAlgorithm::Kll, SketchAlgorithm::DDSketch], - ); - assert_eq!( - ranked, - vec![SketchAlgorithm::DDSketch, SketchAlgorithm::Kll] - ); - } - - #[test] - fn kll_k_matches_bind_kll_quantile_rungs() { - let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(1.0)); - let params = model.size_params(SketchAlgorithm::Kll, &default_quantile(0.99), 0.01, 0.01); - assert_eq!(params, SketchParams::Kll { k: 200 }); - - let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(1.0)); - let tight = planner_types::pre_asap::AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: eps(0.001), - }; - let params = model.size_params(SketchAlgorithm::Kll, &tight, 0.01, 0.01); - assert_eq!(params, SketchParams::Kll { k: 2048 }); - } - - #[test] - fn hll_precision_matches_bind_hll_cardinality_rungs() { - let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(1.0)); - let params = model.size_params(SketchAlgorithm::Hll, &default_cardinality(), 0.01, 0.01); - assert_eq!(params, SketchParams::Hll { precision: 14 }); - } - - #[test] - fn topk_tight_tier_forces_countsketch_even_when_ranked_from_cms() { - let model = ControlPlaneCostModel::new(AccuracyTarget::Exact); - let intent = planner_types::pre_asap::AggIntent::TopK { - k: 5, - accuracy: eps(0.01), - }; - let ranked = model.rank_candidates( - &intent, - &[ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap, - ], - ); - assert_eq!( - ranked, - vec![ - SketchAlgorithm::CountSketchWithHeap, - SketchAlgorithm::CmsWithHeap - ] - ); - } - - // Workload search can enumerate multiple layouts for the same algorithm. - #[test] - fn topk_ranking_preserves_duplicate_candidates() { - let model = ControlPlaneCostModel::new(AccuracyTarget::Exact); - let intent = AggIntent::TopK { - k: 1, - accuracy: eps(0.01), - }; - let mut candidates = vec![ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap, - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap, - ]; - let mut ranked = model.rank_candidates(&intent, &candidates); - candidates.sort(); - ranked.sort(); - assert_eq!(ranked, candidates); - } - - #[test] - fn topk_loose_tier_prefers_cheaper_cms_heap() { - let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(0.1)); - let intent = planner_types::pre_asap::AggIntent::TopK { - k: 5, - accuracy: eps(0.01), - }; - let ranked = model.rank_candidates( - &intent, - &[ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap, - ], - ); - assert_eq!(ranked[0], SketchAlgorithm::CmsWithHeap); - } - - #[test] - fn topk_width_is_rounded_up_to_a_power_of_two() { - let model = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(0.1)); - let intent = planner_types::pre_asap::AggIntent::TopK { - k: 5, - accuracy: eps(0.01), - }; - let params = model.size_params(SketchAlgorithm::CmsWithHeap, &intent, 0.0, 0.0); - let SketchParams::CmsWithHeap { - width, heap_size, .. - } = params - else { - panic!("expected CmsWithHeap params"); - }; - assert!(width.is_power_of_two()); - assert_eq!(heap_size, 5); - } -} diff --git a/control_plane/src/physical/post_asap/deployment_expr.rs b/control_plane/src/physical/post_asap/deployment_expr.rs deleted file mode 100644 index b0372b6a6..000000000 --- a/control_plane/src/physical/post_asap/deployment_expr.rs +++ /dev/null @@ -1,220 +0,0 @@ -//! Deployment wrappers around ASAPPlanner's canonical post-ASAP summary IR. -//! -//! [`PostAsapPlan`] adds named `LetBinding` / `Ref` sharing for sibling walks. -//! [`PhysicalExpr`] adds edge/backend/archive placement. The lowerer produces -//! `Committed`; the stage allocator and emitter also handle explicit raw -//! placements supplied by callers. - -#![allow(dead_code)] - -use std::rc::Rc; -use std::time::Duration; - -use planner_types::post_asap::{SketchAlgorithm, SketchParams, SummaryNode}; - -/// Backend-local wrapper around the Planner-owned post-ASAP tree. -/// -/// It adds only named sharing required during physical placement and emission. -#[derive(Debug, Clone)] -pub enum PostAsapPlan { - /// A committed post-ASAP sub-tree — `SummaryAgg` / `SummaryEstimate` / - /// `SummaryMerge` / `KeepPreAsap`, whatever the replacement strategy - /// (or a deployment-specific pre-pass) produced. - Summary(Rc), - - /// SQL `WITH name AS (expr) SELECT ... FROM name` / sketch-state - /// fan-in: name a sub-expression so multiple parents can reference it. - /// Carries the two-tier fan-in described in design.md §1339 — outer - /// let names a `Window` output, inner let names a `SummaryAgg{Kll}` - /// shared by two `SummaryEstimate` parents reading different quantiles. - LetBinding { - /// Binding name; must be unique within the surrounding scope. - name: String, - /// Bound sub-expression. - expr: Rc, - /// In-scope sub-tree — references the binding via `Ref`. - child: Rc, - }, - - /// Reference a `LetBinding` by name. Resolution is lexical (scope - /// follows the surrounding `LetBinding` chain). - Ref { - /// Bound name. - name: String, - }, -} - -/// Physical placement — "where/how". Wraps an already-committed -/// [`PostAsapPlan`] (summary selection is final by the time it reaches here) -/// with placement -/// info: build at the edge (the common case — `Committed` needs no extra -/// annotation since the `PostAsapPlan` itself is the whole story), or one of -/// Phase ε.1's two backend/archive placements. -#[derive(Debug, Clone)] -pub enum PhysicalExpr { - /// Sketch built at the edge — the default placement. The committed - /// `PostAsapPlan` alone determines the output. - Committed(PostAsapPlan), - - /// Phase ε.1 Mode 2: no sketch processor at the edge — raw OTLP - /// forwards to the backend, which builds the sketch at ingest. The - /// `family` and `params` are the sketch the backend will build, so - /// the backend's `InstalledPrecomputePlan` `aggregation_input` is `raw` for - /// this metric. - RawAtEdgeSketchAtBackend { - /// Sketch family the backend will build at ingest. - family: SketchAlgorithm, - /// Sketch parameters (validated by the catalog at bind time). - params: SketchParams, - /// Input sub-tree — typically `Summary(Logical(Window{...}))` or - /// `Summary(Logical(Scan{...}))`. - child: Box, - }, - - /// Phase ε.1 Mode 3: no sketch processor at the edge — raw OTLP ships - /// directly to Prometheus's native OTLP receiver at - /// `/api/v1/otlp/v1/metrics`. The backend HTTP-forwards queries to - /// Prometheus's `/api/v1/query` endpoint (the `prometheus_remote` - /// engine). Accuracy is exact (ε = 0) — Prometheus owns the raw - /// samples; no sketch math is involved. - /// - /// The variant carries enough projection info for the edge agent's - /// pipeline to ship the right metric with the right labels, and for - /// the backend's storage routing to claim the metric. - RawAtEdgePrometheusArchive { - /// Metric name as it appears at the edge (and in - /// `BackendStorageRouting`). - metric: String, - /// Optional window — when present, the planner pre-bucketed the - /// metric into windowed scrape data. Prometheus stores the raw - /// stream regardless; the field is informational for the L5 - /// emitter so it can size scrape intervals consistently. - window: Option, - /// Label projection — labels promoted from OTLP resource - /// attributes by Prometheus's - /// `otlp.promote_resource_attributes` config. Default - /// `["service.name", "service.namespace", "service.instance.id"]` - /// — see `deploy/configs/prometheus-otlp-receiver.yml`. - label_proj: Vec, - }, -} - -impl PhysicalExpr { - /// Convenience constructor for the common case: a committed - /// `SummaryNode` sketch-built at the edge, no placement wrapper. - pub fn committed(node: Rc) -> Self { - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::enums::WindowKind; - use planner_types::pre_asap::{Column, DataType, QueryExpr, Schema, Source}; - use std::time::Duration; - - fn ts_scan() -> QueryExpr { - let schema = Schema::with_time_index( - vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - table: None, - }, - Column { - name: "service".into(), - dtype: DataType::Utf8, - nullable: false, - table: None, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - table: None, - }, - ], - 0, - vec![vec![0, 1]], - ); - let pred = crate::test_support::label_eq_predicate("service", "api", &schema) - .expect("service column present in schema"); - QueryExpr::Scan { - source: Source::TimeSeries { - metric: "http_request_duration_seconds".into(), - }, - predicates: vec![pred], - schema, - } - } - - fn windowed_scan() -> QueryExpr { - QueryExpr::TimeRange { - range: Duration::from_secs(300), - child: Rc::new(ts_scan()), - } - } - - #[test] - fn committed_wraps_an_implement_tree_result() { - let q = QueryExpr::Aggregate { - reduction: planner_types::pre_asap::Reduction::by(vec![]), - measures: vec![planner_types::pre_asap::AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: crate::types::AccuracyTarget::Epsilon(0.01), - }], - output_names: Vec::new(), - having: None, - child: Rc::new(windowed_scan()), - }; - let node = crate::planner_selection::select_query( - &q, - &crate::physical::post_asap::cost_model::ForcedFamilyCostModel::new( - crate::types::AccuracyTarget::Epsilon(0.01), - planner_types::post_asap::SketchAlgorithm::Kll, - ), - ) - .expect("implements"); - let e = PhysicalExpr::committed(node); - match e { - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) => match &node.expr { - planner_types::post_asap::SummaryExpr::SummaryEstimate { - query, - summary_input, - } => { - assert!( - matches!(query, planner_types::post_asap::SketchQuery::Quantile { q } if *q == 0.99) - ); - match &summary_input.expr { - planner_types::post_asap::SummaryExpr::SummaryAgg { - family, child, .. - } => { - assert_eq!( - family, - &planner_types::post_asap::SummaryFamilyType::Sketch( - planner_types::post_asap::SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { k: 269 }, - ), - planner_types::post_asap::GroupingStrategy::default(), - ) - ); - assert!(matches!( - child.expr, - planner_types::post_asap::SummaryExpr::KeepPreAsap(_) - )); - } - other => panic!("expected SummaryAgg, got {other:?}"), - } - } - other => panic!("expected SummaryEstimate, got {other:?}"), - }, - other => panic!("expected Committed(Summary(_)), got {other:?}"), - } - } -} diff --git a/control_plane/src/physical/post_asap/lower.rs b/control_plane/src/physical/post_asap/lower.rs deleted file mode 100644 index aa572cdf8..000000000 --- a/control_plane/src/physical/post_asap/lower.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! Query binding delegates selection to Planner's costed workload search. - -#![allow(dead_code)] - -use std::rc::Rc; - -use asap_aware_mapping::cost_model::CostModel; -use thiserror::Error; - -use crate::physical::post_asap::cost_model::ControlPlaneCostModel; -use crate::physical::post_asap::deployment_expr::{PhysicalExpr, PostAsapPlan}; -use crate::types::AccuracyTarget; -use planner_types::pre_asap::{AggIntent, QueryExpr}; - -/// Errors surfaced by the `bind_query_expr` lowering. -#[derive(Debug, Error)] -pub enum BindingError { - /// L3 schema derivation failed while lifting an edge to `SummarySchema` — - /// forwarded from `asap_aware_mapping::replacement`. - #[error("L3->L4 implementation failed: {0}")] - Implement(#[from] crate::planner_selection::SelectionError), -} - -/// Lower an L3 query under the supplied accuracy target using -/// [`ControlPlaneCostModel`]. Returns [`PhysicalExpr::Committed`]; placement -/// is separate from summary selection. -pub fn bind_query_expr( - expr: &QueryExpr, - accuracy: AccuracyTarget, -) -> Result { - let cost_model = ControlPlaneCostModel::new(accuracy); - bind_query_expr_with_cost_model(expr, &cost_model) -} - -/// Bind a query using the explicitly supplied planning cost model. -pub fn bind_query_expr_with_cost_model( - expr: &QueryExpr, - cost_model: &dyn CostModel, -) -> Result { - Ok(PhysicalExpr::Committed(bind_recursive(expr, cost_model)?)) -} - -fn bind_recursive( - expr: &QueryExpr, - cost_model: &dyn CostModel, -) -> Result { - match expr { - // These roots describe exact query semantics, not summary candidate - // sites. Preserve the complete expression so archive execution retains - // selector labels, ordering, and comparison filtering. - QueryExpr::Scan { .. } - | QueryExpr::Sort { .. } - | QueryExpr::Filter { .. } - | QueryExpr::BinaryOp { - op: planner_types::pre_asap::BinaryOpKind::Compare(_), - .. - } => Ok(PostAsapPlan::Summary( - crate::planner_selection::keep_pre_asap(expr)?, - )), - - // The canonical L3 IR places `TimeRange` *above* a single-statistic - // sketchable `Aggregate` (`lower_promql`'s window-swap; was - // `Window` before the ASAPPlanner pin migration). The replacement strategy - // only recurses through the `Aggregate` spine (see its module - // docs' "conservative fallbacks" — a logical parent above a - // bindable aggregate subsumes it unbound), so push the range - // down under the aggregate and re-dispatch, exactly as the old - // hand-written walk did — the range then rides along inside the - // bound node's `Logical(...)` child. - QueryExpr::TimeRange { range, child } - if matches!(child.as_ref(), QueryExpr::Aggregate { .. }) => - { - let QueryExpr::Aggregate { - reduction, - measures: aggs, - output_names, - having, - child: agg_child, - } = child.as_ref() - else { - unreachable!("guarded by the `matches!` above") - }; - let pushed = QueryExpr::Aggregate { - reduction: reduction.clone(), - measures: aggs.clone(), - output_names: output_names.clone(), - having: having.clone(), - child: Rc::new(QueryExpr::TimeRange { - range: *range, - child: agg_child.clone(), - }), - }; - bind_recursive(&pushed, cost_model) - } - - // Workload-aware instant selectors retain their source horizon even - // when there is no aggregate to bind. - QueryExpr::TimeRange { .. } => Ok(PostAsapPlan::Summary( - crate::planner_selection::keep_pre_asap(expr)?, - )), - - // Exact Count cannot use this deployment's Sum accumulator: it counts - // values rather than samples. Keep it logical for archive execution. - QueryExpr::Aggregate { - measures: aggs, - having: None, - .. - } if matches!( - aggs.as_slice(), - [AggIntent::Count { - accuracy: AccuracyTarget::Exact - }] - ) => - { - Ok(PostAsapPlan::Summary( - crate::planner_selection::select_query(expr, cost_model)?, - )) - } - - _ => Ok(PostAsapPlan::Summary( - crate::planner_selection::select_query(expr, cost_model)?, - )), - } -} diff --git a/control_plane/src/physical/post_asap/matcher.rs b/control_plane/src/physical/post_asap/matcher.rs deleted file mode 100644 index 29f7d4bc9..000000000 --- a/control_plane/src/physical/post_asap/matcher.rs +++ /dev/null @@ -1,348 +0,0 @@ -//! The reference downstream implementation of [`asap_aware_mapping::Matcher`]. -//! -//! `asap_aware_mapping::replacement::Matcher` is a trait with no default implementation -//! and no shipped instance — deliberately, per its crate doc: which -//! `Realization`s are actually *available* anywhere is entirely a -//! downstream deployment's concern, and even the pure sketch-algebra -//! compatibility rules (a heap-bearing top-k sketch also satisfying a bare -//! frequency point-query) turned out to have deployment-specific competitors -//! (single-vs-multi-population re-aggregation) that don't reduce to a fact -//! about `SummaryKind` alone. -//! -//! [`SummaryFamilyMatcher`] restores exactly the family-compatibility logic -//! that briefly lived as a concrete `Realization::is_satisfied_by` method -//! in `asap-plan` (PR #140) before it was converted into this trait (PR -//! #141) — the `SummaryFamily`/`summary_family` classifier below is a -//! verbatim port. It answers only the kind-family question: "is an -//! available `(SummaryKind, SummaryParams)` pair an acceptable substitute -//! for a required one." It deliberately does **not** attempt the -//! single-vs-multi-population re-aggregation question (e.g. "can a keyed -//! `Sum` accumulator serve an unkeyed `Sum` query") — `Realization` -//! carries no grouping information at all (grouping lives beside the kind, -//! on whatever node carries it, not inside the kind — see -//! `crates/asap_types/src/key_by_label_names.rs`'s module doc for the same -//! design call made on the data-plane side), so a two-`Realization` -//! `Matcher` cannot correctly answer that question. A caller needing that -//! richer, grouping-aware answer must check `AggregationType` compatibility -//! (analogous to `SummaryFamily` here) *and* `grouping_labels` -//! subset-compatibility side by side, composed at the call site rather than -//! inside a single `Matcher::is_satisfied_by`. -//! -//! Retirement note (2026-07) — reverted: this file was briefly gutted down -//! to just [`sketch_family_satisfied`] on the assumption `SummaryFamilyMatcher` -//! had no real consumer (true at the time — no `asap_plan` signature took -//! `impl`/`dyn Matcher` yet). That assumption no longer holds: this struct -//! is the intended `Matcher` implementation for the `SummaryExecutor` -//! serving-time rollout (`data_plane/docs/l4node-plan-executor-design.md`), -//! which is actively being built to replace -//! `storage_engines/sketch_db/query/sketch_reducer.rs`. Restored in full. - -use asap_aware_mapping::{Matcher, Realization}; -use planner_types::post_asap::SketchAlgorithm; - -/// [`Matcher`] impl covering pure sketch-family compatibility. See the -/// module doc for what this deliberately does not cover. -pub struct SummaryFamilyMatcher; - -impl Matcher for SummaryFamilyMatcher { - /// `required` is what an intent needs; `available` is what some - /// downstream inventory (a sketch index, a policy registry, …) - /// already has materialized somewhere. Rules: - /// - `PassThrough` required is vacuously satisfied by anything — no - /// summary is needed at all, so there is nothing to match. - /// - An `ExactAccumulator` is satisfied only by the exact same - /// `SummaryKind` — accumulators carry no notion of "family"; `Sum` - /// and `MinMax` answer different questions, full stop. - /// - A `Sketch` is satisfied by an available sketch in the same family - /// (`Kll`/`DDSketch` are interchangeable quantile answers; - /// `Hll`/`Theta`/`Kmv` are interchangeable cardinality answers), with - /// one asymmetric exception: a heap-bearing top-k sketch - /// (`CmsWithHeap`/`CountSketchWithHeap`) also answers a bare - /// frequency point-query (`Cms`/`CountSketch`) — the heap is - /// additional info layered on top of the same underlying matrix — - /// but not the reverse (a heap-less sketch cannot enumerate top-k - /// items it never tracked). - fn is_satisfied_by(&self, required: &Realization, available: &Realization) -> bool { - // ASAPController#170 had merged `Sketch`/`ExactAccumulator` into - // one `Summary { kind, params }` variant, recoverable via - // `kind.is_exact()`; ASAPPlanner#218 split them back into - // distinct `Sketch`/`ExactAggregate` variants (see - // control_plane/docs/design-asapplanner-pin-migration.md). The - // variant-tag mismatch this match falls through to `_ => false` - // for (comparing a `Sketch` against an `ExactAggregate`) is now - // just the natural consequence of them being separate variants - // again, same behavior as the `is_exact()` mismatch this replaced. - match (required, available) { - (Realization::PassThrough, _) => true, - ( - Realization::ExactAggregate { kind: required, .. }, - Realization::ExactAggregate { kind: have, .. }, - ) => required == have, - (Realization::Sketch(required), Realization::Sketch(have)) => { - sketch_family_satisfied(required.algorithm(), have.algorithm()) - } - _ => false, - } - } -} - -/// Pure `SketchAlgorithm`-to-`SketchAlgorithm` family-compatibility check — the -/// same rule [`SummaryFamilyMatcher::is_satisfied_by`] applies in its -/// `Sketch` arm, exposed directly for callers that only have bare kinds -/// (no [`planner_types::post_asap::SketchParams`]) to compare. -/// `control_plane::physical::runtime_capability::Capability::is_satisfied_by` -/// is the first such caller: its `SketchAlgorithm` query-side dispatch -/// tag never carries params, so constructing a full -/// `Realization::Sketch{kind, params}` just to discard the params -/// would mean fabricating meaningless param values. See that module's -/// doc for why `Capability`/`SketchAlgorithm` themselves aren't deleted -/// outright (`scratchpad/artifacts/enum-unification-plan.md` §8 Step 4). -pub fn sketch_family_satisfied(required: &SketchAlgorithm, available: &SketchAlgorithm) -> bool { - let req_family = summary_family(required); - let have_family = summary_family(available); - req_family.satisfied_by(have_family) -} - -/// Return the family of a [`SketchAlgorithm`] for [`SummaryFamilyMatcher`]. -/// Every variant is a sketch; exact accumulators use a separate type. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum SummaryFamily { - Universal, - Quantile, - Cardinality, - /// Bare per-item frequency point-query — no heavy-hitter heap. - Frequency, - /// Frequency, augmented with a heavy-hitter heap for top-k extraction. - FrequencyTopk, -} - -impl SummaryFamily { - /// True when `available` (this family) satisfies `self` (the required - /// family). Same family always satisfies; the one asymmetric case is - /// `FrequencyTopk` (heap-bearing) satisfying a bare `Frequency` - /// requirement — never the reverse. - fn satisfied_by(self, available: SummaryFamily) -> bool { - self == available - || (self == SummaryFamily::Frequency && available == SummaryFamily::FrequencyTopk) - } -} - -fn summary_family(kind: &SketchAlgorithm) -> SummaryFamily { - match kind { - SketchAlgorithm::UnivMon => SummaryFamily::Universal, - SketchAlgorithm::Kll | SketchAlgorithm::DDSketch => SummaryFamily::Quantile, - SketchAlgorithm::Hll | SketchAlgorithm::Theta | SketchAlgorithm::Kmv => { - SummaryFamily::Cardinality - } - SketchAlgorithm::Cms | SketchAlgorithm::CountSketch => SummaryFamily::Frequency, - SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap => { - SummaryFamily::FrequencyTopk - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use planner_types::post_asap::{ExactKind, ExactParams, SketchParams}; - - /// A valid `SketchParams` for `kind` — `is_satisfied_by` only matches - /// on `kind`, never `params`, but the test values should still be - /// real, constructible `(kind, params)` pairs rather than nonsense - /// combinations (e.g. `Hll` paired with `Kll`'s params) that could - /// never arise from real code. - fn params_for(kind: &SketchAlgorithm) -> SketchParams { - match kind { - SketchAlgorithm::UnivMon => SketchParams::UnivMon { - heap_size: 32, - sketch_rows: 5, - sketch_cols: 1024, - layers: 4, - }, - SketchAlgorithm::Kll => SketchParams::Kll { k: 200 }, - SketchAlgorithm::Cms => SketchParams::Cms { - width: 100, - depth: 5, - }, - SketchAlgorithm::Hll => SketchParams::Hll { precision: 14 }, - SketchAlgorithm::DDSketch => SketchParams::DDSketch { alpha: 0.01 }, - SketchAlgorithm::CmsWithHeap => SketchParams::CmsWithHeap { - width: 100, - depth: 5, - heap_size: 10, - }, - SketchAlgorithm::Kmv => SketchParams::Kmv { k: 1024 }, - SketchAlgorithm::Theta => SketchParams::Theta { k: 1024 }, - SketchAlgorithm::CountSketch => SketchParams::CountSketch { - width: 100, - depth: 5, - }, - SketchAlgorithm::CountSketchWithHeap => SketchParams::CountSketchWithHeap { - width: 100, - depth: 5, - heap_size: 10, - }, - } - } - - fn exact_params_for(kind: &ExactKind) -> ExactParams { - match kind { - ExactKind::Sum => ExactParams::Sum, - ExactKind::Count => ExactParams::Count, - ExactKind::Min => ExactParams::Min, - ExactKind::Max => ExactParams::Max, - ExactKind::Increase => ExactParams::Increase, - ExactKind::Rate => ExactParams::Rate, - ExactKind::IRate => ExactParams::IRate, - } - } - - fn sketch(kind: SketchAlgorithm) -> Realization { - let params = params_for(&kind); - Realization::Sketch(planner_types::post_asap::SketchKind::new(kind, params)) - } - - fn accumulator(kind: ExactKind) -> Realization { - let params = exact_params_for(&kind); - Realization::ExactAggregate { kind, params } - } - - #[test] - fn same_sketch_kind_satisfies_itself() { - let m = SummaryFamilyMatcher; - assert!(m.is_satisfied_by(&sketch(SketchAlgorithm::Kll), &sketch(SketchAlgorithm::Kll))); - assert!(m.is_satisfied_by(&sketch(SketchAlgorithm::Hll), &sketch(SketchAlgorithm::Hll))); - } - - #[test] - fn same_family_alternate_kind_satisfies() { - let m = SummaryFamilyMatcher; - // Kll / DDSketch are interchangeable quantile answers. - assert!(m.is_satisfied_by( - &sketch(SketchAlgorithm::Kll), - &sketch(SketchAlgorithm::DDSketch) - )); - // Hll / Theta / Kmv are interchangeable cardinality answers. - assert!(m.is_satisfied_by( - &sketch(SketchAlgorithm::Hll), - &sketch(SketchAlgorithm::Theta) - )); - assert!(m.is_satisfied_by(&sketch(SketchAlgorithm::Hll), &sketch(SketchAlgorithm::Kmv))); - } - - #[test] - fn cross_family_never_satisfies() { - let m = SummaryFamilyMatcher; - assert!(!m.is_satisfied_by(&sketch(SketchAlgorithm::Kll), &sketch(SketchAlgorithm::Hll))); - assert!(!m.is_satisfied_by(&sketch(SketchAlgorithm::Hll), &sketch(SketchAlgorithm::Kll))); - assert!(!m.is_satisfied_by(&sketch(SketchAlgorithm::Kll), &sketch(SketchAlgorithm::Cms))); - } - - #[test] - fn heap_bearing_available_satisfies_bare_frequency_required() { - let m = SummaryFamilyMatcher; - // A CmsWithHeap instance already carries the plain CMS matrix, so - // it answers a bare frequency point-query too. - assert!(m.is_satisfied_by( - &sketch(SketchAlgorithm::Cms), - &sketch(SketchAlgorithm::CmsWithHeap) - )); - assert!(m.is_satisfied_by( - &sketch(SketchAlgorithm::CountSketch), - &sketch(SketchAlgorithm::CountSketchWithHeap) - )); - } - - #[test] - fn bare_frequency_available_does_not_satisfy_topk_required() { - let m = SummaryFamilyMatcher; - // The reverse does not hold: a heap-less sketch never tracked the - // heavy-hitter heap, so it cannot enumerate top-k items. - assert!(!m.is_satisfied_by( - &sketch(SketchAlgorithm::CmsWithHeap), - &sketch(SketchAlgorithm::Cms) - )); - assert!(!m.is_satisfied_by( - &sketch(SketchAlgorithm::CountSketchWithHeap), - &sketch(SketchAlgorithm::CountSketch) - )); - } - - #[test] - fn cms_and_count_sketch_are_the_same_frequency_family() { - let m = SummaryFamilyMatcher; - assert!(m.is_satisfied_by( - &sketch(SketchAlgorithm::Cms), - &sketch(SketchAlgorithm::CountSketch) - )); - assert!(m.is_satisfied_by( - &sketch(SketchAlgorithm::CmsWithHeap), - &sketch(SketchAlgorithm::CountSketchWithHeap) - )); - } - - #[test] - fn exact_accumulator_requires_the_exact_same_kind() { - let m = SummaryFamilyMatcher; - assert!(m.is_satisfied_by(&accumulator(ExactKind::Sum), &accumulator(ExactKind::Sum))); - assert!(!m.is_satisfied_by(&accumulator(ExactKind::Sum), &accumulator(ExactKind::Max))); - assert!(!m.is_satisfied_by( - &accumulator(ExactKind::Increase), - &accumulator(ExactKind::Rate) - )); - } - - #[test] - fn sketch_and_accumulator_never_satisfy_each_other() { - let m = SummaryFamilyMatcher; - assert!(!m.is_satisfied_by(&sketch(SketchAlgorithm::Kll), &accumulator(ExactKind::Sum))); - assert!(!m.is_satisfied_by(&accumulator(ExactKind::Sum), &sketch(SketchAlgorithm::Kll))); - } - - #[test] - fn pass_through_required_is_vacuously_satisfied() { - let m = SummaryFamilyMatcher; - assert!(m.is_satisfied_by(&Realization::PassThrough, &sketch(SketchAlgorithm::Kll))); - assert!(m.is_satisfied_by(&Realization::PassThrough, &accumulator(ExactKind::Sum))); - assert!(m.is_satisfied_by(&Realization::PassThrough, &Realization::PassThrough)); - } - - #[test] - fn pass_through_available_never_satisfies_a_real_requirement() { - let m = SummaryFamilyMatcher; - assert!(!m.is_satisfied_by(&sketch(SketchAlgorithm::Kll), &Realization::PassThrough)); - assert!(!m.is_satisfied_by(&accumulator(ExactKind::Sum), &Realization::PassThrough)); - } - - // ── sketch_family_satisfied (the bare-kind entry point) ────────────── - - #[test] - fn sketch_family_satisfied_matches_is_satisfied_by_on_the_sketch_arm() { - // The free function is meant to be exactly the logic - // `SummaryFamilyMatcher::is_satisfied_by` applies to its `Sketch` - // arm, just without needing `SketchParams` to call it. - assert!(sketch_family_satisfied( - &SketchAlgorithm::Kll, - &SketchAlgorithm::DDSketch - )); - assert!(sketch_family_satisfied( - &SketchAlgorithm::Cms, - &SketchAlgorithm::CmsWithHeap - )); - assert!(!sketch_family_satisfied( - &SketchAlgorithm::CmsWithHeap, - &SketchAlgorithm::Cms - )); - assert!(!sketch_family_satisfied( - &SketchAlgorithm::Kll, - &SketchAlgorithm::Hll - )); - } - - // The old `sketch_family_satisfied_rejects_exact_accumulator_kinds` - // test (passing `SummaryKind::Sum` -- an exact-accumulator kind -- to - // this sketch-only function) no longer type-checks at all post - // ASAPPlanner#218's split: `SketchAlgorithm` has no exact-accumulator - // variants to construct in the first place, so the property that test - // asserted is now enforced by the type system instead of at runtime. -} diff --git a/control_plane/src/physical/post_asap/mod.rs b/control_plane/src/physical/post_asap/mod.rs deleted file mode 100644 index 469db23e4..000000000 --- a/control_plane/src/physical/post_asap/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! Physical adapters for ASAPPlanner's canonical post-ASAP IR. -//! -//! ASAPPlanner owns summary selection and the post-ASAP `SummaryNode` tree. -//! This module adds only backend-specific concerns required to compile that -//! tree into catalog-backed executable plans: deployment cost input, -//! named sharing/placement wrappers, and runtime family matching. It belongs -//! under `physical`; it is not another logical or sketch-algebra layer. - -#![allow(dead_code, unused_imports)] - -pub mod cost_model; -pub mod deployment_expr; -pub mod lower; -pub mod matcher; - -// Re-exports — `crate::physical::post_asap::*` for downstream callers. -pub use deployment_expr::{PhysicalExpr, PostAsapPlan}; -pub use lower::{bind_query_expr, bind_query_expr_with_cost_model, BindingError}; -pub use matcher::SummaryFamilyMatcher; - -#[cfg(test)] -mod tests; diff --git a/control_plane/src/physical/post_asap/tests.rs b/control_plane/src/physical/post_asap/tests.rs deleted file mode 100644 index 917df4626..000000000 --- a/control_plane/src/physical/post_asap/tests.rs +++ /dev/null @@ -1,904 +0,0 @@ -//! Integration tests for the L4 IR + L3→L4 binding. - -#![cfg(test)] - -use std::rc::Rc; -use std::time::Duration; - -use planner_types::post_asap::{ - ExactKind, ExactParams, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, - SketchQuery, SummaryExpr, SummaryFamilyType, SummaryNode, -}; -use planner_types::pre_asap::expr_ir::ColumnRef; - -use crate::physical::post_asap::cost_model::ForcedFamilyCostModel; -use crate::physical::post_asap::deployment_expr::{PhysicalExpr, PostAsapPlan}; -use crate::physical::post_asap::lower::bind_query_expr; -use crate::types::AccuracyTarget; -use planner_types::pre_asap::{AggIntent, QueryExpr, Reduction, Schema, Source}; -use planner_types::pre_asap::{Column, DataType}; - -// Query outputs are values; inspect the explicitly retained accumulator below them. -fn exact_query_state(node: &SummaryNode) -> &SummaryNode { - let SummaryExpr::ValueOperation { - child, - operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } = &node.expr - else { - panic!("query must finalize its exact state: {:?}", node.expr) - }; - assert!(node - .schema - .fields - .iter() - .all(|field| matches!(field.dtype, SummaryFamilyType::Plain(_)))); - child -} - -fn sketch_family(kind: SketchAlgorithm, params: SketchParams) -> SummaryFamilyType { - SummaryFamilyType::Sketch(SketchKind::new(kind, params), GroupingStrategy::default()) -} - -// ── Test fixtures ───────────────────────────────────────────────────────────── - -fn col(name: &str, dtype: DataType) -> Column { - Column { - name: name.into(), - dtype, - nullable: false, - table: None, - } -} - -fn ts_scan() -> QueryExpr { - let schema = Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ); - let pred = crate::test_support::label_eq_predicate("service", "api", &schema) - .expect("service column present in schema"); - QueryExpr::Scan { - source: Source::TimeSeries { - metric: "http_request_duration_seconds".into(), - }, - predicates: vec![pred], - schema, - } -} - -fn windowed_scan() -> QueryExpr { - QueryExpr::TimeRange { - range: Duration::from_secs(300), - child: Rc::new(ts_scan()), - } -} - -fn agg_quantile(q: f64, accuracy: AccuracyTarget) -> QueryExpr { - QueryExpr::Aggregate { - // No `by()` and windowed (child is `Window`) — the shape - // `quantile_over_time(...)` lowers to: per-series, not a - // cross-series reduction (see #165's `Reduction`). - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Quantile { - col: None, - q, - accuracy, - }], - output_names: Vec::new(), - having: None, - child: Rc::new(windowed_scan()), - } -} - -/// Walk a `PhysicalExpr` tree and report whether any node's `SummaryExpr` -/// wraps an archive-only `AggIntent` in an unbound `Logical(Aggregate)`. -/// Mirrors `emit::mod.rs`'s classification: `Logical`-wrapped aggregates -/// whose sole intent is `archive_only()` route the L5 emitter to the -/// cold-store tier. -fn binding_is_archive(expr: &PhysicalExpr) -> bool { - match expr { - PhysicalExpr::Committed(plan) => plan_is_archive(plan), - PhysicalExpr::RawAtEdgeSketchAtBackend { child, .. } => plan_is_archive(child), - PhysicalExpr::RawAtEdgePrometheusArchive { .. } => false, - } -} - -fn plan_is_archive(plan: &PostAsapPlan) -> bool { - match plan { - PostAsapPlan::Summary(node) => node_is_archive(node), - PostAsapPlan::LetBinding { expr, child, .. } => { - plan_is_archive(expr) || plan_is_archive(child) - } - PostAsapPlan::Ref { .. } => false, - } -} - -fn node_is_archive(node: &Rc) -> bool { - match &node.expr { - SummaryExpr::KeepPreAsap(qe) => match qe.as_ref() { - QueryExpr::Aggregate { measures: aggs, .. } => { - aggs.iter().any(crate::planner_selection::archive_only) - } - _ => false, - }, - SummaryExpr::SummaryAgg { child, .. } => node_is_archive(child), - SummaryExpr::ValueOperation { child, .. } => node_is_archive(child), - SummaryExpr::SummaryEstimate { summary_input, .. } => node_is_archive(summary_input), - SummaryExpr::SummaryMerge { children, .. } => children.iter().any(node_is_archive), - SummaryExpr::SummaryJoin { outer, inner, .. } => { - node_is_archive(outer) || node_is_archive(inner) - } - - SummaryExpr::SummarySubtract { left, right } - | SummaryExpr::RelationalJoin { left, right, .. } - | SummaryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } => node_is_archive(left) || node_is_archive(right), - SummaryExpr::SummaryDelete { summary_input, .. } => node_is_archive(summary_input), - } -} - -// ── Bind rule tests ─────────────────────────────────────────────────────────── - -#[test] -fn bind_kll_quantile_basic() { - // Force KLL to test its binding independently of family ranking. - let expr = agg_quantile(0.99, AccuracyTarget::Epsilon(0.01)); - let cost_model = - ForcedFamilyCostModel::new(AccuracyTarget::Epsilon(0.01), SketchAlgorithm::Kll); - let node = crate::planner_selection::select_query(&expr, &cost_model) - .expect("KLL should bind a Quantile{0.99, ε=0.01}"); - match &node.expr { - SummaryExpr::SummaryEstimate { - query, - summary_input, - } => { - assert!(matches!(query, SketchQuery::Quantile { q } if *q == 0.99)); - match &summary_input.expr { - SummaryExpr::SummaryAgg { family, child, .. } => { - assert_eq!( - family, - &sketch_family(SketchAlgorithm::Kll, SketchParams::Kll { k: 269 }) - ); - assert!(matches!(child.expr, SummaryExpr::KeepPreAsap(_))); - } - other => panic!("expected SummaryAgg, got {other:?}"), - } - } - other => panic!("expected SummaryEstimate, got {other:?}"), - } -} - -#[test] -fn bind_ddsketch_quantile_basic() { - // Force DDSketch to test its quantile binding. - let expr = agg_quantile(0.99, AccuracyTarget::Epsilon(0.01)); - let cost_model = - ForcedFamilyCostModel::new(AccuracyTarget::Epsilon(0.01), SketchAlgorithm::DDSketch); - let node = crate::planner_selection::select_query(&expr, &cost_model) - .expect("DDSketch should bind a Quantile{0.99, ε=0.01}"); - match &node.expr { - SummaryExpr::SummaryEstimate { - query, - summary_input, - } => { - assert!(matches!(query, SketchQuery::Quantile { q } if *q == 0.99)); - match &summary_input.expr { - SummaryExpr::SummaryAgg { family, .. } => match family { - SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &SketchAlgorithm::DDSketch - && matches!(kind.params(), SketchParams::DDSketch { .. }) => - { - let SketchParams::DDSketch { alpha } = kind.params() else { - unreachable!() - }; - assert!((alpha - 0.01).abs() < 1e-12) - } - other => panic!("expected DDSketch family, got {other:?}"), - }, - other => panic!("expected SummaryAgg, got {other:?}"), - } - } - other => panic!("expected SummaryEstimate, got {other:?}"), - } -} - -/// Cost-aware rule selection: the dispatcher should pick DDSketch over -/// KLL for an explicit ε-driven Quantile — `ControlPlaneCostModel::rank_candidates` -/// statically reorders DDSketch first (see its doc comment), matching the -/// legacy `algebra::directory::sketch_type_for_agg` default for SP-2/SP-4. -#[test] -fn bind_picks_ddsketch_over_kll_when_eps_explicit() { - let expr = agg_quantile(0.99, AccuracyTarget::Epsilon(0.01)); - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)) - .expect("bind_query_expr should not error"); - match bound { - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) => match &node.expr { - SummaryExpr::SummaryEstimate { summary_input, .. } => match &summary_input.expr { - SummaryExpr::SummaryAgg { family, .. } => { - assert!( - matches!(family, SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::DDSketch), - "dispatcher should pick DDSketch over KLL on ε-driven Quantile, got {family:?}" - ); - } - other => panic!("expected SummaryAgg, got {other:?}"), - }, - other => panic!("expected SummaryEstimate, got {other:?}"), - }, - other => panic!("expected Committed(Summary(_)), got {other:?}"), - } -} - -/// Build an `Aggregate{TopK{k, accuracy}}` over the windowed scan. -fn agg_topk(k: usize, accuracy: AccuracyTarget) -> QueryExpr { - QueryExpr::Aggregate { - // A ranking always reduces — empty `by` ranks the whole input, - // never per-entity (see `lower.rs`'s `LQueryExpr::TopK` handling). - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::TopK { k, accuracy }], - output_names: Vec::new(), - having: None, - child: Rc::new(windowed_scan()), - } -} - -/// Pull the bound `(SketchAlgorithm, w, d)` out of a top-k binding. -/// `SketchAlgorithm` promotes `with_heap` to kind identity — the top-k cost -/// model always binds `CmsWithHeap`/`CountSketchWithHeap` for a top-k -/// intent, never the bare kind, so there's no separate heap flag to -/// return anymore. -fn topk_binding_family(bound: &PhysicalExpr) -> (SketchAlgorithm, u32, u32) { - match bound { - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) => match &node.expr { - SummaryExpr::SummaryEstimate { - query, - summary_input, - } => { - assert!(matches!(query, SketchQuery::TopK { k } if *k == 10)); - match &summary_input.expr { - SummaryExpr::SummaryAgg { family, .. } => match family { - SummaryFamilyType::Sketch(kind, _) - if matches!( - kind.algorithm(), - SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap - ) => - { - match kind.params() { - SketchParams::CmsWithHeap { width, depth, .. } - | SketchParams::CountSketchWithHeap { width, depth, .. } => { - (kind.algorithm().clone(), *width, *depth) - } - other => panic!("heap algorithm has mismatched params: {other:?}"), - } - } - other => { - panic!("expected CmsWithHeap/CountSketchWithHeap family, got {other:?}") - } - }, - other => panic!("expected SummaryAgg, got {other:?}"), - } - } - other => panic!("expected SummaryEstimate, got {other:?}"), - }, - other => panic!("expected Committed(Summary(_)), got {other:?}"), - } -} - -/// A loose TopK target still needs membership evidence to select a sketch. -#[test] -fn uncertified_topk_keeps_exact_execution() { - let acc = AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.001, - }; - let expr = agg_topk(10, acc.clone()); - assert!(query_is_exact(&committed_node( - bind_query_expr(&expr, acc).unwrap() - ))); -} - -/// An exact workload target cannot accept uncertified TopK membership. -#[test] -fn exact_topk_keeps_exact_execution() { - // Intent requests a normal (non-exact) rank so binding still - // happens; the workload-level policy demands exact recall. - let expr = agg_topk(10, AccuracyTarget::Epsilon(0.01)); - assert!(query_is_exact(&committed_node( - bind_query_expr(&expr, AccuracyTarget::Exact).unwrap() - ))); -} - -/// A cheaper sketch never substitutes for missing membership evidence. -#[test] -fn cheap_topk_does_not_bypass_membership_evidence() { - use crate::physical::deployment_cost::wire::WireCostTable; - let table = WireCostTable::default(); - let cms = table.for_algorithm(&SketchAlgorithm::Cms).per_flush(); - let cs = table - .for_algorithm(&SketchAlgorithm::CountSketch) - .per_flush(); - assert!( - cms < cs, - "CMS-heap ({cms} B) must be cheaper than CountSketch ({cs} B) on the wire" - ); - // The cost gap the Fig-12 harness measured (~66×). - let ratio = cs as f64 / cms as f64; - assert!( - ratio > 50.0, - "CountSketch should be ~66× the CMS-heap wire cost; got {ratio:.1}×" - ); - - // Loose recall → the planner must land on the cost-min family (CMS). - let acc = AccuracyTarget::Epsilon(0.01); - assert!(query_is_exact(&committed_node( - bind_query_expr(&agg_topk(10, acc.clone()), acc).unwrap() - ))); -} - -#[test] -fn uncertified_hll_keeps_exact_execution() { - let expr = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Cardinality { - cols: vec![], - accuracy: AccuracyTarget::Epsilon(0.01), - }], - output_names: Vec::new(), - having: None, - child: Rc::new(windowed_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).expect("no error"); - assert!(query_is_exact(&committed_node(bound))); -} - -#[test] -fn sum_now_binds_to_exact_agg_after_pr_6_followup() { - // `AggIntent::Sum` binds to a readout over `SummaryAgg` with `summary: - // SummaryKind::Sum` and no `SummaryEstimate` wrapper (the partial - // state *is* the value — see `asap_aware_mapping::replacement`'s module docs). The - // old locally-defined `PhysicalExpr::ExactAgg { agg_type, .. }` - // variant (and `asap_types::AggregationType`) no longer exist at - // the L4 IR level: `planner_types::post_asap::SummaryExpr` unifies exact - // accumulators and approximate sketches into the same `SummaryAgg` - // node shape, keyed by `SummaryKind` (see `deployment_expr.rs`'s - // module docs). - let expr = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Sum { col: None }], - output_names: Vec::new(), - having: None, - child: Rc::new(windowed_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Exact).expect("no error"); - match bound { - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) => { - match &exact_query_state(&node).expr { - SummaryExpr::SummaryAgg { family, .. } => { - assert_eq!( - family, - &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - "Sum should bind to SummaryAgg(Sum)" - ); - } - other => panic!("expected Sum state below the query readout, got {other:?}"), - } - } - other => panic!("expected Committed(Summary(_)), got {other:?}"), - } -} - -#[test] -fn bind_exact_accuracy_disables_quantile_binding() { - // Quantile under `AccuracyTarget::Exact` should NOT bind — the - // optimizer falls back to an exact path. (Per design.md §6 line - // ~1254 — "the summary path is selected, not mandated".) - let expr = agg_quantile(0.99, AccuracyTarget::Exact); - let bound = bind_query_expr(&expr, AccuracyTarget::Exact).expect("no error"); - match bound { - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) => { - assert!( - matches!(&node.expr, SummaryExpr::KeepPreAsap(qe) if matches!(**qe, QueryExpr::Aggregate { .. })), - "Exact accuracy should disable summary binding and pass through as Logical, got {:?}", - node.expr - ); - } - other => panic!("expected Committed(Summary(_)), got {other:?}"), - } -} - -// PromQL pattern coverage through canonical parsing and physical binding. - -/// `ONLY_TEMPORAL` — `quantile_over_time(0.99, m[5m])`. -/// asap-planner-rs path: ONLY_TEMPORAL pattern 1 → KLL/DDSketch summary. -/// Control plane path: `Aggregate{Quantile{0.99}}` over `Window` → -/// binds a quantile-capable family → `SummaryAgg{KLL/DDSketch}`. -#[test] -fn phase_b_pattern_only_temporal_quantile_binds_to_sketch() { - let expr = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - output_names: Vec::new(), - having: None, - child: Rc::new(windowed_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); - match bound { - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) => match &node.expr { - SummaryExpr::SummaryEstimate { - query, - summary_input, - } => { - assert!(matches!(query, SketchQuery::Quantile { .. })); - match &summary_input.expr { - SummaryExpr::SummaryAgg { family, .. } => { - assert!(matches!( - family, - SummaryFamilyType::Sketch(kind, _) - if matches!(kind.algorithm(), SketchAlgorithm::Kll | SketchAlgorithm::DDSketch) - )); - } - other => panic!("expected SummaryAgg under SummaryEstimate, got {other:?}"), - } - } - other => panic!("expected SummaryEstimate, got {other:?}"), - }, - other => panic!("expected Committed(Summary(_)), got {other:?}"), - } -} - -/// `ONLY_TEMPORAL` — `sum_over_time(m[5m])` (and the count/avg/min/max -/// variants that legacy `single_query.rs` accepts). -/// -/// Control plane path: `Aggregate{Sum}` over `Window` → binds to a readout over -/// `SummaryAgg{summary: SummaryKind::Sum}` (an exact mergeable -/// accumulator — see `sum_now_binds_to_exact_agg_after_pr_6_followup`'s -/// doc comment for the `ExactAgg` → `SummaryAgg` unification). -#[test] -fn phase_b_pattern_only_temporal_sum_binds_to_exact_agg() { - let expr = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Sum { col: None }], - output_names: Vec::new(), - having: None, - child: Rc::new(windowed_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); - match bound { - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) => { - match &exact_query_state(&node).expr { - SummaryExpr::SummaryAgg { family, .. } => { - assert_eq!( - family, - &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - ); - } - other => panic!("expected SummaryAgg(Sum), got {other:?}"), - } - } - other => panic!("expected Committed(Summary(_)), got {other:?}"), - } -} - -/// `ONLY_SPATIAL` — `sum by (host) (m)`. -/// Control plane path: `Aggregate{Sum, by=[host]}` over a bare `Scan`. -/// -/// Family remains Sum; the reduction carries the grouping columns. -#[test] -fn phase_b_pattern_only_spatial_aggregate_binds_to_grouped_sum() { - let expr = QueryExpr::Aggregate { - reduction: Reduction::by(vec![1]), // service column - measures: vec![AggIntent::Sum { col: None }], - output_names: Vec::new(), - having: None, - child: Rc::new(ts_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); - match bound { - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) => { - match &exact_query_state(&node).expr { - SummaryExpr::SummaryAgg { - family, reduction, .. - } => { - assert_eq!( - family, - &SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - ); - assert_eq!( - reduction.group_keys().map(|k| k.keys()), - Some(&[1][..]), - "Sum reduction must retain the group-by column" - ); - } - other => panic!("expected SummaryAgg(Sum, by=[1]), got {other:?}"), - } - } - other => panic!("expected Committed(Summary(_)), got {other:?}"), - } -} - -/// `ONE_TEMPORAL_ONE_SPATIAL` — `sum by (host) (rate(m[5m]))`. -/// Planner preserves the Rate family and the `by` reduction independently. -#[test] -fn phase_b_pattern_temporal_and_spatial_combined_preserves_rate() { - let expr = QueryExpr::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Rate], - output_names: Vec::new(), - having: None, - child: Rc::new(windowed_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); - match bound { - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) => { - match &exact_query_state(&node).expr { - SummaryExpr::SummaryAgg { - family, reduction, .. - } => { - assert_eq!( - family, - &SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) - ); - assert_eq!(reduction.group_keys().map(|k| k.keys()), Some(&[1][..])); - } - other => panic!("expected SummaryAgg(Rate, by=[1]), got {other:?}"), - } - } - other => panic!("expected Committed(Summary(_)), got {other:?}"), - } -} - -/// Archive-only intents produce logical pass-through plans. -#[test] -fn phase_b_pattern_archive_only_routes_to_archive() { - let intent = AggIntent::Absent; - assert!( - crate::planner_selection::archive_only(&intent), - "Phase β intent must flag archive" - ); - let expr = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![intent.clone()], - output_names: Vec::new(), - having: None, - child: Rc::new(windowed_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); - // The archive-only rule's output is a Logical pass-through carrying - // the original Aggregate. Downstream emitters check archive_only(). - match bound { - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) => match &node.expr { - SummaryExpr::KeepPreAsap(qe) => match qe.as_ref() { - QueryExpr::Aggregate { measures: aggs, .. } => { - assert_eq!(aggs, &vec![intent]); - } - other => panic!("expected Aggregate, got {other:?}"), - }, - other => panic!("expected Logical(Aggregate(Absent)), got {other:?}"), - }, - other => panic!("expected Committed(Summary(_)), got {other:?}"), - } -} - -// Representative PromQL workloads pin the expected sketch or archive binding. - -/// Helper: parse a PromQL string to the canonical L3 `QueryExpr`, bind to -/// L4. Returns the produced `PhysicalExpr` for assertion. -/// `parse_query_expr_canonical` is the real parse path (L1 → L2 relational -/// → L3 canonical via `lower`); `bind_query_expr` is the -/// L3→L4 bottom-up walk under the supplied accuracy target. -fn pipeline_l1_to_l4(query: &str, accuracy: AccuracyTarget) -> PhysicalExpr { - let qe = crate::query_parser::parse_query_expr_canonical(query, accuracy.clone()) - .unwrap_or_else(|e| panic!("parse {query}: {e}")); - bind_query_expr(&qe, accuracy).unwrap_or_else(|e| panic!("bind {query}: {e}")) -} - -/// `quantile_over_time.yaml` — the asap-planner-rs `quantile_over_time` -/// fixture maps to a KLL or DDSketch InstalledPrecomputePlan row. The control plane -/// path: L1 PromQL parse → L3 `Aggregate{Quantile{0.99}}` over `Window` → -/// L4 bind picks Kll (default) or DDSketch. Either is functionally -/// equivalent — both are quantile sketches. -#[test] -fn phase_b_e2e_quantile_over_time_binds_to_quantile_sketch() { - let bound = pipeline_l1_to_l4( - "quantile_over_time(0.99, http_request_duration_seconds[5m])", - AccuracyTarget::Epsilon(0.01), - ); - let kind = crate::emit::extract_root_sketch_algorithm(&bound); - assert!( - matches!( - kind, - Some(SketchAlgorithm::Kll) | Some(SketchAlgorithm::DDSketch) - ), - "expected quantile summary family, got {kind:?}" - ); - assert!( - !binding_is_archive(&bound), - "ASAP-tier quantile must not flag archive" - ); -} - -/// `sum_over_time.yaml` — the legacy planner produces an exact-sum -/// aggregation row (no summary). Control plane path: `Aggregate{Sum}` over -/// `Window` → binds to an exact accumulator (`SummaryAgg{Sum}`), which is -/// neither an approximate summary (so `extract_root_sketch_algorithm`, which -/// excludes exact accumulators — see its doc comment — returns `None`) -/// nor archive-routed. -#[test] -fn phase_b_e2e_sum_over_time_falls_through_to_logical() { - let bound = pipeline_l1_to_l4( - "sum_over_time(http_requests_total[5m])", - AccuracyTarget::Epsilon(0.01), - ); - assert!( - crate::emit::extract_root_sketch_algorithm(&bound).is_none(), - "sum_over_time should not produce an approximate summary" - ); - assert!( - !binding_is_archive(&bound), - "Sum is exact-warm, not archive" - ); -} - -/// `sum_by.yaml` — `sum by (label) (sum_over_time(...))`. Spatial-and- -/// temporal aggregation; the legacy planner emits an exact-sum row keyed -/// on the by-label. Control plane path: `Aggregate{Sum, by=[…]}` over -/// `Window` → binds to an exact accumulator (`SummaryAgg{Sum, by=[…]}`) — -/// no approximate summary family. The by-label is preserved on the L3 -/// group-by-id list, which Phase α's routing emit reads to build the -/// per-label rollup partition. -#[test] -fn phase_b_e2e_sum_by_preserves_grouping_label() { - let bound = pipeline_l1_to_l4( - "sum by (instance) (sum_over_time(http_requests_total[5m]))", - AccuracyTarget::Epsilon(0.01), - ); - // No approximate summary family for plain Sum. - assert!(crate::emit::extract_root_sketch_algorithm(&bound).is_none()); - // The end shape may carry `Logical(Aggregate{by, ...})` beneath a - // `SummaryAgg{Sum}` wrapper, or `Logical(Window{...})` when the - // ParsedQuery → QueryExpr lowering drops the Aggregate (legacy - // ParsedQuery only carries `aggregations: Vec` not the - // by-axis directly). In either case the metric name + label survive - // somewhere in the L3 sub-tree — assert that. - // `PhysicalExpr` is no longer `Serialize` (see its doc) — `Debug` - // output still contains every string literal in the tree, so it - // works just as well for this substring search. - let dbg = format!("{bound:?}"); - assert!( - dbg.contains("http_requests_total"), - "metric name lost through pipeline: {dbg}" - ); - assert!( - dbg.contains("instance"), - "by-label `instance` lost through pipeline: {dbg}" - ); -} - -/// A Rate query keeps Planner's exact Rate family through binding. The -/// physical emitter chooses the runtime processor without changing that -/// family identity. -#[test] -fn phase_b_e2e_rate_falls_through_to_logical() { - let bound = pipeline_l1_to_l4( - "rate(http_requests_total[5m])", - AccuracyTarget::Epsilon(0.01), - ); - assert!(crate::emit::extract_root_sketch_algorithm(&bound).is_none()); - assert!( - !binding_is_archive(&bound), - "Rate is ASAP-tier, not archive" - ); -} - -/// Value-ranked TopK candidates with missing membership evidence remain -/// inspectable but cannot be reported as certified selections. -#[test] -fn value_ranked_topk_without_membership_evidence_is_not_certified() { - let query = "topk(10, sum by (instance) (rate(http_requests_total[5m])))"; - let accuracy = AccuracyTarget::Epsilon(0.05); - let expr = crate::query_parser::parse_query_expr_canonical(query, accuracy.clone()) - .expect("TopK parses"); - let (_, trace) = crate::planner_selection::select_workload_with_accuracy_model_and_trace( - vec![(0, std::rc::Rc::new(expr))], - accuracy.clone(), - &crate::physical::post_asap::cost_model::ControlPlaneCostModel::new(accuracy), - &asap_aware_mapping::NoAccuracyEvidence, - &asap_aware_mapping::DefaultAccuracyModel, - ) - .unwrap(); - assert!(trace["groups"] - .as_array() - .unwrap() - .iter() - .flat_map(|group| group["candidates"].as_array().unwrap()) - .any( - |candidate| candidate["accuracy_status"] == "unknown" && candidate["selected"] == false - )); -} - -/// Archive-only routing through the full L1→L3→L4 pipeline. Asserts the -/// expected functional equivalent of asap-planner-rs's previous -/// `is_supported() == false` behavior (refused outright); the control plane -/// now lifts these to L3 with `archive_only() == true` and the binder -/// emits a Logical pass-through. -/// -/// Replaces the prior `phase_b_e2e_histogram_quantile_e2e_through_parser` -/// — `histogram_quantile(...)` is now a PromQL/MetricsQL language-level -/// operator that the parser substitutes (Step γ5) into a plain -/// `Aggregate { Quantile(φ) }`, NOT an L3 intent of its own. The -/// archive-only routing this test exercises uses `Absent` as a stable -/// proxy (every archive-only variant follows the same code path). -#[test] -fn phase_b_e2e_archive_only_e2e_binding() { - let intent = AggIntent::Absent; - let expr = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![intent.clone()], - output_names: Vec::new(), - having: None, - child: Rc::new(windowed_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); - assert!( - binding_is_archive(&bound), - "archive-only intent must surface archive flag through L4 binding" - ); - // No approximate summary fires for archive-only intents. - assert!(crate::emit::extract_root_sketch_algorithm(&bound).is_none()); -} - -/// Archive-only intents must produce valid logical expressions without errors. -#[test] -fn phase_b_archive_only_intents_round_trip_through_binder() { - let intents = vec![ - AggIntent::Absent, - AggIntent::AbsentOverTime, - AggIntent::PresentOverTime, - AggIntent::Delta, - AggIntent::Deriv, - AggIntent::PredictLinear { seconds: 60.0 }, - AggIntent::DoubleExpSmoothing { - smoothing: 0.3, - trend: 0.3, - }, - AggIntent::IDelta, - AggIntent::Resets, - AggIntent::Changes, - // Spot-check a couple of the Phase 1 IR merge's new archive-only - // intents through the same round-trip. - AggIntent::HistogramCount, - AggIntent::Group, - ]; - for intent in intents { - let expr = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![intent.clone()], - output_names: Vec::new(), - having: None, - child: Rc::new(windowed_scan()), - }; - let bound = - bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).expect("bind should succeed"); - match bound { - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) => match &node.expr { - SummaryExpr::KeepPreAsap(qe) => match qe.as_ref() { - QueryExpr::Aggregate { measures: aggs, .. } => { - assert_eq!(aggs.len(), 1); - assert!( - crate::planner_selection::archive_only(&aggs[0]), - "{intent:?} should preserve archive_only() flag through bind" - ); - } - other => panic!("expected Aggregate({intent:?}), got {other:?}"), - }, - other => panic!("expected Logical(Aggregate({intent:?})), got {other:?}"), - }, - other => panic!("expected Committed(Summary(_)) for {intent:?}, got {other:?}"), - } - } -} - -// ── Deliberate behavior changes (ASAPController#150 / #151) ────────────────── -// -// `AggIntent::Extension` (this deployment's `Frequency` point-query, -// built via `crate::planner_selection::frequency(accuracy, item)`) now binds to a -// real `Cms` summary via `ControlPlaneCostModel::realize_extension`/ -// `readout_extension` (ASAPController#150) — see `frequency_extension_binds_cms` -// below and `physical::workload_planner::mod::tests::typed_binding_endpoint_request_freq_binds_cms`. -// `AggIntent::TopK { accuracy: Exact }` still declines to bind -// (`SummaryExpr::KeepPreAsap`) rather than summary — a REAL, accepted -// behavior change from this migration that remains open -// (`TopK{Exact}`'s `exact_realization` has no accumulator form for it — -// see `lower.rs`'s module docs and `cost_model.rs`'s module docs, -// ASAPController#151, still open). - -#[test] -fn frequency_extension_binds_cms() { - // `ControlPlaneCostModel::realize_extension`/`readout_extension` - // (ASAPController#150) now realize `AggIntent::Extension{"frequency"}` - // as a real `Cms` summary instead of declining to `Logical`. - let intent = crate::planner_selection::frequency(AccuracyTarget::Epsilon(0.01), None); - let expr = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![intent], - output_names: Vec::new(), - having: None, - child: Rc::new(windowed_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).expect("no error"); - match bound { - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) => { - let SummaryExpr::SummaryEstimate { - summary_input, - query, - } = &node.expr - else { - panic!("expected SummaryEstimate, got {:?}", node.expr); - }; - assert!( - matches!( - &summary_input.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::Sketch(kind, _), - .. - } if kind.algorithm() == &SketchAlgorithm::Cms - ), - "expected a Cms SummaryAgg, got {:?}", - summary_input.expr - ); - assert!( - matches!( - query, - SketchQuery::PointCount { - key: ColumnRef::SampleValue, - value: None - } - ), - "no filter value threaded through yet (ASAPQuery-backend Phase 3) -- \ - should read out as the bare bucket total, got {query:?}", - ); - } - other => panic!("expected Committed(Summary(_)), got {other:?}"), - } -} - -#[test] -fn topk_exact_accuracy_declines_to_bind() { - let expr = agg_topk(10, AccuracyTarget::Exact); - let bound = bind_query_expr(&expr, AccuracyTarget::Exact).expect("no error"); - match bound { - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) => { - assert!( - matches!(&node.expr, SummaryExpr::KeepPreAsap(_)), - "TopK{{accuracy: Exact}} should decline pending ASAPController#151, got {:?}", - node.expr - ); - } - other => panic!("expected Committed(Summary(_)), got {other:?}"), - } -} - -fn committed_node(bound: PhysicalExpr) -> Rc { - let PhysicalExpr::Committed(PostAsapPlan::Summary(node)) = bound else { - panic!("expected committed query") - }; - node -} - -fn query_is_exact(node: &SummaryNode) -> bool { - node.guarantee - .as_ref() - .is_some_and(|guarantee| guarantee.is_exact()) -} diff --git a/control_plane/src/physical/publication.rs b/control_plane/src/physical/publication.rs deleted file mode 100644 index 6bd088f35..000000000 --- a/control_plane/src/physical/publication.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! Control-plane construction of the shared catalog publication contract. -use super::compiler::CompiledPhysicalPlan; -pub use asap_types::plan_publication::{PhysicalPlanInstallRequest, PhysicalPlanPublication}; - -impl CompiledPhysicalPlan { - pub fn to_publication_artifact(&self) -> Result { - let artifact = PhysicalPlanPublication { - summary_catalog: self.summary_catalog.clone(), - precompute_plan: self.precompute_plan.clone(), - collector_plans: self.collector_plans.clone(), - transmission_plan: self.transmission_plan.clone(), - query_plan: self.query_plan.clone(), - }; - artifact.validate()?; - Ok(artifact) - } -} diff --git a/control_plane/src/physical/realization.rs b/control_plane/src/physical/realization.rs deleted file mode 100644 index 0f16ebf30..000000000 --- a/control_plane/src/physical/realization.rs +++ /dev/null @@ -1,67 +0,0 @@ -//! Internal realization boundary over the existing compiler and cost contracts. -//! -//! Providers may validate and price physical implementations, never rewrite -//! selected logical roots or infer a pane width from a query's slide. -use super::compiler::{ - CompileError, CompiledPhysicalPlan, DeploymentPlanCompiler, PhysicalCompilationRequest, - PhysicalDeploymentContext, QueryCompilationInput, -}; -use super::workload_cost::{PricedComponents, WorkloadCostEvidence, WorkloadCostManifest}; -use asap_aware_mapping::cost_model::Cost; -use planner_types::post_asap::SummaryWindowFramework; - -pub(crate) trait RealizationProvider { - fn windows( - &self, - query: &QueryCompilationInput, - environment: &PhysicalDeploymentContext, - ) -> Result, CompileError>; - - fn compile( - &self, - request: PhysicalCompilationRequest, - environment: PhysicalDeploymentContext, - frontend: super::compiler::QueryFrontend, - ) -> Result; - - fn price( - &self, - evidence: &WorkloadCostEvidence, - manifest: &WorkloadCostManifest, - ) -> Result; -} - -/// Only the currently implemented deployment paths. Capability validation -/// remains in the compiler; a quote cannot authorize an unsupported runtime. -pub(crate) struct ExistingRealizations; - -impl RealizationProvider for ExistingRealizations { - fn windows( - &self, - query: &QueryCompilationInput, - environment: &PhysicalDeploymentContext, - ) -> Result, CompileError> { - super::compiler::validate_window_implementations(query, environment) - } - - fn compile( - &self, - request: PhysicalCompilationRequest, - environment: PhysicalDeploymentContext, - frontend: super::compiler::QueryFrontend, - ) -> Result { - if frontend == super::compiler::QueryFrontend::MetricsQl { - DeploymentPlanCompiler.compile_metricsql(request, environment) - } else { - DeploymentPlanCompiler.compile_promql(request, environment) - } - } - - fn price( - &self, - evidence: &WorkloadCostEvidence, - manifest: &WorkloadCostManifest, - ) -> Result { - evidence.price(manifest) - } -} diff --git a/control_plane/src/physical/runtime_capability.rs b/control_plane/src/physical/runtime_capability.rs deleted file mode 100644 index 8dc48e18d..000000000 --- a/control_plane/src/physical/runtime_capability.rs +++ /dev/null @@ -1,1019 +0,0 @@ -//! Runtime SID capability tag and query-to-deployed-state routing adapter. -//! -//! Step 2a of the architectural refactor originally consolidated four -//! overlapping capability tables into this module. The performance / -//! cost-model half of that consolidation — [`SketchCapability`] / -//! `default_capability_table` / `load_capability_overrides` -//! — moved to `crate::physical::deployment_cost::sketch_capability` (Stage 4 of the -//! `physical::post_asap` re-layering): it's a cost-model concern read by the -//! optimizer and physical planner, not L4 IR. What's left here: -//! -//! - [`Capability`] / [`SketchAlgorithm`] — query-side capability tag, -//! used by the ASAP-tier reducer in `asap-query-engine` to dispatch -//! PromQL → per-Capability sketch evaluation. -//! - [`capability_for`] — the **semantic** intent → ASAP-tier dispatch -//! bridge. PromQL → intent_algebra::lower → `AggIntent` → (this fn) → -//! `Capability`. The ASAP-tier analyzer is now a thin facade around -//! this single function; PromQL function-name string matching lives -//! only inside the lowerer. - -#![allow(dead_code)] - -use crate::physical::post_asap::matcher::sketch_family_satisfied; -use crate::types::AccuracyTarget; -use asap_types::AggregationType; -pub use planner_types::post_asap::SketchAlgorithm; -#[cfg(test)] -use planner_types::pre_asap::AggIntent; - -// ── Query-side capability tag ──────────────────────────────────────────────── - -/// Warm-tier capability tag. One variant per logical query family the -/// ASAP tier can answer. The inner [`SketchAlgorithm`] is the -/// implementation choice (e.g. DDSketch vs KLL for `QuantileApprox`). -/// Query routing keys on the variant, not the implementation, so two -/// CMS instances and one CountSketch instance for the same metric-and- -/// group-by all map to `FrequencyTopk` and the query path picks any -/// of them. -/// -/// Used by both the control plane (via [`capability_for`] in the ASAP-tier -/// analyzer) and the `asap-query-engine` backend (re-exported as the -/// `sketch_index::Capability` it indexes sketch instances under). One -/// canonical definition; the backend re-exports rather than duplicating. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum Capability { - /// Approximate quantile via DDSketch / KLL / t-digest. The handle's - /// `Any` variant means "any quantile-family sketch satisfies"; a - /// concrete handle means "must be exactly this family". - QuantileApprox(Option), - /// Approximate cardinality via HLL / theta-sketch / linear-counting. - /// No inner handle — cardinality has a single canonical family - /// today (HLL). - CardinalityApprox, - /// Bare per-item frequency estimate (CMS / CountSketch point query, - /// no top-k extraction). Heap-LESS — answers `sum by (item) (rate(m[r]))` - /// with epsilon accuracy. Distinct from [`Capability::FrequencyTopk`]: - /// any heap-bearing variant ALSO satisfies bare frequency (the heap is - /// additional info layered on top of the sketch matrix), so - /// `is_satisfied_by` allows {CountMin, CountSketch, CmsWithHeap, - /// CountSketchWithHeap} on the available side. - FrequencyEstimate(Option), - /// Heavy-hitter top-k via CMS-with-heap or CountSketch-with-heap. - /// Heap-BEARING — only handles that carry an item universe in their - /// wire format can answer this. `Any` required matches either - /// `CmsWithHeap` or `CountSketchWithHeap`. - FrequencyTopk(Option), - /// Exact-aggregation ASAP-tier state — Sum / Count / Min / Max / Avg / - /// Rate / Increase / SetAggregator etc. Backed by a per-accumulator - /// payload (`AggPayload::ExactAgg` in the data plane). One variant - /// per [`AggregationType`] — the inner enum names the concrete - /// accumulator family. - /// - /// Distinct from the `*Approx` variants above: the `*Approx` - /// capabilities serve approximate sketch-bound intents; `ExactAgg` - /// serves the ASAP-tier exact-aggregation path (the data plane's - /// `AggKind::ExactAgg`-backed sids). Routing an analyzer candidate - /// at `Capability::ExactAgg(Sum)` to a sid whose `agg_kind` is - /// `AggKind::ExactAgg { agg_type: Sum, .. }` is what closes the gap - /// between the control plane's vocabulary and the data plane's - /// exact-aggregation state. - /// - /// PR 6 introduces this variant + the matching machinery. The - /// `capability_for(&AggIntent)` lookup deliberately does NOT route - /// `Sum` / `Min` / `Max` / `Rate` / `Increase` / exact-accuracy - /// intents to `ExactAgg` yet — that re-routing is a behavior - /// change deferred to a follow-up. The variant is dormant on the - /// analyzer side until then; the matching half (`is_satisfied_by`) - /// is wired so that sids whose stored `Capability` is - /// `ExactAgg(...)` can be filtered against an `ExactAgg(...)` - /// required capability once callers start populating it. - ExactAgg(AggregationType), -} - -/// PromQL counter-function shape carried to execution without reparsing. -/// The inner counter function wins for composed expressions. -/// -/// Counter state contains per-window deltas: -/// -/// * Plain sum accumulates stored deltas. -/// * Rate sums deltas over the range and divides by covered seconds. -/// * Increase sums deltas over the range. -/// * Sum-over-time requires archive execution because cumulative samples -/// cannot be reconstructed from delta state alone. -/// -/// Rate and Increase have distinct exact-family capabilities. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum OuterFn { - /// No range-style counter function in the expression — bare selector, - /// `sum(metric)`, `sum by (...) (metric)`. PromQL semantics for an - /// instant `sum` over a counter is "current cumulative counter value, - /// summed per group". Over per-window deltas the engine accumulates - /// EVERY window in storage up to `now` into one cumulative number per - /// group. This is the default — `Default::default()` returns `Plain` - /// so candidates built without an explicit outer-fn (test fixtures, - /// fallback paths) get the safe accumulate-all dispatch. - #[default] - Plain, - /// `rate(metric[r])` or `irate(metric[r])` appears in the expression - /// (possibly nested inside an outer `sum by (...) (...)`). The engine - /// dispatches to `evaluate_exact_agg_rate`, which sums the deltas in - /// `[t-r, t]` and divides by `min(r, actual_coverage_seconds)` to - /// produce events-per-second. - Rate, - /// `increase(metric[r])` appears in the expression. PromQL semantics: - /// `counter(t) − counter(t−r)`. Over per-window deltas that is exactly - /// the sum of deltas in `[t-r, t]`. The engine dispatches to the - /// accumulate-across-windows path scoped to the `[t-r, t]` clip, - /// yielding ONE cumulative number per group (no `÷ r`). - Increase, - /// `sum_over_time(metric[r])` appears in the expression. PromQL - /// semantics: Σ of the (cumulative) SAMPLE values in `[r]` — a - /// quadratic over the storage horizon that asap CANNOT reconstruct - /// from stored deltas. The engine returns a capability-miss so the - /// query routes to the archive tier (which has raw samples) rather - /// than fabricating a wrong number. See issue #301 decision (a). - SumOverTime, -} - -/// PromQL outer-aggregation operator carried on each `ASAPTierCandidate` -/// for the shape ` by (labels) ()` where `` is a -/// function the analyzer ALREADY routes to a per-row ASAP-tier -/// candidate (e.g. `quantile_over_time`, `sum_over_time`, `rate`). -/// -/// Background: the analyzer's lowerer captures the INNER intent -/// (`AggIntent::Quantile`, `AggIntent::Sum`, etc.) — that's what -/// `capability_for` maps to a `Capability`. The OUTER aggregation -/// operator (`max`, `min`, `avg`, `count`, etc.) wrapping the inner -/// function is dropped on the floor: the lowerer either folds it into a -/// dedicated `AggIntent` (`Sum` → `ExactAgg(Sum)`, handled separately) -/// or returns no extra intent for the wrapper (`max`, `min`, `avg`, -/// `count` — which are scalar folds over the inner's per-row result). -/// -/// `OuterAgg` carries that wrapper so the engine's evaluator can -/// fold per-row results into one row per `by`-group AFTER the inner -/// function returns its rows. The taxonomy intentionally splits the -/// fold operators that ARE composable on top of a per-row inner result -/// — the inner sketch / accumulator computes per-row values, and the -/// outer aggregation reduces across rows in each `by`-group. -/// -/// Identity case: when the inner result already has exactly one row -/// per `by`-group (e.g. asap's per-zone DDSketch quantile), the fold -/// is the identity — `max(x) = min(x) = avg(x) = x`. The general fold -/// machinery handles this naturally without a special case. -/// -/// `None` is the default — `Default::default()` returns `None` so -/// candidates built without an explicit outer aggregation (test -/// fixtures, plain inner-only queries) keep the prior behavior. -/// -/// Out of scope (separate follow-up): -/// - PromQL `quantile(phi, vec)` (instant) over function results — -/// needs per-group sketch merging, not a scalar fold. -/// - `sum` is NOT included here: `sum by (...) (...)` already routes -/// through `Capability::ExactAgg(Sum)` via the analyzer's lowerer + -/// the `Sum` intent collapse; adding it here would double-dispatch. -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub enum OuterAgg { - /// No outer aggregation operator wraps the inner function — the - /// engine emits the inner result directly. This is the default. - #[default] - None, - /// `max by (labels) ()` — fold each `by`-group's values by - /// taking the maximum. - Max(Vec), - /// `min by (labels) ()` — fold each `by`-group's values by - /// taking the minimum. - Min(Vec), - /// `avg by (labels) ()` — fold each `by`-group's values by - /// taking the arithmetic mean. - Avg(Vec), - /// `count by (labels) ()` — fold each `by`-group's values - /// by counting the contributing rows (cardinality of the group). - Count(Vec), - /// `group by (labels) ()` — PromQL `group` operator returns - /// 1.0 per `by`-group (label preservation, value-erasing fold). - Group(Vec), - /// `stddev by (labels) ()` — fold each `by`-group's values - /// by taking the population standard deviation. - Stddev(Vec), - /// `stdvar by (labels) ()` — fold each `by`-group's values - /// by taking the population variance. - Stdvar(Vec), -} - -impl OuterAgg { - /// True when an outer aggregation operator is set. False for the - /// `None` default. Used by the engine's evaluator to skip the - /// fold pass when no outer aggregation applies. - pub fn is_some(&self) -> bool { - !matches!(self, OuterAgg::None) - } - - /// The `by`-labels carried by every operator variant. `None` returns - /// an empty slice. Caller projects each result row's label map onto - /// these keys to form the group identity. - pub fn by_labels(&self) -> &[String] { - match self { - OuterAgg::None => &[], - OuterAgg::Max(l) - | OuterAgg::Min(l) - | OuterAgg::Avg(l) - | OuterAgg::Count(l) - | OuterAgg::Group(l) - | OuterAgg::Stddev(l) - | OuterAgg::Stdvar(l) => l.as_slice(), - } - } - - /// Fold a slice of f64 values into a single scalar per the operator. - /// Returns `None` only for an empty input slice (caller drops empty - /// groups). All operators are defined on at least one value. - pub fn fold(&self, values: &[f64]) -> Option { - if values.is_empty() { - return None; - } - Some(match self { - // Identity / no-op — engine shouldn't call this when None, - // but defensively return the first value. - OuterAgg::None => values[0], - OuterAgg::Max(_) => values.iter().copied().fold(f64::NEG_INFINITY, f64::max), - OuterAgg::Min(_) => values.iter().copied().fold(f64::INFINITY, f64::min), - OuterAgg::Avg(_) => { - let sum: f64 = values.iter().sum(); - sum / values.len() as f64 - } - OuterAgg::Count(_) => values.len() as f64, - OuterAgg::Group(_) => 1.0, - OuterAgg::Stddev(_) => { - let n = values.len() as f64; - let mean: f64 = values.iter().sum::() / n; - let var: f64 = values.iter().map(|v| (v - mean).powi(2)).sum::() / n; - var.sqrt() - } - OuterAgg::Stdvar(_) => { - let n = values.len() as f64; - let mean: f64 = values.iter().sum::() / n; - values.iter().map(|v| (v - mean).powi(2)).sum::() / n - } - }) - } -} - -impl Capability { - /// True when an indexed sketch instance's capability satisfies the - /// query's required capability. `None` on the query side means any - /// algorithm in that capability family. - /// - /// `self` is the **required** capability (from the analyzer); - /// `indexed` is the **available** capability (from the sketch - /// index). The backend's ASAP-tier hook reads both and routes the - /// query to whichever sids satisfy. - /// - /// The four sketch-family variants (`QuantileApprox`, - /// `CardinalityApprox`, `FrequencyEstimate`, `FrequencyTopk`) - /// delegate their family-compatibility logic to - /// [`sketch_family_satisfied`] (`enum-unification-plan.md` - /// §5/§8 Step 4) — via [`resolve_handle`], which picks a concrete - /// per-family stand-in for the `Any` wildcard since `SketchAlgorithm` - /// has no wildcard concept of its own; family-matching subsumes it. - /// Exact families require identity; keyed layout is checked separately. - pub fn is_satisfied_by(&self, indexed: &Capability) -> bool { - match (self, indexed) { - (Capability::QuantileApprox(req), Capability::QuantileApprox(have)) => { - sketch_algorithms_compatible(req, SketchAlgorithm::Kll, have, SketchAlgorithm::Kll) - } - // Cardinality has no inner handle; family match is total. - (Capability::CardinalityApprox, Capability::CardinalityApprox) => true, - // Top-k: the `Any` stand-in on EITHER side must be the - // heap-bearing `CmsWithHeap`, not bare `Cms` — a bare - // stand-in would let a heap-less available sketch wrongly - // satisfy a top-k requirement (see `resolve_handle`'s doc). - (Capability::FrequencyTopk(req), Capability::FrequencyTopk(have)) => { - sketch_algorithms_compatible( - req, - SketchAlgorithm::CmsWithHeap, - have, - SketchAlgorithm::CmsWithHeap, - ) - } - // Bare frequency: any frequency-family handle works on the - // available side — heap-LESS (CountMin / CountSketch) AND - // heap-bearing (CmsWithHeap / CountSketchWithHeap) all answer - // a point-frequency query (heap is additional info layered on - // the sketch matrix). A heap-bearing `FrequencyTopk` indexed - // capability ALSO satisfies a bare-frequency required capability. - (Capability::FrequencyEstimate(req), Capability::FrequencyEstimate(have)) => { - sketch_algorithms_compatible(req, SketchAlgorithm::Cms, have, SketchAlgorithm::Cms) - } - (Capability::FrequencyEstimate(req), Capability::FrequencyTopk(have)) => { - sketch_algorithms_compatible( - req, - SketchAlgorithm::Cms, - have, - SketchAlgorithm::CmsWithHeap, - ) - } - // Exact family identity must match. Grouping compatibility is - // checked separately by population routing. - (Capability::ExactAgg(req), Capability::ExactAgg(have)) => req == have, - _ => false, - } - } -} - -/// Resolve optional query-side algorithm constraints and delegate family -/// compatibility to the ASAPPlanner algorithm taxonomy. `None` means any -/// algorithm in the capability's category; it is not a fake algorithm. -fn sketch_algorithms_compatible( - required: &Option, - required_any_stand_in: SketchAlgorithm, - available: &Option, - available_any_stand_in: SketchAlgorithm, -) -> bool { - let required = required.as_ref().unwrap_or(&required_any_stand_in); - let available = available.as_ref().unwrap_or(&available_any_stand_in); - sketch_family_satisfied(required, available) -} - -// ── AggIntent → Capability bridge ──────────────────────────────────────────── - -#[cfg(test)] -/// Map a semantic [`AggIntent`] to the ASAP-tier [`Capability`] that can -/// answer it. Returns `None` when no deployed ASAP-tier capability can -/// satisfy the intent. -/// -/// This is a runtime routing requirement, not a summary-selection rule. -/// ASAPPlanner owns legal implementations and candidate enumeration; this -/// adapter describes which already-deployed SID the data plane may read. -pub fn capability_for(intent: &AggIntent) -> Option { - if let Some(accuracy) = crate::planner_selection::as_frequency(intent) { - return if is_exact(&accuracy) { - // Exact aggregation — sketch fallback is only meaningful - // when raw counters aren't kept at the ingest tier; with - // accuracy=Exact the caller wants exact `sum by (label) - // (rate(...))`, which routes to archive. - None - } else { - // Bare frequency point-query uses a frequency-family - // sketch — any of CMS / CountSketch / CmsWithHeap / - // CountSketchWithHeap works (the heap is additional - // info that the FrequencyTopk path uses). `Any` here - // means the optimizer picks the cheapest indexed sid. - Some(Capability::FrequencyEstimate(None)) - }; - } - match intent { - AggIntent::Sum { .. } => Some(Capability::ExactAgg(AggregationType::Sum)), - AggIntent::Count { accuracy } if is_exact(accuracy) => { - Some(Capability::ExactAgg(AggregationType::Count)) - } - // Direction is part of the capability: a stored minimum cannot - // answer `max_over_time` and vice versa, so these must not - // collapse onto one `ExactAgg` the way they did while Planner - // had a single `MinMax` accumulator. - AggIntent::Min { .. } => Some(Capability::ExactAgg(AggregationType::Min)), - AggIntent::Max { .. } => Some(Capability::ExactAgg(AggregationType::Max)), - AggIntent::Increase => Some(Capability::ExactAgg(AggregationType::Increase)), - AggIntent::Rate => Some(Capability::ExactAgg(AggregationType::Rate)), - AggIntent::Quantile { accuracy, .. } if !is_exact(accuracy) => { - Some(Capability::QuantileApprox(None)) - } - AggIntent::Cardinality { accuracy, .. } if !is_exact(accuracy) => { - Some(Capability::CardinalityApprox) - } - AggIntent::TopK { accuracy, .. } if !is_exact(accuracy) => { - Some(Capability::FrequencyTopk(None)) - } - AggIntent::Count { accuracy } if !is_exact(accuracy) => { - Some(Capability::FrequencyEstimate(None)) - } - _ => None, - } -} - -/// True iff the accuracy target forbids approximation. Wraps the match -/// so the call sites read as `if is_exact(accuracy) { ... }`. -fn is_exact(accuracy: &AccuracyTarget) -> bool { - matches!(accuracy, AccuracyTarget::Exact) -} - -// ── Tests ──────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - // ── capability_for: AggIntent → Capability bridge ──────────────────── - - #[test] - fn capability_for_quantile_returns_quantile_approx() { - let intent = AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }; - assert_eq!( - capability_for(&intent), - Some(Capability::QuantileApprox(None)) - ); - } - - #[test] - fn capability_for_quantile_exact_returns_none() { - let intent = AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Exact, - }; - // Exact quantiles must be answered by HashAgg/SortAgg — no - // ASAP-tier sketch in this case. - assert_eq!(capability_for(&intent), None); - } - - #[test] - fn capability_for_cardinality_with_epsilon_returns_cardinality_approx() { - let intent = AggIntent::Cardinality { - cols: vec![], - accuracy: AccuracyTarget::Epsilon(0.01), - }; - assert_eq!(capability_for(&intent), Some(Capability::CardinalityApprox)); - } - - #[test] - fn capability_for_cardinality_with_epsilon_delta_returns_cardinality_approx() { - let intent = AggIntent::Cardinality { - cols: vec![], - accuracy: AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.001, - }, - }; - assert_eq!(capability_for(&intent), Some(Capability::CardinalityApprox)); - } - - #[test] - fn capability_for_cardinality_with_exact_returns_none() { - let intent = AggIntent::Cardinality { - cols: vec![], - accuracy: AccuracyTarget::Exact, - }; - assert_eq!(capability_for(&intent), None); - } - - #[test] - fn capability_for_count_approximate_returns_frequency_estimate() { - // Non-exact `Count` is a relaxed-accuracy `COUNT(*)` point query - // (CMS), matching ASAPController's own `bind.rs::readout` - // (`Count => SketchQuery::PointCount`) and this repo's - // `BindCmsOnCount` rule — not `CardinalityApprox`/HLL, which is - // `AggIntent::Cardinality`'s job (`distinct_over_time`/ - // `COUNT(DISTINCT)` never lower to `Count`). - let intent = AggIntent::Count { - accuracy: AccuracyTarget::Epsilon(0.01), - }; - assert_eq!( - capability_for(&intent), - Some(Capability::FrequencyEstimate(None)) - ); - } - - /// The two intents must never collapse into one capability: an approximate - /// `Count` is a `COUNT(*)` point query (frequency sketch), while - /// `Cardinality` is `COUNT(DISTINCT ...)` (HLL). Routing `Count` to - /// `CardinalityApprox` makes a by-less `count(v)` merge HLL registers and - /// answer the union's distinct count instead of the number of series -- - /// see `query_parser::tests::bare_count_is_series_count_not_distinct_cardinality` - /// for the lowering half of this invariant. - #[test] - fn count_and_cardinality_bind_different_sketch_capabilities() { - let approximate = AccuracyTarget::Epsilon(0.01); - let count = capability_for(&AggIntent::Count { - accuracy: approximate.clone(), - }); - let cardinality = capability_for(&AggIntent::Cardinality { - cols: vec![], - accuracy: approximate, - }); - assert_eq!(count, Some(Capability::FrequencyEstimate(None))); - assert_eq!(cardinality, Some(Capability::CardinalityApprox)); - assert_ne!( - count, cardinality, - "count() must not bind the HLL capability that answers COUNT(DISTINCT)" - ); - } - - #[test] - fn capability_for_count_exact_preserves_count_family() { - let intent = AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }; - assert_eq!( - capability_for(&intent), - Some(Capability::ExactAgg(AggregationType::Count)) - ); - } - - #[test] - fn capability_for_sum_routes_to_exact_agg_sum() { - // PR-6 follow-up: Sum routes to ASAP-tier ExactAgg(Sum) state. - // Pre-follow-up this returned `None`. - assert_eq!( - capability_for(&AggIntent::Sum { col: None }), - Some(Capability::ExactAgg(AggregationType::Sum)) - ); - } - - #[test] - fn capability_for_avg_returns_none() { - // Avg is exact at L3 — no ASAP-tier sketch substitutes for it - // today (a sketch-bound `Avg` would fold onto `Quantile{q=0.5}` - // only when the cost model allows the relaxation, which is a - // follow-up). - assert_eq!(capability_for(&AggIntent::Avg { col: None }), None); - } - - #[test] - fn capability_for_min_returns_exact_agg_min() { - // Min/Max are exact, mergeable accumulators -- no approximation - // needed at all -- matching ASAPController's own - // `crates/plan/src/boundary.rs` treatment. - assert_eq!( - capability_for(&AggIntent::Min { col: None }), - Some(Capability::ExactAgg(AggregationType::Min)) - ); - } - - #[test] - fn capability_for_max_returns_exact_agg_max() { - assert_eq!( - capability_for(&AggIntent::Max { col: None }), - Some(Capability::ExactAgg(AggregationType::Max)) - ); - } - - #[test] - fn exact_agg_min_and_max_do_not_satisfy_each_other() { - // The whole point of splitting the family: a deployed minimum - // sid must never be routed a `max_over_time` read. - assert!(!Capability::ExactAgg(AggregationType::Min) - .is_satisfied_by(&Capability::ExactAgg(AggregationType::Max))); - assert!(!Capability::ExactAgg(AggregationType::Max) - .is_satisfied_by(&Capability::ExactAgg(AggregationType::Min))); - assert!(!Capability::ExactAgg(AggregationType::Min) - .is_satisfied_by(&Capability::ExactAgg(AggregationType::Max))); - } - - #[test] - fn capability_for_rate_and_increase_preserves_family() { - let exact_inc = Some(Capability::ExactAgg(AggregationType::Increase)); - assert_eq!( - capability_for(&AggIntent::Rate), - Some(Capability::ExactAgg(AggregationType::Rate)) - ); - assert_eq!(capability_for(&AggIntent::Increase), exact_inc); - } - - #[test] - fn capability_for_topk_returns_frequency_topk_any() { - // The analyzer no longer pins a concrete heap-bearing variant - // for top-k — `Any` lets `handles_compatible_for_topk` accept - // either `CmsWithHeap` or `CountSketchWithHeap` registered at - // ingest time. Both variants share the heap envelope and the - // reducer dispatches them identically. - let intent = AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::Epsilon(0.05), - }; - assert_eq!( - capability_for(&intent), - Some(Capability::FrequencyTopk(None)) - ); - } - - #[test] - fn capability_for_topk_exact_returns_none() { - // Exact top-k must use HashAgg+Heap; no ASAP-tier sketch. - let intent = AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::Exact, - }; - assert_eq!(capability_for(&intent), None); - } - - #[test] - fn frequency_estimate_with_epsilon_returns_frequency_estimate_approx() { - let intent = crate::planner_selection::frequency(AccuracyTarget::Epsilon(0.01), None); - assert_eq!( - capability_for(&intent), - Some(Capability::FrequencyEstimate(None)) - ); - } - - #[test] - fn frequency_estimate_with_epsilon_delta_returns_frequency_estimate_approx() { - let intent = crate::planner_selection::frequency( - AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.001, - }, - None, - ); - assert_eq!( - capability_for(&intent), - Some(Capability::FrequencyEstimate(None)) - ); - } - - #[test] - fn frequency_estimate_with_exact_returns_none() { - // Exact aggregation routes to archive (sketch fallback only - // meaningful when raw counters aren't kept). - let intent = crate::planner_selection::frequency(AccuracyTarget::Exact, None); - assert_eq!(capability_for(&intent), None); - } - - #[test] - fn capability_for_archive_only_intents_return_none() { - // Spot-check each archive-only variant, including a few added by - // the Phase 1 IR merge. `Irate` is intentionally absent — folded - // into `Rate` (see `agg_intent.rs` module docs); `rate`/`irate` - // now share `capability_for`'s `Rate` arm. - assert_eq!(capability_for(&AggIntent::Absent), None); - assert_eq!(capability_for(&AggIntent::AbsentOverTime), None); - assert_eq!(capability_for(&AggIntent::PresentOverTime), None); - assert_eq!(capability_for(&AggIntent::Delta), None); - assert_eq!(capability_for(&AggIntent::IDelta), None); - assert_eq!(capability_for(&AggIntent::HistogramCount), None); - assert_eq!(capability_for(&AggIntent::Group), None); - } - - // ── Capability::is_satisfied_by ────────────────────────────────────── - - #[test] - fn is_satisfied_by_any_wildcard_matches_concrete() { - let required = Capability::QuantileApprox(None); - let indexed_dd = Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)); - let indexed_kll = Capability::QuantileApprox(Some(SketchAlgorithm::Kll)); - assert!(required.is_satisfied_by(&indexed_dd)); - assert!(required.is_satisfied_by(&indexed_kll)); - } - - #[test] - fn is_satisfied_by_concrete_kind_matches_same_family() { - // Quantile capability matching accepts either KLL or DDSketch, including - // when the required handle names a concrete family. - let required = Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)); - let indexed_dd = Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)); - let indexed_kll = Capability::QuantileApprox(Some(SketchAlgorithm::Kll)); - assert!(required.is_satisfied_by(&indexed_dd)); - assert!(required.is_satisfied_by(&indexed_kll)); - - // Still cross-family-incompatible: a concrete quantile requirement - // is never satisfied by a cardinality-family available handle. - let indexed_hll = Capability::QuantileApprox(Some(SketchAlgorithm::Hll)); - assert!(!required.is_satisfied_by(&indexed_hll)); - } - - #[test] - fn is_satisfied_by_cardinality_is_total() { - let required = Capability::CardinalityApprox; - let indexed = Capability::CardinalityApprox; - assert!(required.is_satisfied_by(&indexed)); - } - - #[test] - fn is_satisfied_by_different_families_are_incompatible() { - let required = Capability::QuantileApprox(None); - let indexed = Capability::CardinalityApprox; - assert!(!required.is_satisfied_by(&indexed)); - - let required = Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)); - let indexed = Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)); - assert!(!required.is_satisfied_by(&indexed)); - } - - #[test] - fn is_satisfied_by_topk_handles_must_match() { - let required = Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)); - let indexed_with_heap = Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)); - let indexed_no_heap = Capability::FrequencyTopk(Some(SketchAlgorithm::Cms)); - assert!(required.is_satisfied_by(&indexed_with_heap)); - assert!(!required.is_satisfied_by(&indexed_no_heap)); - } - - #[test] - fn is_satisfied_by_frequency_topk_rejects_heapless() { - // Top-k REQUIRES a heap-bearing handle. Even when the available - // capability declares itself as `FrequencyTopk(CountMin)` (an - // ill-formed catalog entry), the satisfaction check must reject - // it — top-k cannot enumerate items off a heap-less sketch. - let required_any = Capability::FrequencyTopk(None); - let required_concrete = Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)); - let indexed_heapless = Capability::FrequencyTopk(Some(SketchAlgorithm::Cms)); - let indexed_heapless_cs = Capability::FrequencyTopk(Some(SketchAlgorithm::CountSketch)); - assert!(!required_any.is_satisfied_by(&indexed_heapless)); - assert!(!required_any.is_satisfied_by(&indexed_heapless_cs)); - assert!(!required_concrete.is_satisfied_by(&indexed_heapless)); - } - - #[test] - fn is_satisfied_by_frequency_topk_any_matches_either_heap() { - // `Any` required for top-k accepts either heap-bearing handle. - let required = Capability::FrequencyTopk(None); - let cms_heap = Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)); - let cs_heap = Capability::FrequencyTopk(Some(SketchAlgorithm::CountSketchWithHeap)); - assert!(required.is_satisfied_by(&cms_heap)); - assert!(required.is_satisfied_by(&cs_heap)); - } - - #[test] - fn is_satisfied_by_frequency_estimate_accepts_heap_bearing() { - // Bare frequency point queries can be answered by ANY - // frequency-family sketch — heap-less AND heap-bearing both work - // (the heap is additional metadata; the underlying CMS / CS - // matrix answers the point query either way). - let required = Capability::FrequencyEstimate(None); - let cms = Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms)); - let cs = Capability::FrequencyEstimate(Some(SketchAlgorithm::CountSketch)); - let cms_heap = Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)); - let cs_heap = Capability::FrequencyTopk(Some(SketchAlgorithm::CountSketchWithHeap)); - assert!(required.is_satisfied_by(&cms)); - assert!(required.is_satisfied_by(&cs)); - assert!(required.is_satisfied_by(&cms_heap)); - assert!(required.is_satisfied_by(&cs_heap)); - } - - #[test] - fn is_satisfied_by_frequency_estimate_rejects_non_frequency_family() { - let required = Capability::FrequencyEstimate(None); - // QuantileApprox / CardinalityApprox don't answer frequency. - let q = Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)); - let c = Capability::CardinalityApprox; - // FrequencyEstimate with a non-frequency-family handle on the - // available side is also rejected (defensive). - let bad = Capability::FrequencyEstimate(Some(SketchAlgorithm::Hll)); - assert!(!required.is_satisfied_by(&q)); - assert!(!required.is_satisfied_by(&c)); - assert!(!required.is_satisfied_by(&bad)); - } - - // ── Capability::ExactAgg — matching ────────────────────────────────── - - #[test] - fn is_satisfied_by_exact_agg_same_type_matches() { - // Sum required, Sum indexed → match. Same for every concrete - // AggregationType — the equality check is structural. - let required = Capability::ExactAgg(AggregationType::Sum); - let indexed = Capability::ExactAgg(AggregationType::Sum); - assert!(required.is_satisfied_by(&indexed)); - } - - #[test] - fn is_satisfied_by_exact_agg_different_types_do_not_match() { - // Sum required, Max indexed → no match. No wildcard for - // ExactAgg — every agg_type stands on its own. - let required = Capability::ExactAgg(AggregationType::Sum); - let indexed = Capability::ExactAgg(AggregationType::Max); - assert!(!required.is_satisfied_by(&indexed)); - } - - #[test] - fn is_satisfied_by_exact_agg_does_not_match_other_families() { - // ExactAgg is its own family — no cross-family satisfaction - // with QuantileApprox / CardinalityApprox / FrequencyEstimate / - // FrequencyTopk. - let required = Capability::ExactAgg(AggregationType::Sum); - assert!( - !required.is_satisfied_by(&Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))) - ); - assert!(!required.is_satisfied_by(&Capability::CardinalityApprox)); - assert!( - !required.is_satisfied_by(&Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))) - ); - assert!(!required.is_satisfied_by(&Capability::FrequencyTopk(Some( - SketchAlgorithm::CmsWithHeap - )))); - - // And the reverse — a sketch-family required capability must - // not match an ExactAgg-backed sid. - let sketch_required = Capability::QuantileApprox(None); - let exact_indexed = Capability::ExactAgg(AggregationType::DatasketchesKLL); - assert!(!sketch_required.is_satisfied_by(&exact_indexed)); - } - - #[test] - fn exact_agg_covers_each_canonical_agg_type() { - // Spot-check the full AggregationType surface — each variant - // round-trips through Capability::ExactAgg without losing - // information. Documents the intended coverage of the new - // variant. If a future PR adds an AggregationType variant, this - // test (combined with the exhaustive match in `is_satisfied_by`'s - // `req == have` form) will not require code changes — equality - // is structural. - let cases = [ - AggregationType::Sum, - AggregationType::Increase, - AggregationType::Min, - AggregationType::Max, - AggregationType::DatasketchesKLL, - AggregationType::HydraKLL, - AggregationType::CountMinSketch, - AggregationType::CountMinSketchWithHeap, - AggregationType::CountSketch, - AggregationType::HLL, - AggregationType::DDSketch, - ]; - for t in cases { - let cap = Capability::ExactAgg(t); - assert!( - cap.is_satisfied_by(&Capability::ExactAgg(t)), - "ExactAgg({t:?}) should satisfy itself" - ); - } - } - - #[test] - fn sum_family_cannot_impersonate_exact_counter_state() { - let required = Capability::ExactAgg(AggregationType::Increase); - assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); - assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); - - let required_multi = Capability::ExactAgg(AggregationType::Increase); - assert!(!required_multi.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); - } - - #[test] - fn is_satisfied_by_sum_family_does_not_answer_required_multi_increase_from_single_sum() { - // A different exact family cannot supply counter state. - let required = Capability::ExactAgg(AggregationType::Increase); - assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); - } - - #[test] - fn is_satisfied_by_increase_does_not_answer_required_sum() { - // The reverse direction is NOT sound: required ExactAgg(Sum) is - // sum_over_time semantics (Σ of raw cumulative-counter samples), - // which an Increase-registered sid's per-window deltas cannot - // reconstruct (issue #301) -- refused explicitly at the - // query-dispatch layer, not routed through here. - let required = Capability::ExactAgg(AggregationType::Sum); - assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Increase))); - } - - // ── capability_for: ExactAgg dormancy ──────────────────────────────── - - #[test] - fn exact_agg_routing_keeps_sum_rate_and_increase_distinct() { - assert_eq!( - capability_for(&AggIntent::Sum { col: None }), - Some(Capability::ExactAgg(AggregationType::Sum)) - ); - assert_eq!( - capability_for(&AggIntent::Rate), - Some(Capability::ExactAgg(AggregationType::Rate)) - ); - assert_eq!( - capability_for(&AggIntent::Increase), - Some(Capability::ExactAgg(AggregationType::Increase)) - ); - // Exact count follows the Planner Count family; Avg still needs - // its own composition contract. - assert_eq!( - capability_for(&AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }), - Some(Capability::ExactAgg(AggregationType::Count)) - ); - assert_eq!(capability_for(&AggIntent::Avg { col: None }), None); - } - - #[test] - fn count_sketch_with_heap_handle_round_trips() { - // `CountSketchWithHeap` is the CountSketch counterpart to - // `CmsWithHeap`. Construct a `FrequencyTopk` capability around - // it and verify it satisfies an `Any`-required top-k. - let cap = Capability::FrequencyTopk(Some(SketchAlgorithm::CountSketchWithHeap)); - let required = Capability::FrequencyTopk(None); - assert!(required.is_satisfied_by(&cap)); - // And the concrete-against-concrete (same handle) case matches. - let required_concrete = - Capability::FrequencyTopk(Some(SketchAlgorithm::CountSketchWithHeap)); - assert!(required_concrete.is_satisfied_by(&cap)); - // Intentional broadening vs. this module's pre-`sketch_family_satisfied` - // behavior: CMS and CountSketch are the SAME frequency family in - // `physical::post_asap::matcher`'s reference rule table (both bare and - // heap-bearing — see its - // `cms_and_count_sketch_are_the_same_frequency_family` test), so a - // `CmsWithHeap` requirement IS now satisfied by a - // `CountSketchWithHeap` available (both heap-bearing, same - // family), not just an exact-handle match. Harmless in practice: - // `capability_for` always emits `Any` for `FrequencyTopk`, never a - // concrete handle, so this exact combination never arises from a - // real query. - let required_cms = Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)); - assert!(required_cms.is_satisfied_by(&cap)); - // Still cross-family-incompatible: a heap-bearing requirement is - // never satisfied by a bare (heap-less) available handle. - let bare = Capability::FrequencyTopk(Some(SketchAlgorithm::Cms)); - assert!(!required_cms.is_satisfied_by(&bare)); - } - - // ── OuterAgg fold semantics ────────────────────────────────────────── - // - // Pin the fold-by-operator dispatch shape the engine reads off the - // typed `ASAPTierCandidate.outer_agg` field. The identity case - // (single-value group) is the load-bearing assertion for asap's - // per-zone-DDSketch shape: `max by (zone) (quantile_over_time(...))` - // produces one row per zone, and `OuterAgg::Max.fold([x]) == x` — - // the wrapper is a no-op for already-grouped inner results. - - #[test] - fn outer_agg_default_is_none() { - assert_eq!(OuterAgg::default(), OuterAgg::None); - assert!(!OuterAgg::default().is_some()); - } - - #[test] - fn outer_agg_none_carries_no_by_labels() { - assert!(OuterAgg::None.by_labels().is_empty()); - } - - #[test] - fn outer_agg_max_fold_single_value_is_identity() { - // Issue #296 identity case: inner already emits one row per - // by-group; the outer max fold must return that row unchanged. - let v = OuterAgg::Max(vec!["zone".to_string()]) - .fold(&[42.5]) - .unwrap(); - assert_eq!(v, 42.5); - } - - #[test] - fn outer_agg_min_fold_single_value_is_identity() { - let v = OuterAgg::Min(vec!["zone".to_string()]) - .fold(&[42.5]) - .unwrap(); - assert_eq!(v, 42.5); - } - - #[test] - fn outer_agg_avg_fold_single_value_is_identity() { - let v = OuterAgg::Avg(vec!["zone".to_string()]) - .fold(&[42.5]) - .unwrap(); - assert_eq!(v, 42.5); - } - - #[test] - fn outer_agg_max_fold_multi_picks_largest() { - let v = OuterAgg::Max(vec![]).fold(&[1.0, 5.0, 3.0]).unwrap(); - assert_eq!(v, 5.0); - } - - #[test] - fn outer_agg_min_fold_multi_picks_smallest() { - let v = OuterAgg::Min(vec![]).fold(&[1.0, 5.0, 3.0]).unwrap(); - assert_eq!(v, 1.0); - } - - #[test] - fn outer_agg_avg_fold_multi_is_mean() { - let v = OuterAgg::Avg(vec![]).fold(&[1.0, 5.0, 3.0]).unwrap(); - assert!((v - 3.0).abs() < 1e-9); - } - - #[test] - fn outer_agg_count_fold_returns_cardinality() { - let v = OuterAgg::Count(vec![]).fold(&[1.0, 5.0, 3.0]).unwrap(); - assert_eq!(v, 3.0); - } - - #[test] - fn outer_agg_group_fold_returns_one() { - let v = OuterAgg::Group(vec![]).fold(&[1.0, 5.0, 3.0]).unwrap(); - assert_eq!(v, 1.0); - } - - #[test] - fn outer_agg_stddev_fold_multi_is_population_stddev() { - // population stddev of [1,2,3,4,5] is sqrt(2) ≈ 1.4142 - let v = OuterAgg::Stddev(vec![]) - .fold(&[1.0, 2.0, 3.0, 4.0, 5.0]) - .unwrap(); - assert!((v - 2.0_f64.sqrt()).abs() < 1e-9); - } - - #[test] - fn outer_agg_stdvar_fold_multi_is_population_variance() { - let v = OuterAgg::Stdvar(vec![]) - .fold(&[1.0, 2.0, 3.0, 4.0, 5.0]) - .unwrap(); - assert!((v - 2.0).abs() < 1e-9); - } - - #[test] - fn outer_agg_empty_input_returns_none() { - assert_eq!(OuterAgg::Max(vec![]).fold(&[]), None); - assert_eq!(OuterAgg::Avg(vec![]).fold(&[]), None); - assert_eq!(OuterAgg::Count(vec![]).fold(&[]), None); - } -} diff --git a/control_plane/src/physical/sketch_catalog.rs b/control_plane/src/physical/sketch_catalog.rs deleted file mode 100644 index 9be3c43f4..000000000 --- a/control_plane/src/physical/sketch_catalog.rs +++ /dev/null @@ -1,365 +0,0 @@ -//! Sketch directory — single source of truth for mapping aggregation -//! operations to sketch types, parameters, and memory estimates. -//! -//! Previously this logic was duplicated across: -//! - `planner/rules.rs` (`select_sketch_type`) -//! - `planner/stage_split.rs` (`agg_op_to_sketch_type`, `agg_op_to_sketch_params`, -//! `estimated_sketch_memory_bytes`) -//! - `algebra/allocator.rs` (`sketch_type_for_op`) -//! -//! All callers now go through this module. - -use crate::planner_selection::agg_accuracy; -use crate::types::{AggType, SketchDefaults, SketchParams, SketchType}; -use planner_types::pre_asap::AggIntent; - -/// Physical sizing input for one independently maintained partition. -#[derive(Debug, Clone, PartialEq)] -pub struct PerPartitionWrap { - pub inner: AggIntent, - pub keys: Vec, -} - -// ── AggType → candidate SketchTypes ────────────────────────────────────────── - -/// Candidate sketch families per aggregation type. -/// -/// Each aggregation type has multiple viable sketch implementations. -/// The first entry is the default; the cost-model planner scores all -/// candidates and picks the cheapest that meets the accuracy SLA. -/// -/// | AggType | Candidates (default first) | -/// |-------------|---------------------------- | -/// | Quantile | DDSketch, KLL | -/// | Cardinality | HLL | -/// | Frequency | CountSketch, CountMinSketch | -pub fn candidates_for_agg(agg: &AggType) -> &'static [SketchType] { - match agg { - AggType::Quantile => &[SketchType::DDSketch, SketchType::KLL], - AggType::Cardinality => &[SketchType::HLL], - AggType::Frequency => &[SketchType::CountSketch, SketchType::CountMinSketch], - } -} - -/// All candidate sketch types for a workload (deduped, stable order). -pub fn candidates_for_workload(aggs: &[AggType]) -> Vec { - let mut out = Vec::new(); - for agg in aggs { - for st in candidates_for_agg(agg) { - if !out.contains(st) { - out.push(st.clone()); - } - } - } - out -} - -/// Pick the default sketch family from a list of aggregation types. -/// -/// Returns the first candidate for the highest-priority aggregation. -/// Priority: Quantile → Cardinality → Frequency. Falls back to DDSketch. -/// -/// For cost-optimised selection, use [`candidates_for_workload`] and score -/// each candidate via the cost model. -pub fn sketch_type_for_agg(aggs: &[AggType]) -> SketchType { - for agg in aggs { - let candidates = candidates_for_agg(agg); - if !candidates.is_empty() { - return candidates[0].clone(); - } - } - SketchType::DDSketch -} - -// ── AggIntent → SketchType ─────────────────────────────────────────────────── - -/// Resolve the concrete [`SketchType`] for an [`AggIntent`] IR node. -pub fn sketch_type_for_op(op: &AggIntent) -> SketchType { - if crate::planner_selection::as_frequency(op).is_some() { - return SketchType::CountSketch; - } - match op { - AggIntent::Quantile { .. } => SketchType::DDSketch, - AggIntent::Cardinality { .. } => SketchType::HLL, - // Min/Max/Sum/Count/Avg/TopK/Rate/Increase + archive-only — all - // historically rode the "DDSketch / exact passthrough" rails in - // the legacy resolver. Keep that mapping until Step γ migrates the - // physical planner onto canonical L4 binding. - _ => SketchType::DDSketch, - } -} - -#[cfg(test)] -/// Resolve the concrete [`SketchType`] for a `PerPartitionWrap` shape — -/// delegates to the wrapped inner intent. Kept as a separate function so -/// that the PerPartition structural collapse (Step γ) is a focused -/// removal rather than a recursion-tracking refactor. -pub fn sketch_type_for_per_partition(wrap: &PerPartitionWrap) -> SketchType { - sketch_type_for_op(&wrap.inner) -} - -// ── AggIntent → SketchParams ──────────────────────────────────────────────── - -/// Derive [`SketchParams`] from an [`AggIntent`] IR node. -pub fn sketch_params_for_op(op: &AggIntent) -> SketchParams { - if crate::planner_selection::as_frequency(op).is_some() { - let acc = agg_accuracy(op); - return SketchParams::CountSketch { - epsilon: acc, - delta: 0.01, - }; - } - match op { - AggIntent::Quantile { q, .. } => SketchParams::DDSketch { - relative_accuracy: agg_accuracy(op), - // Canonical Quantile is single-φ post Step α. Multi-φ - // fan-out lives at construction time (Merge of SketchAgg - // siblings), so this carrier always reports its single q. - quantiles: vec![*q], - }, - AggIntent::Cardinality { .. } => { - let acc = agg_accuracy(op).max(f64::MIN_POSITIVE); - // registers ≈ (1.04/accuracy)^2, precision = log2(registers) - let registers = ((1.04 / acc).powi(2) as u32).next_power_of_two(); - let precision = (registers as f64).log2() as u32; - SketchParams::HLL { precision } - } - // Min / Max — preserve the legacy `Extrema` mapping (DDSketch over - // the 0.0 / 1.0 boundary quantiles). - AggIntent::Min { .. } | AggIntent::Max { .. } => SketchParams::DDSketch { - relative_accuracy: 0.01, - quantiles: vec![0.0, 1.0], - }, - // Sum / Count / Avg / TopK / Rate / Increase / archive-only — no - // sketch-specific params; the legacy `Exact(_)` arm returned - // `SketchParams::default()` and we preserve that. - _ => SketchParams::default(), - } -} - -#[cfg(test)] -/// Combined (type, params) lookup — convenience for callers that need both. -pub fn sketch_type_and_params(op: &AggIntent) -> (SketchType, SketchParams) { - (sketch_type_for_op(op), sketch_params_for_op(op)) -} - -// ── AggIntent → memory estimate ───────────────────────────────────────────── - -/// Estimated sketch memory footprint per series (bytes). -/// -/// Used by the physical planner's placement decision to defer a sketch -/// build off a stage when its budget would be exceeded. -pub fn estimated_sketch_memory_bytes(op: &AggIntent) -> u64 { - if crate::planner_selection::as_frequency(op).is_some() { - let acc = agg_accuracy(op).max(f64::MIN_POSITIVE); - // CMS: width ≈ e/accuracy, depth ≈ 5, memory = width*depth*8 - let width = (std::f64::consts::E / acc) as u64; - return width * 5 * 8; - } - match op { - AggIntent::Quantile { .. } => 4_096, - AggIntent::Cardinality { .. } => { - let acc = agg_accuracy(op).max(f64::MIN_POSITIVE); - // HLL: registers ≈ (1.04/accuracy)^2, memory = registers - let registers = ((1.04 / acc).powi(2) as u64).next_power_of_two(); - registers.max(16) - } - // Legacy `Extrema { .. }` (now canonical Min / Max) — 16 bytes. - AggIntent::Min { .. } | AggIntent::Max { .. } => 16, - // Sum / Count / Avg / TopK / Rate / Increase / archive-only — no - // sketch state; preserve the legacy `Exact(_) → 8` mapping. - _ => 8, - } -} - -#[cfg(test)] -/// Memory estimate for a `PerPartitionWrap` — scales the inner-intent -/// estimate by `2 ** keys.len()` (capped at 2**10), matching the legacy -/// `PerPartition` formula. -pub fn estimated_sketch_memory_per_partition(wrap: &PerPartitionWrap) -> u64 { - let factor = 1u64 << wrap.keys.len().min(10); - estimated_sketch_memory_bytes(&wrap.inner).saturating_mul(factor) -} - -// ── SketchType + accuracy SLA → SketchParams (configurable defaults) ───────── - -/// Build default [`SketchParams`] from a [`SketchDefaults`] config and accuracy SLA. -/// -/// Query-specific quantiles override the configured grid when non-empty. -pub fn build_sketch_params( - defaults: &SketchDefaults, - st: &SketchType, - accuracy_sla: f64, - query_quantiles: &[f64], -) -> SketchParams { - let acc = if accuracy_sla <= 0.0 { - defaults.ddsketch.relative_accuracy - } else { - accuracy_sla - }; - let quantiles: Vec = if !query_quantiles.is_empty() { - query_quantiles.to_vec() - } else { - defaults.quantile_grid.clone() - }; - match st { - SketchType::DDSketch => SketchParams::DDSketch { - relative_accuracy: acc, - quantiles, - }, - SketchType::KLL => { - let k = ((1.0 / acc) as u32).max(defaults.kll.min_k); - SketchParams::KLL { k, quantiles } - } - SketchType::HLL => { - let d = &defaults.hll; - let precision = if acc > d.precision_threshold { - d.precision_coarse - } else { - d.precision_fine - }; - SketchParams::HLL { precision } - } - SketchType::CountSketch => SketchParams::CountSketch { - epsilon: defaults.count_sketch.epsilon, - delta: defaults.count_sketch.delta, - }, - SketchType::CountMinSketch => SketchParams::CountMinSketch { - rows: defaults.count_min_sketch.rows, - cols: defaults.count_min_sketch.cols, - metric_name: defaults.count_min_sketch.metric_name.clone(), - }, - } -} - -/// Convenience: build default params using compiled-in defaults. -pub fn default_sketch_params(st: &SketchType, accuracy_sla: f64) -> SketchParams { - build_sketch_params(&SketchDefaults::default(), st, accuracy_sla, &[]) -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn agg_type_quantile_maps_to_ddsketch() { - assert_eq!( - sketch_type_for_agg(&[AggType::Quantile]), - SketchType::DDSketch - ); - } - - #[test] - fn agg_type_cardinality_maps_to_hll() { - assert_eq!( - sketch_type_for_agg(&[AggType::Cardinality]), - SketchType::HLL - ); - } - - #[test] - fn agg_type_frequency_maps_to_countsketch() { - assert_eq!( - sketch_type_for_agg(&[AggType::Frequency]), - SketchType::CountSketch - ); - } - - #[test] - fn empty_aggs_default_to_ddsketch() { - assert_eq!(sketch_type_for_agg(&[]), SketchType::DDSketch); - } - - #[test] - fn op_quantile_yields_ddsketch_type_and_params() { - use crate::types::AccuracyTarget; - let op = AggIntent::Quantile { - col: None, - q: 0.5, - accuracy: AccuracyTarget::Epsilon(0.01), - }; - let (st, p) = sketch_type_and_params(&op); - assert_eq!(st, SketchType::DDSketch); - assert!(matches!(p, SketchParams::DDSketch { .. })); - } - - #[test] - fn op_cardinality_yields_hll_type() { - let op = planner_types::pre_asap::default_cardinality(); - assert_eq!(sketch_type_for_op(&op), SketchType::HLL); - } - - #[test] - fn op_frequency_yields_countsketch() { - let op = crate::planner_selection::default_frequency(); - assert_eq!(sketch_type_for_op(&op), SketchType::CountSketch); - } - - #[test] - fn per_partition_delegates_to_inner() { - let wrap = PerPartitionWrap { - inner: planner_types::pre_asap::default_cardinality(), - keys: vec!["k".into()], - }; - assert_eq!(sketch_type_for_per_partition(&wrap), SketchType::HLL); - } - - #[test] - fn memory_quantile() { - use crate::types::AccuracyTarget; - let op = AggIntent::Quantile { - col: None, - q: 0.5, - accuracy: AccuracyTarget::Epsilon(0.01), - }; - assert_eq!(estimated_sketch_memory_bytes(&op), 4096); - } - - #[test] - fn memory_per_partition_scales_by_keys() { - let inner = planner_types::pre_asap::default_cardinality(); - let base_mem = estimated_sketch_memory_bytes(&inner); - let wrap = PerPartitionWrap { - inner, - keys: vec!["a".into(), "b".into()], - }; - assert_eq!(estimated_sketch_memory_per_partition(&wrap), base_mem * 4); - } - - #[test] - fn configurable_defaults_override_quantile_grid() { - let mut d = SketchDefaults::default(); - d.quantile_grid = vec![0.5, 0.99]; - let p = build_sketch_params(&d, &SketchType::DDSketch, 0.01, &[]); - assert_eq!(p.quantiles(), &[0.5, 0.99]); - } - - #[test] - fn query_quantiles_override_grid() { - let d = SketchDefaults::default(); - let p = build_sketch_params(&d, &SketchType::DDSketch, 0.01, &[0.1, 0.9]); - assert_eq!(p.quantiles(), &[0.1, 0.9]); - } - - #[test] - fn quantile_candidates_include_ddsketch_and_kll() { - let c = candidates_for_agg(&AggType::Quantile); - assert!(c.contains(&SketchType::DDSketch)); - assert!(c.contains(&SketchType::KLL)); - } - - #[test] - fn frequency_candidates_include_cs_and_cms() { - let c = candidates_for_agg(&AggType::Frequency); - assert!(c.contains(&SketchType::CountSketch)); - assert!(c.contains(&SketchType::CountMinSketch)); - } - - #[test] - fn candidates_for_workload_dedupes() { - let c = candidates_for_workload(&[AggType::Quantile, AggType::Quantile]); - assert_eq!(c.iter().filter(|s| **s == SketchType::DDSketch).count(), 1); - } -} diff --git a/control_plane/src/physical/summary_catalog.rs b/control_plane/src/physical/summary_catalog.rs deleted file mode 100644 index 4b2d61cf7..000000000 --- a/control_plane/src/physical/summary_catalog.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Control-plane compatibility surface for the shared catalog wire contract. -//! -//! The authoritative catalog is constructed and published by the control -//! plane, while its immutable contract is shared with every consumer. - -pub use asap_types::summary_catalog::*; diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs deleted file mode 100644 index 045882c94..000000000 --- a/control_plane/src/physical/workload_cost.rs +++ /dev/null @@ -1,1502 +0,0 @@ -//! Compile physical candidates, export quoteable manifests, and select the -//! lowest-cost feasible candidate from the supplied bounded inventory. -//! -//! Planner owns semantic legality. A manifest describes the exact physical -//! demand to price; provider quotes and candidate evaluations are separate. - -mod materialization_candidates; -mod status; -pub use status::{CandidateEvaluationStatus, CandidateSearchScope}; - -#[cfg(test)] -use super::compiler::DeploymentPlanCompiler; - -use std::collections::{BTreeMap, BTreeSet}; - -use asap_aware_mapping::cost_model::Cost; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; - -use super::compiler::{ - CompileError, CompiledPhysicalPlan, PhysicalCompilationRequest, PhysicalDeploymentContext, - QueryCompilationInput, -}; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct CostComponentDemand { - /// Exact implementation/configuration being priced, not merely a family. - pub implementation: Value, - /// `horizon` includes all work in the manifest's source/time scope; - /// `query_evaluation` is one execution of this bound query operator. - #[serde(rename = "unit")] - pub pricing_basis: String, - #[serde(rename = "multiplicity")] - pub occurrences_per_horizon: f64, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct WorkloadCostManifest { - pub plan_id: u64, - pub plan_version: u64, - pub planner_revision: String, - pub capability_snapshot_id: String, - pub backend_compat: String, - pub horizon_seconds: f64, - /// Canonical roots, requirements and demand must match across candidates. - pub workload: BTreeMap, - pub components: BTreeMap, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct WorkloadQuote { - pub manifest: WorkloadCostManifest, - /// Provider attests feasibility for this capability/data generation, - /// including an accessible exact backend when the plan has fallback nodes. - pub executable: bool, - /// Calibrated costs in ONE model's units. All keys are required, including - /// explicit zero costs. Horizon quotes include source cardinality, all - /// maintained groups, retention/spill and the stated partition's work. - pub unit_costs: BTreeMap, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct WorkloadCostEvidence { - #[serde(default)] - pub backend_revision: String, - #[serde(default)] - pub planner_revision: String, - pub data_snapshot_id: String, - pub model_version: String, - pub observed_at_unix_ms: u64, - pub valid_for_ms: u64, - pub quotes: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -/// A candidate diagnostic can precede pricing or record compilation failure. -pub struct CandidatePlanEvaluation { - #[serde(rename = "candidate_id")] - pub candidate_id: Option, - #[serde(default)] - pub logical_root_ids: Vec, - #[serde(rename = "physical_candidate_id")] - pub physical_candidate_id: Option, - pub identity_unavailable_reason: Option, - #[serde(default)] - pub status: CandidateEvaluationStatus, - pub plan_id: Option, - pub total_cost: Option, - pub unavailable_reason: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct MaterializationSearchCoverage { - #[serde(rename = "eligible_leaves")] - pub eligible_materialization_count: usize, - #[serde(rename = "enumerated_local_masks")] - pub enumerated_candidate_key_sets: usize, - pub exhaustive: bool, - #[serde(rename = "scope")] - pub search_scope: CandidateSearchScope, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct CandidatePlanSelectionReport { - #[serde(default)] - #[serde(rename = "logical_selection")] - pub planner_selection_trace: Vec, - #[serde(default)] - pub materialization_search_coverage: Option, - pub data_snapshot_id: String, - pub model_version: String, - pub selected_plan_id: u64, - pub selected_manifest: WorkloadCostManifest, - pub component_costs: BTreeMap, - #[serde(rename = "candidates")] - pub candidate_evaluations: Vec, -} - -fn invalid(reason: impl Into) -> CompileError { - CompileError::Snapshot(format!("complete workload cost: {}", reason.into())) -} - -pub fn manifest( - plan: &CompiledPhysicalPlan, - queries: &[QueryCompilationInput], -) -> Result { - let horizon = queries - .first() - .ok_or_else(|| invalid("empty workload"))? - .summary_lifecycle_inputs - .horizon_seconds; - if !horizon.is_finite() || horizon <= 0.0 { - return Err(invalid("invalid horizon")); - } - let mut workload = BTreeMap::new(); - let mut reads = BTreeMap::new(); - for query in queries { - if query.summary_lifecycle_inputs.horizon_seconds != horizon - || query.summary_lifecycle_inputs.evaluation_interval_ms == 0 - { - return Err(invalid("mixed horizons or unknown recurrence")); - } - let canonical = crate::query_plan::canonical_promql(&query.query_string)?; - if workload - .insert( - query.query_id.clone(), - json!({ - "query": canonical, "accuracy": query.accuracy_target, - "evaluation_interval_ms": query.summary_lifecycle_inputs.evaluation_interval_ms, - "source": query.legacy_query_source, - }), - ) - .is_some() - { - return Err(invalid("duplicate query ID")); - } - reads.insert( - query.query_id.clone(), - horizon * 1000.0 / f64::from(query.summary_lifecycle_inputs.evaluation_interval_ms), - ); - } - let mut components = BTreeMap::new(); - let mut add = - |id: String, implementation: Value, pricing_basis: &str, occurrences_per_horizon: f64| { - components.insert( - id, - CostComponentDemand { - implementation, - pricing_basis: pricing_basis.into(), - occurrences_per_horizon, - }, - ); - }; - // Backend merge/update, storage, and edge maintenance are separate work. - // The raw input is read once per source partition, not once per consumer. - for schema in &plan.precompute_plan.schemas { - let mut locations = vec!["backend".to_string()]; - locations.extend( - plan.collector_plans - .iter() - .filter(|collector| { - collector - .materializations - .iter() - .any(|m| m.materialization == schema.materialization) - }) - .map(|collector| format!("collector:{}", collector.collector_id)), - ); - for location in locations { - let source = json!({"source": schema.source, "location": location, "ingest": plan.precompute_plan.ingest}); - add(format!("source:{}", source), source, "horizon", 1.0); - let physical = plan - .precompute_plan - .materializations - .iter() - .find(|m| m.policy_fingerprint() == schema.materialization.fingerprint()) - .ok_or_else(|| invalid("state has no physical implementation"))?; - let identity = json!({"schema": schema, "location": location, "physical": physical, - "window_implementation": plan.lifecycle_estimates.iter().find(|e| e.materialization == schema.materialization).map(|e| &e.window_realization_id)}); - for operation in ["build", "update", "residency", "retire"] { - add( - format!("state:{location}:{}:{operation}", schema.materialization.0), - json!({"operation": operation, "binding": identity}), - "horizon", - 1.0, - ); - } - } - } - for rule in &plan.transmission_plan.rules { - add( - format!("transport:{}:{}", rule.producer_id, rule.materialization.0), - json!(rule), - "horizon", - 1.0, - ); - } - for entry in plan.query_plan.entries.values() { - let evaluations = *reads - .get(&entry.query_id) - .ok_or_else(|| invalid("unmapped query root"))?; - if entry - .nodes - .values() - .any(|node| matches!(node, crate::query_plan::QueryPlanNode::ExactFallback { .. })) - { - let query = queries - .iter() - .find(|query| query.query_id == entry.query_id) - .ok_or_else(|| invalid("unmapped exact source"))?; - // Charge the exact service's input upkeep/storage separately from - // per-evaluation native execution. Zero is valid only if explicitly - // quoted as already-provisioned/non-incremental for this decision. - let parsed = crate::query_parser::parse_query_expr_canonical( - &query.query_string, - query.accuracy_target.clone(), - ) - .map_err(|error| invalid(error.to_string()))?; - for metric in exact_source_metrics(&parsed)? { - let source = json!({"source": planner_types::pre_asap::Source::TimeSeries { metric }, "location": "exact_backend"}); - add(format!("source:{}", source), source, "horizon", 1.0); - } - } - // Typed local scans require retained input and ingest/update work even - // when no precomputed summary is installed. Deduplicate by source. - for node in entry.nodes.values() { - if let crate::query_plan::QueryPlanNode::Logical { - operator: - crate::query_plan::query_time::QueryTimeOperator::CurrentSeries { - population, .. - }, - .. - } = node - { - let key = population.key(); - for phase in ["build", "update", "residency", "retire"] { - add( - format!("current-series:{key}:{phase}"), - json!({"population": population, "phase": phase}), - "horizon", - 1.0, - ); - } - let source = json!({"source": planner_types::pre_asap::Source::TimeSeries { metric: population.metric.clone() }, "location": "backend", "ingest": plan.precompute_plan.ingest}); - add(format!("source:{source}"), source, "horizon", 1.0); - } - if matches!( - node, - crate::query_plan::QueryPlanNode::Logical { - operator: crate::query_plan::query_time::QueryTimeOperator::Scan { .. }, - .. - } - ) { - return Err(invalid( - "generic backend raw scans are outside the ASAP/Prometheus execution contract", - )); - } - if let crate::query_plan::QueryPlanNode::Logical { - operator: - crate::query_plan::query_time::QueryTimeOperator::ExactSubquery { query } - | crate::query_plan::query_time::QueryTimeOperator::CandidateExactSubquery { - query, - .. - }, - .. - } = node - { - let parsed = crate::query_parser::parse_query_expr_canonical( - query, - crate::types::AccuracyTarget::Exact, - ) - .map_err(|error| invalid(error.to_string()))?; - for metric in exact_source_metrics(&parsed)? { - let source = json!({"source": planner_types::pre_asap::Source::TimeSeries { metric }, "location": "exact_backend"}); - add(format!("source:{}", source), source, "horizon", 1.0); - } - } - } - // Reachability comes from QueryPlan, including materialization reads, - // arithmetic, reduction and a complete engine-native exact fallback. - for node_id in entry.topological_order()? { - add( - format!("query:{}:{}", entry.query_id, node_id.0), - json!({"node": entry.nodes[&node_id], "query": entry.canonical_query, "instant": entry.instant}), - "query_evaluation", - evaluations, - ); - } - add( - format!("result:{}", entry.query_id), - json!({"query_id": entry.query_id, "root": entry.root}), - "query_evaluation", - evaluations, - ); - } - Ok(WorkloadCostManifest { - plan_id: plan.envelope.plan_id, - plan_version: plan.envelope.plan_version, - planner_revision: plan.envelope.planner_revision.clone(), - capability_snapshot_id: plan.envelope.capability_snapshot_id.clone(), - backend_compat: plan.envelope.backend_compat.clone(), - horizon_seconds: horizon, - workload, - components, - }) -} - -/// Walk the canonical relational tree, preserving every input to binary and -/// fan-in operators. Unsupported source discovery must not produce a partial quote. -pub(crate) fn exact_source_metrics( - expr: &planner_types::pre_asap::QueryExpr, -) -> Result, CompileError> { - use planner_types::pre_asap::{QueryExpr, Source}; - fn visit(expr: &QueryExpr, metrics: &mut BTreeSet) -> Result<(), CompileError> { - match expr { - QueryExpr::Scan { - source: Source::TimeSeries { metric }, - .. - } if !metric.is_empty() => { - metrics.insert(metric.clone()); - } - QueryExpr::PromqlScalarBridge(child) - | QueryExpr::PromqlVectorFromScalar(child) - | QueryExpr::PromqlScalarFromVector(child) - | QueryExpr::PromqlRelabel { child, .. } - | QueryExpr::PromqlSeriesSample { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Project { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - | QueryExpr::TimeRange { child, .. } - | QueryExpr::TimeShift { child, .. } => visit(child, metrics)?, - QueryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } - | QueryExpr::Join { left, right, .. } - | QueryExpr::SetOp { left, right, .. } => { - visit(left, metrics)?; - visit(right, metrics)?; - } - QueryExpr::Concat { children, .. } => { - for child in children { - visit(child, metrics)?; - } - } - QueryExpr::Literal(_) | QueryExpr::EvalTimestamp => {} - // In particular, info() has an implicit metadata source that is - // not a Scan child, and unnamed selectors require source discovery. - _ => { - return Err(invalid( - "exact-source pricing cannot enumerate this query's sources", - )) - } - } - Ok(()) - } - let mut metrics = BTreeSet::new(); - visit(expr, &mut metrics)?; - Ok(metrics) -} - -pub(super) type PricedComponents = (Cost, BTreeMap); - -impl WorkloadCostEvidence { - fn validate(&self, env: &PhysicalDeploymentContext) -> Result<(), CompileError> { - if self.backend_revision != super::compiler::BACKEND_REVISION - || self.planner_revision != super::compiler::PLANNER_REVISION - { - return Err(invalid(format!( - "cost evidence compiler mismatch: measured backend/planner {}/{}; running {}/{}", - self.backend_revision, - self.planner_revision, - super::compiler::BACKEND_REVISION, - super::compiler::PLANNER_REVISION - ))); - } - if self.data_snapshot_id.trim().is_empty() - || self.model_version.trim().is_empty() - || self.valid_for_ms == 0 - || self.observed_at_unix_ms > env.observed_at_unix_ms - || env.observed_at_unix_ms - self.observed_at_unix_ms - > self.valid_for_ms.min(env.max_evidence_age_ms) - { - return Err(invalid("missing, future or stale evidence generation")); - } - Ok(()) - } - - pub(super) fn price( - &self, - manifest: &WorkloadCostManifest, - ) -> Result { - let quotes = self - .quotes - .iter() - .filter(|quote| "e.manifest == manifest) - .collect::>(); - if quotes.len() != 1 { - return Err(( - CandidateEvaluationStatus::EvidenceMissing, - "missing or ambiguous quote for exact manifest".into(), - )); - } - let quote = quotes[0]; - if !quote.executable { - return Err(( - CandidateEvaluationStatus::ProviderRejected, - "provider reports unavailable implementation".into(), - )); - } - if !quote.unit_costs.keys().eq(manifest.components.keys()) { - return Err(( - CandidateEvaluationStatus::EvidenceInvalid, - "incomplete or extraneous component evidence".into(), - )); - } - let mut total = 0.0; - let mut components = BTreeMap::new(); - for (id, demand) in &manifest.components { - let unit = quote.unit_costs[id]; - let cost = unit * demand.occurrences_per_horizon; - if !unit.is_finite() || unit < 0.0 || !cost.is_finite() || cost < 0.0 { - return Err(( - CandidateEvaluationStatus::EvidenceInvalid, - format!("invalid cost for {id}"), - )); - } - total += cost; - components.insert(id.clone(), cost); - } - if !total.is_finite() { - return Err(( - CandidateEvaluationStatus::EvidenceInvalid, - "cost overflow".into(), - )); - } - Ok((Cost(total), components)) - } -} - -fn candidate_description(candidate: &PhysicalCompilationRequest) -> CandidatePlanEvaluation { - let root_ids = candidate - .queries - .iter() - .map(|query| { - crate::planner_selection::explained_root_id( - &query.selected_plan_root, - &query.accuracy_target, - ) - }) - .collect::>(); - let complete = root_ids.iter().all(Option::is_some); - let mut logical_root_ids = root_ids.into_iter().flatten().collect::>(); - logical_root_ids.sort(); - logical_root_ids.dedup(); - CandidatePlanEvaluation { - candidate_id: complete.then(|| { - crate::planner_selection::explain_identity( - "alternative", - &( - &logical_root_ids, - candidate.allow_mixed_summary_and_exact_execution, - &candidate.enabled_materialization_keys, - ), - ) - }), - logical_root_ids, - identity_unavailable_reason: (!complete) - .then(|| "lossless canonical executable export unavailable for a logical root".into()), - physical_candidate_id: None, - status: CandidateEvaluationStatus::CompilationFailed, - plan_id: None, - total_cost: None, - unavailable_reason: None, - } -} - -fn compile_candidate_for_pricing( - candidate: PhysicalCompilationRequest, - env: PhysicalDeploymentContext, - frontend: super::compiler::QueryFrontend, -) -> Result< - ( - CompiledPhysicalPlan, - WorkloadCostManifest, - CandidatePlanEvaluation, - ), - Box, -> { - let mut description = candidate_description(&candidate); - let queries = candidate.queries.clone(); - let compiled = super::realization::RealizationProvider::compile( - &super::realization::ExistingRealizations, - candidate, - env, - frontend, - ); - let plan = match compiled { - Ok(plan) => plan, - Err(error) => { - description.unavailable_reason = Some(error.to_string()); - return Err(Box::new(description)); - } - }; - description.plan_id = Some(plan.envelope.plan_id); - // Reuse concrete catalog identities and selected window implementations. - // Deployment epochs/plan IDs and evidence prices do not identify semantics. - let mut bindings = plan - .lifecycle_estimates - .iter() - .map(|item| (item.materialization, item.window_realization_id.clone())) - .collect::>(); - bindings.sort(); - let mut placement = plan - .collector_plans - .iter() - .map(|collector| { - let mut states = collector - .materializations - .iter() - .map(|state| state.materialization) - .collect::>(); - states.sort(); - (&collector.collector_id, states) - }) - .collect::>(); - placement.sort(); - description.physical_candidate_id = - description - .candidate_id - .as_ref() - .map(|logical_candidate_id| { - crate::planner_selection::explain_identity( - "physical", - &( - logical_candidate_id, - bindings, - placement, - &plan.precompute_plan.ingest, - ), - ) - }); - - match manifest(&plan, &queries) { - Ok(manifest) => { - description.status = CandidateEvaluationStatus::AwaitingQuote; - Ok((plan, manifest, description)) - } - Err(error) => { - description.unavailable_reason = Some(error.to_string()); - Err(Box::new(description)) - } - } -} - -/// Preserve failed bindings alongside quoteable manifests. This does not select_lowest_cost_candidate or publish. -pub fn compile_candidates_for_pricing( - candidates: Vec, - env: PhysicalDeploymentContext, - frontend: super::compiler::QueryFrontend, -) -> (Vec, Vec) { - let mut manifests = Vec::new(); - let mut candidate_evaluations = Vec::new(); - for candidate in candidates { - match compile_candidate_for_pricing(candidate, env.clone(), frontend) { - Ok((_, manifest, description)) => { - manifests.push(manifest); - candidate_evaluations.push(description); - } - Err(description) => candidate_evaluations.push(*description), - } - } - (manifests, candidate_evaluations) -} - -/// Compare complete Planner-authorized forests after binding. Infeasible or -/// uncosted candidates are retained as unavailable, never assigned zero. -pub fn select_lowest_cost_candidate( - candidates: Vec, - env: PhysicalDeploymentContext, - evidence: &WorkloadCostEvidence, -) -> Result { - select_candidates( - candidates, - env, - evidence, - super::compiler::QueryFrontend::PromQl, - ) -} - -pub fn select_lowest_cost_metricsql_candidate( - candidates: Vec, - env: PhysicalDeploymentContext, - evidence: &WorkloadCostEvidence, -) -> Result { - select_candidates( - candidates, - env, - evidence, - super::compiler::QueryFrontend::MetricsQl, - ) -} - -fn select_candidates( - candidates: Vec, - env: PhysicalDeploymentContext, - evidence: &WorkloadCostEvidence, - frontend: super::compiler::QueryFrontend, -) -> Result { - evidence.validate(&env)?; - if candidates.is_empty() || candidates.len() > 64 { - return Err(invalid( - "candidate inventory must contain 1..=64 candidates", - )); - } - let candidate_key_sets: BTreeSet<_> = candidates - .iter() - .filter(|c| c.allow_mixed_summary_and_exact_execution) - .filter_map(|c| c.enabled_materialization_keys.clone()) - .collect(); - let eligible_keys: BTreeSet<_> = candidate_key_sets - .iter() - .flat_map(|p| p.iter().cloned()) - .collect(); - let materialization_search_coverage = - (!candidate_key_sets.is_empty()).then(|| MaterializationSearchCoverage { - eligible_materialization_count: eligible_keys.len(), - enumerated_candidate_key_sets: candidate_key_sets.len(), - exhaustive: eligible_keys.len() < usize::BITS as usize - && candidate_key_sets.len() == (1usize << eligible_keys.len()), - search_scope: CandidateSearchScope::PlannerAuthorizedMaterializations, - }); - let planner_selection_trace = candidates[0].planner_selection_trace.clone(); - let mut comparison_workload = None; - let mut candidate_evaluations = Vec::new(); - let mut best_index = 0; - let mut best: Option<( - Cost, - CompiledPhysicalPlan, - WorkloadCostManifest, - BTreeMap, - )> = None; - for candidate in candidates { - let (plan, manifest, mut description) = - match compile_candidate_for_pricing(candidate, env.clone(), frontend) { - Ok(bound) => bound, - Err(description) => { - candidate_evaluations.push(*description); - continue; - } - }; - let scope = (manifest.workload.clone(), manifest.horizon_seconds); - if comparison_workload - .as_ref() - .is_some_and(|previous| previous != &scope) - { - return Err(invalid("candidates describe different workloads/horizons")); - } - comparison_workload = Some(scope); - match super::realization::RealizationProvider::price( - &super::realization::ExistingRealizations, - evidence, - &manifest, - ) { - Ok((cost, components)) => { - description.status = CandidateEvaluationStatus::Unselected; - description.total_cost = Some(cost.0); - candidate_evaluations.push(description); - if best.as_ref().is_none_or(|(previous, ..)| cost < *previous) { - best_index = candidate_evaluations.len() - 1; - best = Some((cost, plan, manifest, components)); - } - } - Err((status, reason)) => { - description.status = status; - description.unavailable_reason = Some(reason); - candidate_evaluations.push(description); - } - } - } - let (_, mut plan, selected_manifest, component_costs) = best.ok_or_else(|| { - CompileError::Candidates( - json!({"status": "all_infeasible", "logical_selection": planner_selection_trace, - "candidates": candidate_evaluations}), - ) - })?; - // Exactly the winner retained by the existing strict-less-than selector. - candidate_evaluations[best_index].status = CandidateEvaluationStatus::Selected; - plan.cost_comparison = Some(CandidatePlanSelectionReport { - planner_selection_trace, - materialization_search_coverage, - data_snapshot_id: evidence.data_snapshot_id.clone(), - model_version: evidence.model_version.clone(), - selected_plan_id: plan.envelope.plan_id, - selected_manifest, - component_costs, - candidate_evaluations, - }); - Ok(plan) -} - -/// The current executor exposes continuously maintained state and the native -/// exact backend. Additional Planner-produced forests can use `select_lowest_cost_candidate` directly. -pub fn enumerate_exact_and_materialized_candidates( - request: PhysicalCompilationRequest, -) -> Result, CompileError> { - let already_selected = super::maintained_population::supported(&request); - let mut candidates = materialization_candidates(request)?; - if already_selected { - return Ok(candidates); - } - let roots: Vec<_> = candidates - .last() - .expect("exact alternative") - .queries - .iter() - .map(|q| match &q.selected_plan_root.expr { - planner_types::post_asap::SummaryExpr::KeepPreAsap(root) => std::rc::Rc::clone(root), - _ => unreachable!("native alternative retains canonical roots"), - }) - .collect(); - let strategy = - asap_aware_mapping::maintained_population::MaintainedPopulationStrategy::new(&roots); - let maintained_roots: Vec<_> = roots - .iter() - .map(|root| { - strategy - .candidate(root) - .filter(|node| super::maintained_population::supported_node(node)) - }) - .collect(); - if maintained_roots.iter().any(Option::is_some) { - // Current-series rules are compatible with window summaries in other - // workload roots. Preserve each priced temporal alternative and mask. - let maintained: Vec<_> = candidates - .iter() - .map(|alternative| { - let mut candidate = alternative.clone(); - for (query, selected) in candidate.queries.iter_mut().zip(&maintained_roots) { - if let Some(selected) = selected { - query.selected_plan_root = std::rc::Rc::clone(selected); - } - } - if maintained_roots.iter().all(Option::is_some) { - candidate.allow_mixed_summary_and_exact_execution = false; - candidate.enabled_materialization_keys = None; - } - candidate - }) - .collect(); - for candidate in maintained { - if !candidates.iter().any(|existing| { - existing.allow_mixed_summary_and_exact_execution - == candidate.allow_mixed_summary_and_exact_execution - && existing.enabled_materialization_keys - == candidate.enabled_materialization_keys - && existing - .queries - .iter() - .zip(&candidate.queries) - .all(|(a, b)| a.selected_plan_root == b.selected_plan_root) - }) { - candidates.push(candidate); - } - } - } - Ok(candidates) -} - -fn materialization_candidates( - request: PhysicalCompilationRequest, -) -> Result, CompileError> { - let mut exact = request.clone(); - exact.allow_mixed_summary_and_exact_execution = false; - exact.enabled_materialization_keys = None; - for (index, query) in exact.queries.iter_mut().enumerate() { - let parsed = if let Some(root) = request.canonical_roots.get(index) { - root.as_ref().clone() - } else { - crate::query_parser::parse_query_expr_canonical( - &query.query_string, - query.accuracy_target.clone(), - ) - .map_err(|error| invalid(error.to_string()))? - }; - query.selected_plan_root = crate::planner_selection::keep_pre_asap(&parsed) - .map_err(|error| invalid(error.to_string()))?; - } - if !request.allow_mixed_summary_and_exact_execution - && request - .queries - .iter() - .zip(&exact.queries) - .all(|(a, b)| a.selected_plan_root == b.selected_plan_root) - { - Ok(vec![request]) - } else { - if !request.allow_mixed_summary_and_exact_execution - || request.enabled_materialization_keys.is_some() - { - return Ok(vec![request, exact]); - } - let mut keys = BTreeSet::new(); - for query in &request.queries { - match crate::query_plan::query_time::eligible_materialization_keys( - &query.query_string, - &query.selected_plan_root, - ) { - Ok(found) => keys.extend(found), - // A failed local projection must not make the native alternative - // disappear. Compile/select_lowest_cost_candidate retains its concrete unavailability. - Err(_) => return Ok(vec![request, exact]), - } - } - if keys.is_empty() { - return Ok(vec![request, exact]); - } - let inventory = materialization_candidates::enumerate(keys); - debug_assert_eq!( - inventory.exhaustive, - inventory.eligible_materialization_count <= 4 - ); - let mut candidate_requests: Vec<_> = inventory - .candidate_key_sets - .into_iter() - .map(|enabled_keys| { - let mut candidate = request.clone(); - candidate.enabled_materialization_keys = Some(enabled_keys); - candidate - }) - .collect(); - candidate_requests.push(exact); - Ok(candidate_requests) - } -} - -#[cfg(test)] -mod tests { - use super::super::compiler::BackendLocalPlanningInput; - use super::*; - - fn fixture() -> BackendLocalPlanningInput { - let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - snapshot.query_workload.repeating_queries.as_mut().unwrap()[0].query = - planner_types::workload::Query("sum(sum_over_time(m[1m]))".into()); - snapshot - } - - /// Input uses one wire contract; removed aliases are not silently accepted. - #[test] - fn snapshot_rejects_removed_input_aliases() { - let value = serde_json::to_value(fixture()).unwrap(); - let decoded: BackendLocalPlanningInput = serde_json::from_value(value.clone()).unwrap(); - assert_eq!(serde_json::to_value(decoded).unwrap(), value); - for (canonical, removed) in [ - ("snapshot_version", "schema_version"), - ("implementation", "physical_inputs"), - ] { - let mut invalid = value.clone(); - let field = invalid.as_object_mut().unwrap().remove(canonical).unwrap(); - invalid[removed] = field; - assert!(serde_json::from_value::(invalid).is_err()); - } - } - - // Fractional demand and unavailable candidate costs preserve the current contract. - #[test] - fn demand_and_evaluation_use_candidate_wire_contracts() { - let old = json!({"implementation": {"op": "read"}, "unit": "query_evaluation", "multiplicity": 2.5}); - let demand: CostComponentDemand = serde_json::from_value(old.clone()).unwrap(); - assert_eq!(demand.pricing_basis, "query_evaluation"); - assert_eq!(demand.occurrences_per_horizon, 2.5); - assert_eq!(serde_json::to_value(demand).unwrap(), old); - let row = json!({ - "candidate_id": null, "physical_candidate_id": null, - "identity_unavailable_reason": null, "plan_id": null, - "total_cost": null, "unavailable_reason": null - }); - let evaluation: CandidatePlanEvaluation = serde_json::from_value(row).unwrap(); - assert_eq!(evaluation.status, CandidateEvaluationStatus::Unspecified); - let encoded = serde_json::to_value(evaluation).unwrap(); - assert_eq!(encoded["status"], ""); - assert!(encoded.get("candidate_id").is_some()); - assert!(encoded.get("alternative_id").is_none()); - } - - // IDs describe semantics; activation/version changes do not create new candidates. - #[test] - fn explain_identity_is_stable_across_activations_and_distinguishes_native() { - let (request, mut env) = fixture().into_physical_compilation_request().unwrap(); - let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); - let (_, first) = compile_candidates_for_pricing( - candidates.clone(), - env.clone(), - crate::physical::compiler::QueryFrontend::PromQl, - ); - env.plan_version += 1; - env.activation_unix_ms += 1; - let (_, second) = compile_candidates_for_pricing( - candidates, - env, - crate::physical::compiler::QueryFrontend::PromQl, - ); - assert_eq!(first.len(), second.len()); - for (a, b) in first.iter().zip(&second) { - assert!(a.candidate_id.is_some()); - assert!(a.physical_candidate_id.is_some()); - assert_eq!(a.candidate_id, b.candidate_id); - assert_eq!(a.physical_candidate_id, b.physical_candidate_id); - } - assert_ne!( - first.first().unwrap().candidate_id, - first.last().unwrap().candidate_id - ); - } - - // Bind failures remain visible even when the native manifest is usable. - #[test] - fn explain_retains_failed_bindings_and_all_missing_quotes() { - let (mut request, env) = fixture().into_physical_compilation_request().unwrap(); - request.allow_mixed_summary_and_exact_execution = false; - request.queries[0].window_realization_candidates.clear(); - let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); - let (manifests, explanations) = compile_candidates_for_pricing( - candidates, - env, - crate::physical::compiler::QueryFrontend::PromQl, - ); - assert_eq!(manifests.len(), 1); - assert_eq!(explanations.len(), 2); - assert_eq!( - explanations[0].status, - CandidateEvaluationStatus::CompilationFailed - ); - assert!(explanations[0].unavailable_reason.is_some()); - assert_eq!( - explanations[1].status, - CandidateEvaluationStatus::AwaitingQuote - ); - - let (candidates, env, mut evidence) = quoted(); - let count = candidates.len(); - evidence.quotes.clear(); - let CompileError::Candidates(report) = - select_lowest_cost_candidate(candidates, env, &evidence).unwrap_err() - else { - panic!("expected structured all-infeasible report") - }; - assert_eq!(report["status"], "all_infeasible"); - let candidates = report["candidates"].as_array().unwrap(); - assert_eq!(candidates.len(), count); - assert!(candidates - .iter() - .all(|item| item["status"] == "evidence_missing")); - assert!(!report["logical_selection"].as_array().unwrap().is_empty()); - } - - // Explanation records the same minimum quote selected by the existing algorithm. - #[test] - fn explain_links_selected_roots_without_changing_cost_choice() { - let (candidates, env, evidence) = quoted(); - let expected = evidence - .quotes - .iter() - .map(|quote| { - let cost = evidence.price("e.manifest).unwrap().0 .0; - (cost, quote.manifest.plan_id) - }) - .min_by(|a, b| a.0.total_cmp(&b.0)) - .unwrap(); - let plan = select_lowest_cost_candidate(candidates, env, &evidence).unwrap(); - assert_eq!(plan.envelope.plan_id, expected.1); - let comparison = plan.cost_comparison.unwrap(); - let selected = comparison - .candidate_evaluations - .iter() - .filter(|item| item.status == CandidateEvaluationStatus::Selected) - .collect::>(); - assert_eq!(selected.len(), 1); - assert_eq!(selected[0].total_cost, Some(expected.0)); - assert!(selected[0].candidate_id.is_some()); - assert!(selected[0].physical_candidate_id.is_some()); - assert_eq!( - comparison.planner_selection_trace, - plan.planner_selection_trace - ); - } - - #[test] - fn filtered_max_materializations_have_distinct_sds_populations() { - let mut snapshot = fixture(); - let entries = snapshot.query_workload.repeating_queries.as_mut().unwrap(); - entries[0].query = planner_types::workload::Query( - "max_over_time(service_retry_queue_depth{job=~\".+\"}[6h])".into(), - ); - entries[0].requirements.accuracy = planner_types::workload::AccuracyRequirement::Explicit( - planner_types::types::AccuracyTarget::Exact, - ); - let mut second = entries[0].clone(); - second.query = planner_types::workload::Query( - "max_over_time(service_retry_queue_depth{job=\"order-service\"}[6h])".into(), - ); - entries.push(second); - let (request, env) = snapshot.into_physical_compilation_request().unwrap(); - let plan = DeploymentPlanCompiler - .compile_promql(request.clone(), env) - .unwrap(); - let costs = manifest(&plan, &request.queries).unwrap(); - assert_eq!( - costs - .components - .keys() - .filter(|id| id.starts_with("state:")) - .count(), - 8 - ); - assert_eq!( - costs - .components - .keys() - .filter(|id| id.starts_with("raw-state:")) - .count(), - 0 - ); - assert_eq!( - plan.query_plan - .entries - .values() - .flat_map(|entry| entry.nodes.values()) - .filter(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ReadMaterialization { .. } - )) - .count(), - 2 - ); - } - - #[test] - fn materialization_masks_reject_mixed_snapshots_before_costing() { - let mut snapshot = fixture(); - let q = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - q.query = - planner_types::workload::Query("max_over_time(a[1m]) + max_over_time(b[1m])".into()); - q.requirements.accuracy = planner_types::workload::AccuracyRequirement::Explicit( - crate::types::AccuracyTarget::Exact, - ); - let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); - assert_eq!( - candidates.len(), - 5, - "four proposed candidate key sets plus native" - ); - let mut identities = BTreeSet::new(); - let mut rejected = 0; - for candidate in &candidates[..4] { - let enabled = candidate - .enabled_materialization_keys - .as_ref() - .unwrap() - .len(); - let result = - DeploymentPlanCompiler.compile_promql(candidate.clone(), environment.clone()); - if enabled == 1 { - let Err(error) = result else { - panic!("mixed snapshots must fail binding"); - }; - assert!( - error.to_string().contains("common snapshot proof"), - "{error}" - ); - rejected += 1; - continue; - } - let plan = result.unwrap(); - assert!(identities.insert(plan.envelope.plan_id)); - let cost = manifest(&plan, &candidate.queries).unwrap(); - assert_eq!( - cost.components - .keys() - .filter(|k| k.starts_with("state:backend:")) - .count(), - enabled * 4 - ); - assert_eq!( - cost.components - .keys() - .filter(|k| k.starts_with("raw-state:")) - .count(), - 0 - ); - assert_eq!( - cost.components - .values() - .filter(|v| v.pricing_basis == "horizon" - && v.implementation.get("location").and_then(Value::as_str) - == Some("exact_backend")) - .count(), - 2 - enabled - ); - assert!(!plan - .query_plan - .entries - .values() - .any(|entry| entry.nodes.values().any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ExactFallback { .. } - )))); - } - assert_eq!(rejected, 2); - assert_eq!(identities.len(), 2); - assert!( - !candidates - .last() - .unwrap() - .allow_mixed_summary_and_exact_execution - ); - } - - fn quoted() -> ( - Vec, - PhysicalDeploymentContext, - WorkloadCostEvidence, - ) { - let (request, env) = fixture().into_physical_compilation_request().unwrap(); - let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); - let quotes = candidates - .iter() - .map(|candidate| { - let plan = DeploymentPlanCompiler - .compile_promql(candidate.clone(), env.clone()) - .unwrap(); - let manifest = manifest(&plan, &candidate.queries).unwrap(); - let unit_costs = manifest - .components - .keys() - .map(|id| (id.clone(), 1.0)) - .collect(); - WorkloadQuote { - manifest, - executable: true, - unit_costs, - } - }) - .collect(); - let evidence = WorkloadCostEvidence { - backend_revision: crate::physical::compiler::BACKEND_REVISION.into(), - planner_revision: crate::physical::compiler::PLANNER_REVISION.into(), - data_snapshot_id: "fixture-data-v1".into(), - model_version: "test-only-unit-costs".into(), - observed_at_unix_ms: env.observed_at_unix_ms, - valid_for_ms: env.max_evidence_age_ms, - quotes, - }; - (candidates, env, evidence) - } - - // Retained local input is priced once per metric, separate from the native service. - #[test] - fn counter_materialization_manifest_prices_owned_state_and_distinct_native_alternative() { - use planner_types::workload::{AccuracyRequirement, Query}; - let mut snapshot = fixture(); - let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query("sum(rate(a{job=\"x\"}[1m])) / sum(rate(a{job!=\"x\"}[5m]))".into()); - entry.requirements.accuracy = - AccuracyRequirement::Explicit(crate::types::AccuracyTarget::Exact); - let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); - assert!(candidates.len() >= 2); - let local = DeploymentPlanCompiler - .compile_promql(candidates[0].clone(), environment.clone()) - .unwrap(); - let native = DeploymentPlanCompiler - .compile_promql(candidates.last().unwrap().clone(), environment) - .unwrap(); - assert_ne!(local.envelope.plan_id, native.envelope.plan_id); - let manifest = manifest(&local, &candidates[0].queries).unwrap(); - assert_eq!( - manifest - .components - .keys() - .filter(|key| key.starts_with("raw-state:a:")) - .count(), - 0 - ); - assert_eq!( - manifest - .components - .values() - .filter(|demand| demand.implementation.get("location") - == Some(&serde_json::json!("backend"))) - .count(), - 1 - ); - // Both ranges are summarized now, so nothing in this workload is - // priced as exact backend execution. The 5m operand used to land there - // only because its window had no implementation candidate. - assert!(!manifest - .components - .values() - .any(|demand| demand.implementation.get("location") - == Some(&serde_json::json!("exact_backend")))); - } - - // All input metrics need upkeep quotes; repeated reads share that upkeep. - #[test] - fn exact_manifest_covers_and_deduplicates_query_sources() { - for (query, expected) in [ - ( - "sum_over_time(m[1m]) + sum_over_time(n[1m])", - vec!["m", "n"], - ), - ("sum_over_time(m[1m]) + count_over_time(m[1m])", vec!["m"]), - ] { - let (mut request, env) = fixture().into_physical_compilation_request().unwrap(); - request.queries[0].query_string = query.into(); - request - .query_workload - .as_mut() - .unwrap() - .repeating_queries - .as_mut() - .unwrap()[0] - .query = planner_types::workload::Query(query.into()); - let exact = enumerate_exact_and_materialized_candidates(request) - .unwrap() - .pop() - .unwrap(); - let plan = DeploymentPlanCompiler - .compile_promql(exact.clone(), env.clone()) - .unwrap(); - let manifest = manifest(&plan, &exact.queries).unwrap(); - let sources: Vec<_> = manifest - .components - .iter() - .filter(|(id, _)| id.starts_with("source:")) - .map(|(_, demand)| { - demand.implementation["source"]["TimeSeries"]["metric"] - .as_str() - .unwrap() - }) - .collect(); - assert_eq!(sources, expected, "{query}"); - let mut evidence = WorkloadCostEvidence { - backend_revision: crate::physical::compiler::BACKEND_REVISION.into(), - planner_revision: crate::physical::compiler::PLANNER_REVISION.into(), - data_snapshot_id: "test-data".into(), - model_version: "test-model".into(), - observed_at_unix_ms: env.observed_at_unix_ms, - valid_for_ms: env.max_evidence_age_ms, - quotes: vec![WorkloadQuote { - unit_costs: manifest - .components - .keys() - .map(|id| (id.clone(), 1.0)) - .collect(), - manifest, - executable: true, - }], - }; - assert!( - select_lowest_cost_candidate(vec![exact.clone()], env.clone(), &evidence).is_ok() - ); - let source_id = evidence.quotes[0] - .unit_costs - .keys() - .rfind(|id| id.starts_with("source:")) - .unwrap() - .clone(); - evidence.quotes[0].unit_costs.remove(&source_id); - assert!( - select_lowest_cost_candidate(vec![exact], env, &evidence).is_err(), - "missing input upkeep must fail closed" - ); - } - } - - // Hidden or unresolved sources must not yield a partially priced manifest. - #[test] - fn exact_source_discovery_rejects_unresolved_inputs() { - let accuracy = fixture() - .into_physical_compilation_request() - .unwrap() - .0 - .queries[0] - .accuracy_target - .clone(); - for query in ["info(m)", "{job=\"api\"}"] { - let parsed = - crate::query_parser::parse_query_expr_canonical(query, accuracy.clone()).unwrap(); - assert!(exact_source_metrics(&parsed).is_err(), "{query}"); - } - } - - #[test] - fn complete_cost_changes_selection_and_reports_shared_work_once() { - let (candidates, env, mut evidence) = quoted(); - assert_ne!(evidence.quotes[0].manifest, evidence.quotes[1].manifest); - assert!(evidence.quotes[0] - .manifest - .components - .keys() - .any(|key| key.starts_with("state:"))); - assert!(!evidence.quotes[1] - .manifest - .components - .keys() - .any(|key| key.starts_with("state:"))); - for cost in evidence.quotes[1].unit_costs.values_mut() { - *cost = 1000.0; - } - let warm = - select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).unwrap(); - assert_eq!(warm.envelope.plan_id, evidence.quotes[0].manifest.plan_id); - let report = warm.cost_comparison.unwrap(); - assert_eq!( - report.component_costs.len(), - report.selected_manifest.components.len() - ); - assert_eq!(report.candidate_evaluations.len(), 2); - assert!(report - .candidate_evaluations - .iter() - .all(|a| a.total_cost.is_some())); - for (id, cost) in &mut evidence.quotes[0].unit_costs { - if id.ends_with(":residency") { - *cost = 1e9; - } - } - let raw = select_lowest_cost_candidate(candidates, env, &evidence).unwrap(); - assert_eq!(raw.envelope.plan_id, evidence.quotes[1].manifest.plan_id); - } - - #[test] - fn incomplete_unavailable_and_wrong_generation_quotes_are_not_free() { - let (candidates, env, mut evidence) = quoted(); - evidence.quotes[0].unit_costs.pop_first(); - let plan = - select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).unwrap(); - assert!(plan.cost_comparison.unwrap().candidate_evaluations[0] - .unavailable_reason - .is_some()); - evidence.quotes[1].executable = false; - assert!(select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).is_err()); - let (_, _, mut evidence) = quoted(); - evidence - .quotes - .iter_mut() - .for_each(|quote| quote.manifest.capability_snapshot_id.push_str("-wrong")); - assert!(select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).is_err()); - let (_, _, mut evidence) = quoted(); - evidence.observed_at_unix_ms = env.observed_at_unix_ms + 1; - assert!(select_lowest_cost_candidate(candidates, env, &evidence).is_err()); - } - - #[test] - fn evidence_from_a_different_compiler_build_is_rejected_before_matching_quotes() { - let (candidates, env, mut evidence) = quoted(); - evidence.backend_revision = "stale-backend-build".into(); - let error = select_lowest_cost_candidate(candidates, env, &evidence) - .unwrap_err() - .to_string(); - assert!(error.contains("cost evidence compiler mismatch"), "{error}"); - } - - #[test] - fn snapshot_requires_quotes_and_roundtrips_selection() { - let mut snapshot = fixture(); - assert!(snapshot.clone().compile_promql().is_err()); - let (_, _, evidence) = quoted(); - snapshot.workload_cost_evidence = Some(evidence); - let snapshot: BackendLocalPlanningInput = - serde_json::from_str(&serde_json::to_string(&snapshot).unwrap()).unwrap(); - assert!(snapshot.compile_promql().unwrap().cost_comparison.is_some()); - } - - #[test] - fn second_consumer_adds_reads_not_another_shared_state() { - let (request, env) = fixture().into_physical_compilation_request().unwrap(); - let first = manifest( - &DeploymentPlanCompiler - .compile_promql(request.clone(), env.clone()) - .unwrap(), - &request.queries, - ) - .unwrap(); - let mut shared = request.clone(); - let mut second = shared.queries[0].clone(); - second.query_id = "second-consumer".into(); - second.query_string = "sum(sum_over_time(m[1m])) * 2".into(); - let entries = shared - .query_workload - .as_mut() - .unwrap() - .repeating_queries - .as_mut() - .unwrap(); - let mut demand = entries[0].clone(); - demand.query = planner_types::workload::Query(second.query_string.clone()); - entries.push(demand); - shared.queries.push(second); - let roots = shared - .queries - .iter() - .map(|query| { - std::rc::Rc::new( - crate::query_parser::parse_query_expr_canonical( - &query.query_string, - query.accuracy_target.clone(), - ) - .unwrap(), - ) - }) - .collect(); - super::super::compiler::select_logical_roots_for_queries( - &mut shared.queries, - roots, - &shared.topk_membership_evidence_by_query_id, - &shared.exact_composition_costs, - ) - .unwrap(); - let plan = DeploymentPlanCompiler - .compile_promql(shared.clone(), env) - .unwrap(); - assert_eq!(plan.precompute_plan.materializations.len(), 1); - let second = manifest(&plan, &shared.queries).unwrap(); - let states = |m: &WorkloadCostManifest| { - m.components - .keys() - .filter(|key| key.starts_with("state:")) - .cloned() - .collect::>() - }; - assert_eq!(states(&first), states(&second)); - assert!(second.components.len() > first.components.len()); - } - - #[test] - fn exact_alternative_does_not_require_unused_state_implementation_evidence() { - let (candidates, env, evidence) = quoted(); - let mut exact = candidates[1].clone(); - assert_eq!( - enumerate_exact_and_materialized_candidates(exact.clone()) - .unwrap() - .len(), - 1 - ); - exact.queries[0].window_realization_candidates.clear(); - assert!(select_lowest_cost_candidate(vec![exact], env, &evidence).is_ok()); - } - - #[test] - fn altered_horizon_duplicate_and_invalid_costs_are_rejected() { - let (candidates, env, mut evidence) = quoted(); - evidence - .quotes - .iter_mut() - .for_each(|quote| quote.manifest.horizon_seconds += 1.0); - assert!(select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).is_err()); - let (_, _, mut evidence) = quoted(); - evidence.quotes.extend(evidence.quotes.clone()); - assert!(select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).is_err()); - let (_, _, mut evidence) = quoted(); - for quote in &mut evidence.quotes { - *quote.unit_costs.values_mut().next().unwrap() = -1.0; - } - assert!(select_lowest_cost_candidate(candidates, env, &evidence).is_err()); - } -} diff --git a/control_plane/src/physical/workload_cost/materialization_candidates.rs b/control_plane/src/physical/workload_cost/materialization_candidates.rs deleted file mode 100644 index 2344192f2..000000000 --- a/control_plane/src/physical/workload_cost/materialization_candidates.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! Bounded physical implementation search; semantic leaf legality is established -//! by the Planner-witness lowering before these stable keys are supplied. -use std::collections::BTreeSet; - -#[derive(Debug)] -pub(super) struct MaterializationCandidateSets { - pub candidate_key_sets: Vec>, - pub exhaustive: bool, - pub eligible_materialization_count: usize, -} - -/// Enumerate every enabled_keys for up to four leaves. Larger forests retain all-materialized, -/// all-exact, then singleton/complement pairs in stable key order. The caller must -/// disclose bounded coverage; no unenumerated optimum is claimed. Reserve one -/// of the selector's 64 candidate slots for native execution. -pub(super) fn enumerate(keys: BTreeSet) -> MaterializationCandidateSets { - let eligible_materialization_count = keys.len(); - let ordered: Vec<_> = keys.iter().cloned().collect(); - let exhaustive = eligible_materialization_count <= 4; - let mut candidate_key_sets = vec![keys.clone()]; - if exhaustive { - for bits in 0..(1usize << eligible_materialization_count) { - let enabled_keys = ordered - .iter() - .enumerate() - .filter(|(i, _)| bits & (1 << i) != 0) - .map(|(_, key)| key.clone()) - .collect(); - if !candidate_key_sets.contains(&enabled_keys) { - candidate_key_sets.push(enabled_keys); - } - } - } else { - candidate_key_sets.push(BTreeSet::new()); - for key in ordered { - for enabled_keys in [ - BTreeSet::from([key.clone()]), - keys.difference(&BTreeSet::from([key])).cloned().collect(), - ] { - if candidate_key_sets.len() >= 63 { - break; - } - if !candidate_key_sets.contains(&enabled_keys) { - candidate_key_sets.push(enabled_keys); - } - } - } - } - MaterializationCandidateSets { - candidate_key_sets, - exhaustive, - eligible_materialization_count, - } -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn small_forests_cover_every_mixed_path_once() { - let result = enumerate(BTreeSet::from(["a".into(), "b".into()])); - assert!(result.exhaustive); - assert_eq!(result.eligible_materialization_count, 2); - assert_eq!(result.candidate_key_sets.len(), 4); - assert_eq!( - result - .candidate_key_sets - .iter() - .collect::>() - .len(), - 4 - ); - assert!(result - .candidate_key_sets - .contains(&BTreeSet::from(["a".into()]))); - assert!(result - .candidate_key_sets - .contains(&BTreeSet::from(["b".into()]))); - } - #[test] - fn large_inventory_reserves_native_slot_and_discloses_truncation() { - let keys = (0..100).map(|i| format!("{i:03}")).collect(); - let result = enumerate(keys); - assert!(!result.exhaustive); - assert_eq!(result.candidate_key_sets.len(), 63); - assert_eq!(result.candidate_key_sets[0].len(), 100); - assert!(result.candidate_key_sets[1].is_empty()); - assert_eq!( - result - .candidate_key_sets - .iter() - .collect::>() - .len(), - 63 - ); - } - #[test] - fn no_materialization_has_one_exact_implementation() { - assert_eq!( - enumerate(BTreeSet::new()).candidate_key_sets, - vec![BTreeSet::new()] - ); - } -} diff --git a/control_plane/src/physical/workload_cost/status.rs b/control_plane/src/physical/workload_cost/status.rs deleted file mode 100644 index e73051e8a..000000000 --- a/control_plane/src/physical/workload_cost/status.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! Typed candidate diagnostics with the existing string wire representation. -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] -#[serde(from = "String", into = "String")] -pub enum CandidateEvaluationStatus { - #[default] - Unspecified, - CompilationFailed, - AwaitingQuote, - EvidenceMissing, - ProviderRejected, - EvidenceInvalid, - Unselected, - Selected, - /// Preserve diagnostics from other producer versions during migration. - Other(String), -} - -impl CandidateEvaluationStatus { - pub fn as_str(&self) -> &str { - match self { - Self::Unspecified => "", - Self::CompilationFailed => "bind_failed", - Self::AwaitingQuote => "bound", - Self::EvidenceMissing => "evidence_missing", - Self::ProviderRejected => "rejected", - Self::EvidenceInvalid => "evidence_invalid", - Self::Unselected => "unselected", - Self::Selected => "selected", - Self::Other(value) => value, - } - } -} - -impl From for CandidateEvaluationStatus { - fn from(value: String) -> Self { - match value.as_str() { - "" => Self::Unspecified, - "bind_failed" => Self::CompilationFailed, - "bound" => Self::AwaitingQuote, - "evidence_missing" => Self::EvidenceMissing, - "rejected" => Self::ProviderRejected, - "evidence_invalid" => Self::EvidenceInvalid, - "unselected" => Self::Unselected, - "selected" => Self::Selected, - _ => Self::Other(value), - } - } -} - -impl From for String { - fn from(value: CandidateEvaluationStatus) -> Self { - value.as_str().to_owned() - } -} - -// Preserve the existing report text, including historical terminology, on the -// wire. The typed variant describes the actual scope for new Rust consumers. -const MATERIALIZATION_SEARCH_SCOPE: &str = "Backend materialization versus Prometheus exact-subquery masks over Planner-authorized leaves; native alternative separate; bounded inventory does not claim an unenumerated optimum"; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(from = "String", into = "String")] -pub enum CandidateSearchScope { - PlannerAuthorizedMaterializations, - Other(String), -} - -impl From for CandidateSearchScope { - fn from(value: String) -> Self { - if value == MATERIALIZATION_SEARCH_SCOPE { - Self::PlannerAuthorizedMaterializations - } else { - Self::Other(value) - } - } -} - -impl From for String { - fn from(value: CandidateSearchScope) -> Self { - match value { - CandidateSearchScope::PlannerAuthorizedMaterializations => { - MATERIALIZATION_SEARCH_SCOPE.to_owned() - } - CandidateSearchScope::Other(value) => value, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Renaming diagnostic variants does not change known or future wire values. - #[test] - fn status_wire_values_round_trip() { - for value in [ - "", - "bind_failed", - "bound", - "evidence_missing", - "rejected", - "evidence_invalid", - "unselected", - "selected", - "future_status", - ] { - let status: CandidateEvaluationStatus = - serde_json::from_value(serde_json::json!(value)).unwrap(); - assert_eq!(status.as_str(), value); - assert_eq!(serde_json::to_value(status).unwrap(), value); - } - } - - /// Old search descriptions retain their exact serialized representation. - #[test] - fn scope_wire_values_round_trip() { - for value in [MATERIALIZATION_SEARCH_SCOPE, "future_scope"] { - let scope: CandidateSearchScope = - serde_json::from_value(serde_json::json!(value)).unwrap(); - assert_eq!(serde_json::to_value(scope).unwrap(), value); - } - } -} diff --git a/control_plane/src/planner_selection.rs b/control_plane/src/planner_selection.rs deleted file mode 100644 index 81ead342b..000000000 --- a/control_plane/src/planner_selection.rs +++ /dev/null @@ -1,932 +0,0 @@ -//! Deployment-owned selection at the latest ASAPPlanner boundary. -//! -//! ASAPPlanner enumerates a ranked candidate space and deliberately does not -//! commit to one deployment plan. The backend owns that decision because it -//! also owns placement, runtime capabilities, and the physical wire contract. - -use std::rc::Rc; - -use crate::physical::post_asap::cost_model::ControlPlaneCostModel; -use crate::types::AccuracyTarget; -use asap_aware_mapping::{ - AccuracyEvidenceProvider, AccuracyModel, CostModel, Replacement, ReplacementStrategy, - SketchAlgorithmStrategy, TargetSubDAG, -}; -use planner_types::post_asap::{ - SummaryExpr, SummaryFamilyType, SummaryField, SummaryNode, SummarySchema, -}; -use planner_types::pre_asap::{agg_accuracy as planner_agg_accuracy, AggIntent}; -use planner_types::pre_asap::{QueryExpr, QueryExprError}; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum SelectionError { - #[error("ASAPPlanner workload materialization failed: {0}")] - Workload(String), - #[error("failed to derive the pre-ASAP schema: {0}")] - Schema(#[from] QueryExprError), - #[error("ASAPPlanner produced no legal summary candidate for the target")] - NoLegalCandidate, - #[error("ASAPPlanner sketch strategy produced a logical rewrite instead of a summary")] - UnexpectedRewrite, -} - -/// Versioned diagnostic identity over existing canonical IR, never Rc or rank IDs. -/// Evidence/activation generations are reported separately from semantic identity. -pub(crate) fn explain_identity(kind: &str, value: &impl serde::Serialize) -> String { - use sha2::{Digest, Sha256}; - fn canonical(value: serde_json::Value) -> serde_json::Value { - match value { - serde_json::Value::Object(fields) => { - let sorted: std::collections::BTreeMap<_, _> = fields - .into_iter() - .map(|(key, value)| (key, canonical(value))) - .collect(); - serde_json::Value::Object(sorted.into_iter().collect()) - } - serde_json::Value::Array(values) => { - serde_json::Value::Array(values.into_iter().map(canonical).collect()) - } - value => value, - } - } - let value = canonical(serde_json::to_value(value).expect("canonical IR serializes")); - let bytes = serde_json::to_vec(&serde_json::json!({"kind": kind, - "planner_revision": crate::physical::compiler::PLANNER_REVISION, "value": value})) - .expect("JSON serializes"); - format!("asap-explain-v1:{kind}:{:x}", Sha256::digest(bytes)) -} - -// serde_json maps non-finite floats to null. A lossy encoding must never -// become a semantic identity, even when the runtime keeps an exact fallback. -fn lossless_json( - value: &T, -) -> Option { - let encoded = serde_json::to_value(value).ok()?; - let restored: T = serde_json::from_value(encoded.clone()).ok()?; - (restored == *value).then_some(encoded) -} - -fn target_identity(target: &QueryExpr, accuracy: &AccuracyTarget) -> Option { - Some(explain_identity( - "target", - &(lossless_json(target)?, lossless_json(accuracy)?), - )) -} - -fn summary_identity(node: &SummaryNode) -> Option { - // Canonical exporter owns operator payloads and edge semantics. Hash its - // structure, not assigned node IDs or the incidental sharing of Rc values. - let dag = planner_types::post_asap::compile_executable_dag(&Rc::new(node.clone())).ok()?; - lossless_json(&dag)?; - fn visit( - dag: &planner_types::post_asap::ExecutableDag, - id: planner_types::post_asap::PostAsapNodeId, - memo: &mut std::collections::HashMap, - ) -> String { - if let Some(hash) = memo.get(&id) { - return hash.clone(); - } - let node = dag - .nodes - .iter() - .find(|node| node.id == id) - .expect("exported node exists"); - let mut inputs = dag.edges.iter().filter(|edge| edge.consumer == id).map(|edge| { - let child = visit(dag, edge.producer, memo); - serde_json::json!({"child": child, "role": edge.role, "schema": edge.intermediate_schema, - "state": edge.data_state, "grouping": edge.grouping, "window": edge.window}) - }).collect::>(); - inputs.sort_by_cached_key(|input| input.to_string()); - let hash = explain_identity( - "summary_node", - &serde_json::json!({ - "payload": node.payload, "state": node.output_state, - "schema": node.output_schema, "guarantee": node.guarantee, "inputs": inputs}), - ); - memo.insert(id, hash.clone()); - hash - } - Some(visit(&dag, dag.root, &mut std::collections::HashMap::new())) -} - -pub(crate) fn explained_root_id(node: &SummaryNode, accuracy: &AccuracyTarget) -> Option { - Some(explain_identity( - "root", - &(summary_identity(node)?, lossless_json(accuracy)?), - )) -} - -fn replacement_identity( - target: &QueryExpr, - replacement: &Replacement, - accuracy: &AccuracyTarget, -) -> Option { - let target = lossless_json(target)?; - let accuracy = lossless_json(accuracy)?; - let value = match replacement { - Replacement::Summary(node) => serde_json::json!({"summary": summary_identity(node)?}), - Replacement::Rewrite(node) => serde_json::json!({"rewrite": lossless_json(node.as_ref())?}), - Replacement::ExactComposition(composition) => serde_json::json!({ - "exact_composition": {"placement": format!("{:?}", composition.placement), - "operation": lossless_json(&composition.op)?, "child": lossless_json(composition.child_target.as_ref())?, "schema": lossless_json(&composition.schema)?}}), - }; - Some(explain_identity("candidate", &(target, accuracy, value))) -} - -/// Deployment extension tag for keyed point-frequency queries. ASAPPlanner -/// intentionally treats extension payloads as opaque; this adapter is the one -/// backend-owned interpretation point. -pub(crate) const FREQUENCY_EXT_KIND: &str = "frequency"; - -pub fn frequency(accuracy: AccuracyTarget, item: Option<(String, String)>) -> AggIntent { - let mut payload = serde_json::json!({ "accuracy": accuracy }); - if let Some((label, value)) = item { - payload["item_label"] = serde_json::Value::String(label); - payload["item_value"] = serde_json::Value::String(value); - } - AggIntent::Extension { - ext_kind: FREQUENCY_EXT_KIND.to_string(), - payload, - } -} - -pub fn default_frequency() -> AggIntent { - frequency(AccuracyTarget::Epsilon(std::f64::consts::E / 2000.0), None) -} - -pub fn as_frequency(intent: &AggIntent) -> Option { - match intent { - AggIntent::Extension { ext_kind, payload } if ext_kind == FREQUENCY_EXT_KIND => { - serde_json::from_value(payload.get("accuracy")?.clone()).ok() - } - _ => None, - } -} - -pub fn agg_accuracy(intent: &AggIntent) -> f64 { - match as_frequency(intent) { - Some(AccuracyTarget::Exact) => 0.0, - Some(AccuracyTarget::Epsilon(epsilon)) - | Some(AccuracyTarget::EpsilonDelta { epsilon, .. }) => epsilon, - None => planner_agg_accuracy(intent), - } -} - -pub fn archive_only(intent: &AggIntent) -> bool { - if as_frequency(intent).is_some() { - return false; - } - matches!( - intent, - AggIntent::Absent - | AggIntent::AbsentOverTime - | AggIntent::PresentOverTime - | AggIntent::Delta - | AggIntent::Deriv - | AggIntent::PredictLinear { .. } - | AggIntent::DoubleExpSmoothing { .. } - | AggIntent::IDelta - | AggIntent::Resets - | AggIntent::Changes - | AggIntent::HistogramCount - | AggIntent::HistogramSum - | AggIntent::HistogramAvg - | AggIntent::HistogramStdDev - | AggIntent::HistogramStdVar - | AggIntent::HistogramFraction { .. } - | AggIntent::HistogramQuantile { .. } - | AggIntent::Math(_) - | AggIntent::TimeFn(_) - | AggIntent::Group - | AggIntent::CountValues { .. } - | AggIntent::LastOverTime - | AggIntent::FirstOverTime - | AggIntent::MadOverTime - | AggIntent::TsOfMinOverTime - | AggIntent::TsOfMaxOverTime - | AggIntent::TsOfFirstOverTime - | AggIntent::TsOfLastOverTime - | AggIntent::Extension { .. } - ) -} - -/// Preserve an unsupported subtree explicitly at the post-ASAP boundary. -pub fn keep_pre_asap(expr: &QueryExpr) -> Result, SelectionError> { - let schema = expr.output_schema()?; - Ok(Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(Rc::new(expr.clone())), - schema: SummarySchema { - fields: schema - .columns - .into_iter() - .map(|column| SummaryField { - name: column.name, - dtype: SummaryFamilyType::Plain(column.dtype), - nullable: column.nullable, - }) - .collect(), - time_index: schema.time_index, - }, - guarantee: None, - })) -} - -/// Select a complete query through the same costed Planner search as workloads. -/// Per-operator requirements remain in the canonical expression. -pub fn select_query( - expr: &QueryExpr, - cost_model: &dyn CostModel, -) -> Result, SelectionError> { - select_query_with_models( - expr, - cost_model, - &asap_aware_mapping::DefaultAccuracyModel, - &asap_aware_mapping::NoAccuracyEvidence, - ) -} - -#[cfg(test)] -pub(crate) fn plan_test_query(expr: &QueryExpr) -> Result, SelectionError> { - select_query( - expr, - &ControlPlaneCostModel::new(AccuracyTarget::Epsilon(0.01)), - ) -} - -#[cfg(test)] -/// Search a same-requirement workload cohort through Planner's canonical CSE -/// and replacement inventory. Physical implementation compatibility is checked -/// later, before publication; this function never assigns runtime identities. -pub fn select_workload( - roots: Vec<(usize, Rc)>, - accuracy: AccuracyTarget, - cost_model: &dyn CostModel, -) -> Result)>, SelectionError> { - select_workload_with_evidence( - roots, - accuracy, - cost_model, - &asap_aware_mapping::NoAccuracyEvidence, - ) -} - -#[cfg(test)] -/// The entire cohort uses the same scoped accuracy certificate; callers must -/// not spread one query's evidence to unrelated workload roots. -pub fn select_workload_with_evidence( - roots: Vec<(usize, Rc)>, - accuracy: AccuracyTarget, - cost_model: &dyn CostModel, - evidence: &dyn AccuracyEvidenceProvider, -) -> Result)>, SelectionError> { - select_workload_with_accuracy_model( - roots, - accuracy, - cost_model, - evidence, - &asap_aware_mapping::DefaultAccuracyModel, - ) -} - -#[cfg(test)] -/// Keep replacement legality and workload-root validation on the same model. -pub fn select_workload_with_accuracy_model( - roots: Vec<(usize, Rc)>, - accuracy: AccuracyTarget, - cost_model: &dyn CostModel, - evidence: &dyn AccuracyEvidenceProvider, - accuracy_model: &dyn AccuracyModel, -) -> Result)>, SelectionError> { - select_workload_impl( - roots, - accuracy, - cost_model, - evidence, - accuracy_model, - None, - None, - ) -} - -/// Return the candidate ranking and committed choices from the same search -/// that produces the installed roots. Missing numeric costs remain explicit. -pub fn select_workload_with_accuracy_model_and_trace( - roots: Vec<(usize, Rc)>, - accuracy: AccuracyTarget, - cost_model: &ControlPlaneCostModel, - evidence: &dyn AccuracyEvidenceProvider, - accuracy_model: &dyn AccuracyModel, -) -> Result<(Vec<(usize, Rc)>, serde_json::Value), SelectionError> { - let mut trace = serde_json::Value::Null; - let selected = select_workload_impl( - roots, - accuracy, - cost_model, - evidence, - accuracy_model, - Some(&mut trace), - Some(cost_model), - )?; - Ok((selected, trace)) -} - -/// The [`ReplacementStrategy`] set this deployment registers, carrying its own -/// cost and accuracy models. This is deliberately not Planner's -/// `default_strategies_with`: two of the strategies that list would give us are -/// excluded below, for two unrelated reasons. -/// -/// The three workload-dependent strategies — `RollupStrategy`, -/// `AccuracyReconciliationStrategy` and `TopKLimitReuseStrategy` — are -/// deliberately absent and must stay absent: they are constructed from the -/// post-CSE sibling set, which only `search_cse_workload_with` owns, so -/// `search_workload_with_targets` registers them itself against every -/// discovered target. Adding them here would ask each target with an empty -/// sibling list and report nothing. -/// -/// `SharedSubtreeStrategy` is withheld for a reason the pinned Planner has to -/// fix first, not as a cost policy this deployment could express. Both of its -/// arms are `Replacement::Rewrite`s of the target itself, and `rank_group` -/// ranks that pair (its "Shape 1") ahead of every `Replacement::Summary` in -/// the same group and returns before the sketch-family ranking runs. The -/// winning rewrite then materializes as `KeepPreAsap`, so registering the -/// strategy silently downgrades every shared aggregate from its selected -/// sketch to raw execution — `shared_aggregates_keep_their_summary` pins -/// exactly that. Sharing must decide how many copies of the summary state -/// exist, never whether the state is a summary at all; until Planner can -/// carry the selected summary through the share rewrite, canonical CSE -/// inside `search_workload_with_targets` already shares these subtrees -/// structurally and correctly. -fn replacement_strategies<'a>( - cost_model: &'a dyn CostModel, - evidence: &'a dyn AccuracyEvidenceProvider, - accuracy_model: &'a dyn AccuracyModel, -) -> Vec> { - vec![ - Box::new( - SketchAlgorithmStrategy::new_with_planning_inputs_and_evidence( - cost_model, - accuracy_model, - &asap_aware_mapping::EqualSplitAllocator, - evidence, - ), - ), - Box::new( - asap_aware_mapping::HydraGroupingStrategy::new_with_planning_inputs_and_evidence( - cost_model, - accuracy_model, - &asap_aware_mapping::EqualSplitAllocator, - evidence, - ), - ), - Box::new(asap_aware_mapping::ExactCompositionStrategy::new( - cost_model, - )), - Box::new(asap_aware_mapping::SemanticEquivalentRewriteStrategy), - ] -} - -fn select_workload_impl( - roots: Vec<(usize, Rc)>, - accuracy: AccuracyTarget, - cost_model: &dyn CostModel, - evidence: &dyn AccuracyEvidenceProvider, - accuracy_model: &dyn AccuracyModel, - mut trace: Option<&mut serde_json::Value>, - backend_cost_model: Option<&ControlPlaneCostModel>, -) -> Result)>, SelectionError> { - let strategies = replacement_strategies(cost_model, evidence, accuracy_model); - let space = asap_aware_mapping::search_workload_with_targets( - roots - .into_iter() - .map(|(id, root)| (id, root, Some(accuracy.clone()))) - .collect(), - &strategies, - accuracy_model, - ); - let selection = space.global_selection(cost_model); - if let Some(trace) = trace.as_deref_mut() { - let groups = space.cost_sorted(cost_model).iter().enumerate().map(|(index, group)| { - let chosen = selection.target_selections().find(|selected| Rc::ptr_eq(selected.target, group.target)) - .and_then(|selected| selected.chosen); - let target = TargetSubDAG::with_consumer_count(group.target, group.consumer_count); - let candidates = group.candidates.iter().enumerate() - .map(|(rank, candidate)| { - let candidate_cost = cost_model.candidate_cost(candidate, &target); - let accuracy_status = if candidate.has_missing_accuracy_evidence() { - "unknown" - } else { - "known" - }; - let runtime_support_status = match candidate.runtime_support_evidence(cost_model) { - Some(true) => "supported", - Some(false) => "unsupported", - None => "unknown_pending_backend_binding", - }; - let decision_reason = if candidate.has_missing_accuracy_evidence() { - "missing_accuracy_evidence" - } else if candidate.runtime_support_evidence(cost_model) == Some(false) { - "unsupported_runtime_operation" - } else if chosen.is_some_and(|chosen| std::ptr::eq(chosen, *candidate)) { - "planner_selected_pending_backend_binding" - } else if candidate_cost.is_none() { - "missing_comparable_cost" - } else { - "not_selected_by_planner" - }; - let guarantee = match &candidate.replacement { - Replacement::Summary(node) => node.guarantee.as_ref(), - _ => None, - }; - serde_json::json!({ - "rank": rank, - "candidate_id": replacement_identity(group.target, &candidate.replacement, &accuracy), - "status": if chosen.is_some_and(|chosen| std::ptr::eq(chosen, *candidate)) { "selected" } else { "unselected" }, - "strategy": candidate.strategy, - "provenance": format!("{:?}", candidate.provenance), - "rationale": candidate.rationale, - "replacement_kind": match &candidate.replacement { - Replacement::Summary(_) => "summary", - Replacement::Rewrite(_) => "rewrite", - Replacement::ExactComposition(_) => "exact_composition", - }, - "accuracy_status": accuracy_status, - "guarantee": guarantee, - "runtime_support_status": runtime_support_status, - "decision_reason": decision_reason, - "estimated_cost": candidate_cost.map(|cost| cost.0), - "estimated_cost_status": if candidate_cost.is_some() { "available" } else { "unavailable" }, - "cost_estimate": backend_cost_model.and_then(|model| model.candidate_cost_estimate(candidate)), - "selected": chosen.is_some_and(|chosen| std::ptr::eq(chosen, *candidate)), - })}).collect::>(); - let rejected = space.target_subdag_candidates().find(|memo| Rc::ptr_eq(&memo.target, group.target)) - .into_iter().flat_map(|memo| &memo.rejected).map(|candidate| serde_json::json!({ - "status": "rejected", "strategy": candidate.strategy, - "description": candidate.description, "reason": candidate.error.to_string() - })).collect::>(); - serde_json::json!({ "group_id": index, - "target_id": target_identity(group.target, &accuracy), - "consumer_count": group.consumer_count, "candidates": candidates, "rejected": rejected }) - }).collect::>(); - *trace = serde_json::json!({ "schema_version": 1, "group_id_scope": "this_selection", "groups": groups }); - } - let roots = space - .roots - .iter() - .map(|(id, root)| { - selection - .assemble_selected_query(root) - .map_err(|error| SelectionError::Workload(error.to_string()))? - .map(|node| (*id, node)) - .ok_or_else(|| SelectionError::Workload(format!("missing query root {id}"))) - }) - .collect::, _>>()?; - let roots = planner_types::post_asap::share_common_summary_subtrees(roots); - if let Some(trace) = trace { - trace["roots"] = serde_json::json!(roots - .iter() - .map(|(id, node)| serde_json::json!({ - "query_index": id, "logical_root_id": explained_root_id(node, &accuracy) - })) - .collect::>()); - } - Ok(roots) -} - -/// Evidence and accuracy hooks feed canonical global selection; enumeration -/// order never authorizes a query plan. -pub fn select_query_with_models( - expr: &QueryExpr, - cost_model: &dyn CostModel, - accuracy_model: &dyn AccuracyModel, - evidence: &dyn AccuracyEvidenceProvider, -) -> Result, SelectionError> { - let strategies = replacement_strategies(cost_model, evidence, accuracy_model); - let space = asap_aware_mapping::search_workload_with_targets( - vec![(0, Rc::new(expr.clone()), None)], - &strategies, - accuracy_model, - ); - space - .global_selection(cost_model) - .assemble_selected_query(&space.roots[0].1) - .map_err(|error| SelectionError::Workload(error.to_string()))? - .ok_or(SelectionError::NoLegalCandidate) -} - -#[cfg(test)] -mod workload_tests { - use super::*; - use crate::physical::post_asap::cost_model::ControlPlaneCostModel; - - fn plan(queries: &[&str], accuracy: AccuracyTarget) -> Vec<(usize, Rc)> { - let roots = queries - .iter() - .enumerate() - .map(|(index, query)| { - ( - index, - Rc::new( - crate::query_parser::parse_query_expr_canonical(query, accuracy.clone()) - .unwrap(), - ), - ) - }) - .collect(); - select_workload( - roots, - accuracy.clone(), - &ControlPlaneCostModel::new(accuracy), - ) - .unwrap() - } - - // Allocation identities and ranking ordinals are not semantic candidate identities. - #[test] - fn explain_candidate_ids_survive_reparse_and_preserve_accuracy() { - fn trace(accuracy: AccuracyTarget) -> serde_json::Value { - let root = crate::query_parser::parse_query_expr_canonical( - "quantile_over_time(0.9, m[1m])", - accuracy.clone(), - ) - .unwrap(); - select_workload_with_accuracy_model_and_trace( - vec![(0, Rc::new(root))], - accuracy.clone(), - &ControlPlaneCostModel::new(accuracy), - &asap_aware_mapping::NoAccuracyEvidence, - &asap_aware_mapping::DefaultAccuracyModel, - ) - .unwrap() - .1 - } - let a = trace(AccuracyTarget::Epsilon(0.05)); - let b = trace(AccuracyTarget::Epsilon(0.05)); - assert_eq!(a, b); - assert!(a["groups"] - .as_array() - .unwrap() - .iter() - .flat_map(|group| group["candidates"].as_array().unwrap()) - .any(|candidate| candidate["candidate_id"].is_string())); - assert_ne!( - a["roots"][0]["logical_root_id"], - trace(AccuracyTarget::Epsilon(0.1))["roots"][0]["logical_root_id"] - ); - } - - /// Explain availability from the candidate-cost API, even if a display - /// estimate exists, and never label an unknown guarantee as selected. - #[test] - fn explain_keeps_uncertified_and_uncosted_candidates_explicit() { - struct Uncosted; - impl CostModel for Uncosted { - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[planner_types::post_asap::SketchAlgorithm], - ) -> Vec { - candidates.to_vec() - } - - fn candidate_cost( - &self, - _candidate: &asap_aware_mapping::ReplacementSubDAG, - _target: &TargetSubDAG<'_>, - ) -> Option { - None - } - } - - let accuracy = AccuracyTarget::Epsilon(0.05); - let root = crate::query_parser::parse_query_expr_canonical( - "quantile_over_time(0.9,m[1m]) / quantile_over_time(0.5,m[1m])", - accuracy.clone(), - ) - .unwrap(); - let mut trace = serde_json::Value::Null; - select_workload_impl( - vec![(0, Rc::new(root))], - accuracy, - &Uncosted, - &asap_aware_mapping::NoAccuracyEvidence, - &asap_aware_mapping::DefaultAccuracyModel, - Some(&mut trace), - None, - ) - .unwrap(); - let candidates = trace["groups"] - .as_array() - .unwrap() - .iter() - .flat_map(|group| group["candidates"].as_array().unwrap()); - let mut saw_unknown = false; - for candidate in candidates { - assert_eq!(candidate["estimated_cost_status"], "unavailable"); - assert!(candidate["estimated_cost"].is_null()); - if candidate["accuracy_status"] == "unknown" { - saw_unknown = true; - assert_eq!(candidate["selected"], false); - } - } - assert!(saw_unknown); - } - - // JSON must not alias NaN and infinity through its null representation. - #[test] - fn explain_nonfinite_identity_is_unavailable() { - for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { - let root = QueryExpr::Literal(planner_types::pre_asap::ScalarValue::Float64(value)); - assert!(replacement_identity( - &root, - &Replacement::Rewrite(Rc::new(root.clone())), - &AccuracyTarget::Exact - ) - .is_none()); - } - } - - // Hashing excludes incidental allocation sharing but retains operand roles. - #[test] - fn explain_summary_identity_preserves_roles_and_ignores_rc_sharing() { - let root = plan( - &["sum_over_time(m[1m]) - sum_over_time(n[1m])"], - AccuracyTarget::Exact, - )[0] - .1 - .clone(); - let id = summary_identity(&root).expect("canonical binary exports"); - let mut reversed = (*root).clone(); - let SummaryExpr::BinaryOp { lhs, rhs, .. } = &mut reversed.expr else { - panic!("binary expected") - }; - std::mem::swap(lhs, rhs); - assert_ne!(Some(id), summary_identity(&reversed)); - let root = plan( - &["sum_over_time(m[1m]) + sum_over_time(m[1m])"], - AccuracyTarget::Exact, - )[0] - .1 - .clone(); - let id = summary_identity(&root).expect("canonical shared binary exports"); - let mut unshared = (*root).clone(); - let SummaryExpr::BinaryOp { rhs, .. } = &mut unshared.expr else { - panic!("binary expected") - }; - *rhs = Rc::new((**rhs).clone()); - assert_eq!(Some(id), summary_identity(&unshared)); - } - - // Distinct quantile roots retain their readouts while sharing one selected sketch. - #[test] - fn quantile_roots_share_selected_producer() { - let roots = plan( - &[ - "quantile_over_time(0.90, m[1m])", - "quantile_over_time(0.99, m[1m])", - ], - AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }, - ); - let SummaryExpr::SummaryEstimate { - summary_input: first, - query: q1, - } = &roots[0].1.expr - else { - panic!("{:?}", roots[0].1) - }; - let SummaryExpr::SummaryEstimate { - summary_input: second, - query: q2, - } = &roots[1].1.expr - else { - panic!("{:?}", roots[1].1) - }; - assert!(Rc::ptr_eq(first, second)); - assert_ne!(q1, q2); - } - - // Sharing must not collapse different source or logical-window requirements. - #[test] - fn distinct_windows_and_sources_do_not_share() { - let roots = plan( - &[ - "sum_over_time(m[1m])", - "sum_over_time(m[2m])", - "sum_over_time(n[1m])", - ], - AccuracyTarget::Exact, - ); - assert!(!Rc::ptr_eq(&roots[0].1, &roots[1].1)); - assert!(!Rc::ptr_eq(&roots[0].1, &roots[2].1)); - } - - // Arithmetic keeps its exact operands visible and reuses the standalone sum. - #[test] - fn weighted_mean_retains_shared_sum_operand() { - let roots = plan( - &[ - "sum_over_time(m[1m])", - "sum_over_time(m[1m]) / count_over_time(m[1m])", - ], - AccuracyTarget::Exact, - ); - let SummaryExpr::BinaryOp { lhs, .. } = &roots[1].1.expr else { - panic!("{:?}", roots[1].1) - }; - let shared = match &lhs.expr { - SummaryExpr::ValueOperation { - child, - operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - .. - } => child, - _ => lhs, - }; - let SummaryExpr::ValueOperation { - child: standalone, - operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - .. - } = &roots[0].1.expr - else { - panic!("standalone query must finalize its shared state") - }; - assert!(Rc::ptr_eq(standalone, shared)); - } - - // The registered set is exactly the four strategies this deployment - // supports, in discovery order — which is also the tie-break order - // `rank_group` falls back to. Both exclusions are load-bearing: - // `SharedSubtreeStrategy` would downgrade shared aggregates to raw - // execution (see `shared_aggregates_keep_their_summary`), and the - // workload-dependent three are already registered by - // `search_cse_workload_with`, which alone owns their sibling set. - #[test] - fn registered_strategies_are_exactly_the_supported_set() { - let accuracy = AccuracyTarget::Epsilon(0.01); - let cost_model = ControlPlaneCostModel::new(accuracy); - let names: Vec<&str> = replacement_strategies( - &cost_model, - &asap_aware_mapping::NoAccuracyEvidence, - &asap_aware_mapping::DefaultAccuracyModel, - ) - .iter() - .map(|strategy| strategy.name()) - .collect(); - assert_eq!( - names, - [ - "SketchAlgorithmStrategy", - "HydraGroupingStrategy", - "ExactCompositionStrategy", - "SemanticEquivalentRewriteStrategy", - ] - ); - } - - // Missing shared-grid evidence keeps Hydra candidates visible but - // uncertified. Supplying a certificate makes them eligible for selection. - #[test] - fn hydra_candidates_remain_visible_without_shared_grid_evidence() { - use asap_aware_mapping::{AccuracyEvidenceProvider, PropagationStats}; - use planner_types::post_asap::{CompositionOperator, SketchQuery}; - use planner_types::pre_asap::query_expr::Source; - use planner_types::pre_asap::{Column, DataType, GroupKeys, Reduction, Schema}; - - struct MeasuredSharedGrid; - impl AccuracyEvidenceProvider for MeasuredSharedGrid { - fn propagation_stats( - &self, - _op: &CompositionOperator, - _family: &SummaryFamilyType, - _query: Option<&SketchQuery>, - ) -> PropagationStats { - PropagationStats { - hydra_shared_grid_collision_bound: Some(0.0), - hydra_shared_grid_failure_probability: Some(0.0), - ..Default::default() - } - } - } - - let accuracy = AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }; - let grouped_count = Rc::new(QueryExpr::Aggregate { - reduction: Reduction::Reduce(GroupKeys::by(vec![2])), - measures: vec![AggIntent::Count { - accuracy: accuracy.clone(), - }], - output_names: vec![], - having: None, - child: Rc::new(QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Column::new("ts", DataType::Timestamp, false), - Column::new("value", DataType::Float64, false), - Column::new("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - }), - }); - let cost_model = ControlPlaneCostModel::new(accuracy); - let target = TargetSubDAG::new(&grouped_count); - let hydra = |evidence: &dyn AccuracyEvidenceProvider| { - asap_aware_mapping::HydraGroupingStrategy::new_with_planning_inputs_and_evidence( - &cost_model, - &asap_aware_mapping::DefaultAccuracyModel, - &asap_aware_mapping::EqualSplitAllocator, - evidence, - ) - .replacements(&target) - }; - let unknown = hydra(&asap_aware_mapping::NoAccuracyEvidence); - assert_eq!(unknown.len(), 2); - assert!(unknown - .iter() - .all(|candidate| candidate.has_missing_accuracy_evidence())); - // HydraCms over Cms and HydraCountSketch over CountSketch. - let certified = hydra(&MeasuredSharedGrid); - assert_eq!(certified.len(), 2); - assert!(certified - .iter() - .all(|candidate| !candidate.has_missing_accuracy_evidence())); - } - - // A shared aggregate must keep the sketch plan an unshared one gets: - // sharing decides how many copies of the state exist, never whether the - // state is a summary at all. - #[test] - fn shared_aggregates_keep_their_summary() { - use planner_types::pre_asap::query_expr::Source; - use planner_types::pre_asap::{Column, DataType, GroupKeys, Reduction, Schema}; - - let accuracy = AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }; - let grouped_count = || { - Rc::new(QueryExpr::Aggregate { - reduction: Reduction::Reduce(GroupKeys::by(vec![2])), - measures: vec![AggIntent::Count { - accuracy: accuracy.clone(), - }], - output_names: vec![], - having: None, - child: Rc::new(QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Column::new("ts", DataType::Timestamp, false), - Column::new("value", DataType::Float64, false), - Column::new("job", DataType::Utf8, true), - ], - 0, - vec![vec![2]], - ), - }), - }) - }; - let summarized = |roots: Vec<(usize, Rc)>| -> Vec { - select_workload( - roots, - accuracy.clone(), - &ControlPlaneCostModel::new(accuracy.clone()), - ) - .unwrap() - .into_iter() - .map(|(_, node)| { - fn is_summary(node: &SummaryNode) -> bool { - match &node.expr { - SummaryExpr::SummaryAgg { .. } => true, - SummaryExpr::SummaryEstimate { summary_input, .. } => { - is_summary(summary_input) - } - SummaryExpr::ValueOperation { - child, - operation: - planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - .. - } => is_summary(child), - _ => false, - } - } - is_summary(&node) - }) - .collect() - }; - assert_eq!(summarized(vec![(0, grouped_count())]), [true]); - assert_eq!( - summarized(vec![(0, grouped_count()), (1, grouped_count())]), - [true, true], - "sharing must not downgrade a summarized aggregate to raw execution" - ); - } -} diff --git a/control_plane/src/query_parser/mod.rs b/control_plane/src/query_parser/mod.rs deleted file mode 100644 index a8371a629..000000000 --- a/control_plane/src/query_parser/mod.rs +++ /dev/null @@ -1,677 +0,0 @@ -//! PromQL workload extraction through `asap_frontend_promql::lower_promql_workload`. -//! -//! ASAPPlanner owns parsing and intent classification. Both entry points require -//! an explicit `AccuracyTarget` because lowering can size summaries. -//! -//! * [`parse_query_expr_canonical`] returns the canonical query expression. -//! * [`parse_query`] projects it into `ParsedQuery` for workload analysis. - -use std::collections::HashMap; -use std::time::Duration; - -use crate::types::AccuracyTarget; -use crate::types::AggType; -use planner_types::pre_asap::AggIntent; -use planner_types::pre_asap::{ColumnId, CompareOpKind, ScalarValue}; -use planner_types::pre_asap::{Predicate, QueryExpr, Source}; -use planner_types::workload::{ - AccuracyRequirement, BatchEntry, DataWorkload, DurationMs, Evidence, Query, QueryLanguage, - QueryRequirements, QueryWorkload, TimeSelection, -}; - -const COMPATIBILITY_DATA_INGESTION_INTERVAL_MS: u64 = 1_000; - -// ── Output types (legacy — consumed by analyzer and planner) ────────────────── - -/// Flat intermediate representation consumed by [`crate::analyzer::Analyzer`]. -/// -/// Produced by [`parse_query`] via [`QueryExpr`] tree walking. -#[derive(Debug, Clone)] -pub struct ParsedQuery { - /// Metric name (from the PromQL selector). - pub metric_name: String, - /// Aggregation types inferred from the query. - pub aggregations: Vec, - /// Dimensions that must be preserved for GROUP BY / `by (dims)`. - pub group_by_labels: Vec, - /// Equality label filters extracted from the query. - pub label_filters: HashMap, - /// Time window extracted from the range vector or query context. - pub time_window: Duration, - /// True when the query requires per-sample exact values. - pub exact_required: bool, - /// Quantile φ values implied by the query. - pub quantiles: Vec, - /// Named pattern hint for domain-specific planner defaults. - pub hint: Option, -} - -/// Named query pattern recognised by the DEBS-aware planner. -#[derive(Debug, Clone)] -pub enum QueryHint { - // ── DEBS 2022 financial queries ─────────────────────────────────────────── - /// Q1 – per-symbol EMA via quantile proxy (DDSketch / KLL). - DebsEma, - /// Q3 – top-K symbols by event count or price move (CountSketch). - DebsTopK { k: u64 }, - /// Q4 – per-symbol high / low / last / range (extreme-quantile DDSketch). - DebsPriceStats, - /// Q5 / Q9 – realized volatility / Bollinger bands via IQR proxy. - DebsVolatility, - /// Q6 – distinct active symbols per window (HLL). - DebsCardinality, - /// Q7 – TWAP as median / p50 (DDSketch). - DebsTwap, - /// Q8 – price anomaly detection via IQR (DDSketch). - DebsAnomaly, - // ── Exact-only patterns ─────────────────────────────────────────────────── - /// Query requires stateful per-sample computation; no sketch benefit. - ExactRequired { reason: String }, -} - -// ── Public entry points ─────────────────────────────────────────────────────── - -/// Parse a PromQL query string into the **canonical** L3 -/// [`query_expr::QueryExpr`](planner_types::pre_asap::QueryExpr) IR. -/// -/// Compatibility entry point for callers that do not own a complete workload. -/// Plan-ready compilation lowers the complete caller-supplied workload instead. -pub fn parse_query_expr_canonical( - query: &str, - accuracy: AccuracyTarget, -) -> anyhow::Result { - parse_query_expr_with_interval(query, accuracy, COMPATIBILITY_DATA_INGESTION_INTERVAL_MS) -} - -pub(crate) fn parse_query_expr_with_interval( - query: &str, - accuracy: AccuracyTarget, - interval_ms: u64, -) -> anyhow::Result { - let workload = planner_types::workload::PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.trim().into()), - requirements: QueryRequirements { - accuracy: AccuracyRequirement::Explicit(accuracy), - ..Default::default() - }, - predictability: Default::default(), - invocations: 1, - execute_at: None, - time_selection: TimeSelection::default(), - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload { - data_ingestion_interval: Evidence { - value: Some(DurationMs(interval_ms)), - ..Default::default() - }, - ..Default::default() - }), - }; - asap_frontend_promql::lower_promql_workload(&workload, 0)? - .into_iter() - .next() - .ok_or_else(|| anyhow::anyhow!("PromQL compatibility workload produced no query")) -} - -/// Parse a PromQL query string into a [`ParsedQuery`]. -/// -/// Flat compatibility entry point for workload metadata extraction. Internally it parses via -/// [`parse_query_expr_canonical`] and extracts the flat summary by walking -/// the canonical [`QueryExpr`] tree. -pub fn parse_query(query: &str, accuracy: AccuracyTarget) -> anyhow::Result { - let qe = parse_query_expr_canonical(query, accuracy)?; - Ok(qe_to_parsed_query(&qe)) -} - -/// Extract a flat [`ParsedQuery`] by walking a canonical [`QueryExpr`] tree. -/// -/// Step γ7: the collector walks the canonical IR. `Aggregate` carries -/// `Vec` directly (the legacy `AggFunc` is gone), so the -/// `collect_agg_func*` helpers are replaced by per-intent [`collect_op`] -/// calls. Canonical `Scan` also carries `label_filters` inline, so they -/// are picked up at the scan leaf as well as from any `Filter` predicate. -pub(crate) fn qe_to_parsed_query(qe: &QueryExpr) -> ParsedQuery { - // The SchemaResolver-built `Scan.schema` is the complete, self-contained - // column universe every `ColumnId` in the tree indexes into. We grab - // it up-front so the `Aggregate` walk can recover group-by *names* - // from positional `by` ids. - let schema = root_scan_schema(qe); - let mut c = QeCollector::default(); - c.visit(qe, schema); - c.build() -} - -/// Find the schema carried by the tree's first `Scan` leaf. Every `Scan` -/// in a converted tree carries the same SchemaResolver schema, so the first one -/// found is representative. -fn root_scan_schema(qe: &QueryExpr) -> Option<&planner_types::pre_asap::Schema> { - match qe { - QueryExpr::Scan { schema, .. } => Some(schema), - QueryExpr::Filter { child, .. } - // `TimeRange`/`TimeShift` are `lower_promql`'s range-vector-selector - // and offset/@ markers (L1 adoption, design-target-architecture.md - // Part B) -- pass-through wrappers over the same `Scan`. - | QueryExpr::TimeRange { child, .. } - | QueryExpr::TimeShift { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Project { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } => root_scan_schema(child), - QueryExpr::Concat { children, .. } => children.iter().find_map(root_scan_schema), - QueryExpr::Join { left, right, .. } - | QueryExpr::SetOp { left, right, .. } - | QueryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } => root_scan_schema(left).or_else(|| root_scan_schema(right)), - // The remaining PromQL-surface superset (Scalar/EvalTime/ - // VectorFromScalar/ScalarFromVector/Relabel/InfoJoin/Sample/ - // WindowFunc) is real, new capability `lower_promql` adds but this - // crate's flat `ParsedQuery` extraction doesn't attempt to unpack - // yet -- accepted gap (design-target-architecture.md Part B): these - // shapes weren't reachable at all before this swap, so nothing - // regresses; `ParsedQuery` may come back incomplete for them. - _ => None, - } -} - -#[derive(Default)] -struct QeCollector { - metric_name: Option, - agg_types: Vec, - group_by_labels: Vec, - label_filters: HashMap, - time_window: Option, - exact_required: bool, - quantiles: Vec, - topk: Option, -} - -impl QeCollector { - /// `schema` is the SchemaResolver-built `Scan.schema` — the complete column - /// universe `Aggregate.by` positional ids index into. Threaded - /// unchanged through the walk; only the `Aggregate` arm reads it. - fn visit(&mut self, expr: &QueryExpr, schema: Option<&planner_types::pre_asap::Schema>) { - match expr { - QueryExpr::Scan { - source, - predicates, - schema: scan_schema, - } => { - if self.metric_name.is_none() { - self.metric_name = Some(match source { - Source::TimeSeries { metric } => metric.clone(), - Source::Table { table_ref } => table_ref.clone(), - }); - } - // Canonical `Scan.predicates` carries equality label - // filters as typed `Predicate(QueryExpr::Compare{Column, Eq, - // Literal(Utf8)})` trees (the shape - // `query_expr::label_filter_to_predicate` builds) — resolve - // each `Column` id back to its name via the Scan's own - // schema to recover the flat name/value map this legacy - // `ParsedQuery` output still wants. - for p in predicates { - collect_filters_from_expr(&p.0, Some(scan_schema), &mut self.label_filters); - } - } - QueryExpr::Filter { pred, child } => { - // Extract equality label filters from the predicate tree. - collect_filters_from_scalar(pred, schema, &mut self.label_filters); - self.visit(child, schema); - } - // `TimeRange` is `lower_promql`'s range-vector-selector marker - // (`m[5m]` in `quantile_over_time(φ, m[5m])`) -- the range-window - // duration this collector's `time_window` field wants, same as - // `Window::size` above. `TimeShift` (`offset`/`@`) is a pure - // pass-through, no window/label information of its own. - QueryExpr::TimeRange { range, child } => { - if self.time_window.is_none() { - self.time_window = Some(*range); - } - self.visit(child, schema); - } - QueryExpr::TimeShift { child, .. } => self.visit(child, schema), - QueryExpr::Aggregate { - reduction, - measures: aggs, - child, - .. - } => { - // The canonical IR folds legacy SketchAgg / WindowedAgg-inner - // / TopK / Aggregate into one variant carrying `AggIntent`s. - // `by` is positional — recover the group-by label *names* - // from the SchemaResolver schema (the legacy `Aggregate.keys` were - // names; multi-agg aggregates reach here with a non-empty - // `by` after the SchemaResolver resolves them). A per-entity - // reduction (ASAPController#163/#165) has no `by` at all — - // same as an empty one here, no label names to recover. - let by: &[ColumnId] = reduction.group_keys().map(|k| k.keys()).unwrap_or(&[]); - if let Some(s) = schema { - for &id in by { - if let Some(col) = s.columns.get(id) { - if !self.group_by_labels.contains(&col.name) { - self.group_by_labels.push(col.name.clone()); - } - } - } - } - for intent in aggs { - if let AggIntent::TopK { k, .. } = intent { - self.topk = Some(*k as u64); - } else { - self.collect_op(intent); - } - } - self.visit(child, schema); - } - QueryExpr::Dedup { child, .. } => self.visit(child, schema), - QueryExpr::Concat { children, .. } => { - for c in children { - self.visit(c, schema); - } - } - QueryExpr::Project { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } => self.visit(child, schema), - QueryExpr::Join { left, right, .. } - | QueryExpr::SetOp { left, right, .. } - | QueryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } => { - self.visit(left, schema); - self.visit(right, schema); - } - _ => {} - } - } - - fn collect_op(&mut self, op: &AggIntent) { - // Canonical Quantile is single-φ post Step α (multi-φ legacy - // intents fan out into sibling SketchAggs at construction time — - // each one routes through this collector independently). The - // legacy `Extrema { min, max }` enum split into `Min` / `Max` - // canonical variants; map each to the corresponding boundary - // quantile for legacy compat. - match op { - AggIntent::Cardinality { .. } => { - if !self.agg_types.contains(&AggType::Cardinality) { - self.agg_types.push(AggType::Cardinality); - } - } - op if crate::planner_selection::as_frequency(op).is_some() => { - if !self.agg_types.contains(&AggType::Frequency) { - self.agg_types.push(AggType::Frequency); - } - } - AggIntent::Quantile { q, .. } => { - if !self.agg_types.contains(&AggType::Quantile) { - self.agg_types.push(AggType::Quantile); - } - if !self.quantiles.contains(q) { - self.quantiles.push(*q); - } - } - AggIntent::Min { .. } => { - if !self.agg_types.contains(&AggType::Quantile) { - self.agg_types.push(AggType::Quantile); - } - if !self.quantiles.contains(&0.0) { - self.quantiles.push(0.0); - } - } - AggIntent::Max { .. } => { - if !self.agg_types.contains(&AggType::Quantile) { - self.agg_types.push(AggType::Quantile); - } - if !self.quantiles.contains(&1.0) { - self.quantiles.push(1.0); - } - } - // Sum / Count / Avg / TopK / Rate / Increase / archive-only — - // all flip the exact_required flag (no sketch benefit at the - // legacy planner's level). - _ => { - self.exact_required = true; - } - } - } - - fn build(self) -> ParsedQuery { - let metric_name = self.metric_name.unwrap_or_default(); - let mut qs = self.quantiles; - qs.sort_by(|a, b| a.partial_cmp(b).unwrap()); - qs.dedup(); - let hint = debs_hint( - &metric_name, - &self.agg_types, - &qs, - self.exact_required, - self.topk, - ); - ParsedQuery { - metric_name, - aggregations: self.agg_types, - group_by_labels: self.group_by_labels, - label_filters: self.label_filters, - time_window: self.time_window.unwrap_or(Duration::from_secs(300)), - exact_required: self.exact_required, - quantiles: qs, - hint, - } - } -} - -fn collect_filters_from_scalar( - pred: &Predicate, - schema: Option<&planner_types::pre_asap::Schema>, - out: &mut HashMap, -) { - collect_filters_from_expr(&pred.0, schema, out); -} - -/// Recover a flat name/value equality map from a canonical `QueryExpr` -/// predicate tree — `Column(id) == Literal(Utf8(v))` conjuncts, `id` -/// resolved back to a name via `schema` (positional `Column` carries no -/// name of its own, unlike the pre-merge name-based `Predicate::Column`). -fn collect_filters_from_expr( - expr: &QueryExpr, - schema: Option<&planner_types::pre_asap::Schema>, - out: &mut HashMap, -) { - match expr { - QueryExpr::Compare { - left, - op: CompareOpKind::Eq, - right, - } => { - if let (QueryExpr::Column(id), QueryExpr::Literal(ScalarValue::Utf8(v))) = - (left.as_ref(), right.as_ref()) - { - if let Some(col) = schema.and_then(|s| s.columns.get(*id)) { - out.entry(col.name.clone()).or_insert_with(|| v.clone()); - } - } - } - QueryExpr::BoolAnd(parts) => { - for p in parts { - collect_filters_from_expr(p, schema, out); - } - } - _ => {} - } -} - -// ── DEBS hint classifier (shared by both parsers via to_parsed_query) ───────── - -/// Returns the DEBS-specific hint for `financial.last_trade_price` queries. -pub(super) fn debs_hint( - metric: &str, - aggs: &[AggType], - quantiles: &[f64], - exact_required: bool, - topk: Option, -) -> Option { - let is_debs = metric == "financial.last_trade_price" || metric == "financial_last_trade_price"; - if !is_debs { - return None; - } - - if exact_required { - return Some(QueryHint::ExactRequired { - reason: "query requires per-sample stateful computation".into(), - }); - } - if let Some(k) = topk { - return Some(QueryHint::DebsTopK { k }); - } - let primary = aggs.first()?; - match primary { - AggType::Cardinality => Some(QueryHint::DebsCardinality), - AggType::Frequency => Some(QueryHint::DebsTopK { k: 10 }), - AggType::Quantile => { - let qs: std::collections::HashSet = quantiles - .iter() - .map(|&q| (q * 100.0).round() as i32) - .collect(); - if qs.contains(&50) && qs.len() == 1 { - Some(QueryHint::DebsTwap) - } else if qs.contains(&0) || qs.contains(&100) { - Some(QueryHint::DebsPriceStats) - } else if qs.contains(&25) && qs.contains(&75) && qs.contains(&50) { - Some(QueryHint::DebsAnomaly) - } else if qs.contains(&25) && qs.contains(&75) { - Some(QueryHint::DebsVolatility) - } else { - Some(QueryHint::DebsEma) - } - } - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - const ACC: AccuracyTarget = AccuracyTarget::Epsilon(0.01); - - // Focused unit contracts for the PromQL parse entry point. - - #[test] - fn promql_dispatched_correctly() { - // `by` belongs to the aggregate operator, not the function call. - let pq = parse_query("sum by (host) (quantile_over_time(0.99, latency[5m]))", ACC).unwrap(); - assert!(pq.aggregations.contains(&AggType::Quantile)); - assert_eq!(pq.quantiles, vec![0.99]); - } - - #[test] - fn parse_query_expr_returns_expr() { - let pq = parse_query( - "topk by (symbol) (10, count_over_time(financial_last_trade_price[5m]))", - ACC, - ) - .unwrap(); - // Should parse without error and extract the metric name. - assert_eq!(pq.metric_name, "financial_last_trade_price"); - } - - // ── canonical-IR entry point ───────────────────────────────────────────── - - #[test] - fn canonical_query_quantile_yields_aggregate_over_time_range() { - use planner_types::pre_asap::QueryExpr as CQueryExpr; - // Range-vector selectors lower to `TimeRange`; `Window` represents streaming - // windows. The aggregate sits directly above the range selector. - let expr = parse_query_expr_canonical( - "quantile_over_time(0.99, http_request_duration{env=\"prod\"}[5m])", - ACC, - ) - .unwrap(); - match expr { - CQueryExpr::Aggregate { child, .. } => { - assert!(matches!(*child, CQueryExpr::TimeRange { .. })); - } - other => panic!("expected canonical Aggregate, got {other:?}"), - } - } - - #[test] - fn canonical_query_avg_over_time_yields_aggregate_over_time_range() { - use planner_types::pre_asap::QueryExpr as CQueryExpr; - // A bare `avg_over_time(m[w])` (no `by`) lowers to canonical - // `Aggregate { child: TimeRange { child: Scan } }`. - let expr = - parse_query_expr_canonical("avg_over_time(cpu_seconds_total[10m])", ACC).unwrap(); - match expr { - CQueryExpr::Aggregate { child, .. } => match (*child).clone() { - CQueryExpr::TimeRange { child, .. } => { - assert!(matches!(*child, CQueryExpr::Scan { .. })); - } - other => panic!("expected canonical TimeRange, got {other:?}"), - }, - other => panic!("expected canonical Aggregate, got {other:?}"), - } - } - /// PromQL `count(v)` counts SERIES; it is not a distinct-item count. - /// - /// This mapping has flip-flopped across ASAPPlanner revisions: `ca7546de` - /// (and several around it) lowered a bare `count by (d) (...)` to - /// `AggIntent::Cardinality`, which binds `Capability::CardinalityApprox` - /// and is served by merging HLL registers. For two series holding - /// {a,b,c} and {d,e,f} that answers 6 -- the union's distinct cardinality -- - /// where PromQL's answer is 2, the number of series. - /// - /// `Cardinality` belongs to the distinct-count idiom only: `distinct_over_time`, - /// `COUNT(DISTINCT ...)`, or an explicit `count(distinct_over_time(...))` - /// whose INNER node carries the intent. Keeping the two apart is what lets - /// `capability_for` route `Count` to a frequency sketch and `Cardinality` - /// to HLL, as `runtime_capability`'s own contract states. - #[test] - fn bare_count_is_series_count_not_distinct_cardinality() { - fn intents(query: &str) -> Vec { - fn walk(expr: &QueryExpr, out: &mut Vec) { - match expr { - QueryExpr::Aggregate { - measures, child, .. - } => { - out.extend(measures.iter().cloned()); - walk(child, out); - } - QueryExpr::TimeRange { child, .. } => walk(child, out), - _ => {} - } - } - let mut out = vec![]; - walk(&parse_query_expr_canonical(query, ACC).unwrap(), &mut out); - out - } - - for query in ["count(unique_users)", "count by (svc) (unique_users)"] { - let found = intents(query); - assert!( - found - .iter() - .all(|intent| !matches!(intent, AggIntent::Cardinality { .. })), - "{query} must not carry a Cardinality intent -- that binds HLL and \ - answers the union's distinct count instead of the series count; got {found:?}" - ); - assert!( - found - .iter() - .any(|intent| matches!(intent, AggIntent::Count { .. })), - "{query} must lower to a Count intent, got {found:?}" - ); - } - - // The distinct-count idiom keeps its cardinality intent, on the inner node. - for query in [ - "distinct_over_time(unique_users[5m])", - "count(distinct_over_time(unique_users[5m]))", - ] { - let found = intents(query); - assert!( - found - .iter() - .any(|intent| matches!(intent, AggIntent::Cardinality { .. })), - "{query} is the distinct-count idiom and must keep Cardinality, got {found:?}" - ); - } - } -} - -#[cfg(test)] -mod doc_verify_all { - // Pins the design.md §6 worked examples against the canonical IR - // `asap_frontend_promql::lower_promql` produces — the only algebra IR - // the parse path now emits. - use super::parse_query_expr_canonical; - use crate::types::AccuracyTarget; - use planner_types::pre_asap::QueryExpr; - use planner_types::pre_asap::{AggIntent, Reduction}; - - const ACC: AccuracyTarget = AccuracyTarget::Epsilon(0.01); - - #[test] - fn example4_promql_quantile() { - let expr = parse_query_expr_canonical( - "quantile_over_time(0.99, http_request_duration{env=\"prod\"}[5m])", - ACC, - ) - .unwrap(); - // `Aggregate { reduction: PerEntity, [Quantile], child: TimeRange }` - // — no explicit `by()`, and ranged over a single non-per-series - // intent, so there's no grouping concept at all (see #165's - // `Reduction`). No `Window` wrapper -- `asap_l2::lower` models the - // range-vector selector itself as `TimeRange`, not `Window`. - match &expr { - QueryExpr::Aggregate { - reduction, - measures: aggs, - child, - .. - } => { - assert!(matches!(reduction, Reduction::PerEntity)); - assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { .. }])); - assert!(matches!(child.as_ref(), QueryExpr::TimeRange { .. })); - } - other => panic!("expected Aggregate, got {other:?}"), - } - } - - #[test] - fn example5_promql_topk() { - let expr = parse_query_expr_canonical( - "topk by (service) (10, count_over_time(requests{env=\"prod\"}[1m]))", - ACC, - ) - .unwrap(); - // `topk(...)` ranking by `count_over_time(...)` is the heavy-hitter - // shape both this deployment and `asap_frontend_promql` route to a - // canonical `Aggregate` carrying `AggIntent::TopK`, over the - // `Window { Aggregate { by } } }` the grouped windowed count lowers - // to — regardless of what intent that INNER aggregate now carries - // (see this module's doc: no longer necessarily the `Frequency` - // extension), the outer `TopK` shape itself is unaffected. - match &expr { - QueryExpr::Aggregate { - measures: aggs, - child, - .. - } => { - assert!(matches!(aggs.as_slice(), [AggIntent::TopK { k: 10, .. }])); - // Inner reduction the TopK ranks by: `Aggregate { Count, - // child: TimeRange { child: Scan } }` -- same `TimeRange` - // shape as the other tests above, one level down. `Count` - // carries its own `AccuracyTarget` field (here `Epsilon(0.01)`, - // threaded from this call's `accuracy` argument) rather than - // being forced exact -- adopted as-is per this module's doc. - match child.as_ref() { - QueryExpr::Aggregate { - measures: inner_aggs, - child: inner_child, - .. - } => { - assert!(matches!(inner_aggs.as_slice(), [AggIntent::Count { .. }])); - assert!(matches!(inner_child.as_ref(), QueryExpr::TimeRange { .. })); - } - other => panic!("expected inner Aggregate, got {other:?}"), - } - } - other => panic!("expected Aggregate with TopK intent, got {other:?}"), - } - } -} diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs deleted file mode 100644 index 04a7afd85..000000000 --- a/control_plane/src/query_plan.rs +++ /dev/null @@ -1,1514 +0,0 @@ -//! Control-plane lowering from Planner IR to the shared installed query DAG. -//! Serving consumes asap_types::query_plan; compilation stays in this component. - -mod clickhouse_exact; -pub mod query_time; - -pub use asap_types::query_plan::*; -#[cfg(test)] -use asap_types::PolicyFingerprint; -use planner_types::post_asap::{SummaryExpr, SummaryFamilyType, SummaryNode}; -use planner_types::pre_asap::Reduction; -use std::collections::BTreeMap; -#[cfg(test)] -use std::collections::BTreeSet; -use std::rc::Rc; - -pub fn compile_bound_mapped( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - mut bind: F, - mut lowered: G, -) -> Result -where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - G: FnMut(&Rc, QueryNodeId), -{ - let mut compiler = DagCompiler { - next_id: 0, - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - bind: &mut bind, - logical_source: None, - preserve_relational: false, - lowered: Some(&mut lowered), - }; - let root = compiler.lower(root)?; - Ok(QueryPlanEntry { - language: QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: compiler.nodes, - instant, - fallback, - }) -} - -/// Preserve Planner-to-runtime node identities for installed SQL DAGs. -pub fn compile_bound_relational_mapped( - query_id: String, - canonical_query: String, - root: &Rc, - fixed_evaluation: FixedEvaluationRange, - instant: InstantExecution, - fallback: FallbackPolicy, - mut bind: F, - mut lowered: G, -) -> Result -where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - G: FnMut(&Rc, QueryNodeId), -{ - let mut compiler = DagCompiler { - next_id: 0, - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - bind: &mut bind, - logical_source: None, - preserve_relational: true, - lowered: Some(&mut lowered), - }; - let root = compiler.lower(root)?; - Ok(QueryPlanEntry { - language: QueryLanguage::ClickHouseSql, - query_id, - canonical_query, - fixed_evaluation: Some(fixed_evaluation), - root, - nodes: compiler.nodes, - instant, - fallback, - }) -} - -/// Compile a composable query while exposing the stable mapping from -/// Planner semantic nodes to installed query nodes. The control-plane -/// physical compiler uses this to persist backend placement without -/// relying on pointer values or reconstructing query shape later. -pub fn compile_bound_composable_mapped( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - mut bind: F, - mut lowered: G, -) -> Result -where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - G: FnMut(&Rc, QueryNodeId), -{ - let mut compiler = DagCompiler { - next_id: 0, - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - bind: &mut bind, - logical_source: Some(canonical_query.clone()), - preserve_relational: false, - lowered: Some(&mut lowered), - }; - let root = compiler.lower(root)?; - let mut entry = QueryPlanEntry { - language: QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: compiler.nodes, - instant, - fallback, - }; - query_time::finalize_query_time_nodes(&mut entry)?; - Ok(entry) -} - -fn compile_native_fragment( - root: &Rc, - query_inputs: &[QueryNodeId], -) -> Result { - use asap_physical_operators::physical_planner::{compile, InputContract}; - use planner_types::post_asap::{compile_executable_dag, EdgeRole}; - let invalid = |e: String| QueryPlanError::Invalid(e); - let dag = compile_executable_dag(root).map_err(|e| invalid(e.to_string()))?; - let mut edges = dag - .edges - .iter() - .filter(|e| e.consumer == dag.root) - .collect::>(); - edges.sort_by_key(|e| match e.role { - EdgeRole::Left => 0, - EdgeRole::Input => 1, - EdgeRole::Right => 2, - }); - if edges.len() != query_inputs.len() { - return Err(invalid("physical frontier arity mismatch".into())); - } - let bindings = edges - .iter() - .zip(query_inputs) - .map(|(edge, &id)| (u64::from(edge.producer.0), id)) - .collect::>(); - let contracts = edges - .iter() - .map(|edge| { - ( - u64::from(edge.producer.0), - InputContract::bounded(std::sync::Arc::new(edge.intermediate_schema.clone())), - ) - }) - .collect(); - let physical = - compile(&dag, contracts, &[u64::from(dag.root.0)]).map_err(|e| invalid(e.to_string()))?; - let row_input = physical - .input_contracts() - .position(|(id, _)| bindings[&id] == query_inputs[0]) - .unwrap(); - let pruning = if let SummaryExpr::RelationalJoin { - left, - right, - pred, - pruning: Some(completeness), - .. - } = &root.expr - { - Some(asap_types::query_plan::PruningInputContract { - candidate_input: physical - .input_contracts() - .position(|(id, _)| bindings[&id] == query_inputs[1]) - .unwrap(), - keys: asap_physical_operators::physical_planner::equijoin_keys( - pred, - &left.schema, - &right.schema, - ) - .map_err(|e| invalid(e.to_string()))?, - completeness: completeness.clone(), - }) - } else { - None - }; - Ok(QueryPlanNode::PhysicalFragment { - pruning, - row_input, - inputs: physical - .input_contracts() - .map(|(id, _)| bindings[&id]) - .collect(), - dag: physical.encode().map_err(|e| invalid(e.to_string()))?, - }) -} - -struct DagCompiler<'a, F> { - next_id: u64, - nodes: BTreeMap, - seen: BTreeMap, - bind: &'a mut F, - logical_source: Option, - preserve_relational: bool, - lowered: Option<&'a mut dyn FnMut(&Rc, QueryNodeId)>, -} - -impl DagCompiler<'_, F> -where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, -{ - fn graft( - &mut self, - id: QueryNodeId, - root: QueryNodeId, - nodes: BTreeMap, - ) -> Result { - let mut remap = BTreeMap::new(); - for local in nodes.keys() { - let global = if *local == root { - id - } else { - let next = QueryNodeId(self.next_id); - self.next_id += 1; - next - }; - remap.insert(*local, global); - } - for (local, mut physical) in nodes { - match &mut physical { - QueryPlanNode::PhysicalFragment { inputs, .. } - | QueryPlanNode::Logical { inputs, .. } - | QueryPlanNode::SummaryMerge { inputs } - | QueryPlanNode::ExternalExact { inputs, .. } => { - for input in inputs { - *input = remap[input]; - } - } - QueryPlanNode::Binary { inputs, .. } - | QueryPlanNode::RelationalJoin { inputs, .. } => { - for input in inputs { - *input = remap[input]; - } - } - QueryPlanNode::SummaryEstimate { input, .. } - | QueryPlanNode::ExactReadout { input, .. } - | QueryPlanNode::ReduceSum { input, .. } - | QueryPlanNode::Relational { input, .. } => *input = remap[input], - QueryPlanNode::Scalar { .. } - | QueryPlanNode::ReadMaterialization { .. } - | QueryPlanNode::ExactFallback { .. } => {} - } - self.nodes.insert(remap[&local], physical); - } - Ok(id) - } - - fn lower(&mut self, node: &Rc) -> Result { - let identity = Rc::as_ptr(node) as usize; - if let Some(id) = self.seen.get(&identity) { - return Ok(*id); - } - if let SummaryExpr::ValueOperation { - child, - operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - .. - } = &node.expr - { - if exact_accumulator_value_source(node).is_none() { - return Err(QueryPlanError::UnsupportedNode( - "exact finalization requires a direct exact accumulator".into(), - )); - } - // SummaryAgg lowering already emits the family-specific ExactReadout. - // Preserve the Planner's explicit state boundary without adding a - // second runtime readout node. - let child_id = self.lower(child)?; - self.seen.insert(identity, child_id); - if let Some(lowered) = &mut self.lowered { - lowered(node, child_id); - } - return Ok(child_id); - } - let id = QueryNodeId(self.next_id); - self.next_id += 1; - self.seen.insert(identity, id); - let query_time = match (&self.logical_source, &node.expr) { - (Some(original), SummaryExpr::KeepPreAsap(expr)) => { - Some(query_time::query_time_nodes(original, expr)?) - } - (Some(original), SummaryExpr::SummaryAgg { child, .. }) - if matches!(child.expr, SummaryExpr::KeepPreAsap(_)) - && !matches!( - crate::physical::compiler::raw_materialization_input_contract(node), - Ok((_, Some(_), _)) - ) => - { - Some(query_time::selected_query_time_nodes(original, node)?) - } - _ => None, - }; - if let Some((root, nodes)) = query_time { - let id = self.graft(id, root, nodes)?; - if let Some(lowered) = &mut self.lowered { - lowered(node, id); - } - return Ok(id); - } - - let physical = match &node.expr { - SummaryExpr::RelationalJoin { - left, - right, - kind, - pred, - pruning, - } if self.preserve_relational => QueryPlanNode::RelationalJoin { - inputs: [self.lower(left)?, self.lower(right)?], - join_kind: kind.clone(), - pruning: pruning.clone(), - pred: serde_json::to_value(pred).map_err(|error| { - QueryPlanError::Invalid(format!( - "cannot serialize relational join predicate: {error}" - )) - })?, - left_schema: left.schema.clone(), - right_schema: right.schema.clone(), - output_schema: node.schema.clone(), - }, - SummaryExpr::ValueOperation { - child, operation, .. - } if self.preserve_relational - && matches!( - operation, - planner_types::post_asap::ValueOperation::Project { .. } - | planner_types::post_asap::ValueOperation::Filter { .. } - | planner_types::post_asap::ValueOperation::Sort { .. } - | planner_types::post_asap::ValueOperation::Limit { .. } - | planner_types::post_asap::ValueOperation::Exact( - planner_types::post_asap::ExactOperation::Aggregate { .. } - ) - ) => - { - QueryPlanNode::Relational { - input: self.lower(child)?, - operation: serde_json::to_value(operation).map_err(|error| { - QueryPlanError::UnsupportedNode(format!( - "cannot serialize relational operation: {error}" - )) - })?, - input_schema: child.schema.clone(), - output_schema: node.schema.clone(), - } - } - SummaryExpr::ValueOperation { - child, - operation: - planner_types::post_asap::ValueOperation::Exact( - planner_types::post_asap::ExactOperation::Aggregate { - reduction, - measures, - having: None, - .. - }, - ), - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } if measures.len() == 1 => { - use planner_types::pre_asap::AggIntent; - let operation = match &measures[0] { - AggIntent::Sum { .. } => Some(query_time::Aggregation::Sum), - AggIntent::Count { .. } => Some(query_time::Aggregation::Count), - AggIntent::Min { .. } => Some(query_time::Aggregation::Min), - AggIntent::Max { .. } => Some(query_time::Aggregation::Max), - AggIntent::Avg { .. } => Some(query_time::Aggregation::Avg), - _ => { - return Err(QueryPlanError::Invalid( - "unsupported exact value aggregation".into(), - )) - } - }; - let keys = reduction.group_keys().ok_or_else(|| { - QueryPlanError::Invalid( - "per-entity exact value aggregation has no grouping".into(), - ) - })?; - let labels = keys - .keys() - .iter() - .map(|&column| { - child - .schema - .fields - .get(column) - .map(|field| field.name.clone()) - .ok_or_else(|| { - QueryPlanError::Invalid( - "unresolved exact aggregation column".into(), - ) - }) - }) - .collect::, _>>()?; - let grouping = query_time::Grouping { - labels, - without: keys.is_without(), - }; - let operator = query_time::QueryTimeOperator::Aggregate { - operation: operation.expect("aggregate operation"), - grouping, - }; - QueryPlanNode::Logical { - operator, - inputs: vec![self.lower(child)?], - } - } - SummaryExpr::ValueOperation { - child, - operation: - planner_types::post_asap::ValueOperation::Limit { .. } - | planner_types::post_asap::ValueOperation::Sort { .. } - | planner_types::post_asap::ValueOperation::Filter { .. }, - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } => { - let input = self.lower(child)?; - compile_native_fragment(node, &[input])? - } - SummaryExpr::ValueOperation { .. } => QueryPlanNode::ExactFallback { - reason: "unsupported post-ASAP value operation".into(), - }, - SummaryExpr::RelationalJoin { - right: candidates, - left: values, - kind: planner_types::pre_asap::JoinKind::Semi, - pruning, - .. - } => { - let candidate_input = self.lower(candidates)?; - let value_input = if let Some(original) = - self.logical_source.as_ref().filter(|_| pruning.is_some()) - { - let exact_expression = - query_time::selected_native_expression(original, values)?; - let value_id = QueryNodeId(self.next_id); - self.next_id += 1; - self.nodes.insert( - value_id, - QueryPlanNode::ExternalExact { - request: ExternalExactRequest { - language: QueryLanguage::PromQl, - expression: exact_expression.to_string(), - output: ExternalExactOutput::InstantVector, - parameters: BTreeMap::new(), - start_parameter: None, - end_parameter: None, - // The external source provides values; the Planner DAG owns matching. - input_contracts: vec![], - }, - inputs: vec![], - }, - ); - value_id - } else { - self.lower(values)? - }; - compile_native_fragment(node, &[value_input, candidate_input])? - } - SummaryExpr::RelationalJoin { .. } => QueryPlanNode::ExactFallback { - reason: "unsupported join in vector adapter".into(), - }, - SummaryExpr::BinaryOp { - lhs, - rhs, - operator, - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } if matches!( - operator.kind, - planner_types::pre_asap::BinaryOpKind::Arithmetic(_) - ) && operator.vector_match.is_none() - && !operator.checked_relative_division - && !operator.checked_finite_division - && node.guarantee.as_ref().is_some_and(|g| !g.has_unknown()) - && [lhs, rhs].iter().all(|operand| { - matches!( - operand.expr, - SummaryExpr::SummaryEstimate { - query: planner_types::post_asap::SketchQuery::Quantile { .. }, - .. - } - ) - }) => - { - let planner_types::pre_asap::BinaryOpKind::Arithmetic(kind) = &operator.kind else { - unreachable!() - }; - QueryPlanNode::Binary { - inputs: [self.lower(lhs)?, self.lower(rhs)?], - operator: kind.clone(), - } - } - SummaryExpr::BinaryOp { - lhs, - rhs, - operator, - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } if self.logical_source.is_some() - || operator.checked_relative_division - || operator.checked_finite_division => - { - let operator = query_time::binary_operator(operator)?; - QueryPlanNode::Logical { - operator, - inputs: vec![self.lower(lhs)?, self.lower(rhs)?], - } - } - - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(kind, _), - child, - reduction, - .. - } if self.logical_source.is_some() - && !matches!(child.expr, SummaryExpr::KeepPreAsap(_)) => - { - if !matches!( - kind, - planner_types::post_asap::ExactKind::Sum - | planner_types::post_asap::ExactKind::Count - ) { - let operator = query_time::selected_aggregate_operator( - self.logical_source.as_deref().unwrap(), - node, - )?; - let input = self.lower(child)?; - self.nodes.insert( - id, - QueryPlanNode::Logical { - operator, - inputs: vec![input], - }, - ); - return Ok(id); - } - let operation = match kind { - planner_types::post_asap::ExactKind::Sum => query_time::Aggregation::Sum, - planner_types::post_asap::ExactKind::Count => query_time::Aggregation::Count, - _ => { - return Err(QueryPlanError::Invalid( - "unsupported aggregation over selected summary values".into(), - )) - } - }; - let keys = reduction.group_keys().ok_or_else(|| { - QueryPlanError::Invalid( - "per-entity summary reduction requires a temporal operator".into(), - ) - })?; - let labels = keys - .keys() - .iter() - .map(|&column| { - child - .schema - .fields - .get(column) - .map(|field| field.name.clone()) - .ok_or_else(|| { - QueryPlanError::Invalid("unresolved logical grouping column".into()) - }) - }) - .collect::, _>>()?; - QueryPlanNode::Logical { - operator: query_time::QueryTimeOperator::Aggregate { - operation, - grouping: query_time::Grouping { - labels, - without: keys.is_without(), - }, - }, - inputs: vec![self.lower(child)?], - } - } - SummaryExpr::BinaryOp { - lhs, - rhs, - operator, - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } if exact_value_executable(node) => { - let planner_types::pre_asap::BinaryOpKind::Arithmetic(operator) = &operator.kind - else { - unreachable!() - }; - QueryPlanNode::Binary { - inputs: [self.lower(lhs)?, self.lower(rhs)?], - operator: operator.clone(), - } - } - SummaryExpr::KeepPreAsap(expr) if scalar_literal(expr).is_some() => { - QueryPlanNode::Scalar { - value: scalar_literal(expr).unwrap(), - } - } - SummaryExpr::KeepPreAsap(expr) if self.preserve_relational => { - let mut expression = - clickhouse_exact::render(expr).map_err(QueryPlanError::UnsupportedNode)?; - let mut bounded = false; - if let planner_types::pre_asap::QueryExpr::Scan { - predicates, schema, .. - } = expr.as_ref() - { - if predicates.is_empty() { - if let Some(column) = schema.time_index.and_then(|i| schema.columns.get(i)) - { - let name = format!("`{}`", column.name.replace('`', "``")); - expression.push_str(&format!( - " WHERE {name} >= {{from:UInt64}} AND {name} <= {{to:UInt64}}" - )); - bounded = true; - } - } - } - QueryPlanNode::ExternalExact { - request: ExternalExactRequest { - language: QueryLanguage::ClickHouseSql, - expression, - output: ExternalExactOutput::Relation { - schema: serde_json::to_value(&node.schema).map_err(|error| { - QueryPlanError::Invalid(format!( - "cannot serialize external exact schema: {error}" - )) - })?, - }, - parameters: BTreeMap::new(), - start_parameter: bounded.then(|| "from".into()), - end_parameter: bounded.then(|| "to".into()), - input_contracts: Vec::new(), - }, - inputs: Vec::new(), - } - } - SummaryExpr::SummaryAgg { - family: - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Sum, _), - child, - reduction, - .. - } if !matches!(child.expr, SummaryExpr::KeepPreAsap(_)) - && exact_value_executable(node) => - { - QueryPlanNode::ReduceSum { - input: self.lower(child)?, - grouping: physical_grouping(reduction, child)?, - } - } - SummaryExpr::SummaryAgg { - child, - family, - reduction, - .. - } if !matches!(child.expr, SummaryExpr::KeepPreAsap(_)) => { - // A compiled immutable dependency is already materialized. Its - // query reads that binding instead of replaying maintenance. - match (self.bind)(node, family) { - Ok(mut binding) => { - binding.output_grouping = physical_grouping(reduction, child)?; - if let Some(readout) = exact_readout(family) { - let input = QueryNodeId(self.next_id); - self.next_id += 1; - self.nodes - .insert(input, QueryPlanNode::ReadMaterialization { binding }); - QueryPlanNode::ExactReadout { input, readout } - } else { - QueryPlanNode::ReadMaterialization { binding } - } - } - Err(_) => QueryPlanNode::ExactFallback { - reason: "unsupported exact operation over summary output".into(), - }, - } - } - // Relational count bindings validate their row population and value - // projection in the SQL compiler; the temporal restriction belongs - // to the PromQL observation-count path. - SummaryExpr::SummaryAgg { - family: - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Count, _), - .. - } if !self.preserve_relational && !exact_value_executable(node) => { - QueryPlanNode::ExactFallback { - reason: "only temporal observation counts are supported".into(), - } - } - SummaryExpr::BinaryOp { .. } => QueryPlanNode::ExactFallback { - reason: "summary binary operation is not executable by the warm tier".into(), - }, - SummaryExpr::KeepPreAsap(_) => QueryPlanNode::ExactFallback { - reason: "post-ASAP node requires exact execution".into(), - }, - SummaryExpr::SummaryAgg { - family, - reduction, - child, - .. - } => match family { - SummaryFamilyType::ExactAggregate(..) | SummaryFamilyType::Sketch(..) => { - let mut binding = match (self.bind)(node, family) { - Ok(binding) => binding, - Err(error) => { - if let Some(original) = &self.logical_source { - let (root, nodes) = - query_time::selected_query_time_nodes(original, node)?; - return self.graft(id, root, nodes); - } - return Err(error); - } - }; - binding.output_grouping = physical_grouping(reduction, child)?; - if let Some(readout) = exact_readout(family) { - let existing = self.nodes.iter().find_map(|(id, node)| { - matches!(node, QueryPlanNode::ReadMaterialization { binding: other } if other == &binding).then_some(*id) - }); - let input = existing.unwrap_or_else(|| { - let input = QueryNodeId(self.next_id); - self.next_id += 1; - self.nodes - .insert(input, QueryPlanNode::ReadMaterialization { binding }); - input - }); - QueryPlanNode::ExactReadout { input, readout } - } else { - QueryPlanNode::ReadMaterialization { binding } - } - } - other => QueryPlanNode::ExactFallback { - reason: format!("summary family {other:?} is not executable by the warm tier"), - }, - }, - SummaryExpr::SummaryEstimate { - summary_input, - query, - } => QueryPlanNode::SummaryEstimate { - input: self.lower(summary_input)?, - query: query.clone().into(), - }, - SummaryExpr::SummaryMerge { children, .. } => { - if children.is_empty() { - QueryPlanNode::ExactFallback { - reason: "empty summary_merge".into(), - } - } else { - QueryPlanNode::SummaryMerge { - inputs: children - .iter() - .map(|child| self.lower(child)) - .collect::>()?, - } - } - } - SummaryExpr::SummaryJoin { .. } => QueryPlanNode::ExactFallback { - reason: "summary_join is not executable by the warm tier".into(), - }, - SummaryExpr::SummarySubtract { .. } => QueryPlanNode::ExactFallback { - reason: "summary_subtract is not executable by the warm tier".into(), - }, - SummaryExpr::SummaryDelete { .. } => QueryPlanNode::ExactFallback { - reason: "summary_delete is not executable by the warm tier".into(), - }, - }; - self.nodes.insert(id, physical); - if let Some(lowered) = &mut self.lowered { - lowered(node, id); - } - Ok(id) - } -} - -fn exact_readout(family: &SummaryFamilyType) -> Option { - use planner_types::post_asap::ExactKind; - match family { - SummaryFamilyType::ExactAggregate(ExactKind::Sum, _) => Some(ExactReadout::Sum), - SummaryFamilyType::ExactAggregate(ExactKind::Count, _) => Some(ExactReadout::Count), - SummaryFamilyType::ExactAggregate(ExactKind::Increase, _) => Some(ExactReadout::Increase), - SummaryFamilyType::ExactAggregate(ExactKind::Rate, _) => Some(ExactReadout::Rate), - SummaryFamilyType::ExactAggregate(ExactKind::Min, _) => Some(ExactReadout::Min), - SummaryFamilyType::ExactAggregate(ExactKind::Max, _) => Some(ExactReadout::Max), - _ => None, - } -} - -fn scalar_literal(expr: &planner_types::pre_asap::QueryExpr) -> Option { - use planner_types::pre_asap::{QueryExpr, ScalarValue}; - let value = match expr { - QueryExpr::PromqlScalarBridge(child) => return scalar_literal(child), - QueryExpr::Literal(ScalarValue::Float64(value)) => *value, - QueryExpr::Literal(ScalarValue::Int64(value)) => *value as f64, - _ => return None, - }; - value.is_finite().then_some(value) -} - -/// A value edge may explicitly finalize an exact accumulator at either -/// execution time. Inspect only this typed boundary; other value operations -/// cannot be treated as transparent producer identity. -pub(crate) fn exact_accumulator_value_source(node: &SummaryNode) -> Option<&SummaryNode> { - let source = match &node.expr { - SummaryExpr::ValueOperation { - child, - operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - .. - } => child.as_ref(), - _ => node, - }; - matches!( - source.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(..), - .. - } - ) - .then_some(source) -} - -/// The current exact arithmetic adapter is deliberately narrower than PromQL: -/// default vector matching, scalar literals and additive temporal readouts. -/// Unsupported operands make the complete expression fall back. -pub(crate) fn exact_value_executable(node: &SummaryNode) -> bool { - use planner_types::post_asap::ExactKind; - if !node - .guarantee - .as_ref() - .is_some_and(|guarantee| guarantee.is_exact()) - { - return false; - } - match &node.expr { - SummaryExpr::ValueOperation { .. } => { - exact_accumulator_value_source(node).is_some_and(exact_value_executable) - } - SummaryExpr::KeepPreAsap(expr) => scalar_literal(expr).is_some(), - SummaryExpr::BinaryOp { - lhs, - rhs, - operator, - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } => { - matches!( - operator.kind, - planner_types::pre_asap::BinaryOpKind::Arithmetic(_) - ) && operator.vector_match.is_none() - && exact_value_executable(lhs) - && exact_value_executable(rhs) - && value_grouping(node).is_ok() - && match (value_source(lhs), value_source(rhs)) { - (Some(left), Some(right)) => left == right, - _ => true, - } - } - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(kind, _), - child, - reduction, - .. - } => { - if matches!(child.expr, SummaryExpr::KeepPreAsap(_)) { - matches!(&child.expr, SummaryExpr::KeepPreAsap(expr) if matches!(expr.as_ref(), planner_types::pre_asap::QueryExpr::TimeRange { child, .. } if matches!(child.as_ref(), planner_types::pre_asap::QueryExpr::Scan { .. }))) - && matches!(reduction, Reduction::PerEntity) - && matches!( - kind, - ExactKind::Sum - | ExactKind::Count - | ExactKind::Increase - | ExactKind::Rate - | ExactKind::Min - | ExactKind::Max - ) - } else { - // Raw producer grouping may move through additive reductions, - // but never through division or other value arithmetic. - matches!(kind, ExactKind::Sum) - && exact_accumulator_value_source(child).is_some() - && exact_value_executable(child) - } - } - _ => false, - } -} - -fn value_grouping(node: &SummaryNode) -> Result, QueryPlanError> { - if matches!(node.expr, SummaryExpr::ValueOperation { .. }) { - if let Some(source) = exact_accumulator_value_source(node) { - return value_grouping(source); - } - } - match &node.expr { - SummaryExpr::KeepPreAsap(_) => Ok(None), - SummaryExpr::SummaryAgg { - reduction, child, .. - } => physical_grouping(reduction, child).map(Some), - SummaryExpr::BinaryOp { lhs, rhs, .. } => { - let left = value_grouping(lhs)?; - let right = value_grouping(rhs)?; - match (left, right) { - (Some(left), Some(right)) if left != right => Err(QueryPlanError::Invalid( - "arithmetic operands require different producer grouping contracts".into(), - )), - (left, right) => Ok(left.or(right)), - } - } - _ => Err(QueryPlanError::Invalid( - "unsupported exact value grouping".into(), - )), - } -} - -// The MVP QueryPlan evaluates all operands over one interval. Different -// selectors/windows need per-operand time binding before they can be warm. -fn value_source(node: &SummaryNode) -> Option<&planner_types::pre_asap::QueryExpr> { - match &node.expr { - SummaryExpr::ValueOperation { .. } => { - exact_accumulator_value_source(node).and_then(value_source) - } - SummaryExpr::SummaryAgg { child, .. } => match &child.expr { - SummaryExpr::KeepPreAsap(expr) => Some(expr), - _ => value_source(child), - }, - SummaryExpr::BinaryOp { lhs, rhs, .. } => value_source(lhs).or_else(|| value_source(rhs)), - _ => None, - } -} - -fn physical_grouping( - reduction: &Reduction, - child: &SummaryNode, -) -> Result { - let Some(keys) = reduction.group_keys() else { - return Ok(PhysicalGrouping::PerEntity); - }; - let names = keys - .keys() - .iter() - .map(|&id| { - child - .schema - .fields - .get(id) - .map(|f| f.name.clone()) - .ok_or_else(|| QueryPlanError::Invalid(format!("unresolved grouping column {id}"))) - }) - .collect::>()?; - Ok(PhysicalGrouping::Reduce(names)) -} - -#[cfg(test)] -mod catalog_binding_tests { - use super::*; - use crate::physical::summary_catalog::SummaryCatalog; - use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - - fn fixture() -> (QueryPlan, SummaryCatalog) { - let mut config = PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), - KeyByLabelNames::new(vec!["job".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 10, - 10, - WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, - ); - config.pane_origin_ms = Some(0); - let catalog = SummaryCatalog::from_materializations(7, 2, &[config.clone()]).unwrap(); - let entry = QueryPlanEntry { - language: crate::query_plan::QueryLanguage::PromQl, - query_id: "q".into(), - canonical_query: "sum_over_time(m[1m])".into(), - fixed_evaluation: None, - root: QueryNodeId(1), - nodes: BTreeMap::from([( - QueryNodeId(1), - QueryPlanNode::ReadMaterialization { - binding: MaterializationBinding { - full_window_slide_ms: None, - item_labels: Vec::new(), - materialization: config.policy_fingerprint().into(), - stored_output_reference: catalog - .output_reference(config.policy_fingerprint().into()) - .unwrap(), - output_grouping: PhysicalGrouping::PerEntity, - window_ms: 10_000, - pane_origin_ms: Some(0), - readout_lookback_ms: Some(60_000), - }, - }, - )]), - instant: InstantExecution { - lookback_ms: 60_000, - full_history: false, - cumulative_readout: true, - }, - fallback: FallbackPolicy::ExactBackend, - }; - ( - QueryPlan { - plan_id: 7, - plan_version: 2, - clickhouse_context: None, - selected_dags: Default::default(), - entries: BTreeMap::from([(entry.canonical_query.clone(), entry)]), - }, - catalog, - ) - } - fn binding(plan: &mut QueryPlan) -> &mut MaterializationBinding { - let QueryPlanNode::ReadMaterialization { binding } = plan - .entries - .values_mut() - .next() - .unwrap() - .nodes - .values_mut() - .next() - .unwrap() - else { - panic!("fixture") - }; - binding - } - - // One pane ID is compatible with a longer semantic readout window. - #[test] - fn catalog_binding_round_trip_preserves_pane_and_readout_windows() { - let (plan, catalog) = fixture(); - let wire = serde_json::to_vec(&plan).unwrap(); - let mut decoded: QueryPlan = serde_json::from_slice(&wire).unwrap(); - decoded.validate_against_catalog(&catalog).unwrap(); - assert_eq!( - decoded - .lookup("sum_over_time(m[1m])") - .unwrap() - .canonical_query, - "sum_over_time(m[1m])" - ); - assert!(String::from_utf8(wire).unwrap().contains("canonical_query")); - assert_eq!(binding(&mut decoded).window_ms, 10_000); - assert_eq!(binding(&mut decoded).readout_lookback_ms, Some(60_000)); - } - - // The catalog owns source and grouping; the binding owns only its stable ID. - #[test] - fn catalog_binding_rejects_source_grouping_and_identity_drift() { - let (plan, catalog) = fixture(); - let mut broken = plan.clone(); - binding(&mut broken).materialization = PolicyFingerprint(123).into(); - assert!(broken.validate_against_catalog(&catalog).is_err()); - let mut broken = plan.clone(); - broken.plan_version += 1; - assert!(broken.validate_against_catalog(&catalog).is_err()); - let mut broken = plan; - binding(&mut broken).window_ms = 0; - assert!(broken.validate_against_catalog(&catalog).is_err()); - } - - // Catalog descriptor corruption must fail even if the materialization exists. - #[test] - fn catalog_binding_rejects_broken_descriptor_reference() { - let (plan, mut catalog) = fixture(); - catalog.summary_descriptors.clear(); - assert!(plan.validate_against_catalog(&catalog).is_err()); - } - - #[test] - fn counter_readout_requires_counter_sds_fidelity() { - fn as_rate_plan(mut plan: QueryPlan) -> QueryPlan { - let entry = plan.entries.values_mut().next().unwrap(); - let read = entry.root; - let root = QueryNodeId(2); - entry.root = root; - entry.nodes.insert( - root, - QueryPlanNode::ExactReadout { - input: read, - readout: ExactReadout::Rate, - }, - ); - plan - } - - let (sum_plan, sum_catalog) = fixture(); - assert!(as_rate_plan(sum_plan) - .validate_against_catalog(&sum_catalog) - .unwrap_err() - .to_string() - .contains("exact counter SDS")); - - let mut counter = PrecomputeMaterialization::new( - AggregationType::Increase, - String::new(), - Default::default(), - KeyByLabelNames::new(vec!["job".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 10, - 10, - WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, - ); - counter.pane_origin_ms = Some(0); - let counter_catalog = - SummaryCatalog::from_materializations(7, 2, &[counter.clone()]).unwrap(); - let (mut counter_plan, _) = fixture(); - binding(&mut counter_plan).materialization = counter.policy_fingerprint().into(); - binding(&mut counter_plan).stored_output_reference = counter_catalog - .output_reference(counter.policy_fingerprint().into()) - .unwrap(); - as_rate_plan(counter_plan) - .validate_against_catalog(&counter_catalog) - .unwrap(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn guarded_division_retains_checks_in_both_query_compilers() { - // Both compilers retain the finite/relative guard supplied by Planner. - let query = "avg_over_time(m[5m])"; - let canonical = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let root = crate::planner_selection::plan_test_query(&canonical).unwrap(); - let SummaryExpr::BinaryOp { operator, .. } = &root.expr else { - panic!("expected the Planner's average rewrite"); - }; - assert!(operator.checked_finite_division); - for relative in [false, true] { - let mut guarded = root.as_ref().clone(); - let SummaryExpr::BinaryOp { operator, .. } = &mut guarded.expr else { - unreachable!(); - }; - operator.checked_finite_division = !relative; - operator.checked_relative_division = relative; - let guarded = Rc::new(guarded); - for composable in [false, true] { - let instant = InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }; - let bind = |_: &Rc, _: &SummaryFamilyType| { - Ok(MaterializationBinding { - full_window_slide_ms: None, - materialization: PolicyFingerprint(7).into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_output( - PolicyFingerprint(7).into(), - ), - output_grouping: PhysicalGrouping::PerEntity, - window_ms: 300_000, - pane_origin_ms: Some(0), - readout_lookback_ms: Some(300_000), - item_labels: Vec::new(), - }) - }; - let entry = if composable { - compile_bound_composable_mapped( - "guarded".into(), - query.into(), - &guarded, - instant, - FallbackPolicy::ExactBackend, - bind, - |_, _| {}, - ) - } else { - compile_bound_mapped( - "guarded".into(), - query.into(), - &guarded, - instant, - FallbackPolicy::ExactBackend, - bind, - |_, _| {}, - ) - } - .unwrap(); - let QueryPlanNode::Logical { - operator: query_time::QueryTimeOperator::Binary { operation, .. }, - .. - } = &entry.nodes[&entry.root] - else { - panic!( - "expected guarded division (composable={composable}): {:?}", - entry.nodes - ); - }; - assert_eq!( - *operation, - if relative { - query_time::BinaryOperation::CheckedDiv - } else { - query_time::BinaryOperation::FiniteDiv - } - ); - assert!(!entry.materialization_bindings().is_empty()); - } - } - } - - #[test] - fn canonical_identity_ignores_formatting() { - assert_eq!( - canonical_promql("sum by (service) ( rate(http_requests_total[5m]) )").unwrap(), - canonical_promql("sum by(service)(rate(http_requests_total[5m]))").unwrap() - ); - } - - #[test] - fn language_tag_preserves_query_entry_serde() { - let entry = QueryPlanEntry { - language: crate::query_plan::QueryLanguage::PromQl, - query_id: "q".into(), - canonical_query: canonical_promql("up").unwrap(), - fixed_evaluation: None, - root: QueryNodeId(0), - nodes: BTreeMap::from([( - QueryNodeId(0), - QueryPlanNode::ExactFallback { - reason: "fixture".into(), - }, - )]), - instant: InstantExecution { - lookback_ms: 1, - full_history: false, - cumulative_readout: false, - }, - fallback: FallbackPolicy::ExactBackend, - }; - let before = serde_json::to_value(&entry).unwrap(); - assert_eq!(before, serde_json::to_value(&entry).unwrap()); - assert!(before.get("canonical_query").is_some()); - assert!(before.get("executable").is_none()); - } - - #[test] - fn language_catalog_keys_keep_equal_query_text_distinct() { - let base = QueryPlanEntry { - language: QueryLanguage::PromQl, - query_id: "prom".into(), - canonical_query: "shared".into(), - fixed_evaluation: None, - root: QueryNodeId(0), - nodes: BTreeMap::from([( - QueryNodeId(0), - QueryPlanNode::ExactFallback { - reason: "fixture".into(), - }, - )]), - instant: InstantExecution { - lookback_ms: 1, - full_history: false, - cumulative_readout: false, - }, - fallback: FallbackPolicy::ExactBackend, - }; - let mut metricsql = base.clone(); - metricsql.language = QueryLanguage::MetricsQl; - metricsql.query_id = "metrics".into(); - let mut clickhouse = base.clone(); - clickhouse.language = QueryLanguage::ClickHouseSql; - clickhouse.query_id = "sql".into(); - clickhouse.fixed_evaluation = Some(FixedEvaluationRange { - start_ms: 1, - end_ms: 2, - cumulative: true, - }); - let plan = QueryPlan { - plan_id: 0, - plan_version: 0, - clickhouse_context: Some(ClickHousePlanningContext { - window_templates: Default::default(), - tables: Default::default(), - accuracy: planner_types::types::AccuracyTarget::Exact, - }), - selected_dags: Default::default(), - entries: [base, metricsql, clickhouse] - .into_iter() - .map(|entry| (QueryPlan::catalog_key(entry.language, "shared"), entry)) - .collect(), - }; - - assert_eq!(plan.entries.len(), 3); - assert_eq!( - plan.lookup_canonical(QueryLanguage::PromQl, "shared") - .unwrap() - .query_id, - "prom" - ); - assert_eq!( - plan.lookup_canonical(QueryLanguage::MetricsQl, "shared") - .unwrap() - .query_id, - "metrics" - ); - assert_eq!(plan.lookup_clickhouse("shared").unwrap().query_id, "sql"); - } - - // A protocol-vector binding cannot silently invent fields or reconstruct changed rows. - #[test] - fn physical_binding_rejects_invented_samples_and_changed_rows() { - use asap_physical_operators::{ - operators::{Expression, Operator}, - physical_planner::{CompiledPhysicalDag, InputContract}, - }; - use planner_types::{ - post_asap::{SummaryFamilyType, SummaryField, SummarySchema}, - pre_asap::DataType, - }; - for duplicate_sample in [false, true] { - let schema = std::sync::Arc::new(SummarySchema { - fields: (0..if duplicate_sample { 2 } else { 1 }) - .map(|i| SummaryField { - name: format!("v{i}"), - dtype: SummaryFamilyType::Plain(DataType::Float64), - nullable: false, - }) - .collect(), - time_index: None, - }); - let op = if duplicate_sample { - Operator::limit(schema.clone(), 1, 0, vec![]).unwrap() - } else { - Operator::project(schema.clone(), vec![("v0".into(), Expression::Column(0))]) - .unwrap() - }; - let compiled = CompiledPhysicalDag::from_operators( - [(0, InputContract::bounded(schema))].into(), - [(1, (vec![0], op))].into(), - vec![1], - ) - .unwrap(); - let entry = QueryPlanEntry { - language: QueryLanguage::PromQl, - query_id: "q".into(), - canonical_query: "topk(1, m)".into(), - fixed_evaluation: None, - root: QueryNodeId(1), - nodes: BTreeMap::from([ - ( - QueryNodeId(0), - QueryPlanNode::ExactFallback { - reason: "prepared source".into(), - }, - ), - ( - QueryNodeId(1), - QueryPlanNode::PhysicalFragment { - inputs: vec![QueryNodeId(0)], - dag: compiled.encode().unwrap(), - row_input: 0, - pruning: None, - }, - ), - ]), - instant: InstantExecution { - lookback_ms: 0, - full_history: false, - cumulative_readout: false, - }, - fallback: FallbackPolicy::Reject, - }; - let error = entry.validate(&BTreeSet::new()).unwrap_err().to_string(); - assert!( - error.contains(if duplicate_sample { - "one numeric sample" - } else { - "preserve its bound input rows" - }), - "{error}" - ); - } - } - - #[test] - fn graph_validation_rejects_cycles() { - let mut nodes = BTreeMap::new(); - nodes.insert( - QueryNodeId(0), - QueryPlanNode::SummaryMerge { - inputs: vec![QueryNodeId(0)], - }, - ); - let entry = QueryPlanEntry { - language: crate::query_plan::QueryLanguage::PromQl, - query_id: "q".into(), - canonical_query: "up".into(), - fixed_evaluation: None, - root: QueryNodeId(0), - nodes, - instant: InstantExecution { - lookback_ms: 0, - full_history: false, - cumulative_readout: false, - }, - fallback: FallbackPolicy::Reject, - }; - assert!(entry - .validate(&BTreeSet::new()) - .unwrap_err() - .to_string() - .contains("cycle")); - } - - #[test] - fn semi_join_rejects_invalid_completeness_contract() { - let leaf = QueryPlanNode::ExactFallback { - reason: "prepared".into(), - }; - let entry = QueryPlanEntry { - language: crate::query_plan::QueryLanguage::PromQl, - query_id: "q".into(), - canonical_query: "topk(2, rate(m[5m]))".into(), - fixed_evaluation: None, - root: QueryNodeId(2), - nodes: BTreeMap::from([ - (QueryNodeId(0), leaf.clone()), - (QueryNodeId(1), leaf), - (QueryNodeId(2), { - let schema = planner_types::post_asap::SummarySchema { - fields: vec![planner_types::post_asap::SummaryField { - name: "pod".into(), - dtype: planner_types::post_asap::SummaryFamilyType::Plain( - planner_types::pre_asap::DataType::Utf8, - ), - nullable: false, - }], - time_index: None, - }; - QueryPlanNode::RelationalJoin { - inputs: [QueryNodeId(1), QueryNodeId(0)], - join_kind: planner_types::pre_asap::JoinKind::Semi, - pred: serde_json::to_value(planner_types::pre_asap::Predicate( - std::rc::Rc::new(planner_types::pre_asap::QueryExpr::Compare { - left: std::rc::Rc::new(planner_types::pre_asap::QueryExpr::Column( - 0, - )), - op: planner_types::pre_asap::CompareOpKind::Eq, - right: std::rc::Rc::new( - planner_types::pre_asap::QueryExpr::Column(1), - ), - }), - )) - .unwrap(), - pruning: Some(CandidateCompleteness::Certified { - guarantee: planner_types::post_asap::ResultGuarantee { - metric: planner_types::post_asap::ErrorMetric::Frequency, - bound: planner_types::post_asap::BoundExpr::Unknown { - statistic: "membership margin".into(), - }, - failure_probability: - planner_types::post_asap::ProbabilityExpr::Unknown { - statistic: "membership confidence".into(), - }, - provenance: vec![], - }, - }), - left_schema: schema.clone(), - right_schema: schema.clone(), - output_schema: schema, - } - }), - ]), - instant: InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }, - fallback: FallbackPolicy::ExactBackend, - }; - assert!(entry.validate(&BTreeSet::new()).is_err()); - } -} diff --git a/control_plane/src/query_plan/clickhouse_exact.rs b/control_plane/src/query_plan/clickhouse_exact.rs deleted file mode 100644 index bcd31fb85..000000000 --- a/control_plane/src/query_plan/clickhouse_exact.rs +++ /dev/null @@ -1,472 +0,0 @@ -//! Render a supported canonical relational cut without changing its row population. -//! Unsupported operators remain admission errors, never guessed SQL semantics. -use planner_types::pre_asap::{ - AggIntent, ArithmeticOpKind, CompareOpKind, QueryExpr, Reduction, ScalarValue, Schema, Source, -}; - -fn quoted(name: &str) -> String { - format!("`{}`", name.replace('`', "``")) -} -fn column(index: usize, schema: &Schema) -> Result { - schema - .columns - .get(index) - .map(|c| quoted(&c.name)) - .ok_or_else(|| format!("unresolved exact column {index}")) -} -fn scalar(expr: &QueryExpr, schema: &Schema) -> Result { - Ok(match expr { - QueryExpr::Column(id) => column(*id, schema)?, - QueryExpr::Literal(value) => match value { - ScalarValue::Int64(v) => v.to_string(), - ScalarValue::Float64(v) if v.is_finite() => format!("toFloat64('{}')", v), - ScalarValue::Utf8(v) => format!("'{}'", v.replace('\\', "\\\\").replace('\'', "\\'")), - ScalarValue::Boolean(v) => if *v { "true" } else { "false" }.into(), - ScalarValue::Null => "NULL".into(), - _ => return Err("nonfinite exact literal".into()), - }, - QueryExpr::Arithmetic { op, left, right } => { - let op = match op { - ArithmeticOpKind::Add => "+", - ArithmeticOpKind::Sub => "-", - ArithmeticOpKind::Mul => "*", - ArithmeticOpKind::Div => "/", - ArithmeticOpKind::Mod => "%", - _ => return Err("unsupported exact arithmetic".into()), - }; - format!( - "({} {op} {})", - scalar(left, schema)?, - scalar(right, schema)? - ) - } - QueryExpr::Compare { op, left, right } => { - let op = match op { - CompareOpKind::Eq => "=", - CompareOpKind::Ne => "!=", - CompareOpKind::Lt => "<", - CompareOpKind::Le => "<=", - CompareOpKind::Gt => ">", - CompareOpKind::Ge => ">=", - _ => return Err("unsupported exact comparison".into()), - }; - format!( - "({} {op} {})", - scalar(left, schema)?, - scalar(right, schema)? - ) - } - QueryExpr::BoolAnd(args) | QueryExpr::BoolOr(args) => { - let and = matches!(expr, QueryExpr::BoolAnd(_)); - if args.is_empty() { - if and { "true" } else { "false" }.into() - } else { - format!( - "({})", - args.iter() - .map(|e| scalar(e, schema)) - .collect::, _>>()? - .join(if and { " AND " } else { " OR " }) - ) - } - } - QueryExpr::Not(arg) => format!("NOT ({})", scalar(arg, schema)?), - QueryExpr::IsNull(arg) => format!("({} IS NULL)", scalar(arg, schema)?), - QueryExpr::IsNotNull(arg) => format!("({} IS NOT NULL)", scalar(arg, schema)?), - QueryExpr::FunctionCall { name, args } => { - let function = match name.as_str() { - "map" => "map", - "mapconcat" => "mapConcat", - "asap_map_access" | "asap_element_access" => "arrayElement", - "asap_struct_field" => "tupleElement", - _ => return Err(format!("unsupported exact scalar function {name}")), - }; - expr.scalar_type(schema).map_err(|e| e.to_string())?; - format!( - "{function}({})", - args.iter() - .map(|e| scalar(e, schema)) - .collect::, _>>()? - .join(", ") - ) - } - _ => return Err("unsupported exact scalar expression".into()), - }) -} -fn aggregate(intent: &AggIntent, schema: &Schema) -> Result { - if let Some((arg, order)) = intent.arg_selector_columns(schema)? { - let AggIntent::Extension { ext_kind, .. } = intent else { - unreachable!() - }; - let function = if ext_kind == "arg_max" { - "argMax" - } else { - "argMin" - }; - return Ok(format!( - "{function}({}, {})", - column(arg, schema)?, - column(order, schema)? - )); - } - let (function, col) = match intent { - AggIntent::Count { .. } => return Ok("count()".into()), - AggIntent::Sum { col } => ("sum", col), - AggIntent::Min { col } => ("min", col), - AggIntent::Max { col } => ("max", col), - AggIntent::Avg { col } => ("avg", col), - _ => return Err("unsupported exact aggregate contract".into()), - }; - Ok(format!( - "{function}({})", - column( - col.ok_or("SQL aggregate requires explicit input column")?, - schema - )? - )) -} -/// Composite cuts retain their own canonical predicates; caller bounds are not -/// injected into descendant scans (which may belong to independent windows). -pub(super) fn render(expr: &QueryExpr) -> Result { - let output = expr.output_schema().map_err(|e| e.to_string())?; - // SQL names must identify a unique positional field at each nested boundary. - let mut names = std::collections::BTreeSet::new(); - if output.columns.iter().any(|c| !names.insert(&c.name)) { - return Err("ambiguous exact output column names".into()); - } - match expr { - QueryExpr::Scan { - source: Source::Table { table_ref }, - predicates, - schema, - } => { - let table = table_ref - .split('.') - .map(quoted) - .collect::>() - .join("."); - let columns = schema - .columns - .iter() - .map(|c| quoted(&c.name)) - .collect::>() - .join(", "); - let filters = predicates - .iter() - .map(|p| scalar(&p.0, schema)) - .collect::, _>>()?; - Ok(format!( - "SELECT {columns} FROM {table}{}", - if filters.is_empty() { - String::new() - } else { - format!(" WHERE {}", filters.join(" AND ")) - } - )) - } - QueryExpr::Filter { pred, child } => { - let schema = child.output_schema().map_err(|e| e.to_string())?; - Ok(format!( - "SELECT * FROM ({}) WHERE {}", - render(child)?, - scalar(&pred.0, &schema)? - )) - } - QueryExpr::Project { cols, child, .. } => { - let schema = child.output_schema().map_err(|e| e.to_string())?; - if cols.len() != output.columns.len() { - return Err("exact projection width mismatch".into()); - } - let columns = cols - .iter() - .zip(&output.columns) - .map(|(item, col)| { - Ok(format!( - "{} AS {}", - scalar(&item.expr, &schema)?, - quoted(&col.name) - )) - }) - .collect::, String>>()?; - Ok(format!( - "SELECT {} FROM ({})", - columns.join(", "), - render(child)? - )) - } - QueryExpr::Aggregate { - reduction: Reduction::Reduce(keys), - measures, - having: None, - child, - .. - } if !keys.is_without() => { - let schema = child.output_schema().map_err(|e| e.to_string())?; - let groups = keys - .keys() - .iter() - .map(|id| column(*id, &schema)) - .collect::, _>>()?; - let mut values = groups.clone(); - values.extend( - measures - .iter() - .map(|m| aggregate(m, &schema)) - .collect::, _>>()?, - ); - if values.len() != output.columns.len() { - return Err("exact aggregate width mismatch".into()); - } - let values = values - .iter() - .zip(&output.columns) - .map(|(v, c)| format!("{v} AS {}", quoted(&c.name))) - .collect::>(); - Ok(format!( - "SELECT {} FROM ({}){}", - values.join(", "), - render(child)?, - if groups.is_empty() { - String::new() - } else { - format!(" GROUP BY {}", groups.join(", ")) - } - )) - } - QueryExpr::Sort { - keys, - partition_by, - child, - } if partition_by.keys().is_empty() && !partition_by.is_without() => { - let schema = child.output_schema().map_err(|e| e.to_string())?; - let keys = keys - .iter() - .map(|key| { - Ok(format!( - "{} {} NULLS {}", - scalar(&key.expr, &schema)?, - if key.ascending { "ASC" } else { "DESC" }, - if key.nulls_first { "FIRST" } else { "LAST" } - )) - }) - .collect::, String>>()?; - if keys.is_empty() { - return Err("empty exact sort".into()); - } - Ok(format!( - "SELECT * FROM ({}) ORDER BY {}", - render(child)?, - keys.join(", ") - )) - } - QueryExpr::Limit { n, offset, child } => Ok(format!( - "SELECT * FROM ({}) LIMIT {n} OFFSET {offset}", - render(child)? - )), - _ => Err("unsupported canonical ClickHouse exact subtree".into()), - } -} - -#[cfg(test)] -mod original_tests { - use super::*; - use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; - use planner_types::{ - pre_asap::{Column, DataType}, - types::AccuracyTarget, - workload::SqlDialect, - }; - #[tokio::test] - async fn original_exact_shapes_retain_native_aggregates_and_bounds() { - let catalog = SqlCatalog::new().with_table( - "raw_samples", - Schema::new(vec![ - Column::new("metric", DataType::Utf8, false), - Column::new("ts_ms", DataType::Int64, false), - Column::new("value", DataType::Float64, false), - Column::new( - "labels", - DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Utf8), - value_nullable: false, - }, - false, - ), - ]), - ); - for sql in [ - include_str!("../../tests/fixtures/sql_exact_cuts/q07.sql"), - include_str!("../../tests/fixtures/sql_exact_cuts/q09.sql"), - include_str!("../../tests/fixtures/sql_exact_cuts/q12.sql"), - include_str!("../../tests/fixtures/sql_exact_cuts/q27.sql"), - ] { - let canonical = lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .unwrap(); - let rendered = render(&canonical).unwrap(); - assert!(rendered.contains("1788891296000")); - assert!(rendered.contains("`ts_ms`")); - assert!(!rendered.contains("{from:")); - if sql.contains("sum(value)") && sql.contains("argMax") { - use crate::physical::post_asap::{PhysicalExpr, PostAsapPlan}; - use crate::query_plan::{ - FallbackPolicy, FixedEvaluationRange, InstantExecution, QueryPlanError, - QueryPlanNode, - }; - let planned = - crate::clickhouse::plan_clickhouse_sql(sql, &catalog, AccuracyTarget::Exact) - .await - .unwrap(); - let PhysicalExpr::Committed(PostAsapPlan::Summary(root)) = planned.physical else { - panic!("missing selected SQL DAG") - }; - let entry = crate::query_plan::compile_bound_relational_mapped( - "test".into(), - planned.canonical_sql, - &root, - FixedEvaluationRange { - start_ms: 1788890996000, - end_ms: 1788891296000, - cumulative: false, - }, - InstantExecution { - lookback_ms: 300000, - full_history: false, - cumulative_readout: false, - }, - FallbackPolicy::ExactBackend, - |_, _| Err(QueryPlanError::Invalid("unexpected summary binding".into())), - |_, _| {}, - ) - .unwrap(); - assert!( - !entry - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::Logical { .. })), - "SQL must not acquire PromQL operators" - ); - assert!(entry.nodes.values().any(|node| match node { - QueryPlanNode::Relational { operation, .. } => matches!( - serde_json::from_value::( - operation.clone() - ) - .unwrap(), - planner_types::post_asap::ValueOperation::Exact( - planner_types::post_asap::ExactOperation::Aggregate { .. } - ) - ), - _ => false, - })); - } - } - } - #[test] - fn literal_quotes_and_backslashes_are_escaped_independently() { - let rendered = scalar( - &QueryExpr::Literal(ScalarValue::Utf8("a\\'b\n".into())), - &Schema::new(vec![]), - ) - .unwrap(); - assert_eq!(rendered, "'a\\\\\\'b\n'"); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use planner_types::pre_asap::{Column, DataType, Predicate, ProjectItem}; - use std::rc::Rc; - #[test] - fn composite_cut_preserves_branch_time_and_positional_projection() { - let scan = Rc::new(QueryExpr::Scan { - source: Source::Table { - table_ref: "db.samples".into(), - }, - schema: Schema::new(vec![ - Column::new("ts", DataType::Int64, false), - Column::new("v", DataType::Float64, false), - ]), - predicates: vec![Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(0)), - op: CompareOpKind::Lt, - right: Rc::new(QueryExpr::Literal(ScalarValue::Int64(-100))), - }))], - }); - let project = QueryExpr::Project { - cols: vec![ProjectItem { - alias: Some("result".into()), - expr: QueryExpr::Column(1), - }], - qualifier: None, - child: scan, - }; - let sql = render(&project).unwrap(); - assert!(sql.contains("`ts` < -100")); - assert!(sql.starts_with("SELECT `v` AS `result`")); - assert!(!sql.contains("{from:")); - assert!(!sql.contains("{to:")); - } - #[test] - fn typed_list_access_renders_native_element_lookup() { - let schema = Schema::new(vec![Column::new( - "samples", - DataType::List { - element: Box::new(Column::new("item", DataType::Float64, false)), - }, - false, - )]); - let expr = QueryExpr::FunctionCall { - name: "asap_element_access".into(), - args: vec![ - QueryExpr::Column(0), - QueryExpr::Literal(ScalarValue::Int64(-1)), - ], - }; - assert_eq!( - scalar(&expr, &schema).unwrap(), - "arrayElement(`samples`, -1)" - ); - } - - #[test] - fn typed_struct_field_renders_native_lookup() { - let schema = Schema::new(vec![Column::new( - "sample", - DataType::Struct { - fields: vec![ - Column::new("ts", DataType::Int64, false), - Column::new("value", DataType::Float64, true), - ], - }, - false, - )]); - let expr = QueryExpr::FunctionCall { - name: "asap_struct_field".into(), - args: vec![ - QueryExpr::Column(0), - QueryExpr::Literal(ScalarValue::Utf8("value".into())), - ], - }; - assert_eq!( - scalar(&expr, &schema).unwrap(), - "tupleElement(`sample`, 'value')" - ); - } - - #[test] - fn unsupported_scalar_is_not_forwarded_as_arbitrary_native_code() { - let schema = Schema::new(vec![]); - let expr = QueryExpr::FunctionCall { - name: "unreviewedFunction".into(), - args: vec![], - }; - assert!(scalar(&expr, &schema).is_err()); - } -} diff --git a/control_plane/src/query_plan/query_time.rs b/control_plane/src/query_plan/query_time.rs deleted file mode 100644 index 7f1afc89e..000000000 --- a/control_plane/src/query_plan/query_time.rs +++ /dev/null @@ -1,1767 +0,0 @@ -//! The query-time half of a plan: everything a summary did not replace. -//! -//! Typed operations compiled once by the control plane, never parsed at -//! serving time. This is the counterpart to the precompute half, which runs -//! ahead of the query and ends at stored outputs. -use super::{ - FallbackPolicy, InstantExecution, QueryNodeId, QueryPlanEntry, QueryPlanError, QueryPlanNode, -}; -use promql_parser::{ - label::MatchOp, - parser::{self, Expr, LabelModifier, Offset, VectorSelector}, -}; -use std::collections::BTreeMap; - -pub use asap_types::query_plan::query_time::*; - -/// Stable identity of a Planner-authorized materializable DAG leaf. This is a -/// workload-selection key, not another physical materialization definition. -#[derive(Debug, Clone, serde::Serialize, PartialEq)] -struct MaterializationCandidateIdentity { - metric: String, - matchers: Vec, - range_ms: u64, - offset_ms: i64, - operation: TemporalOperation, -} - -fn invalid(message: impl Into) -> QueryPlanError { - QueryPlanError::Invalid(message.into()) -} -fn millis(duration: std::time::Duration) -> Result { - u64::try_from(duration.as_millis()).map_err(|_| invalid("logical duration overflow")) -} -fn offset(value: &Option) -> Result { - match value { - None => Ok(0), - Some(Offset::Pos(d)) => i64::try_from(millis(*d)?).map_err(|_| invalid("offset overflow")), - Some(Offset::Neg(d)) => i64::try_from(millis(*d)?) - .map(|v| -v) - .map_err(|_| invalid("offset overflow")), - } -} - -struct Lower { - nodes: BTreeMap, - seen: BTreeMap, -} -impl Lower { - fn add(&mut self, node: QueryPlanNode) -> Result { - let key = serde_json::to_string(&node).map_err(|e| invalid(e.to_string()))?; - if let Some(id) = self.seen.get(&key) { - return Ok(*id); - } - let id = QueryNodeId(self.nodes.len() as u64); - self.nodes.insert(id, node); - self.seen.insert(key, id); - Ok(id) - } - fn operation( - &mut self, - operator: QueryTimeOperator, - inputs: Vec, - ) -> Result { - operator.validate(inputs.len())?; - self.add(QueryPlanNode::Logical { operator, inputs }) - } - fn scan( - &mut self, - s: &VectorSelector, - range_ms: Option, - ) -> Result { - if s.at.is_some() || !s.matchers.or_matchers.is_empty() { - return Err(invalid("logical @/OR selector is not supported")); - } - let matchers = s - .matchers - .matchers - .iter() - .map(|m| LabelMatcher { - name: m.name.clone(), - value: m.value.clone(), - operation: match m.op { - MatchOp::Equal => LabelMatch::Equal, - MatchOp::NotEqual => LabelMatch::NotEqual, - MatchOp::Re(_) => LabelMatch::Regex, - MatchOp::NotRe(_) => LabelMatch::NotRegex, - }, - }) - .collect(); - self.operation( - QueryTimeOperator::Scan { - metric: s.name.clone(), - matchers, - range_ms, - offset_ms: offset(&s.offset)?, - }, - vec![], - ) - } - fn lower(&mut self, expr: &Expr) -> Result { - match expr { - Expr::NumberLiteral(n) if n.val.is_finite() => { - self.add(QueryPlanNode::Scalar { value: n.val }) - } - Expr::Paren(p) => self.lower(&p.expr), - Expr::Unary(u) => { - let input = self.lower(&u.expr)?; - self.operation(QueryTimeOperator::UnaryNegate, vec![input]) - } - Expr::VectorSelector(s) => self.scan(s, None), - Expr::MatrixSelector(s) => self.scan(&s.vs, Some(millis(s.range)?)), - Expr::Subquery(s) => { - if s.at.is_some() { - return Err(invalid("logical subquery @ is unsupported")); - } - let input = self.lower(&s.expr)?; - self.operation( - QueryTimeOperator::Subquery { - range_ms: millis(s.range)?, - // Prometheus uses its configured default evaluation - // interval when `[range:]` omits the resolution. The - // backend-local deployment uses the Prometheus default - // of one minute; unsupported subquery operands are - // externalized as one exact subtree before execution. - step_ms: millis( - s.step.unwrap_or_else(|| std::time::Duration::from_secs(60)), - )?, - offset_ms: offset(&s.offset)?, - }, - vec![input], - ) - } - Expr::Aggregate(a) => { - let grouping = match &a.modifier { - None => Grouping { - labels: vec![], - without: false, - }, - Some(LabelModifier::Include(labels)) => Grouping { - labels: labels.labels.clone(), - without: false, - }, - Some(LabelModifier::Exclude(labels)) => Grouping { - labels: labels.labels.clone(), - without: true, - }, - }; - if a.op.to_string() == "topk" { - let Some(Expr::NumberLiteral(parameter)) = a.param.as_deref() else { - return Err(invalid("topk requires a literal scalar parameter")); - }; - if !parameter.val.is_finite() { - return Err(invalid("topk requires a finite scalar parameter")); - } - // Prometheus converts the scalar parameter to int64 before - // selection. Values below one produce an empty vector. - let k = parameter.val as i64; - // Keep the selection node local even when its operand has - // unsupported syntax (for example a subquery with an - // implicit resolution). Prometheus evaluates that maximal - // instant-vector child; the backend still performs topk. - let nodes_before = self.nodes.clone(); - let seen_before = self.seen.clone(); - let input = match self.lower(&a.expr) { - Ok(input) => input, - Err(_) => { - self.nodes = nodes_before; - self.seen = seen_before; - self.operation( - QueryTimeOperator::ExactSubquery { - query: a.expr.to_string(), - }, - vec![], - )? - } - }; - let sorted = self.operation( - QueryTimeOperator::Sort { - descending: true, - grouping: grouping.clone(), - }, - vec![input], - )?; - return self.operation( - QueryTimeOperator::Limit { - n: u64::try_from(k).unwrap_or(0), - offset: 0, - grouping, - }, - vec![sorted], - ); - } - if a.param.is_some() { - return Err(invalid("unsupported parameterized aggregate")); - } - let operation = match a.op.to_string().as_str() { - "sum" => Aggregation::Sum, - "max" => Aggregation::Max, - "min" => Aggregation::Min, - "avg" => Aggregation::Avg, - "count" => Aggregation::Count, - other => return Err(invalid(format!("unsupported logical aggregate {other}"))), - }; - let input = self.lower(&a.expr)?; - self.operation( - QueryTimeOperator::Aggregate { - operation, - grouping, - }, - vec![input], - ) - } - Expr::Call(c) => { - let operator = match c.func.name { - "scalar" => QueryTimeOperator::VectorToScalar, - "histogram_quantile" => QueryTimeOperator::HistogramQuantile, - "sort" => QueryTimeOperator::Sort { - descending: false, - grouping: Grouping { - labels: vec![], - without: false, - }, - }, - "sort_desc" => QueryTimeOperator::Sort { - descending: true, - grouping: Grouping { - labels: vec![], - without: false, - }, - }, - name => QueryTimeOperator::Temporal { - operation: match name { - "rate" => TemporalOperation::Rate, - "increase" => TemporalOperation::Increase, - "avg_over_time" => TemporalOperation::Avg, - "max_over_time" => TemporalOperation::Max, - "min_over_time" => TemporalOperation::Min, - "sum_over_time" => TemporalOperation::Sum, - "count_over_time" => TemporalOperation::Count, - _ => { - return Err(invalid(format!("unsupported logical function {name}"))) - } - }, - }, - }; - let inputs = c - .args - .args - .iter() - .map(|e| self.lower(e)) - .collect::, _>>()?; - self.operation(operator, inputs) - } - Expr::Binary(b) => { - if b.modifier.as_ref().is_some_and(|m| { - m.matching.is_some() - || !matches!(m.card, parser::VectorMatchCardinality::OneToOne) - }) { - return Err(invalid("logical explicit vector matching unsupported")); - } - let operation = match b.op.to_string().as_str() { - "+" => BinaryOperation::Add, - "-" => BinaryOperation::Sub, - "*" => BinaryOperation::Mul, - "/" => BinaryOperation::Div, - "%" => BinaryOperation::Mod, - "^" => BinaryOperation::Pow, - "==" => BinaryOperation::Equal, - "!=" => BinaryOperation::NotEqual, - "<" => BinaryOperation::Less, - "<=" => BinaryOperation::LessEqual, - ">" => BinaryOperation::Greater, - ">=" => BinaryOperation::GreaterEqual, - other => return Err(invalid(format!("unsupported logical binary {other}"))), - }; - let inputs = vec![self.lower(&b.lhs)?, self.lower(&b.rhs)?]; - self.operation( - QueryTimeOperator::Binary { - operation, - return_bool: b.return_bool(), - }, - inputs, - ) - } - _ => Err(invalid("unsupported logical expression")), - } - } -} - -/// Lower a Planner-authorized native fragment into typed backend operations. -/// Callers retain a separate external-native alternative for cost comparison. -pub fn compile_logical( - query_id: String, - canonical_query: String, - instant: InstantExecution, - fallback: FallbackPolicy, -) -> Result { - let expr = parser::parse(&canonical_query).map_err(|e| invalid(e.to_string()))?; - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let root = lower.lower(&expr)?; - let entry = QueryPlanEntry { - language: super::QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: lower.nodes, - instant, - fallback, - }; - entry.validate(&Default::default())?; - Ok(entry) -} - -fn horizons(expr: &planner_types::pre_asap::QueryExpr, out: &mut Vec) { - use planner_types::pre_asap::QueryExpr; - if let QueryExpr::TimeRange { range, .. } = expr { - if let Ok(ms) = u64::try_from(range.as_millis()) { - out.push(ms); - } - } - match expr { - QueryExpr::PromqlScalarBridge(child) - | QueryExpr::PromqlVectorFromScalar(child) - | QueryExpr::PromqlScalarFromVector(child) - | QueryExpr::PromqlRelabel { child, .. } - | QueryExpr::PromqlSeriesSample { child, .. } - | QueryExpr::PromqlInfoEnrich { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Project { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - | QueryExpr::TimeRange { child, .. } - | QueryExpr::TimeShift { child, .. } => horizons(child, out), - QueryExpr::BinaryOp { lhs, rhs, .. } => { - horizons(lhs, out); - horizons(rhs, out); - } - QueryExpr::Join { left, right, .. } | QueryExpr::SetOp { left, right, .. } => { - horizons(left, out); - horizons(right, out); - } - _ => {} - } -} - -/// Match fragments by semantic IR equality, not display text or source names. -/// This ensures a subtree parsed for physical lowering is the subtree Planner kept. -/// Does this re-parsed subtree denote the same computation as the Planner -/// fragment? -/// -/// Not `==`, and deliberately so. Planner documents a PromQL leaf's `schema` as -/// usage-derived: "the `(ts, value)` floor + the labels the query references", -/// and marks it `closed: false` precisely because it does not enumerate the -/// row. A fragment resolved inside the whole query therefore carries every -/// label the *query* mentions, while the same fragment re-parsed on its own -/// carries only the labels *it* mentions. -/// -/// So `sum by (label_0) (rate(data[1m]))` yields a fragment whose leaf scan has -/// columns `[ts, value, label_0]`, while re-parsing the subtree `rate(data[1m])` -/// yields `[ts, value]`. Identical source, predicates, range, measures and -/// reduction; one extra column that the isolated parse had no way to know -/// about. Requiring equality there asks an isolated parse to reproduce -/// whole-query context, and every other part of the comparison is what actually -/// discriminates: a different matcher, range or metric still fails. -/// -/// Open leaf schemas are therefore compared by containment. Containment is a -/// *prefix*, not an arbitrary subset, because `ColumnId`s are positional: the -/// floor comes first and context only appends, so a prefix keeps every column -/// id in `predicates` and grouping keys meaning the same column on both sides. -/// Closed (catalog-backed SQL) schemas do enumerate the row, so they keep exact -/// equality. -fn fragment_matches( - candidate: &planner_types::pre_asap::QueryExpr, - fragment: &planner_types::pre_asap::QueryExpr, -) -> bool { - match ( - serde_json::to_value(candidate), - serde_json::to_value(fragment), - ) { - (Ok(candidate), Ok(fragment)) => same_modulo_open_leaf_schema(&candidate, &fragment), - // Fall back to the strict comparison rather than accepting anything we - // could not inspect. - _ => candidate == fragment, - } -} - -fn same_modulo_open_leaf_schema( - candidate: &serde_json::Value, - fragment: &serde_json::Value, -) -> bool { - use serde_json::Value; - match (candidate, fragment) { - (Value::Object(candidate), Value::Object(fragment)) => { - if is_open_schema(candidate) && is_open_schema(fragment) { - return open_schema_is_widened(candidate, fragment); - } - candidate.len() == fragment.len() - && candidate.iter().all(|(key, value)| { - fragment - .get(key) - .is_some_and(|other| same_modulo_open_leaf_schema(value, other)) - }) - } - (Value::Array(candidate), Value::Array(fragment)) => { - candidate.len() == fragment.len() - && candidate - .iter() - .zip(fragment) - .all(|(a, b)| same_modulo_open_leaf_schema(a, b)) - } - _ => candidate == fragment, - } -} - -fn is_open_schema(value: &serde_json::Map) -> bool { - value.get("closed") == Some(&serde_json::Value::Bool(false)) && value.contains_key("columns") -} - -/// The isolated parse's columns must be a prefix of the whole-query ones: -/// context appends the labels it references, it never removes or reorders the -/// floor. Every other schema field still has to agree exactly. -fn open_schema_is_widened( - candidate: &serde_json::Map, - fragment: &serde_json::Map, -) -> bool { - let (Some(narrow), Some(wide)) = ( - candidate.get("columns").and_then(|v| v.as_array()), - fragment.get("columns").and_then(|v| v.as_array()), - ) else { - return false; - }; - candidate - .iter() - .filter(|(key, _)| key.as_str() != "columns") - .all(|(key, value)| fragment.get(key) == Some(value)) - && narrow.len() <= wide.len() - && narrow.iter().zip(wide).all(|(a, b)| a == b) -} - -pub(super) fn query_time_nodes( - original: &str, - fragment: &planner_types::pre_asap::QueryExpr, -) -> Result<(QueryNodeId, BTreeMap), QueryPlanError> { - fn visit<'a>(expr: &'a Expr, out: &mut Vec<&'a Expr>) { - out.push(expr); - match expr { - Expr::Paren(e) => visit(&e.expr, out), - Expr::Unary(e) => visit(&e.expr, out), - Expr::Subquery(e) => visit(&e.expr, out), - Expr::Aggregate(e) => visit(&e.expr, out), - Expr::Binary(e) => { - visit(&e.lhs, out); - visit(&e.rhs, out); - } - Expr::Call(e) => { - for input in &e.args.args { - visit(input, out); - } - } - _ => {} - } - } - let original = parser::parse(original).map_err(|e| invalid(e.to_string()))?; - let mut expressions = Vec::new(); - visit(&original, &mut expressions); - // Reconstruct equality witnesses with the selected IR's source horizon, - // not the compatibility parser's default. Explicit matrix ranges remain - // query-owned and equality still checks the complete tree. - let mut intervals = vec![1_000]; - horizons(fragment, &mut intervals); - intervals.sort_unstable(); - intervals.dedup(); - // Accuracy annotations select a candidate, but exact execution still - // implements that candidate's computation. Reconstruct the same typed IR - // before comparing it; do not erase operators or source predicates. - let accuracy = match fragment { - planner_types::pre_asap::QueryExpr::Aggregate { measures, .. } => measures - .iter() - .find_map(|intent| { - use planner_types::pre_asap::AggIntent; - match intent { - AggIntent::Quantile { accuracy, .. } - | AggIntent::Cardinality { accuracy, .. } - | AggIntent::Count { accuracy } - | AggIntent::TopK { accuracy, .. } - | AggIntent::FrequencyL2 { accuracy, .. } - | AggIntent::FrequencyEntropy { accuracy, .. } => Some(accuracy.clone()), - _ => None, - } - }) - .unwrap_or(planner_types::types::AccuracyTarget::Exact), - _ => planner_types::types::AccuracyTarget::Exact, - }; - for expression in expressions { - for interval in &intervals { - if let Ok(candidate) = crate::query_parser::parse_query_expr_with_interval( - &expression.to_string(), - accuracy.clone(), - *interval, - ) { - if fragment_matches(&candidate, fragment) { - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let root = lower.lower(expression)?; - return Ok((root, lower.nodes)); - } - } - } - } - Err(invalid( - "Planner fragment does not match any original query subtree", - )) -} - -pub(super) fn binary_operator( - operator: &planner_types::post_asap::BinaryOperator, -) -> Result { - if operator.checked_relative_division || operator.checked_finite_division { - if (operator.checked_relative_division && operator.checked_finite_division) - || operator.vector_match.is_some() - || !matches!( - operator.kind, - planner_types::pre_asap::BinaryOpKind::Arithmetic( - planner_types::pre_asap::ArithmeticOpKind::Div - ) - ) - { - return Err(invalid("invalid Planner checked division contract")); - } - return Ok(QueryTimeOperator::Binary { - operation: if operator.checked_finite_division { - BinaryOperation::FiniteDiv - } else { - BinaryOperation::CheckedDiv - }, - return_bool: false, - }); - } - if operator.vector_match.is_some() { - return Err(invalid("explicit fragment vector matching unsupported")); - } - let operation = match operator.kind.to_string().as_str() { - "+" => BinaryOperation::Add, - "-" => BinaryOperation::Sub, - "*" => BinaryOperation::Mul, - "/" => BinaryOperation::Div, - "%" => BinaryOperation::Mod, - "^" => BinaryOperation::Pow, - "=" | "==" => BinaryOperation::Equal, - "<>" | "!=" => BinaryOperation::NotEqual, - "<" => BinaryOperation::Less, - "<=" => BinaryOperation::LessEqual, - ">" => BinaryOperation::Greater, - ">=" => BinaryOperation::GreaterEqual, - other => { - return Err(invalid(format!( - "unsupported Planner binary operator {other}" - ))) - } - }; - Ok(QueryTimeOperator::Binary { - operation, - return_bool: false, - }) -} - -/// Prove a physical-native substitute represents exactly the selected summary leaf. -/// A second Planner invocation is an equality witness, not a replacement selection. -pub(crate) fn selected_query_time_nodes( - original: &str, - selected: &planner_types::post_asap::SummaryNode, -) -> Result<(QueryNodeId, BTreeMap), QueryPlanError> { - let expression = selected_native_expression(original, selected)?; - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let root = lower.lower(&expression)?; - Ok((root, lower.nodes)) -} - -/// Resolve the selected exact subtree to a verified native expression before -/// binding an external input. Never substitute the top-level query's child. -pub(super) fn selected_native_expression( - original: &str, - selected: &planner_types::post_asap::SummaryNode, -) -> Result { - if !selected.guarantee.as_ref().is_some_and(|g| g.is_exact()) { - return Err(invalid( - "native fragment substitution requires an exact selected value", - )); - } - let selected = match &selected.expr { - planner_types::post_asap::SummaryExpr::ValueOperation { - child, - operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - .. - } => child.as_ref(), - _ => selected, - }; - fn visit<'a>(expr: &'a Expr, output: &mut Vec<&'a Expr>) { - output.push(expr); - match expr { - Expr::Paren(e) => visit(&e.expr, output), - Expr::Unary(e) => visit(&e.expr, output), - Expr::Subquery(e) => visit(&e.expr, output), - Expr::Aggregate(e) => visit(&e.expr, output), - Expr::Binary(e) => { - visit(&e.lhs, output); - visit(&e.rhs, output); - } - Expr::Call(e) => { - for input in &e.args.args { - visit(input, output); - } - } - _ => {} - } - } - let parsed = parser::parse(original).map_err(|e| invalid(e.to_string()))?; - let mut expressions = Vec::new(); - visit(&parsed, &mut expressions); - fn selected_horizons(node: &planner_types::post_asap::SummaryNode, out: &mut Vec) { - use planner_types::post_asap::SummaryExpr; - match &node.expr { - SummaryExpr::KeepPreAsap(expr) => horizons(expr, out), - SummaryExpr::ValueOperation { child, .. } | SummaryExpr::SummaryAgg { child, .. } => { - selected_horizons(child, out) - } - SummaryExpr::SummaryEstimate { summary_input, .. } - | SummaryExpr::SummaryDelete { summary_input, .. } => { - selected_horizons(summary_input, out) - } - SummaryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } - | SummaryExpr::RelationalJoin { left, right, .. } - | SummaryExpr::SummaryJoin { - outer: left, - inner: right, - .. - } - | SummaryExpr::SummarySubtract { left, right } => { - selected_horizons(left, out); - selected_horizons(right, out); - } - SummaryExpr::SummaryMerge { children, .. } => { - for child in children { - selected_horizons(child, out); - } - } - } - } - let mut intervals = vec![1_000]; - selected_horizons(selected, &mut intervals); - intervals.sort_unstable(); - intervals.dedup(); - let mut matched = None; - for expression in expressions { - for interval in &intervals { - let Ok(canonical) = crate::query_parser::parse_query_expr_with_interval( - &expression.to_string(), - planner_types::types::AccuracyTarget::Exact, - *interval, - ) else { - continue; - }; - // Match provenance against all exact candidates. Do not make a - // second selection or assume the first enumerated candidate won. - use asap_aware_mapping::{ - Replacement, ReplacementStrategy, SketchAlgorithmStrategy, TargetSubDAG, - }; - let root = std::rc::Rc::new(canonical); - let candidates = SketchAlgorithmStrategy::new(&asap_aware_mapping::DefaultCostModel) - .replacements(&TargetSubDAG::new(&root)); - if candidates.iter().any(|candidate| matches!(&candidate.replacement, Replacement::Summary(node) if node.as_ref() == selected)) { - let candidate = expression.clone(); - if matched - .as_ref() - .is_some_and(|previous| previous != &candidate) - { - return Err(invalid( - "ambiguous original subtrees share a Planner summary representation", - )); - } - matched = Some(candidate); - } - } - } - matched.ok_or_else(|| { - invalid("selected summary leaf has no semantically identical original subtree witness") - }) -} - -/// Read the original aggregate operation only after proving its selected-node identity. -/// Min and max share a Planner accumulator family, so the family name alone is insufficient. -pub(super) fn selected_aggregate_operator( - original: &str, - selected: &planner_types::post_asap::SummaryNode, -) -> Result { - let (root, nodes) = selected_query_time_nodes(original, selected)?; - match nodes.get(&root) { - Some(QueryPlanNode::Logical { - operator: operator @ QueryTimeOperator::Aggregate { .. }, - .. - }) => Ok(operator.clone()), - _ => Err(invalid( - "selected value aggregation has no verified original aggregate operator", - )), - } -} - -#[cfg(test)] -mod hybrid_tests { - use super::*; - use crate::query_plan::{MaterializationBinding, PhysicalGrouping}; - #[test] - fn external_binding_rejects_an_unrelated_selected_exact_subtree() { - let exact = crate::query_parser::parse_query_expr_with_interval( - "sum_over_time(other_metric[5m])", - planner_types::types::AccuracyTarget::Exact, - 1_000, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&exact).unwrap(); - assert!(selected_native_expression("topk(2, sum_over_time(m[5m]))", &selected).is_err()); - assert_eq!( - selected_native_expression("sum_over_time(other_metric[5m])", &selected) - .unwrap() - .to_string(), - "sum_over_time(other_metric[5m])" - ); - } - - #[test] - fn selected_summary_and_filtered_residual_share_installed_binary() { - // Both filtered and unfiltered leaves bind independently. - let query = "sum_over_time(m[5m]) + sum_over_time(m{job=\"api\"}[5m])"; - let canonical = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&canonical).unwrap(); - let entry = - crate::query_plan::compile_bound_composable_mapped( - "hybrid".into(), - query.into(), - &selected, - InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }, - FallbackPolicy::Reject, - |node, _| { - let (_, _, spatial_filter) = - crate::physical::compiler::raw_materialization_input_contract(node) - .map_err(QueryPlanError::Invalid)?; - Ok(MaterializationBinding { - full_window_slide_ms: None, - item_labels: Vec::new(), - materialization: asap_types::PolicyFingerprint( - if spatial_filter.is_empty() { 7 } else { 8 }, - ) - .into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_output( - asap_types::PolicyFingerprint(if spatial_filter.is_empty() { - 7 - } else { - 8 - }) - .into(), - ), - output_grouping: PhysicalGrouping::PerEntity, - window_ms: 300_000, - pane_origin_ms: Some(0), - readout_lookback_ms: Some(300_000), - }) - }, - |_, _| {}, - ) - .unwrap(); - assert_eq!(entry.materialization_bindings().len(), 2); - assert!(!entry.nodes.values().any(|node| matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { .. }, - .. - } - ))); - assert!(!entry.nodes.values().any(|node| matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::Scan { .. }, - .. - } - ))); - assert!(matches!( - entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Binary { .. }, - .. - } - )); - entry - .validate( - &[ - asap_types::PolicyFingerprint(7), - asap_types::PolicyFingerprint(8), - ] - .into_iter() - .collect(), - ) - .unwrap(); - } - - #[test] - fn different_filter_cannot_witness_selected_residual() { - // Equality includes filter predicates, not just family, source, or window. - let canonical = crate::query_parser::parse_query_expr_canonical( - "sum_over_time(m{job=\"api\"}[5m])", - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&canonical).unwrap(); - assert!( - selected_query_time_nodes("sum_over_time(m{job=\"worker\"}[5m])", &selected).is_err() - ); - } -} - -#[cfg(test)] -mod planner_workload_tests { - use super::*; - use crate::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; - - fn compile_one(query: &str) -> crate::physical::compiler::CompiledPhysicalPlan { - let mut fixture: serde_json::Value = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let mut entry = fixture["query_workload"]["repeating_queries"][0].clone(); - entry["query"] = query.into(); - entry["requirements"]["accuracy"] = serde_json::json!({"explicit":"Exact"}); - fixture["query_workload"]["repeating_queries"] = vec![entry].into(); - let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); - let (request, environment) = snapshot - .into_physical_compilation_request() - .unwrap_or_else(|error| panic!("{query}: {error}")); - DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap_or_else(|error| panic!("{query}: {error}")) - } - - fn assert_local_limit(node: &QueryPlanNode) { - match node { - QueryPlanNode::PhysicalFragment { dag, .. } => { - let plan = - asap_physical_operators::physical_planner::CompiledPhysicalDag::decode(dag) - .unwrap(); - assert_eq!(plan.operator_name(plan.roots()[0]), Some("Limit")); - } - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { .. }, - .. - } => {} - _ => panic!("expected local Limit, got {node:?}"), - } - } - - #[test] - fn evaluation_topk_queries_retain_a_local_selection_root() { - for query in [ - "topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h])))", - "topk(2, sum by (job) (backend_process_resident_memory_bytes))", - "topk(2, max_over_time(backend_retry_backlog_depth[6h]))", - "topk(1, sum by (job) (rate(backend_process_cpu_seconds_total[6h])))", - "topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:]))", - ] { - let plan = compile_one(query); - let entry = plan.query_plan.entries.values().next().unwrap(); - assert_local_limit(&entry.nodes[&entry.root]); - assert!( - !entry - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. })), - "{query}" - ); - } - } - - #[test] - fn planner_value_topk_preserves_direct_summary_children() { - for query in [ - "topk(2, rate(backend_process_cpu_seconds_total[1h]))", - "topk(2, max_over_time(backend_retry_backlog_depth[6h]))", - ] { - let plan = compile_one(query); - let entry = plan.query_plan.entries.values().next().unwrap(); - assert_local_limit(&entry.nodes[&entry.root]); - assert!( - !entry.materialization_bindings().is_empty(), - "{query} must retain its SummaryStore child: {:?}", - entry.nodes - ); - } - } - - #[test] - fn whole_o11y_planner_candidate_lowers_every_original_query() { - // AST support alone is insufficient: the actual selected Planner forest must bind too. - let corpus: serde_json::Value = - serde_json::from_str(include_str!("../../tests/fixtures/o11y_queries.json")).unwrap(); - let mut fixture: serde_json::Value = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let template = fixture["query_workload"]["repeating_queries"][0].clone(); - let mut seen = std::collections::BTreeSet::new(); - let mut entries = Vec::new(); - for row in corpus["queries"].as_array().unwrap() { - let query = row["query"].as_str().unwrap(); - if !seen.insert(query.to_string()) { - continue; - } - let mut entry = template.clone(); - entry["query"] = query.into(); - entry["requirements"]["accuracy"] = serde_json::json!({"explicit":"Exact"}); - entries.push(entry); - } - fixture["query_workload"]["repeating_queries"] = entries.into(); - let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); - let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - assert!(request.allow_mixed_summary_and_exact_execution); - let plan = DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap(); - assert_eq!(plan.query_plan.entries.len(), 24); - assert!(plan.query_plan.entries.values().all(|entry| !entry - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. })))); - } - - #[test] - fn max_of_selected_values_uses_original_max_operator() { - // MinMax storage type does not authorize choosing min or replacing the selected operand graph. - let query = "max(sum_over_time(m[5m]) / 2)"; - let canonical = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - use asap_aware_mapping::{ - Replacement, ReplacementStrategy, SketchAlgorithmStrategy, TargetSubDAG, - }; - let root = std::rc::Rc::new(canonical); - let candidates = SketchAlgorithmStrategy::new(&asap_aware_mapping::DefaultCostModel) - .replacements(&TargetSubDAG::new(&root)); - let [candidate] = candidates.as_slice() else { - panic!("expected one exact aggregate candidate") - }; - let Replacement::Summary(selected) = &candidate.replacement else { - panic!("expected exact summary fixture") - }; - let operator = selected_aggregate_operator(query, selected).unwrap(); - assert!(matches!( - operator, - QueryTimeOperator::Aggregate { - operation: Aggregation::Max, - .. - } - )); - } - - #[test] - fn ambiguous_extremum_witness_is_rejected() { - // Different readouts over the same MinMax state cannot be resolved by taking the first AST match. - let canonical = crate::query_parser::parse_query_expr_canonical( - "min(m)", - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&canonical).unwrap(); - let maximum = crate::query_parser::parse_query_expr_canonical( - "max(m)", - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let maximum = crate::planner_selection::plan_test_query(&maximum).unwrap(); - let result = selected_query_time_nodes("min(m) + max(m)", &selected); - if selected == maximum { - assert!(result.is_err()); - } else { - let (root, nodes) = result.unwrap(); - assert!(matches!( - nodes[&root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Aggregate { - operation: Aggregation::Min, - .. - }, - .. - } - )); - } - } -} - -/// Preserve the exact original operator direction because MinMax family alone -/// does not distinguish min from max. The full Planner-node witness is required. -pub(crate) fn selected_range_max_materialization( - original: &str, - node: &planner_types::post_asap::SummaryNode, -) -> Result, QueryPlanError> { - use planner_types::post_asap::{ExactKind, SummaryExpr, SummaryFamilyType}; - let node = match &node.expr { - SummaryExpr::ValueOperation { - child, - operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - .. - } => child.as_ref(), - _ => node, - }; - if !matches!( - &node.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(ExactKind::Max, _), - reduction: planner_types::pre_asap::Reduction::PerEntity, - .. - } - ) { - return Ok(None); - } - let (root, nodes) = selected_query_time_nodes(original, node)?; - let Some(QueryPlanNode::Logical { - operator: - QueryTimeOperator::Temporal { - operation: TemporalOperation::Max, - }, - inputs, - }) = nodes.get(&root) - else { - return Ok(None); - }; - if inputs.len() != 1 || nodes.len() != 2 { - return Ok(None); - } - let Some(QueryPlanNode::Logical { - operator: - QueryTimeOperator::Scan { - metric: Some(metric), - matchers, - range_ms: Some(range_ms), - offset_ms: 0, - }, - .. - }) = nodes.get(&inputs[0]) - else { - return Ok(None); - }; - Ok(Some(materialization_candidate_key( - MaterializationCandidateIdentity { - metric: metric.clone(), - matchers: matchers.clone(), - range_ms: *range_ms, - offset_ms: 0, - operation: TemporalOperation::Max, - }, - )?)) -} - -#[cfg(test)] -mod range_max_materialization_tests { - use super::*; - #[test] - fn real_gauge_queries_have_planner_authorized_exact_materializations() { - for (query, metric, range_ms) in [ - ( - r#"max_over_time(service_cache_refresh_lag_seconds{job="user-service"}[12h])"#, - "service_cache_refresh_lag_seconds", - 43_200_000, - ), - ( - r#"max_over_time(service_retry_queue_depth{job=~".+"}[6h])"#, - "service_retry_queue_depth", - 21_600_000, - ), - ( - r#"max_over_time(service_retry_queue_depth{job="order-service"}[6h])"#, - "service_retry_queue_depth", - 21_600_000, - ), - ] { - let original = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&original).unwrap(); - let key = selected_range_max_materialization(query, &selected) - .unwrap() - .unwrap(); - assert!(key.contains(metric)); - assert!(key.contains(&range_ms.to_string())); - } - } - #[test] - fn min_and_shifted_or_nested_windows_do_not_become_max_materializations() { - for query in [ - "min_over_time(m[1m])", - "max_over_time(m[1m] offset 1m)", - "max_over_time((m + m)[1m:1s])", - ] { - let original = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&original).unwrap(); - assert!( - selected_range_max_materialization(query, &selected) - .unwrap() - .is_none(), - "{query}" - ); - } - } -} - -/// Stable contract identity used by priced physical candidates, independent of node IDs. -fn materialization_candidate_key( - candidate: MaterializationCandidateIdentity, -) -> Result { - let mut value = serde_json::to_value(candidate).map_err(|e| invalid(e.to_string()))?; - if let Some(object) = value.as_object_mut() { - // Retention is a consumer lifetime requirement, not the materialization read's - // semantics. Equivalent matcher conjunctions must share policy keys. - object.remove("retention_ms"); - if let Some(matchers) = object.get_mut("matchers").and_then(|v| v.as_array_mut()) { - matchers.sort_by_cached_key(|m| m.to_string()); - } - } - serde_json::to_string(&value).map_err(|e| invalid(e.to_string())) -} - -fn counter_contract( - root: QueryNodeId, - nodes: &BTreeMap, -) -> Option { - let QueryPlanNode::Logical { - operator: QueryTimeOperator::Temporal { operation }, - inputs, - } = nodes.get(&root)? - else { - return None; - }; - if !matches!( - operation, - TemporalOperation::Rate | TemporalOperation::Increase - ) || inputs.len() != 1 - { - return None; - } - let QueryPlanNode::Logical { - operator: - QueryTimeOperator::Scan { - metric: Some(metric), - matchers, - range_ms: Some(range_ms), - offset_ms, - }, - .. - } = nodes.get(&inputs[0])? - else { - return None; - }; - Some(MaterializationCandidateIdentity { - metric: metric.clone(), - matchers: matchers.clone(), - range_ms: *range_ms, - offset_ms: *offset_ms, - operation: *operation, - }) -} - -pub(crate) fn selected_counter_materialization( - original: &str, - node: &planner_types::post_asap::SummaryNode, -) -> Result, QueryPlanError> { - use planner_types::post_asap::{ExactKind, SummaryExpr, SummaryFamilyType}; - if !matches!( - &node.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(ExactKind::Rate | ExactKind::Increase, _), - reduction: planner_types::pre_asap::Reduction::PerEntity, - .. - } - ) { - return Ok(None); - } - let (root, nodes) = selected_query_time_nodes(original, node)?; - counter_contract(root, &nodes) - .map(materialization_candidate_key) - .transpose() -} - -fn prune(entry: &mut QueryPlanEntry) { - let mut seen = std::collections::BTreeSet::new(); - let mut pending = vec![entry.root]; - while let Some(id) = pending.pop() { - if seen.insert(id) { - if let Some(node) = entry.nodes.get(&id) { - pending.extend(node.inputs()); - } - } - } - entry.nodes.retain(|id, _| seen.contains(id)); -} - -/// Finish the installed DAG by externalizing every fragment raw subtree. -pub fn finalize_query_time_nodes(entry: &mut QueryPlanEntry) -> Result<(), QueryPlanError> { - externalize_query_time_nodes(entry)?; - assign_retention(entry) -} - -pub fn eligible_materialization_keys( - original: &str, - selected: &std::rc::Rc, -) -> Result, QueryPlanError> { - use planner_types::post_asap::SummaryExpr; - fn visit( - original: &str, - node: &std::rc::Rc, - keys: &mut std::collections::BTreeSet, - ) -> Result<(), QueryPlanError> { - if let Some(key) = selected_counter_materialization(original, node)? - .or(selected_range_max_materialization(original, node)?) - { - keys.insert(key); - } - match &node.expr { - SummaryExpr::BinaryOp { lhs, rhs, .. } => { - visit(original, lhs, keys)?; - visit(original, rhs, keys)?; - } - SummaryExpr::RelationalJoin { left, right, .. } => { - visit(original, left, keys)?; - visit(original, right, keys)?; - } - - SummaryExpr::ValueOperation { child, .. } => visit(original, child, keys)?, - SummaryExpr::SummaryAgg { child, .. } => visit(original, child, keys)?, - SummaryExpr::SummaryEstimate { summary_input, .. } - | SummaryExpr::SummaryDelete { summary_input, .. } => { - visit(original, summary_input, keys)? - } - SummaryExpr::SummaryMerge { children, .. } => { - for child in children { - visit(original, child, keys)?; - } - } - SummaryExpr::SummaryJoin { outer, inner, .. } => { - visit(original, outer, keys)?; - visit(original, inner, keys)?; - } - SummaryExpr::SummarySubtract { left, right } => { - visit(original, left, keys)?; - visit(original, right, keys)?; - } - SummaryExpr::KeepPreAsap(_) => {} - } - Ok(()) - } - let mut keys = std::collections::BTreeSet::new(); - visit(original, selected, &mut keys)?; - Ok(keys) -} - -fn expression_shape( - id: QueryNodeId, - nodes: &BTreeMap, -) -> Result { - let node = nodes - .get(&id) - .ok_or_else(|| invalid("missing expression node"))?; - if let QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { query }, - .. - } = node - { - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let expr = parser::parse(query).map_err(|e| invalid(e.to_string()))?; - let root = lower.lower(&expr)?; - return expression_shape(root, &lower.nodes); - } - let value = match node { - QueryPlanNode::Logical { operator, .. } => { - serde_json::to_string(operator).map_err(|e| invalid(e.to_string()))? - } - QueryPlanNode::Scalar { value } => format!("scalar:{:x}", value.to_bits()), - _ => return Err(invalid("summary node has no raw expression shape")), - }; - let children = node - .inputs() - .iter() - .map(|child| expression_shape(*child, nodes)) - .collect::, _>>()?; - Ok(format!("{value}({})", children.join(";"))) -} - -/// Collapse only maximal exact fragment subtrees whose full typed expression is -/// witnessed in the original query. Matrix boundaries remain inside Prometheus. -pub fn externalize_query_time_nodes(entry: &mut QueryPlanEntry) -> Result<(), QueryPlanError> { - fn gather(expr: &Expr, out: &mut Vec) { - if !matches!(expr, Expr::MatrixSelector(_) | Expr::Subquery(_)) { - out.push(expr.clone()); - } - match expr { - Expr::Paren(e) => gather(&e.expr, out), - Expr::Unary(e) => gather(&e.expr, out), - Expr::Subquery(e) => gather(&e.expr, out), - Expr::Aggregate(e) => gather(&e.expr, out), - Expr::Binary(e) => { - gather(&e.lhs, out); - gather(&e.rhs, out); - } - Expr::Call(e) => { - for arg in &e.args.args { - gather(arg, out); - } - } - _ => {} - } - } - let expr = parser::parse(&entry.canonical_query).map_err(|e| invalid(e.to_string()))?; - let mut expressions = Vec::new(); - gather(&expr, &mut expressions); - let mut witnesses = BTreeMap::new(); - for expression in expressions { - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - if let Ok(root) = lower.lower(&expression) { - witnesses.insert( - expression_shape(root, &lower.nodes)?, - expression.to_string(), - ); - } - } - fn flags(id: QueryNodeId, nodes: &BTreeMap) -> (bool, bool) { - let Some(node) = nodes.get(&id) else { - return (true, false); - }; - let mut indexed = !matches!( - node, - QueryPlanNode::Logical { .. } | QueryPlanNode::Scalar { .. } - ) || matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { .. }, - .. - } - ); - let mut exact = false; - if let QueryPlanNode::Logical { operator, .. } = node { - exact = matches!( - operator, - QueryTimeOperator::Scan { .. } - | QueryTimeOperator::ExactSubquery { .. } - | QueryTimeOperator::CandidateExactSubquery { .. } - ); - } - for child in node.inputs() { - let (a, b) = flags(*child, nodes); - indexed |= a; - exact |= b; - } - (indexed, exact) - } - let mut pending = vec![entry.root]; - while let Some(id) = pending.pop() { - if matches!( - entry.nodes.get(&id), - Some(QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { .. } - | QueryTimeOperator::CandidateExactSubquery { .. }, - .. - }) - ) { - continue; - } - let (indexed, exact) = flags(id, &entry.nodes); - if !indexed && exact { - if let Ok(shape) = expression_shape(id, &entry.nodes) { - if let Some(query) = witnesses.get(&shape) { - entry.nodes.insert( - id, - QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { - query: query.clone(), - }, - inputs: vec![], - }, - ); - continue; - } - } - } - pending.extend(entry.nodes[&id].inputs()); - } - prune(entry); - if entry.nodes.values().any(|node| { - matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::Scan { .. }, - .. - } - ) - }) { - return Err(invalid( - "local Scan is not deployable; exact subtree requires a complete Prometheus boundary", - )); - } - Ok(()) -} - -fn assign_retention(entry: &mut QueryPlanEntry) -> Result<(), QueryPlanError> { - let mut pending = vec![(entry.root, 0u64)]; - let mut depths = BTreeMap::new(); - while let Some((id, depth)) = pending.pop() { - if depths.get(&id).is_some_and(|prior| *prior >= depth) { - continue; - } - depths.insert(id, depth); - let node = entry - .nodes - .get_mut(&id) - .ok_or_else(|| invalid("missing index ancestor"))?; - let mut child_depth = depth; - if let QueryPlanNode::Logical { operator, .. } = node { - if let QueryTimeOperator::Subquery { - range_ms, - offset_ms, - .. - } = operator - { - child_depth = depth - .checked_add(*range_ms) - .and_then(|v| v.checked_add((*offset_ms).max(0) as u64)) - .ok_or_else(|| invalid("retention overflow"))?; - } - } - pending.extend(node.inputs().iter().map(|child| (*child, child_depth))); - } - Ok(()) -} - -#[cfg(test)] -mod remote_boundary_regressions { - use super::*; - - #[test] - fn stored_output_identity_is_independent_of_matcher_order() { - let first = LabelMatcher { - name: "job".into(), - value: "orders".into(), - operation: LabelMatch::Equal, - }; - let second = LabelMatcher { - name: "status".into(), - value: "5..".into(), - operation: LabelMatch::Regex, - }; - let a = MaterializationCandidateIdentity { - metric: "requests".into(), - matchers: vec![first.clone(), second.clone()], - range_ms: 300_000, - offset_ms: 0, - operation: TemporalOperation::Rate, - }; - let b = MaterializationCandidateIdentity { - metric: "requests".into(), - matchers: vec![second, first], - range_ms: 300_000, - offset_ms: 0, - operation: TemporalOperation::Rate, - }; - assert_eq!( - materialization_candidate_key(a).unwrap(), - materialization_candidate_key(b).unwrap() - ); - } - - #[test] - fn real_error_ratio_exposes_two_independent_materialization_candidates() { - let query = "sum(rate(http_requests_total{job=\"order-service\",status=~\"5..\"}[5m])) / sum(rate(http_requests_total{job=\"order-service\"}[5m]))"; - let parsed = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&parsed).unwrap(); - assert_eq!( - eligible_materialization_keys(query, &selected) - .unwrap() - .len(), - 2 - ); - } -} - -#[cfg(test)] -mod tests { - use super::*; - // A grouped query's fragment carries the grouping label in its leaf scan - // schema, because Planner resolves that schema against the whole query. - // Re-parsing the subtree alone cannot know the label, so requiring equal - // column sets rejected a fragment that is the subtree. - #[test] - fn grouped_query_residual_matches_its_own_subtree() { - for (query, subtree) in [ - ("sum by (label_0) (rate(data[1m]))", "rate(data[1m])"), - ( - "sum by (label_0) (sum_over_time(data[1m]))", - "sum_over_time(data[1m])", - ), - ] { - let fragment = crate::query_parser::parse_query_expr_with_interval( - subtree, - planner_types::types::AccuracyTarget::Exact, - 60_000, - ) - .unwrap(); - assert!( - query_time_nodes(query, &fragment).is_ok(), - "{query}: fragment {subtree} must resolve against its own query" - ); - } - } - - // Widening is only accepted for the context-derived leaf columns. Anything - // that actually identifies the computation still has to match exactly. - #[test] - fn widened_leaf_schema_does_not_excuse_a_different_computation() { - let fragment = crate::query_parser::parse_query_expr_with_interval( - "rate(data[1m])", - planner_types::types::AccuracyTarget::Exact, - 60_000, - ) - .unwrap(); - // Different metric. - assert!(query_time_nodes("sum by (label_0) (rate(other[1m]))", &fragment).is_err()); - // Different range. - assert!(query_time_nodes("sum by (label_0) (rate(data[2m]))", &fragment).is_err()); - // Different function. - assert!(query_time_nodes("sum by (label_0) (increase(data[1m]))", &fragment).is_err()); - // Different matcher. - let filtered = crate::query_parser::parse_query_expr_with_interval( - "rate(data{job=\"api\"}[1m])", - planner_types::types::AccuracyTarget::Exact, - 60_000, - ) - .unwrap(); - assert!(query_time_nodes( - "sum by (label_0) (rate(data{job=\"worker\"}[1m]))", - &filtered - ) - .is_err()); - } - - // A workload horizon changes the equality witness, never its filter or explicit range. - #[test] - fn workload_horizon_residual_keeps_semantic_equality() { - let fragment = crate::query_parser::parse_query_expr_with_interval( - "sum(m{job=\"api\"})", - planner_types::types::AccuracyTarget::Exact, - 5_000, - ) - .unwrap(); - assert!(query_time_nodes("sum(m{job=\"api\"})", &fragment).is_ok()); - assert!(query_time_nodes("sum(m{job=\"worker\"})", &fragment).is_err()); - let range = crate::query_parser::parse_query_expr_with_interval( - "sum_over_time(m[1m])", - planner_types::types::AccuracyTarget::Exact, - 5_000, - ) - .unwrap(); - assert!(query_time_nodes("sum_over_time(m[2m])", &range).is_err()); - } - - fn instant() -> InstantExecution { - InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - } - } - - #[test] - fn complete_o11y_corpus_lowers_to_serialized_operations() { - // Every original workload occurrence must compile to an executable typed graph. - let corpus: serde_json::Value = - serde_json::from_str(include_str!("../../tests/fixtures/o11y_queries.json")).unwrap(); - for row in corpus["queries"].as_array().unwrap() { - let query = row["query"].as_str().unwrap(); - let entry = crate::query_plan::query_time::compile_logical( - row["id"].as_str().unwrap().into(), - query.into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap_or_else(|error| panic!("{query}: {error}")); - let encoded = serde_json::to_string(&entry).unwrap(); - let restored: QueryPlanEntry = serde_json::from_str(&encoded).unwrap(); - restored.validate(&Default::default()).unwrap(); - assert!(!restored - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. }))); - } - } - #[test] - fn query_time_mapping_preserves_filters_and_rejects_different_sources() { - // Physical lowering must prove correspondence with the Planner-kept semantic subtree. - let query = "sum(rate(requests_total{job=\"api\"}[5m]))"; - let fragment = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let (_, nodes) = query_time_nodes(query, &fragment).unwrap(); - assert!(nodes.values().any(|node| matches!(node, QueryPlanNode::Logical { operator: QueryTimeOperator::Scan { matchers, .. }, .. } if matchers.iter().any(|m| m.name == "job" && m.value == "api")))); - assert!(query_time_nodes("sum(rate(other_total[5m]))", &fragment).is_err()); - } - #[test] - fn repeated_subexpressions_share_node_identity() { - // Serialized edges must retain CSE rather than duplicating raw work. - let entry = crate::query_plan::query_time::compile_logical( - "q".into(), - "sum(up) / sum(up)".into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap(); - let QueryPlanNode::Logical { inputs, .. } = &entry.nodes[&entry.root] else { - panic!("binary expected") - }; - assert_eq!(inputs[0], inputs[1]); - } - #[test] - fn malformed_operator_arity_is_rejected_at_installation() { - // A serialized graph cannot bypass the operation's input contract. - assert!(QueryTimeOperator::HistogramQuantile.validate(1).is_err()); - assert!(QueryTimeOperator::Subquery { - range_ms: 60_000, - step_ms: 0, - offset_ms: 0 - } - .validate(1) - .is_err()); - } - - #[test] - fn real_topk_queries_lower_to_value_selection() { - for (query, k) in [ - ( - "topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h])))", - 2, - ), - ( - "topk(2, sum by (job) (backend_process_resident_memory_bytes))", - 2, - ), - ("topk(2, max_over_time(backend_retry_backlog_depth[6h]))", 2), - ( - "topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", - 1, - ), - ( - "topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:]))", - 3, - ), - ] { - let entry = crate::query_plan::query_time::compile_logical( - "topk".into(), - query.into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap_or_else(|error| panic!("{query}: {error}")); - assert!(matches!( - entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { n: actual, .. }, - .. - } if actual == k - )); - } - } - - #[test] - fn topk_keeps_unsupported_child_as_exact_leaf() { - let entry = crate::query_plan::query_time::compile_logical( - "topk-subquery".into(), - "topk(3, label_replace(memory_bytes, \"dst\", \"$1\", \"src\", \"(.*)\"))".into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap(); - assert!(matches!( - entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { n: 3, .. }, - .. - } - )); - assert!(entry.nodes.values().any(|node| matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { .. }, - .. - } - ))); - } - - #[test] - fn topk_preserves_by_and_without_partitioning() { - for (query, labels, without) in [ - ("topk by (cluster) (2, m)", vec!["cluster"], false), - ("topk without (pod) (2, m)", vec!["pod"], true), - ] { - let entry = crate::query_plan::query_time::compile_logical( - "topk-group".into(), - query.into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap(); - assert!(matches!( - &entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { grouping, .. }, - .. - } if grouping.labels == labels && grouping.without == without - )); - } - } -} diff --git a/control_plane/src/runtime_samples.rs b/control_plane/src/runtime_samples.rs deleted file mode 100644 index 3812b6ce9..000000000 --- a/control_plane/src/runtime_samples.rs +++ /dev/null @@ -1,524 +0,0 @@ -//! Receiver for agents' [`sketch-runtime::GrpcExporter`] -//! batches — the **push side** of the control plane's real-time -//! decision loop. -//! -//! Agents running an embedded `sketch-runtime` Sampler call the -//! `asap.runtime.v1.RuntimeSamples.Push` RPC with batched -//! records. The handler appends each record to a bounded ring -//! buffer keyed by `(source, sketch, impl)`. Decision loops -//! peek the tail of that buffer to see the freshest throughput -//! / latency / accuracy signal. -//! -//! ## Why gRPC (vs the earlier HTTP+JSONL) -//! -//! HTTP/2 flow control surfaces control plane back-pressure to -//! the agent — critical when the decision loop is real-time. -//! See the design discussion thread for the full trade-off; -//! the summary is in -//! [`sketch-runtime::exporter::grpc`](https://github.com/ProjectASAP/sketch-bench/blob/main/sketch-runtime/src/exporter/grpc.rs). -//! -//! ## Why a ring buffer, not a stream -//! -//! Real-time decisions want the freshest N records, not a full -//! replay. Bounded memory, O(1) append + peek, FIFO eviction. -//! Longer windows live in the agent's `FileExporter` artifact. - -use std::collections::{HashMap, VecDeque}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; - -use parking_lot::RwLock; -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -// Generated from proto/feedback.proto by build.rs. -pub mod feedback { - tonic::include_proto!("asap.runtime.v1"); -} -use feedback::runtime_samples_server::{RuntimeSamples, RuntimeSamplesServer}; -use feedback::{PushAck, PushBatch}; - -/// One record as stored in the ring buffer. Kept as -/// `serde_json::Value` payload for schema forward-compat — new -/// fields on `sketch-core::report::Record` flow through without -/// a control plane rebump. Decision loops extract numeric fields -/// on demand. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RuntimeRecord { - pub source: String, - pub sketch: String, - #[serde(rename = "impl")] - pub impl_name: String, - #[serde(default)] - pub schema_version: u32, - /// The full `sketch-core::Record` payload, minus the - /// labelling fields already in this struct. Parsed from the - /// `RuntimeRecord.payload_json` field of the proto. - #[serde(flatten)] - pub payload: Value, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct SampleKey { - pub source: String, - pub sketch: String, - pub impl_name: String, -} - -#[derive(Debug, Default)] -pub struct RuntimeSamplesStats { - pub batches_received: AtomicU64, - pub records_stored: AtomicU64, - pub records_evicted: AtomicU64, - pub decode_errors: AtomicU64, - pub records_rejected: AtomicU64, -} - -impl RuntimeSamplesStats { - pub fn snapshot(&self) -> RuntimeSamplesStatsSnapshot { - RuntimeSamplesStatsSnapshot { - batches_received: self.batches_received.load(Ordering::Relaxed), - records_stored: self.records_stored.load(Ordering::Relaxed), - records_evicted: self.records_evicted.load(Ordering::Relaxed), - decode_errors: self.decode_errors.load(Ordering::Relaxed), - records_rejected: self.records_rejected.load(Ordering::Relaxed), - } - } -} - -#[derive(Debug, Clone, Copy, Serialize)] -pub struct RuntimeSamplesStatsSnapshot { - pub batches_received: u64, - pub records_stored: u64, - pub records_evicted: u64, - pub decode_errors: u64, - pub records_rejected: u64, -} - -/// Bounded FIFO ring buffer of runtime records, keyed by -/// `(source, sketch, impl)`. Each key has its own record cap. Global key and -/// serialized-byte limits evict whole oldest keys; consumers see missing -/// evidence rather than a partially retained observation. -pub struct RuntimeSamplesStore { - buffers: RwLock, - per_key_capacity: usize, - max_keys: usize, - max_json_bytes: usize, - stats: Arc, -} - -#[derive(Default)] -struct RuntimeBuffers { - by_key: HashMap>, - insertion_order: VecDeque, - json_bytes: usize, -} - -impl RuntimeBuffers { - fn remove(&mut self, key: &SampleKey) -> usize { - self.insertion_order.retain(|stored| stored != key); - let Some(records) = self.by_key.remove(key) else { - return 0; - }; - self.json_bytes -= records.iter().map(|(_, bytes)| bytes).sum::(); - records.len() - } -} - -// Count serialized bytes without allocating another copy of the observation. -fn encoded_size(record: &RuntimeRecord, limit: usize) -> Option { - struct Counter { - bytes: usize, - limit: usize, - } - impl std::io::Write for Counter { - fn write(&mut self, bytes: &[u8]) -> std::io::Result { - self.bytes = self - .bytes - .checked_add(bytes.len()) - .filter(|size| *size <= self.limit) - .ok_or_else(|| std::io::Error::other("runtime record exceeds byte budget"))?; - Ok(bytes.len()) - } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } - } - let mut count = Counter { bytes: 0, limit }; - serde_json::to_writer(&mut count, record).ok()?; - Some(count.bytes) -} - -impl RuntimeSamplesStore { - pub fn new(per_key_capacity: usize) -> Arc { - Self::with_limits(per_key_capacity, 1024, 16 * 1024 * 1024) - } - - /// Limits serialized retained metadata bytes, not allocator RSS. A zero - /// limit disables retention; rejected latest records invalidate older fits. - pub fn with_limits( - per_key_capacity: usize, - max_keys: usize, - max_json_bytes: usize, - ) -> Arc { - Arc::new(Self { - buffers: RwLock::new(RuntimeBuffers::default()), - per_key_capacity, - max_keys, - max_json_bytes, - stats: Arc::new(RuntimeSamplesStats::default()), - }) - } - - pub fn stats(&self) -> Arc { - Arc::clone(&self.stats) - } - - pub fn stats_handle(&self) -> Arc { - Arc::clone(&self.stats) - } - - #[cfg(test)] - pub(crate) fn append_for_test(&self, rec: RuntimeRecord) { - self.append(rec); - } - - fn invalidate(&self, key: &SampleKey) { - let removed = self.buffers.write().remove(key); - self.stats - .records_evicted - .fetch_add(removed as u64, Ordering::Relaxed); - } - - fn append(&self, rec: RuntimeRecord) -> bool { - let key = SampleKey { - source: rec.source.clone(), - sketch: rec.sketch.clone(), - impl_name: rec.impl_name.clone(), - }; - let size = encoded_size(&rec, self.max_json_bytes); - if self.per_key_capacity == 0 || self.max_keys == 0 || size.is_none() { - self.invalidate(&key); - self.stats.records_rejected.fetch_add(1, Ordering::Relaxed); - return false; - } - let size = size.unwrap(); - let mut buffers = self.buffers.write(); - if let Some(records) = buffers.by_key.get_mut(&key) { - if records.len() >= self.per_key_capacity { - let (_, bytes) = records.pop_front().unwrap(); - buffers.json_bytes -= bytes; - self.stats.records_evicted.fetch_add(1, Ordering::Relaxed); - } - } - while (buffers.by_key.len() >= self.max_keys && !buffers.by_key.contains_key(&key)) - || buffers.json_bytes.saturating_add(size) > self.max_json_bytes - { - let oldest = buffers - .insertion_order - .front() - .cloned() - .expect("retained keys have an order"); - let removed = buffers.remove(&oldest); - self.stats - .records_evicted - .fetch_add(removed as u64, Ordering::Relaxed); - } - if !buffers.by_key.contains_key(&key) { - buffers.insertion_order.push_back(key.clone()); - } - buffers - .by_key - .entry(key) - .or_default() - .push_back((rec, size)); - buffers.json_bytes += size; - self.stats.records_stored.fetch_add(1, Ordering::Relaxed); - true - } - - /// Peek the latest record for a given key, or `None` if the - /// key has never been seen. Used by the replanner / - /// decision loop to read freshness signals. - pub fn latest(&self, key: &SampleKey) -> Option { - self.buffers - .read() - .by_key - .get(key) - .and_then(|b| b.back().map(|(record, _)| record.clone())) - } - - /// Snapshot the full ring for a key. O(n) clone; non-hot-path only. - pub fn snapshot(&self, key: &SampleKey) -> Vec { - self.buffers - .read() - .by_key - .get(key) - .map(|b| b.iter().map(|(record, _)| record.clone()).collect()) - .unwrap_or_default() - } - - pub fn keys(&self) -> Vec { - self.buffers.read().by_key.keys().cloned().collect() - } -} - -/// tonic service impl. One instance wraps the shared store and -/// is added to a tonic `Server` listening on the runtime-samples -/// port. -pub struct RuntimeSamplesService { - store: Arc, -} - -impl RuntimeSamplesService { - pub fn new(store: Arc) -> Self { - Self { store } - } - - /// Return the server as a tonic-routed service with gzip - /// compression negotiated on both sides. - pub fn into_server(self) -> RuntimeSamplesServer { - RuntimeSamplesServer::new(self) - .accept_compressed(tonic::codec::CompressionEncoding::Gzip) - .send_compressed(tonic::codec::CompressionEncoding::Gzip) - } -} - -#[tonic::async_trait] -impl RuntimeSamples for RuntimeSamplesService { - async fn push( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - self.store - .stats - .batches_received - .fetch_add(1, Ordering::Relaxed); - let batch = request.into_inner(); - let mut accepted = 0_u64; - for pb in batch.records { - // The agent's RuntimeRecord carries the full v1 - // `sketch-core::Record` as JSON in `payload_json`. - // Parse it into our `Value`-flattened store struct; - // reject individual malformed records rather than - // failing the whole batch. - let mut payload: Value = match serde_json::from_str(&pb.payload_json) { - Ok(v) => v, - Err(e) => { - self.store - .stats - .decode_errors - .fetch_add(1, Ordering::Relaxed); - self.store.invalidate(&SampleKey { - source: pb.source.clone(), - sketch: pb.sketch.clone(), - impl_name: pb.impl_name.clone(), - }); - tracing::warn!(error = %e, "runtime-samples: malformed payload_json, invalidating source evidence"); - continue; - } - }; - // Strip the labelling fields out of `payload` — our - // RuntimeRecord carries them as typed fields, so - // duplicates inside payload would confuse downstream - // consumers. - if let Some(obj) = payload.as_object_mut() { - obj.remove("source"); - obj.remove("sketch"); - obj.remove("impl"); - } - let rec = RuntimeRecord { - source: pb.source, - sketch: pb.sketch, - impl_name: pb.impl_name, - schema_version: pb.schema_version, - payload, - }; - if self.store.append(rec) { - accepted += 1; - } - } - Ok(tonic::Response::new(PushAck { accepted })) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use feedback::RuntimeRecord as PbRecord; - - fn make_pb_record(source: &str, sketch: &str, impl_name: &str, tp: f64) -> PbRecord { - let payload = serde_json::json!({ - "schema_version": 1, - "mode": "runtime", - "timestamp": "2026-04-21T19:00:00Z", - "bench": { - "throughput_items_per_sec": { "mean": tp, "stddev": 0.0 }, - "latency_ns": { "p50": 10, "p99": 100 }, - "memory_bytes": 2048 - } - }); - PbRecord { - source: source.into(), - sketch: sketch.into(), - impl_name: impl_name.into(), - schema_version: 1, - payload_json: payload.to_string(), - } - } - - #[tokio::test] - async fn push_stores_records_under_per_key_rings() { - let store = RuntimeSamplesStore::new(16); - let svc = RuntimeSamplesService::new(Arc::clone(&store)); - let batch = PushBatch { - records: vec![ - make_pb_record("dc-a", "cms", "oxide", 1e6), - make_pb_record("dc-b", "hll", "lib", 2e6), - make_pb_record("dc-a", "cms", "oxide", 1.5e6), - ], - }; - let resp = svc.push(tonic::Request::new(batch)).await.expect("ok"); - assert_eq!(resp.into_inner().accepted, 3); - assert_eq!(store.keys().len(), 2); - let key = SampleKey { - source: "dc-a".into(), - sketch: "cms".into(), - impl_name: "oxide".into(), - }; - assert_eq!(store.snapshot(&key).len(), 2); - } - - #[tokio::test] - async fn push_rejects_malformed_payload_individually_but_accepts_rest() { - let store = RuntimeSamplesStore::new(16); - let svc = RuntimeSamplesService::new(Arc::clone(&store)); - let good1 = make_pb_record("dc-a", "cms", "oxide", 1e6); - let bad = PbRecord { - source: "dc-a".into(), - sketch: "cms".into(), - impl_name: "oxide".into(), - schema_version: 1, - payload_json: "{not json".into(), - }; - let good2 = make_pb_record("dc-a", "cms", "oxide", 2e6); - let batch = PushBatch { - records: vec![good1, bad, good2], - }; - let resp = svc.push(tonic::Request::new(batch)).await.expect("ok"); - assert_eq!(resp.into_inner().accepted, 2); - let snap = store.stats.snapshot(); - assert_eq!(snap.records_stored, 2); - assert_eq!(snap.decode_errors, 1); - } - - /// Replanning can create new definition keys forever; retain only a bounded - /// set and expose eviction as missing evidence. - #[tokio::test] - async fn global_key_limit_evicts_whole_oldest_source() { - let store = RuntimeSamplesStore::with_limits(4, 2, 16_384); - let svc = RuntimeSamplesService::new(Arc::clone(&store)); - for source in ["old", "current", "new"] { - let ack = svc - .push(tonic::Request::new(PushBatch { - records: vec![make_pb_record(source, "hll", "runtime", 10.0)], - })) - .await - .unwrap(); - assert_eq!(ack.into_inner().accepted, 1); - } - assert_eq!(store.keys().len(), 2); - assert!(store - .latest(&SampleKey { - source: "old".into(), - sketch: "hll".into(), - impl_name: "runtime".into() - }) - .is_none()); - assert_eq!(store.stats.snapshot().records_evicted, 1); - } - - /// Byte pressure never truncates population fits inside an observation. - #[tokio::test] - async fn byte_budget_and_oversized_latest_invalidate_old_evidence() { - let store = RuntimeSamplesStore::with_limits(10, 10, 600); - let svc = RuntimeSamplesService::new(Arc::clone(&store)); - for source in ["a", "b", "c", "d"] { - svc.push(tonic::Request::new(PushBatch { - records: vec![make_pb_record(source, "hll", "runtime", 10.0)], - })) - .await - .unwrap(); - assert!(store.buffers.read().json_bytes <= 600); - } - assert!(store.stats.snapshot().records_evicted > 0); - let key = SampleKey { - source: "d".into(), - sketch: "hll".into(), - impl_name: "runtime".into(), - }; - assert!(store.latest(&key).is_some()); - let mut oversized = make_pb_record("d", "hll", "runtime", 10.0); - oversized.payload_json = serde_json::json!({"large": "x".repeat(1000)}).to_string(); - let ack = svc - .push(tonic::Request::new(PushBatch { - records: vec![oversized], - })) - .await - .unwrap(); - assert_eq!(ack.into_inner().accepted, 0); - assert!(store.latest(&key).is_none()); - assert_eq!(store.stats.snapshot().records_rejected, 1); - } - - #[tokio::test] - async fn malformed_latest_and_disabled_retention_cannot_leave_old_fit() { - let store = RuntimeSamplesStore::new(2); - let svc = RuntimeSamplesService::new(Arc::clone(&store)); - let mut record = make_pb_record("a", "hll", "runtime", 10.0); - svc.push(tonic::Request::new(PushBatch { - records: vec![record.clone()], - })) - .await - .unwrap(); - record.payload_json = "{".into(); - svc.push(tonic::Request::new(PushBatch { - records: vec![record], - })) - .await - .unwrap(); - assert!(store.keys().is_empty()); - let disabled = RuntimeSamplesStore::with_limits(0, 2, 600); - let service = RuntimeSamplesService::new(Arc::clone(&disabled)); - let ack = service - .push(tonic::Request::new(PushBatch { - records: vec![make_pb_record("a", "hll", "runtime", 10.0)], - })) - .await - .unwrap(); - assert_eq!(ack.into_inner().accepted, 0); - assert!(disabled.keys().is_empty()); - } - - #[tokio::test] - async fn ring_evicts_oldest_past_capacity() { - let store = RuntimeSamplesStore::new(3); - let svc = RuntimeSamplesService::new(Arc::clone(&store)); - for _ in 0..5 { - svc.push(tonic::Request::new(PushBatch { - records: vec![make_pb_record("dc-a", "cms", "oxide", 1e6)], - })) - .await - .unwrap(); - } - let snap = store.stats.snapshot(); - assert_eq!(snap.records_stored, 5); - assert_eq!(snap.records_evicted, 2); - let key = SampleKey { - source: "dc-a".into(), - sketch: "cms".into(), - impl_name: "oxide".into(), - }; - assert_eq!(store.snapshot(&key).len(), 3); - } -} diff --git a/control_plane/src/types.rs b/control_plane/src/types.rs deleted file mode 100644 index cf9276f10..000000000 --- a/control_plane/src/types.rs +++ /dev/null @@ -1,823 +0,0 @@ -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use std::time::Duration; - -// ── Workload characteristics ─────────────────────────────────────────────────── - -pub use planner_types::workload::DataDistribution; - -/// Observable characteristics of the incoming data stream. -/// -/// Compatibility input and transient cost projection of canonical data facts. -/// The registry stores Planner DataWorkload; cost routines use this view to -/// compare raw vs. sketch-full vs. sketch-delta transmission costs and -/// estimate the CPU / memory overhead at the SDK or agent collector. -/// -/// All fields have conservative defaults so callers can omit the struct -/// entirely and still get a valid (if pessimistic) decision. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WorkloadCharacteristics { - /// Number of distinct active time series for this metric. - pub series_count: u64, - /// Sample rate per series at the SDK / agent (Hz). - pub samples_per_sec_per_series: f64, - /// Wire size of one raw OTLP metric data point after protobuf encoding - /// (bytes). Typical range: 50–200 bytes. - pub bytes_per_raw_sample: u32, - /// Known distinct key values per flush period for frequency / cardinality - /// sketches. `None` → inferred analytically from inserts and distribution. - pub distinct_keys_per_window: Option, - /// Statistical distribution of keys in the stream. - pub data_distribution: DataDistribution, - /// Optional memory cap at the SDK / agent collector (bytes). - /// `None` → no budget constraint applied. - pub memory_budget_bytes: Option, -} - -impl Default for WorkloadCharacteristics { - fn default() -> Self { - Self { - series_count: 1_000, - samples_per_sec_per_series: 100.0, - bytes_per_raw_sample: 100, - distinct_keys_per_window: None, - data_distribution: DataDistribution::Zipf, - memory_budget_bytes: None, - } - } -} - -// ── Delta transmission decision ──────────────────────────────────────────────── - -/// Reason the planner chose not to enable delta encoding. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum DeltaSkipReason { - /// Estimated fill rate is so high that delta compression ratio < 2×, - /// making snapshot overhead unjustifiable. - FillRateTooHigh, - /// Snapshot memory for all series × sketches would exceed the configured - /// memory budget at the agent. - MemoryBudgetExceeded, - /// This sketch type has no delta implementation (e.g. KLL). - SketchTypeUnsupported, - /// Compression ratio fell below the minimum acceptable threshold. - CompressionRatioBelowThreshold, -} - -/// Reason the planner chose raw pass-through over sketch transmission. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RawDataReason { - /// Series count × sample rate is so small that sketch CPU / memory - /// overhead is not justified by the bandwidth savings. - WorkloadTooSmall, -} - -/// The control plane's resolved decision on transmission mode. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "mode", rename_all = "snake_case")] -pub enum DeltaDecision { - /// Enable delta-compressed sketch payloads. - UseDelta { - /// Minimum absolute cell change included in a delta payload (T). - threshold: f64, - /// Estimated compression ratio (full_bytes / delta_bytes). - estimated_compression_ratio: f64, - /// Estimated outbound bandwidth with delta enabled (bytes/sec). - estimated_delta_bytes_per_sec: f64, - /// Additional CPU at the agent per sample due to snapshot diff and - /// sparse encoding (µs/sample), amortised over the flush period. - delta_cpu_overhead_micros_per_sample: f64, - /// Additional memory at the agent for storing snapshots (bytes). - delta_memory_overhead_bytes: f64, - }, - /// Transmit full (non-delta) sketch payloads each flush. - UseFullSketch { - reason: DeltaSkipReason, - /// Estimated outbound bandwidth with full sketches (bytes/sec). - estimated_full_bytes_per_sec: f64, - }, - /// Skip sketch aggregation; pass raw OTLP samples through. - UseRaw { - reason: RawDataReason, - /// Estimated outbound bandwidth with raw samples (bytes/sec). - estimated_raw_bytes_per_sec: f64, - }, -} - -impl Default for DeltaDecision { - fn default() -> Self { - DeltaDecision::UseFullSketch { - reason: DeltaSkipReason::CompressionRatioBelowThreshold, - estimated_full_bytes_per_sec: 0.0, - } - } -} - -/// Bandwidth and overhead estimates for all three transmission strategies. -/// Carried on every [`CollectionPlan`] for observability and debugging. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct TransmissionCostSummary { - /// Raw OTLP pass-through bandwidth (bytes/sec). - pub raw_bytes_per_sec: f64, - /// Full-sketch transmission bandwidth (bytes/sec). - pub sketch_full_bytes_per_sec: f64, - /// Delta-sketch transmission bandwidth (bytes/sec); 0 if delta not viable. - pub sketch_delta_bytes_per_sec: f64, - /// Extra CPU at the agent per sample in delta mode (µs/sample). - pub delta_cpu_overhead_micros_per_sample: f64, - /// Extra memory at the agent for delta snapshots (bytes). - pub delta_memory_overhead_bytes: f64, - /// Estimated fill rate (fraction of sketch cells changed per flush). - pub estimated_fill_rate: f64, - /// Flush rate derived from window_duration or repeat_every (Hz). - pub flush_rate_hz: f64, -} - -// ── Enumerations ────────────────────────────────────────────────────────────── - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum AggType { - Quantile, - Cardinality, - Frequency, -} - -impl std::fmt::Display for AggType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - AggType::Quantile => write!(f, "quantile"), - AggType::Cardinality => write!(f, "cardinality"), - AggType::Frequency => write!(f, "frequency"), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum SketchType { - DDSketch, - KLL, - HLL, - CountSketch, - CountMinSketch, -} - -impl std::fmt::Display for SketchType { - /// Returns the OTel Collector component type string (must match the Go - /// factory's `component.MustNewType(…)` in each processor). - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - SketchType::DDSketch => write!(f, "ddsketch"), - SketchType::KLL => write!(f, "KLL"), - SketchType::HLL => write!(f, "HLL"), - SketchType::CountSketch => write!(f, "countsketch"), - SketchType::CountMinSketch => write!(f, "countmin"), - } - } -} - -// Only the five sketch families in the deployment capability table can be -// converted to this deployment vocabulary; exact accumulators are not inputs. -impl From for SketchType { - fn from(k: planner_types::post_asap::SketchAlgorithm) -> Self { - use planner_types::post_asap::SketchAlgorithm; - match k { - SketchAlgorithm::Kll => SketchType::KLL, - SketchAlgorithm::DDSketch => SketchType::DDSketch, - SketchAlgorithm::Hll => SketchType::HLL, - SketchAlgorithm::Cms => SketchType::CountMinSketch, - SketchAlgorithm::CountSketch => SketchType::CountSketch, - other => unreachable!( - "SketchType::from(SketchAlgorithm): no legacy equivalent for {other:?} \ - (callers only pass the 5 canonical families)" - ), - } - } -} - -/// Inverse of `From for SketchType`. Round-trippable for the 5 -/// canonical families: `SketchAlgorithm::from(SketchType::from(k)) == k`. -impl From for planner_types::post_asap::SketchAlgorithm { - fn from(t: SketchType) -> Self { - use planner_types::post_asap::SketchAlgorithm; - match t { - SketchType::KLL => SketchAlgorithm::Kll, - SketchType::DDSketch => SketchAlgorithm::DDSketch, - SketchType::HLL => SketchAlgorithm::Hll, - SketchType::CountMinSketch => SketchAlgorithm::Cms, - SketchType::CountSketch => SketchAlgorithm::CountSketch, - } - } -} - -#[derive(Debug, Clone, PartialEq)] -pub enum OutputMode { - Raw, - Sketch, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ProcessorMode { - Batch, - Window, -} - -impl std::fmt::Display for ProcessorMode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ProcessorMode::Batch => write!(f, "batch"), - ProcessorMode::Window => write!(f, "window"), - } - } -} - -// ── Core types ──────────────────────────────────────────────────────────────── - -// ── Sketch defaults (YAML-configurable) ────────────────────────────────────── - -/// Per-sketch-type default parameters. Loaded from a YAML config file at -/// startup; falls back to compile-time defaults when the file is absent. -/// -/// Example `sketch_params_default.yml`: -/// ```yaml -/// quantile_grid: [0.0, 0.25, 0.5, 0.75, 0.9, 0.99, 1.0] -/// ddsketch: -/// relative_accuracy: 0.01 -/// kll: -/// min_k: 32 -/// hll: -/// precision_coarse: 10 -/// precision_fine: 14 -/// precision_threshold: 0.02 -/// count_sketch: -/// epsilon: 0.022 -/// delta: 0.007 -/// count_min_sketch: -/// rows: 5 -/// cols: 2048 -/// metric_name: "countsketch_partition" -/// ``` -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct SketchDefaults { - /// Quantile grid used when no query-specific φ values are available. - pub quantile_grid: Vec, - pub ddsketch: DDSketchDefaults, - pub kll: KLLDefaults, - pub hll: HLLDefaults, - pub count_sketch: CountSketchDefaults, - pub count_min_sketch: CountMinSketchDefaults, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct DDSketchDefaults { - pub relative_accuracy: f64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct KLLDefaults { - /// Minimum k value (clamped from 1/accuracy_sla). - pub min_k: u32, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct HLLDefaults { - /// Precision for coarse SLA (accuracy > threshold). - pub precision_coarse: u32, - /// Precision for fine SLA (accuracy ≤ threshold). - pub precision_fine: u32, - /// SLA boundary between coarse and fine precision. - pub precision_threshold: f64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct CountSketchDefaults { - /// Relative error bound (ε ≈ 1/√cols). - pub epsilon: f64, - /// Error probability (δ ≈ e^(−rows)). - pub delta: f64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct CountMinSketchDefaults { - pub rows: u32, - pub cols: u32, - pub metric_name: String, -} - -impl Default for SketchDefaults { - fn default() -> Self { - Self { - quantile_grid: vec![0.0, 0.25, 0.5, 0.75, 0.9, 0.99, 1.0], - ddsketch: DDSketchDefaults::default(), - kll: KLLDefaults::default(), - hll: HLLDefaults::default(), - count_sketch: CountSketchDefaults::default(), - count_min_sketch: CountMinSketchDefaults::default(), - } - } -} - -impl Default for DDSketchDefaults { - fn default() -> Self { - Self { - relative_accuracy: 0.01, - } - } -} - -impl Default for KLLDefaults { - fn default() -> Self { - Self { min_k: 32 } - } -} - -impl Default for HLLDefaults { - fn default() -> Self { - Self { - precision_coarse: 10, - precision_fine: 14, - precision_threshold: 0.02, - } - } -} - -impl Default for CountSketchDefaults { - fn default() -> Self { - Self { - epsilon: 0.022, - delta: 0.007, - } - } -} - -impl Default for CountMinSketchDefaults { - fn default() -> Self { - Self { - rows: 5, - cols: 2048, - metric_name: "countsketch_partition".into(), - } - } -} - -impl SketchDefaults { - /// Load from a YAML file, falling back to compiled defaults on any error. - pub fn load(path: &str) -> Self { - match std::fs::read_to_string(path) { - Ok(contents) => serde_yaml::from_str(&contents).unwrap_or_else(|e| { - tracing::warn!(path, error = %e, "invalid sketch_defaults YAML; using built-in defaults"); - Self::default() - }), - Err(_) => Self::default(), - } - } -} - -/// Per-sketch-type parameters. Each variant carries only the fields relevant -/// to that sketch family, avoiding the "bag of unrelated fields" problem. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum SketchParams { - DDSketch { - relative_accuracy: f64, - quantiles: Vec, - }, - KLL { - k: u32, - quantiles: Vec, - }, - HLL { - precision: u32, - }, - CountSketch { - /// Relative error bound (ε). - epsilon: f64, - /// Error probability (δ). - delta: f64, - }, - CountMinSketch { - rows: u32, - cols: u32, - /// Metric name required by the CMS processor. - metric_name: String, - }, -} - -impl Default for SketchParams { - fn default() -> Self { - let d = SketchDefaults::default(); - SketchParams::DDSketch { - relative_accuracy: d.ddsketch.relative_accuracy, - quantiles: d.quantile_grid, - } - } -} - -impl SketchParams { - /// Extract quantiles if this sketch type supports them. - pub fn quantiles(&self) -> &[f64] { - match self { - SketchParams::DDSketch { quantiles, .. } | SketchParams::KLL { quantiles, .. } => { - quantiles - } - _ => &[], - } - } -} - -/// GOS delta-gating knobs pushed to the edge (Count-Sketch families). -/// `epsilon` = the staleness share ε_st of the metric's accuracy budget; -/// `sites` = k (fleet size for the merged-error bound); `anisotropic` selects -/// the per-cell {T_j} water-filling (O(d·w) edge memory) over the isotropic -/// scalar (O(1)). -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct GosKnobs { - pub epsilon: f64, - pub sites: u32, - pub anisotropic: bool, -} - -impl GosKnobs { - /// Derive the edge delta-gate knobs from the metric's ε budget and the - /// edge-CPU-vs-communication cost weights: `epsilon` is the **staleness - /// share** ε_st from [`crate::epsilon_alloc::split_budget`] (the rest, ε_sa, - /// would fund sampling). `sites` = fleet size k (≥1); `anisotropic` selects - /// per-cell {T_j} over the isotropic scalar at the edge. - pub fn derive(epsilon: f64, sites: u32, w_edge: f64, w_comm: f64, anisotropic: bool) -> Self { - // The sketch's own ε is accounted separately (accuracy.rs), so the delta - // gate splits the full accuracy budget (eps_sketch=0 here). - let (_eps_sa, eps_st) = crate::epsilon_alloc::split_budget(epsilon, 0.0, w_edge, w_comm); - Self { - epsilon: eps_st, - sites: sites.max(1), - anisotropic, - } - } -} - -#[derive(Debug, Clone)] -pub struct AgentCollectorConfig { - pub output_mode: OutputMode, - pub sketch_type: SketchType, - pub sketch_params: SketchParams, - pub aggregate_by: Vec, - pub label_matchers: Vec, - pub window_duration: Option, - pub mode: ProcessorMode, - pub enable_self_monitoring: bool, - pub transmit_sketch: bool, - pub drop_original: bool, - /// Whether to enable the series ID (UID) registry on the OTLP receiver. - /// When true, the receiver caches metric name + attributes per series and - /// assigns a compact `series_id`. Subsequent exports can omit attributes - /// and send only the ID, saving ~120 bytes per sample. - pub enable_series_id: bool, - /// TTL for series ID cache entries (seconds). 0 = use receiver default. - pub series_id_ttl_secs: u64, - /// Whether the agent processor should enable delta encoding. - /// Set by the delta cost model after sketch type selection. - pub delta_transmission: bool, - /// Minimum absolute cell change included in a delta payload (T). - /// Ignored when `delta_transmission` is false. - pub delta_threshold: f64, - /// GOS relative delta gating (design-gos-unified-edge-telemetry.md §7). - /// When set, the edge replaces the fixed `delta_threshold` with the - /// norm-adaptive GOS threshold; `None` = fixed threshold (unchanged). - /// Produced by the ε-budget split (`epsilon_alloc::split_budget`). - pub gos: Option, - /// Data sink the planner wants the agent to emit to. Decoupled - /// from the planner output (which sketch / window / projection) - /// because where the data goes is a deployment-scope concern, - /// not a planning concern. The previous hardcoded - /// "prometheus exporter on :8889" approach broke the moment we - /// tried to ship sketch types — stock Prometheus exporter - /// silently drops `DDSketchDataPoint` / `HLLSketchDataPoint` - /// etc. — so emit OTLP-to-backend for sketch deployments and - /// keep the prometheus path only for legacy raw-scalar pipelines. - pub data_sink: AgentDataSink, -} - -/// What the agent's collector exports to. -/// -/// `Otlp` — the agent's pipeline ends with an OTLP exporter -/// pointed at the configured endpoint. Required for sketch -/// transport: the modified-OTLP `Data::Ddsketch` / `KLLSketch` / -/// etc. variants are carried natively over OTLP and decoded by -/// the backend's `OtlpReceiver` + the per-sketch -/// `from_sketchlib_proto_bytes` / `from_msgpack_bytes` decoders. -/// -/// `PrometheusScrape` — agent exposes `/metrics` on the listed -/// host:port for an external scraper. Loses sketch types at the -/// translation step; only useful for raw-scalar pipelines. -#[derive(Debug, Clone)] -pub enum AgentDataSink { - /// `endpoint` is an OTLP gRPC endpoint, e.g. `data-plane:4317`. - /// `compression` is the transport-level codec; the canonical - /// path uses `none` because the backend's tonic gRPC server - /// rejects gzip-compressed bodies (returns Unimplemented). - Otlp { - endpoint: String, - compression: String, - }, - /// Pre-existing path: prometheus exporter at `endpoint`. Kept - /// for back-compat with the legacy raw-scalar deployment. - PrometheusScrape { endpoint: String }, -} - -impl Default for AgentDataSink { - /// Default is OTLP-to-backend at the canonical compose - /// hostname. Override per-deployment via the planner's - /// `--agent-data-sink` flag (or future config push). - fn default() -> Self { - AgentDataSink::Otlp { - endpoint: "data-plane:4317".to_string(), - compression: "none".to_string(), - } - } -} - -#[derive(Debug, Clone)] -pub struct GatewayCollectorConfig { - pub passthrough: bool, -} - -// ── Per-stage resource budgets ──────────────────────────────────────────────── - -/// Per-stage resource caps. The agent cap feeds the physical planner's -/// placement decision (sketch build deferred off the agent when it would -/// exceed the cap). -/// -/// `None` means unbounded (no cap enforced). Typically sourced from -/// [`WorkloadCharacteristics::memory_budget_bytes`] for the agent stage and -/// from a runtime config file for backend / precompute stages. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct StageResourceBudgets { - /// Max sketch memory at the agent OTel Collector (bytes). - pub agent_memory_bytes: Option, - /// Max sketch-insertion CPU budget at the agent (µs/sample). - pub agent_cpu_micros_per_sample: Option, - /// Max sketch memory at the backend OTel Collector (bytes). - pub backend_memory_bytes: Option, - /// Max memory at the ASAPQuery Precompute Engine (bytes). - pub precompute_memory_bytes: Option, -} - -impl StageResourceBudgets {} - -// ── Collection plan ─────────────────────────────────────────────────────────── - -#[derive(Debug, Clone)] -pub struct CollectionPlan { - pub agent_config: AgentCollectorConfig, - pub gateway_config: GatewayCollectorConfig, - pub valid_until: DateTime, - /// Resolved delta transmission decision and rationale. - pub delta_decision: DeltaDecision, - /// Bandwidth and overhead estimates for all three transmission strategies. - pub transmission_cost_summary: TransmissionCostSummary, -} - -pub use asap_types::QueryLanguage; - -// ── AccuracyTarget ──────────────────────────────────────────────────────────── - -/// Per-target accuracy SLA, shared with ASAPPlanner's `AggIntent` fields. -/// Uses Planner's externally tagged serde representation, e.g. -/// `{"Epsilon": 0.02}`; `EpsilonDelta` names its bound `epsilon`. -pub use planner_types::types::AccuracyTarget; - -/// Translate the legacy `accuracy_sla: f64` field -- a fractional -/// "1.0 = exact, 0.0 = anything goes" SLA -- into the typed form. -/// `accuracy_sla == 1.0` round-trips to `Exact`; everything else becomes -/// `Epsilon(1.0 - accuracy_sla)` (the implied error bound). Free function, -/// not `impl AccuracyTarget` -- Rust's orphan rules don't allow inherent -/// impls on a foreign type. -pub fn accuracy_target_from_legacy_accuracy_sla(accuracy_sla: f64) -> AccuracyTarget { - if accuracy_sla >= 1.0 { - AccuracyTarget::Exact - } else { - AccuracyTarget::Epsilon((1.0 - accuracy_sla).max(0.0)) - } -} - -/// Resolve the public compatibility input once, preserving typed delta/exact semantics. -pub fn resolve_accuracy_target( - typed: Option<&AccuracyTarget>, - legacy_confidence: f64, -) -> Result { - let target = if let Some(target) = typed { - target.clone() - } else { - if !legacy_confidence.is_finite() || !(0.0..=1.0).contains(&legacy_confidence) { - return Err("accuracy_sla must be finite and in [0,1]".into()); - } - accuracy_target_from_legacy_accuracy_sla(legacy_confidence) - }; - let valid = match target { - AccuracyTarget::Exact => true, - AccuracyTarget::Epsilon(epsilon) => epsilon.is_finite() && (0.0..=1.0).contains(&epsilon), - AccuracyTarget::EpsilonDelta { epsilon, delta } => { - epsilon.is_finite() - && (0.0..=1.0).contains(&epsilon) - && delta.is_finite() - && delta > 0.0 - && delta < 1.0 - } - }; - if valid { - Ok(target) - } else { - Err("invalid typed accuracy requirement".into()) - } -} - -// ── QueryShape ──────────────────────────────────────────────────────────────── - -/// How the query is *evaluated*: one-shot, continuous, or scheduled. -/// -/// Drives L4 binding (mergeable vs one-shot sketch family) and the L5 -/// wire format (`OneShot` emits config + result; `Streaming` and -/// `Periodic` emit a config that keeps running). Distinct from -/// [`DataShape`] — see `design.md` §6. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum QueryShape { - /// Evaluate once. Plan, execute, return result, discard state. - /// One-off PromQL via `POST /plan`. - #[default] - OneShot, - /// Continuous query — output stream that the executor keeps emitting - /// as new data arrives. No fixed cadence; the runtime emits whenever - /// the underlying state changes. Streaming dashboards, alerting - /// expressions evaluated by the agent rather than by a poller. - Streaming, - /// Re-evaluated at a fixed cadence — Prometheus recording rules, - /// scheduled dashboard panels, alerting evaluation cycles. The - /// planner amortises sketch / aggregate build cost across - /// evaluations within the cadence and reuses state between - /// adjacent windows. - Periodic { - /// Re-evaluation cadence. - every: Duration, - }, -} - -// ── DataShape ───────────────────────────────────────────────────────────────── - -/// Shape of the *data* feeding the query. Workload-level summary; per-leaf -/// detail rides on `Source::data_shape` per `design.md` §6 L3. -/// -/// Drives L4 binding choices: an `AppendOnlyStream` unlocks incremental, -/// mergeable sketches and retraction-free aggregation; `Batch` lets the -/// planner pick a non-mergeable estimator (e.g. exact percentile over a -/// sort) that wouldn't survive a distributed streaming setting; `Mutable` -/// requires retraction-aware operators (out of scope today — the planner -/// refuses sketch binding and falls back to re-scan). -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] -#[serde(rename_all = "snake_case")] -pub enum DataShape { - /// Bounded relation, fully materialised at plan time. Parquet / CSV - /// files, in-process columnar tables. - Batch, - /// Append-only stream — events arrive over time, never updated or - /// deleted. Metrics, logs, event streams. The common case for the - /// asap-collector / asap-query deployments and so the control plane - /// default when the field is omitted. - #[default] - AppendOnlyStream, - /// Mutable relation — inserts + updates + deletes. Operational - /// databases, CRUD-style tables. Sketch binding is currently - /// refused for this shape (no retraction-aware sketches in the - /// catalog yet). - Mutable, - /// Join across sources of differing shape. The planner consults - /// `Source::data_shape` per leaf during L4; this variant exists so - /// callers don't have to flatten a workload-level summary. - Mixed, -} - -// ── QueryId ─────────────────────────────────────────────────────────────────── - -/// Stable identifier preserved across `replan` cycles so the runtime can -/// correlate plan outputs with the originating spec, and L4 reuse rules -/// can name shared producers across consumers in the same workload. -/// -/// Idiomatically a string here — the control plane already round-trips -/// metric names + agent IDs as strings (see `monitor::Endpoint`, -/// `opamp::AgentRole`), so adding a `Uuid` dependency for one field -/// with no DB-side semantics would be churn for no benefit. The HTTP -/// API is JSON; callers can supply any string they want, including a -/// stringified UUID. When omitted, `Analyzer::analyze` derives a -/// deterministic id from the parsed metric name + accuracy target. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] -#[serde(transparent)] -pub struct QueryId(pub String); - -impl std::fmt::Display for QueryId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.0) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn query_language_serde_roundtrip() { - let variant = QueryLanguage::PromQl; - let json = serde_json::to_string(&variant).unwrap(); - let back: QueryLanguage = serde_json::from_str(&json).unwrap(); - assert_eq!(variant, back, "round-trip failed for {variant:?}"); - } - - #[test] - fn accuracy_target_serde_roundtrip() { - let cases = [ - AccuracyTarget::Exact, - AccuracyTarget::Epsilon(0.05), - AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.001, - }, - ]; - for variant in cases { - let json = serde_json::to_string(&variant).unwrap(); - let back: AccuracyTarget = serde_json::from_str(&json).unwrap(); - assert_eq!(variant, back, "round-trip failed for {variant:?}"); - } - } - - #[test] - fn accuracy_target_from_legacy() { - // 1.0 means exact in the legacy schema. - assert_eq!( - accuracy_target_from_legacy_accuracy_sla(1.0), - AccuracyTarget::Exact - ); - // 0.99 SLA → ε = 0.01. - match accuracy_target_from_legacy_accuracy_sla(0.99) { - AccuracyTarget::Epsilon(eps) => { - assert!((eps - 0.01).abs() < 1e-9, "got eps={eps}"); - } - other => panic!("expected Epsilon, got {other:?}"), - } - // Out-of-range guard — analyzer rejects these upstream, but the - // helper itself must not panic on a 0.0 SLA. - match accuracy_target_from_legacy_accuracy_sla(0.0) { - AccuracyTarget::Epsilon(eps) => assert!((eps - 1.0).abs() < 1e-9), - other => panic!("expected Epsilon, got {other:?}"), - } - } - - #[test] - fn query_shape_serde_roundtrip_and_default() { - assert_eq!(QueryShape::default(), QueryShape::OneShot); - let cases = [ - QueryShape::OneShot, - QueryShape::Streaming, - QueryShape::Periodic { - every: Duration::from_secs(60), - }, - ]; - for variant in cases { - let json = serde_json::to_string(&variant).unwrap(); - let back: QueryShape = serde_json::from_str(&json).unwrap(); - assert_eq!(variant, back, "round-trip failed for {variant:?}"); - } - } - - #[test] - fn data_shape_serde_roundtrip_and_default() { - assert_eq!(DataShape::default(), DataShape::AppendOnlyStream); - for variant in [ - DataShape::Batch, - DataShape::AppendOnlyStream, - DataShape::Mutable, - DataShape::Mixed, - ] { - let json = serde_json::to_string(&variant).unwrap(); - let back: DataShape = serde_json::from_str(&json).unwrap(); - assert_eq!(variant, back, "round-trip failed for {variant:?}"); - } - } - - #[test] - fn query_id_transparent_string_serde() { - let id = QueryId("metric-x@0.99".into()); - let json = serde_json::to_string(&id).unwrap(); - // `#[serde(transparent)]` should serialize as a bare string. - assert_eq!(json, "\"metric-x@0.99\""); - let back: QueryId = serde_json::from_str(&json).unwrap(); - assert_eq!(id, back); - assert_eq!(id.0, "metric-x@0.99"); - } -} diff --git a/control_plane/src/workload.rs b/control_plane/src/workload.rs deleted file mode 100644 index 395ce65ca..000000000 --- a/control_plane/src/workload.rs +++ /dev/null @@ -1,1150 +0,0 @@ -//! Declarative workload registration. -//! -//! Loads a YAML file describing workloads and their assignments so the -//! control plane can pre-populate the plan store and assign workloads to -//! agents on connect without requiring an explicit HTTP `POST /api/v1/plan`. - -use serde::{Deserialize, Deserializer, Serialize}; -use tracing::{info, warn}; - -use crate::types::SketchType; -use planner_types::pre_asap::AggIntent; - -/// Aggregation role a single (metric, query-shape) pair plays in the planner. -/// -/// **Why this exists** (B2 full restructure): a single metric can carry -/// MULTIPLE aggregation roles when the workload YAML registers more than -/// one PromQL shape for it. The canonical case is -/// `http_requests_total`, which the MVP demo's `mvp-workload.yaml` -/// registers three times (entries 2/3/4 of [`deploy/configs/mvp-workload.yaml`]): -/// * `sum by (zone) (http_requests_total)` → [`AggRole::Sum`] -/// * `sum by (zone) (rate(http_requests_total[5m]))` → [`AggRole::Rate`] -/// * `count(http_requests_total{zone="z0"})` → [`AggRole::Count`] -/// -/// Before this enum: the `WorkloadStore` was keyed by metric name alone -/// and `set(metric, …)` overwrote on collision — only the LAST entry -/// survived, so the `sum by (zone)` query (entries 2 + 3) lost its plan -/// and the data-plane refused it with `ExactAgg(Sum) capability not -/// satisfied` (the surviving plan was DDSketch from entry 1's quantile -/// shape). -/// -/// After: the store is keyed by `(metric, role)` so each shape gets its -/// own plan, its own `PrecomputeMaterialization` on the backend's streaming -/// config, and its own routing-connector pipeline. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AggRole { - /// `quantile_over_time`, `quantile(...)`, `histogram_quantile(...)`, - /// or workload entries with `sketch_family_override: DDSketch | KLL`. - /// Routes to a quantile-shaped sketch (DDSketch / KLL). - Quantile, - /// Bare selector or Sum-shaped exact aggregation. - Sum, - /// Reset-aware per-second counter rate. - Rate, - /// Reset-aware counter increase over the selected window. - Increase, - /// `count(...)`, `count_over_time(...)`, `count_distinct_over_time(...)`, - /// or workload entries with `sketch_family_override: HLL`. Routes - /// to HLL when a sketch is appropriate, otherwise to exact Count. - Count, - /// `topk(...)`, or workload entries with - /// `sketch_family_override: CountSketch | CountMinSketch`. Routes - /// to CountSketch / CMS-with-heap. (`topk_over_time(...)` isn't a - /// real function this parser recognizes — dropped from this list; - /// see `agg_role_topk_query_strings`'s test comment.) - Topk, - /// Fallback bucket — specialized sketch families that don't fit the - /// four shapes above (e.g. CountMinSketch frequency without a topk - /// outer), or PromQL shapes the role-classifier can't recognise - /// today. Preserves "unique" semantics for the (metric, role) key - /// so multiple unrecognised entries still don't collide. - Other, -} - -impl AggRole { - /// Stable lowercase tag for routing-connector pipeline names, log - /// fields, and HTTP path discriminators. - pub fn as_str(&self) -> &'static str { - match self { - AggRole::Quantile => "quantile", - AggRole::Sum => "sum", - AggRole::Rate => "rate", - AggRole::Increase => "increase", - AggRole::Count => "count", - AggRole::Topk => "topk", - AggRole::Other => "other", - } - } -} - -impl std::fmt::Display for AggRole { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} - -/// Classify a single workload entry into its `AggRole`. -/// -/// Resolution order: -/// 1. **`sketch_family_override`** wins when present: -/// * `DDSketch` / `KLL` → [`AggRole::Quantile`] -/// * `HLL` → [`AggRole::Count`] -/// * `CountSketch` → [`AggRole::Topk`] -/// * `CountMinSketch` → [`AggRole::Other`] (frequency — no -/// single canonical shape; we keep it out of `Topk` so the topk -/// variant stays semantically pure for sketch-with-heap families) -/// 2. **Real `AggIntent` classification** — `query_string` is parsed -/// through the same canonical pipeline the live serving path uses -/// (`query_parser::parse_query_expr_canonical` → -/// `asap_tier_analysis::collect_agg_intents`), and the OUTERMOST -/// intent is matched, except that a Sum wrapping a counter function -/// keeps the inner Rate or Increase role: -/// * [`AggIntent::Quantile`] → [`AggRole::Quantile`] -/// * [`AggIntent::TopK`] → [`AggRole::Topk`] -/// * [`AggIntent::Cardinality`], [`AggIntent::Count`], or the -/// windowed-Count-as-Frequency extension -/// (`intent_algebra::as_frequency`) → [`AggRole::Count`] -/// * [`AggIntent::Sum`], [`AggIntent::Rate`], [`AggIntent::Increase`] -/// → their respective roles -/// * Anything else recognised but not one of the listed shapes above -/// (`Min`/`Max`/`Avg`/`StdDev`/histogram accessors/…) → -/// [`AggRole::Other`]. -/// * Bare metric selector (no `Aggregate` node at all) → -/// [`AggRole::Sum`] (matches PromQL's instant-vector semantics — -/// a bare counter sums values across time-aligned samples). -/// 3. Unparseable / no query string / no override → [`AggRole::Other`]. -/// -/// This used to be a PromQL-string leading-token sniff — the same -/// duplicate-classifier smell already retired from the live serving path -/// (see the former `analyzer-parity-matrix.md`'s "engine (duplicate)" -/// analyzer). `AggIntent` classification is now the single source of -/// truth for "what shape is this query," here and on the serving path. -/// -/// Ambiguous case decisions (documented for the B2 PR, still holds under -/// `AggIntent` classification): -/// * `count_over_time(metric)` — Count (cardinality/frequency -/// semantics, whichever the lowerer picks). The Sum-shaped -/// alternative is rare in practice; users who want it write -/// `sum_over_time(count(...))` which classifies as Sum. -/// * `irate` currently folds onto `AggIntent::Rate` in the frontend. -pub fn derive_agg_role(entry: &WorkloadEntry) -> AggRole { - // 1. `sketch_family_override` wins. - if let Some(family) = entry.sketch_family_override.as_ref() { - return match family { - SketchType::DDSketch | SketchType::KLL => AggRole::Quantile, - SketchType::HLL => AggRole::Count, - SketchType::CountSketch => AggRole::Topk, - SketchType::CountMinSketch => AggRole::Other, - }; - } - - // 2. Real AggIntent classification via the canonical parse/lower - // pipeline — the same one `capability_for`/serving uses. Canonical - // structural lowering is owned by ASAPPlanner's frontend. - let Some(qs) = entry.query_string.as_ref() else { - return AggRole::Other; - }; - let accuracy = crate::types::accuracy_target_from_legacy_accuracy_sla(entry.accuracy_sla); - let Ok(expr) = crate::query_parser::parse_query_expr_canonical(qs, accuracy) else { - return AggRole::Other; - }; - let mut intents: Vec = Vec::new(); - collect_agg_intents(&expr, &mut intents); - let Some(outer) = intents.first() else { - // No Aggregate node at all — a bare metric selector (or a - // window-only shape with no AggType to map onto). Bare - // selectors default to Sum per PromQL's instant-vector - // semantics; anything else falls through to Other below. - let trimmed = qs.trim_start(); - if !trimmed.is_empty() && !trimmed.starts_with('{') { - return AggRole::Sum; - } - return AggRole::Other; - }; - if crate::planner_selection::as_frequency(outer).is_some() { - return AggRole::Count; - } - // A spatial `sum by (...)` around a counter function still requires the - // counter family's state; using Sum as the registration key would let it - // overwrite a bare Sum workload for the same metric. - if matches!(outer, AggIntent::Sum { .. }) { - if intents - .iter() - .any(|intent| matches!(intent, AggIntent::Rate)) - { - return AggRole::Rate; - } - if intents - .iter() - .any(|intent| matches!(intent, AggIntent::Increase)) - { - return AggRole::Increase; - } - } - match outer { - AggIntent::Quantile { .. } => AggRole::Quantile, - AggIntent::TopK { .. } => AggRole::Topk, - AggIntent::Cardinality { .. } | AggIntent::Count { .. } => AggRole::Count, - AggIntent::Sum { .. } => AggRole::Sum, - AggIntent::Rate => AggRole::Rate, - AggIntent::Increase => AggRole::Increase, - _ => AggRole::Other, - } -} - -fn collect_agg_intents(expr: &planner_types::pre_asap::QueryExpr, out: &mut Vec) { - use planner_types::pre_asap::QueryExpr; - match expr { - QueryExpr::Aggregate { - measures, child, .. - } => { - out.extend(measures.iter().cloned()); - collect_agg_intents(child, out); - } - QueryExpr::Filter { child, .. } - | QueryExpr::Project { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - | QueryExpr::TimeRange { child, .. } - | QueryExpr::TimeShift { child, .. } - | QueryExpr::SQLWindowFunc { child, .. } => collect_agg_intents(child, out), - QueryExpr::Concat { children, .. } => { - for child in children { - collect_agg_intents(child, out); - } - } - QueryExpr::Join { left, right, .. } - | QueryExpr::SetOp { left, right, .. } - | QueryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } => { - collect_agg_intents(left, out); - collect_agg_intents(right, out); - } - _ => {} - } -} - -/// A single workload entry from the workloads YAML file. -/// -/// `deny_unknown_fields`: a misspelled or unsupported key is a planning input -/// the controller cannot honour. Accepting it silently would let the operator -/// believe a declared cadence / hint reached the planner when it never left the -/// YAML, so the registry rejects the file instead (see -/// [`WorkloadRegistry::try_load`]). -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct WorkloadEntry { - /// Metric name this workload targets (e.g. `http_request_duration_seconds`). - pub metric_name: String, - /// PromQL / SQL query string for the planner. - #[serde(default)] - pub query_string: Option, - /// Required accuracy SLA (0.0 – 1.0). - #[serde(default = "default_accuracy_sla")] - pub accuracy_sla: f64, - /// Role that should receive this workload (e.g. `"agent"`, `"backend"`). - #[serde(default = "default_role")] - pub assign_to_role: String, - /// Optional explicit sketch family override. When set, the planner pins - /// this family for the metric (modulo `(sketch, statistic)` validity - /// by ASAPPlanner's legal candidate enumeration). Threaded - /// into `RegisteredWorkload::sketch_type_override` by the registry pre-pop - /// path so the typed L4 binding (`bind_workload_typed`) honours it. - /// - /// MVP-§46 contract entries 5–8 in `deploy/configs/mvp-workload.yaml` - /// rely on this field to pin HLL / CountSketch / CountMinSketch - /// against metrics whose name-classified statistic class is - /// `Cardinality` / `TopK` / `Frequency`. - #[serde(default, deserialize_with = "deserialize_sketch_family")] - pub sketch_family_override: Option, - /// Optional storage tier hint (e.g. `"warm"`, `"archive"`). Round-trips - /// silently for now — kept here so the YAML schema matches the - /// capability_matching agent's expected shape (no rename step at - /// integration). Not yet read by the planner. - #[serde(default)] - pub target_path: Option, - /// MVP blocker B3 — declarative grouping labels for the wire-attr - /// allowlist the agent applies before sketching. Mirrors the - /// streaming-config's `grouping_labels` contract: the agent's - /// `transform/keep_for_` OTTL processor calls - /// `keep_keys(datapoint.attributes, [...])` on this list, stripping - /// every other attr BEFORE the sketch processor mints sids. - /// - /// Why this is a separate field (not parsed from `query_string`): - /// the canonical MVP workload `quantile_over_time(0.99, - /// http_requests_total_latency_ms[30s])` carries no `by (...)` - /// clause, so the PromQL parser surfaces an EMPTY group_by_labels. - /// Without a declarative field the analyzer ends up with an empty - /// `RegisteredWorkload.group_by_labels` → an empty `keep_keys` list → - /// the agent strips ALL attrs and mints a single sid per metric - /// (instead of one per `(metric, zone)`), defeating the streaming- - /// config contract. - /// - /// Threaded into `RegisteredWorkload::group_by_labels` by the registry - /// pre-pop loop in `main`, so it merges with any `by (...)` keys - /// the PromQL parser surfaces. Empty / missing ⇒ same behaviour as - /// pre-B3 (no allowlist injected). - #[serde(default)] - pub grouping_labels: Vec, - /// Optional per-metric sketch sampling probability `p` in `(0, 1]`. - /// - /// Activates the warm-sketch sampling layer (geometric admission for - /// the frequency families / hash-threshold for HLL) the agent's - /// sketch processors carry: the encoder admits a `p` fraction of - /// updates, stores the RAW sampled state, stamps `p` on the - /// `SketchEnvelope`, and the backend rescales count-like estimates by - /// `1/p` at query time. `1.0` (the default — also the value `0`/unset - /// normalises to) disables sampling so the emitted agent config and - /// wire bytes are byte-identical to today. - /// - /// Today this is a static operator-set knob; a dynamic - /// optimizer-driven `p` (tuned from runtime samples against an - /// accuracy/bandwidth budget) is a follow-up and is intentionally out - /// of scope here. - /// - /// Threaded into `EdgeStageConfig::metric_to_sample_p` by the registry - /// pre-pop loop in `main`, which the L5 edge emitter reads in - /// `build_edge_processor_block` and writes onto the per-metric - /// sketch-processor block (`sample_p`) only when `< 1.0`. - #[serde(default = "default_sample_p")] - pub sample_p: f64, - /// Optional **known distinct key (item) count per flush window** for the - /// cardinality / frequency sketch families — the declarative twin of - /// [`crate::types::WorkloadCharacteristics::distinct_keys_per_window`]. - /// - /// Today this refines ONLY the HLL sparse-vs-dense base selection at the - /// edge: a *per-series* HLL is emitted sparse by default (PR #358), but a - /// per-series HLL whose cardinality is known to exceed the in-memory - /// sparse→dense promotion point pays only promotion churn from the sparse - /// base, so when this hint is `Some(n)` with `n` above that crossover the - /// L5 emitter emits it dense instead (completing the #358 follow-up). - /// - /// Threaded into `EdgeStageConfig::metric_to_distinct_keys` by - /// [`crate::emit::collect_metric_to_distinct_keys`] (registry pre-pop loop - /// in `main` + the replan companion), which the L5 `asap_edge` emitter - /// reads in `emit_edge_yaml_asap_edge`'s HLL branch. `None` / missing ⇒ - /// the scope-based default of PR #358 (per-series ⇒ sparse), so the - /// emitted config stays byte-identical when the hint is absent. - #[serde(default)] - pub distinct_keys_per_window: Option, - /// Optional **inner high-cardinality dimension** for the item-counting - /// sketch families (HLL / CountSketch / CountMinSketch): the data-point - /// attribute whose VALUE is the "item" the sketch counts/ranks, as - /// opposed to a [`grouping_labels`](Self::grouping_labels) key that - /// splits the sketch into per-group series. - /// - /// Why this is a separate declarative field (not derivable from the - /// metric name or `query_string`): the inner dimension is a producer - /// data-point attribute name (`user_id` for the HLL metric - /// `unique_users_per_min`, `endpoint` for the CountSketch metric - /// `top_endpoint_qps` and the Count-Min metric `endpoint_request_freq`). - /// Neither the metric NAME nor the PromQL carries it — `count(...)` / - /// `topk(...)` / `rate(...)` name no attribute. Without a declarative - /// field the high-cardinality inner attribute stays in the sketch's - /// series key (one cardinality-1 HLL per `user_id` instead of one HLL - /// per zone), so the HLL/CMS warm queries return semantically wrong / - /// empty results. - /// - /// Threaded into `EdgeStageConfig::metric_to_item_label` by the registry - /// pre-pop loop in `main` (and the replan companion stitch), which the - /// L5 edge emitter reads in `emit_edge_yaml_asap_edge` and writes onto - /// the per-metric sketch entry as `item_label`. For CountSketch the - /// emitter falls back to the metric-name convention - /// (`countsketch_item_label_for`) when this field is unset, preserving - /// the prior behaviour. `None` / missing ⇒ no `item_label` is emitted - /// for HLL/CMS (backward-compatible). - #[serde(default)] - pub item_label: Option, - - /// Optional continuous-monitoring (CDM) declaration: when set, this metric - /// becomes a monitored standing query — the controller auto-emits the - /// backend coordinator's `monitors:` entry (authoritative τ) and, in - /// future, the edge `threshold:` block. `None` / missing ⇒ no monitor. - #[serde(default)] - pub monitor: Option, - - /// Optional evaluation cadence for this declared query class, in the same - /// `1h5m30s` spelling [`crate::pipeline::parse_duration`] accepts. - /// - /// Why this exists: the cadence is a **cost input**, not a scheduler knob. - /// [`crate::physical::deployment_cost::delta`] uses it as the batch-mode - /// flush-period proxy, so the same declaration costed through the HTTP - /// `POST /api/v1/plan` path (whose [`crate::pipeline::QuerySpec`] has - /// carried `repeat_every` all along) and through this YAML registry used - /// to reach the planner with *different* flush periods — the startup path - /// hardcoded `None`. Threaded into `QuerySpec::repeat_every` by the - /// registry pre-pop loop in `main`, which the analyzer parses into - /// [`crate::types::RegisteredWorkload::repeat_every`]. - /// - /// `None` / missing ⇒ unchanged behaviour (the cost model falls back to - /// its window-derived flush rate). - #[serde(default)] - pub repeat_every: Option, -} - -/// User-facing continuous-monitoring declaration on a [`WorkloadEntry`]. τ/ε and -/// the window are authoritative at the coordinator; this is the controller's -/// source for emitting them. See `crate::emit::monitor`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MonitorDecl { - /// Threshold τ the global aggregate is monitored against. - pub tau: f64, - /// Additive functional: "sum" (default), "cms_point", or "linear_buckets". - #[serde(default)] - pub functional: String, - /// CMS point-frequency key x (functional = cms_point). - #[serde(default)] - pub key: String, - /// Relative tolerance ε (the alert fires at (1−ε)τ). - #[serde(default = "default_monitor_epsilon")] - pub epsilon: f64, - /// Tumbling epoch length in seconds; MUST match the metric's edge window. - pub window_secs: u64, -} - -fn default_monitor_epsilon() -> f64 { - 0.05 -} - -fn default_accuracy_sla() -> f64 { - 0.01 -} - -/// Default per-metric sampling probability — `1.0` (sampling disabled, -/// exact). Keeps the emitted config byte-identical to pre-sampling when a -/// workload entry omits `sample_p`. -fn default_sample_p() -> f64 { - 1.0 -} -fn default_role() -> String { - "agent".into() -} - -/// Case-insensitive `SketchType` deserialiser. The wire YAML in -/// `deploy/configs/mvp-workload.yaml` spells the variants in mixed case -/// (`KLL`, `HLL`, `CountSketch`, `CountMinSketch`, `DDSketch`) to match -/// the capability-matching agent's schema, while `SketchType`'s -/// `#[serde(rename_all = "lowercase")]` would otherwise reject those -/// strings. Accepts both spellings. -fn deserialize_sketch_family<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let opt: Option = Option::deserialize(deserializer)?; - let Some(s) = opt else { return Ok(None) }; - let kind = match s.trim().to_ascii_lowercase().as_str() { - "ddsketch" => SketchType::DDSketch, - "kll" => SketchType::KLL, - "hll" => SketchType::HLL, - "countsketch" => SketchType::CountSketch, - "countminsketch" | "countmin" | "cms" => SketchType::CountMinSketch, - other => { - return Err(serde::de::Error::custom(format!( - "unknown sketch_family_override `{other}`; expected one of \ - DDSketch / KLL / HLL / CountSketch / CountMinSketch" - ))) - } - }; - Ok(Some(kind)) -} - -/// Registry of declarative workloads loaded from a YAML file. -#[derive(Debug, Clone)] -pub struct WorkloadRegistry { - entries: Vec, - /// Runtime-injected entries — e.g. from the autonomous `/api/v1/plan/auto` - /// apply path. Kept SEPARATE from the static YAML `entries` so the static - /// accessors (`entries()`, `for_role()`) are byte-for-byte unchanged; only - /// monitor emission ([`monitor_intents`]) overlays these. `Arc>` - /// so cloned registries (AppState + Replanner hold clones) share the same - /// runtime injections. - runtime: std::sync::Arc>>, -} - -impl WorkloadRegistry { - /// Collect the continuous-monitoring intents declared across all workload - /// entries, resolved against the given coordinator endpoint. The controller - /// feeds these to the backend `monitors:` emitter (and, later, the edge - /// `threshold:` emitter); each carries the content-addressed agg_id derived - /// from the metric name so it lines up with the edge's per-window sketch. - pub fn monitor_intents( - &self, - coordinator_url: &str, - ) -> Vec { - use crate::emit::monitor::{Functional, MonitorIntent}; - let runtime = self.runtime.read().expect("runtime registry lock poisoned"); - self.entries - .iter() - .chain(runtime.iter()) - .filter_map(|e| { - let m = e.monitor.as_ref()?; - Some(MonitorIntent { - metric: e.metric_name.clone(), - functional: Functional::from_name(&m.functional), - key: m.key.clone(), - coeffs: Vec::new(), - coordinator_url: coordinator_url.to_string(), - tau: m.tau, - epsilon: m.epsilon, - window_ms: m.window_secs.saturating_mul(1000), - }) - }) - .collect() - } - - /// Inject (or replace, keyed by `metric_name`) a runtime workload entry. - /// Used by the autonomous-allocation apply path to register a synthesized - /// monitor so the next replan/repost emits it into the backend - /// `InstalledPrecomputePlan` (the coordinator then derives the ε-floor `p`). Shared - /// across registry clones via the `Arc>` overlay. - pub fn insert_runtime(&self, entry: WorkloadEntry) { - let mut rt = self - .runtime - .write() - .expect("runtime registry lock poisoned"); - rt.retain(|e| e.metric_name != entry.metric_name); - rt.push(entry); - } - - /// Load from a YAML file. Returns an empty registry on any error. - /// - /// Prefer [`Self::try_load`] on the startup path: a registry that parses - /// into nothing is indistinguishable, at every later step, from an - /// operator who declared no workloads at all. - pub fn load(path: &str) -> Self { - match Self::try_load(path) { - Ok(registry) => registry, - Err(error) => { - warn!(path, error = %error, "invalid workloads YAML; using empty registry"); - Self::empty() - } - } - } - - /// Load from a YAML file, reporting an unusable file instead of degrading - /// to an empty registry. - /// - /// A **missing** file stays non-fatal — the controller is expected to run - /// without a declarative registry (the process e2e tests boot it with - /// `CONTROLLER_WORKLOADS` pointing at a path that does not exist). A file - /// that exists but does not parse is fatal: it carries planning input the - /// operator wrote down, and silently planning *nothing* from it has the - /// same observable shape as a controller that planned everything. - pub fn try_load(path: &str) -> Result { - let contents = match std::fs::read_to_string(path) { - Ok(contents) => contents, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - info!(path, "workloads file not found; using empty registry"); - return Ok(Self::empty()); - } - Err(error) => return Err(format!("cannot read workload registry: {error}")), - }; - let entries = serde_yaml::from_str::>(&contents) - .map_err(|error| format!("cannot parse workload registry: {error}"))?; - info!(path, count = entries.len(), "loaded workload registry"); - Ok(Self { - entries, - runtime: Default::default(), - }) - } - - /// Create an empty registry (no file). - pub fn empty() -> Self { - Self { - entries: vec![], - runtime: Default::default(), - } - } - - /// Create a registry from in-memory entries (useful for tests and - /// programmatic construction). - pub fn from_entries(entries: Vec) -> Self { - Self { - entries, - runtime: Default::default(), - } - } - - /// Returns all workload entries. - pub fn entries(&self) -> &[WorkloadEntry] { - &self.entries - } - - #[cfg(test)] - /// Returns workload entries assigned to a given role. - pub fn for_role(&self, role: &str) -> Vec<&WorkloadEntry> { - self.entries - .iter() - .filter(|e| e.assign_to_role.eq_ignore_ascii_case(role)) - .collect() - } - - /// Returns the first workload entry for a given role, if any. - pub fn first_for_role(&self, role: &str) -> Option<&WorkloadEntry> { - self.entries - .iter() - .find(|e| e.assign_to_role.eq_ignore_ascii_case(role)) - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - /// An unsupported key is planning input the controller cannot honour; - /// accepting the file would report a cadence / hint that never left the YAML. - #[test] - fn unknown_registry_key_is_rejected() { - let path = std::env::temp_dir().join(format!( - "asap_workload_unknown_key_{}.yaml", - std::process::id() - )); - std::fs::write( - &path, - "- metric_name: http_requests_total - accuracy_sla: 0.99 - repeat_evry: 30s -", - ) - .unwrap(); - let error = WorkloadRegistry::try_load(path.to_str().unwrap()) - .expect_err("unknown key must not load"); - assert!(error.contains("repeat_evry"), "unexpected error: {error}"); - // The lenient wrapper still degrades, which is why startup uses `try_load`. - assert!(WorkloadRegistry::load(path.to_str().unwrap()) - .entries() - .is_empty()); - std::fs::remove_file(&path).unwrap(); - } - - /// Running without a declarative registry stays supported: the process e2e - /// tests boot the controller with `CONTROLLER_WORKLOADS` pointing nowhere. - #[test] - fn missing_registry_file_is_not_a_startup_failure() { - let registry = WorkloadRegistry::try_load("/definitely/missing/workloads.yaml") - .expect("a missing registry is not an error"); - assert!(registry.entries().is_empty()); - } - - #[test] - fn load_empty_on_missing_file() { - let reg = WorkloadRegistry::load("/nonexistent/workloads.yaml"); - assert!(reg.entries().is_empty()); - } - - #[test] - fn monitor_intents_collected_from_workload_decls() { - // Only the entry carrying a `monitor:` block produces an intent; the - // intent's agg_id MUST match the Go edge's fnv64 for the same metric. - let yaml = r#" -- metric_name: bytes_sent - accuracy_sla: 0.95 - assign_to_role: agent - monitor: - tau: 1000.0 - functional: sum - window_secs: 60 -- metric_name: plain_metric - accuracy_sla: 0.95 - assign_to_role: agent -"#; - let entries: Vec = serde_yaml::from_str(yaml).expect("parse workload yaml"); - let reg = WorkloadRegistry::from_entries(entries); - let intents = reg.monitor_intents("data-plane:4319"); - assert_eq!(intents.len(), 1, "only the entry with a monitor decl"); - let i = &intents[0]; - assert_eq!(i.metric, "bytes_sent"); - assert_eq!(i.tau, 1000.0); - assert_eq!(i.window_ms, 60_000); - assert_eq!(i.epsilon, 0.05); // serde default - assert_eq!(i.coordinator_url, "data-plane:4319"); - assert!(matches!( - i.functional, - crate::emit::monitor::Functional::Sum - )); - let entry = crate::emit::monitor::streaming_config_monitor_entry(i); - assert_eq!( - entry["agg_id"].as_u64().unwrap(), - crate::emit::monitor::agg_id_for_metric("bytes_sent") - ); - } - - #[test] - fn empty_registry() { - let reg = WorkloadRegistry::empty(); - assert!(reg.entries().is_empty()); - assert!(reg.first_for_role("agent").is_none()); - } - - #[test] - fn runtime_injected_monitor_is_emitted() { - // autonomous apply path: insert a runtime entry carrying a monitor and - // assert monitor_intents() overlays it (the static accessors stay empty). - let reg = WorkloadRegistry::empty(); - assert!(reg.monitor_intents("dp:4319").is_empty()); - reg.insert_runtime(WorkloadEntry { - metric_name: "auto_latency".into(), - query_string: None, - accuracy_sla: 0.01, - assign_to_role: "agent".into(), - sketch_family_override: Some(crate::types::SketchType::DDSketch), - target_path: None, - grouping_labels: vec![], - sample_p: 1.0, - distinct_keys_per_window: None, - item_label: None, - monitor: Some(MonitorDecl { - tau: 7000.0, - functional: "sum".into(), - key: String::new(), - epsilon: 0.05, - window_secs: 30, - }), - repeat_every: None, - }); - let intents = reg.monitor_intents("dp:4319"); - assert_eq!(intents.len(), 1); - assert_eq!(intents[0].metric, "auto_latency"); - assert_eq!(intents[0].tau, 7000.0); - assert_eq!(intents[0].epsilon, 0.05); - // static accessors untouched by the runtime overlay - assert!(reg.entries().is_empty()); - // replace-by-metric: re-inserting the same metric does not duplicate - reg.insert_runtime(WorkloadEntry { - metric_name: "auto_latency".into(), - query_string: None, - accuracy_sla: 0.01, - assign_to_role: "agent".into(), - sketch_family_override: Some(crate::types::SketchType::DDSketch), - target_path: None, - grouping_labels: vec![], - sample_p: 1.0, - distinct_keys_per_window: None, - item_label: None, - monitor: Some(MonitorDecl { - tau: 9000.0, - functional: "sum".into(), - key: String::new(), - epsilon: 0.05, - window_secs: 30, - }), - repeat_every: None, - }); - let intents = reg.monitor_intents("dp:4319"); - assert_eq!(intents.len(), 1, "replaced, not duplicated"); - assert_eq!(intents[0].tau, 9000.0); - } - - #[test] - fn deserialize_entries() { - let yaml = r#" -- metric_name: latency - query_string: "histogram_quantile(0.99, rate(http_duration_bucket[5m]))" - accuracy_sla: 0.01 - assign_to_role: agent -- metric_name: error_count - accuracy_sla: 0.05 - assign_to_role: backend -"#; - let entries: Vec = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(entries.len(), 2); - assert_eq!(entries[0].metric_name, "latency"); - assert_eq!(entries[0].assign_to_role, "agent"); - assert!(entries[0].query_string.is_some()); - assert_eq!(entries[1].metric_name, "error_count"); - assert!(entries[1].query_string.is_none()); - } - - #[test] - fn for_role_filters_correctly() { - let reg = WorkloadRegistry::from_entries(vec![ - WorkloadEntry { - metric_name: "a".into(), - query_string: None, - accuracy_sla: 0.01, - assign_to_role: "agent".into(), - sketch_family_override: None, - target_path: None, - grouping_labels: vec![], - sample_p: 1.0, - distinct_keys_per_window: None, - item_label: None, - monitor: None, - repeat_every: None, - }, - WorkloadEntry { - metric_name: "b".into(), - query_string: None, - accuracy_sla: 0.05, - assign_to_role: "backend".into(), - sketch_family_override: None, - target_path: None, - grouping_labels: vec![], - sample_p: 1.0, - distinct_keys_per_window: None, - item_label: None, - monitor: None, - repeat_every: None, - }, - WorkloadEntry { - metric_name: "c".into(), - query_string: None, - accuracy_sla: 0.02, - assign_to_role: "agent".into(), - sketch_family_override: None, - target_path: None, - grouping_labels: vec![], - sample_p: 1.0, - distinct_keys_per_window: None, - item_label: None, - monitor: None, - repeat_every: None, - }, - ]); - assert_eq!(reg.for_role("agent").len(), 2); - assert_eq!(reg.for_role("backend").len(), 1); - assert_eq!(reg.first_for_role("agent").unwrap().metric_name, "a"); - } - - #[test] - fn deserialize_sample_p_default_and_explicit() { - // sample_p is optional and defaults to 1.0 (sampling disabled); - // an explicit value round-trips. This is the operator-facing - // per-metric sampling knob. - let yaml = r#" -- metric_name: freq_metric - sketch_family_override: CountMinSketch - sample_p: 0.1 -- metric_name: card_metric - sketch_family_override: HLL - sample_p: 0.25 -- metric_name: unset_metric - sketch_family_override: HLL -"#; - let entries: Vec = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(entries.len(), 3); - assert!((entries[0].sample_p - 0.1).abs() < 1e-12); - assert!((entries[1].sample_p - 0.25).abs() < 1e-12); - // Unset ⇒ default 1.0 (sampling disabled / byte-identical). - assert!((entries[2].sample_p - 1.0).abs() < 1e-12); - } - - #[test] - fn deserialize_item_label_default_and_explicit() { - // item_label is optional (None by default) and round-trips when - // declared. It names the inner high-cardinality data-point - // attribute the HLL/CountSketch/CMS family counts or ranks - // (e.g. user_id / endpoint), as opposed to a grouping_labels key. - let yaml = r#" -- metric_name: unique_users_per_min - sketch_family_override: HLL - grouping_labels: [zone] - item_label: user_id -- metric_name: endpoint_request_freq - sketch_family_override: CountMinSketch - grouping_labels: [zone] - item_label: endpoint -- metric_name: http_requests_total_latency_ms - sketch_family_override: KLL -"#; - let entries: Vec = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(entries.len(), 3); - assert_eq!(entries[0].item_label.as_deref(), Some("user_id")); - assert_eq!(entries[1].item_label.as_deref(), Some("endpoint")); - // Unset ⇒ None (no inner item dimension; byte-identical to before). - assert_eq!(entries[2].item_label, None); - } - - #[test] - fn deserialize_sketch_family_override_mixed_case() { - // The live wire YAML in `deploy/configs/mvp-workload.yaml` spells - // the override values in mixed case (KLL / HLL / CountSketch / - // CountMinSketch). Verify deserialization picks them up — without - // this, MVP §46 entries 5–8 silently drop their family override - // (the original stitching-gap symptom). - let yaml = r#" -- metric_name: a - sketch_family_override: KLL -- metric_name: b - sketch_family_override: HLL -- metric_name: c - sketch_family_override: CountSketch -- metric_name: d - sketch_family_override: CountMinSketch -- metric_name: e - sketch_family_override: DDSketch -- metric_name: f -"#; - let entries: Vec = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(entries.len(), 6); - assert_eq!(entries[0].sketch_family_override, Some(SketchType::KLL)); - assert_eq!(entries[1].sketch_family_override, Some(SketchType::HLL)); - assert_eq!( - entries[2].sketch_family_override, - Some(SketchType::CountSketch) - ); - assert_eq!( - entries[3].sketch_family_override, - Some(SketchType::CountMinSketch) - ); - assert_eq!( - entries[4].sketch_family_override, - Some(SketchType::DDSketch) - ); - assert_eq!(entries[5].sketch_family_override, None); - } - - // ── AggRole tests (B2 full restructure) ────────────────────────────── - - fn entry(metric: &str, q: Option<&str>, override_: Option) -> WorkloadEntry { - WorkloadEntry { - metric_name: metric.into(), - query_string: q.map(|s| s.to_string()), - accuracy_sla: 0.01, - assign_to_role: "agent".into(), - sketch_family_override: override_, - target_path: None, - grouping_labels: vec![], - sample_p: 1.0, - distinct_keys_per_window: None, - item_label: None, - monitor: None, - repeat_every: None, - } - } - - #[test] - fn agg_role_override_pins_quantile_for_dd_and_kll() { - assert_eq!( - derive_agg_role(&entry("m", None, Some(SketchType::DDSketch))), - AggRole::Quantile - ); - assert_eq!( - derive_agg_role(&entry("m", None, Some(SketchType::KLL))), - AggRole::Quantile - ); - } - - #[test] - fn agg_role_override_pins_count_for_hll() { - assert_eq!( - derive_agg_role(&entry("m", None, Some(SketchType::HLL))), - AggRole::Count - ); - } - - #[test] - fn agg_role_override_pins_topk_for_countsketch() { - assert_eq!( - derive_agg_role(&entry("m", None, Some(SketchType::CountSketch))), - AggRole::Topk - ); - } - - #[test] - fn agg_role_override_pins_other_for_cms() { - // CountMinSketch is a frequency estimator without an inherent - // topk shape — keep it in `Other` so the topk variant stays - // pure for sketch-with-heap families. - assert_eq!( - derive_agg_role(&entry("m", None, Some(SketchType::CountMinSketch))), - AggRole::Other - ); - } - - #[test] - fn agg_role_quantile_query_strings() { - for q in ["quantile_over_time(0.99, m[5m])", "quantile(0.5, m)"] { - assert_eq!( - derive_agg_role(&entry("m", Some(q), None)), - AggRole::Quantile, - "query `{q}` should classify as Quantile" - ); - } - } - - #[test] - fn agg_role_classic_bucket_histogram_quantile_is_other() { - // Classic `_bucket` + `rate(...)` histogram quantiles lower to the - // exact-only `HistogramQuantile` intent, classified here as `Other`. - assert_eq!( - derive_agg_role(&entry( - "m", - Some("histogram_quantile(0.99, rate(m_bucket[5m]))"), - None - )), - AggRole::Other - ); - } - - #[test] - fn agg_role_sum_query_strings() { - for q in ["sum by (zone) (m)", "sum_over_time(m[5m])"] { - assert_eq!( - derive_agg_role(&entry("m", Some(q), None)), - AggRole::Sum, - "query `{q}` should classify as Sum" - ); - } - } - - #[test] - fn counter_functions_have_distinct_workload_roles() { - for (query, expected) in [ - ("rate(m[5m])", AggRole::Rate), - ("sum by (zone) (rate(m[5m]))", AggRole::Rate), - ("increase(m[5m])", AggRole::Increase), - ("sum by (zone) (increase(m[5m]))", AggRole::Increase), - ] { - assert_eq!(derive_agg_role(&entry("m", Some(query), None)), expected); - } - } - - #[test] - fn agg_role_count_query_strings() { - for q in [ - "count(m)", - "count_over_time(m[5m])", - r#"count(m{zone="z0"})"#, - ] { - assert_eq!( - derive_agg_role(&entry("m", Some(q), None)), - AggRole::Count, - "query `{q}` should classify as Count" - ); - } - } - - #[test] - fn agg_role_handles_a_parenthesized_query_the_old_string_sniff_could_not() { - // A leading-token PromQL-string sniff (the pre-migration - // implementation) reads the FIRST character to find the outer - // function name; a redundant wrapping paren isn't an identifier - // character, so the old code's `token_end` search finds an - // empty leading token, falls through to "not a bare selector - // either" and mis-defaults to `Sum`. Real `AggIntent` - // classification parses the whole expression regardless of - // surface punctuation and correctly resolves the inner - // `quantile_over_time` shape. - assert_eq!( - derive_agg_role(&entry("m", Some("(quantile_over_time(0.9, m[5m]))"), None)), - AggRole::Quantile - ); - } - - #[test] - fn agg_role_topk_query_strings() { - // `topk_over_time` was in this list pre-migration, but it isn't a - // real function this parser (or vanilla PromQL) recognizes at - // all — confirmed via grep, it appears nowhere in - // query_parser/promql.rs or intent_algebra/lower.rs. A workload - // entry with that query_string would fail to parse anywhere else - // in the real pipeline too (main.rs's handle_plan included), so - // the old string-sniffing heuristic classifying it as Topk was - // itself the bug, not something this test should keep pinning. - // ASAPPlanner distinguishes heavy-hitter TopK from generic PromQL - // ranking. Only the former is a sketchable TopK intent. - { - let q = "topk(5, count_over_time(m[1m]))"; - assert_eq!( - derive_agg_role(&entry("m", Some(q), None)), - AggRole::Topk, - "query `{q}` should classify as Topk" - ); - } - } - - #[test] - fn agg_role_bare_metric_selector_is_sum() { - assert_eq!( - derive_agg_role(&entry( - "http_requests_total", - Some("http_requests_total"), - None - )), - AggRole::Sum - ); - } - - #[test] - fn agg_role_no_query_string_no_override_is_other() { - assert_eq!(derive_agg_role(&entry("m", None, None)), AggRole::Other); - } - - #[test] - fn agg_role_override_beats_query_string() { - // An explicit `sketch_family_override: HLL` paired with a - // `count(...)` query — both happen to classify as Count, but - // we exercise the override priority with a deliberately - // mismatched pair (override KLL on a count query) to pin the - // override-first rule. - let e = entry("m", Some("count(m)"), Some(SketchType::KLL)); - assert_eq!(derive_agg_role(&e), AggRole::Quantile); - } - - #[test] - fn deserialize_sketch_family_override_lowercase_aliases() { - // Lowercase / kebab-case spellings also accepted, plus the two - // CMS aliases (`countmin`, `cms`). - let yaml = r#" -- metric_name: a - sketch_family_override: ddsketch -- metric_name: b - sketch_family_override: countmin -- metric_name: c - sketch_family_override: cms -"#; - let entries: Vec = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - entries[0].sketch_family_override, - Some(SketchType::DDSketch) - ); - assert_eq!( - entries[1].sketch_family_override, - Some(SketchType::CountMinSketch) - ); - assert_eq!( - entries[2].sketch_family_override, - Some(SketchType::CountMinSketch) - ); - } - - #[test] - fn three_synthetic_http_requests_total_entries_keep_distinct_roles() { - // Synthetic mirror of `deploy/configs/mvp-workload.yaml` - // entries 2/3/4 — proves `derive_agg_role` produces distinct - // roles for the three http_requests_total shapes. Pre-B2 the - // workload store collapsed these onto one key and only the - // last entry's plan survived; the (metric, role) keyed store - // + per-entry role classification fixes that. - let entries = [ - entry( - "http_requests_total", - Some("sum by (zone) (http_requests_total)"), - None, - ), - entry( - "http_requests_total", - Some("sum by (zone) (rate(http_requests_total[5m]))"), - None, - ), - entry( - "http_requests_total", - Some(r#"count(http_requests_total{zone="z0"})"#), - None, - ), - ]; - let roles: Vec = entries.iter().map(derive_agg_role).collect(); - assert_eq!(roles, vec![AggRole::Sum, AggRole::Rate, AggRole::Count]); - let distinct: std::collections::HashSet<_> = roles.iter().copied().collect(); - assert_eq!(distinct.len(), 3); - } -} diff --git a/control_plane/tests/discovery_snapshot.rs b/control_plane/tests/discovery_snapshot.rs deleted file mode 100644 index c6e4a69b0..000000000 --- a/control_plane/tests/discovery_snapshot.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! Discovery output must be accepted by the production snapshot planner. -use control_plane::physical::compiler::BackendLocalPlanningInput; -use std::{fs, path::PathBuf, process::Command}; - -struct TempDirectory(PathBuf); -impl Drop for TempDirectory { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.0); - } -} - -#[test] -fn discovered_snapshot_plans_with_observed_cadence_and_promql_history() { - let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .to_path_buf(); - let temporary = TempDirectory( - std::env::temp_dir().join(format!("asap-discovery-snapshot-{}", std::process::id())), - ); - fs::create_dir(&temporary.0).unwrap(); - let corpus = temporary.0.join("corpus.json"); - let metrics = temporary.0.join("metrics.prom"); - let output = temporary.0.join("snapshot.json"); - fs::write(&corpus, r#"{"upstream_revision":"test","queries":[{"id":"q","query":"max_over_time(m[1m] offset 1h)","eval_timestamp_ms":120000}]}"#).unwrap(); - fs::write(&metrics, "m{job=\"test\"} 1 60\nm{job=\"test\"} 2 120\n").unwrap(); - let result = Command::new("python3") - .env("PYTHONDONTWRITEBYTECODE", "1") - .arg(root.join("tools/o11y-execution/discover_snapshot.py")) - .arg("--corpus") - .arg(&corpus) - .arg("--metrics") - .arg(&metrics) - .arg("--template") - .arg(root.join("docs/examples/asapquery-planning-snapshot.json")) - .arg("--output") - .arg(&output) - .args(["--repetitions", "1"]) - .output() - .unwrap(); - assert!( - result.status.success(), - "{}", - String::from_utf8_lossy(&result.stderr) - ); - let snapshot: BackendLocalPlanningInput = - serde_json::from_str(&fs::read_to_string(&output).unwrap()).unwrap(); - assert_eq!(snapshot.physical_inputs.scrape_interval_ms, 60_000); - assert_eq!( - snapshot.data_workload.data_ingestion_interval.value, - Some(planner_types::workload::DurationMs(60_000)) - ); - let (request, _) = snapshot - .clone() - .into_physical_compilation_request() - .unwrap(); - assert_eq!(request.scrape_interval_ms, Some(60_000)); - assert_eq!(request.queries[0].query_lookback_seconds, 3660); - let roundtrip: BackendLocalPlanningInput = - serde_json::from_value(serde_json::to_value(&snapshot).unwrap()).unwrap(); - assert_eq!(snapshot, roundtrip); -} diff --git a/control_plane/tests/fixtures/erp-kll-measured.json b/control_plane/tests/fixtures/erp-kll-measured.json deleted file mode 100644 index 154627f29..000000000 --- a/control_plane/tests/fixtures/erp-kll-measured.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "schema_version": 1, - "producer_version": "dedc2d4", - "records": [ - { - "id": "erp-0", - "sketch": "kll-percall", - "implementation": "lib", - "parameters": { - "algorithm": "kll-percall", - "params": { - "k": 32 - } - }, - "distribution": { - "synthetic": { - "description": { - "column_num": 1, - "column_label": [ - "key" - ], - "column_spec": [ - { - "distribution": { - "kind": "uniform", - "lower_bound": 0.0, - "upper_bound": 1.0, - "seed": 42 - }, - "special_rule": 0, - "data_type": "f64" - } - ], - "row_num": 1000 - } - } - }, - "trials": 10, - "error_metrics": { - "accuracy_runs": 1.0, - "grid_points": 101.0, - "items": 1000.0, - "max_rank_err": 0.051, - "mean_rank_err": 0.01618811881188117 - }, - "resources": { - "memory_bytes": 1024.0, - "update_cpu_seconds": 5.588000000000001e-7, - "merge_cpu_seconds": 0.000051599999999999994, - "query_cpu_seconds": 0.000031625742574257424 - } - } - ] -} diff --git a/control_plane/tests/fixtures/o11y_queries.json b/control_plane/tests/fixtures/o11y_queries.json deleted file mode 100644 index 9ea29a17d..000000000 --- a/control_plane/tests/fixtures/o11y_queries.json +++ /dev/null @@ -1,296 +0,0 @@ -{ - "upstream_revision": "3b9567319296366c6a73640d21826fc2fb26bc7e", - "eval_timestamp_ms": 1788891296000, - "source_files": [ - { - "path": "tasks-spec/prometheus_query/check-service-health.yaml", - "sha256": "387dff16abf294107f92cb81384c4ca3964206f1ba47d773671fd6ac7aed785c" - }, - { - "path": "tasks-spec/prometheus_query/promql-burn-rate-assessment.yaml", - "sha256": "6b3aee1e2adfc3c5b6004f29abfe374edc7294b86a55792d669f03fd122e878b" - }, - { - "path": "tasks-spec/prometheus_query/promql-cache-lag-vs-user-latency.yaml", - "sha256": "606e0dd923a013bc8475a0efbaafc72afa1ee6035e2f48d1c4bd5e23f164cec3" - }, - { - "path": "tasks-spec/prometheus_query/promql-cache-refresh-lag-peak.yaml", - "sha256": "6a88045045cab7ac6aba1c84968ecf8e7474cdd77315f2b4db1f2c89869243b3" - }, - { - "path": "tasks-spec/prometheus_query/promql-capacity-analysis.yaml", - "sha256": "0760992a9f1ad15bfcf91fe14ccab3d130232773cd4a9feaadb96ffeec2982fd" - }, - { - "path": "tasks-spec/prometheus_query/promql-discover-http-metric.yaml", - "sha256": "5a70c6fb1d575ec794252085afcef88e70b66214cf89aae3bfed8cc6600c78f9" - }, - { - "path": "tasks-spec/prometheus_query/promql-error-rate.yaml", - "sha256": "7756d896deddf03a8c3ebafc8b677d4d0caf953a55837d63b93c914f0dbb9d0a" - }, - { - "path": "tasks-spec/prometheus_query/promql-highest-backend-error-ratio.yaml", - "sha256": "39748917439d0201c5eca505f37b48f37d0fa39200c12d3ccd1fc2ce5f20183f" - }, - { - "path": "tasks-spec/prometheus_query/promql-label-matchers-service-errors.yaml", - "sha256": "db72f53039fba07ebc0b77eb16cdfa10a7cd57b06384c6a8540d95d7d6a94a2b" - }, - { - "path": "tasks-spec/prometheus_query/promql-offset-traffic-compare.yaml", - "sha256": "dca456d4948de758427725139c5123194d6e6de151815365eb12b5b801194f62" - }, - { - "path": "tasks-spec/prometheus_query/promql-order-latency-vs-traffic.yaml", - "sha256": "ee1df5d3d7727358c60236091e4f6c24bba77499f3928ae2a337983ad94e4018" - }, - { - "path": "tasks-spec/prometheus_query/promql-retry-backlog-triage.yaml", - "sha256": "3b11a1d3afb871558908b53a19d79bd38a659f890f244320b34eea8450cfef25" - }, - { - "path": "tasks-spec/prometheus_query/promql-subquery-peak-error-rate.yaml", - "sha256": "fbc6d1ea4386784ce9c6f1830f33640aa9f517c808d0909209d26d1d829bbee7" - }, - { - "path": "tasks-spec/prometheus_query/promql-topk-5xx-share.yaml", - "sha256": "d19c40a904b4d09560ad6782adf61dccf8ce0aa060a18da6c835fc81cdf3779e" - }, - { - "path": "tasks-spec/prometheus_query/query-cpu-metrics.yaml", - "sha256": "51058f02c2bd8616cdde085519e9139668b2eb5b5915afbabe40952062f8459c" - }, - { - "path": "tasks-spec/prometheus_query/query-memory-usage.yaml", - "sha256": "7ebf61f36fa97d5398cb64ba6543737349d087b91a8719ee62e3568ca23436b9" - } - ], - "queries": [ - { - "id": "check-service-health:rubric:0", - "task_id": "check-service-health", - "source_path": "tasks-spec/prometheus_query/check-service-health.yaml", - "rubric_index": 0, - "query": "sum(up)", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "check-service-health:rubric:1", - "task_id": "check-service-health", - "source_path": "tasks-spec/prometheus_query/check-service-health.yaml", - "rubric_index": 1, - "query": "max((sum by (job) (rate(http_requests_total{status=~\"5..\",job=~\"user-service|order-service|payment-service\"}[6h]))) / (sum by (job) (rate(http_requests_total{job=~\"user-service|order-service|payment-service\"}[6h]))))", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-burn-rate-assessment:rubric:0", - "task_id": "promql-burn-rate-assessment", - "source_path": "tasks-spec/prometheus_query/promql-burn-rate-assessment.yaml", - "rubric_index": 0, - "query": "sum(increase(http_requests_total{job=\"payment-service\",status=~\"5..\"}[1h])) / sum(increase(http_requests_total{job=\"payment-service\"}[1h]))", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-burn-rate-assessment:rubric:1", - "task_id": "promql-burn-rate-assessment", - "source_path": "tasks-spec/prometheus_query/promql-burn-rate-assessment.yaml", - "rubric_index": 1, - "query": "sum(increase(http_requests_total{job=\"payment-service\",status=~\"5..\"}[1h] offset 6h)) / sum(increase(http_requests_total{job=\"payment-service\"}[1h] offset 6h))", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-cache-lag-vs-user-latency:rubric:0", - "task_id": "promql-cache-lag-vs-user-latency", - "source_path": "tasks-spec/prometheus_query/promql-cache-lag-vs-user-latency.yaml", - "rubric_index": 0, - "query": "max_over_time(service_cache_refresh_lag_seconds{job=\"user-service\"}[12h])", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-cache-lag-vs-user-latency:rubric:1", - "task_id": "promql-cache-lag-vs-user-latency", - "source_path": "tasks-spec/prometheus_query/promql-cache-lag-vs-user-latency.yaml", - "rubric_index": 1, - "query": "max_over_time(histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"user-service\"}[5m])) by (le))[12h:1m])", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-cache-lag-vs-user-latency:rubric:2", - "task_id": "promql-cache-lag-vs-user-latency", - "source_path": "tasks-spec/prometheus_query/promql-cache-lag-vs-user-latency.yaml", - "rubric_index": 2, - "query": "service_cache_refresh_lag_seconds{job=\"user-service\"}", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-cache-refresh-lag-peak:rubric:0", - "task_id": "promql-cache-refresh-lag-peak", - "source_path": "tasks-spec/prometheus_query/promql-cache-refresh-lag-peak.yaml", - "rubric_index": 0, - "query": "max_over_time(service_cache_refresh_lag_seconds{job=\"user-service\"}[12h])", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-cache-refresh-lag-peak:rubric:1", - "task_id": "promql-cache-refresh-lag-peak", - "source_path": "tasks-spec/prometheus_query/promql-cache-refresh-lag-peak.yaml", - "rubric_index": 1, - "query": "service_cache_refresh_lag_seconds{job=\"user-service\"}", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-capacity-analysis:rubric:0", - "task_id": "promql-capacity-analysis", - "source_path": "tasks-spec/prometheus_query/promql-capacity-analysis.yaml", - "rubric_index": 0, - "query": "sum(rate(process_cpu_seconds_total[1h]))", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-capacity-analysis:rubric:1", - "task_id": "promql-capacity-analysis", - "source_path": "tasks-spec/prometheus_query/promql-capacity-analysis.yaml", - "rubric_index": 1, - "query": "sum(process_resident_memory_bytes)", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-capacity-analysis:rubric:2", - "task_id": "promql-capacity-analysis", - "source_path": "tasks-spec/prometheus_query/promql-capacity-analysis.yaml", - "rubric_index": 2, - "query": "max_over_time(sum(process_resident_memory_bytes)[6h:1m])", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-discover-http-metric:rubric:0", - "task_id": "promql-discover-http-metric", - "source_path": "tasks-spec/prometheus_query/promql-discover-http-metric.yaml", - "rubric_index": 0, - "query": "sum(rate(http_requests_total{job=~\"user-service|order-service|payment-service\"}[5m]))", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-error-rate:rubric:0", - "task_id": "promql-error-rate", - "source_path": "tasks-spec/prometheus_query/promql-error-rate.yaml", - "rubric_index": 0, - "query": "sum(rate(http_requests_total{status=~\"5..\",job=~\"user-service|order-service|payment-service\"}[1h])) / sum(rate(http_requests_total{job=~\"user-service|order-service|payment-service\"}[1h]))", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-highest-backend-error-ratio:rubric:2", - "task_id": "promql-highest-backend-error-ratio", - "source_path": "tasks-spec/prometheus_query/promql-highest-backend-error-ratio.yaml", - "rubric_index": 2, - "query": "(sum by (job) (increase(http_requests_total{status=~\"5..\",job=~\"user-service|order-service|payment-service\"}[6h]))) / (sum by (job) (increase(http_requests_total{job=~\"user-service|order-service|payment-service\"}[6h])))", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-highest-backend-error-ratio:rubric:3", - "task_id": "promql-highest-backend-error-ratio", - "source_path": "tasks-spec/prometheus_query/promql-highest-backend-error-ratio.yaml", - "rubric_index": 3, - "query": "(sum by (job) (increase(http_requests_total{status=~\"5..\",job=~\"user-service|order-service|payment-service\"}[6h]))) / (sum by (job) (increase(http_requests_total{job=~\"user-service|order-service|payment-service\"}[6h])))", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-label-matchers-service-errors:rubric:0", - "task_id": "promql-label-matchers-service-errors", - "source_path": "tasks-spec/prometheus_query/promql-label-matchers-service-errors.yaml", - "rubric_index": 0, - "query": "sum by (job) (increase(http_requests_total{status=~\"5..\",job=~\".+-service\"}[24h])) > 0", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-offset-traffic-compare:rubric:0", - "task_id": "promql-offset-traffic-compare", - "source_path": "tasks-spec/prometheus_query/promql-offset-traffic-compare.yaml", - "rubric_index": 0, - "query": "sum(rate(http_requests_total{job=\"order-service\"}[5m]))", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-offset-traffic-compare:rubric:1", - "task_id": "promql-offset-traffic-compare", - "source_path": "tasks-spec/prometheus_query/promql-offset-traffic-compare.yaml", - "rubric_index": 1, - "query": "sum(rate(http_requests_total{job=\"order-service\"}[5m] offset 1h))", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-order-latency-vs-traffic:rubric:0", - "task_id": "promql-order-latency-vs-traffic", - "source_path": "tasks-spec/prometheus_query/promql-order-latency-vs-traffic.yaml", - "rubric_index": 0, - "query": "sum(rate(http_requests_total{job=\"order-service\"}[5m]))", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-order-latency-vs-traffic:rubric:1", - "task_id": "promql-order-latency-vs-traffic", - "source_path": "tasks-spec/prometheus_query/promql-order-latency-vs-traffic.yaml", - "rubric_index": 1, - "query": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"order-service\"}[5m])) by (le))", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-retry-backlog-triage:rubric:0", - "task_id": "promql-retry-backlog-triage", - "source_path": "tasks-spec/prometheus_query/promql-retry-backlog-triage.yaml", - "rubric_index": 0, - "query": "max_over_time(service_retry_queue_depth{job=~\".+\"}[6h])", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-retry-backlog-triage:rubric:1", - "task_id": "promql-retry-backlog-triage", - "source_path": "tasks-spec/prometheus_query/promql-retry-backlog-triage.yaml", - "rubric_index": 1, - "query": "max_over_time(service_retry_queue_depth{job=\"order-service\"}[6h])", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-subquery-peak-error-rate:rubric:0", - "task_id": "promql-subquery-peak-error-rate", - "source_path": "tasks-spec/prometheus_query/promql-subquery-peak-error-rate.yaml", - "rubric_index": 0, - "query": "max_over_time((sum(rate(http_requests_total{job=\"order-service\",status=~\"5..\"}[5m])) / sum(rate(http_requests_total{job=\"order-service\"}[5m])))[6h:1m])", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "promql-topk-5xx-share:rubric:0", - "task_id": "promql-topk-5xx-share", - "source_path": "tasks-spec/prometheus_query/promql-topk-5xx-share.yaml", - "rubric_index": 0, - "query": "sum(increase(http_requests_total{job=\"order-service\",status=~\"5..\"}[24h])) / sum(increase(http_requests_total{status=~\"5..\"}[24h]))", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "query-cpu-metrics:rubric:0", - "task_id": "query-cpu-metrics", - "source_path": "tasks-spec/prometheus_query/query-cpu-metrics.yaml", - "rubric_index": 0, - "query": "sum by (job) (rate(process_cpu_seconds_total{job=~\".+\"}[6h]))", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "query-cpu-metrics:rubric:2", - "task_id": "query-cpu-metrics", - "source_path": "tasks-spec/prometheus_query/query-cpu-metrics.yaml", - "rubric_index": 2, - "query": "sort_desc(sum by (job) (rate(process_cpu_seconds_total{job=~\".+\"}[6h])))", - "eval_timestamp_ms": 1788891296000 - }, - { - "id": "query-memory-usage:rubric:0", - "task_id": "query-memory-usage", - "source_path": "tasks-spec/prometheus_query/query-memory-usage.yaml", - "rubric_index": 0, - "query": "sum(avg_over_time(process_resident_memory_bytes{job=~\".+\"}[6h]))", - "eval_timestamp_ms": 1788891296000 - } - ] -} diff --git a/control_plane/tests/fixtures/sql_exact_cuts/q07.sql b/control_plane/tests/fixtures/sql_exact_cuts/q07.sql deleted file mode 100644 index fd214356e..000000000 --- a/control_plane/tests/fixtures/sql_exact_cuts/q07.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT mapConcat(labels,map('__name__','user_service_cache_refresh_lag_seconds')) AS labels, argMax(value,ts_ms) AS value FROM raw_samples WHERE metric='user_service_cache_refresh_lag_seconds' AND ts_ms>1788891296000-300000 AND ts_ms<=1788891296000 GROUP BY labels ORDER BY labels diff --git a/control_plane/tests/fixtures/sql_exact_cuts/q09.sql b/control_plane/tests/fixtures/sql_exact_cuts/q09.sql deleted file mode 100644 index d134dc552..000000000 --- a/control_plane/tests/fixtures/sql_exact_cuts/q09.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT map() AS labels, sum(value) AS value FROM (SELECT labels,argMax(value,ts_ms) AS value FROM raw_samples WHERE metric='backend_process_resident_memory_bytes' AND ts_ms>1788891296000-300000 AND ts_ms<=1788891296000 GROUP BY labels) diff --git a/control_plane/tests/fixtures/sql_exact_cuts/q12.sql b/control_plane/tests/fixtures/sql_exact_cuts/q12.sql deleted file mode 100644 index 3d860f5eb..000000000 --- a/control_plane/tests/fixtures/sql_exact_cuts/q12.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT map('job',job) AS labels,sum(value) AS value FROM (SELECT labels['job'] AS job,labels,argMax(value,ts_ms) AS value FROM raw_samples WHERE metric='backend_process_resident_memory_bytes' AND ts_ms>1788891296000-300000 AND ts_ms<=1788891296000 GROUP BY job,labels) GROUP BY job ORDER BY value DESC,job LIMIT 2 diff --git a/control_plane/tests/fixtures/sql_exact_cuts/q27.sql b/control_plane/tests/fixtures/sql_exact_cuts/q27.sql deleted file mode 100644 index f3fbb3d0a..000000000 --- a/control_plane/tests/fixtures/sql_exact_cuts/q27.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT map('job',job) labels,avg(value) value FROM (SELECT labels['job'] job,ts_ms,sum(value) value FROM raw_samples WHERE metric='backend_process_resident_memory_bytes' AND ts_ms+4000>=1788891296000-21600000 AND ts_ms+4000<=1788891296000 AND modulo(ts_ms+4000,60000)=0 GROUP BY job,ts_ms) GROUP BY job ORDER BY value DESC,job LIMIT 3 diff --git a/control_plane/tests/o11y_exact_fallback.rs b/control_plane/tests/o11y_exact_fallback.rs deleted file mode 100644 index 3122ea088..000000000 --- a/control_plane/tests/o11y_exact_fallback.rs +++ /dev/null @@ -1,42 +0,0 @@ -use control_plane::{ - physical::post_asap::{bind_query_expr, PhysicalExpr, PostAsapPlan}, - query_parser::parse_query_expr_canonical, - types::AccuracyTarget, -}; -use planner_types::post_asap::SummaryExpr; - -/// Non-summary o11y roots preserve labels, ordering and comparison filtering -/// as the original exact IR, instead of failing summary candidate selection. -#[test] -fn o11y_non_summary_roots_preserve_exact_query_semantics() { - for query in [ - "service_cache_refresh_lag_seconds{job=\"user-service\"}", - "sort_desc(sum by (job) (rate(process_cpu_seconds_total{job=~\".+\"}[6h])))", - "sum by (job) (increase(http_requests_total{status=~\"5..\",job=~\".+-service\"}[24h])) > 0", - ] { - for accuracy in [AccuracyTarget::Exact, AccuracyTarget::Epsilon(0.01)] { - let expr = parse_query_expr_canonical(query, accuracy.clone()).unwrap(); - let result = bind_query_expr(&expr, accuracy) - .unwrap_or_else(|error| panic!("{query}: {error}")); - let PhysicalExpr::Committed(PostAsapPlan::Summary(node)) = result else { - panic!("expected an explicit exact fallback for {query}"); - }; - let SummaryExpr::KeepPreAsap(original) = &node.expr else { - panic!("expected original exact query for {query}"); - }; - assert_eq!(original.as_ref(), &expr, "query semantics changed: {query}"); - assert_eq!(node.schema.fields.len(), expr.output_schema().unwrap().columns.len()); - let executable = control_plane::query_plan::compile_bound_mapped( - "fixture".into(), query.into(), &node, - control_plane::query_plan::InstantExecution { - lookback_ms: 300_000, full_history: false, cumulative_readout: false, - }, - control_plane::query_plan::FallbackPolicy::Reject, - |_, _| panic!("exact fallback must not request summary materializations"), - |_, _| {}).unwrap(); - assert!(matches!(executable.nodes[&executable.root], - control_plane::query_plan::QueryPlanNode::ExactFallback { .. })); - assert!(executable.materialization_bindings().is_empty()); - } - } -} diff --git a/control_plane/tests/offline_evidence.rs b/control_plane/tests/offline_evidence.rs deleted file mode 100644 index f24dddeec..000000000 --- a/control_plane/tests/offline_evidence.rs +++ /dev/null @@ -1,405 +0,0 @@ -//! Synthetic fixtures test integration decisions, not measured sketch benefits. -use asap_aware_mapping::{ - empirical_cost::{EmpiricalEvidenceProvider, EvidenceArtifact, EvidenceContext}, - CostModel, -}; -use control_plane::{ - physical::post_asap::{ - bind_query_expr_with_cost_model, cost_model::ControlPlaneCostModel, PhysicalExpr, - PostAsapPlan, - }, - query_parser::parse_query_expr_canonical, - types::AccuracyTarget, -}; -use planner_types::{ - post_asap::{SketchAlgorithm, SketchParams, SummaryExpr, SummaryFamilyType, SummaryNode}, - pre_asap::AggIntent, -}; -use serde_json::json; - -fn frequency_comparison() -> ( - asap_aware_mapping::empirical_comparison::OfflineComparisonEvidence, - asap_aware_mapping::empirical_comparison::OfflineComparisonRequest, -) { - let (mut artifact, context) = fixture(&model(), &intent()); - let query = - json!({"kind":"point_frequency", "value_type":"i64", "probe_set":"all_distinct_keys"}); - artifact.records[0].algorithm = SketchAlgorithm::Cms; - artifact.records[0].params = SketchParams::Cms { - width: 512, - depth: 5, - }; - artifact.records[0].error.as_mut().unwrap().mean = Some(0.2); - artifact.records[1].algorithm = SketchAlgorithm::Cms; - artifact.records[1].params = SketchParams::Cms { - width: 1024, - depth: 5, - }; - artifact.records[1].error.as_mut().unwrap().mean = Some(0.001); - for row in &mut artifact.records { - row.error.as_mut().unwrap().query = query.clone(); - row.metrics.resources.cpu.build_cpu_ns = - serde_json::from_value(json!({"value":1.0,"samples":3,"stddev":0.0})).unwrap(); - row.metrics.resources.cpu.read_cpu_ns = row.metrics.resources.cpu.build_cpu_ns.clone(); - } - let bindings: Vec<_> = artifact - .records - .iter() - .map(|r| json!({"record_id":r.id,"query":query})) - .collect(); - let evidence = serde_json::from_value(json!({"schema_version":1,"timing_contract":"disjoint_live_state_v1", - "sketch_evidence":artifact, "query_bindings":bindings, - "exact_records":[{"id":"exact-test", "distribution":context.distribution, - "environment":context.environment, "query":query, - "measured_at_unix_seconds":100,"valid_until_unix_seconds":200, - "provenance":{"command":"synthetic test","dataset":"test","source_revision":"test","repetitions":3}, - "metrics":{"empty_build_cpu_ns":{"value":1.0,"samples":3}, - "update_cpu_ns":{"value":100.0,"samples":3}, - "prepare_cpu_ns":{"value":1.0,"samples":3}, - "read_cpu_ns":{"value":100.0,"samples":3}}}]})).unwrap(); - let request = serde_json::from_value(json!({"context":context, - "exact_environment":context.environment,"query":query, - "accuracy":{"metric":"absolute relative frequency error","max_observed_mean":0.01,"minimum_trials":3}, - "workload":{"input_items_per_state":1000,"reads_per_state":100,"merges_per_state":0,"state_instances":1,"horizon_seconds":3600.0}, - "weights":{"cpu_ns_weight":1.0,"retained_byte_seconds_weight":0.0},"formal_minimums":null})).unwrap(); - (evidence, request) -} - -fn point_frequency(accuracy: AccuracyTarget) -> AggIntent { - control_plane::planner_selection::frequency(accuracy, Some(("key".into(), "7".into()))) -} - -/// A query-matched mean-error budget selects a larger measured CMS, while -/// preserving the formal workload bound and the requested point readout. -#[test] -fn offline_frequency_error_budget_changes_configuration() { - let (evidence, request) = frequency_comparison(); - let empirical = model().with_offline_frequency_comparison(evidence, request); - let intent = point_frequency(AccuracyTarget::Epsilon(0.01)); - let AggIntent::Extension { ext_kind, payload } = &intent else { - unreachable!() - }; - let asap_aware_mapping::Realization::Sketch(selected) = - empirical.realize_extension(ext_kind, payload) - else { - panic!( - "expected a measured frequency sketch: {:?}", - empirical.offline_frequency_recommendation(payload) - ); - }; - assert_eq!( - selected.params(), - &SketchParams::Cms { - width: 1024, - depth: 5 - } - ); - let recommendation = empirical.offline_frequency_recommendation(payload).unwrap(); - assert!(recommendation.checked_formal_minimums); - assert!(recommendation.candidates[0] - .rejection - .as_ref() - .unwrap() - .contains("observed error")); - - let scan = - parse_query_expr_canonical("offline_metric{key=~\".+\"}", AccuracyTarget::Exact).unwrap(); - let expr = planner_types::pre_asap::QueryExpr::Aggregate { - reduction: planner_types::pre_asap::Reduction::PerEntity, - measures: vec![intent], - output_names: vec![], - having: None, - child: std::rc::Rc::new(scan), - }; - let PhysicalExpr::Committed(PostAsapPlan::Summary(node)) = - bind_query_expr_with_cost_model(&expr, &empirical).unwrap() - else { - panic!("expected summary") - }; - assert_eq!( - sketch(&node).1, - &SketchParams::Cms { - width: 1024, - depth: 5 - } - ); - assert!( - matches!(&node.expr, SummaryExpr::SummaryEstimate { query: planner_types::post_asap::SketchQuery::PointCount { - key: planner_types::pre_asap::ColumnRef::Named(key), value: Some(value) - }, .. } if key == "key" && value == "7") - ); -} - -/// An infeasible empirical budget, stale context, exact request, or missing -/// item filter must retain exact execution, never invent a default sketch. -#[test] -fn offline_frequency_comparison_falls_back_when_inapplicable() { - for scenario in ["error", "stale", "exact", "missing_item", "formal_minimum"] { - let (evidence, mut request) = frequency_comparison(); - if scenario == "error" { - request.accuracy.max_observed_mean = 0.0; - } - if scenario == "stale" { - request.context.now_unix_seconds = 201; - } - let empirical = model().with_offline_frequency_comparison(evidence, request); - let intent = match scenario { - "exact" => point_frequency(AccuracyTarget::Exact), - "missing_item" => { - control_plane::planner_selection::frequency(AccuracyTarget::Epsilon(0.01), None) - } - "formal_minimum" => point_frequency(AccuracyTarget::Epsilon(0.001)), - _ => point_frequency(AccuracyTarget::Epsilon(0.01)), - }; - let AggIntent::Extension { ext_kind, payload } = intent else { - unreachable!() - }; - assert!( - matches!( - empirical.realize_extension(&ext_kind, &payload), - asap_aware_mapping::Realization::PassThrough - ), - "{scenario}" - ); - } -} - -/// An unsupported cheap CMS width must not hide a valid measured alternative. -#[test] -fn offline_frequency_filters_backend_layout_before_comparison() { - let (mut evidence, request) = frequency_comparison(); - let mut unsupported = evidence.sketch_evidence.records[1].clone(); - unsupported.id = "unsupported-cheap-width".into(); - unsupported.params = SketchParams::Cms { - width: 1500, - depth: 5, - }; - unsupported - .metrics - .resources - .cpu - .update_cpu_ns - .as_mut() - .unwrap() - .value = 0.001; - let mut query_binding = evidence.query_bindings[1].clone(); - query_binding.record_id = unsupported.id.clone(); - evidence.query_bindings.push(query_binding); - evidence.sketch_evidence.records.push(unsupported); - let empirical = model().with_offline_frequency_comparison(evidence, request); - let AggIntent::Extension { payload, .. } = point_frequency(AccuracyTarget::Epsilon(0.01)) - else { - unreachable!() - }; - let recommendation = empirical - .offline_frequency_recommendation(&payload) - .unwrap(); - assert_eq!( - recommendation.selected_sketch().unwrap().params, - SketchParams::Cms { - width: 1024, - depth: 5 - } - ); - assert!(recommendation - .candidates - .iter() - .all(|c| c.record_id != "unsupported-cheap-width")); -} - -fn fixture( - model: &ControlPlaneCostModel, - intent: &AggIntent, -) -> (EvidenceArtifact, EvidenceContext) { - let distribution = json!({"id":"synthetic-unit-test", "family":"uniform", "sample_count":1000, - "distinct_count":100, "parameters":{"seed":42}}); - let environment = json!({"id":"test-machine", "cpu":"test", "os":"test", "runtime":"test", - "implementation":"synthetic-test-fixture", "implementation_version":"v1"}); - let records: Vec<_> = [(SketchAlgorithm::Cms,20.0), (SketchAlgorithm::CountSketch,10.0), (SketchAlgorithm::UnivMon,30.0)].into_iter().map(|(algorithm,value)| { - let params = model.size_params(algorithm.clone(), intent, 0.01, 0.01); - json!({"id":format!("test-{algorithm:?}"), "algorithm":algorithm, "params":params, - "distribution":distribution, "environment":environment, - "measured_at_unix_seconds":100, "valid_until_unix_seconds":200, - "provenance":{"command":"synthetic integration fixture, not a real benchmark", "dataset":"test", "source_revision":"test", "repetitions":3}, - "metrics":{"update_cpu_ns":{"value":value,"stddev":1.0,"samples":3}}, - "error":{"metric":"absolute relative frequency error", "mean":0.0, "max":null, "trials":3, - "ground_truth_method":"synthetic fixture", "query":{"kind":"point_frequency"}}}) - }).collect(); - let artifact = serde_json::from_value( - json!({"schema_version":1,"benchmark_version":"synthetic-test-v1", - "model_version":"empirical-update-cpu-v1","records":records}), - ) - .unwrap(); - let context = serde_json::from_value( - json!({"distribution":distribution,"environment":environment,"now_unix_seconds":150}), - ) - .unwrap(); - (artifact, context) -} - -fn intent() -> AggIntent { - AggIntent::Count { - accuracy: AccuracyTarget::Epsilon(0.01), - } -} -fn candidates() -> Vec { - vec![SketchAlgorithm::Cms, SketchAlgorithm::CountSketch] -} -fn model() -> ControlPlaneCostModel { - ControlPlaneCostModel::new(AccuracyTarget::Epsilon(0.01)) -} - -fn bound(model: &ControlPlaneCostModel) -> std::rc::Rc { - let query = parse_query_expr_canonical( - "count_over_time(offline_metric[5m])", - AccuracyTarget::Epsilon(0.01), - ) - .unwrap(); - let PhysicalExpr::Committed(PostAsapPlan::Summary(node)) = - bind_query_expr_with_cost_model(&query, model).unwrap() - else { - panic!("expected summary binding") - }; - node -} - -fn sketch(node: &SummaryNode) -> (&SketchAlgorithm, &SketchParams) { - match &node.expr { - SummaryExpr::SummaryEstimate { summary_input, .. } => sketch(summary_input), - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::Sketch(kind, _), - .. - } => (kind.algorithm(), kind.params()), - other => panic!("expected sketch, got {other:?}"), - } -} - -fn assert_exact_count(node: &SummaryNode) { - match &node.expr { - SummaryExpr::ValueOperation { - child, - operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } => { - assert!(node - .schema - .fields - .iter() - .all(|field| matches!(field.dtype, SummaryFamilyType::Plain(_)))); - assert_exact_count(child); - } - - SummaryExpr::SummaryEstimate { summary_input, .. } => assert_exact_count(summary_input), - SummaryExpr::SummaryAgg { - family: - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Count, - planner_types::post_asap::ExactParams::Count, - ), - .. - } => {} - other => panic!("expected exact total-count state, got {other:?}"), - } -} - -/// Update evidence ranks frequency sketches but does not replace an exact -/// total-count accumulator with a point-frequency sketch. -#[test] -fn offline_update_evidence_preserves_exact_count_binding() { - let default = model(); - let (artifact, context) = fixture(&default, &intent()); - let empirical = - model().with_offline_evidence(EmpiricalEvidenceProvider::new(artifact, context).unwrap()); - assert_eq!( - default.rank_candidates(&intent(), &candidates()), - candidates() - ); - assert_eq!( - empirical.rank_candidates(&intent(), &candidates()), - vec![SketchAlgorithm::CountSketch, SketchAlgorithm::Cms] - ); - let default_bound = bound(&default); - let measured_bound = bound(&empirical); - assert_exact_count(&default_bound); - assert_exact_count(&measured_bound); - assert!(default_bound.guarantee.is_some()); - assert!(measured_bound.guarantee.is_some()); - // Observed zero point-frequency error has no effect on formal sizing. - for kind in candidates() { - assert_eq!( - empirical.size_params(kind.clone(), &intent(), 0.01, 0.01), - default.size_params(kind, &intent(), 0.01, 0.01) - ); - } -} - -/// Stale/missing/environment-mismatched data and parameters for another -/// workload accuracy all fall back to the deployment's existing preference. -#[test] -fn incompatible_evidence_preserves_deployment_behavior() { - for scenario in ["missing", "stale", "environment", "workload_sizing"] { - let (mut artifact, mut context) = fixture(&model(), &intent()); - let mut deployment = model(); - match scenario { - "missing" => { - artifact.records.remove(1); - } - "stale" => context.now_unix_seconds = 201, - "environment" => context.environment.cpu = "different CPU".into(), - "workload_sizing" => { - deployment = ControlPlaneCostModel::new(AccuracyTarget::Epsilon(0.001)) - } - _ => unreachable!(), - } - let empirical = deployment - .with_offline_evidence(EmpiricalEvidenceProvider::new(artifact, context).unwrap()); - assert_eq!( - empirical.rank_candidates(&intent(), &candidates()), - candidates(), - "{scenario}" - ); - assert_exact_count(&bound(&empirical)); - } -} - -/// Binary operations over these approximate sketch values retain explicit -/// fallback even though the warm tier now supports exact additive binaries. -#[test] -fn binary_summary_has_explicit_warm_tier_fallback() { - use control_plane::query_plan::{FallbackPolicy, InstantExecution, QueryPlanNode}; - use planner_types::{post_asap::BinaryOperator, pre_asap::BinaryOpKind}; - let child = bound(&model()); - let root = std::rc::Rc::new(SummaryNode { - expr: SummaryExpr::BinaryOp { - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - lhs: child.clone(), - rhs: child.clone(), - operator: BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: BinaryOpKind::Arithmetic(planner_types::pre_asap::ArithmeticOpKind::Div), - vector_match: None, - }, - }, - schema: child.schema.clone(), - guarantee: None, - }); - let plan = control_plane::query_plan::compile_bound_mapped( - "test".into(), - "left / right".into(), - &root, - InstantExecution { - lookback_ms: 300000, - full_history: false, - cumulative_readout: false, - }, - FallbackPolicy::Reject, - |_, _| panic!("unsupported binary plan must not bind a materialization"), - |_, _| {}, - ) - .unwrap(); - assert!( - matches!(&plan.nodes[&plan.root], QueryPlanNode::ExactFallback { reason } if !reason.is_empty()) - ); - assert!(plan.materialization_bindings().is_empty()); -} diff --git a/control_plane/tests/plan_workload.rs b/control_plane/tests/plan_workload.rs new file mode 100644 index 000000000..d052c994a --- /dev/null +++ b/control_plane/tests/plan_workload.rs @@ -0,0 +1,63 @@ +use std::path::{Path, PathBuf}; + +use control_plane::{backend_capabilities, plan_workload, InstalledPlan, PlanError}; +use planner_types::workload::{PlanningWorkload, QueryLanguage, SqlDialect}; + +fn example1() -> PlanningWorkload { + let path = + Path::new(env!("CARGO_MANIFEST_DIR")).join("../examples/planner-layering-example1.json"); + serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() +} + +fn temp_path(name: &str) -> PathBuf { + std::env::temp_dir().join(format!("control-plane-{}-{name}", std::process::id())) +} + +/// The Planner selects one physical DAG for #509 Example 1, with one root per +/// query, and the plan survives the file the data plane loads. +#[test] +fn plans_example1_and_round_trips_the_plan() { + let workload = example1(); + let plan = plan_workload(&workload, &backend_capabilities()).unwrap(); + assert_eq!( + plan.queries, + workload + .query_workload + .entries() + .map(|entry| entry.query.0) + .collect::>() + ); + assert_eq!(plan.dag.dag.roots.len(), 2); + let path = temp_path("example1.json"); + plan.save(&path).unwrap(); + assert_eq!(InstalledPlan::load(&path).unwrap(), plan); + std::fs::remove_file(path).unwrap(); +} + +/// Only PromQL workloads are planned; SQL support comes later. +#[test] +fn rejects_sql_workloads() { + let mut workload = example1(); + workload.query_workload.language = QueryLanguage::SQL(SqlDialect::ClickhouseSQL); + assert!(matches!( + plan_workload(&workload, &backend_capabilities()), + Err(PlanError::UnsupportedLanguage(_)) + )); +} + +/// A plan whose queries do not match the DAG's roots is not installed. +#[test] +fn load_rejects_a_plan_without_a_query_per_root() { + let mut plan = plan_workload(&example1(), &backend_capabilities()).unwrap(); + plan.queries.pop(); + let path = temp_path("mismatched.json"); + plan.save(&path).unwrap(); + assert!(matches!( + InstalledPlan::load(&path), + Err(PlanError::RootCount { + roots: 2, + queries: 1 + }) + )); + std::fs::remove_file(path).unwrap(); +} diff --git a/control_plane/tests/snapshot_version.rs b/control_plane/tests/snapshot_version.rs deleted file mode 100644 index 9ae7cac20..000000000 --- a/control_plane/tests/snapshot_version.rs +++ /dev/null @@ -1,80 +0,0 @@ -//! Every producer of a planning snapshot must declare the one schema version -//! the backend accepts. -//! -//! The version had drifted before: two tools in `tools/o11y-execution/` -//! overwrote it with a stale literal while the backend had moved on, so a -//! snapshot built from a current template was relabelled to an old version and -//! then rejected by the backend that had just been handed it. Nothing failed -//! until a test deep in the stack did, with a message about the version rather -//! than about the producer that wrote it. -//! -//! Producers are written in Rust, Python and JSON, so they cannot share a -//! constant. They can share this test. -use control_plane::physical::compiler::WORKLOAD_SNAPSHOT_VERSION; -use std::path::{Path, PathBuf}; - -fn repository_root() -> PathBuf { - Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("control_plane sits in the workspace") - .to_path_buf() -} - -/// Shipped snapshots a user or a tool starts from. -const EXAMPLE_SNAPSHOTS: &[&str] = &[ - "docs/examples/asapquery-planning-snapshot.json", - "docs/examples/asapquery-compatibility-demo-snapshot.json", -]; - -/// Tools that accept or emit a snapshot and state the version they expect. -const TOOLS: &[&str] = &[ - "tools/o11y-execution/discover_snapshot.py", - "tools/o11y-execution/calibrate.py", - "tools/shared-workload/planned_run.py", -]; - -#[test] -fn shipped_snapshots_declare_the_supported_version() { - let root = repository_root(); - for relative in EXAMPLE_SNAPSHOTS { - let path = root.join(relative); - let snapshot: serde_json::Value = serde_json::from_str( - &std::fs::read_to_string(&path).unwrap_or_else(|error| panic!("{relative}: {error}")), - ) - .unwrap_or_else(|error| panic!("{relative}: {error}")); - assert_eq!( - snapshot["snapshot_version"].as_u64(), - Some(u64::from(WORKLOAD_SNAPSHOT_VERSION)), - "{relative} declares a version the backend does not accept" - ); - } -} - -#[test] -fn snapshot_tools_expect_the_supported_version() { - let root = repository_root(); - let expected = format!("snapshot_version\") != {WORKLOAD_SNAPSHOT_VERSION}"); - for relative in TOOLS { - let path = root.join(relative); - let source = - std::fs::read_to_string(&path).unwrap_or_else(|error| panic!("{relative}: {error}")); - assert!( - source.contains("snapshot_version"), - "{relative} no longer mentions snapshot_version; \ - remove it from TOOLS or restore the check" - ); - assert!( - source.contains(&expected), - "{relative} does not reject snapshots other than version \ - {WORKLOAD_SNAPSHOT_VERSION}; a producer that guesses the version is \ - how it drifted last time" - ); - // A tool that restates the version while writing is how the literal got - // stale: it changes content, not schema, so it must not relabel. - assert!( - !source.contains("[\"snapshot_version\"] ="), - "{relative} overwrites snapshot_version; carry the input's version \ - through instead of restating it" - ); - } -} diff --git a/crates/asap_otel_proto/Cargo.toml b/crates/asap_otel_proto/Cargo.toml deleted file mode 100644 index 563d742f2..000000000 --- a/crates/asap_otel_proto/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -name = "asap_otel_proto" -version.workspace = true -edition.workspace = true - -# Vendored Rust bindings for DataCollector's modified opentelemetry-proto. -# See ../../../../docs/pipeline-query-catalog.md §5.4 for the reasoning. -# -# The .proto files in proto/opentelemetry/proto/ are copied verbatim from -# DataCollector's opentelemetry-proto submodule and add first-class sketch -# variants (DDSketch, KLLSketch, CountSketch, CountMinSketch, HLLSketch) to -# the standard OTLP Metric.data oneof. To re-vendor when DataCollector -# updates its proto schema, copy the four files referenced in build.rs -# from the corresponding paths in the DataCollector worktree. - -[lib] -path = "src/lib.rs" - -[dependencies] -prost = "0.13" -tonic = { version = "0.12", features = ["transport", "codegen", "prost"] } - -[build-dependencies] -prost-build = "0.13" -tonic-build = { version = "0.12", features = ["prost"] } -# Ship a modern protoc so the build does not depend on the host's system -# protoc version. The system protoc on many distros is < 3.15 which -# rejects proto3 `optional` keyword used in opentelemetry-proto. -protoc-bin-vendored = "3" diff --git a/crates/asap_otel_proto/build.rs b/crates/asap_otel_proto/build.rs deleted file mode 100644 index e2ae992e3..000000000 --- a/crates/asap_otel_proto/build.rs +++ /dev/null @@ -1,42 +0,0 @@ -fn main() -> Result<(), Box> { - let proto_root = "proto"; - let proto_files = [ - "proto/opentelemetry/proto/common/v1/common.proto", - "proto/opentelemetry/proto/resource/v1/resource.proto", - "proto/opentelemetry/proto/metrics/v1/metrics.proto", - "proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto", - // sketchlib.v1 delta messages — vendored locally because - // the upstream asap_sketchlib crate only exposes state - // types today, not delta types. See the .proto file - // headers for the sketchlib-go source of truth. - "proto/sketchlib_delta/ddsketch_delta.proto", - "proto/sketchlib_delta/hll_delta.proto", - "proto/sketchlib_delta/countsketch_delta.proto", - "proto/sketchlib_delta/countminsketch_delta.proto", - ]; - for f in &proto_files { - println!("cargo:rerun-if-changed={f}"); - } - println!("cargo:rerun-if-changed=build.rs"); - - // Use the vendored protoc binary so the build is independent of the - // host's system protoc version. Many distros still ship protoc < 3.15 - // which rejects proto3 `optional` keyword used in opentelemetry-proto. - std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path()?); - - tonic_build::configure() - .build_server(true) - .build_client(false) - .compile_protos(&proto_files, &[proto_root])?; - - // Monitor service (CDM edge↔coordinator reverse channel) — needs BOTH a - // server (coordinator) and a client (edge sim / tests), unlike the OTLP - // ingest protos above which are server-only. - let monitor_proto = "proto/monitor/monitor.proto"; - println!("cargo:rerun-if-changed={monitor_proto}"); - tonic_build::configure() - .build_server(true) - .build_client(true) - .compile_protos(&[monitor_proto], &[proto_root])?; - Ok(()) -} diff --git a/crates/asap_otel_proto/proto/monitor/monitor.proto b/crates/asap_otel_proto/proto/monitor/monitor.proto deleted file mode 100644 index 781cb2e52..000000000 --- a/crates/asap_otel_proto/proto/monitor/monitor.proto +++ /dev/null @@ -1,117 +0,0 @@ -// Continuous distributed monitoring (CDM) wire contract — the edge↔coordinator -// reverse channel for Discipline B (continuous intra-window monitoring) from -// ASAPCollector docs/continuous-monitoring-tumbling-cost-analysis.md. -// -// This is the SOURCE OF TRUTH for the protocol. A byte-identical copy is -// vendored into the Go edge runtime at -// ASAPCollector/asap-precompute-go/monitor/monitorpb/monitor.proto -// for `protoc --go_out --go-grpc_out` codegen. Keep the two in sync; the -// message field numbers and the service name are the cross-language contract. -// -// Protocol (Cormode–Muthukrishnan–Yi distributed functional monitoring over a -// tumbling window; one window = one independent monitoring epoch, reset at the -// boundary). Per monitor (agg_id, key) the coordinator runs rounds: -// 1. Δ = τ − global_estimate. If Δ ≤ ε·τ → fire GlobalAlert, epoch done. -// 2. local_slack = Δ/(2k), k = #edges. Broadcast SlackGrant to every edge. -// 3. An edge sends MonitorReport when local_value − round_baseline ≥ local_slack. -// 4. When reported mass crosses ~Δ/2 the coordinator closes the round -// (PollLocal every edge), sets global_estimate = Σ local_value, round += 1, -// and goto 1 with shrinking slack. -// Every report/grant carries window_start_ms; messages for a non-current epoch -// are dropped on both sides. -// -// NOTE: avoids the proto3 `optional` keyword so host protoc < 3.15 can compile -// the Go side (same constraint that drives PROTOC=vendored on the Rust side). - -syntax = "proto3"; - -package asap.monitor.v1; - -// Go import path for the vendored edge-side copy (ignored by tonic_build). The -// generated stubs live in a SEPARATE nested Go module (monitor/grpcclient) so -// the gRPC dependency tree never contaminates the core runtime's module graph. -option go_package = "github.com/ProjectASAP/asap-precompute-go/monitor/grpcclient/monitorpb;monitorpb"; - -// edge → coordinator: open-stream registration. Sent on connect and at the -// start of every new tumbling epoch (after a window boundary). -message MonitorRegister { - string edge_id = 1; // stable per-edge identity (collector instance id) - uint64 agg_id = 2; // content-addressed aggregation id (matches edge + coordinator) - bytes key = 3; // CMS point-frequency key x; empty for Sum - uint64 epoch_window_ms = 4; // edge's tumbling window size — coordinator alignment guard - uint64 window_start_ms = 5; // edge's current epoch start — avoids coordinator clock skew -} - -// edge → coordinator: the current local additive value for the named monitor, -// emitted when the edge's local increase since round start crossed its slack, -// or in response to a PollLocal at round close. -message MonitorReport { - string edge_id = 1; - uint64 agg_id = 2; - bytes key = 3; - uint64 window_start_ms = 4; // epoch this value belongs to (stale-epoch drop) - double local_value = 5; // current additive local value at report time - uint64 round = 6; // round this report answers - uint64 seq = 7; // per-edge monotonic counter; idempotent-retransmit dedup - double rate = 8; // edge's observed items/window — feeds the coordinator's p_i allocation -} - -// coordinator → edge: this round's per-edge slack budget. The edge reports once -// its local increase since the grant reaches local_slack. -message SlackGrant { - uint64 agg_id = 1; - bytes key = 2; - uint64 round = 3; - double local_slack = 4; - uint64 window_start_ms = 5; - double sample_p = 6; // distributed-NitroSketch update-sampling prob the coordinator allocates (0=unset/p=1); edge applies via WithSampleP at EpochReset -} - -// coordinator → edge: demand the current local value now (round close). -message PollLocal { - uint64 agg_id = 1; - bytes key = 2; - uint64 round = 3; - uint64 window_start_ms = 4; -} - -// coordinator → edge: round closed; edge advances its round baseline. -message RoundClose { - uint64 agg_id = 1; - uint64 round = 2; - uint64 window_start_ms = 3; -} - -// Out-of-band (NOT carried on the Monitor stream): the coordinator serializes -// this into the control-plane violation/notifier sink when the global aggregate -// crosses τ. Defined here so both sides share the shape for logging/tests. -message GlobalAlert { - uint64 agg_id = 1; - bytes key = 2; - double global_estimate = 3; - double tau = 4; - uint64 window_start_ms = 5; -} - -// Upstream envelope (edge → coordinator). -message EdgeToCoord { - oneof msg { - MonitorRegister reg = 1; - MonitorReport report = 2; - } -} - -// Downstream envelope (coordinator → edge). -message CoordToEdge { - oneof msg { - SlackGrant grant = 1; - PollLocal poll = 2; - RoundClose close = 3; - } -} - -// One bidirectional stream per edge; the coordinator multiplexes every monitor -// for that edge over the single stream. -service MonitorService { - rpc Monitor(stream EdgeToCoord) returns (stream CoordToEdge); -} diff --git a/crates/asap_otel_proto/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto b/crates/asap_otel_proto/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto deleted file mode 100644 index 0b4d12fe9..000000000 --- a/crates/asap_otel_proto/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2019, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package opentelemetry.proto.collector.metrics.v1; - -import "opentelemetry/proto/metrics/v1/metrics.proto"; - -option csharp_namespace = "OpenTelemetry.Proto.Collector.Metrics.V1"; -option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.collector.metrics.v1"; -option java_outer_classname = "MetricsServiceProto"; -option go_package = "go.opentelemetry.io/proto/otlp/collector/metrics/v1"; - -// Service that can be used to push metrics between one Application -// instrumented with OpenTelemetry and a collector, or between a collector and a -// central collector. -service MetricsService { - rpc Export(ExportMetricsServiceRequest) returns (ExportMetricsServiceResponse) {} - - // Refactor-2026-05 (Phase 4): Eager batch resolution of series identifiers. - // Senders that have an attribute-only batch ready before the next Export - // can pre-resolve sids in one round-trip, then attach the returned sids - // to subsequent Exports without sending attributes again. The resolver - // is idempotent: same `(metric_name, attributes_fingerprint)` always - // produces the same sid for the lifetime of the cache (see design doc - // §5.4 "Idempotency invariant on `ResolveSeriesIDs`"). - rpc ResolveSeriesIDs(ResolveSeriesIDsRequest) returns (ResolveSeriesIDsResponse) {} -} - -message ResolveSeriesIDsRequest { - repeated SeriesQuery queries = 1; -} - -message SeriesQuery { - // The metric this attribute set belongs to. - string metric_name = 1; - // Opaque fingerprint of the attribute set. Senders MUST use the same - // canonical fingerprint algorithm as the receiver (sorted keys joined - // by `key=value;`); the receiver re-derives the cache key from this - // bytes blob, so any disagreement on canonicalization produces a - // cache miss and a fresh sid. - bytes attributes_fingerprint = 2; -} - -message ResolveSeriesIDsResponse { - repeated SeriesAssignment assignments = 1; -} - -message ExportMetricsServiceRequest { - // An array of ResourceMetrics. - // For data coming from a single resource this array will typically contain one - // element. Intermediary nodes (such as OpenTelemetry Collector) that receive - // data from multiple origins typically batch the data before forwarding further and - // in that case this array will contain multiple elements. - repeated opentelemetry.proto.metrics.v1.ResourceMetrics resource_metrics = 1; -} - -message ExportMetricsServiceResponse { - // The details of a partially successful export request. - // - // If the request is only partially accepted - // (i.e. when the server accepts only parts of the data and rejects the rest) - // the server MUST initialize the `partial_success` field and MUST - // set the `rejected_` with the number of items it rejected. - // - // Servers MAY also make use of the `partial_success` field to convey - // warnings/suggestions to senders even when the request was fully accepted. - // In such cases, the `rejected_` MUST have a value of `0` and - // the `error_message` MUST be non-empty. - // - // A `partial_success` message with an empty value (rejected_ = 0 and - // `error_message` = "") is equivalent to it not being set/present. Senders - // SHOULD interpret it the same way as in the full success case. - ExportMetricsPartialSuccess partial_success = 1; - - // List of compact descriptors for series IDs that were minted by the - // collector while processing this request. Exporters SHOULD cache these - // assignments and omit the associated attributes on subsequent exports once - // a series_id is known. - repeated SeriesAssignment series_assignments = 2; - - // Refactor-2026-05: sids the receiver did NOT recognize in this Export. - // Sent when the receiver's series-id cache has diverged from the - // sender's (e.g., after backend restart without persistence, or any - // sid-cache eviction). Senders MUST evict the listed sids from their - // local cache; subsequent emits MUST re-attach `attributes` so the - // receiver can re-resolve and return fresh SeriesAssignments. This - // field is the universal recovery primitive — same code path that - // handles cold-start bootstrap also handles every cache-divergence - // failure mode. - repeated uint64 unknown_series_ids = 3; -} - -message ExportMetricsPartialSuccess { - // The number of rejected data points. - // - // A `rejected_` field holding a `0` value indicates that the - // request was fully accepted. - int64 rejected_data_points = 1; - - // A developer-facing human-readable message in English. It should be used - // either to explain why the server rejected parts of the data during a partial - // success or to convey warnings/suggestions during a full success. The message - // should offer guidance on how users can address such issues. - // - // error_message is an optional field. An error_message with an empty value - // is equivalent to it not being set. - string error_message = 2; -} - -// SeriesAssignment communicates collector-assigned series identifiers back to -// exporters. Identifiers are expressed using the same deterministic key format -// as the collector's attribute de-duplication logic: resource_key and scope_key -// are built by concatenating the schema URL with sorted attribute pairs, -// metric_name is the OTLP Metric name, metric_type is the string form of -// opentelemetry.proto.metrics.v1.Metric.MetricType, and attributes_fingerprint -// is an opaque fingerprint of the sorted point-level attributes. -message SeriesAssignment { - // Deterministic encoding of the Resource associated with the series. The - // collector MUST build this using schema_url + sorted attributes separated - // by '|' to match the lookup key it uses internally. - string resource_key = 1; - - // Deterministic encoding of the instrumentation scope. Built from - // scope.name + scope.version + scope.schema_url + sorted attributes separated - // by '|'. - string scope_key = 2; - - // Fully-qualified metric identity. - string metric_name = 3; - - // String representation of the metric type (e.g. "Gauge", "Histogram"). - string metric_type = 4; - - // Opaque fingerprint of the data point's attribute set. The exporter MUST - // use the same fingerprinting algorithm when looking up cached series_ids. - bytes attributes_fingerprint = 5; - - // The collector-assigned series identifier. - uint64 series_id = 6; -} diff --git a/crates/asap_otel_proto/proto/opentelemetry/proto/common/v1/common.proto b/crates/asap_otel_proto/proto/opentelemetry/proto/common/v1/common.proto deleted file mode 100644 index 7f9ffab37..000000000 --- a/crates/asap_otel_proto/proto/opentelemetry/proto/common/v1/common.proto +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2019, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package opentelemetry.proto.common.v1; - -option csharp_namespace = "OpenTelemetry.Proto.Common.V1"; -option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.common.v1"; -option java_outer_classname = "CommonProto"; -option go_package = "go.opentelemetry.io/proto/otlp/common/v1"; - -// Represents any type of attribute value. AnyValue may contain a -// primitive value such as a string or integer or it may contain an arbitrary nested -// object containing arrays, key-value lists and primitives. -message AnyValue { - // The value is one of the listed fields. It is valid for all values to be unspecified - // in which case this AnyValue is considered to be "empty". - oneof value { - string string_value = 1; - bool bool_value = 2; - int64 int_value = 3; - double double_value = 4; - ArrayValue array_value = 5; - KeyValueList kvlist_value = 6; - bytes bytes_value = 7; - } -} - -// ArrayValue is a list of AnyValue messages. We need ArrayValue as a message -// since oneof in AnyValue does not allow repeated fields. -message ArrayValue { - // Array of values. The array may be empty (contain 0 elements). - repeated AnyValue values = 1; -} - -// KeyValueList is a list of KeyValue messages. We need KeyValueList as a message -// since `oneof` in AnyValue does not allow repeated fields. Everywhere else where we need -// a list of KeyValue messages (e.g. in Span) we use `repeated KeyValue` directly to -// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches -// are semantically equivalent. -message KeyValueList { - // A collection of key/value pairs of key-value pairs. The list may be empty (may - // contain 0 elements). - // - // The keys MUST be unique (it is not allowed to have more than one - // value with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. - repeated KeyValue values = 1; -} - -// Represents a key-value pair that is used to store Span attributes, Link -// attributes, etc. -message KeyValue { - // The key name of the pair. - string key = 1; - - // The value of the pair. - AnyValue value = 2; -} - -// InstrumentationScope is a message representing the instrumentation scope information -// such as the fully qualified name and version. -message InstrumentationScope { - // A name denoting the Instrumentation scope. - // An empty instrumentation scope name means the name is unknown. - string name = 1; - - // Defines the version of the instrumentation scope. - // An empty instrumentation scope version means the version is unknown. - string version = 2; - - // Additional attributes that describe the scope. [Optional]. - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. - repeated KeyValue attributes = 3; - - // The number of attributes that were discarded. Attributes - // can be discarded because their keys are too long or because there are too many - // attributes. If this value is 0, then no attributes were dropped. - uint32 dropped_attributes_count = 4; -} - -// A reference to an Entity. -// Entity represents an object of interest associated with produced telemetry: e.g spans, metrics, profiles, or logs. -// -// Status: [Development] -message EntityRef { - // The Schema URL, if known. This is the identifier of the Schema that the entity data - // is recorded in. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url - // - // This schema_url applies to the data in this message and to the Resource attributes - // referenced by id_keys and description_keys. - // TODO: discuss if we are happy with this somewhat complicated definition of what - // the schema_url applies to. - // - // This field obsoletes the schema_url field in ResourceMetrics/ResourceSpans/ResourceLogs. - string schema_url = 1; - - // Defines the type of the entity. MUST not change during the lifetime of the entity. - // For example: "service" or "host". This field is required and MUST not be empty - // for valid entities. - string type = 2; - - // Attribute Keys that identify the entity. - // MUST not change during the lifetime of the entity. The Id must contain at least one attribute. - // These keys MUST exist in the containing {message}.attributes. - repeated string id_keys = 3; - - // Descriptive (non-identifying) attribute keys of the entity. - // MAY change over the lifetime of the entity. MAY be empty. - // These attribute keys are not part of entity's identity. - // These keys MUST exist in the containing {message}.attributes. - repeated string description_keys = 4; -} \ No newline at end of file diff --git a/crates/asap_otel_proto/proto/opentelemetry/proto/metrics/v1/metrics.proto b/crates/asap_otel_proto/proto/opentelemetry/proto/metrics/v1/metrics.proto deleted file mode 100644 index 8aa6e443a..000000000 --- a/crates/asap_otel_proto/proto/opentelemetry/proto/metrics/v1/metrics.proto +++ /dev/null @@ -1,1047 +0,0 @@ -// Copyright 2019, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package opentelemetry.proto.metrics.v1; - -import "opentelemetry/proto/common/v1/common.proto"; -import "opentelemetry/proto/resource/v1/resource.proto"; - -option csharp_namespace = "OpenTelemetry.Proto.Metrics.V1"; -option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.metrics.v1"; -option java_outer_classname = "MetricsProto"; -option go_package = "go.opentelemetry.io/proto/otlp/metrics/v1"; - -// MetricsData represents the metrics data that can be stored in a persistent -// storage, OR can be embedded by other protocols that transfer OTLP metrics -// data but do not implement the OTLP protocol. -// -// MetricsData -// └─── ResourceMetrics -// ├── Resource -// ├── SchemaURL -// └── ScopeMetrics -// ├── Scope -// ├── SchemaURL -// └── Metric -// ├── Name -// ├── Description -// ├── Unit -// └── data -// ├── Gauge -// ├── Sum -// ├── Histogram -// ├── ExponentialHistogram -// ├── DDSketch -// └── Summary -// -// The main difference between this message and collector protocol is that -// in this message there will not be any "control" or "metadata" specific to -// OTLP protocol. -// -// When new fields are added into this message, the OTLP request MUST be updated -// as well. -message MetricsData { - // An array of ResourceMetrics. - // For data coming from a single resource this array will typically contain - // one element. Intermediary nodes that receive data from multiple origins - // typically batch the data before forwarding further and in that case this - // array will contain multiple elements. - repeated ResourceMetrics resource_metrics = 1; -} - -// A collection of ScopeMetrics from a Resource. -message ResourceMetrics { - reserved 1000; - - // The resource for the metrics in this message. - // If this field is not set then no resource info is known. - opentelemetry.proto.resource.v1.Resource resource = 1; - - // A list of metrics that originate from a resource. - repeated ScopeMetrics scope_metrics = 2; - - // The Schema URL, if known. This is the identifier of the Schema that the resource data - // is recorded in. Notably, the last part of the URL path is the version number of the - // schema: http[s]://server[:port]/path/. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url - // This schema_url applies to the data in the "resource" field. It does not apply - // to the data in the "scope_metrics" field which have their own schema_url field. - string schema_url = 3; -} - -// A collection of Metrics produced by an Scope. -message ScopeMetrics { - // The instrumentation scope information for the metrics in this message. - // Semantically when InstrumentationScope isn't set, it is equivalent with - // an empty instrumentation scope name (unknown). - opentelemetry.proto.common.v1.InstrumentationScope scope = 1; - - // A list of metrics that originate from an instrumentation library. - repeated Metric metrics = 2; - - // The Schema URL, if known. This is the identifier of the Schema that the metric data - // is recorded in. Notably, the last part of the URL path is the version number of the - // schema: http[s]://server[:port]/path/. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url - // This schema_url applies to the data in the "scope" field and all metrics in the - // "metrics" field. - string schema_url = 3; -} - -// Defines a Metric which has one or more timeseries. The following is a -// brief summary of the Metric data model. For more details, see: -// -// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/data-model.md -// -// The data model and relation between entities is shown in the -// diagram below. Here, "DataPoint" is the term used to refer to any -// one of the specific data point value types, and "points" is the term used -// to refer to any one of the lists of points contained in the Metric. -// -// - Metric is composed of a metadata and data. -// - Metadata part contains a name, description, unit. -// - Data is one of the possible types (Sum, Gauge, Histogram, Summary). -// - DataPoint contains timestamps, attributes, and one of the possible value type -// fields. -// -// Metric -// +------------+ -// |name | -// |description | -// |unit | +------------------------------------+ -// |data |---> |Gauge, Sum, Histogram, Summary, ... | -// +------------+ +------------------------------------+ -// -// Data [One of Gauge, Sum, Histogram, Summary, ...] -// +-----------+ -// |... | // Metadata about the Data. -// |points |--+ -// +-----------+ | -// | +---------------------------+ -// | |DataPoint 1 | -// v |+------+------+ +------+ | -// +-----+ ||label |label |...|label | | -// | 1 |-->||value1|value2|...|valueN| | -// +-----+ |+------+------+ +------+ | -// | . | |+-----+ | -// | . | ||value| | -// | . | |+-----+ | -// | . | +---------------------------+ -// | . | . -// | . | . -// | . | . -// | . | +---------------------------+ -// | . | |DataPoint M | -// +-----+ |+------+------+ +------+ | -// | M |-->||label |label |...|label | | -// +-----+ ||value1|value2|...|valueN| | -// |+------+------+ +------+ | -// |+-----+ | -// ||value| | -// |+-----+ | -// +---------------------------+ -// -// Each distinct type of DataPoint represents the output of a specific -// aggregation function, the result of applying the DataPoint's -// associated function of to one or more measurements. -// -// All DataPoint types have three common fields: -// - Attributes includes key-value pairs associated with the data point -// - TimeUnixNano is required, set to the end time of the aggregation -// - StartTimeUnixNano is optional, but strongly encouraged for DataPoints -// having an AggregationTemporality field, as discussed below. -// -// Both TimeUnixNano and StartTimeUnixNano values are expressed as -// UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. -// -// # TimeUnixNano -// -// This field is required, having consistent interpretation across -// DataPoint types. TimeUnixNano is the moment corresponding to when -// the data point's aggregate value was captured. -// -// Data points with the 0 value for TimeUnixNano SHOULD be rejected -// by consumers. -// -// # StartTimeUnixNano -// -// StartTimeUnixNano in general allows detecting when a sequence of -// observations is unbroken. This field indicates to consumers the -// start time for points with cumulative and delta -// AggregationTemporality, and it should be included whenever possible -// to support correct rate calculation. Although it may be omitted -// when the start time is truly unknown, setting StartTimeUnixNano is -// strongly encouraged. -message Metric { - reserved 4, 6, 8; - - // The name of the metric. - string name = 1; - - // A description of the metric, which can be used in documentation. - string description = 2; - - // The unit in which the metric value is reported. Follows the format - // described by https://unitsofmeasure.org/ucum.html. - string unit = 3; - - // Data determines the aggregation type (if any) of the metric, what is the - // reported value type for the data points, as well as the relatationship to - // the time interval over which they are reported. - oneof data { - Gauge gauge = 5; - Sum sum = 7; - Histogram histogram = 9; - ExponentialHistogram exponential_histogram = 10; - DDSketch ddsketch = 13; - Summary summary = 11; - KLLSketch kllsketch = 14; - CountSketch countsketch = 15; - CountMinSketch countminsketch = 16; - HLLSketch hllsketch = 17; - // Scalar Sum aggregate (AggregationKind = Sum); field 18 mirrors the - // collector pdata SumAgg oneof tag. Decoded into ExactAgg(Sum). - SumAgg sum_agg = 18; - } - - // Additional metadata attributes that describe the metric. [Optional]. - // Attributes are non-identifying. - // Consumers SHOULD NOT need to be aware of these attributes. - // These attributes MAY be used to encode information allowing - // for lossless roundtrip translation to / from another data model. - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. - repeated opentelemetry.proto.common.v1.KeyValue metadata = 12; -} - -// Gauge represents the type of a scalar metric that always exports the -// "current value" for every data point. It should be used for an "unknown" -// aggregation. -// -// A Gauge does not support different aggregation temporalities. Given the -// aggregation is unknown, points cannot be combined using the same -// aggregation, regardless of aggregation temporalities. Therefore, -// AggregationTemporality is not included. Consequently, this also means -// "StartTimeUnixNano" is ignored for all data points. -message Gauge { - // The time series data points. - // Note: Multiple time series may be included (same timestamp, different attributes). - repeated NumberDataPoint data_points = 1; -} - -// Sum represents the type of a scalar metric that is calculated as a sum of all -// reported measurements over a time interval. -message Sum { - // The time series data points. - // Note: Multiple time series may be included (same timestamp, different attributes). - repeated NumberDataPoint data_points = 1; - - // aggregation_temporality describes if the aggregator reports delta changes - // since last report time, or cumulative changes since a fixed start time. - AggregationTemporality aggregation_temporality = 2; - - // Represents whether the sum is monotonic. - bool is_monotonic = 3; -} - -// Histogram represents the type of a metric that is calculated by aggregating -// as a Histogram of all reported measurements over a time interval. -message Histogram { - // The time series data points. - // Note: Multiple time series may be included (same timestamp, different attributes). - repeated HistogramDataPoint data_points = 1; - - // aggregation_temporality describes if the aggregator reports delta changes - // since last report time, or cumulative changes since a fixed start time. - AggregationTemporality aggregation_temporality = 2; -} - -// ExponentialHistogram represents the type of a metric that is calculated by aggregating -// as a ExponentialHistogram of all reported double measurements over a time interval. -message ExponentialHistogram { - // The time series data points. - // Note: Multiple time series may be included (same timestamp, different attributes). - repeated ExponentialHistogramDataPoint data_points = 1; - - // aggregation_temporality describes if the aggregator reports delta changes - // since last report time, or cumulative changes since a fixed start time. - AggregationTemporality aggregation_temporality = 2; -} - -// DDSketch represents the type of a metric that encodes measurements using DDSketch buckets. -message DDSketch { - // The time series data points. - // Note: Multiple time series may be included (same timestamp, different attributes). - repeated DDSketchDataPoint data_points = 1; - - // aggregation_temporality describes if the aggregator reports delta changes - // since last report time, or cumulative changes since a fixed start time. - AggregationTemporality aggregation_temporality = 2; - - // Refactor-2026-05: sketch instance config — DDSketch's relative - // accuracy (alpha). Sent ONCE per Metric emit (not per DataPoint), - // since this is a per-sketch-instance parameter, not per-window - // mutable state. Backend reads this into - // SketchInstanceMetadata.sketch_config on first sight and caches. - double relative_accuracy = 3; -} - -// SumAgg represents a first-class scalar Sum aggregate (AggregationKind = Sum), -// carried as a portable {sum,count} envelope in the sketch bytes. Not a sketch; -// shares the modified-OTLP metric data oneof so the backend decodes it via the -// same envelope path (into the ExactAgg(Sum) accumulator). -message SumAgg { - repeated SumAggDataPoint data_points = 1; - AggregationTemporality aggregation_temporality = 2; -} - -// SumAggDataPoint carries a scalar Sum aggregate as a {sum,count} envelope. -message SumAggDataPoint { - repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; - fixed64 start_time_unix_nano = 2; - fixed64 time_unix_nano = 3; - // Serialized SumState envelope (sketchlib SketchEnvelope{sum:SumState}). - bytes sketch = 8; - SumAggEncoding encoding = 10; - repeated Exemplar exemplars = 11; - uint32 flags = 15; - uint64 series_id = 16; -} - -// SumAggEncoding identifies how the SumAgg payload bytes are encoded. -enum SumAggEncoding { - SUM_AGG_ENCODING_UNSPECIFIED = 0; - SUM_AGG_ENCODING_PROTO = 1; - SUM_AGG_ENCODING_PROTO_DELTA = 2; - SUM_AGG_ENCODING_MSGPACK = 3; - SUM_AGG_ENCODING_MSGPACK_DELTA = 4; -} - -// Summary metric data are used to convey quantile summaries, -// a Prometheus (see: https://prometheus.io/docs/concepts/metric_types/#summary) -// and OpenMetrics (see: https://github.com/prometheus/OpenMetrics/blob/4dbf6075567ab43296eed941037c12951faafb92/protos/prometheus.proto#L45) -// data type. These data points cannot always be merged in a meaningful way. -// While they can be useful in some applications, histogram data points are -// recommended for new applications. -// Summary metrics do not have an aggregation temporality field. This is -// because the count and sum fields of a SummaryDataPoint are assumed to be -// cumulative values. -message Summary { - // The time series data points. - // Note: Multiple time series may be included (same timestamp, different attributes). - repeated SummaryDataPoint data_points = 1; -} - -// DDSketchDataPoint is a single data point that encodes a distribution using the DDSketch format. -// -// Refactor-2026-05: removed precomputed/duplicated fields (count, sum, -// min, max) — they are derivable from the sketch payload, so storing -// them per-DataPoint duplicates state and risks cache-invalidation -// bugs. Field numbers reserved for forward/backward compat. -message DDSketchDataPoint { - reserved 1; // legacy "labels" (replaced by `attributes`) - reserved 4, 5, 6, 7, 12, 13, 14; // refactor-2026-05: count, sum oneof, min oneof, max oneof — derivable from sketch state - - // The set of key/value pairs that uniquely identify the timeseries from - // where this point belongs. The list MUST be omitted (or present but empty) - // whenever `series_id` is non-zero, and MUST be populated when `series_id` - // is zero so the receiver can mint a new entry. - // Attribute keys MUST be unique. After group-by rollup at the agent's - // sketch processor, `attributes.keys()` IS the group-by KEY set; - // `attributes` (as a map) is the group-by VALUES for this series. - repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; - - // StartTimeUnixNano is optional but strongly encouraged, see the - // the detailed comments above Metric. - fixed64 start_time_unix_nano = 2; - - // TimeUnixNano is required, see the detailed comments above Metric. - fixed64 time_unix_nano = 3; - - // Serialized DDSketch payload. The encoding is described by the encoding field. - bytes sketch = 8; - - // Encoding that describes how the sketch bytes should be decoded. - DDSketchEncoding encoding = 10; - - // (Optional) List of exemplars collected from - // measurements that were used to form the data point - repeated Exemplar exemplars = 11; - - // Flags that apply to this specific data point. See DataPointFlags - // for the available flags and their meaning. - uint32 flags = 15; - - // series_id references a collector-assigned series descriptor. Receivers MUST - // reject the point unless exactly one of (`series_id` != 0, `attributes` populated) - // is true. - // After Phase 5 (centralized resolver at asap-query-backend), this id - // is the BACKEND-assigned global identifier for `(metric_name, attributes)`, - // shared across all hops in the pipeline. - uint64 series_id = 16; -} - -// DDSketchEncoding identifies how the DDSketch payload bytes are encoded. -enum DDSketchEncoding { - DDSKETCH_ENCODING_UNSPECIFIED = 0; - - // DDSKETCH_ENCODING_PROTO indicates the sketch bytes contain the serialized - // github.com/DataDog/sketches-go/ddsketch/pb/sketchpb.DDSketch message. - DDSKETCH_ENCODING_PROTO = 1; - DDSKETCH_ENCODING_PROTO_DELTA = 2; - // DDSKETCH_ENCODING_MSGPACK indicates the sketch bytes are the MessagePack - // serialization of the cross-language sketch-core `DdSketch` struct - // (matching ASAPQuery-backend's `sketch_core::dd_sketch::DdSketch` and - // sketchlib-go's `wire/asapmsgpack.DDSketchState`). - DDSKETCH_ENCODING_MSGPACK = 3; - DDSKETCH_ENCODING_MSGPACK_DELTA = 4; -} - -// KLLSketch represents the type of a metric that encodes measurements using the KLL quantile sketch. -message KLLSketch { - repeated KLLSketchDataPoint data_points = 1; - AggregationTemporality aggregation_temporality = 2; - - // Refactor-2026-05: KLL sketch parameter `k` — controls quantile - // error / space tradeoff. Sent once per Metric emit. - uint32 k = 3; -} - -// KLLSketchDataPoint is a single data point that encodes a distribution using the KLL sketch format. -// -// Refactor-2026-05: removed count, sum, min, max — derivable from sketch state. -message KLLSketchDataPoint { - reserved 4, 5, 6, 7; // count, sum, min, max - repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; - fixed64 start_time_unix_nano = 2; - fixed64 time_unix_nano = 3; - bytes sketch = 8; - KLLSketchEncoding encoding = 9; - uint32 flags = 10; - uint64 series_id = 11; -} - -// KLLSketchEncoding identifies how the KLL sketch payload bytes are encoded. -enum KLLSketchEncoding { - KLL_SKETCH_ENCODING_UNSPECIFIED = 0; - KLL_SKETCH_ENCODING_PROTO = 1; - // KLL_SKETCH_ENCODING_MSGPACK is reserved for the cross-language - // sketch-core `KllSketch` msgpack wire format. Not currently usable - // because sketchlib-go's KLL does not share a byte-level backend - // with ASAPQuery-backend's datasketches-rs-backed `KllSketch` - // (tracked as a follow-up — for KLL today, use PROTO). - KLL_SKETCH_ENCODING_MSGPACK = 3; - KLL_SKETCH_ENCODING_MSGPACK_DELTA = 4; -} - -// CountSketch represents the type of a metric that encodes frequency estimations using CountSketch. -message CountSketch { - repeated CountSketchDataPoint data_points = 1; - AggregationTemporality aggregation_temporality = 2; - - // Refactor-2026-05: CountSketch matrix dimensions. depth × width. - // (epsilon, delta) deriveable: eps = sqrt(e/cols) for L2 estimation; - // delta ≈ 1/(2^(rows/2)). Backend computes these for display. - int32 rows = 3; // depth - int32 cols = 4; // width -} - -// CountSketchDataPoint is a single data point encoding frequency estimations. -// -// Refactor-2026-05: removed `dimension` (legacy partition-key -// descriptor — replaced by `attributes` keys = group-by labels); -// removed `epsilon`/`delta` (derivable from container's rows/cols). -message CountSketchDataPoint { - reserved 6, 7, 8; // dimension, epsilon, delta - repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; - fixed64 start_time_unix_nano = 2; - fixed64 time_unix_nano = 3; - bytes sketch = 4; - CountSketchEncoding encoding = 5; - uint32 flags = 9; - uint64 series_id = 10; -} - -// CountSketchEncoding identifies how the CountSketch payload bytes are encoded. -enum CountSketchEncoding { - COUNT_SKETCH_ENCODING_UNSPECIFIED = 0; - COUNT_SKETCH_ENCODING_PROTO = 1; - COUNT_SKETCH_ENCODING_DELTA = 2; - // COUNT_SKETCH_ENCODING_MSGPACK indicates the sketch bytes are the - // MessagePack serialization of the cross-language sketch-core - // `CountSketch` struct (matching ASAPQuery-backend's - // `sketch_core::count_sketch::CountSketch` and sketchlib-go's - // `wire/asapmsgpack.MarshalCountSketch`). - COUNT_SKETCH_ENCODING_MSGPACK = 3; - COUNT_SKETCH_ENCODING_MSGPACK_DELTA = 4; -} - -// CountMinSketch represents the type of a metric encoding frequency estimations using CountMinSketch. -message CountMinSketch { - repeated CountMinSketchDataPoint data_points = 1; - AggregationTemporality aggregation_temporality = 2; - - // Refactor-2026-05: CMS matrix dimensions. depth × width. - // (epsilon, delta) deriveable: eps ≈ e/cols; delta ≈ exp(-rows). - int32 rows = 3; // depth = ⌈ln(1/delta)⌉ - int32 cols = 4; // width = ⌈e/epsilon⌉ -} - -// CountMinSketchDataPoint is a single data point encoding frequency estimations. -// -// Refactor-2026-05: removed `sample_count` (derivable from sketch -// row sums); removed `rows`/`cols` (lifted to parent container, sent -// once per Metric emit instead of per DataPoint). -message CountMinSketchDataPoint { - reserved 4, 7, 8; // sample_count, rows, cols - repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; - fixed64 start_time_unix_nano = 2; - fixed64 time_unix_nano = 3; - bytes sketch = 5; - CountMinSketchEncoding encoding = 6; - uint32 flags = 9; - uint64 series_id = 10; -} - -// CountMinSketchEncoding identifies how the CountMinSketch payload bytes are encoded. -enum CountMinSketchEncoding { - COUNT_MIN_SKETCH_ENCODING_UNSPECIFIED = 0; - COUNT_MIN_SKETCH_ENCODING_PROTO = 1; - COUNT_MIN_SKETCH_ENCODING_DELTA = 2; - // COUNT_MIN_SKETCH_ENCODING_MSGPACK indicates the sketch bytes are - // the MessagePack serialization of the cross-language sketch-core - // `CountMinSketch` wire struct (matching ASAPQuery-backend's - // `sketch_core::count_min::CountMinSketch` and sketchlib-go's - // `wire/asapmsgpack.MarshalCountMinSketch`). - COUNT_MIN_SKETCH_ENCODING_MSGPACK = 3; - COUNT_MIN_SKETCH_ENCODING_MSGPACK_DELTA = 4; -} - -// HLLSketch represents the type of a metric that encodes cardinality estimations using HyperLogLog. -message HLLSketch { - repeated HLLSketchDataPoint data_points = 1; - AggregationTemporality aggregation_temporality = 2; - - // Refactor-2026-05: HLL register-bucket precision (= log2(register count)). - // Standard error ≈ 1.04 / sqrt(2^precision). Sent once per Metric emit. - uint32 precision = 3; -} - -// HLLSketchDataPoint is a single data point encoding cardinality estimations. -// -// Refactor-2026-05: removed `count` (derivable from sketch register -// state), removed `cardinality` (precomputed cache of -// sketch.estimate() — let the receiver evaluate on demand to avoid -// cache drift). `precision` lifted to parent container. -message HLLSketchDataPoint { - reserved 4, 5, 8; // count, cardinality, precision - repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; - fixed64 start_time_unix_nano = 2; - fixed64 time_unix_nano = 3; - bytes sketch = 6; - HLLSketchEncoding encoding = 7; - uint32 flags = 9; - uint64 series_id = 10; -} - -// HLLSketchEncoding identifies how the HLL sketch payload bytes are encoded. -enum HLLSketchEncoding { - HLL_SKETCH_ENCODING_UNSPECIFIED = 0; - HLL_SKETCH_ENCODING_PROTO = 1; - HLL_SKETCH_ENCODING_DELTA = 2; - // HLL_SKETCH_ENCODING_MSGPACK indicates the sketch bytes are the - // MessagePack serialization of the cross-language sketch-core - // `HllSketch` struct — registers + variant + precision + HIP state - // (matching ASAPQuery-backend's `sketch_core::hll_sketch::HllSketch` - // and sketchlib-go's `wire/asapmsgpack.MarshalHLLSketch`). - HLL_SKETCH_ENCODING_MSGPACK = 3; - HLL_SKETCH_ENCODING_MSGPACK_DELTA = 4; -} - -// AggregationTemporality defines how a metric aggregator reports aggregated -// values. It describes how those values relate to the time interval over -// which they are aggregated. -enum AggregationTemporality { - // UNSPECIFIED is the default AggregationTemporality, it MUST not be used. - AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; - - // DELTA is an AggregationTemporality for a metric aggregator which reports - // changes since last report time. Successive metrics contain aggregation of - // values from continuous and non-overlapping intervals. - // - // The values for a DELTA metric are based only on the time interval - // associated with one measurement cycle. There is no dependency on - // previous measurements like is the case for CUMULATIVE metrics. - // - // For example, consider a system measuring the number of requests that - // it receives and reports the sum of these requests every second as a - // DELTA metric: - // - // 1. The system starts receiving at time=t_0. - // 2. A request is received, the system measures 1 request. - // 3. A request is received, the system measures 1 request. - // 4. A request is received, the system measures 1 request. - // 5. The 1 second collection cycle ends. A metric is exported for the - // number of requests received over the interval of time t_0 to - // t_0+1 with a value of 3. - // 6. A request is received, the system measures 1 request. - // 7. A request is received, the system measures 1 request. - // 8. The 1 second collection cycle ends. A metric is exported for the - // number of requests received over the interval of time t_0+1 to - // t_0+2 with a value of 2. - AGGREGATION_TEMPORALITY_DELTA = 1; - - // CUMULATIVE is an AggregationTemporality for a metric aggregator which - // reports changes since a fixed start time. This means that current values - // of a CUMULATIVE metric depend on all previous measurements since the - // start time. Because of this, the sender is required to retain this state - // in some form. If this state is lost or invalidated, the CUMULATIVE metric - // values MUST be reset and a new fixed start time following the last - // reported measurement time sent MUST be used. - // - // For example, consider a system measuring the number of requests that - // it receives and reports the sum of these requests every second as a - // CUMULATIVE metric: - // - // 1. The system starts receiving at time=t_0. - // 2. A request is received, the system measures 1 request. - // 3. A request is received, the system measures 1 request. - // 4. A request is received, the system measures 1 request. - // 5. The 1 second collection cycle ends. A metric is exported for the - // number of requests received over the interval of time t_0 to - // t_0+1 with a value of 3. - // 6. A request is received, the system measures 1 request. - // 7. A request is received, the system measures 1 request. - // 8. The 1 second collection cycle ends. A metric is exported for the - // number of requests received over the interval of time t_0 to - // t_0+2 with a value of 5. - // 9. The system experiences a fault and loses state. - // 10. The system recovers and resumes receiving at time=t_1. - // 11. A request is received, the system measures 1 request. - // 12. The 1 second collection cycle ends. A metric is exported for the - // number of requests received over the interval of time t_1 to - // t_0+1 with a value of 1. - // - // Note: Even though, when reporting changes since last report time, using - // CUMULATIVE is valid, it is not recommended. This may cause problems for - // systems that do not use start_time to determine when the aggregation - // value was reset (e.g. Prometheus). - AGGREGATION_TEMPORALITY_CUMULATIVE = 2; -} - -// DataPointFlags is defined as a protobuf 'uint32' type and is to be used as a -// bit-field representing 32 distinct boolean flags. Each flag defined in this -// enum is a bit-mask. To test the presence of a single flag in the flags of -// a data point, for example, use an expression like: -// -// (point.flags & DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK) == DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK -// -enum DataPointFlags { - // The zero value for the enum. Should not be used for comparisons. - // Instead use bitwise "and" with the appropriate mask as shown above. - DATA_POINT_FLAGS_DO_NOT_USE = 0; - - // This DataPoint is valid but has no recorded value. This value - // SHOULD be used to reflect explicitly missing data in a series, as - // for an equivalent to the Prometheus "staleness marker". - DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK = 1; - - // Bits 2-31 are reserved for future use. -} - -// NumberDataPoint is a single data point in a timeseries that describes the -// time-varying scalar value of a metric. -message NumberDataPoint { - reserved 1; - - // The set of key/value pairs that uniquely identify the timeseries from - // where this point belongs. The list MUST be omitted (or present but empty) - // whenever `series_id` is non-zero, and MUST be populated when `series_id` - // is zero so the receiver can mint a new entry. - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. - repeated opentelemetry.proto.common.v1.KeyValue attributes = 7; - - // StartTimeUnixNano is optional but strongly encouraged, see the - // the detailed comments above Metric. - // - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January - // 1970. - fixed64 start_time_unix_nano = 2; - - // TimeUnixNano is required, see the detailed comments above Metric. - // - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January - // 1970. - fixed64 time_unix_nano = 3; - - // The value itself. A point is considered invalid when one of the recognized - // value fields is not present inside this oneof. - oneof value { - double as_double = 4; - sfixed64 as_int = 6; - } - - // (Optional) List of exemplars collected from - // measurements that were used to form the data point - repeated Exemplar exemplars = 5; - - // Flags that apply to this specific data point. See DataPointFlags - // for the available flags and their meaning. - uint32 flags = 8; - - // series_id references a collector-assigned series descriptor. Receivers MUST - // reject the point unless exactly one of (`series_id` != 0, `attributes` populated) - // is true. - uint64 series_id = 9; -} - -// HistogramDataPoint is a single data point in a timeseries that describes the -// time-varying values of a Histogram. A Histogram contains summary statistics -// for a population of values, it may optionally contain the distribution of -// those values across a set of buckets. -// -// If the histogram contains the distribution of values, then both -// "explicit_bounds" and "bucket counts" fields must be defined. -// If the histogram does not contain the distribution of values, then both -// "explicit_bounds" and "bucket_counts" must be omitted and only "count" and -// "sum" are known. -message HistogramDataPoint { - reserved 1; - - // The set of key/value pairs that uniquely identify the timeseries from - // where this point belongs. The list MUST be omitted (or present but empty) - // whenever `series_id` is non-zero, and MUST be populated when `series_id` - // is zero so the receiver can mint a new entry. - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. - repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; - - // StartTimeUnixNano is optional but strongly encouraged, see the - // the detailed comments above Metric. - // - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January - // 1970. - fixed64 start_time_unix_nano = 2; - - // TimeUnixNano is required, see the detailed comments above Metric. - // - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January - // 1970. - fixed64 time_unix_nano = 3; - - // count is the number of values in the population. Must be non-negative. This - // value must be equal to the sum of the "count" fields in buckets if a - // histogram is provided. - fixed64 count = 4; - - // sum of the values in the population. If count is zero then this field - // must be zero. - // - // Note: Sum should only be filled out when measuring non-negative discrete - // events, and is assumed to be monotonic over the values of these events. - // Negative events *can* be recorded, but sum should not be filled out when - // doing so. This is specifically to enforce compatibility w/ OpenMetrics, - // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram - optional double sum = 5; - - // bucket_counts is an optional field contains the count values of histogram - // for each bucket. - // - // The sum of the bucket_counts must equal the value in the count field. - // - // The number of elements in bucket_counts array must be by one greater than - // the number of elements in explicit_bounds array. The exception to this rule - // is when the length of bucket_counts is 0, then the length of explicit_bounds - // must also be 0. - repeated fixed64 bucket_counts = 6; - - // explicit_bounds specifies buckets with explicitly defined bounds for values. - // - // The boundaries for bucket at index i are: - // - // (-infinity, explicit_bounds[i]] for i == 0 - // (explicit_bounds[i-1], explicit_bounds[i]] for 0 < i < size(explicit_bounds) - // (explicit_bounds[i-1], +infinity) for i == size(explicit_bounds) - // - // The values in the explicit_bounds array must be strictly increasing. - // - // Histogram buckets are inclusive of their upper boundary, except the last - // bucket where the boundary is at infinity. This format is intentionally - // compatible with the OpenMetrics histogram definition. - // - // If bucket_counts length is 0 then explicit_bounds length must also be 0, - // otherwise the data point is invalid. - repeated double explicit_bounds = 7; - - // (Optional) List of exemplars collected from - // measurements that were used to form the data point - repeated Exemplar exemplars = 8; - - // Flags that apply to this specific data point. See DataPointFlags - // for the available flags and their meaning. - uint32 flags = 10; - - // min is the minimum value over (start_time, end_time]. - optional double min = 11; - - // max is the maximum value over (start_time, end_time]. - optional double max = 12; - - // series_id references a collector-assigned series descriptor. Receivers MUST - // reject the point unless exactly one of (`series_id` != 0, `attributes` populated) - // is true. - uint64 series_id = 13; -} - -// ExponentialHistogramDataPoint is a single data point in a timeseries that describes the -// time-varying values of a ExponentialHistogram of double values. A ExponentialHistogram contains -// summary statistics for a population of values, it may optionally contain the -// distribution of those values across a set of buckets. -// -message ExponentialHistogramDataPoint { - // The set of key/value pairs that uniquely identify the timeseries from - // where this point belongs. The list MUST be omitted (or present but empty) - // whenever `series_id` is non-zero, and MUST be populated when `series_id` - // is zero so the receiver can mint a new entry. - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. - repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; - - // StartTimeUnixNano is optional but strongly encouraged, see the - // the detailed comments above Metric. - // - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January - // 1970. - fixed64 start_time_unix_nano = 2; - - // TimeUnixNano is required, see the detailed comments above Metric. - // - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January - // 1970. - fixed64 time_unix_nano = 3; - - // The number of values in the population. Must be - // non-negative. This value must be equal to the sum of the "bucket_counts" - // values in the positive and negative Buckets plus the "zero_count" field. - fixed64 count = 4; - - // The sum of the values in the population. If count is zero then this field - // must be zero. - // - // Note: Sum should only be filled out when measuring non-negative discrete - // events, and is assumed to be monotonic over the values of these events. - // Negative events *can* be recorded, but sum should not be filled out when - // doing so. This is specifically to enforce compatibility w/ OpenMetrics, - // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram - optional double sum = 5; - - // scale describes the resolution of the histogram. Boundaries are - // located at powers of the base, where: - // - // base = (2^(2^-scale)) - // - // The histogram bucket identified by `index`, a signed integer, - // contains values that are greater than (base^index) and - // less than or equal to (base^(index+1)). - // - // The positive and negative ranges of the histogram are expressed - // separately. Negative values are mapped by their absolute value - // into the negative range using the same scale as the positive range. - // - // scale is not restricted by the protocol, as the permissible - // values depend on the range of the data. - sint32 scale = 6; - - // The count of values that are either exactly zero or - // within the region considered zero by the instrumentation at the - // tolerated degree of precision. This bucket stores values that - // cannot be expressed using the standard exponential formula as - // well as values that have been rounded to zero. - // - // Implementations MAY consider the zero bucket to have probability - // mass equal to (zero_count / count). - fixed64 zero_count = 7; - - // positive carries the positive range of exponential bucket counts. - Buckets positive = 8; - - // negative carries the negative range of exponential bucket counts. - Buckets negative = 9; - - // Buckets are a set of bucket counts, encoded in a contiguous array - // of counts. - message Buckets { - // The bucket index of the first entry in the bucket_counts array. - // - // Note: This uses a varint encoding as a simple form of compression. - sint32 offset = 1; - - // An array of count values, where bucket_counts[i] carries - // the count of the bucket at index (offset+i). bucket_counts[i] is the count - // of values greater than base^(offset+i) and less than or equal to - // base^(offset+i+1). - // - // Note: By contrast, the explicit HistogramDataPoint uses - // fixed64. This field is expected to have many buckets, - // especially zeros, so uint64 has been selected to ensure - // varint encoding. - repeated uint64 bucket_counts = 2; - } - - // Flags that apply to this specific data point. See DataPointFlags - // for the available flags and their meaning. - uint32 flags = 10; - - // (Optional) List of exemplars collected from - // measurements that were used to form the data point - repeated Exemplar exemplars = 11; - - // The minimum value over (start_time, end_time]. - optional double min = 12; - - // The maximum value over (start_time, end_time]. - optional double max = 13; - - // ZeroThreshold may be optionally set to convey the width of the zero - // region. Where the zero region is defined as the closed interval - // [-ZeroThreshold, ZeroThreshold]. - // When ZeroThreshold is 0, zero count bucket stores values that cannot be - // expressed using the standard exponential formula as well as values that - // have been rounded to zero. - double zero_threshold = 14; - - // series_id references a collector-assigned series descriptor. Receivers MUST - // reject the point unless exactly one of (`series_id` != 0, `attributes` populated) - // is true. - uint64 series_id = 15; -} - -// SummaryDataPoint is a single data point in a timeseries that describes the -// time-varying values of a Summary metric. The count and sum fields represent -// cumulative values. -message SummaryDataPoint { - reserved 1; - - // The set of key/value pairs that uniquely identify the timeseries from - // where this point belongs. The list MUST be omitted (or present but empty) - // whenever `series_id` is non-zero, and MUST be populated when `series_id` - // is zero so the receiver can mint a new entry. - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. - repeated opentelemetry.proto.common.v1.KeyValue attributes = 7; - - // StartTimeUnixNano is optional but strongly encouraged, see the - // the detailed comments above Metric. - // - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January - // 1970. - fixed64 start_time_unix_nano = 2; - - // TimeUnixNano is required, see the detailed comments above Metric. - // - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January - // 1970. - fixed64 time_unix_nano = 3; - - // count is the number of values in the population. Must be non-negative. - fixed64 count = 4; - - // sum of the values in the population. If count is zero then this field - // must be zero. - // - // Note: Sum should only be filled out when measuring non-negative discrete - // events, and is assumed to be monotonic over the values of these events. - // Negative events *can* be recorded, but sum should not be filled out when - // doing so. This is specifically to enforce compatibility w/ OpenMetrics, - // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#summary - double sum = 5; - - // Represents the value at a given quantile of a distribution. - // - // To record Min and Max values following conventions are used: - // - The 1.0 quantile is equivalent to the maximum value observed. - // - The 0.0 quantile is equivalent to the minimum value observed. - // - // See the following issue for more context: - // https://github.com/open-telemetry/opentelemetry-proto/issues/125 - message ValueAtQuantile { - // The quantile of a distribution. Must be in the interval - // [0.0, 1.0]. - double quantile = 1; - - // The value at the given quantile of a distribution. - // - // Quantile values must NOT be negative. - double value = 2; - } - - // (Optional) list of values at different quantiles of the distribution calculated - // from the current snapshot. The quantiles must be strictly increasing. - repeated ValueAtQuantile quantile_values = 6; - - // Flags that apply to this specific data point. See DataPointFlags - // for the available flags and their meaning. - uint32 flags = 8; - - // series_id references a collector-assigned series descriptor. Receivers MUST - // reject the point unless exactly one of (`series_id` != 0, `attributes` populated) - // is true. - uint64 series_id = 9; -} - -// A representation of an exemplar, which is a sample input measurement. -// Exemplars also hold information about the environment when the measurement -// was recorded, for example the span and trace ID of the active span when the -// exemplar was recorded. -message Exemplar { - reserved 1; - - // The set of key/value pairs that were filtered out by the aggregator, but - // recorded alongside the original measurement. Only key/value pairs that were - // filtered out by the aggregator should be included - repeated opentelemetry.proto.common.v1.KeyValue filtered_attributes = 7; - - // time_unix_nano is the exact time when this exemplar was recorded - // - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January - // 1970. - fixed64 time_unix_nano = 2; - - // The value of the measurement that was recorded. An exemplar is - // considered invalid when one of the recognized value fields is not present - // inside this oneof. - oneof value { - double as_double = 3; - sfixed64 as_int = 6; - } - - // (Optional) Span ID of the exemplar trace. - // span_id may be missing if the measurement is not recorded inside a trace - // or if the trace is not sampled. - bytes span_id = 4; - - // (Optional) Trace ID of the exemplar trace. - // trace_id may be missing if the measurement is not recorded inside a trace - // or if the trace is not sampled. - bytes trace_id = 5; -} diff --git a/crates/asap_otel_proto/proto/opentelemetry/proto/resource/v1/resource.proto b/crates/asap_otel_proto/proto/opentelemetry/proto/resource/v1/resource.proto deleted file mode 100644 index 42c5913cf..000000000 --- a/crates/asap_otel_proto/proto/opentelemetry/proto/resource/v1/resource.proto +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2019, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package opentelemetry.proto.resource.v1; - -import "opentelemetry/proto/common/v1/common.proto"; - -option csharp_namespace = "OpenTelemetry.Proto.Resource.V1"; -option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.resource.v1"; -option java_outer_classname = "ResourceProto"; -option go_package = "go.opentelemetry.io/proto/otlp/resource/v1"; - -// Resource information. -message Resource { - // Set of attributes that describe the resource. - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. - repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; - - // The number of dropped attributes. If the value is 0, then - // no attributes were dropped. - uint32 dropped_attributes_count = 2; - - // Set of entities that participate in this Resource. - // - // Note: keys in the references MUST exist in attributes of this message. - // - // Status: [Development] - repeated opentelemetry.proto.common.v1.EntityRef entity_refs = 3; -} diff --git a/crates/asap_otel_proto/proto/sketchlib_delta/countminsketch_delta.proto b/crates/asap_otel_proto/proto/sketchlib_delta/countminsketch_delta.proto deleted file mode 100644 index 49581a33b..000000000 --- a/crates/asap_otel_proto/proto/sketchlib_delta/countminsketch_delta.proto +++ /dev/null @@ -1,25 +0,0 @@ -// countminsketch_delta.proto — Delta wire format for CountMin. -// -// Mirrors the new packed encoding of `CountMinDelta` in -// `sketchlib-go/proto/countminsketch/countminsketch.proto`. -// Deprecated `cells_legacy` cells + their `d_sum` / `d_sum2` -// accessories are omitted — new sketchlib-go producers write -// the packed arrays instead. -// -// Semantics: matrix[cell_rows[i]][cell_cols[i]] += d_counts[i]. -// L1 and L2 are per-row norm deltas, transmitted for queryside -// accuracy-accounting; not needed for the core merge. - -syntax = "proto3"; - -package sketchlib.v1; - -message CountMinDelta { - uint32 rows = 1; - uint32 cols = 2; - repeated double l1 = 4 [packed = true]; - repeated double l2 = 5 [packed = true]; - repeated uint32 cell_rows = 9 [packed = true]; - repeated uint32 cell_cols = 10 [packed = true]; - repeated sint64 d_counts = 11 [packed = true]; -} diff --git a/crates/asap_otel_proto/proto/sketchlib_delta/countsketch_delta.proto b/crates/asap_otel_proto/proto/sketchlib_delta/countsketch_delta.proto deleted file mode 100644 index c9d31042e..000000000 --- a/crates/asap_otel_proto/proto/sketchlib_delta/countsketch_delta.proto +++ /dev/null @@ -1,33 +0,0 @@ -// countsketch_delta.proto — Delta wire format for CountSketch. -// -// Mirrors the new packed encoding of `CountSketchDelta` in -// `sketchlib-go/proto/countsketch/countsketch.proto`. Only the -// packed `cell_rows` / `cell_cols` / `d_counts` path is modeled -// here — the deprecated `cells_legacy` / `l2_legacy` fields and -// the `topk` / `hh_keys` top-K carriers are omitted because: -// -// * Legacy cells: new producers (sketchlib-go v0 and later) -// don't emit them. -// * Top-K on delta path: CountSketch's top-K is non-linear, so -// sketchlib-go transmits it in full (TopKState), not via -// delta. Merging top-K deltas would require re-querying the -// CS accumulator for each candidate key, which the backend -// doesn't wire today. A follow-up PR can pick this up once -// the §6.4 accuracy experiments flag top-K as worth chasing. -// -// Bucket additive semantics: `cell_rows[i]`, `cell_cols[i]` → -// matrix[row][col] += d_counts[i]. L2 norm is per-row -// additive. - -syntax = "proto3"; - -package sketchlib.v1; - -message CountSketchDelta { - uint32 rows = 1; - uint32 cols = 2; - repeated uint32 cell_rows = 9 [packed = true]; - repeated uint32 cell_cols = 10 [packed = true]; - repeated sint64 d_counts = 11 [packed = true]; - repeated double l2 = 12 [packed = true]; -} diff --git a/crates/asap_otel_proto/proto/sketchlib_delta/ddsketch_delta.proto b/crates/asap_otel_proto/proto/sketchlib_delta/ddsketch_delta.proto deleted file mode 100644 index f2aef92fc..000000000 --- a/crates/asap_otel_proto/proto/sketchlib_delta/ddsketch_delta.proto +++ /dev/null @@ -1,38 +0,0 @@ -// ddsketch_delta.proto — Delta wire format for DDSketch. -// -// Mirrors `sketchlib-go/proto/ddsketch/ddsketch.proto` DDSketchDelta + -// DDSketchBucketDelta messages byte-for-byte. Kept as a separate tiny -// proto in this crate because the upstream `asap_sketchlib` Rust crate -// only generates the state types today, not the delta types. Once -// `asap_sketchlib` picks these up upstream this vendoring can be removed. -// -// See `sketches/DDSketch/delta.go::ApplyDelta` for the canonical -// merge semantics — additive bucket counts only. The DataPoint-level -// METRIC scalars (count/sum/min/max) were dropped from the wire format -// (ProjectASAP/sketchlib-go#243 / asap_sketchlib#57): the total count is -// recoverable by summing the bucket counts and the remaining aggregates -// are carried by controller-provisioned exact aggregations. - -syntax = "proto3"; - -package sketchlib.v1; - -// DDSketchDelta carries only the buckets that changed above threshold. -// Bucket counts are additive. The former count/sum (tags 2-3) and -// min/max scalars (tags 4-7) were removed from the wire format -// (ProjectASAP/sketchlib-go#243): the count is recoverable by summing -// the merged bucket counts; Sum/Min/Max move to exact aggregations. -message DDSketchDelta { - repeated DDSketchBucketDelta buckets = 1; - - // Tags 2-3 previously carried the additive count/sum deltas and tags - // 4-7 the lossless min/max scalars + their changed flags. Dropped from - // the wire format (ProjectASAP/sketchlib-go#243). - reserved 2, 3, 4, 5, 6, 7; -} - -// DDSketchBucketDelta is the delta for one log-scale bucket. -message DDSketchBucketDelta { - sint32 index = 1; // signed bucket index (negative for values < 1) - uint64 d_count = 2; // Δcount ≥ threshold T -} diff --git a/crates/asap_otel_proto/proto/sketchlib_delta/hll_delta.proto b/crates/asap_otel_proto/proto/sketchlib_delta/hll_delta.proto deleted file mode 100644 index 6e49f9c86..000000000 --- a/crates/asap_otel_proto/proto/sketchlib_delta/hll_delta.proto +++ /dev/null @@ -1,17 +0,0 @@ -// hll_delta.proto — Delta wire format for HLL. -// -// The increased registers are varint-packed as (index_delta, value) pairs in -// ascending index order (index_delta from the previous index, starting at 0) — -// the same layout the full sparse register state uses. HLL applies a delta with -// max semantics: registers[idx] = max(registers[idx], value). -// -// No threshold is meaningful for HLL — any skipped register update causes -// permanent underestimation. - -syntax = "proto3"; - -package sketchlib.v1; - -message HLLDelta { - bytes packed_updates = 1; -} diff --git a/crates/asap_otel_proto/src/lib.rs b/crates/asap_otel_proto/src/lib.rs deleted file mode 100644 index 400d3099f..000000000 --- a/crates/asap_otel_proto/src/lib.rs +++ /dev/null @@ -1,63 +0,0 @@ -//! Vendored Rust bindings for DataCollector's modified OTLP proto. -//! -//! Adds first-class `DDSketch`, `KLLSketch`, `CountSketch`, `CountMinSketch`, -//! and `HLLSketch` variants to the `Metric.data` oneof on tags 13–17, plus -//! per-sketch encoding enums (with full and `*_DELTA` variants) and a -//! `series_id` field on every data point. See -//! `docs/pipeline-query-catalog.md` §5.4 in the DataCollector repo for the -//! full motivation and the corresponding upstream proto path. -//! -//! The generated Rust modules mirror `opentelemetry_proto::tonic::*` so call -//! sites only need to swap `opentelemetry_proto` for `asap_otel_proto`. - -#![allow(clippy::all)] - -pub mod tonic { - pub mod common { - pub mod v1 { - tonic::include_proto!("opentelemetry.proto.common.v1"); - } - } - - pub mod resource { - pub mod v1 { - tonic::include_proto!("opentelemetry.proto.resource.v1"); - } - } - - pub mod metrics { - pub mod v1 { - tonic::include_proto!("opentelemetry.proto.metrics.v1"); - } - } - - pub mod collector { - pub mod metrics { - pub mod v1 { - tonic::include_proto!("opentelemetry.proto.collector.metrics.v1"); - } - } - } -} - -/// Delta wire-format types for DDSketch + HLL, kept separate from the -/// main `tonic` tree because they live in `sketchlib.v1` (not -/// `opentelemetry.proto.*`). Vendored locally until the upstream -/// `asap_sketchlib` crate picks up the same proto definitions — see -/// `proto/sketchlib_delta/*.proto` for the file headers that explain -/// why the local copy exists. -pub mod sketchlib { - pub mod v1 { - tonic::include_proto!("sketchlib.v1"); - } -} - -/// Continuous distributed monitoring (CDM) edge↔coordinator reverse channel. -/// Generated from `proto/monitor/monitor.proto` with both server (coordinator, -/// in `data_plane`) and client (edge sim / tests) stubs. See the .proto header -/// for the protocol; the Go edge runtime vendors a byte-identical copy. -pub mod monitor { - pub mod v1 { - tonic::include_proto!("asap.monitor.v1"); - } -} diff --git a/crates/asap_otel_proto/tests/coupling_wire_compat.rs b/crates/asap_otel_proto/tests/coupling_wire_compat.rs deleted file mode 100644 index d585a34ba..000000000 --- a/crates/asap_otel_proto/tests/coupling_wire_compat.rs +++ /dev/null @@ -1,64 +0,0 @@ -//! Cross-language wire-compat gate for the dynamic coordinator<->sampling -//! coupling fields: bytes are emitted by the Go edge -//! (asap-precompute-go/monitor/grpcclient/monitorpb TestCouplingFixture) and -//! MUST decode field-for-field here via prost — proving SlackGrant.sample_p -//! (coord->edge) and MonitorReport.rate (edge->coord) cross the wire. -use asap_otel_proto::monitor::v1::{coord_to_edge, edge_to_coord, CoordToEdge, EdgeToCoord}; -use prost::Message; - -fn unhex(s: &str) -> Vec { - (0..s.len()) - .step_by(2) - .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) - .collect() -} - -#[test] -fn go_grant_sample_p_decodes_in_rust() { - let bytes = unhex("0a1d082a1807210000000000000c402880d095ffbc3131000000000000d03f"); - let env = CoordToEdge::decode(&bytes[..]).expect("decode CoordToEdge"); - match env.msg.expect("msg") { - coord_to_edge::Msg::Grant(g) => { - assert_eq!(g.agg_id, 42); - assert_eq!(g.round, 7); - assert!((g.sample_p - 0.25).abs() < 1e-12, "sample_p={}", g.sample_p); - } - other => panic!("expected Grant, got {:?}", other), - } -} - -#[test] -fn go_report_rate_decodes_in_rust() { - let bytes = unhex("12200a06656467652d37102a2900000000004a934030033809410000000000c09240"); - let env = EdgeToCoord::decode(&bytes[..]).expect("decode EdgeToCoord"); - match env.msg.expect("msg") { - edge_to_coord::Msg::Report(r) => { - assert_eq!(r.edge_id, "edge-7"); - assert_eq!(r.agg_id, 42); - assert!((r.rate - 1200.0).abs() < 1e-9, "rate={}", r.rate); - } - other => panic!("expected Report, got {:?}", other), - } -} - -#[test] -fn rust_grant_sample_p_emit_for_go() { - // The REAL coord->edge direction: Rust(prost) encodes a grant; Go decodes it. - use asap_otel_proto::monitor::v1::SlackGrant; - let env = CoordToEdge { - msg: Some(coord_to_edge::Msg::Grant(SlackGrant { - agg_id: 99, - round: 4, - local_slack: 1.0, - window_start_ms: 1_700_000_000_000, - sample_p: 0.3, - ..Default::default() - })), - }; - let mut buf = Vec::new(); - env.encode(&mut buf).unwrap(); - println!( - "RUST_GRANT_HEX={}", - buf.iter().map(|b| format!("{:02x}", b)).collect::() - ); -} diff --git a/crates/asap_otel_proto/tests/monitor_interop.rs b/crates/asap_otel_proto/tests/monitor_interop.rs deleted file mode 100644 index 19ca9eb39..000000000 --- a/crates/asap_otel_proto/tests/monitor_interop.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! Cross-language Phase-1 gate: decode the wire bytes produced by the Go edge -//! (asap-precompute-go/monitor/grpcclient/monitorpb interop_test.go) and assert -//! field-for-field equality. Proves the vendored Go copy and the canonical Rust -//! proto agree on field numbers / encoding. -use asap_otel_proto::monitor::v1::{edge_to_coord::Msg, EdgeToCoord}; -use prost::Message; - -#[test] -fn decodes_go_fixture() { - // FIXTURE_HEX emitted by the Go TestInteropFixture. - let hex = "122c0a06656467652d37102a1a0c7376633d636865636b6f75742080d095ffbc312900000000004a934030033809"; - let bytes: Vec = (0..hex.len()) - .step_by(2) - .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap()) - .collect(); - - let env = EdgeToCoord::decode(&*bytes).expect("decode"); - let report = match env.msg.expect("msg") { - Msg::Report(r) => r, - other => panic!("expected report, got {other:?}"), - }; - assert_eq!(report.edge_id, "edge-7"); - assert_eq!(report.agg_id, 42); - assert_eq!(report.key, b"svc=checkout"); - assert_eq!(report.window_start_ms, 1_700_000_000_000); - assert_eq!(report.local_value, 1234.5); - assert_eq!(report.round, 3); - assert_eq!(report.seq, 9); -} diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml deleted file mode 100644 index d5c83ef67..000000000 --- a/crates/asap_types/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "asap_types" -version.workspace = true -edition.workspace = true - -[dependencies] -promql-parser.workspace = true -base64 = "0.21" -tracing.workspace = true -serde.workspace = true -serde_json.workspace = true -serde_yaml.workspace = true -anyhow.workspace = true -thiserror.workspace = true -clap.workspace = true -xxhash-rust = { version = "0.8", features = ["xxh64"] } -sha2 = "0.10" - -# Use the same Planner IR as the control plane at compilation boundaries. -planner-types.workspace = true - -# Accuracy metadata projects the authoritative Planner guarantees. -asap-aware-mapping.workspace = true - -asap-physical-operators.workspace = true diff --git a/crates/asap_types/src/accumulator_spec.rs b/crates/asap_types/src/accumulator_spec.rs deleted file mode 100644 index bdf5949b4..000000000 --- a/crates/asap_types/src/accumulator_spec.rs +++ /dev/null @@ -1,820 +0,0 @@ -//! Validate stored materialization descriptors against Planner summary families. -//! -//! This projection supports catalog identity and imported state metadata. It is -//! not an execution program. Raw and maintenance execution dispatch directly -//! on the selected post-ASAP DAG payload; the descriptor must agree with it. - -use serde_json::Value; - -use crate::aggregation_config::PrecomputeMaterialization; -use crate::key_by_label_names::KeyByLabelNames; -use crate::AggregationType; -use planner_types::post_asap::{ - ExactKind, ExactParams, GroupingStrategy, HydraKind, HydraParams, SketchAlgorithm, SketchKind, - SketchParams, SummaryFamilyType, -}; - -/// Data_plane's typed replacement for -/// `(aggregation_type, aggregation_sub_type, parameters)`: which -/// accumulator to run (`kind`), with what tuning (`params`), and -/// whether it's keyed by a group-by label set (`grouping`). -/// -/// Computed on demand from an [`PrecomputeMaterialization`] via -/// [`PrecomputeMaterialization::accumulator_spec`] — not stored on the config -/// itself, so there is exactly one source of truth for the fields that -/// feed [`crate::policy_fingerprint::PolicyFingerprint`]. -#[derive(Debug, Clone, PartialEq)] -pub struct AccumulatorSpec { - /// Planner-owned committed summary identity. For sketches this is a - /// validated `SketchKind` (category + algorithm + params), following the - /// ASAP-aware-mapping vocabulary. - pub family: SummaryFamilyType, - /// Physical keyed-state layout, independent of semantic family. - pub grouping: Option, -} - -/// Typed rule for turning an input sample into an accumulator update. -/// -/// This is execution semantics, separate from the summary family: the same -/// CMS-with-heap state can count events, sum sample values, or sum reset-aware -/// counter deltas. Stored descriptors encode the rule in -/// `parameters`; callers use [`PrecomputeMaterialization::sample_update_rule`] so the -/// runtime does not branch on ad-hoc strings. -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum SampleUpdateRule { - Value { scale: f64 }, - Count, - CounterDelta { scale: f64 }, -} - -/// The implemented raw frequency domain counts occurrences of sample values. -pub fn is_unit_sample_frequency(update: &planner_types::post_asap::SummaryUpdate) -> bool { - use planner_types::post_asap::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; - matches!( - update.item, - Some(SummaryInputExpr::Column( - planner_types::pre_asap::ColumnRef::SampleValue - )) - ) && matches!(update.weight, SummaryInputExpr::Constant(1.0)) - && matches!( - update.weight_domain, - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount - } - ) -} - -/// Raw HLL hashes the scalar sample; it does not interpret it as a frequency weight. -pub fn is_scalar_sample_value(update: &planner_types::post_asap::SummaryUpdate) -> bool { - update.item.is_none() - && matches!( - update.weight, - planner_types::post_asap::SummaryInputExpr::Column( - planner_types::pre_asap::ColumnRef::SampleValue - ) - ) -} - -impl PrecomputeMaterialization { - pub fn sample_update_rule(&self) -> SampleUpdateRule { - let scale = self - .parameters - .get("weight_scale") - .and_then(Value::as_f64) - .filter(|scale| scale.is_finite() && *scale > 0.0) - .unwrap_or(1.0); - match self - .parameters - .get("weight_mode") - .or_else(|| self.parameters.get("topk_weight")) - .and_then(Value::as_str) - .map(str::to_ascii_lowercase) - .as_deref() - { - Some("count" | "frequency" | "freq") => SampleUpdateRule::Count, - Some("counter_delta") => SampleUpdateRule::CounterDelta { scale }, - _ => SampleUpdateRule::Value { scale }, - } - } -} - -/// A storage descriptor cannot be resolved to a supported Planner family. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AccumulatorSpecError { - /// `aggregation_type` was `SingleSubpopulation` with an - /// `aggregation_sub_type` string not in the recognized alias list. - UnknownSingleSubpopulationSubType(String), - /// `aggregation_type` was `MultipleSubpopulation` with an - /// unrecognized `aggregation_sub_type`. - UnknownMultipleSubpopulationSubType(String), - /// `aggregation_type` itself has no accumulator-dispatch mapping. - /// Also returned for an invalid HLL precision. A resolved family identifies - /// stored state; it does not imply raw-value updater support. - UnmappedAggregationType(AggregationType), -} - -impl std::fmt::Display for AccumulatorSpecError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::UnknownSingleSubpopulationSubType(s) => { - write!(f, "Unknown SingleSubpopulation sub_type '{s}'") - } - Self::UnknownMultipleSubpopulationSubType(s) => { - write!(f, "Unknown MultipleSubpopulation sub_type '{s}'") - } - Self::UnmappedAggregationType(t) => write!(f, "Unknown aggregation_type '{t:?}'"), - } - } -} - -impl std::error::Error for AccumulatorSpecError {} - -impl PrecomputeMaterialization { - /// Resolve this config's `(aggregation_type, aggregation_sub_type, - /// parameters)` triple into a typed [`AccumulatorSpec`]. - /// - /// Unsupported descriptors return an error; this projection never chooses - /// a fallback family and cannot authorize DAG execution. - pub fn accumulator_spec(&self) -> Result { - use AggregationType::*; - - let sub_type = self.aggregation_sub_type.as_str(); - - let independent_sketch = |algorithm, params| { - SummaryFamilyType::Sketch( - SketchKind::new(algorithm, params), - GroupingStrategy::PerSubpopulationInstance, - ) - }; - let (family, keyed) = match self.aggregation_type { - Sum | Count | Increase | Rate | Min | Max => ( - self.aggregation_type.planner_exact_family().unwrap(), - !self.aggregated_labels.is_empty(), - ), - DatasketchesKLL => ( - independent_sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: kll_k_param(self) as u32, - }, - ), - false, - ), - HydraKLL => { - let k = kll_k_param(self) as u32; - ( - SummaryFamilyType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k }), - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraKll, - params: HydraParams::HydraKll { - k, - shared_buckets: k, - }, - }, - ), - true, - ) - } - CountMinSketch => { - let (row_num, col_num) = cms_params(self); - ( - independent_sketch( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: col_num as u32, - depth: row_num as u32, - }, - ), - true, - ) - } - CountMinSketchWithHeap => { - let (row_num, col_num) = cms_params(self); - let heap_size = heap_size_param(self); - ( - independent_sketch( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: col_num as u32, - depth: row_num as u32, - heap_size: heap_size as u32, - }, - ), - true, - ) - } - CountSketch => { - let (row_num, col_num) = cms_params(self); - ( - independent_sketch( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { - width: col_num as u32, - depth: row_num as u32, - }, - ), - true, - ) - } - CountSketchWithHeap => { - let (row_num, col_num) = cms_params(self); - let heap_size = heap_size_param(self); - ( - independent_sketch( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: col_num as u32, - depth: row_num as u32, - heap_size: heap_size as u32, - }, - ), - true, - ) - } - DDSketch => ( - independent_sketch( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { - alpha: ddsketch_alpha_param(self), - }, - ), - false, - ), - UnivMon => { - let param = - |name: &str, default: u64, max: u64| -> Result { - let value = self - .parameters - .get(name) - .map_or(Some(default), |v| v.as_u64()) - .ok_or(AccumulatorSpecError::UnmappedAggregationType(UnivMon))?; - if value == 0 || value > max { - return Err(AccumulatorSpecError::UnmappedAggregationType(UnivMon)); - } - Ok(value as u32) - }; - ( - independent_sketch( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: param("heap_size", 32, u32::MAX as u64)?, - sketch_rows: param("sketch_rows", 5, 20)?, - sketch_cols: param("sketch_cols", 1024, u32::MAX as u64)?, - layers: param("layers", 4, 64)? as u8, - }, - ), - false, - ) - } - HLL => { - let precision = match self.parameters.get("precision") { - None => 14, - Some(value) => value - .as_u64() - .ok_or(AccumulatorSpecError::UnmappedAggregationType(HLL))?, - }; - if !(4..=18).contains(&precision) { - return Err(AccumulatorSpecError::UnmappedAggregationType(HLL)); - } - ( - independent_sketch( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: precision as u8, - }, - ), - false, - ) - } - SingleSubpopulation => match sub_type { - "Sum" | "sum" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - false, - ), - "Min" | "min" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - false, - ), - "Max" | "max" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - false, - ), - "Increase" | "increase" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - false, - ), - "DatasketchesKLL" | "datasketches_kll" | "KLL" | "kll" => ( - independent_sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: kll_k_param(self) as u32, - }, - ), - false, - ), - other => { - return Err(AccumulatorSpecError::UnknownSingleSubpopulationSubType( - other.to_string(), - )) - } - }, - MultipleSubpopulation => match sub_type { - "Sum" | "sum" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - true, - ), - "Min" | "min" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - true, - ), - "Max" | "max" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - true, - ), - "Increase" | "increase" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - true, - ), - "CountMinSketch" | "count_min_sketch" | "CMS" | "cms" => { - let (row_num, col_num) = cms_params(self); - ( - independent_sketch( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: col_num as u32, - depth: row_num as u32, - }, - ), - true, - ) - } - "HydraKLL" | "hydra_kll" => ( - independent_sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: kll_k_param(self) as u32, - }, - ), - true, - ), - other => { - return Err(AccumulatorSpecError::UnknownMultipleSubpopulationSubType( - other.to_string(), - )) - } - }, - }; - - let grouping = if keyed { - Some(self.grouping_labels.label_names()) - } else { - None - }; - - Ok(AccumulatorSpec { family, grouping }) - } -} - -// --------------------------------------------------------------------------- -// Raw-parameter extraction helpers. -// -// Relocated verbatim from `data_plane::precompute_engine::accumulator_factory` -// (same names, same behavior, same defaults) — this is now their one -// definition; `accumulator_factory.rs` re-exports them via `use` so its -// existing unit tests (`cms_params_reads_canonical_w_d_keys`, -// `test_kll_k_param_capital_k`, ...) keep passing unchanged. -// --------------------------------------------------------------------------- - -/// Extract the KLL `k` parameter. Capital `"K"` takes precedence over -/// lowercase `"k"` to match the convention used by the top-level -/// aggregation type arms. Defaults to 200. -pub fn kll_k_param(config: &PrecomputeMaterialization) -> u16 { - config - .parameters - .get("K") - .or_else(|| config.parameters.get("k")) - .and_then(|v| v.as_u64()) - .and_then(|v| u16::try_from(v).ok()) - .unwrap_or(200) -} - -/// Extract `(row_num, col_num)` for CMS / HydraKLL configs. -/// -/// Reads canonical `d` (depth = rows) / `w` (width = cols) keys — -/// matches what the control plane's `sketch_params_to_json` emits and -/// what `sketch_config_to_params` uses for OTLP policy_fp content -/// matching. Defaults to `(4, 1000)`. -pub fn cms_params(config: &PrecomputeMaterialization) -> (usize, usize) { - let row_num = config - .parameters - .get("d") - .and_then(|v| v.as_u64()) - .unwrap_or(4) as usize; - let col_num = config - .parameters - .get("w") - .and_then(|v| v.as_u64()) - .unwrap_or(1000) as usize; - (row_num, col_num) -} - -/// Top-k heap size for the `*WithHeap` configs. Reads `heap_size` / `k` -/// from `parameters`; defaults to 20 (the heap holds the top-k -/// candidates — it must be >= the largest `k` a query asks for). -pub fn heap_size_param(config: &PrecomputeMaterialization) -> usize { - config - .parameters - .get("heap_size") - .or_else(|| config.parameters.get("k")) - .or_else(|| config.parameters.get("K")) - .and_then(|v| v.as_u64()) - .map(|v| v as usize) - .filter(|&v| v > 0) - .unwrap_or(20) -} - -/// Pull `relativeAccuracy` (or canonical aliases) out of a -/// streaming-config aggregation entry. Defaults to 0.01 (1% rel-err, -/// the same default the agent's `ddsketchprocessor` uses). -pub fn ddsketch_alpha_param(config: &PrecomputeMaterialization) -> f64 { - let parsed = param_f64(config, "relativeAccuracy") - .or_else(|| param_f64(config, "relative_accuracy")) - .or_else(|| param_f64(config, "alpha")) - .unwrap_or(0.01); - if parsed > 0.0 && parsed < 1.0 { - parsed - } else { - tracing::warn!( - "DDSketch relativeAccuracy {} out of (0,1); using default 0.01", - parsed - ); - 0.01 - } -} - -fn param_f64(config: &PrecomputeMaterialization, key: &str) -> Option { - config.parameters.get(key).and_then(Value::as_f64) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::enums::WindowKind; - use crate::key_by_label_names::KeyByLabelNames; - use std::collections::HashMap; - - fn assert_exact(spec: &AccumulatorSpec, expected: ExactKind) { - assert!(matches!( - &spec.family, - SummaryFamilyType::ExactAggregate(kind, _) if kind == &expected - )); - } - - fn assert_sketch( - spec: &AccumulatorSpec, - expected_algorithm: planner_types::post_asap::SketchAlgorithm, - expected_params: planner_types::post_asap::SketchParams, - ) { - match &spec.family { - SummaryFamilyType::Sketch(kind, _) => { - assert_eq!(kind.algorithm(), &expected_algorithm); - assert_eq!(kind.params(), &expected_params); - } - other => panic!("expected sketch family, got {other:?}"), - } - } - - #[allow(clippy::too_many_arguments)] - fn make_config( - agg_type: AggregationType, - sub_type: &str, - params: HashMap, - grouping_labels: Vec<&str>, - ) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - agg_type, - sub_type.to_string(), - params, - KeyByLabelNames::new(grouping_labels.into_iter().map(|s| s.to_string()).collect()), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - "m".to_string(), - None, - None, - None, - ) - } - - // ---- direct (non-wrapper) variants ------------------------------ - - #[test] - fn sum_is_unkeyed_sum() { - let cfg = make_config(AggregationType::Sum, "", HashMap::new(), vec![]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_exact(&spec, ExactKind::Sum); - assert!(spec.grouping.is_none()); - } - - #[test] - fn keyed_layout_preserves_sum_family() { - let mut cfg = make_config(AggregationType::Sum, "", HashMap::new(), vec!["zone"]); - cfg.aggregated_labels = KeyByLabelNames::new(vec!["host".into()]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_exact(&spec, ExactKind::Sum); - assert_eq!( - spec.grouping, - Some(KeyByLabelNames::new(vec!["zone".to_string()])) - ); - } - - #[test] - fn datasketches_kll_reads_k_param() { - let mut params = HashMap::new(); - params.insert("k".to_string(), serde_json::json!(128)); - let cfg = make_config(AggregationType::DatasketchesKLL, "", params, vec![]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_sketch( - &spec, - planner_types::post_asap::SketchAlgorithm::Kll, - planner_types::post_asap::SketchParams::Kll { k: 128 }, - ); - assert!(spec.grouping.is_none()); - } - - #[test] - fn hydra_kll_is_keyed_kll() { - let mut params = HashMap::new(); - params.insert("k".to_string(), serde_json::json!(64)); - let cfg = make_config(AggregationType::HydraKLL, "", params, vec!["host"]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_sketch( - &spec, - planner_types::post_asap::SketchAlgorithm::Kll, - planner_types::post_asap::SketchParams::Kll { k: 64 }, - ); - assert!(spec.grouping.is_some()); - } - - #[test] - fn count_min_sketch_maps_to_cms_params() { - let mut params = HashMap::new(); - params.insert("d".to_string(), serde_json::json!(7)); - params.insert("w".to_string(), serde_json::json!(2048)); - let cfg = make_config(AggregationType::CountMinSketch, "", params, vec!["host"]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_sketch( - &spec, - planner_types::post_asap::SketchAlgorithm::Cms, - planner_types::post_asap::SketchParams::Cms { - width: 2048, - depth: 7, - }, - ); - assert!(spec.grouping.is_some()); - } - - #[test] - fn count_min_sketch_with_heap_maps_to_cms_with_heap_params() { - let mut params = HashMap::new(); - params.insert("d".to_string(), serde_json::json!(4)); - params.insert("w".to_string(), serde_json::json!(256)); - params.insert("heap_size".to_string(), serde_json::json!(8)); - let cfg = make_config( - AggregationType::CountMinSketchWithHeap, - "", - params, - vec!["host"], - ); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_sketch( - &spec, - planner_types::post_asap::SketchAlgorithm::CmsWithHeap, - planner_types::post_asap::SketchParams::CmsWithHeap { - width: 256, - depth: 4, - heap_size: 8, - }, - ); - } - - /// Documented existing quirk (see `accumulator_factory.rs`): bare - /// `CountSketch` gets its own `SketchAlgorithm` identity here, but - /// `accumulator_factory` routes it through the same - /// `CmsAccumulatorUpdater` as bare CMS — no dedicated heap-less - /// Count-Sketch accumulator exists. This test locks in the *identity* - /// resolution only; the shared-accumulator behavior is exercised in - /// `accumulator_factory.rs`'s own tests. - #[test] - fn count_sketch_gets_its_own_kind_identity() { - let cfg = make_config(AggregationType::CountSketch, "", HashMap::new(), vec!["h"]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert!(matches!( - &spec.family, - SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::CountSketch - )); - } - - #[test] - fn ddsketch_reads_relative_accuracy_alpha() { - let mut params = HashMap::new(); - params.insert("relativeAccuracy".to_string(), serde_json::json!(0.02)); - let cfg = make_config(AggregationType::DDSketch, "", params, vec![]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_sketch( - &spec, - planner_types::post_asap::SketchAlgorithm::DDSketch, - planner_types::post_asap::SketchParams::DDSketch { alpha: 0.02 }, - ); - assert!(spec.grouping.is_none()); - } - - #[test] - fn hll_has_catalog_identity_for_sketch_envelope_ingest() { - let cfg = make_config(AggregationType::HLL, "", HashMap::new(), vec![]); - let spec = cfg.accumulator_spec().unwrap(); - assert_sketch( - &spec, - SketchAlgorithm::Hll, - SketchParams::Hll { precision: 14 }, - ); - for precision in [ - serde_json::json!(3), - serde_json::json!(19), - serde_json::json!(4.5), - serde_json::json!("bad"), - ] { - let cfg = make_config( - AggregationType::HLL, - "", - HashMap::from([("precision".into(), precision)]), - vec![], - ); - assert!(cfg.accumulator_spec().is_err()); - } - } - - // ---- wrapper (Single/MultipleSubpopulation) variants ------------ - - #[test] - fn single_subpopulation_sum_alias() { - for alias in ["Sum", "sum"] { - let cfg = make_config( - AggregationType::SingleSubpopulation, - alias, - HashMap::new(), - vec![], - ); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_exact(&spec, ExactKind::Sum); - assert!(spec.grouping.is_none()); - } - } - - #[test] - fn multiple_subpopulation_cms_alias() { - for alias in ["CountMinSketch", "count_min_sketch", "CMS", "cms"] { - let cfg = make_config( - AggregationType::MultipleSubpopulation, - alias, - HashMap::new(), - vec!["host"], - ); - let spec = cfg.accumulator_spec().expect("resolves"); - assert!(matches!( - &spec.family, - SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::Cms - )); - assert!(spec.grouping.is_some()); - } - } - - #[test] - fn multiple_subpopulation_hydra_kll_alias() { - for alias in ["HydraKLL", "hydra_kll"] { - let cfg = make_config( - AggregationType::MultipleSubpopulation, - alias, - HashMap::new(), - vec!["host"], - ); - let spec = cfg.accumulator_spec().expect("resolves"); - assert!(matches!( - &spec.family, - SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::Kll - )); - assert!(spec.grouping.is_some()); - } - } - - #[test] - fn unknown_single_subpopulation_sub_type_errors() { - let cfg = make_config( - AggregationType::SingleSubpopulation, - "Bogus", - HashMap::new(), - vec![], - ); - let err = cfg.accumulator_spec().expect_err("unknown sub_type"); - assert_eq!( - err, - AccumulatorSpecError::UnknownSingleSubpopulationSubType("Bogus".to_string()) - ); - } - - #[test] - fn unknown_multiple_subpopulation_sub_type_errors() { - let cfg = make_config( - AggregationType::MultipleSubpopulation, - "Bogus", - HashMap::new(), - vec!["host"], - ); - let err = cfg.accumulator_spec().expect_err("unknown sub_type"); - assert_eq!( - err, - AccumulatorSpecError::UnknownMultipleSubpopulationSubType("Bogus".to_string()) - ); - } - - #[test] - fn error_display_matches_pre_step5_warning_text() { - assert_eq!( - AccumulatorSpecError::UnknownSingleSubpopulationSubType("Bogus".to_string()) - .to_string(), - "Unknown SingleSubpopulation sub_type 'Bogus'" - ); - assert_eq!( - AccumulatorSpecError::UnknownMultipleSubpopulationSubType("Bogus".to_string()) - .to_string(), - "Unknown MultipleSubpopulation sub_type 'Bogus'" - ); - assert_eq!( - AccumulatorSpecError::UnmappedAggregationType(AggregationType::HLL).to_string(), - "Unknown aggregation_type 'HLL'" - ); - } - - #[test] - fn sample_update_rule_decodes_legacy_parameters_once() { - let mut parameters = HashMap::new(); - parameters.insert("weight_mode".into(), Value::String("counter_delta".into())); - parameters.insert("weight_scale".into(), Value::from(1_000_000.0)); - let counter = make_config( - AggregationType::CountMinSketchWithHeap, - "", - parameters, - vec![], - ); - assert_eq!( - counter.sample_update_rule(), - SampleUpdateRule::CounterDelta { scale: 1_000_000.0 } - ); - - let mut count_parameters = HashMap::new(); - count_parameters.insert("topk_weight".into(), Value::String("frequency".into())); - let count = make_config( - AggregationType::CountSketchWithHeap, - "", - count_parameters, - vec![], - ); - assert_eq!(count.sample_update_rule(), SampleUpdateRule::Count); - } - - // ---- PolicyFingerprint stability guard --------------------------- - - /// `accumulator_spec()` must be a pure, additional *read* of - /// `PrecomputeMaterialization` — it must not change what - /// `PolicyFingerprint::from_config` hashes. This locks in a fixed - /// fingerprint for a fixed config as a tripwire: if this test ever - /// needs its expected constant updated, `policy_fingerprint.rs` - /// changed in a way that breaks on-disk sid compatibility, which is - /// exactly what Step 5 was required not to do. - #[test] - fn accumulator_spec_does_not_perturb_policy_fingerprint() { - let cfg = make_config(AggregationType::DDSketch, "", HashMap::new(), vec![]); - let fp_before = cfg.policy_fp_u64(); - let _ = cfg.accumulator_spec(); // takes &self — must not mutate `cfg` - let fp_after = cfg.policy_fp_u64(); - assert_eq!( - fp_before, fp_after, - "calling accumulator_spec() must not change the fingerprint \ - PolicyFingerprint::from_config computes from this config" - ); - - // An independently-built config with identical content must - // still agree — proves accumulator_spec() reads, never writes, - // the fields PolicyFingerprint::from_config hashes. - let cfg2 = make_config(AggregationType::DDSketch, "", HashMap::new(), vec![]); - assert_eq!(fp_after, cfg2.policy_fp_u64()); - } -} diff --git a/crates/asap_types/src/accuracy.rs b/crates/asap_types/src/accuracy.rs deleted file mode 100644 index 723fa5cd8..000000000 --- a/crates/asap_types/src/accuracy.rs +++ /dev/null @@ -1,123 +0,0 @@ -//! Accuracy wire metadata projected from ASAPPlanner guarantees. -//! Formula ownership stays in ASAPPlanner; unknown confidence remains explicit. - -use serde::{Deserialize, Serialize}; - -/// Interpretation of the reported epsilon; serialized identically across planners and serving. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AccuracyKind { - Uncalibrated, - Exact, - AdditiveFrequency, - RelativeCardinality, - RankQuantile, - RelativeQuantile, - TopK, -} - -impl AccuracyKind { - pub fn as_str(self) -> &'static str { - match self { - Self::Uncalibrated => "uncalibrated", - Self::Exact => "exact", - Self::AdditiveFrequency => "additive_frequency", - Self::RelativeCardinality => "relative_cardinality", - Self::RankQuantile => "rank_quantile", - Self::RelativeQuantile => "relative_quantile", - Self::TopK => "top_k", - } - } -} - -/// Accuracy metadata shared by planning and serving. -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -pub struct AccuracyProfile { - pub epsilon: f64, - /// Failure probability; None means the planner cannot establish it. - pub delta: Option, - pub kind: AccuracyKind, -} - -impl AccuracyProfile { - /// Exact (ε = δ = 0). - pub fn exact() -> Self { - Self { - epsilon: 0.0, - delta: Some(0.0), - kind: AccuracyKind::Exact, - } - } - - /// One-line summary for dashboards and logs. - pub fn summary(&self) -> String { - format!( - "accuracy: ε={}, δ={}, kind={}", - self.epsilon, - self.delta - .map(|value| value.to_string()) - .unwrap_or_else(|| "unknown".into()), - self.kind.as_str() - ) - } - - /// Project the four shared family contracts without reimplementing their - /// error formulas. Other families keep their backend-specific adapters. - pub fn from_sketch_params(params: &planner_types::post_asap::SketchParams) -> Option { - use asap_aware_mapping::accuracy::DefaultAccuracyModel; - use planner_types::post_asap::{SketchAlgorithm, SketchParams, SketchQuery}; - use planner_types::pre_asap::ColumnRef; - let (algorithm, query, kind) = match params { - SketchParams::Cms { .. } => ( - SketchAlgorithm::Cms, - SketchQuery::PointCount { - key: ColumnRef::SampleValue, - value: None, - }, - AccuracyKind::AdditiveFrequency, - ), - SketchParams::Hll { .. } => ( - SketchAlgorithm::Hll, - SketchQuery::Cardinality, - AccuracyKind::RelativeCardinality, - ), - SketchParams::Kll { .. } => ( - SketchAlgorithm::Kll, - SketchQuery::Quantile { q: 0.5 }, - AccuracyKind::RankQuantile, - ), - SketchParams::DDSketch { .. } => ( - SketchAlgorithm::DDSketch, - SketchQuery::Quantile { q: 0.5 }, - AccuracyKind::RelativeQuantile, - ), - _ => return None, - }; - let guarantee = DefaultAccuracyModel::sketch_guarantee(&algorithm, params, &query)?; - Some(Self { - epsilon: guarantee.bound.evaluate()?, - delta: guarantee.failure_probability.evaluate(), - kind, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn wire_contract_round_trips_with_stable_kind_name() { - let profile = AccuracyProfile { - epsilon: 0.01, - delta: Some(0.001), - kind: AccuracyKind::AdditiveFrequency, - }; - let json = serde_json::to_string(&profile).unwrap(); - assert_eq!( - serde_json::from_str::(&json).unwrap(), - profile - ); - assert!(json.contains("additive_frequency")); - } -} diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs deleted file mode 100644 index f256d74fd..000000000 --- a/crates/asap_types/src/aggregation_config.rs +++ /dev/null @@ -1,792 +0,0 @@ -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use serde_yaml; -use std::collections::HashMap; - -use crate::enums::{QueryLanguage, WindowKind}; -use crate::policy_fingerprint::PolicyFingerprint; -use crate::utils::normalize_spatial_filter; -use crate::AggregationType; -use crate::KeyByLabelNames; - -/// Physical maintenance layout for one semantic windowed summary. -/// -/// `window_size` and `slide_interval` on [`PrecomputeMaterialization`] retain -/// the query's window and evaluation cadence. This enum describes how that -/// semantic window is represented in storage; it must never be inferred by -/// overloading either semantic duration. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum WindowMaterializationLayout { - /// Store disjoint mergeable states and compose a query window at read time. - Pane { pane_secs: u64 }, - /// Maintain one complete state for every evaluation point. - FullWindow, - /// Store base panes plus coarser mergeable rollups. Every level is a - /// duration in seconds and is an integer multiple of its predecessor. - HierarchicalRollup { - base_pane_secs: u64, - levels_secs: Vec, - }, -} - -impl WindowMaterializationLayout { - pub fn base_pane_secs(&self) -> u64 { - match self { - Self::Pane { pane_secs } => *pane_secs, - Self::FullWindow => 0, - Self::HierarchicalRollup { base_pane_secs, .. } => *base_pane_secs, - } - } - - pub fn validate(&self, window_secs: u64, slide_secs: u64) -> Result<(), String> { - if window_secs == 0 || slide_secs == 0 { - return Err("window and slide must be positive".into()); - } - match self { - Self::Pane { pane_secs } => { - if *pane_secs == 0 - || !window_secs.is_multiple_of(*pane_secs) - || !slide_secs.is_multiple_of(*pane_secs) - { - return Err("pane size must divide both window size and slide".into()); - } - } - Self::FullWindow => {} - Self::HierarchicalRollup { - base_pane_secs, - levels_secs, - } => { - if *base_pane_secs == 0 - || !window_secs.is_multiple_of(*base_pane_secs) - || !slide_secs.is_multiple_of(*base_pane_secs) - || levels_secs.is_empty() - { - return Err( - "rollup base pane must divide window and slide, with at least one level" - .into(), - ); - } - let mut previous = *base_pane_secs; - for level in levels_secs { - if *level <= previous - || *level % previous != 0 - || !window_secs.is_multiple_of(*level) - { - return Err( - "rollup levels must increase by integral factors and divide the window" - .into(), - ); - } - previous = *level; - } - } - } - Ok(()) - } -} - -/// Physical materialization metadata with content-derived identity. -/// This descriptor cannot authorize execution: the enclosing PrecomputePlan -/// must bind it to a compatible Planner DAG producer. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PrecomputeMaterialization { - /// Explicit deployment output allocation; independent of semantic identity. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stored_output_id: Option, - /// Planner-selected dependency closure ending at the persisted output. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub semantic_fragment: Option, - pub aggregation_type: AggregationType, - pub aggregation_sub_type: String, - pub parameters: HashMap, - #[serde(serialize_with = "crate::grouping_projection::serialize_config_grouping")] - pub grouping_labels: crate::GroupingProjection, - #[serde( - default, - skip_serializing_if = "crate::grouping_projection::PopulationKeyEncoding::is_legacy" - )] - pub population_key_encoding: crate::grouping_projection::PopulationKeyEncoding, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub partitioning: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub derived_input: Option, - pub aggregated_labels: KeyByLabelNames, - pub rollup_labels: KeyByLabelNames, - pub original_yaml: String, - - pub window_size: u64, // Window size in seconds (e.g., 900s for 15m) - pub slide_interval: u64, // Slide/hop interval in seconds (e.g., 30s) - pub window_type: WindowKind, // Tumbling or Sliding - pub window_layout: WindowMaterializationLayout, - /// Unix millisecond timestamp on the pane-boundary grid selected from - /// the consuming query workload. An absent origin cannot authorize certified - /// pane-only reads. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub pane_origin_ms: Option, - - pub spatial_filter: String, - pub spatial_filter_normalized: String, - pub metric: String, // PromQL mode: metric name; SQL mode: derived from table_name.value_column - pub num_aggregates_to_retain: Option, - - // SQL-specific fields (optional, used when query_language=sql) - pub table_name: Option, // SQL mode: table name - #[serde(default)] - pub value_projection: Option, - /// Table timestamp projection, in Unix milliseconds. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub table_timestamp_column: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub table_population: Option, - /// Producer typing for a SQL table value projection: the source column's - /// declared type and nullability. - /// - /// **Why this is separate from [`Self::value_projection`]**: the - /// projection is the materialization's *identity* — which column or - /// constant is summarised, and part of the policy fingerprint. This is how - /// the ingest path must *read* that column, which the identity does not - /// determine: an integer column needs an exactness guard on its way into - /// f64 summary state, and a nullable column needs SQL's "aggregates skip - /// NULL" rule reproduced at the reader rather than a decode failure on the - /// first NULL row. Two materializations over the same column are the same - /// policy either way, so this deliberately stays out of the fingerprint. - /// - /// `None` ⇒ PromQL-mode materializations and legacy SQL definitions, which - /// keep the pre-typed behaviour (read the column as it comes). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub value_source_column: Option, -} - -/// Policy-match handles for both the key and value dimensions of a -/// query. For single-population queries, key and value share the same -/// fingerprint and type. For multi-population queries (e.g. Topk), they -/// differ. -/// -/// **PR 5**: the `aggregation_id_for_*: u64` fields now carry the -/// `PolicyFingerprint::as_u64()` form of the matched config, NOT a -/// controller-allocated id. Callers that need typed identity can call -/// [`AggregationIdInfo::policy_fp_for_key`] / -/// [`AggregationIdInfo::policy_fp_for_value`]. -#[derive(Debug, Clone)] -pub struct AggregationIdInfo { - /// `PolicyFingerprint::as_u64()` of the key aggregation's config. - pub key_policy_fingerprint: u64, - /// `PolicyFingerprint::as_u64()` of the value aggregation's config. - pub value_policy_fingerprint: u64, - pub aggregation_type_for_key: AggregationType, - pub aggregation_type_for_value: AggregationType, -} - -impl AggregationIdInfo {} - -impl PrecomputeMaterialization { - pub fn effective_value_projection(&self) -> &crate::sds::ValueProjectionIdentity { - self.value_projection - .as_ref() - .unwrap_or(&crate::sds::ValueProjectionIdentity::SampleValue) - } - - /// Temporal extent of one stored base state, independent of emission cadence. - pub fn stored_window_ms(&self) -> u64 { - match &self.window_layout { - WindowMaterializationLayout::FullWindow => self.window_size, - layout => layout.base_pane_secs(), - } - .saturating_mul(1_000) - } - - pub fn source_identity(&self) -> crate::sds::DataSourceIdentity { - use crate::sds::DataSourceIdentity; - if let Some(input) = &self.derived_input { - DataSourceIdentity::Derived { - input: input.clone(), - } - } else if let Some(table_ref) = &self.table_name { - DataSourceIdentity::Table { - table_ref: table_ref.clone(), - } - } else { - DataSourceIdentity::TimeSeries { - metric: self.metric.clone(), - } - } - } - - pub fn population_filter_canonical(&self) -> Result { - if let Some(input) = &self.derived_input { - input.validate()?; - if self.table_name.is_some() - || self.table_population.is_some() - || self.table_timestamp_column.is_some() - || !self.spatial_filter.is_empty() - { - return Err("derived inputs cannot also declare a raw source/filter".into()); - } - } - self.effective_value_projection().validate()?; - if self.value_projection.is_some() - && self.table_name.is_none() - && self.derived_input.is_none() - { - return Err("explicit table value projection requires a table source".into()); - } - if let Some(column) = &self.table_timestamp_column { - if self.table_name.is_none() || column.is_empty() { - return Err("table timestamp projection requires a table and a column".into()); - } - crate::table_population::validate_column_name(column)?; - } - if self.table_name.is_some() && !self.spatial_filter.is_empty() { - return Err("table populations cannot use a PromQL label filter".into()); - } - if let Some(population) = &self.table_population { - if self.table_name.is_none() || !self.spatial_filter.is_empty() { - return Err( - "typed table population requires a table and no PromQL label filter".into(), - ); - } - population.validate()?; - Ok(population.canonical()) - } else { - Ok(normalize_spatial_filter(&self.spatial_filter)) - } - } - - #[allow(clippy::too_many_arguments)] - pub fn new( - aggregation_type: AggregationType, - aggregation_sub_type: String, - parameters: HashMap, - grouping_labels: impl Into, - aggregated_labels: KeyByLabelNames, - rollup_labels: KeyByLabelNames, - original_yaml: String, - window_size: u64, - slide_interval: u64, - window_type: WindowKind, - spatial_filter: String, - metric: String, - num_aggregates_to_retain: Option, - // SQL-specific fields - table_name: Option, - value_column: Option, - ) -> Self { - // Generate normalized spatial filter (placeholder implementation) - let spatial_filter_normalized = normalize_spatial_filter(&spatial_filter); - - Self { - stored_output_id: None, - semantic_fragment: None, - aggregation_type, - aggregation_sub_type, - parameters, - grouping_labels: grouping_labels.into(), - population_key_encoding: Default::default(), - partitioning: None, - derived_input: None, - aggregated_labels, - rollup_labels, - original_yaml, - window_size, - slide_interval, - window_type, - window_layout: WindowMaterializationLayout::Pane { - pane_secs: if slide_interval == 0 { - window_size - } else { - slide_interval - }, - }, - pane_origin_ms: None, - spatial_filter, - spatial_filter_normalized, - metric, - num_aggregates_to_retain, - table_name, - value_projection: value_column - .map(|name| crate::sds::ValueProjectionIdentity::Column { name }), - table_population: None, - table_timestamp_column: None, - value_source_column: None, - } - } - - /// Content-addressed identity for this config. Sugar over - /// [`PolicyFingerprint::from_config`]. - pub fn policy_fingerprint(&self) -> PolicyFingerprint { - PolicyFingerprint::from_config(self) - } - - /// `PolicyFingerprint::as_u64()` — the u64-form handle used by the - /// policy-fingerprint-keyed call sites (e.g. `InstalledPrecomputePlan`'s - /// `HashMap` keys). **Always** equal to - /// `self.policy_fingerprint().as_u64()`. The value is content- - /// addressed identity, NOT a controller-allocated counter id. - pub fn policy_fp_u64(&self) -> u64 { - self.policy_fingerprint().as_u64() - } - - pub fn from_yaml_data( - aggregation_data: &serde_yaml::Value, - num_aggregates_to_retain: Option, - query_language: QueryLanguage, - ) -> Result { - // `aggregationId` is silently dropped — identity is - // content-addressed via PolicyFingerprint (PR 5). Pre-PR-5 - // fixtures that still spell out the field parse cleanly. - - let labels = &aggregation_data["labels"]; - let grouping_labels: crate::GroupingProjection = - serde_yaml::from_value(labels["grouping"].clone())?; - let aggregated_labels = KeyByLabelNames::new( - labels["aggregated"] - .as_sequence() - .ok_or_else(|| anyhow::anyhow!("Missing aggregated labels"))? - .iter() - .filter_map(|v| v.as_str()) - .map(|s| s.to_string()) - .collect(), - ); - let rollup_labels = KeyByLabelNames::new( - labels["rollup"] - .as_sequence() - .ok_or_else(|| anyhow::anyhow!("Missing rollup labels"))? - .iter() - .filter_map(|v| v.as_str()) - .map(|s| s.to_string()) - .collect(), - ); - - let aggregation_type: AggregationType = aggregation_data["aggregationType"] - .as_str() - .ok_or_else(|| anyhow::anyhow!("Missing aggregationType"))? - .parse() - .map_err(|e: String| anyhow::anyhow!(e))?; - - let aggregation_sub_type = aggregation_data["aggregationSubType"] - .as_str() - .ok_or_else(|| anyhow::anyhow!("Missing aggregationSubType"))? - .to_string(); - - // Convert serde_yaml::Value to serde_json::Value for parameters - let parameters: HashMap = aggregation_data["parameters"] - .as_mapping() - .ok_or_else(|| anyhow::anyhow!("Missing parameters"))? - .iter() - .map(|(k, v)| { - let key = k.as_str().unwrap_or("").to_string(); - let value = serde_json::to_value(v).unwrap_or(Value::Null); - (key, value) - }) - .collect(); - - let window_size = aggregation_data["windowSize"] - .as_u64() - .ok_or_else(|| anyhow::anyhow!("Missing windowSize"))?; - - let window_type = aggregation_data - .get("windowType") - .and_then(|v| v.as_str()) - .unwrap_or("tumbling") - .parse::() - .unwrap_or_default(); - - let slide_interval = aggregation_data - .get("slideInterval") - .and_then(|v| v.as_u64()) - .unwrap_or(window_size); - - let pane_origin_ms = aggregation_data - .get("paneOriginMs") - .or_else(|| aggregation_data.get("pane_origin_ms")) - .and_then(|v| v.as_i64()); - - let spatial_filter = aggregation_data["spatialFilter"] - .as_str() - .unwrap_or("") - .to_string(); - - if [ - "valueColumn", - "value_column", - "valueProjection", - "value_projection", - ] - .iter() - .filter(|key| { - aggregation_data - .get(**key) - .is_some_and(|value| !value.is_null()) - }) - .count() - > 1 - { - return Err(anyhow::anyhow!( - "multiple value projection fields are not allowed" - )); - } - let typed_projection: Option = aggregation_data - .get("valueProjection") - .or_else(|| aggregation_data.get("value_projection")) - .filter(|value| !value.is_null()) - .map(|value| serde_json::to_value(value).and_then(serde_json::from_value)) - .transpose()?; - let (metric, table_name, value_column) = match query_language { - QueryLanguage::PromQl | QueryLanguage::MetricsQl => { - let metric = aggregation_data["metric"] - .as_str() - .ok_or_else(|| { - anyhow::anyhow!("Missing metric for time-series query language") - })? - .to_string(); - (metric, None, None) - } - QueryLanguage::ClickHouseSql => { - let table = aggregation_data["tableName"] - .as_str() - .ok_or_else(|| anyhow::anyhow!("Missing tableName for ClickHouse SQL"))? - .to_string(); - let column = aggregation_data["valueColumn"] - .as_str() - .or_else(|| { - typed_projection - .as_ref() - .and_then(|projection| projection.column()) - }) - .map(str::to_owned); - if column.is_none() && typed_projection.is_none() { - return Err(anyhow::anyhow!( - "Missing value projection for ClickHouse SQL" - )); - } - ( - format!("{table}.{}", column.as_deref().unwrap_or("constant")), - Some(table), - column, - ) - } - }; - - let mut config = Self::new( - aggregation_type, - aggregation_sub_type, - parameters, - grouping_labels, - aggregated_labels, - rollup_labels, - String::new(), // original_yaml - empty as in Python - window_size, - slide_interval, - window_type, - spatial_filter, - metric, - num_aggregates_to_retain, - table_name, - value_column, - ); - if aggregation_data.get("windowLayout").is_some() - && aggregation_data.get("window_layout").is_some() - { - anyhow::bail!("multiple window layout fields are not allowed"); - } - if let Some(layout) = aggregation_data - .get("windowLayout") - .or_else(|| aggregation_data.get("window_layout")) - { - config.window_layout = serde_yaml::from_value(layout.clone())?; - config - .window_layout - .validate(config.window_size, config.slide_interval) - .map_err(anyhow::Error::msg)?; - } - config.population_key_encoding = aggregation_data - .get("population_key_encoding") - .map(|value| serde_yaml::from_value(value.clone())) - .transpose()? - .unwrap_or_default(); - config.derived_input = aggregation_data - .get("derived_input") - .filter(|v| !v.is_null()) - .map(|v| serde_yaml::from_value(v.clone())) - .transpose()?; - config.partitioning = aggregation_data - .get("partitioning") - .filter(|value| !value.is_null()) - .map(|value| serde_yaml::from_value(value.clone())) - .transpose()?; - if let Some(projection) = typed_projection { - config.value_projection = Some(projection); - } - config.pane_origin_ms = pane_origin_ms; - config.table_timestamp_column = aggregation_data - .get("tableTimestampColumn") - .or_else(|| aggregation_data.get("table_timestamp_column")) - .and_then(serde_yaml::Value::as_str) - .map(str::to_owned); - config.table_population = aggregation_data - .get("tablePopulation") - .or_else(|| aggregation_data.get("table_population")) - .filter(|value| !value.is_null()) - .cloned() - .map(serde_yaml::from_value) - .transpose()?; - config - .population_filter_canonical() - .map_err(anyhow::Error::msg)?; - Ok(config) - } -} - -#[cfg(test)] -mod window_layout_tests { - use super::WindowMaterializationLayout; - - #[test] - fn validates_multiple_slides_and_rejects_uncomposable_panes() { - for slide in [5, 10, 15, 30] { - WindowMaterializationLayout::Pane { pane_secs: 5 } - .validate(60, slide) - .unwrap(); - WindowMaterializationLayout::FullWindow - .validate(60, slide) - .unwrap(); - } - assert!(WindowMaterializationLayout::Pane { pane_secs: 7 } - .validate(60, 10) - .is_err()); - assert!(WindowMaterializationLayout::HierarchicalRollup { - base_pane_secs: 5, - levels_secs: vec![10, 30], - } - .validate(60, 10) - .is_ok()); - assert!(WindowMaterializationLayout::HierarchicalRollup { - base_pane_secs: 5, - levels_secs: vec![12], - } - .validate(60, 10) - .is_err()); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn sample_yaml(with_id: bool) -> serde_yaml::Value { - let id_line = if with_id { "aggregationId: 42\n" } else { "" }; - let yaml = format!( - "{id_line}aggregationType: DDSketch\naggregationSubType: ''\nmetric: http_latency_ms\nlabels:\n grouping: [zone]\n rollup: []\n aggregated: []\nparameters:\n relative_accuracy: 0.01\nwindowSize: 30\nwindowType: tumbling\nspatialFilter: ''\n", - id_line = id_line - ); - serde_yaml::from_str(&yaml).expect("yaml parses") - } - - /// PR 5: `aggregationId` in the YAML is silently dropped — identity - /// is derived from content. A fixture with the field parses to the - /// SAME config as a fixture without it. - #[test] - fn explicit_aggregation_id_in_yaml_is_ignored() { - let with = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(true), - None, - QueryLanguage::PromQl, - ) - .expect("parse ok"); - let without = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse ok"); - assert_eq!( - with.policy_fingerprint(), - without.policy_fingerprint(), - "explicit aggregationId in the YAML must not change the policy fingerprint", - ); - } - - /// Round-tripping the same content yields the same fingerprint. - #[test] - fn fingerprint_is_deterministic_per_content() { - let a = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse a"); - let b = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse b"); - assert_eq!(a.policy_fingerprint(), b.policy_fingerprint()); - assert_ne!( - a.policy_fingerprint().as_u64(), - 0, - "fingerprint is never the 0 sentinel for a real config", - ); - } - - #[test] - fn explicit_window_layout_survives_custom_json_and_yaml_transport() { - use super::WindowMaterializationLayout; - let mut yaml = sample_yaml(false); - yaml["windowSize"] = serde_yaml::to_value(60).unwrap(); - yaml["slideInterval"] = serde_yaml::to_value(10).unwrap(); - for layout in [ - WindowMaterializationLayout::FullWindow, - WindowMaterializationLayout::Pane { pane_secs: 5 }, - WindowMaterializationLayout::HierarchicalRollup { - base_pane_secs: 5, - levels_secs: vec![10, 30], - }, - ] { - yaml["windowLayout"] = serde_yaml::to_value(&layout).unwrap(); - let config = - PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl) - .unwrap(); - assert_eq!(config.window_layout, layout); - let mut wire = serde_json::to_value(&config).unwrap(); - let decoded: PrecomputeMaterialization = serde_json::from_value(wire.clone()).unwrap(); - assert_eq!(decoded.window_layout, layout); - assert_eq!(decoded.stored_window_ms(), config.stored_window_ms()); - assert_eq!(decoded.policy_fingerprint(), config.policy_fingerprint()); - wire["windowLayout"] = wire["window_layout"].clone(); - assert!(serde_json::from_value::(wire).is_err()); - } - yaml.as_mapping_mut() - .unwrap() - .remove(serde_yaml::Value::from("windowLayout")); - let legacy = - PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl).unwrap(); - assert_eq!( - legacy.window_layout, - WindowMaterializationLayout::Pane { pane_secs: 10 } - ); - yaml["window_layout"] = serde_yaml::from_str("{kind: pane, pane_secs: 7}").unwrap(); - assert!( - PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl).is_err() - ); - } - - #[test] - fn pane_origin_round_trips_and_changes_definition_identity() { - let mut epoch = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse"); - let unknown = epoch.policy_fingerprint(); - epoch.pane_origin_ms = Some(7_000); - let planned = epoch.policy_fingerprint(); - assert_ne!(unknown, planned); - - let wire = serde_json::to_value(&epoch).unwrap(); - assert_eq!(wire["pane_origin_ms"], serde_json::json!(7_000)); - let mut derived = serde_json::to_value(&epoch).unwrap(); - let origin = derived - .as_object_mut() - .unwrap() - .remove("pane_origin_ms") - .unwrap(); - derived - .as_object_mut() - .unwrap() - .insert("paneOriginMs".into(), origin); - assert!(serde_json::from_value::(derived).is_err()); - let decoded: PrecomputeMaterialization = - serde_json::from_value(serde_json::to_value(&epoch).unwrap()).unwrap(); - assert_eq!(decoded.pane_origin_ms, Some(7_000)); - } - - /// The `policy_fp_u64()` accessor is exactly the fingerprint u64. - #[test] - fn policy_fp_u64_accessor_equals_fingerprint_u64() { - let cfg = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse"); - assert_eq!(cfg.policy_fp_u64(), cfg.policy_fingerprint().as_u64()); - } - - /// Installed materializations reject the removed externally assigned identity. - #[test] - fn canonical_wire_rejects_aggregation_id() { - let cfg = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse"); - let mut json = serde_json::to_value(&cfg).unwrap(); - assert!( - json.get("aggregationId").is_none(), - "aggregationId must not appear in the canonical wire format" - ); - json["aggregationId"] = serde_json::json!(42); - assert!(serde_json::from_value::(json).is_err()); - } - - #[test] - fn typed_projection_roundtrips_and_untyped_column_is_rejected() { - use crate::sds::ValueProjectionIdentity; - use planner_types::pre_asap::ScalarValue; - let mut config = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .unwrap(); - config.table_name = Some("telemetry".into()); - config.value_projection = Some(ValueProjectionIdentity::Column { - name: "value".into(), - }); - let column_identity = config.policy_fingerprint(); - let mut legacy = serde_json::to_value(&config).unwrap(); - legacy.as_object_mut().unwrap().remove("value_projection"); - legacy["value_column"] = serde_json::json!("value"); - assert!(serde_json::from_value::(legacy).is_err()); - let mut untyped = serde_json::to_value(&config).unwrap(); - untyped["value_projection"] = serde_json::json!("value"); - assert!(serde_json::from_value::(untyped).is_err()); - config.value_projection = Some(ValueProjectionIdentity::Constant { - value: ScalarValue::Int64(1), - }); - let wire = serde_json::to_value(&config).unwrap(); - let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); - assert_eq!( - decoded.effective_value_projection(), - config.effective_value_projection() - ); - let mut yaml = sample_yaml(false); - yaml["tableName"] = serde_yaml::to_value("telemetry").unwrap(); - yaml["valueProjection"] = - serde_yaml::to_value(serde_json::to_value(&config.value_projection).unwrap()).unwrap(); - let decoded = - PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::ClickHouseSql) - .unwrap(); - assert_eq!( - decoded.effective_value_projection(), - config.effective_value_projection() - ); - yaml["valueColumn"] = serde_yaml::to_value("other_column").unwrap(); - assert!(PrecomputeMaterialization::from_yaml_data( - &yaml, - None, - QueryLanguage::ClickHouseSql - ) - .is_err()); - assert_ne!(config.policy_fingerprint(), column_identity); - config.value_projection = Some(ValueProjectionIdentity::Constant { - value: ScalarValue::Float64(f64::NAN), - }); - assert!(config.population_filter_canonical().is_err()); - } -} diff --git a/crates/asap_types/src/aggregation_type.rs b/crates/asap_types/src/aggregation_type.rs deleted file mode 100644 index 7ff2ce7ea..000000000 --- a/crates/asap_types/src/aggregation_type.rs +++ /dev/null @@ -1,2 +0,0 @@ -//! Kernel identity is owned by the shared physical operator library. -pub use asap_physical_operators::AggregationType; diff --git a/crates/asap_types/src/derived_input.rs b/crates/asap_types/src/derived_input.rs deleted file mode 100644 index e7aba6906..000000000 --- a/crates/asap_types/src/derived_input.rs +++ /dev/null @@ -1,386 +0,0 @@ -//! Content identity for a maintenance sub-DAG cut at existing summary inputs. -//! The executable program remains in OwnedPostAsapDag; this is its catalog key. -use std::collections::{BTreeMap, BTreeSet}; - -use planner_types::post_asap::PostAsapNodeId; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; - -use crate::{executable_plan::OwnedPostAsapDag, sds::StoredOutputId}; - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct DerivedInputIdentity { - pub inputs: BTreeSet, - pub program_sha256: String, -} - -impl DerivedInputIdentity { - pub fn validate(&self) -> Result<(), String> { - if self.inputs.is_empty() - || self.inputs.iter().any(|id| id.fingerprint().is_unset()) - || self.program_sha256.len() != 64 - || !self - .program_sha256 - .bytes() - .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) - { - return Err("derived input requires summary references and a canonical SHA-256".into()); - } - Ok(()) - } - - /// Hash semantic nodes and edge roles, replacing input frontiers with stable - /// catalog IDs. Query IDs, node numbering, and catalog generations are absent. - pub fn from_dag( - document: &OwnedPostAsapDag, - root: PostAsapNodeId, - frontiers: &BTreeMap, - ) -> Result { - if ![ - crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION, - crate::executable_plan::PRECOMPUTE_DAG_SCHEMA_VERSION, - ] - .contains(&document.schema_version) - { - return Err("unsupported derived program document version".into()); - } - let decoded = document.decode()?; - let literals: BTreeSet<_> = decoded - .nodes - .iter() - .filter_map(|node| { - matches!( - &node.payload, - planner_types::post_asap::ExecutableOperatorPayload::Fallback { - expression: planner_types::pre_asap::QueryExpr::Literal(_), - } - ) - .then_some(node.id) - }) - .collect(); - let mut incoming: BTreeMap<_, Vec<_>> = BTreeMap::new(); - for edge in &document.edges { - incoming.entry(edge.consumer).or_default().push(edge); - } - let nodes: BTreeMap<_, _> = document.nodes.iter().map(|n| (n.id, n)).collect(); - if frontiers.contains_key(&root) { - return Err("derived program requires a non-frontier root".into()); - } - let mut hashes = BTreeMap::new(); - let mut visiting = BTreeSet::new(); - let mut inputs = BTreeSet::new(); - let mut stack = vec![(root, false)]; - while let Some((id, finish)) = stack.pop() { - if hashes.contains_key(&id) { - continue; - } - let node = nodes - .get(&id) - .ok_or("derived program references missing node")?; - if let Some(summary) = frontiers.get(&id) { - inputs.insert(*summary); - hashes.insert(id, serde_json::json!({"summary": summary})); - continue; - } - if !finish { - if !visiting.insert(id) { - return Err("derived program has a cycle".into()); - } - stack.push((id, true)); - if !incoming.contains_key(&id) && !literals.contains(&id) { - return Err("derived program has an unbound input leaf".into()); - } - for edge in incoming.get(&id).into_iter().flatten() { - stack.push((edge.producer, false)); - } - continue; - } - let mut edges = Vec::new(); - for edge in incoming.get(&id).into_iter().flatten() { - let input = hashes - .get(&edge.producer) - .ok_or("derived input was not evaluated")?; - edges.push( - serde_json::to_vec(&serde_json::json!({ - "input": input, "role": edge.role, "schema": edge.intermediate_schema, - "state": edge.data_state, "grouping": edge.grouping, "window": edge.window, - })) - .map_err(|e| e.to_string())?, - ); - } - edges.sort(); - let bytes = serde_json::to_vec(&serde_json::json!({ - "version": 2, "payload": node.payload, - "state": node.output_state, "schema": node.output_schema, - "guarantee": node.guarantee, "inputs": edges, - })) - .map_err(|e| e.to_string())?; - hashes.insert( - id, - serde_json::json!(format!("{:x}", Sha256::digest(bytes))), - ); - visiting.remove(&id); - } - let identity = Self { - inputs, - program_sha256: hashes[&root] - .as_str() - .ok_or("derived root has no semantic hash")? - .into(), - }; - identity.validate()?; - Ok(identity) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::summary_catalog::SummaryCatalog; - use crate::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - - fn config() -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 10, - 10, - WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, - ) - } - - #[test] - fn derived_source_is_distinct_and_generation_independent() { - let raw = config(); - let raw_id = StoredOutputId::from(raw.policy_fingerprint()); - let mut derived = raw.clone(); - derived.derived_input = Some(DerivedInputIdentity { - inputs: BTreeSet::from([raw_id]), - program_sha256: "a".repeat(64), - }); - assert_ne!(raw.policy_fingerprint(), derived.policy_fingerprint()); - let a = - SummaryCatalog::from_materializations(1, 1, &[raw.clone(), derived.clone()]).unwrap(); - let b = SummaryCatalog::from_materializations(2, 9, &[raw, derived.clone()]).unwrap(); - assert_eq!(a.outputs, b.outputs); - assert_eq!(a.data_descriptors, b.data_descriptors); - let mut renamed = derived.clone(); - renamed.metric = "output_alias".into(); - assert_eq!(renamed.policy_fingerprint(), derived.policy_fingerprint()); - let json = serde_json::to_value(&derived).unwrap(); - let decoded: PrecomputeMaterialization = serde_json::from_value(json).unwrap(); - assert_eq!(decoded.policy_fingerprint(), derived.policy_fingerprint()); - } - - #[test] - fn raw_utf8_metric_cannot_impersonate_derived_policy_domain() { - let mut derived = config(); - derived.derived_input = Some(DerivedInputIdentity { - inputs: BTreeSet::from([StoredOutputId::from(derived.policy_fingerprint())]), - program_sha256: "d".repeat(64), - }); - let mut raw = derived.clone(); - raw.metric = format!( - "derived-input-v1:{}", - serde_json::to_string(&derived.source_identity()).unwrap() - ); - raw.derived_input = None; - assert_ne!(raw.policy_fingerprint(), derived.policy_fingerprint()); - } - - #[test] - fn catalog_rejects_missing_derived_dependencies_and_raw_source_conflicts() { - let mut derived = config(); - derived.derived_input = Some(DerivedInputIdentity { - inputs: BTreeSet::from([StoredOutputId::from(derived.policy_fingerprint())]), - program_sha256: "b".repeat(64), - }); - assert!(SummaryCatalog::from_materializations(1, 1, &[derived.clone()]).is_err()); - derived.table_name = Some("table".into()); - assert!(derived.population_filter_canonical().is_err()); - } - fn program(source: u32, root: u32) -> OwnedPostAsapDag { - use crate::executable_plan::{OwnedPostAsapEdge, OwnedPostAsapNode}; - use planner_types::post_asap::{ - EdgeRole, ExecutionDataState, GroupingEdgeCompatibility, WindowEdgeCompatibility, - }; - let state = ExecutionDataState::INGESTION_SUMMARY; - OwnedPostAsapDag { - schema_version: crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION, - query_id: "query-a".into(), - root: PostAsapNodeId(root), - nodes: [source, root] - .into_iter() - .map(|id| OwnedPostAsapNode { - id: PostAsapNodeId(id), - payload: serde_json::json!({"kind":"summary_merge"}), - output_state: state, - output_schema: serde_json::json!({"fields":[],"time_index":null}), - guarantee: None, - }) - .collect(), - edges: vec![OwnedPostAsapEdge { - producer: PostAsapNodeId(source), - consumer: PostAsapNodeId(root), - role: EdgeRole::Input, - intermediate_schema: serde_json::json!({"fields":[],"time_index":null}), - data_state: state, - grouping: GroupingEdgeCompatibility::Identical, - window: WindowEdgeCompatibility::NotApplicable, - }], - } - } - - #[test] - fn semantic_signature_ignores_node_and_query_numbering_but_not_inputs() { - let source = StoredOutputId::from(config().policy_fingerprint()); - let a = program(1, 2); - let first = DerivedInputIdentity::from_dag( - &a, - a.root, - &BTreeMap::from([(PostAsapNodeId(1), source)]), - ) - .unwrap(); - let mut b = program(900, 42); - b.query_id = "another-query".into(); - b.nodes.reverse(); - let second = DerivedInputIdentity::from_dag( - &b, - b.root, - &BTreeMap::from([(PostAsapNodeId(900), source)]), - ) - .unwrap(); - assert_eq!(first, second); - b.nodes - .iter_mut() - .find(|n| n.id == b.root) - .unwrap() - .output_schema = serde_json::json!({"fields":[],"time_index":0}); - assert_ne!( - first, - DerivedInputIdentity::from_dag( - &b, - b.root, - &BTreeMap::from([(PostAsapNodeId(900), source)]) - ) - .unwrap() - ); - assert!(DerivedInputIdentity::from_dag(&a, a.root, &BTreeMap::new()).is_err()); - let mut unsupported = a.clone(); - unsupported.schema_version = 999; - assert!(DerivedInputIdentity::from_dag( - &unsupported, - unsupported.root, - &BTreeMap::from([(PostAsapNodeId(1), source)]) - ) - .is_err()); - unsupported = a.clone(); - unsupported.nodes[1].payload = serde_json::json!({"kind":"unknown_operator"}); - assert!(DerivedInputIdentity::from_dag( - &unsupported, - unsupported.root, - &BTreeMap::from([(PostAsapNodeId(1), source)]) - ) - .is_err()); - let mut cycle = a.clone(); - cycle.edges[0].producer = cycle.root; - assert!(DerivedInputIdentity::from_dag(&cycle, cycle.root, &BTreeMap::new()).is_err()); - cycle.edges[0].producer = PostAsapNodeId(999); - assert!(DerivedInputIdentity::from_dag(&cycle, cycle.root, &BTreeMap::new()).is_err()); - } - - #[test] - fn catalog_rejects_self_referential_summary() { - use crate::sds::{ - DataDescriptor, DataSourceIdentity, SummaryDescriptor, ValueProjectionIdentity, - }; - let config = config(); - let input = DerivedInputIdentity { - inputs: BTreeSet::from([StoredOutputId::from(config.policy_fingerprint())]), - program_sha256: "f".repeat(64), - }; - let data = DataDescriptor::new_typed( - DataSourceIdentity::Derived { input }, - ValueProjectionIdentity::SampleValue, - "", - Vec::::new(), - "derived", - ); - assert!(SummaryCatalog::build( - 1, - 1, - [( - config.policy_fingerprint(), - SummaryDescriptor::from_config(&config).unwrap(), - data, - )] - ) - .is_err()); - } - #[test] - fn installation_rejects_derived_inputs_without_an_installed_maintenance_dag() { - use crate::precompute_plan::{PlanEnvelope, PrecomputePlan}; - let mut config = config(); - config.derived_input = Some(DerivedInputIdentity { - inputs: BTreeSet::from([StoredOutputId::from(config.policy_fingerprint())]), - program_sha256: "c".repeat(64), - }); - let envelope = PlanEnvelope { - plan_id: 1, - plan_version: 1, - generated_at_unix_ms: 0, - activation_unix_ms: 0, - expiry_unix_ms: None, - backend_compat: "asap-query-backend.v1".into(), - planner_revision: "test".into(), - capability_snapshot_id: "test".into(), - }; - let error = - PrecomputePlan::build(envelope, vec![config], &["producer".into()]).unwrap_err(); - assert!(error - .to_string() - .contains("installed aligned immutable maintenance DAG")); - } - #[test] - fn literal_leaves_are_hashed_without_inventing_materialization_references() { - use planner_types::{ - post_asap::ExecutableOperatorPayload, - pre_asap::{QueryExpr, ScalarValue}, - }; - let mut dag = program(1, 2); - let mut literal = dag.nodes[0].clone(); - literal.id = PostAsapNodeId(3); - literal.payload = serde_json::to_value(ExecutableOperatorPayload::Fallback { - expression: QueryExpr::Literal(ScalarValue::Int64(2)), - }) - .unwrap(); - dag.nodes.push(literal); - let mut edge = dag.edges[0].clone(); - edge.producer = PostAsapNodeId(3); - dag.edges.push(edge); - let source = StoredOutputId::from(config().policy_fingerprint()); - let frontiers = BTreeMap::from([(PostAsapNodeId(1), source)]); - let first = DerivedInputIdentity::from_dag(&dag, dag.root, &frontiers).unwrap(); - assert_eq!(first.inputs, BTreeSet::from([source])); - dag.nodes[2].payload = serde_json::to_value(ExecutableOperatorPayload::Fallback { - expression: QueryExpr::Literal(ScalarValue::Int64(3)), - }) - .unwrap(); - assert_ne!( - first, - DerivedInputIdentity::from_dag(&dag, dag.root, &frontiers).unwrap() - ); - assert!(DerivedInputIdentity::from_dag(&dag, PostAsapNodeId(3), &frontiers).is_err()); - } -} diff --git a/crates/asap_types/src/enums.rs b/crates/asap_types/src/enums.rs deleted file mode 100644 index ebf03534b..000000000 --- a/crates/asap_types/src/enums.rs +++ /dev/null @@ -1,96 +0,0 @@ -use std::fmt; -use std::str::FromStr; - -pub use asap_physical_operators::Statistic; - -#[derive( - clap::ValueEnum, - Clone, - Copy, - Debug, - Default, - PartialEq, - Eq, - Hash, - serde::Serialize, - serde::Deserialize, -)] -#[serde(rename_all = "snake_case")] -pub enum QueryLanguage { - #[default] - #[value(alias = "PROMQL", alias = "promql")] - PromQl, - MetricsQl, - ClickHouseSql, -} - -/// Policy for cleaning up old aggregates from the store. -/// Must be explicitly specified in inference_config.yaml. -#[derive(Clone, Debug, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum CleanupPolicy { - /// Keep only the N most recent aggregates (circular buffer behavior) - CircularBuffer, - /// Never clean up aggregates - NoCleanup, -} - -impl fmt::Display for CleanupPolicy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - CleanupPolicy::CircularBuffer => write!(f, "circular_buffer"), - CleanupPolicy::NoCleanup => write!(f, "no_cleanup"), - } - } -} - -impl FromStr for CleanupPolicy { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "circular_buffer" => Ok(CleanupPolicy::CircularBuffer), - "no_cleanup" => Ok(CleanupPolicy::NoCleanup), - _ => Err(format!("Unknown cleanup policy: '{s}'")), - } - } -} - -/// Window lifecycle and flush semantics owned by streaming configuration. -/// Tumbling is the default; wire names are lowercase. -#[derive( - Debug, Clone, Copy, Default, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -#[serde(rename_all = "snake_case")] -pub enum WindowKind { - /// Fixed, non-overlapping windows (e.g. a new 30s bucket every 30s). - #[default] - Tumbling, - /// Overlapping windows that advance by less than their width. - Sliding, - /// Gap-based windows that close after a period of inactivity. - Session, -} - -impl fmt::Display for WindowKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - WindowKind::Tumbling => write!(f, "tumbling"), - WindowKind::Sliding => write!(f, "sliding"), - WindowKind::Session => write!(f, "session"), - } - } -} - -impl FromStr for WindowKind { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "tumbling" => Ok(WindowKind::Tumbling), - "sliding" => Ok(WindowKind::Sliding), - "session" => Ok(WindowKind::Session), - _ => Err(format!("Unknown window kind: '{s}'")), - } - } -} diff --git a/crates/asap_types/src/erp_observation.rs b/crates/asap_types/src/erp_observation.rs deleted file mode 100644 index df23f6f1b..000000000 --- a/crates/asap_types/src/erp_observation.rs +++ /dev/null @@ -1,229 +0,0 @@ -//! Versioned runtime evidence about the inputs of one installed summary. -//! Shape is generic so the transport contract does not depend on a planner. -use crate::sds::{CatalogGeneration, StoredOutputId, SummaryInstanceId}; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ErpObservationInputSemantics { - ScalarSampleValue, - UnitSampleFrequency, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ErpPopulationObservation { - pub population_id: SummaryInstanceId, - pub shape: Shape, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ErpPopulationObservations { - pub schema_version: u32, - pub catalog_generation: CatalogGeneration, - pub stored_output_id: StoredOutputId, - pub observed_at_unix_ms: u64, - pub window_start_ms: i64, - pub window_end_ms: i64, - pub input_semantics: ErpObservationInputSemantics, - /// Any overflow or unsupported input invalidates the entire envelope. - /// Invalid envelopes contain no population fits. - pub invalid_reason: Option, - pub populations: Vec>, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ErpObservationFreshness { - pub max_age_ms: u64, - pub max_future_skew_ms: u64, -} - -impl ErpPopulationObservations { - pub fn validate_identity_and_freshness( - &self, - expected_generation: &CatalogGeneration, - expected_definition: StoredOutputId, - now_ms: u64, - freshness: ErpObservationFreshness, - ) -> Result<(), &'static str> { - if self.schema_version != 1 - || &self.catalog_generation != expected_generation - || self.stored_output_id != expected_definition - { - return Err("ERP observation belongs to a different catalog or summary"); - } - if self.invalid_reason.is_some() - || self.populations.is_empty() - || self.window_start_ms >= self.window_end_ms - { - return Err("ERP observation is incomplete or invalid"); - } - if freshness.max_age_ms == 0 - || self.observed_at_unix_ms > now_ms.saturating_add(freshness.max_future_skew_ms) - || now_ms.saturating_sub(self.observed_at_unix_ms) > freshness.max_age_ms - { - return Err("ERP observation is stale or future-dated"); - } - let mut seen = std::collections::BTreeSet::new(); - if self - .populations - .iter() - .any(|p| !seen.insert(&p.population_id)) - { - return Err("ERP observation repeats a population"); - } - Ok(()) - } -} - -/// Empirical sufficient statistics for frequency-shape fitting. No raw keys, -/// raw samples, distribution-family assertion, or planner configuration crosses the wire. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct EmpiricalFrequencyObservation { - pub sorted_counts: Vec, - pub interval_counts: Vec, -} -impl EmpiricalFrequencyObservation { - pub fn event_count(&self) -> Option { - if self.sorted_counts.len() > 65_536 - || self.interval_counts.len() > 256 - || self.sorted_counts.is_empty() - || self.sorted_counts.contains(&0) - || self.sorted_counts.windows(2).any(|w| w[0] < w[1]) - { - return None; - } - let total = self - .sorted_counts - .iter() - .try_fold(0u64, |sum, n| sum.checked_add(*n))?; - let intervals = self - .interval_counts - .iter() - .try_fold(0u64, |sum, n| sum.checked_add(*n))?; - (total == intervals).then_some(total) - } -} - -/// Bounded exact frequency counting. Fitting and ERP selection remain control-plane work. -#[derive(Debug)] -pub struct BoundedFrequencyObserver { - frequencies: std::collections::HashMap, - intervals: std::collections::HashMap, - max_keys: usize, - max_intervals: usize, - events: u64, - invalid: bool, -} -impl BoundedFrequencyObserver { - pub fn new(max_keys: usize, max_intervals: usize) -> Result { - if max_keys == 0 || max_intervals == 0 { - return Err("observation limits must be positive"); - } - Ok(Self { - frequencies: Default::default(), - intervals: Default::default(), - max_keys, - max_intervals, - events: 0, - invalid: false, - }) - } - pub fn observed_key_count(&self) -> usize { - self.frequencies.len() - } - pub fn invalidate(&mut self) { - self.invalid = true; - } - pub fn observe(&mut self, key: K, interval: usize) -> Result<(), &'static str> { - if self.invalid { - return Err("ERP observation was invalidated"); - } - if (!self.frequencies.contains_key(&key) && self.frequencies.len() >= self.max_keys) - || (!self.intervals.contains_key(&interval) - && self.intervals.len() >= self.max_intervals) - { - self.invalid = true; - return Err("ERP observation key or interval budget exceeded"); - } - let Some(events) = self.events.checked_add(1) else { - self.invalid = true; - return Err("ERP observation count overflow"); - }; - *self.frequencies.entry(key).or_default() += 1; - *self.intervals.entry(interval).or_default() += 1; - self.events = events; - Ok(()) - } - pub fn snapshot(&self) -> Option { - if self.invalid || self.frequencies.is_empty() { - return None; - } - let mut sorted_counts: Vec<_> = self.frequencies.values().copied().collect(); - sorted_counts.sort_unstable_by(|a, b| b.cmp(a)); - Some(EmpiricalFrequencyObservation { - sorted_counts, - interval_counts: self.intervals.values().copied().collect(), - }) - } -} - -impl ErpPopulationObservations { - pub fn try_map_shapes( - self, - mut convert: impl FnMut(S) -> Option, - ) -> Option> { - let populations = self - .populations - .into_iter() - .map(|population| { - Some(ErpPopulationObservation { - population_id: population.population_id, - shape: convert(population.shape)?, - }) - }) - .collect::>>()?; - Some(ErpPopulationObservations { - schema_version: self.schema_version, - catalog_generation: self.catalog_generation, - stored_output_id: self.stored_output_id, - observed_at_unix_ms: self.observed_at_unix_ms, - window_start_ms: self.window_start_ms, - window_end_ms: self.window_end_ms, - input_semantics: self.input_semantics, - invalid_reason: self.invalid_reason, - populations, - }) - } -} - -#[cfg(test)] -mod empirical_tests { - use super::*; - #[test] - fn count_overflow_invalidates_instead_of_publishing_prefix() { - let mut observer = BoundedFrequencyObserver::new(2, 2).unwrap(); - observer.observe(1u64, 0).unwrap(); - observer.events = u64::MAX; - assert!(observer.observe(1, 0).is_err()); - assert!(observer.snapshot().is_none()); - } - #[test] - fn malformed_frequency_counts_fail_closed() { - assert!(EmpiricalFrequencyObservation { - sorted_counts: vec![2, 1], - interval_counts: vec![2] - } - .event_count() - .is_none()); - assert!(EmpiricalFrequencyObservation { - sorted_counts: vec![1, 2], - interval_counts: vec![3] - } - .event_count() - .is_none()); - } -} diff --git a/crates/asap_types/src/executable_plan.rs b/crates/asap_types/src/executable_plan.rs deleted file mode 100644 index aff1ced0d..000000000 --- a/crates/asap_types/src/executable_plan.rs +++ /dev/null @@ -1,401 +0,0 @@ -//! Shared backend installation contract for semantic DAGs and physical bindings. -//! -//! `OwnedPostAsapDag` is the Send/Sync installed representation of Planner IR; -//! it is distinct from Planner's `PostAsapDagDocument` transport envelope. -//! Planner payloads contain process-local `Rc` values, so they are decoded only -//! when executing or validating a DAG. Compilation and QueryPlan cross-checks -//! remain control-plane responsibilities. - -use std::collections::{BTreeMap, BTreeSet}; - -use crate::sds::StoredOutputId; -use planner_types::post_asap::{ - EdgeRole, ExecutableDag, ExecutableDagEdge, ExecutableDagNode, ExecutionDataState, - ExecutionTiming, GroupingEdgeCompatibility, PostAsapNodeId, WindowEdgeCompatibility, -}; -use serde::{Deserialize, Serialize}; - -/// Identity within an installed query entry, separate from semantic Planner IDs. -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[serde(transparent)] -pub struct QueryNodeId(pub u64); - -pub const OWNED_POST_ASAP_DAG_SCHEMA_VERSION: u32 = 5; -pub const PRECOMPUTE_DAG_SCHEMA_VERSION: u32 = 6; - -/// Versioned, language-neutral Planner DAG persisted with an installed plan. -/// Plan lifecycle belongs to the enclosing `PrecomputePlan`; this document -/// carries semantic identity only and does not duplicate its envelope. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct OwnedPostAsapDag { - pub schema_version: u32, - pub query_id: String, - pub nodes: Vec, - pub edges: Vec, - pub root: PostAsapNodeId, -} - -/// Send/Sync wire form of one Planner node. Planner's in-memory payload may -/// contain `Rc`; the canonical JSON payload preserves its tagged type without -/// leaking that process-local ownership choice into installed runtime state. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct OwnedPostAsapNode { - pub id: PostAsapNodeId, - pub payload: serde_json::Value, - pub output_state: ExecutionDataState, - pub output_schema: serde_json::Value, - pub guarantee: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct OwnedPostAsapEdge { - pub producer: PostAsapNodeId, - pub consumer: PostAsapNodeId, - pub role: EdgeRole, - pub intermediate_schema: serde_json::Value, - pub data_state: ExecutionDataState, - pub grouping: GroupingEdgeCompatibility, - pub window: WindowEdgeCompatibility, -} - -impl OwnedPostAsapDag { - pub fn from_executable(query_id: String, dag: &ExecutableDag) -> Result { - let nodes = dag - .nodes - .iter() - .map(|node| { - Ok(OwnedPostAsapNode { - id: node.id, - payload: serde_json::to_value(&node.payload).map_err(|e| e.to_string())?, - output_state: node.output_state, - output_schema: serde_json::to_value(&node.output_schema) - .map_err(|e| e.to_string())?, - guarantee: node - .guarantee - .as_ref() - .map(serde_json::to_value) - .transpose() - .map_err(|e| e.to_string())?, - }) - }) - .collect::, String>>()?; - let edges = dag - .edges - .iter() - .map(|edge| { - Ok(OwnedPostAsapEdge { - producer: edge.producer, - consumer: edge.consumer, - role: edge.role, - intermediate_schema: serde_json::to_value(&edge.intermediate_schema) - .map_err(|e| e.to_string())?, - data_state: edge.data_state, - grouping: edge.grouping, - window: edge.window, - }) - }) - .collect::, String>>()?; - Ok(Self { - schema_version: OWNED_POST_ASAP_DAG_SCHEMA_VERSION, - query_id, - nodes, - edges, - root: dag.root, - }) - } - - pub fn decode(&self) -> Result { - let node_ids = self - .nodes - .iter() - .map(|node| node.id) - .collect::>(); - if node_ids.len() != self.nodes.len() { - return Err("duplicate post-ASAP DAG node ID".into()); - } - let edge_ids = self - .edges - .iter() - .map(|edge| { - ( - edge.producer, - edge.consumer, - serde_json::to_string(&edge.role).expect("EdgeRole is serializable"), - ) - }) - .collect::>(); - if edge_ids.len() != self.edges.len() { - return Err("duplicate post-ASAP DAG edge".into()); - } - let nodes = self - .nodes - .iter() - .map(|node| { - let payload: planner_types::post_asap::ExecutableOperatorPayload = - serde_json::from_value(node.payload.clone()).map_err(|e| e.to_string())?; - Ok(ExecutableDagNode { - id: node.id, - payload, - output_state: node.output_state, - output_schema: serde_json::from_value(node.output_schema.clone()) - .map_err(|e| e.to_string())?, - guarantee: node - .guarantee - .clone() - .map(serde_json::from_value) - .transpose() - .map_err(|e| e.to_string())?, - }) - }) - .collect::, String>>()?; - let edges = self - .edges - .iter() - .map(|edge| { - Ok(ExecutableDagEdge { - producer: edge.producer, - consumer: edge.consumer, - role: edge.role, - intermediate_schema: serde_json::from_value(edge.intermediate_schema.clone()) - .map_err(|e| e.to_string())?, - data_state: edge.data_state, - grouping: edge.grouping, - window: edge.window, - }) - }) - .collect::, String>>()?; - Ok(ExecutableDag { - nodes, - edges, - root: self.root, - }) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct InstalledPostAsapDag { - pub document: OwnedPostAsapDag, - pub binding: BackendExecutableBinding, -} - -impl InstalledPostAsapDag { - pub fn validate(&self) -> Result<(), String> { - if self.document.query_id.trim().is_empty() { - return Err("invalid post-ASAP DAG document identity/version".into()); - } - if self.document.schema_version != PRECOMPUTE_DAG_SCHEMA_VERSION { - return Err("unsupported maintenance DAG document version".into()); - } - self.binding.validate_precompute(&self.document.decode()?) - } - - /// Project the selected semantic DAG onto the maintenance ancestors of its - /// stored outputs. - pub fn precompute_projection(mut self) -> Result { - if self.document.schema_version != OWNED_POST_ASAP_DAG_SCHEMA_VERSION { - return Err("selected DAG has an unsupported document version".into()); - } - self.binding.validate(&self.document.decode()?)?; - if self.binding.precompute_sinks.is_empty() { - return Err("cannot project a DAG without maintenance sinks".into()); - } - let mut included = self - .binding - .precompute_sinks - .iter() - .copied() - .collect::>(); - let mut frontier = self.binding.precompute_sinks.clone(); - while let Some(consumer) = frontier.pop() { - for edge in self - .document - .edges - .iter() - .filter(|edge| edge.consumer == consumer) - { - if included.insert(edge.producer) { - frontier.push(edge.producer); - } - } - } - self.document - .nodes - .retain(|node| included.contains(&node.id)); - self.document - .edges - .retain(|edge| included.contains(&edge.producer) && included.contains(&edge.consumer)); - self.binding.nodes.retain(|id, _| included.contains(id)); - self.document.root = *self.binding.precompute_sinks.first().unwrap(); - self.document.schema_version = PRECOMPUTE_DAG_SCHEMA_VERSION; - self.validate()?; - Ok(self) - } -} - -/// Backend-owned physical identities and sink placement for one Planner -/// semantic DAG. Planner node IDs remain stable semantic references; the -/// backend never assumes they equal its independently allocated query IDs. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct BackendExecutableBinding { - pub nodes: BTreeMap, - pub query_sink: PostAsapNodeId, - pub query_plan_sink: QueryNodeId, - pub precompute_sinks: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "placement", rename_all = "snake_case", deny_unknown_fields)] -pub enum BackendNodeBinding { - Query { - query_node: QueryNodeId, - }, - /// Semantic read node absorbed into a larger installed query node, such - /// as an external exact subtree boundary. - QueryInput, - MaintenanceInput, - Materialization { - stored_output: StoredOutputId, - }, -} - -impl BackendExecutableBinding { - pub fn node(&self, id: PostAsapNodeId) -> Option<&BackendNodeBinding> { - self.nodes.get(&id) - } - - pub fn validate_precompute(&self, dag: &ExecutableDag) -> Result<(), String> { - let ids = dag - .nodes - .iter() - .map(|node| node.id) - .collect::>(); - if ids.is_empty() || self.nodes.keys().copied().collect::>() != ids { - return Err("maintenance binding does not cover its projected DAG".into()); - } - if self.precompute_sinks.is_empty() - || self.precompute_sinks.iter().any(|id| !ids.contains(id)) - { - return Err("maintenance projection has missing sinks".into()); - } - if self.precompute_sinks.iter().any(|id| { - !matches!( - self.node(*id), - Some(BackendNodeBinding::Materialization { .. }) - ) - }) { - return Err("maintenance sink lacks a stored-output binding".into()); - } - for node in &dag.nodes { - match (node.output_state.timing, self.node(node.id)) { - ( - ExecutionTiming::IngestionTime, - Some( - BackendNodeBinding::MaintenanceInput - | BackendNodeBinding::Materialization { .. }, - ), - ) => {} - _ => { - return Err(format!( - "query-owned node {} in maintenance projection", - node.id.0 - )) - } - } - } - if dag - .edges - .iter() - .any(|edge| !ids.contains(&edge.producer) || !ids.contains(&edge.consumer)) - { - return Err("maintenance projection has dangling edges".into()); - } - Ok(()) - } - - pub fn validate(&self, dag: &ExecutableDag) -> Result<(), String> { - let semantic = dag - .nodes - .iter() - .map(|node| node.id.0) - .collect::>(); - if !semantic.contains(&self.query_sink.0) { - return Err("backend query sink is absent from semantic DAG".into()); - } - if self.nodes.keys().map(|id| id.0).collect::>() != semantic { - return Err("backend executable binding does not cover semantic DAG exactly".into()); - } - for node in &dag.nodes { - match (node.output_state.timing, self.node(node.id).unwrap()) { - (ExecutionTiming::QueryTime, BackendNodeBinding::Query { .. }) - | (ExecutionTiming::QueryTime, BackendNodeBinding::QueryInput) - | (ExecutionTiming::IngestionTime, BackendNodeBinding::MaintenanceInput) - | (ExecutionTiming::IngestionTime, BackendNodeBinding::Materialization { .. }) => {} - _ => { - return Err(format!( - "backend placement disagrees with node {} mode", - node.id.0 - )) - } - } - } - for sink in &self.precompute_sinks { - if !matches!( - self.node(*sink), - Some(BackendNodeBinding::Materialization { .. }) - ) { - return Err(format!( - "backend precompute sink {} is not materialized", - sink.0 - )); - } - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - // Shared installation metadata must be safe to retain in cross-thread - // RuntimePhysicalPlan snapshots without importing the compiler crate. - #[test] - fn installed_contract_is_send_sync_and_preserves_wire_identity() { - fn send_sync() {} - send_sync::(); - let wire = serde_json::json!({ - "schema_version": 2, "query_id": "q", "nodes": [], "edges": [], "root": 0 - }); - let document: OwnedPostAsapDag = serde_json::from_value(wire.clone()).unwrap(); - assert_eq!(serde_json::to_value(document).unwrap(), wire); - assert_eq!(serde_json::to_value(QueryNodeId(9)).unwrap(), 9); - } - - #[test] - fn installed_maintenance_dag_rejects_complete_dag_version() { - let installed = InstalledPostAsapDag { - document: OwnedPostAsapDag { - schema_version: OWNED_POST_ASAP_DAG_SCHEMA_VERSION, - query_id: "q".into(), - nodes: Vec::new(), - edges: Vec::new(), - root: PostAsapNodeId(0), - }, - binding: BackendExecutableBinding { - nodes: BTreeMap::new(), - query_sink: PostAsapNodeId(0), - query_plan_sink: QueryNodeId(0), - precompute_sinks: Vec::new(), - }, - }; - assert!(installed - .validate() - .unwrap_err() - .contains("unsupported maintenance DAG")); - } -} diff --git a/crates/asap_types/src/grouping_projection.rs b/crates/asap_types/src/grouping_projection.rs deleted file mode 100644 index 23c0ae721..000000000 --- a/crates/asap_types/src/grouping_projection.rs +++ /dev/null @@ -1,455 +0,0 @@ -//! Typed source columns defining one summary population key. -use crate::KeyByLabelNames; -use planner_types::pre_asap::{Column, DataType}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -/// Version of the population-key routing contract. Choosing a new version -/// changes materialization and data identity; it never migrates old SIDs. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum PopulationKeyEncoding { - #[default] - LegacyDelimited, - CanonicalLabelsV1, -} -impl PopulationKeyEncoding { - pub fn is_legacy(&self) -> bool { - *self == Self::LegacyDelimited - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct GroupingProjection(Vec); - -impl GroupingProjection { - pub fn new(mut columns: Vec) -> Self { - columns.sort_by(|a, b| a.name.cmp(&b.name)); - Self(columns) - } - pub fn validate(&self) -> Result<(), String> { - let mut previous: Option<&str> = None; - for column in &self.0 { - if column.name.is_empty() - || column.table.is_some() - || previous == Some(column.name.as_str()) - { - return Err("grouping columns must be unqualified and unique".into()); - } - previous = Some(&column.name); - } - Ok(()) - } - pub fn validate_table_group_codec(&self) -> Result<(), String> { - self.validate()?; - for column in self.columns() { - crate::table_population::validate_column_name(&column.name)?; - if column.nullable { - return Err( - "nullable table grouping requires an explicit null-key encoding".into(), - ); - } - if !table_group_codec_type(&column.dtype) { - return Err( - "table group codec requires integer, string, boolean, or supported Map values" - .into(), - ); - } - } - Ok(()) - } - pub fn push(&mut self, name: String) { - self.0.push(Column::new(name, DataType::Utf8, false)); - self.0.sort_by(|a, b| a.name.cmp(&b.name)); - self.0.dedup(); - } - pub fn columns(&self) -> &[Column] { - &self.0 - } - pub fn iter(&self) -> impl Iterator { - self.0.iter().map(|c| &c.name) - } - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - pub fn names(&self) -> Vec { - self.0.iter().map(|c| c.name.clone()).collect() - } - pub fn label_names(&self) -> KeyByLabelNames { - KeyByLabelNames::new(self.names()) - } - pub fn is_legacy_labels(&self) -> bool { - self.0 - .iter() - .all(|c| c.dtype == DataType::Utf8 && !c.nullable && c.table.is_none()) - } - pub fn serialize_to_json(&self) -> serde_json::Value { - if self.is_legacy_labels() { - serde_json::json!(self.names()) - } else { - serde_json::to_value(self).expect("grouping projection serializes") - } - } - pub fn deserialize_from_json(value: &serde_json::Value) -> Result { - serde_json::from_value(value.clone()) - } -} -// Float grouping requires SQL equality canonicalization (+0/-0 and NaNs), -// and timestamps need explicit unit/timezone semantics. Neither is claimed by v1. -fn table_group_codec_type(dtype: &DataType) -> bool { - match dtype { - DataType::Int64 | DataType::Utf8 | DataType::Bool => true, - DataType::Map { key, value, .. } => { - matches!( - key.as_ref(), - DataType::Int64 | DataType::Utf8 | DataType::Bool - ) && table_group_codec_type(value) - } - _ => false, - } -} - -impl From for GroupingProjection { - fn from(mut names: KeyByLabelNames) -> Self { - names.labels.sort(); - names.labels.dedup(); - Self::new( - names - .labels - .into_iter() - .map(|name| Column::new(name, DataType::Utf8, false)) - .collect(), - ) - } -} -impl FromIterator for GroupingProjection { - fn from_iter>(iter: T) -> Self { - KeyByLabelNames::new(iter.into_iter().collect()).into() - } -} -impl<'de> Deserialize<'de> for GroupingProjection { - fn deserialize>(deserializer: D) -> Result { - #[derive(Deserialize)] - #[serde(untagged)] - enum Wire { - Columns(Vec), - Names(Vec), - Legacy(KeyByLabelNames), - } - Ok(match Wire::deserialize(deserializer)? { - Wire::Columns(columns) => Self::new(columns), - Wire::Names(names) => KeyByLabelNames::new(names).into(), - Wire::Legacy(names) => names.into(), - }) - } -} - -#[cfg(test)] -mod identity_tests { - use super::*; - /// A type or nullability change cannot reuse an incompatible summary population. - #[test] - fn descriptor_identity_includes_group_type_and_nullability() { - let legacy = crate::sds::DataDescriptor::new("m", "", vec!["tenant".into()]); - let same = legacy - .clone() - .with_grouping_projection(KeyByLabelNames::new(vec!["tenant".into()]).into()); - assert_eq!(legacy.id, same.id); - let numeric = legacy - .clone() - .with_grouping_projection(GroupingProjection::new(vec![Column::new( - "tenant", - DataType::Int64, - false, - )])); - let nullable = legacy - .clone() - .with_grouping_projection(GroupingProjection::new(vec![Column::new( - "tenant", - DataType::Utf8, - true, - )])); - assert_ne!(legacy.id, numeric.id); - assert_ne!(legacy.id, nullable.id); - assert_ne!(numeric.id, nullable.id); - assert!(numeric - .validate() - .unwrap_err() - .to_string() - .contains("string labels")); - assert!(nullable - .validate() - .unwrap_err() - .to_string() - .contains("string labels")); - for grouping in [numeric.group_by_keys, nullable.group_by_keys] { - let table = crate::sds::DataDescriptor::new_typed( - crate::sds::DataSourceIdentity::Table { - table_ref: "samples".into(), - }, - crate::sds::ValueProjectionIdentity::Column { - name: "value".into(), - }, - "", - Vec::::new(), - "table.samples.v1", - ) - .with_grouping_projection(grouping); - table.validate().unwrap(); - } - } -} - -impl Serialize for GroupingProjection { - fn serialize(&self, serializer: S) -> Result { - if self.is_legacy_labels() { - self.names().serialize(serializer) - } else { - self.0.serialize(serializer) - } - } -} - -/// Preserve the older precompute-config object shape for ordinary label groups. -pub(crate) fn serialize_config_grouping( - grouping: &GroupingProjection, - serializer: S, -) -> Result { - if grouping.is_legacy_labels() { - grouping.label_names().serialize(serializer) - } else { - grouping.serialize(serializer) - } -} - -/// Table group values travel through the string-key index as lossless JSON bytes. -/// Map values use ordered key/value pairs, preserving duplicate keys. -pub const TABLE_GROUP_OBSERVATION_SEMANTICS: &str = "asap.table-column-groups.base64-json.v1"; - -pub fn encode_table_group_value(value: &serde_json::Value) -> Result { - use base64::Engine; - let bytes = serde_json::to_vec(value).map_err(|error| error.to_string())?; - Ok(base64::engine::general_purpose::STANDARD.encode(bytes)) -} - -pub fn decode_table_group_value(encoded: &str) -> Result { - use base64::Engine; - let bytes = base64::engine::general_purpose::STANDARD - .decode(encoded) - .map_err(|error| error.to_string())?; - serde_json::from_slice(&bytes).map_err(|error| error.to_string()) -} - -#[cfg(test)] -mod codec_tests { - use super::*; - #[test] - fn table_group_codec_rejects_noncanonical_float_and_timestamp_keys() { - for dtype in [ - DataType::Float64, - DataType::Timestamp, - DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Float64), - value_nullable: false, - }, - ] { - assert!( - GroupingProjection::new(vec![Column::new("g", dtype, false)]) - .validate_table_group_codec() - .is_err() - ); - } - assert!(GroupingProjection::new(vec![Column::new( - "g", - DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Int64), - value_nullable: true, - }, - false - )]) - .validate_table_group_codec() - .is_ok()); - } - - #[test] - fn equivalent_json_spelling_has_one_shared_group_encoding() { - use base64::Engine; - let first = base64::engine::general_purpose::STANDARD.encode(br#""a\/b""#); - let second = base64::engine::general_purpose::STANDARD.encode(br#""a/b""#); - assert_ne!(first, second); - let normalize = |input: &str| { - encode_table_group_value(&decode_table_group_value(input).unwrap()).unwrap() - }; - assert_eq!(normalize(&first), normalize(&second)); - } - - /// The grouping codec preserves null, escaping, duplicate map keys and exact integers. - #[test] - fn typed_group_codec_is_lossless_at_numeric_and_string_boundaries() { - for value in [ - serde_json::Value::Null, - serde_json::json!("a,\"b\\c\n"), - serde_json::json!(i64::MAX), - serde_json::json!(i64::MIN), - serde_json::json!([["k", i64::MAX], ["k", null]]), - ] { - let encoded = encode_table_group_value(&value).unwrap(); - assert!(!encoded.contains(['\"', ',', '\\'])); - assert_eq!(decode_table_group_value(&encoded).unwrap(), value); - } - assert!(decode_table_group_value("not base64").is_err()); - } -} - -/// Canonical label-population identity. This codec does not replace any -/// persisted routing key implicitly: consumers must select its version as part -/// of their contract. This key remains scoped by the existing summary/data -/// definition; it does not replace source identity, partitioning or schema. -/// Typed table values retain their existing value codec. -pub const LABEL_POPULATION_KEY_PREFIX: &str = "asap.label-population.v1:"; - -pub fn encode_label_population_key( - labels: &std::collections::BTreeMap, -) -> Result { - use base64::Engine; - let pairs: Vec<_> = labels.iter().collect(); - let bytes = serde_json::to_vec(&pairs).map_err(|error| error.to_string())?; - Ok(format!( - "{LABEL_POPULATION_KEY_PREFIX}{}", - base64::engine::general_purpose::STANDARD.encode(bytes) - )) -} - -#[cfg(test)] -/// Reject legacy, unknown-version, duplicate-name and noncanonical encodings. -/// In particular, decoding must never silently collapse duplicate populations. -pub fn decode_label_population_key( - encoded: &str, -) -> Result, String> { - use base64::Engine; - let payload = encoded - .strip_prefix(LABEL_POPULATION_KEY_PREFIX) - .ok_or("unsupported label population key version")?; - let bytes = base64::engine::general_purpose::STANDARD - .decode(payload) - .map_err(|error| error.to_string())?; - let pairs: Vec<(String, String)> = - serde_json::from_slice(&bytes).map_err(|error| error.to_string())?; - let mut labels = std::collections::BTreeMap::new(); - for (key, value) in pairs { - if labels.insert(key, value).is_some() { - return Err("duplicate label population key".into()); - } - } - if encode_label_population_key(&labels)? != encoded { - return Err("noncanonical label population key".into()); - } - Ok(labels) -} - -#[cfg(test)] -mod population_key_tests { - use super::*; - use std::collections::BTreeMap; - #[test] - fn delimiter_collisions_and_order_are_unambiguous() { - let one = BTreeMap::from([("a".into(), "b;c=d".into())]); - let two = BTreeMap::from([("a".into(), "b".into()), ("c".into(), "d".into())]); - assert_ne!( - encode_label_population_key(&one).unwrap(), - encode_label_population_key(&two).unwrap() - ); - let reversed = [("c".into(), "d".into()), ("a".into(), "b".into())] - .into_iter() - .collect(); - assert_eq!( - encode_label_population_key(&two), - encode_label_population_key(&reversed) - ); - for labels in [ - one, - two, - BTreeMap::new(), - BTreeMap::from([("=;\0雪".into(), "\"\\\n\0".into())]), - ] { - let key = encode_label_population_key(&labels).unwrap(); - assert_eq!(decode_label_population_key(&key).unwrap(), labels); - let serialized = serde_json::to_string(&key).unwrap(); - assert_eq!(serde_json::from_str::(&serialized).unwrap(), key); - } - } - #[test] - fn unknown_legacy_duplicate_and_noncanonical_keys_fail_closed() { - use base64::Engine; - for json in [ - r#"[["a","1"],["a","2"]]"#, - r#"[["z","1"],["a","2"]]"#, - r#"[ ["a","1"] ]"#, - ] { - let key = format!( - "{LABEL_POPULATION_KEY_PREFIX}{}", - base64::engine::general_purpose::STANDARD.encode(json) - ); - assert!(decode_label_population_key(&key).is_err()); - } - for key in [ - "a=b;", - "asap.label-population.v2:W10=", - "asap.label-population.v1:!", - ] { - assert!(decode_label_population_key(key).is_err()); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - /// Legacy label lists become one non-null string column per name. - #[test] - fn legacy_and_typed_groups_share_one_projection() { - let legacy: GroupingProjection = - serde_json::from_value(serde_json::json!({"labels":["job"]})).unwrap(); - assert_eq!( - legacy.columns(), - &[Column::new("job", DataType::Utf8, false)] - ); - let typed: GroupingProjection = serde_json::from_value( - serde_json::json!([{"name":"job","dtype":"utf8","nullable":false}]), - ) - .unwrap(); - assert_eq!(typed, legacy); - assert_eq!(typed.serialize_to_json(), serde_json::json!(["job"])); - assert_eq!( - serde_json::to_value(&typed).unwrap(), - serde_json::json!(["job"]) - ); - #[derive(Serialize)] - struct LegacyConfig { - #[serde(serialize_with = "serialize_config_grouping")] - grouping_labels: GroupingProjection, - } - assert_eq!( - serde_json::to_value(LegacyConfig { - grouping_labels: typed - }) - .unwrap(), - serde_json::json!({"grouping_labels":{"labels":["job"]}}) - ); - } - /// Numeric grouping types must survive wire transport rather than become labels. - #[test] - fn typed_group_retains_type_and_rejects_duplicate_columns() { - let typed = GroupingProjection::new(vec![Column::new("tenant", DataType::Int64, true)]); - let decoded: GroupingProjection = - serde_json::from_value(typed.serialize_to_json()).unwrap(); - assert_eq!(decoded, typed); - assert!(!decoded.is_legacy_labels()); - let duplicate = GroupingProjection::new(vec![ - Column::new("tenant", DataType::Int64, true), - Column::new("tenant", DataType::Utf8, false), - ]); - assert!(duplicate.validate().is_err()); - } -} diff --git a/crates/asap_types/src/key_by_label_names.rs b/crates/asap_types/src/key_by_label_names.rs deleted file mode 100644 index 5cd902b7d..000000000 --- a/crates/asap_types/src/key_by_label_names.rs +++ /dev/null @@ -1,147 +0,0 @@ -//! A sorted, set-algebra-bearing label-name key. -//! -//! Formerly `promql_utilities::data_model::key_by_label_names` — moved -//! here for the same reason as [`crate::Statistic`]: `asap_types` -//! (`PrecomputeMaterialization::grouping_labels`, `PolicyFingerprint`, -//! `PolicyRegistry`, `capability_matching`) is its real center of -//! gravity and the shared foundation both `control_plane`'s ecosystem -//! and `data_plane` can depend on without a cycle. Closer to a runtime -//! index key than a planning IR node — ASAPController's -//! `QueryExpr::Aggregate.by` is the nearest relative in spirit, but -//! carries positional `ColumnId`s, not a sorted, deduplicated label-name -//! set with `Vec`-style set algebra. - -use serde::{Deserialize, Serialize}; -use tracing::debug; - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct KeyByLabelNames { - pub labels: Vec, // Renamed from label_names to match query_logics usage -} - -impl KeyByLabelNames { - pub fn new(label_names: Vec) -> Self { - debug!("Creating KeyByLabelNames with {} labels", label_names.len()); - let mut sorted_names = label_names; - sorted_names.sort(); // Match Python behavior - keys are sorted - debug!("Sorted labels: {:?}", sorted_names); - Self { - labels: sorted_names, - } - } - - pub fn empty() -> Self { - Self::new(Vec::new()) - } - - pub fn from_names(names: Vec) -> Self { - Self::new(names) - } - - pub fn push(&mut self, name: String) { - debug!("Adding label: {}", name); - self.labels.push(name); - self.labels.sort(); // Keep sorted - } - - /// Set difference operation - remove labels that are in the other set - /// Based on Python implementation: KeyByLabelNames.__sub__ - pub fn difference(&self, other: &KeyByLabelNames) -> KeyByLabelNames { - debug!( - "Computing difference between {:?} and {:?}", - self.labels, other.labels - ); - let other_set: std::collections::HashSet<_> = other.labels.iter().collect(); - let result: Vec = self - .labels - .iter() - .filter(|label| !other_set.contains(label)) - .cloned() - .collect(); - KeyByLabelNames::new(result) - } - - /// Set union operation - combine labels from both sets - /// Based on Python implementation: KeyByLabelNames.__add__ - pub fn union(&self, other: &KeyByLabelNames) -> KeyByLabelNames { - debug!( - "Computing union between {:?} and {:?}", - self.labels, other.labels - ); - let mut combined = std::collections::HashSet::new(); - for label in &self.labels { - combined.insert(label.clone()); - } - for label in &other.labels { - combined.insert(label.clone()); - } - KeyByLabelNames::new(combined.into_iter().collect()) - } - - pub fn serialize_to_json(&self) -> serde_json::Value { - serde_json::to_value(&self.labels).unwrap_or(serde_json::Value::Null) - } - - pub fn deserialize_from_json(data: &serde_json::Value) -> Result { - let names: Vec = serde_json::from_value(data.clone())?; - Ok(Self::new(names)) - } - - pub fn is_empty(&self) -> bool { - self.labels.is_empty() - } - - pub fn len(&self) -> usize { - self.labels.len() - } -} - -impl Default for KeyByLabelNames { - fn default() -> Self { - Self::empty() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_key_by_label_names() { - let key = KeyByLabelNames::new(vec!["instance".to_string(), "job".to_string()]); - - assert_eq!(key.len(), 2); - assert_eq!(key.labels, vec!["instance".to_string(), "job".to_string()]); - - let mut key = KeyByLabelNames::new(vec!["instance".to_string(), "job".to_string()]); - key.push("new_label".to_string()); - assert_eq!(key.len(), 3); - // After sorting, should be in alphabetical order - assert!(key.labels.contains(&"instance".to_string())); - assert!(key.labels.contains(&"job".to_string())); - assert!(key.labels.contains(&"new_label".to_string())); - } - - #[test] - fn test_difference() { - let key1 = KeyByLabelNames::new(vec!["a".to_string(), "b".to_string(), "c".to_string()]); - let key2 = KeyByLabelNames::new(vec!["b".to_string(), "c".to_string()]); - - let diff = key1.difference(&key2); - assert_eq!(diff.len(), 1); - assert_eq!(diff.labels, vec!["a".to_string()]); - } - - #[test] - fn test_union() { - let key1 = KeyByLabelNames::new(vec!["a".to_string(), "b".to_string()]); - let key2 = KeyByLabelNames::new(vec!["b".to_string(), "c".to_string()]); - - let union = key1.union(&key2); - assert_eq!(union.len(), 3); - assert_eq!( - union.labels, - vec!["a".to_string(), "b".to_string(), "c".to_string()] - ); - } -} diff --git a/crates/asap_types/src/lib.rs b/crates/asap_types/src/lib.rs deleted file mode 100644 index 1d4af2fef..000000000 --- a/crates/asap_types/src/lib.rs +++ /dev/null @@ -1,43 +0,0 @@ -pub mod accumulator_spec; -pub mod accuracy; -pub mod aggregation_config; -pub mod aggregation_type; -pub mod derived_input; -pub mod enums; -pub mod erp_observation; -pub mod executable_plan; -pub mod grouping_projection; -pub mod key_by_label_names; -pub mod monitor_spec; -pub mod plan_publication; -pub mod policy_fingerprint; -pub mod policy_registry; -pub mod producer_plan; -pub mod query_requirements; -pub mod routing_index; -pub mod sds; -pub mod semantic_fragment; -pub mod storage_backend; -pub mod summary_catalog; -pub mod summary_semantics; -pub mod table_population; -pub mod traits; -pub mod utils; - -pub use accumulator_spec::{AccumulatorSpec, AccumulatorSpecError, SampleUpdateRule}; -pub use accuracy::{AccuracyKind, AccuracyProfile}; -pub use aggregation_config::*; -pub use aggregation_type::AggregationType; -pub use enums::*; -pub use grouping_projection::{GroupingProjection, PopulationKeyEncoding}; -pub use key_by_label_names::KeyByLabelNames; -pub use monitor_spec::{MonitorFunctional, MonitorSpec}; -pub use policy_fingerprint::PolicyFingerprint; -pub use policy_registry::PolicyRegistry; -pub use query_requirements::*; -pub use routing_index::RoutingIndex; -pub use storage_backend::*; - -pub mod precompute_plan; - -pub mod query_plan; diff --git a/crates/asap_types/src/monitor_spec.rs b/crates/asap_types/src/monitor_spec.rs deleted file mode 100644 index 6aad5ad06..000000000 --- a/crates/asap_types/src/monitor_spec.rs +++ /dev/null @@ -1,84 +0,0 @@ -use serde::{Deserialize, Serialize}; - -/// Continuous-monitoring readout shared by the control and data planes. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum MonitorFunctional { - #[default] - Sum, - CmsPoint, - LinearBuckets, - F2, -} - -impl MonitorFunctional { - pub const fn as_str(self) -> &'static str { - match self { - Self::Sum => "sum", - Self::CmsPoint => "cms_point", - Self::LinearBuckets => "linear_buckets", - Self::F2 => "f2", - } - } - - pub fn from_name(value: &str) -> Self { - match value.trim().to_ascii_lowercase().as_str() { - "cms_point" | "cms" => Self::CmsPoint, - "linear_buckets" | "linear" => Self::LinearBuckets, - "f2" | "l2" => Self::F2, - _ => Self::Sum, - } - } -} - -/// One continuous-monitoring (CDM) threshold spec. The data-plane monitor -/// coordinator owns the AUTHORITATIVE `tau`/`epsilon`/`window_ms` (the edge -/// copy is advisory), keyed by the same content-addressed `agg_id` the edge and -/// coordinator share. `key` is the CMS point-frequency key for point monitors -/// (empty for Sum / whole-stream). See -/// `ASAPCollector/docs/continuous-monitoring-tumbling-cost-analysis.md`. -/// -/// Stays here (unlike `data_plane::storage_engines::types::InstalledPrecomputePlan`, -/// which holds a `Vec` field) because `control_plane` genuinely -/// needs it: `emit/monitor.rs` builds the `InstalledPrecomputePlan.monitors[]` JSON -/// entry by hand and has a regression test asserting that JSON deserializes -/// into this exact type. `control_plane` cannot depend on `data_plane` (the -/// dependency runs the other way), so this type has to live somewhere both -/// sides can reach — same reasoning as `PrecomputeMaterialization`/`PolicyFingerprint`. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct MonitorSpec { - pub agg_id: u64, - /// Additive readout the edge reports: "sum" (default), "cms_point", "f2". - /// Pass-through metadata so the edge can auto-learn its reporting mode from - /// the pushed config; the coordinator allocation is value-driven and does not - /// branch on it (p_i ∝ √(value/rate) is the F2 allocation when value=‖f‖²). - #[serde(default)] - pub functional: String, - /// CMS point-frequency key x; empty (default) for Sum / whole-stream / F2. - #[serde(default)] - pub key: String, - /// Threshold τ (authoritative here, not at the edge). - pub tau: f64, - /// Relative tolerance ε; the alert fires when the estimate reaches (1−ε)τ. - #[serde(default = "default_monitor_epsilon")] - pub epsilon: f64, - /// Tumbling epoch length in ms; MUST match the edge window for this agg_id. - pub window_ms: u64, - /// Count-Sketch depth (rows) for whole-sketch `functional="f2"` monitors. - /// 0 (default) for scalar monitors; MUST match the edge's Count-Sketch for - /// this agg when F2 (both sides square/merge the same cell matrix). - #[serde(default)] - pub d: usize, - /// Count-Sketch width (buckets/row) for F2 monitors; 0 for scalar. - #[serde(default)] - pub w: usize, - /// F2 monitoring variant: "distributed" (default, ship every window) or - /// "geometric" (Sharfman–Schuster–Keren safe-zone, ship on local violation). - /// Ignored by scalar monitors. - #[serde(default)] - pub mode: String, -} - -fn default_monitor_epsilon() -> f64 { - 0.05 -} diff --git a/crates/asap_types/src/plan_publication.rs b/crates/asap_types/src/plan_publication.rs deleted file mode 100644 index d678a23d5..000000000 --- a/crates/asap_types/src/plan_publication.rs +++ /dev/null @@ -1,211 +0,0 @@ -//! Canonical publication document for one catalog generation. -use crate::precompute_plan::PrecomputePlan; -use crate::producer_plan::{CollectorPlan, TransmissionPlan}; -use crate::query_plan::QueryPlan; -use crate::summary_catalog::SummaryCatalog; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalPlanPublication { - pub summary_catalog: SummaryCatalog, - pub precompute_plan: PrecomputePlan, - pub collector_plans: Vec, - pub transmission_plan: TransmissionPlan, - pub query_plan: QueryPlan, -} - -/// Complete typed envelope accepted by the data-plane install endpoint. -/// -/// Runtime-only routing and adaptation evidence decorate the shared -/// publication without changing its catalog generation. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalPlanInstallRequest { - pub summary_catalog: SummaryCatalog, - #[serde(default)] - pub collector_plans: Vec, - pub precompute_plan: PrecomputePlan, - pub transmission_plan: TransmissionPlan, - pub query_plan: QueryPlan, - pub storage_routing: Option, - #[serde(default)] - pub adaptation_evidence: Vec, -} - -/// A projected writer must refer to the query entry installed in the same -/// generation. -pub fn validate_maintenance_query_bindings( - precompute: &PrecomputePlan, - query: &QueryPlan, -) -> Result<(), String> { - for (query_id, installed) in &precompute.executable_dags { - installed.validate()?; - let mut entries = query - .entries - .values() - .filter(|entry| &entry.query_id == query_id); - let entry = entries - .next() - .ok_or("maintenance projection has no query entry")?; - if entries.next().is_some() { - return Err("maintenance projection has ambiguous query entries".into()); - } - if entry.root != installed.binding.query_plan_sink { - return Err("maintenance projection and query entry have different roots".into()); - } - let selected = query - .selected_dags - .get(query_id) - .ok_or("maintenance projection has no selected semantic provenance")?; - if selected.schema_version != crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION - || selected.query_id != *query_id - { - return Err("selected semantic provenance has invalid identity/version".into()); - } - selected.decode()?; - if installed - .document - .nodes - .iter() - .any(|node| !selected.nodes.contains(node)) - || installed - .document - .edges - .iter() - .any(|edge| !selected.edges.contains(edge)) - { - return Err("maintenance projection differs from its selected DAG".into()); - } - } - Ok(()) -} - -/// Every query read must target the installed writer's exact stored output and -/// physical window contract. The catalog describes semantics; these choices -/// belong to the executable plans. -pub fn validate_stored_output_references( - precompute: &PrecomputePlan, - query: &QueryPlan, -) -> Result<(), String> { - let writers = precompute - .schemas - .iter() - .map(|schema| (schema.materialization, schema)) - .collect::>(); - let configs = precompute - .materializations - .iter() - .map(|config| (config.policy_fingerprint().into(), config)) - .collect::>(); - for entry in query.entries.values() { - for binding in entry.materialization_bindings() { - let writer = writers.get(&binding.materialization).ok_or_else(|| { - format!( - "query binding for definition {} has no precompute stored-summary writer", - binding.materialization.as_u64() - ) - })?; - if binding.stored_output_reference != writer.stored_output_reference { - return Err( - "query read and precompute writer have different stored outputs".into(), - ); - } - let config = configs.get(&binding.materialization).ok_or_else(|| { - format!( - "query binding for definition {} has no precompute configuration", - binding.materialization.as_u64() - ) - })?; - let full_slide = matches!( - config.window_layout, - crate::WindowMaterializationLayout::FullWindow - ) - .then_some(config.slide_interval.saturating_mul(1_000)); - if binding.full_window_slide_ms != full_slide { - return Err("query full-window cadence differs from precompute definition".into()); - } - if config.stored_window_ms() != binding.window_ms { - return Err("query pane differs from precompute stored window".into()); - } - if config.pane_origin_ms != binding.pane_origin_ms { - return Err("query pane origin differs from precompute definition".into()); - } - } - } - Ok(()) -} - -impl PhysicalPlanPublication { - /// Validate every plan against the shared catalog snapshot. - pub fn validate(&self) -> Result<(), String> { - let catalog = &self.summary_catalog; - self.precompute_plan - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - self.transmission_plan - .validate(&self.precompute_plan) - .map_err(|e| e.to_string())?; - self.transmission_plan - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - self.query_plan - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - validate_maintenance_query_bindings(&self.precompute_plan, &self.query_plan)?; - validate_stored_output_references(&self.precompute_plan, &self.query_plan)?; - let mut collectors = std::collections::BTreeSet::new(); - for collector in &self.collector_plans { - if collector.envelope != self.precompute_plan.envelope - || !collectors.insert(&collector.collector_id) - { - return Err("collector envelope mismatch or duplicate collector".into()); - } - collector - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - for rule in &collector.transmission_rules { - if !self.transmission_plan.rules.contains(rule) { - return Err( - "collector transmission rule absent from published transmission plan" - .into(), - ); - } - } - } - for producer in &self.precompute_plan.producers { - let collector = self - .collector_plans - .iter() - .find(|c| c.collector_id == producer.collector_id) - .ok_or("precompute producer has no published CollectorPlan")?; - if !collector - .materializations - .iter() - .any(|m| m.materialization == producer.materialization) - { - return Err( - "collector does not produce referenced precompute materialization".into(), - ); - } - } - Ok(()) - } - - pub fn install_request( - &self, - storage_routing: Option, - adaptation_evidence: Vec, - ) -> Result { - self.validate()?; - Ok(PhysicalPlanInstallRequest { - summary_catalog: self.summary_catalog.clone(), - collector_plans: self.collector_plans.clone(), - precompute_plan: self.precompute_plan.clone(), - transmission_plan: self.transmission_plan.clone(), - query_plan: self.query_plan.clone(), - storage_routing, - adaptation_evidence, - }) - } -} diff --git a/crates/asap_types/src/policy_fingerprint.rs b/crates/asap_types/src/policy_fingerprint.rs deleted file mode 100644 index 42fa2e475..000000000 --- a/crates/asap_types/src/policy_fingerprint.rs +++ /dev/null @@ -1,504 +0,0 @@ -//! Legacy routing wrapper for a deployed stored output. -//! -//! An explicit `PrecomputeMaterialization::stored_output_id` takes precedence. -//! Otherwise the compiler allocates a deterministic default from the existing -//! policy fields (including pane layout and cadence). This identifier is not -//! semantic identity: `SummaryDefinitionId` hashes the versioned semantic -//! definition, and several deployed outputs may share that definition. -//! Catalog installation checks that an output is never assigned conflicting -//! computation or format contracts. - -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; -use xxhash_rust::xxh64::xxh64; - -use crate::aggregation_config::PrecomputeMaterialization; - -/// Routing handle for one deployed stored output. See the module contract. -#[derive( - Debug, Clone, Copy, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize, -)] -#[serde(transparent)] -pub struct PolicyFingerprint(pub u64); - -impl PolicyFingerprint { - /// Unset sentinel returned by `Default`. Sinks cannot resolve an unset policy; - /// construct valid fingerprints with [`Self::from_config`]. - pub const UNSET: PolicyFingerprint = PolicyFingerprint(0); - - /// True when this fingerprint is the [`Self::UNSET`] sentinel. - pub fn is_unset(self) -> bool { - self.0 == 0 - } -} - -impl PolicyFingerprint { - /// Compute the fingerprint of an [`PrecomputeMaterialization`]. - /// - /// Hash inputs are concatenated with `\0` byte separators and - /// canonicalized so that map/iteration order can't affect the - /// outcome. Parameter values are rendered via `serde_json::to_string` - /// for nested-shape determinism (matches the existing - /// `parameters_canonical` form used in `AggKind::ExactAgg`). - pub fn from_config(cfg: &PrecomputeMaterialization) -> Self { - if let Some(output) = cfg.stored_output_id { - return output.fingerprint(); - } - let mut buf: Vec = Vec::with_capacity(512); - - if !cfg.population_key_encoding.is_legacy() { - // UTF-8 raw metrics cannot alias this nonlegacy domain prefix. - buf.extend_from_slice(b"\xffpopulation-key-canonical-labels-v1\0"); - } - // 1. metric name - if cfg.derived_input.is_some() { - // Raw policies start with UTF-8 metric bytes; 0xff is impossible - // there, so a metric cannot impersonate this source domain. - buf.extend_from_slice(b"\xffderived-input-v1:"); - buf.extend_from_slice( - &serde_json::to_vec(&cfg.source_identity()).expect("typed source identity"), - ); - } else { - buf.extend_from_slice(cfg.metric.as_bytes()); - } - buf.push(0); - - // 2. aggregation_type (Serialize impl is the stable form) - buf.extend_from_slice( - serde_json::to_string(&cfg.aggregation_type) - .unwrap_or_default() - .as_bytes(), - ); - buf.push(0); - - // 3. aggregation_sub_type - buf.extend_from_slice(cfg.aggregation_sub_type.as_bytes()); - buf.push(0); - - // 4. parameters — canonicalized (sorted keys, JSON-rendered values) - let sorted: BTreeMap<&String, &serde_json::Value> = cfg.parameters.iter().collect(); - for (k, v) in sorted { - buf.extend_from_slice(k.as_bytes()); - buf.push(b'='); - buf.extend_from_slice(serde_json::to_string(v).unwrap_or_default().as_bytes()); - buf.push(b';'); - } - buf.push(0); - - if let Some(partitioning) = cfg.partitioning { - buf.extend_from_slice(format!("partition:{partitioning:?}\0").as_bytes()); - } - - // 5. grouping_labels (already sorted at construction per - // KeyByLabelNames invariant; encode as `,`-joined list) - for l in &cfg.grouping_labels.names() { - buf.extend_from_slice(l.as_bytes()); - buf.push(b','); - } - buf.push(0); - - if cfg.table_name.is_some() && !cfg.grouping_labels.is_empty() { - buf.extend_from_slice( - crate::grouping_projection::TABLE_GROUP_OBSERVATION_SEMANTICS.as_bytes(), - ); - buf.push(0); - } - if !cfg.grouping_labels.is_legacy_labels() { - buf.extend_from_slice(b"typed-grouping:"); - buf.extend_from_slice( - serde_json::to_string(&cfg.grouping_labels) - .expect("group projection serializes") - .as_bytes(), - ); - buf.push(0); - } - - // 6. aggregated_labels - for l in &cfg.aggregated_labels.labels { - buf.extend_from_slice(l.as_bytes()); - buf.push(b','); - } - buf.push(0); - - // 7. rollup_labels - for l in &cfg.rollup_labels.labels { - buf.extend_from_slice(l.as_bytes()); - buf.push(b','); - } - buf.push(0); - - // 8. window_size + slide_interval + window_type (cadence) - buf.extend_from_slice(&cfg.window_size.to_le_bytes()); - buf.push(0); - buf.extend_from_slice(&cfg.slide_interval.to_le_bytes()); - buf.push(0); - buf.extend_from_slice( - serde_json::to_string(&cfg.window_type) - .unwrap_or_default() - .as_bytes(), - ); - buf.push(0); - - // Physical representation is part of state identity. A full overlapping - // window and a mergeable pane layout may share semantic descriptors but - // never share payload instances or lifecycle accounting. - buf.extend_from_slice( - serde_json::to_string(&cfg.window_layout) - .unwrap_or_default() - .as_bytes(), - ); - buf.push(0); - - // 9. pane origin. Presence is explicit so a legacy definition with - // unknown phase cannot alias an epoch-aligned definition. - match cfg.pane_origin_ms { - Some(origin) => { - buf.push(1); - buf.extend_from_slice(&origin.to_le_bytes()); - } - None => buf.push(0), - } - buf.push(0); - - // 10. spatial_filter_normalized — canonicalized predicate - buf.extend_from_slice(cfg.spatial_filter_normalized.as_bytes()); - if let Some(table) = &cfg.table_name { - buf.extend_from_slice(b"\0sql-source-v1\0"); - buf.extend_from_slice(table.as_bytes()); - buf.push(0); - if let Some(column) = cfg.effective_value_projection().column() { - buf.extend_from_slice(column.as_bytes()); - } - if matches!( - cfg.effective_value_projection(), - crate::sds::ValueProjectionIdentity::Constant { .. } - ) { - buf.extend_from_slice(b"\0constant-projection-v1\0"); - buf.extend_from_slice( - serde_json::to_string(cfg.effective_value_projection()) - .expect("finite validated projection serializes") - .as_bytes(), - ); - } - } - if let Some(population) = &cfg.table_population { - let canonical = population.canonical(); - if !canonical.is_empty() { - buf.push(0); - buf.extend_from_slice(canonical.as_bytes()); - } - } - if let Some(column) = &cfg.table_timestamp_column { - buf.extend_from_slice(b"\0timestamp-ms\0"); - buf.extend_from_slice(column.as_bytes()); - } - - Self(xxh64(&buf, 0)) - } - - /// The raw u64. Use sparingly — prefer comparing `PolicyFingerprint` - /// values directly. - pub fn as_u64(self) -> u64 { - self.0 - } -} - -impl std::fmt::Display for PolicyFingerprint { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // Hex form so logs distinguish a fingerprint from a decimal - // counter id at a glance. - write!(f, "policy_fp:{:016x}", self.0) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::enums::WindowKind; - use crate::AggregationType; - use crate::KeyByLabelNames; - use std::collections::HashMap; - - fn cfg( - metric: &str, - agg_type: AggregationType, - params: HashMap, - group_by: Vec<&str>, - window_size: u64, - spatial_filter: &str, - ) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - agg_type, - String::new(), - params, - KeyByLabelNames::new(group_by.into_iter().map(|s| s.to_string()).collect()), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - window_size, - window_size, - WindowKind::Tumbling, - spatial_filter.to_string(), - metric.to_string(), - None, - None, - None, - ) - } - - #[test] - fn population_encoding_preserves_legacy_wire_and_separates_identity() { - use crate::grouping_projection::PopulationKeyEncoding; - let legacy = cfg( - "m", - AggregationType::Sum, - HashMap::new(), - vec!["host"], - 60, - "", - ); - let wire = serde_json::to_value(&legacy).unwrap(); - assert!(wire.get("population_key_encoding").is_none()); - let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); - assert!(decoded.population_key_encoding.is_legacy()); - assert_eq!(legacy.policy_fingerprint(), decoded.policy_fingerprint()); - let mut canonical = legacy.clone(); - canonical.population_key_encoding = PopulationKeyEncoding::CanonicalLabelsV1; - assert_ne!(legacy.policy_fingerprint(), canonical.policy_fingerprint()); - let wire = serde_json::to_value(&canonical).unwrap(); - assert_eq!(wire["population_key_encoding"], "canonical_labels_v1"); - let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); - assert_eq!(decoded.policy_fingerprint(), canonical.policy_fingerprint()); - } - - #[test] - fn same_config_yields_same_fingerprint() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec!["zone"], - 60, - "", - ); - let b = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec!["zone"], - 60, - "", - ); - assert_eq!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - #[test] - fn physical_window_layout_is_part_of_state_identity() { - let panes = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec!["zone"], - 60, - "", - ); - let mut full = panes.clone(); - full.window_layout = crate::WindowMaterializationLayout::FullWindow; - assert_ne!( - PolicyFingerprint::from_config(&panes), - PolicyFingerprint::from_config(&full) - ); - } - - #[test] - fn different_metric_yields_different_fingerprint() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - let b = cfg( - "cpu_pct", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - assert_ne!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - #[test] - fn different_window_yields_different_fingerprint() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - let b = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 300, - "", - ); - assert_ne!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - #[test] - fn different_spatial_filter_yields_different_fingerprint() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - let b = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - r#"status="200""#, - ); - assert_ne!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - #[test] - fn different_group_by_yields_different_fingerprint() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec!["zone"], - 60, - "", - ); - let b = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec!["zone", "service"], - 60, - "", - ); - assert_ne!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - /// Pre-PR-5 the `aggregation_id` field on `PrecomputeMaterialization` was - /// excluded from the fingerprint hash. PR 5 deletes the field - /// entirely — identity *is* the fingerprint — so this is now - /// vacuously true. Kept as a doc-comment anchor; no runtime test - /// is needed. - #[test] - fn policy_fp_u64_accessor_equals_fingerprint_u64() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - assert_eq!( - a.policy_fp_u64(), - PolicyFingerprint::from_config(&a).as_u64(), - ); - } - - #[test] - fn num_aggregates_to_retain_does_not_affect_fingerprint() { - let mut a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - let mut b = a.clone(); - a.num_aggregates_to_retain = Some(100); - b.num_aggregates_to_retain = Some(500); - assert_eq!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b), - "retention is a separate concern from policy identity" - ); - } - - #[test] - fn parameter_map_order_does_not_affect_fingerprint() { - // HashMap iteration order is non-deterministic; the fingerprint - // must be order-independent. - let mut p1 = HashMap::new(); - p1.insert("a".into(), serde_json::json!(1)); - p1.insert("b".into(), serde_json::json!(2)); - let mut p2 = HashMap::new(); - p2.insert("b".into(), serde_json::json!(2)); - p2.insert("a".into(), serde_json::json!(1)); - let a = cfg("http_lat", AggregationType::Sum, p1, vec![], 60, ""); - let b = cfg("http_lat", AggregationType::Sum, p2, vec![], 60, ""); - assert_eq!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - #[test] - fn spatial_filter_canonicalization_drives_fingerprint() { - // Two filters that differ only in matcher ordering produce the - // SAME normalized form, hence the SAME fingerprint. The - // canonicalization step in `PrecomputeMaterialization::new` (via - // `normalize_spatial_filter`) sorts matchers by key. - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - r#"status="200",zone="us""#, - ); - let b = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - r#"zone="us",status="200""#, - ); - assert_eq!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - #[test] - fn display_format_is_hex_with_prefix() { - let fp = PolicyFingerprint(0xdeadbeef); - assert_eq!(format!("{}", fp), "policy_fp:00000000deadbeef"); - } -} diff --git a/crates/asap_types/src/policy_registry.rs b/crates/asap_types/src/policy_registry.rs deleted file mode 100644 index 90138b139..000000000 --- a/crates/asap_types/src/policy_registry.rs +++ /dev/null @@ -1,169 +0,0 @@ -//! Content-addressed policy registry. -//! -//! Derived view over a collection of `PrecomputeMaterialization`s that maps -//! [`PolicyFingerprint`] → [`PrecomputeMaterialization`]. This is the -//! merged-sid-identity-chain replacement for the controller-allocated -//! `aggregation_id`-keyed `HashMap` that `data_plane`'s `InstalledPrecomputePlan` -//! carries (see `data_plane::storage_engines::types::installed_precompute_plan`'s -//! module doc for why that type lives there, not here). -//! -//! ## Dual-keyed transition -//! -//! PR 3 (where this lives): the registry exists alongside the -//! `aggregation_id`-keyed map. Callers can opt into either lookup. -//! Construction is `O(N)` over the source configs; computation is -//! pure (no I/O, no mutation of the source). -//! -//! Subsequent PRs migrate one set of callers at a time off -//! `aggregation_id` → `PolicyFingerprint`, until the final PR can -//! delete the legacy index. -//! -//! ## Identity invariants -//! -//! Two `PrecomputeMaterialization`s that produce the same `PolicyFingerprint` -//! ARE the same policy. The registry treats this as a *deduplication* -//! invariant — if two distinct entries in the source `materializations_by_output` -//! map produce the same fingerprint, the later one wins (last-write -//! semantics). In practice the source should never contain duplicates; -//! if it does, that's a control-plane bug worth surfacing in telemetry -//! (see [`PolicyRegistry::from_configs_with_collisions`]). - -use std::collections::HashMap; - -use crate::aggregation_config::PrecomputeMaterialization; -use crate::policy_fingerprint::PolicyFingerprint; - -/// Content-addressed lookup table for active aggregation policies. -#[derive(Debug, Clone, Default)] -pub struct PolicyRegistry { - policies: HashMap, -} - -impl PolicyRegistry { - /// Construct from a list of configs. Duplicates (same fingerprint) - /// collapse to the last entry; use - /// [`Self::from_configs_with_collisions`] when you want to detect - /// them. - pub fn from_configs(configs: I) -> Self - where - I: IntoIterator, - { - let mut policies = HashMap::new(); - for cfg in configs { - policies.insert(PolicyFingerprint::from_config(&cfg), cfg); - } - Self { policies } - } - - #[cfg(test)] - /// Construct + report the count of duplicate fingerprints (entries - /// where the source contained two configs producing the same - /// fingerprint and the later one displaced the earlier). Zero in - /// the well-formed case; non-zero is a control-plane bug worth - /// surfacing. - pub fn from_configs_with_collisions(configs: I) -> (Self, usize) - where - I: IntoIterator, - { - let mut policies = HashMap::new(); - let mut collisions = 0usize; - for cfg in configs { - let fp = PolicyFingerprint::from_config(&cfg); - if policies.insert(fp, cfg).is_some() { - collisions += 1; - } - } - (Self { policies }, collisions) - } - - /// Look up the config for a fingerprint. - pub fn get(&self, fp: PolicyFingerprint) -> Option<&PrecomputeMaterialization> { - self.policies.get(&fp) - } - - /// Iterate fingerprint → config pairs. - pub fn iter(&self) -> impl Iterator { - self.policies.iter() - } - - /// Live policy count. - pub fn len(&self) -> usize { - self.policies.len() - } - - pub fn is_empty(&self) -> bool { - self.policies.is_empty() - } - - /// All fingerprints currently registered. Useful for diffing two - /// registries during a hot-reload swap. - pub fn fingerprints(&self) -> impl Iterator + '_ { - self.policies.keys().copied() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::enums::WindowKind; - use crate::AggregationType; - use crate::KeyByLabelNames; - use std::collections::HashMap as StdHashMap; - - fn cfg(_id: u64, metric: &str) -> PrecomputeMaterialization { - // `_id` is unused after PR 5 — identity is derived from - // content. Kept as a parameter so existing call sites in the - // tests below don't churn. - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - StdHashMap::new(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ) - } - - #[test] - fn from_configs_round_trips_lookup() { - let a = cfg(1, "http_lat"); - let b = cfg(2, "cpu_pct"); - let fp_a = PolicyFingerprint::from_config(&a); - let fp_b = PolicyFingerprint::from_config(&b); - let reg = PolicyRegistry::from_configs(vec![a.clone(), b.clone()]); - assert_eq!(reg.len(), 2); - assert_eq!(reg.get(fp_a).unwrap().metric, "http_lat"); - assert_eq!(reg.get(fp_b).unwrap().metric, "cpu_pct"); - } - - #[test] - fn identical_configs_collapse_to_one_entry() { - // PR 5: identity IS the content. Two configs that are - // byte-for-byte identical on the policy-relevant fields - // collapse to ONE entry — there's no way to distinguish them - // anymore. - let a = cfg(1, "http_lat"); - let b = cfg(99, "http_lat"); - let (reg, collisions) = PolicyRegistry::from_configs_with_collisions(vec![a, b]); - assert_eq!(reg.len(), 1); - assert_eq!(collisions, 1); - } - - #[test] - fn distinct_policies_keep_distinct_entries() { - let a = cfg(1, "http_lat"); - let b = cfg(1, "cpu_pct"); // different metric → distinct policies - let (reg, collisions) = PolicyRegistry::from_configs_with_collisions(vec![a, b]); - assert_eq!(reg.len(), 2); - assert_eq!(collisions, 0); - } -} diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs deleted file mode 100644 index 05227b679..000000000 --- a/crates/asap_types/src/precompute_plan.rs +++ /dev/null @@ -1,1161 +0,0 @@ -//! Shared precompute installation contract and catalog consistency checks. -//! Compilation chooses these values; runtime consumers validate the same DTO. - -mod catalog; - -use planner_types::post_asap::{SketchAlgorithm, SketchParams, SummaryFamilyType}; -use planner_types::pre_asap::Source; -use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, BTreeSet, HashMap}; -use thiserror::Error; - -/// Check the initial executable maintenance population reductions against the -/// installed configuration. Indexed reductions need authoritative column mapping -/// and are not admitted by this bounded capability check. -pub fn validate_maintenance_reduction( - config: &crate::PrecomputeMaterialization, - node: &planner_types::post_asap::ExecutableDagNode, -) -> Result<(), String> { - use crate::sds::PopulationPartitioning; - use planner_types::{post_asap::ExecutableOperatorPayload, pre_asap::Reduction}; - match &node.payload { - ExecutableOperatorPayload::SummaryAgg { - reduction: Reduction::PerEntity, - .. - } if config.partitioning == Some(PopulationPartitioning::PerEntity) => Ok(()), - ExecutableOperatorPayload::SummaryAgg { - reduction: Reduction::Reduce(keys), - .. - } if keys.is_empty() - && config.partitioning == Some(PopulationPartitioning::Grouped) - && config.grouping_labels.is_empty() => - { - Ok(()) - } - _ => Err( - "maintenance reduction does not match supported configured population grouping".into(), - ), - } -} - -/// Validate the physical windows consumed by one derived input program. -/// This returns the existing source definitions, not a second serialized -/// contract. It does not prove completion or authorize runtime execution. -/// Full-window sliding is lossless here; pane merging requires a separate -/// explicit operator and is deliberately not inferred from window sizes. -pub fn validated_source_window_cohort<'a>( - target: &crate::PrecomputeMaterialization, - sources: &[&'a crate::PrecomputeMaterialization], -) -> Result, PrecomputePlanError> { - let invalid = || { - PrecomputePlanError::CatalogContract( - "derived inputs require matching explicit full stored windows".into(), - ) - }; - let full_ms = target.window_size.checked_mul(1000).ok_or_else(invalid)?; - if sources.is_empty() - || full_ms == 0 - || target.slide_interval == 0 - || target.slide_interval > target.window_size - || (target.slide_interval < target.window_size && target.pane_origin_ms.is_none()) - || target.stored_window_ms() != full_ms - || (target.slide_interval < target.window_size - && !matches!( - target.window_layout, - crate::WindowMaterializationLayout::FullWindow - )) - { - return Err(invalid()); - } - let mut identities = BTreeSet::new(); - for source in sources { - if source.derived_input.is_some() - || !identities.insert(source.policy_fingerprint()) - || source.window_size != target.window_size - || source.slide_interval != target.slide_interval - || source.pane_origin_ms != target.pane_origin_ms - || source.stored_window_ms() != full_ms - || (source.slide_interval < source.window_size - && !matches!( - source.window_layout, - crate::WindowMaterializationLayout::FullWindow - )) - { - return Err(invalid()); - } - } - Ok(sources.to_vec()) -} - -pub const BACKEND_COMPAT: &str = "asap-query-backend.v1"; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct PlanEnvelope { - pub plan_id: u64, - pub plan_version: u64, - pub generated_at_unix_ms: u64, - pub activation_unix_ms: u64, - pub expiry_unix_ms: Option, - pub backend_compat: String, - pub planner_revision: String, - pub capability_snapshot_id: String, -} - -/// DAG-format precompute installation. Planner node payloads and dependency -/// edges define execution; materializations attach storage/window placement. -/// Raw source-to-SummaryAgg paths lower to streaming kernels. Derived paths -/// execute through the maintenance DAG scheduler at stored-state frontiers. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PrecomputePlan { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub summary_catalog: Option, - pub envelope: PlanEnvelope, - pub ingest: IngestContract, - pub schemas: Vec, - pub producers: Vec, - pub materializations: Vec, - /// Maintenance projections ending at stored outputs. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub executable_dags: BTreeMap, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum IngestProtocol { - ModifiedOtlpMetricsV1, - PrometheusRemoteWriteV1, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum TimestampUnit { - UnixNanoseconds, - UnixMilliseconds, -} - -/// Backend ingress semantics installed with the precompute projection. This -/// replaces implicit knowledge formerly hidden in the streaming-config path. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct IngestContract { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub dataset_identity: Option, - pub protocol: IngestProtocol, - pub endpoint_path: String, - pub timestamp_unit: TimestampUnit, - pub require_plan_identity: bool, - pub require_stored_output_identity: bool, - pub require_registered_producer: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -#[serde(rename_all = "snake_case")] -pub enum StateEncoding { - SketchlibProtobufV1, - SketchCoreMsgpackV1, - ExactAccumulatorV1, - /// Persisted backend state with explicit Planner family and population layout. - PlannerExactAccumulatorV1, - ExactCounterAccumulatorV2, -} - -/// Serializable physical state identity derived from Planner's canonical -/// summary family. This is a wire DTO, not a second planning algebra. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "family", rename_all = "snake_case", deny_unknown_fields)] -pub enum StateFamilyContract { - Exact { - kind: ExactStateKind, - }, - Sketch { - algorithm: SketchAlgorithm, - parameters: SketchParams, - }, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum ExactStateKind { - Sum, - Count, - /// Exact minimum. Distinct state from [`ExactStateKind::Max`] -- a - /// stored minimum cannot answer a maximum query, so the two must - /// never share a content address. - Min, - /// Exact maximum. Planner still spells this `ExactKind::Max` for - /// historical reasons; it is a maximum accumulator. - Max, - Increase, - Rate, - IRate, -} - -impl TryFrom<&SummaryFamilyType> for StateFamilyContract { - type Error = (); - - fn try_from(family: &SummaryFamilyType) -> Result { - use planner_types::post_asap::ExactKind; - Ok(match family { - SummaryFamilyType::ExactAggregate(kind, _) => Self::Exact { - kind: match kind { - ExactKind::Sum => ExactStateKind::Sum, - ExactKind::Count => ExactStateKind::Count, - ExactKind::Max => ExactStateKind::Max, - ExactKind::Min => ExactStateKind::Min, - ExactKind::Increase => ExactStateKind::Increase, - ExactKind::Rate => ExactStateKind::Rate, - ExactKind::IRate => ExactStateKind::IRate, - }, - }, - SummaryFamilyType::Sketch(kind, _) => Self::Sketch { - algorithm: kind.algorithm().clone(), - parameters: kind.params().clone(), - }, - _ => return Err(()), - }) - } -} - -/// Decoder/schema contract for one content-addressed materialization. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct StateSchemaContract { - pub stored_output_reference: crate::sds::StoredOutputReference, - pub schema_id: String, - pub schema_version: u32, - pub materialization: crate::sds::StoredOutputId, - pub family: StateFamilyContract, - pub source: Source, - pub value_projection: crate::sds::ValueProjectionIdentity, - pub group_by: crate::GroupingProjection, - pub window: StateWindowContract, - pub encodings: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct StateWindowContract { - pub kind: crate::WindowKind, - pub size_ms: u64, - pub slide_ms: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub pane_origin_ms: Option, -} - -/// A collector authorized to produce state for one materialization. Runtime -/// producer epochs and frame sequences belong to TransmissionPlan. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -#[serde(deny_unknown_fields)] -pub struct ProducerContract { - pub producer_id: String, - pub collector_id: String, - pub materialization: crate::sds::StoredOutputId, - pub schema_id: String, - /// Authoritative partitions for completion barriers. An explicitly empty - /// roster cannot authorize completion claims. - pub partition_ids: BTreeSet, -} - -#[derive(Debug, Error, PartialEq, Eq)] -pub enum PrecomputePlanError { - #[error("invalid precompute catalog contract: {0}")] - CatalogContract(String), - #[error("PrecomputePlan envelope does not match its SummaryCatalog identity/lifecycle")] - PlanIdentityMismatch, - #[error("unsupported precompute ingest protocol/endpoint/identity contract")] - UnsupportedIngestEndpoint, - #[error("duplicate materialization {0}")] - DuplicateMaterialization(u64), - #[error("schema set does not exactly match the materialization set")] - SchemaSetMismatch, - #[error("schema {schema_id} has invalid version or no encoding")] - InvalidSchema { schema_id: String }, - #[error("materialization {0} uses a summary family unsupported by the runtime schema")] - UnsupportedFamily(u64), - #[error("materialization {materialization} has an invalid window layout: {reason}")] - InvalidWindowLayout { - materialization: u64, - reason: String, - }, - #[error("producer {producer_id} references an unknown materialization or schema")] - InvalidProducer { producer_id: String }, - #[error("materialization {0} has no registered producer")] - MissingProducer(u64), - #[error("duplicate producer binding {0}")] - DuplicateProducer(String), -} - -impl PrecomputePlan { - pub fn build( - envelope: PlanEnvelope, - materializations: Vec, - producer_ids: &[String], - ) -> Result { - Self::build_complete( - envelope, - materializations, - producer_ids, - false, - BTreeMap::new(), - ) - } - - fn build_complete( - envelope: PlanEnvelope, - materializations: Vec, - producer_ids: &[String], - backend_local: bool, - executable_dags: BTreeMap, - ) -> Result { - let schemas = materializations - .iter() - .map(|materialization| { - let fingerprint = materialization.policy_fingerprint(); - let accumulator = materialization - .accumulator_spec() - .map_err(|_| PrecomputePlanError::UnsupportedFamily(fingerprint.0))?; - let family = StateFamilyContract::try_from(&accumulator.family) - .map_err(|_| PrecomputePlanError::UnsupportedFamily(fingerprint.0))?; - let source = materialization.table_name.as_ref().map_or_else( - || Source::TimeSeries { - metric: materialization.metric.clone(), - }, - |table_ref| Source::Table { - table_ref: table_ref.clone(), - }, - ); - let value_projection = materialization.effective_value_projection().clone(); - Ok(StateSchemaContract { - stored_output_reference: crate::sds::StoredOutputReference::for_output( - fingerprint.into(), - ), - schema_id: state_schema_id(fingerprint), - schema_version: 1, - materialization: fingerprint.into(), - family, - source, - value_projection, - group_by: materialization.grouping_labels.clone(), - window: StateWindowContract { - kind: materialization.window_type, - size_ms: materialization.window_size.saturating_mul(1_000), - slide_ms: match materialization.window_type { - crate::WindowKind::Tumbling => None, - crate::WindowKind::Sliding => { - Some(materialization.slide_interval.saturating_mul(1_000)) - } - crate::WindowKind::Session => None, - }, - pane_origin_ms: materialization.pane_origin_ms, - }, - encodings: state_encodings(&accumulator.family), - }) - }) - .collect::, PrecomputePlanError>>()?; - let producers = producer_ids - .iter() - .flat_map(|producer_id| { - schemas.iter().map(move |schema| ProducerContract { - producer_id: producer_id.clone(), - collector_id: producer_id.clone(), - materialization: schema.materialization, - schema_id: schema.schema_id.clone(), - partition_ids: BTreeSet::new(), - }) - }) - .collect(); - let dataset_identity = materializations - .iter() - .filter_map(|m| m.semantic_fragment.as_ref()?.dataset_identity.clone()) - .next(); - let plan = Self { - summary_catalog: None, - envelope, - ingest: if backend_local { - IngestContract { - dataset_identity: dataset_identity.clone(), - protocol: IngestProtocol::PrometheusRemoteWriteV1, - endpoint_path: "/api/v1/write".into(), - timestamp_unit: TimestampUnit::UnixMilliseconds, - require_plan_identity: false, - require_stored_output_identity: false, - require_registered_producer: false, - } - } else { - IngestContract { - dataset_identity: dataset_identity.clone(), - protocol: IngestProtocol::ModifiedOtlpMetricsV1, - endpoint_path: "/v1/metrics".into(), - timestamp_unit: TimestampUnit::UnixNanoseconds, - require_plan_identity: true, - require_stored_output_identity: true, - require_registered_producer: true, - } - }, - schemas, - producers, - materializations, - executable_dags, - }; - plan.validate()?; - Ok(plan) - } - - /// Build the backend-local projection used when raw Prometheus samples - /// are precomputed inside ASAPQuery rather than by ASAPCollector. - pub fn build_backend_local( - envelope: PlanEnvelope, - materializations: Vec, - ) -> Result { - Self::build_backend_local_with_dags(envelope, materializations, BTreeMap::new()) - } - - /// Construct the complete backend-local contract before validation. Derived - /// definitions are never validated without their actual executable bindings. - pub fn build_backend_local_with_dags( - envelope: PlanEnvelope, - materializations: Vec, - executable_dags: BTreeMap, - ) -> Result { - Self::build_complete(envelope, materializations, &[], true, executable_dags) - } - - pub fn runtime_materializations( - &self, - ) -> Result< - HashMap, - PrecomputePlanError, - > { - self.validate()?; - Ok(self - .materializations - .iter() - .cloned() - .map(|materialization| { - ( - crate::sds::StoredOutputId(materialization.policy_fp_u64()), - materialization, - ) - }) - .collect()) - } - - /// Validate a barrier's installed scope, not authentication, durability or - /// monotonic progress. The runtime must enforce those before accepting it. - pub fn validate_watermark_scope( - &self, - materialization: crate::sds::StoredOutputId, - barrier: &crate::sds::SummaryWatermarkBarrier, - ) -> Result<(), PrecomputePlanError> { - self.validate()?; - barrier - .validate() - .map_err(|error| PrecomputePlanError::CatalogContract(error.to_string()))?; - if self.summary_catalog.as_ref() != Some(&barrier.catalog_generation) - || self.envelope.plan_id != barrier.catalog_generation.plan_id - || self.envelope.plan_version != barrier.catalog_generation.plan_version - { - return Err(PrecomputePlanError::CatalogContract( - "watermark catalog generation does not match installed plan".into(), - )); - } - if !self.producers.iter().any(|producer| { - producer.materialization == materialization - && producer.producer_id == barrier.source.producer_id - && producer - .partition_ids - .contains(&barrier.source.partition_id) - }) { - return Err(PrecomputePlanError::CatalogContract( - "watermark source is outside the authoritative partition roster".into(), - )); - } - Ok(()) - } - - pub fn validate(&self) -> Result<(), PrecomputePlanError> { - if let Some(dataset) = &self.ingest.dataset_identity { - dataset - .validate() - .map_err(PrecomputePlanError::CatalogContract)?; - } - for materialization in &self.materializations { - if let Some(fragment) = &materialization.semantic_fragment { - fragment - .validate() - .map_err(PrecomputePlanError::CatalogContract)?; - if fragment.dataset_identity != self.ingest.dataset_identity { - return Err(PrecomputePlanError::CatalogContract( - "semantic dataset differs from installed input binding".into(), - )); - } - } - } - let valid_ingest = match self.ingest.protocol { - IngestProtocol::ModifiedOtlpMetricsV1 => { - self.ingest.endpoint_path == "/v1/metrics" - && self.ingest.timestamp_unit == TimestampUnit::UnixNanoseconds - && self.ingest.require_plan_identity - && self.ingest.require_stored_output_identity - && self.ingest.require_registered_producer - } - IngestProtocol::PrometheusRemoteWriteV1 => { - self.ingest.endpoint_path == "/api/v1/write" - && self.ingest.timestamp_unit == TimestampUnit::UnixMilliseconds - && !self.ingest.require_plan_identity - && !self.ingest.require_stored_output_identity - && !self.ingest.require_registered_producer - } - }; - if !valid_ingest { - return Err(PrecomputePlanError::UnsupportedIngestEndpoint); - } - let canonical_cohort_members: BTreeSet<_> = self - .materializations - .iter() - .filter(|config| !config.population_key_encoding.is_legacy()) - .filter_map(|config| config.derived_input.as_ref().map(|input| (config, input))) - .flat_map(|(config, input)| { - std::iter::once(config.policy_fingerprint().into()) - .chain(input.inputs.iter().copied()) - }) - .collect(); - for config in &self.materializations { - if !config.population_key_encoding.is_legacy() - && (self.ingest.protocol != IngestProtocol::PrometheusRemoteWriteV1 - || !canonical_cohort_members.contains(&config.policy_fingerprint().into())) - { - return Err(PrecomputePlanError::CatalogContract( - "population key encoding is not supported by the installed runtime".into(), - )); - } - let Some(derived) = &config.derived_input else { - continue; - }; - let invalid = || { - PrecomputePlanError::CatalogContract( - "derived summary input requires an installed aligned immutable maintenance DAG" - .into(), - ) - }; - if self.ingest.protocol != IngestProtocol::PrometheusRemoteWriteV1 - || derived.inputs.is_empty() - { - return Err(invalid()); - } - let sources = derived - .inputs - .iter() - .map(|id| { - self.materializations - .iter() - .find(|candidate| candidate.policy_fingerprint() == id.fingerprint()) - .ok_or_else(invalid) - }) - .collect::, _>>()?; - if !config.population_key_encoding.is_legacy() { - if config.partitioning != Some(crate::sds::PopulationPartitioning::Grouped) - || !config.grouping_labels.is_empty() - || sources.iter().any(|source| { - source.population_key_encoding != config.population_key_encoding - }) - { - return Err(invalid()); - } - } else if sources - .iter() - .any(|source| !source.population_key_encoding.is_legacy()) - { - return Err(invalid()); - } - validated_source_window_cohort(config, &sources)?; - if sources - .iter() - .any(|source| !matches!(source.aggregation_type, crate::AggregationType::Sum)) - { - return Err(invalid()); - } - if config.window_size != config.slide_interval { - return Err(invalid()); - } - let mut matched = false; - for installed in self.executable_dags.values() { - let dag = installed - .document - .decode() - .map_err(PrecomputePlanError::CatalogContract)?; - for sink in &installed.binding.precompute_sinks { - if !matches!(installed.binding.node(*sink), - Some(crate::executable_plan::BackendNodeBinding::Materialization { stored_output }) - if stored_output.fingerprint() == config.policy_fingerprint()) - { - continue; - } - let target_node = dag - .nodes - .iter() - .find(|node| node.id == *sink) - .ok_or_else(invalid)?; - validate_maintenance_reduction(config, target_node) - .map_err(PrecomputePlanError::CatalogContract)?; - let inputs: Vec<_> = dag - .edges - .iter() - .filter(|edge| edge.consumer == *sink) - .collect(); - let [edge] = inputs.as_slice() else { - return Err(invalid()); - }; - let frontiers = installed - .binding - .nodes - .iter() - .filter_map(|(node, binding)| match binding { - crate::executable_plan::BackendNodeBinding::Materialization { - stored_output, - } if derived.inputs.contains(stored_output) => { - Some((*node, *stored_output)) - } - _ => None, - }) - .collect(); - let actual = crate::derived_input::DerivedInputIdentity::from_dag( - &installed.document, - edge.producer, - &frontiers, - ) - .map_err(PrecomputePlanError::CatalogContract)?; - if &actual != derived { - return Err(invalid()); - } - let mut pending = vec![edge.producer]; - let mut visited = BTreeSet::new(); - while let Some(id) = pending.pop() { - if !visited.insert(id) { - continue; - } - let node = dag - .nodes - .iter() - .find(|node| node.id == id) - .ok_or_else(invalid)?; - let children: Vec<_> = dag - .edges - .iter() - .filter(|edge| edge.consumer == id) - .collect(); - use planner_types::post_asap::{ - ExecutableOperatorPayload as Payload, ValueOperation, - }; - if node.output_state - != planner_types::post_asap::ExecutionDataState::INGESTION_ROWS - { - return Err(invalid()); - } - match &node.payload { - Payload::Value { - operation: ValueOperation::FinalizeExactAccumulator, - } if children.len() == 1 - && frontiers.contains_key(&children[0].producer) => {} - Payload::Binary { operator } - if children.len() == 2 - && children - .iter() - .filter(|edge| { - edge.role == planner_types::post_asap::EdgeRole::Left - }) - .count() - == 1 - && children - .iter() - .filter(|edge| { - edge.role == planner_types::post_asap::EdgeRole::Right - }) - .count() - == 1 - && operator.vector_match.is_none() - && matches!( - operator.kind, - planner_types::pre_asap::BinaryOpKind::Arithmetic(_) - ) => - { - pending.extend(children.iter().map(|edge| edge.producer)); - } - _ => return Err(invalid()), - } - } - matched = true; - } - } - if !matched { - return Err(invalid()); - } - } - for (query_id, installed) in &self.executable_dags { - if query_id != &installed.document.query_id { - return Err(PrecomputePlanError::CatalogContract( - "post-ASAP DAG map key differs from document query ID".into(), - )); - } - installed - .validate() - .map_err(PrecomputePlanError::CatalogContract)?; - let dag = installed - .document - .decode() - .map_err(PrecomputePlanError::CatalogContract)?; - for node in &dag.nodes { - let Some(crate::executable_plan::BackendNodeBinding::Materialization { - stored_output, - }) = installed.binding.node(node.id) - else { - continue; - }; - let Some(config) = self - .materializations - .iter() - .find(|config| config.policy_fingerprint() == stored_output.fingerprint()) - else { - return Err(PrecomputePlanError::CatalogContract( - "DAG materialization has no runtime configuration".into(), - )); - }; - if self.ingest.protocol == IngestProtocol::PrometheusRemoteWriteV1 - && matches!( - config.aggregation_type, - crate::AggregationType::HLL | crate::AggregationType::UnivMon - ) - { - if let planner_types::post_asap::ExecutableOperatorPayload::SummaryAgg { - input, - .. - } = &node.payload - { - let supported = match config.aggregation_type { - crate::AggregationType::HLL => { - crate::accumulator_spec::is_scalar_sample_value(input) - || crate::accumulator_spec::is_unit_sample_frequency(input) - } - crate::AggregationType::UnivMon => { - crate::accumulator_spec::is_unit_sample_frequency(input) - } - _ => unreachable!(), - }; - if !supported { - return Err(PrecomputePlanError::CatalogContract( - "raw materialization input does not match its accumulator update semantics".into(), - )); - } - } - } - if let Some(partitioning) = config.partitioning { - if let planner_types::post_asap::ExecutableOperatorPayload::SummaryAgg { - reduction, - .. - } = &node.payload - { - let expected = match reduction { - planner_types::pre_asap::Reduction::PerEntity => { - crate::sds::PopulationPartitioning::PerEntity - } - planner_types::pre_asap::Reduction::Reduce(_) => { - crate::sds::PopulationPartitioning::Grouped - } - }; - if partitioning != expected { - return Err(PrecomputePlanError::CatalogContract( - "runtime population partition disagrees with Planner reduction" - .into(), - )); - } - } - } - } - } - let mut materializations = BTreeSet::new(); - for materialization in &self.materializations { - if materialization.table_name.is_none() - && !materialization.grouping_labels.is_legacy_labels() - { - return Err(PrecomputePlanError::CatalogContract( - "time-series grouping requires non-null string labels".into(), - )); - } - - materialization - .grouping_labels - .validate() - .map_err(PrecomputePlanError::CatalogContract)?; - materialization - .window_layout - .validate(materialization.window_size, materialization.slide_interval) - .map_err(|reason| PrecomputePlanError::InvalidWindowLayout { - materialization: materialization.policy_fp_u64(), - reason, - })?; - let expected_kind = if materialization.slide_interval == materialization.window_size { - crate::WindowKind::Tumbling - } else { - crate::WindowKind::Sliding - }; - if materialization.window_type != expected_kind { - return Err(PrecomputePlanError::InvalidWindowLayout { - materialization: materialization.policy_fp_u64(), - reason: "window kind disagrees with size and slide".into(), - }); - } - if !materializations.insert(materialization.policy_fingerprint().into()) { - return Err(PrecomputePlanError::DuplicateMaterialization( - materialization.policy_fp_u64(), - )); - } - } - let mut schema_ids = BTreeSet::new(); - let mut stored_outputs = BTreeSet::new(); - for schema in &self.schemas { - if schema.schema_id.trim().is_empty() - || !schema_ids.insert(schema.schema_id.as_str()) - || !stored_outputs.insert(schema.stored_output_reference.stored_output_id) - || schema.schema_version == 0 - || schema.encodings.is_empty() - || schema.stored_output_reference.stored_output_id != schema.materialization - { - return Err(PrecomputePlanError::InvalidSchema { - schema_id: schema.schema_id.clone(), - }); - } - } - let schemas: BTreeSet<_> = self - .schemas - .iter() - .map(|schema| schema.materialization) - .collect(); - if schemas != materializations || schemas.len() != self.schemas.len() { - return Err(PrecomputePlanError::SchemaSetMismatch); - } - let schema_by_materialization: BTreeMap<_, _> = self - .schemas - .iter() - .map(|schema| (schema.materialization, schema.schema_id.as_str())) - .collect(); - for schema in &self.schemas { - let materialization = self - .materializations - .iter() - .find(|candidate| { - candidate.policy_fingerprint() == schema.materialization.fingerprint() - }) - .ok_or(PrecomputePlanError::SchemaSetMismatch)?; - let accumulator = materialization.accumulator_spec().map_err(|_| { - PrecomputePlanError::UnsupportedFamily(schema.materialization.as_u64()) - })?; - let family = StateFamilyContract::try_from(&accumulator.family).map_err(|_| { - PrecomputePlanError::UnsupportedFamily(schema.materialization.as_u64()) - })?; - let source = materialization.table_name.as_ref().map_or_else( - || Source::TimeSeries { - metric: materialization.metric.clone(), - }, - |table_ref| Source::Table { - table_ref: table_ref.clone(), - }, - ); - let value_projection = materialization.effective_value_projection().clone(); - if schema.schema_id != state_schema_id(schema.materialization.fingerprint()) - || schema.family != family - || schema.source != source - || schema.value_projection != value_projection - || schema.group_by != materialization.grouping_labels - || schema.window.kind != materialization.window_type - || schema.window.size_ms != materialization.window_size.saturating_mul(1_000) - || schema.window.slide_ms - != match materialization.window_type { - crate::WindowKind::Tumbling => None, - crate::WindowKind::Sliding => { - Some(materialization.slide_interval.saturating_mul(1_000)) - } - crate::WindowKind::Session => None, - } - || schema.window.pane_origin_ms != materialization.pane_origin_ms - || schema.encodings != state_encodings(&accumulator.family) - { - return Err(PrecomputePlanError::InvalidSchema { - schema_id: schema.schema_id.clone(), - }); - } - } - let mut producers = BTreeSet::new(); - let mut produced = BTreeSet::new(); - for producer in &self.producers { - if producer.producer_id.trim().is_empty() - || producer.collector_id.trim().is_empty() - || producer.partition_ids.iter().any(|id| id.trim().is_empty()) - || !materializations.contains(&producer.materialization) - || schema_by_materialization - .get(&producer.materialization) - .copied() - != Some(producer.schema_id.as_str()) - { - return Err(PrecomputePlanError::InvalidProducer { - producer_id: producer.producer_id.clone(), - }); - } - let key = ( - producer.producer_id.as_str(), - producer.materialization, - producer.schema_id.as_str(), - ); - if !producers.insert(key) { - return Err(PrecomputePlanError::DuplicateProducer( - producer.producer_id.clone(), - )); - } - produced.insert(producer.materialization); - } - if self.ingest.require_registered_producer { - if let Some(missing) = materializations.difference(&produced).next() { - return Err(PrecomputePlanError::MissingProducer(missing.as_u64())); - } - } - Ok(()) - } -} - -pub(crate) fn state_schema_id(fingerprint: crate::PolicyFingerprint) -> String { - format!("{}:summary-state:v1:{}", BACKEND_COMPAT, fingerprint.0) -} - -pub(crate) fn state_encodings(family: &SummaryFamilyType) -> Vec { - match family { - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Increase - | planner_types::post_asap::ExactKind::Rate, - _, - ) => vec![ - StateEncoding::ExactCounterAccumulatorV2, - StateEncoding::PlannerExactAccumulatorV1, - ], - SummaryFamilyType::ExactAggregate(..) => vec![ - StateEncoding::ExactAccumulatorV1, - StateEncoding::PlannerExactAccumulatorV1, - ], - SummaryFamilyType::Sketch(kind, _) - if matches!( - kind.algorithm(), - SketchAlgorithm::CmsWithHeap - | SketchAlgorithm::CountSketchWithHeap - | SketchAlgorithm::UnivMon - ) => - { - vec![StateEncoding::SketchCoreMsgpackV1] - } - SummaryFamilyType::Sketch(..) => vec![ - StateEncoding::SketchlibProtobufV1, - StateEncoding::SketchCoreMsgpackV1, - ], - _ => Vec::new(), - } -} - -#[cfg(test)] -mod source_window_cohort_tests { - use super::*; - fn full_window() -> crate::PrecomputeMaterialization { - serde_json::from_value(serde_json::json!({ - "aggregation_type":"Sum", "aggregation_sub_type":"", "parameters":{}, - "grouping_labels":{"labels":[]}, "aggregated_labels":{"labels":[]}, - "rollup_labels":{"labels":[]}, "original_yaml":"", - "window_size":60, "slide_interval":10, "window_type":"sliding", - "window_layout":{"kind":"full_window"}, "pane_origin_ms":0, - "spatial_filter":"", "spatial_filter_normalized":"", "metric":"m", - "num_aggregates_to_retain":null, "table_name":null, "value_projection":null - })) - .unwrap() - } - #[test] - fn producer_roster_roundtrip_and_watermark_scope() { - let envelope = PlanEnvelope { - plan_id: 1, - plan_version: 2, - generated_at_unix_ms: 0, - activation_unix_ms: 0, - expiry_unix_ms: None, - backend_compat: "test".into(), - planner_revision: "test".into(), - capability_snapshot_id: "test".into(), - }; - let mut config = full_window(); - config.slide_interval = config.window_size; - config.window_type = crate::WindowKind::Tumbling; - let mut plan = PrecomputePlan::build_backend_local(envelope, vec![config]).unwrap(); - let generation = crate::sds::CatalogGeneration { - schema_version: 1, - plan_id: 1, - plan_version: 2, - snapshot_sha256: "snapshot".into(), - }; - plan.summary_catalog = Some(generation.clone()); - let materialization = plan.schemas[0].materialization; - let legacy = serde_json::json!({ - "producer_id":"p", "collector_id":"c", - "materialization":materialization, "schema_id":plan.schemas[0].schema_id, - }); - assert!(serde_json::from_value::(legacy.clone()).is_err()); - let mut current = legacy; - current["partition_ids"] = serde_json::json!([]); - let producer: ProducerContract = serde_json::from_value(current).unwrap(); - assert!(producer.partition_ids.is_empty()); - plan.producers.push(producer); - let barrier = crate::sds::SummaryWatermarkBarrier { - catalog_generation: generation, - source: crate::sds::SummarySourcePartition { - producer_id: "p".into(), - partition_id: "a".into(), - producer_epoch: 1, - }, - sequence: 1, - watermark_ms: 100, - }; - assert!(plan - .validate_watermark_scope(materialization, &barrier) - .is_err()); - plan.producers[0].partition_ids = BTreeSet::from(["b".into(), "a".into()]); - let wire = serde_json::to_value(&plan).unwrap(); - assert_eq!( - wire["producers"][0]["partition_ids"], - serde_json::json!(["a", "b"]) - ); - let restored: PrecomputePlan = serde_json::from_value(wire).unwrap(); - assert!(restored - .validate_watermark_scope(materialization, &barrier) - .is_ok()); - for case in 0..6 { - let mut wrong = barrier.clone(); - match case { - 0 => wrong.source.producer_id = "unknown".into(), - 1 => wrong.source.partition_id = "unknown".into(), - 2 => wrong.catalog_generation.plan_version += 1, - 3 => wrong.source.producer_epoch = 0, - 4 => wrong.sequence = 0, - _ => wrong.catalog_generation.snapshot_sha256 = "other".into(), - } - assert!(restored - .validate_watermark_scope(materialization, &wrong) - .is_err()); - } - let other_materialization = crate::sds::StoredOutputId::from(crate::PolicyFingerprint( - materialization.as_u64().wrapping_add(1), - )); - assert!(restored - .validate_watermark_scope(other_materialization, &barrier) - .is_err()); - let mut wrong_envelope = restored.clone(); - wrong_envelope.envelope.plan_version += 1; - assert!(wrong_envelope - .validate_watermark_scope(materialization, &barrier) - .is_err()); - plan.producers[0].partition_ids.insert(" ".into()); - assert!(plan.validate().is_err()); - } - - #[test] - fn maintenance_reduction_requires_matching_explicit_population_contract() { - use planner_types::post_asap::*; - use planner_types::pre_asap::Reduction; - let mut config = full_window(); - config.partitioning = Some(crate::sds::PopulationPartitioning::Grouped); - let mut node = ExecutableDagNode { - id: PostAsapNodeId(1), - payload: ExecutableOperatorPayload::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - input: SummaryUpdate { - item: None, - weight: SummaryInputExpr::Constant(1.0), - weight_domain: Default::default(), - }, - reduction: Reduction::by(vec![]), - grouping: Default::default(), - }, - output_state: ExecutionDataState::INGESTION_SUMMARY, - output_schema: SummarySchema { - fields: vec![], - time_index: None, - }, - guarantee: None, - }; - assert!(validate_maintenance_reduction(&config, &node).is_ok()); - config.partitioning = Some(crate::sds::PopulationPartitioning::PerEntity); - assert!(validate_maintenance_reduction(&config, &node).is_err()); - if let ExecutableOperatorPayload::SummaryAgg { reduction, .. } = &mut node.payload { - *reduction = Reduction::PerEntity; - } - assert!(validate_maintenance_reduction(&config, &node).is_ok()); - config.partitioning = Some(crate::sds::PopulationPartitioning::Grouped); - if let ExecutableOperatorPayload::SummaryAgg { reduction, .. } = &mut node.payload { - *reduction = Reduction::by(vec![0]); - } - assert!(validate_maintenance_reduction(&config, &node).is_err()); - config.partitioning = None; - assert!(validate_maintenance_reduction(&config, &node).is_err()); - node.payload = ExecutableOperatorPayload::SummaryMerge; - assert!(validate_maintenance_reduction(&config, &node).is_err()); - } - - #[test] - fn canonical_population_encoding_is_not_yet_installable() { - let mut config = full_window(); - config.slide_interval = config.window_size; - config.population_key_encoding = - crate::grouping_projection::PopulationKeyEncoding::CanonicalLabelsV1; - let envelope = PlanEnvelope { - plan_id: 1, - plan_version: 1, - generated_at_unix_ms: 0, - activation_unix_ms: 0, - expiry_unix_ms: None, - backend_compat: "test".into(), - planner_revision: "test".into(), - capability_snapshot_id: "test".into(), - }; - let error = PrecomputePlan::build_backend_local(envelope, vec![config]).unwrap_err(); - assert!( - error.to_string().contains("population key encoding"), - "{error}" - ); - } - - #[test] - fn full_sliding_cohort_preserves_explicit_windows_and_identity() { - let target = full_window(); - let source = full_window(); - let mut missing_origin = source.clone(); - missing_origin.pane_origin_ms = None; - assert!(validated_source_window_cohort(&missing_origin, &[&missing_origin]).is_err()); - missing_origin.slide_interval = missing_origin.window_size; - assert!(validated_source_window_cohort(&missing_origin, &[&missing_origin]).is_ok()); - let result = validated_source_window_cohort(&target, &[&source]).unwrap(); - assert!(std::ptr::eq(result[0], &source)); - let mut other = source.clone(); - other.metric = "other".into(); - assert_eq!( - validated_source_window_cohort(&target, &[&source, &other]) - .unwrap() - .len(), - 2 - ); - assert!(validated_source_window_cohort(&target, &[&source, &source]).is_err()); - for mutation in 0..3 { - let mut changed = source.clone(); - match mutation { - 0 => changed.pane_origin_ms = Some(1), - 1 => changed.slide_interval = 20, - _ => { - changed.window_layout = - crate::WindowMaterializationLayout::Pane { pane_secs: 10 } - } - } - assert_ne!(source.policy_fingerprint(), changed.policy_fingerprint()); - assert!(validated_source_window_cohort(&target, &[&changed]).is_err()); - } - } -} diff --git a/crates/asap_types/src/precompute_plan/catalog.rs b/crates/asap_types/src/precompute_plan/catalog.rs deleted file mode 100644 index bad95e7e5..000000000 --- a/crates/asap_types/src/precompute_plan/catalog.rs +++ /dev/null @@ -1,232 +0,0 @@ -//! Catalog consistency checks for the precompute execution plan. -use super::*; -use crate::sds::{StoredOutputId, SummaryDescriptor}; -use crate::summary_catalog::SummaryCatalog; -use planner_types::pre_asap::Source; -use std::collections::BTreeSet; -fn invalid(reason: impl Into) -> PrecomputePlanError { - PrecomputePlanError::CatalogContract(reason.into()) -} - -impl PrecomputePlan { - /// Bind references after retention decisions are finalized. The plan keeps - /// only the immutable snapshot reference; descriptors are installed once. - pub fn bind_catalog(&mut self, catalog: &SummaryCatalog) -> Result<(), PrecomputePlanError> { - for config in &self.materializations { - let id = StoredOutputId::from(config.policy_fingerprint()); - catalog.outputs.get(&id).ok_or_else(|| { - invalid(format!("missing catalog materialization {}", id.as_u64())) - })?; - } - for schema in &mut self.schemas { - schema.stored_output_reference = catalog - .output_reference(schema.materialization) - .map_err(|e| invalid(e.to_string()))?; - } - self.summary_catalog = Some( - catalog - .reference() - .map_err(|error| invalid(error.to_string()))?, - ); - self.validate_against_catalog(catalog) - } - /// Strict validation resolves this plan's reference against the one catalog - /// snapshot carried by the enclosing physical-plan installation. - pub fn validate_against_catalog( - &self, - catalog: &SummaryCatalog, - ) -> Result<(), PrecomputePlanError> { - self.validate()?; - self.validate_catalog_contents(catalog) - } - fn validate_catalog_contents( - &self, - catalog: &SummaryCatalog, - ) -> Result<(), PrecomputePlanError> { - catalog.validate().map_err(|e| invalid(e.to_string()))?; - let expected = SummaryCatalog::from_materializations( - catalog.plan_id, - catalog.plan_version, - &self.materializations, - ) - .map_err(|e| invalid(e.to_string()))?; - if catalog.outputs != expected.outputs || catalog.definitions != expected.definitions { - return Err(invalid( - "stored output semantics differ from installed writer computation", - )); - } - for config in &self.materializations { - if config.derived_input.is_some() && config.semantic_fragment.is_none() { - return Err(invalid( - "derived stored output requires its complete Planner semantic closure", - )); - } - if let Some(expected) = &config.semantic_fragment { - let mut found = false; - for installed in self.executable_dags.values() { - let dag = installed.document.decode().map_err(invalid)?; - for (id, binding) in &installed.binding.nodes { - if matches!(binding, crate::executable_plan::BackendNodeBinding::Materialization { stored_output } - if stored_output.fingerprint() == config.policy_fingerprint()) - { - found = true; - let actual = match &self.ingest.dataset_identity { - Some(dataset) => crate::semantic_fragment::SemanticFragment::from_stored_output_in_dataset(&dag, *id, dataset.clone()), - None => crate::semantic_fragment::SemanticFragment::from_stored_output(&dag, *id), - }.map_err(invalid)?; - if &actual != expected { - return Err(invalid( - "semantic definition differs from Planner-selected producer", - )); - } - } - } - } - if !found { - return Err(invalid( - "semantic definition has no Planner-selected producer", - )); - } - } - } - let expected_reference = catalog - .reference() - .map_err(|error| invalid(error.to_string()))?; - if self.summary_catalog.as_ref() != Some(&expected_reference) - || catalog.plan_id != self.envelope.plan_id - || catalog.plan_version != self.envelope.plan_version - { - return Err(PrecomputePlanError::PlanIdentityMismatch); - } - let ids: BTreeSet<_> = self - .materializations - .iter() - .map(|m| StoredOutputId::from(m.policy_fingerprint())) - .collect(); - if ids != catalog.outputs.keys().copied().collect() { - return Err(invalid("catalog/reference/materialization sets differ")); - } - for config in &self.materializations { - let id = StoredOutputId::from(config.policy_fingerprint()); - let binding = &catalog.outputs[&id]; - let expected = - SummaryDescriptor::from_config(config).map_err(|e| invalid(e.to_string()))?; - if binding.summary_descriptor_id != expected.id { - return Err(invalid( - "summary operator/update contract differs from catalog", - )); - } - let data = &catalog.data_descriptors[&binding.data_descriptor_id]; - let expected_source = config.source_identity(); - if config.table_name.is_some() - && !config.grouping_labels.is_empty() - && data.observation_semantics - != crate::grouping_projection::TABLE_GROUP_OBSERVATION_SEMANTICS - { - return Err(invalid( - "table grouping codec differs from installed contract", - )); - } - if config.table_name.is_some() { - config - .grouping_labels - .validate_table_group_codec() - .map_err(invalid)?; - } - let expected_projection = config.effective_value_projection(); - if data.population_key_encoding != config.population_key_encoding - || data.partitioning != config.partitioning - || data.timestamp_column != config.table_timestamp_column - || data.source != expected_source - || &data.value_projection != expected_projection - || data.population_filter_canonical - != config.population_filter_canonical().map_err(invalid)? - || data.group_by_keys != config.grouping_labels - { - return Err(invalid("source/population/grouping differs from catalog")); - } - if config.spatial_filter_normalized - != crate::utils::normalize_spatial_filter(&config.spatial_filter) - { - return Err(invalid("normalized population predicate drift")); - } - let schema = self - .schemas - .iter() - .find(|s| s.materialization == id) - .ok_or(PrecomputePlanError::SchemaSetMismatch)?; - let family = config - .accumulator_spec() - .map_err(|e| invalid(e.to_string()))? - .family; - let expected_family = StateFamilyContract::try_from(&family) - .map_err(|_| PrecomputePlanError::UnsupportedFamily(id.as_u64()))?; - let source = if let Some(table) = &config.table_name { - Source::Table { - table_ref: table.clone(), - } - } else { - Source::TimeSeries { - metric: config.metric.clone(), - } - }; - let projection = config.effective_value_projection(); - let size = config - .window_size - .checked_mul(1000) - .filter(|v| *v > 0) - .ok_or_else(|| invalid("invalid window size"))?; - let slide = config - .slide_interval - .checked_mul(1000) - .filter(|v| *v > 0) - .ok_or_else(|| invalid("invalid slide interval"))?; - let expected_slide = match config.window_type { - crate::WindowKind::Tumbling => None, - crate::WindowKind::Sliding => Some(slide), - crate::WindowKind::Session => { - return Err(invalid("session lifecycle is not supported")) - } - }; - if schema.stored_output_reference - != catalog - .output_reference(id) - .map_err(|e| invalid(e.to_string()))? - || schema.schema_id != state_schema_id(id.fingerprint()) - || schema.family != expected_family - || schema.source != source - || &schema.value_projection != projection - || schema.group_by != config.grouping_labels - || schema.window.kind != config.window_type - || schema.window.size_ms != size - || schema.window.slide_ms != expected_slide - || schema.window.pane_origin_ms != config.pane_origin_ms - { - return Err(PrecomputePlanError::InvalidSchema { - schema_id: schema.schema_id.clone(), - }); - } - if schema.encodings != state_encodings(&family) { - return Err(invalid("encoding does not match state family")); - } - if config.num_aggregates_to_retain == Some(0) { - return Err(invalid("materialization lifecycle/retention mismatch")); - } - } - if self.ingest.protocol == IngestProtocol::PrometheusRemoteWriteV1 - && !self.producers.is_empty() - { - return Err(invalid( - "backend-local plan cannot declare collector placement", - )); - } - if self.producers.iter().any(|p| { - p.producer_id.trim().is_empty() - || p.collector_id.trim().is_empty() - || p.producer_id != p.collector_id - }) { - return Err(invalid("invalid collector placement identity")); - } - Ok(()) - } -} diff --git a/crates/asap_types/src/producer_plan.rs b/crates/asap_types/src/producer_plan.rs deleted file mode 100644 index 846c4df83..000000000 --- a/crates/asap_types/src/producer_plan.rs +++ /dev/null @@ -1,855 +0,0 @@ -//! Installed collector and transmission contracts shared across producers and consumers. -//! Deployment choices and accuracy-budget allocation remain in the control plane. -use crate::precompute_plan::{PlanEnvelope, PrecomputePlan, StateEncoding, StateFamilyContract}; -use planner_types::post_asap::{SketchAlgorithm, SummaryWindowFramework}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use std::collections::BTreeSet; -use thiserror::Error; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct CollectorMaterialization { - pub query_id: String, - pub materialization: crate::sds::StoredOutputId, - pub metric: String, - pub algorithm: String, - pub parameters: Value, - pub group_by: Vec, - pub window_secs: u64, - pub abstract_window_framework: SummaryWindowFramework, - #[serde(rename = "window_implementation_id")] - pub window_realization_id: String, - pub slide_secs: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub pane_origin_ms: Option, - pub window_layout: crate::WindowMaterializationLayout, - pub evidence_source: Option, - pub lifecycle: CollectorLifecycle, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct CollectorLifecycle { - pub kind: String, - pub maintenance_mode: String, - pub evaluation_schedule: String, - pub output_representation: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct CollectorPlan { - /// Set when binding the plan; required for catalog-authorized installation. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub summary_catalog: Option, - pub collector_id: String, - pub envelope: PlanEnvelope, - pub materializations: Vec, - pub transmission_rules: Vec, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -#[serde(rename_all = "snake_case")] -pub enum TransmissionMode { - Full, - Delta, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum SequenceScope { - MaterializationSeriesProducerEpoch, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct FrameIdentityContract { - pub identity_version: u32, - pub sequence_scope: SequenceScope, - pub require_checkpoint_for_full: bool, - pub require_base_checkpoint_for_delta: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct TransmissionRule { - pub materialization: crate::sds::StoredOutputId, - pub producer_id: String, - pub schema_id: String, - pub mode: TransmissionMode, - pub encoding: StateEncoding, - pub emit_every_ms: u64, - pub full_checkpoint_every_ms: Option, - pub destination_ref: String, - /// Plan-owned runtime knobs. These values are part of the immutable plan - /// generation; live feedback may only change them by publishing a - /// successor generation accepted by [`TransmissionPlan::authorize_successor`]. - #[serde(default)] - pub runtime_policy: RuntimeRulePolicy, -} - -/// How the collector admits updates before sketch maintenance. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] -#[serde(tag = "mode", rename_all = "snake_case", deny_unknown_fields)] -pub enum SamplingPolicy { - #[default] - Disabled, - Fixed { - /// Probability in `(0, 1]`; `1` is valid but should normally be - /// represented by `Disabled`. - probability: f64, - estimator: SamplingEstimator, - }, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum SamplingEstimator { - /// Hash-threshold element sampling, used by cardinality summaries. - HashThreshold, - /// Geometric admission/Nitro-style update sampling, used by frequency - /// summaries. The sketch readout carries the corresponding correction. - GeometricAdmission, -} - -/// Norm-adaptive Group-of-Sketches delta gating. GOS is meaningful only for -/// CountSketch families and only when the transmission rule is in delta mode. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct GosPolicy { - pub epsilon_staleness: f64, - pub sites: u32, - pub threshold_mode: GosThresholdMode, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum GosThresholdMode { - Isotropic, - Anisotropic, -} - -/// Sparse-delta semantics within a delta transmission rule. An absolute -/// threshold of zero sends every changed cell. When `gos` is present it -/// replaces the fixed threshold with the GOS norm-adaptive threshold. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct DeltaPolicy { - pub absolute_threshold: f64, - pub gos: Option, -} - -/// Inclusive bounds for one floating-point adaptation knob. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct AdaptiveF64Bounds { - pub min: f64, - pub max: f64, - pub max_step: f64, -} - -/// Inclusive bounds for one integer adaptation knob. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct AdaptiveU64Bounds { - pub min: u64, - pub max: u64, - pub max_step: u64, -} - -/// Guardrails for telemetry-driven runtime adaptation. This is an -/// authorization contract, not an instruction to mutate the active plan. -/// Every accepted change becomes a staged successor CompiledPhysicalPlan. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] -#[serde(deny_unknown_fields)] -pub struct RuntimeAdaptationPolicy { - pub enabled: bool, - pub not_before_unix_ms: u64, - pub max_evidence_age_ms: u64, - pub min_evidence_samples: u64, - pub sample_probability: Option, - pub emit_every_ms: Option, - pub delta_threshold: Option, - pub gos_epsilon_staleness: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] -#[serde(deny_unknown_fields)] -pub struct RuntimeRulePolicy { - #[serde(default)] - pub sampling: SamplingPolicy, - pub delta: Option, - #[serde(default)] - pub adaptation: RuntimeAdaptationPolicy, -} - -/// Identity and sufficiency information for evidence authorizing one rule's -/// successor knobs. Raw measurements remain in the runtime-samples store; the -/// authorization boundary needs only their exact provenance and sample count. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct RuntimeAdaptationEvidence { - pub plan_id: u64, - pub plan_version: u64, - pub materialization: crate::sds::StoredOutputId, - pub producer_id: String, - pub schema_id: String, - pub producer_version: String, - pub observed_at_unix_ms: u64, - pub sample_count: u64, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct TransmissionPlan { - /// Set when binding the plan; required for catalog-authorized installation. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub summary_catalog: Option, - pub envelope: PlanEnvelope, - pub frame_identity: FrameIdentityContract, - pub rules: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum SummaryFrameKind { - Full, - Delta, -} - -/// Identity attached to every summary record. Window bounds come from the -/// data point; the remaining fields are carried as reserved `asap.frame.*` -/// attributes until the modified-OTLP schema gains a dedicated message. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SummaryFrameIdentity { - pub identity_version: u32, - pub plan_id: u64, - pub plan_version: u64, - pub backend_compat: String, - pub materialization: crate::sds::StoredOutputId, - /// Canonical producer-side identity for one concrete retained-label group. - pub series_identity: String, - pub schema_id: String, - pub producer_id: String, - pub producer_epoch: String, - pub window_start_unix_nano: u64, - pub window_end_unix_nano: u64, - pub sequence: u64, - pub kind: SummaryFrameKind, - pub encoding: StateEncoding, - pub checkpoint_id: Option, - pub base_checkpoint_id: Option, -} - -#[derive(Debug, Error, PartialEq, Eq)] -pub enum TransmissionPlanError { - #[error("summary catalog mismatch: {0}")] - Catalog(String), - #[error("TransmissionPlan envelope differs from PrecomputePlan")] - EnvelopeMismatch, - #[error("transmission rules do not exactly match precompute producer bindings")] - ProducerSetMismatch, - #[error("invalid transmission rule for producer {0}")] - InvalidRule(String), - #[error("frame identity is invalid: {0}")] - InvalidFrame(String), - #[error("frame has no matching transmission rule")] - UnknownFrame, - #[error("runtime policy for producer {producer_id} is invalid: {reason}")] - InvalidRuntimePolicy { producer_id: String, reason: String }, - #[error("runtime adaptation successor is invalid: {0}")] - InvalidSuccessor(String), - #[error("runtime adaptation evidence for producer {0} is missing or invalid")] - InvalidAdaptationEvidence(String), - #[error("runtime adaptation for producer {producer_id} exceeds guardrails: {knob}")] - AdaptationOutOfBounds { - producer_id: String, - knob: &'static str, - }, -} - -fn validate_catalog_projection( - reference: Option<&crate::sds::CatalogGeneration>, - envelope: &PlanEnvelope, - materializations: impl IntoIterator, - catalog: &crate::summary_catalog::SummaryCatalog, -) -> Result<(), TransmissionPlanError> { - let expected = catalog - .reference() - .map_err(|error| TransmissionPlanError::Catalog(error.to_string()))?; - if reference != Some(&expected) - || envelope.plan_id != catalog.plan_id - || envelope.plan_version != catalog.plan_version - { - return Err(TransmissionPlanError::Catalog( - "missing or different snapshot reference".into(), - )); - } - for id in materializations { - if !catalog.outputs.contains_key(&id) { - return Err(TransmissionPlanError::Catalog(format!( - "unknown materialization {}", - id.as_u64() - ))); - } - } - Ok(()) -} - -impl CollectorPlan { - pub fn validate_against_catalog( - &self, - catalog: &crate::summary_catalog::SummaryCatalog, - ) -> Result<(), TransmissionPlanError> { - validate_catalog_projection( - self.summary_catalog.as_ref(), - &self.envelope, - self.materializations - .iter() - .map(|m| m.materialization) - .chain(self.transmission_rules.iter().map(|r| r.materialization)), - catalog, - ) - } -} - -impl TransmissionPlan { - pub fn validate_against_catalog( - &self, - catalog: &crate::summary_catalog::SummaryCatalog, - ) -> Result<(), TransmissionPlanError> { - validate_catalog_projection( - self.summary_catalog.as_ref(), - &self.envelope, - self.rules.iter().map(|r| r.materialization), - catalog, - ) - } - - pub fn validate(&self, precompute: &PrecomputePlan) -> Result<(), TransmissionPlanError> { - if self.summary_catalog != precompute.summary_catalog { - return Err(TransmissionPlanError::Catalog( - "transmission and precompute plans reference different catalog snapshots".into(), - )); - } - if self.envelope != precompute.envelope { - return Err(TransmissionPlanError::EnvelopeMismatch); - } - let expected: BTreeSet<_> = precompute - .producers - .iter() - .map(|producer| { - ( - producer.materialization, - producer.producer_id.as_str(), - producer.schema_id.as_str(), - ) - }) - .collect(); - let actual: BTreeSet<_> = self - .rules - .iter() - .map(|rule| { - ( - rule.materialization, - rule.producer_id.as_str(), - rule.schema_id.as_str(), - ) - }) - .collect(); - if expected != actual || actual.len() != self.rules.len() { - return Err(TransmissionPlanError::ProducerSetMismatch); - } - for rule in &self.rules { - let valid_checkpoint_cadence = match (rule.mode, rule.full_checkpoint_every_ms) { - (TransmissionMode::Full, None) => true, - (TransmissionMode::Delta, Some(full_every)) => { - rule.emit_every_ms > 0 - && full_every >= rule.emit_every_ms - && full_every % rule.emit_every_ms == 0 - } - _ => false, - }; - if rule.emit_every_ms == 0 - || rule.destination_ref.is_empty() - || !valid_checkpoint_cadence - { - return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); - } - let schema = precompute - .schemas - .iter() - .find(|schema| schema.materialization == rule.materialization) - .expect("producer set validation guarantees a matching schema"); - if !schema.encodings.contains(&rule.encoding) { - return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); - } - validate_runtime_rule_policy(rule, &schema.family)?; - } - Ok(()) - } - - /// Authorize a telemetry-driven successor without mutating this active - /// plan. Semantic identity, codecs, destination, transmission mode and - /// checkpoint cadence remain fixed. Only explicitly bounded runtime knobs - /// may move, and each changed rule needs fresh evidence attributed to the - /// exact active generation. - pub fn authorize_successor( - &self, - successor: &TransmissionPlan, - evidence: &[RuntimeAdaptationEvidence], - now_unix_ms: u64, - ) -> Result<(), TransmissionPlanError> { - if successor.envelope.plan_id != self.envelope.plan_id - || successor.envelope.plan_version != self.envelope.plan_version.saturating_add(1) - || successor.envelope.backend_compat != self.envelope.backend_compat - || successor.envelope.planner_revision != self.envelope.planner_revision - || successor.envelope.capability_snapshot_id != self.envelope.capability_snapshot_id - || successor.envelope.generated_at_unix_ms < self.envelope.generated_at_unix_ms - || successor.envelope.activation_unix_ms < successor.envelope.generated_at_unix_ms - || successor.rules.len() != self.rules.len() - || successor.frame_identity != self.frame_identity - { - return Err(TransmissionPlanError::InvalidSuccessor( - "successor must be the next version of the same semantic/capability generation" - .into(), - )); - } - - for current in &self.rules { - let Some(next) = successor.rules.iter().find(|candidate| { - candidate.materialization == current.materialization - && candidate.producer_id == current.producer_id - && candidate.schema_id == current.schema_id - }) else { - return Err(TransmissionPlanError::InvalidSuccessor(format!( - "missing rule for producer {}", - current.producer_id - ))); - }; - if current.mode != next.mode - || current.encoding != next.encoding - || current.full_checkpoint_every_ms != next.full_checkpoint_every_ms - || current.destination_ref != next.destination_ref - || current.runtime_policy.adaptation != next.runtime_policy.adaptation - || sampling_estimator(¤t.runtime_policy.sampling) - != sampling_estimator(&next.runtime_policy.sampling) - || delta_shape(¤t.runtime_policy.delta) - != delta_shape(&next.runtime_policy.delta) - { - return Err(TransmissionPlanError::InvalidSuccessor(format!( - "rule identity/codec/mode/guardrails drifted for producer {}", - current.producer_id - ))); - } - if current.emit_every_ms == next.emit_every_ms - && current.runtime_policy.sampling == next.runtime_policy.sampling - && current.runtime_policy.delta == next.runtime_policy.delta - { - continue; - } - let policy = ¤t.runtime_policy.adaptation; - if !policy.enabled || now_unix_ms < policy.not_before_unix_ms { - return Err(TransmissionPlanError::AdaptationOutOfBounds { - producer_id: current.producer_id.clone(), - knob: "adaptation_disabled_or_in_cooldown", - }); - } - let has_evidence = evidence.iter().any(|item| { - item.plan_id == self.envelope.plan_id - && item.plan_version == self.envelope.plan_version - && item.materialization == current.materialization - && item.producer_id == current.producer_id - && item.schema_id == current.schema_id - && !item.producer_version.trim().is_empty() - && item.sample_count >= policy.min_evidence_samples - && item.observed_at_unix_ms <= now_unix_ms - && now_unix_ms.saturating_sub(item.observed_at_unix_ms) - <= policy.max_evidence_age_ms - }); - if !has_evidence { - return Err(TransmissionPlanError::InvalidAdaptationEvidence( - current.producer_id.clone(), - )); - } - authorize_f64_change( - sampling_probability(¤t.runtime_policy.sampling), - sampling_probability(&next.runtime_policy.sampling), - policy.sample_probability.as_ref(), - ¤t.producer_id, - "sample_probability", - )?; - authorize_u64_change( - current.emit_every_ms, - next.emit_every_ms, - policy.emit_every_ms.as_ref(), - ¤t.producer_id, - "emit_every_ms", - )?; - authorize_f64_change( - delta_threshold(¤t.runtime_policy.delta), - delta_threshold(&next.runtime_policy.delta), - policy.delta_threshold.as_ref(), - ¤t.producer_id, - "delta_threshold", - )?; - authorize_f64_change( - gos_epsilon(¤t.runtime_policy.delta), - gos_epsilon(&next.runtime_policy.delta), - policy.gos_epsilon_staleness.as_ref(), - ¤t.producer_id, - "gos_epsilon_staleness", - )?; - } - Ok(()) - } - - pub fn validate_frame( - &self, - frame: &SummaryFrameIdentity, - ) -> Result<(), TransmissionPlanError> { - if frame.identity_version != self.frame_identity.identity_version - || frame.plan_id != self.envelope.plan_id - || frame.plan_version != self.envelope.plan_version - || frame.backend_compat != self.envelope.backend_compat - || frame.series_identity.is_empty() - || frame.producer_epoch.is_empty() - || frame.sequence == 0 - || frame.window_start_unix_nano >= frame.window_end_unix_nano - || (frame.kind == SummaryFrameKind::Full - && self.frame_identity.require_checkpoint_for_full - && frame.checkpoint_id.is_none()) - || (frame.kind == SummaryFrameKind::Delta - && self.frame_identity.require_base_checkpoint_for_delta - && frame.base_checkpoint_id.is_none()) - { - return Err(TransmissionPlanError::InvalidFrame( - "identity/lifecycle/window/checkpoint fields do not satisfy the active contract" - .into(), - )); - } - if self.rules.iter().any(|rule| { - rule.materialization == frame.materialization - && rule.producer_id == frame.producer_id - && rule.schema_id == frame.schema_id - // A delta rule necessarily emits periodic full checkpoints; - // a full-only rule must never emit deltas. - && (frame.kind == SummaryFrameKind::Full - || rule.mode == TransmissionMode::Delta) - && rule.encoding == frame.encoding - }) { - Ok(()) - } else { - Err(TransmissionPlanError::UnknownFrame) - } - } -} - -fn validate_runtime_rule_policy( - rule: &TransmissionRule, - family: &StateFamilyContract, -) -> Result<(), TransmissionPlanError> { - let invalid = |reason: &str| TransmissionPlanError::InvalidRuntimePolicy { - producer_id: rule.producer_id.clone(), - reason: reason.into(), - }; - if let SamplingPolicy::Fixed { - probability, - estimator, - } = &rule.runtime_policy.sampling - { - if !probability.is_finite() || !(0.0..=1.0).contains(probability) || *probability == 0.0 { - return Err(invalid("sample probability must be finite and in (0, 1]")); - } - let supported = matches!( - (family, *estimator), - ( - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::Hll, - .. - }, - SamplingEstimator::HashThreshold, - ) | ( - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap, - .. - }, - SamplingEstimator::GeometricAdmission, - ) - ); - if !supported { - return Err(invalid( - "sampling estimator is not implemented for the materialization family", - )); - } - } - - if (rule.mode == TransmissionMode::Delta) != rule.runtime_policy.delta.is_some() { - return Err(invalid( - "delta policy must be present exactly when transmission mode is delta", - )); - } - if let Some(delta) = &rule.runtime_policy.delta { - if !delta.absolute_threshold.is_finite() || delta.absolute_threshold < 0.0 { - return Err(invalid("delta threshold must be finite and non-negative")); - } - if !matches!( - family, - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::DDSketch - | SketchAlgorithm::Hll - | SketchAlgorithm::Cms - | SketchAlgorithm::CmsWithHeap - | SketchAlgorithm::CountSketch - | SketchAlgorithm::CountSketchWithHeap, - .. - } - ) { - return Err(invalid( - "delta transmission is not implemented for the materialization family", - )); - } - if let Some(gos) = &delta.gos { - if !matches!( - family, - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap, - .. - } - ) || !gos.epsilon_staleness.is_finite() - || !(0.0..=1.0).contains(&gos.epsilon_staleness) - || gos.epsilon_staleness == 0.0 - || gos.sites == 0 - { - return Err(invalid( - "GOS requires a CountSketch family, epsilon in (0, 1], and at least one site", - )); - } - } - } - - let adaptation = &rule.runtime_policy.adaptation; - if adaptation.enabled - && (adaptation.max_evidence_age_ms == 0 || adaptation.min_evidence_samples == 0) - { - return Err(invalid( - "enabled adaptation requires non-zero evidence age and sample-count requirements", - )); - } - validate_f64_bounds(adaptation.sample_probability.as_ref(), 0.0, 1.0).map_err(&invalid)?; - validate_f64_bounds(adaptation.delta_threshold.as_ref(), 0.0, f64::MAX).map_err(&invalid)?; - validate_f64_bounds(adaptation.gos_epsilon_staleness.as_ref(), 0.0, 1.0).map_err(&invalid)?; - if let Some(bounds) = &adaptation.emit_every_ms { - if bounds.min == 0 - || bounds.min > bounds.max - || bounds.max_step == 0 - || !(bounds.min..=bounds.max).contains(&rule.emit_every_ms) - { - return Err(invalid("emit interval guardrails are invalid")); - } - } - if let Some(bounds) = &adaptation.sample_probability { - let current = sampling_probability(&rule.runtime_policy.sampling); - if current < bounds.min || current > bounds.max { - return Err(invalid( - "current sampling probability is outside guardrails", - )); - } - } - if let Some(bounds) = &adaptation.delta_threshold { - let Some(current) = rule - .runtime_policy - .delta - .as_ref() - .map(|policy| policy.absolute_threshold) - else { - return Err(invalid("delta guardrails require an active delta policy")); - }; - if current < bounds.min || current > bounds.max { - return Err(invalid("current delta threshold is outside guardrails")); - } - } - if let Some(bounds) = &adaptation.gos_epsilon_staleness { - let Some(current) = rule - .runtime_policy - .delta - .as_ref() - .and_then(|policy| policy.gos.as_ref()) - .map(|gos| gos.epsilon_staleness) - else { - return Err(invalid("GOS guardrails require an active GOS policy")); - }; - if current < bounds.min || current > bounds.max { - return Err(invalid("current GOS epsilon is outside guardrails")); - } - } - Ok(()) -} - -fn validate_f64_bounds( - bounds: Option<&AdaptiveF64Bounds>, - domain_min: f64, - domain_max: f64, -) -> Result<(), &'static str> { - let Some(bounds) = bounds else { - return Ok(()); - }; - if !bounds.min.is_finite() - || !bounds.max.is_finite() - || !bounds.max_step.is_finite() - || bounds.min < domain_min - || bounds.max > domain_max - || bounds.min > bounds.max - || bounds.max_step <= 0.0 - { - Err("floating-point adaptation guardrails are invalid") - } else { - Ok(()) - } -} - -fn sampling_probability(policy: &SamplingPolicy) -> f64 { - match policy { - SamplingPolicy::Disabled => 1.0, - SamplingPolicy::Fixed { probability, .. } => *probability, - } -} - -fn sampling_estimator(policy: &SamplingPolicy) -> Option { - match policy { - SamplingPolicy::Disabled => None, - SamplingPolicy::Fixed { estimator, .. } => Some(*estimator), - } -} - -fn delta_threshold(policy: &Option) -> f64 { - policy - .as_ref() - .map(|policy| policy.absolute_threshold) - .unwrap_or(0.0) -} - -fn gos_epsilon(policy: &Option) -> f64 { - policy - .as_ref() - .and_then(|policy| policy.gos.as_ref()) - .map(|gos| gos.epsilon_staleness) - .unwrap_or(0.0) -} - -fn delta_shape(policy: &Option) -> Option<(Option<(u32, GosThresholdMode)>,)> { - policy.as_ref().map(|policy| { - (policy - .gos - .as_ref() - .map(|gos| (gos.sites, gos.threshold_mode)),) - }) -} - -fn authorize_f64_change( - current: f64, - next: f64, - bounds: Option<&AdaptiveF64Bounds>, - producer_id: &str, - knob: &'static str, -) -> Result<(), TransmissionPlanError> { - if current == next { - return Ok(()); - } - let allowed = bounds.is_some_and(|bounds| { - next.is_finite() - && (bounds.min..=bounds.max).contains(&next) - && (next - current).abs() <= bounds.max_step - }); - if allowed { - Ok(()) - } else { - Err(TransmissionPlanError::AdaptationOutOfBounds { - producer_id: producer_id.into(), - knob, - }) - } -} - -fn authorize_u64_change( - current: u64, - next: u64, - bounds: Option<&AdaptiveU64Bounds>, - producer_id: &str, - knob: &'static str, -) -> Result<(), TransmissionPlanError> { - if current == next { - return Ok(()); - } - let allowed = bounds.is_some_and(|bounds| { - (bounds.min..=bounds.max).contains(&next) && current.abs_diff(next) <= bounds.max_step - }); - if allowed { - Ok(()) - } else { - Err(TransmissionPlanError::AdaptationOutOfBounds { - producer_id: producer_id.into(), - knob, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use planner_types::post_asap::SketchParams; - - #[test] - fn runtime_policy_is_family_and_mode_checked() { - let mut rule = TransmissionRule { - materialization: crate::PolicyFingerprint(1).into(), - producer_id: "test".into(), - schema_id: "test".into(), - mode: TransmissionMode::Full, - encoding: StateEncoding::SketchlibProtobufV1, - emit_every_ms: 60_000, - full_checkpoint_every_ms: None, - destination_ref: "backend".into(), - runtime_policy: Default::default(), - }; - rule.runtime_policy.sampling = SamplingPolicy::Fixed { - probability: 0.5, - estimator: SamplingEstimator::HashThreshold, - }; - let hll = StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::Hll, - parameters: SketchParams::Hll { precision: 14 }, - }; - let count_sketch = StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::CountSketch, - parameters: SketchParams::CountSketch { - width: 128, - depth: 4, - }, - }; - validate_runtime_rule_policy(&rule, &hll).expect("HLL supports hash-threshold sampling"); - assert!(matches!( - validate_runtime_rule_policy(&rule, &count_sketch), - Err(TransmissionPlanError::InvalidRuntimePolicy { .. }) - )); - - rule.runtime_policy.sampling = SamplingPolicy::Disabled; - rule.mode = TransmissionMode::Delta; - rule.full_checkpoint_every_ms = Some(300_000); - rule.runtime_policy.delta = Some(DeltaPolicy { - absolute_threshold: 0.0, - gos: Some(GosPolicy { - epsilon_staleness: 0.02, - sites: 2, - threshold_mode: GosThresholdMode::Isotropic, - }), - }); - validate_runtime_rule_policy(&rule, &count_sketch).expect("CountSketch supports delta GOS"); - assert!(matches!( - validate_runtime_rule_policy(&rule, &hll), - Err(TransmissionPlanError::InvalidRuntimePolicy { .. }) - )); - } -} diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs deleted file mode 100644 index 8cea4e1a5..000000000 --- a/crates/asap_types/src/query_plan.rs +++ /dev/null @@ -1,991 +0,0 @@ -//! Shared installed QueryPlan contract and activation validation. -//! -//! ASAPPlanner owns semantic post-ASAP IR. Physical compilation binds every -//! maintained-summary leaf to one materialization and lowers edges to stable -//! node IDs. Serving executes this graph without reconstructing Planner IR or -//! searching for compatible materializations. - -pub mod current_series; -pub mod query_time; - -use std::collections::{BTreeMap, BTreeSet}; - -use planner_types::post_asap::SketchQuery; -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -pub use crate::QueryLanguage; -use crate::{sds::StoredOutputId, PolicyFingerprint}; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct QueryPlan { - pub plan_id: u64, - pub plan_version: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub clickhouse_context: Option, - /// Selected semantic roots retained for provenance; serving executes - /// `entries` and never reconstructs a plan from these documents. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub selected_dags: BTreeMap, - pub entries: BTreeMap, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ClickHousePlanningContext { - pub tables: std::collections::HashMap, - pub accuracy: planner_types::types::AccuracyTarget, - /// A time template can have several concrete plans with different pane - /// origins or materializations. Keep their physical bindings independent. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub window_templates: BTreeMap>, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct FixedEvaluationRange { - pub start_ms: u64, - pub end_ms: u64, - pub cumulative: bool, -} - -impl QueryPlan { - pub fn empty() -> Self { - Self { - plan_id: 0, - plan_version: 0, - clickhouse_context: None, - selected_dags: BTreeMap::new(), - entries: BTreeMap::new(), - } - } - - pub fn lookup(&self, promql: &str) -> Result<&QueryPlanEntry, QueryPlanError> { - // Installed entries already carry validated canonical identities. The - // common exact spelling requires no serving-time parser invocation. - if let Some(entry) = self.entries.get(promql).filter(|entry| { - entry.language == QueryLanguage::PromQl && entry.canonical_query == promql - }) { - return Ok(entry); - } - let identity = canonical_promql(promql)?; - self.lookup_canonical(QueryLanguage::PromQl, &identity) - } - - pub fn lookup_canonical( - &self, - language: QueryLanguage, - identity: &str, - ) -> Result<&QueryPlanEntry, QueryPlanError> { - let key = Self::catalog_key(language, identity); - self.entries - .get(&key) - .filter(|entry| entry.language == language) - .ok_or_else(|| QueryPlanError::QueryNotPlanned(identity.into())) - } - - pub fn catalog_key(language: QueryLanguage, identity: &str) -> String { - match language { - QueryLanguage::PromQl => identity.to_owned(), - QueryLanguage::MetricsQl => format!("metricsql:{identity}"), - QueryLanguage::ClickHouseSql => format!("clickhouse:{identity}"), - } - } - - pub fn lookup_clickhouse( - &self, - canonical_sql: &str, - ) -> Result<&QueryPlanEntry, QueryPlanError> { - self.lookup_canonical(QueryLanguage::ClickHouseSql, canonical_sql) - } - - pub fn bind_catalog( - &mut self, - catalog: &crate::summary_catalog::SummaryCatalog, - ) -> Result<(), QueryPlanError> { - catalog - .validate() - .map_err(|e| QueryPlanError::Invalid(e.to_string()))?; - for entry in self.entries.values_mut() { - for node in entry.nodes.values_mut() { - if let QueryPlanNode::ReadMaterialization { binding } = node { - binding.stored_output_reference = catalog - .output_reference(binding.materialization) - .map_err(|e| QueryPlanError::Invalid(e.to_string()))?; - } - } - } - self.validate_against_catalog(catalog) - } - - /// Validate semantic bindings against the authoritative snapshot before use. - pub fn validate_against_catalog( - &self, - catalog: &crate::summary_catalog::SummaryCatalog, - ) -> Result<(), QueryPlanError> { - catalog - .validate() - .map_err(|error| QueryPlanError::Invalid(error.to_string()))?; - if self.plan_id != catalog.plan_id || self.plan_version != catalog.plan_version { - return Err(QueryPlanError::Invalid( - "QueryPlan and SummaryCatalog have different plan identity/version".into(), - )); - } - let available = catalog.outputs.keys().copied().map(Into::into).collect(); - self.validate(&available)?; - for entry in self.entries.values() { - for binding in entry.materialization_bindings() { - let identity = catalog - .outputs - .get(&binding.materialization) - .ok_or_else(|| { - QueryPlanError::Invalid( - "query binding references absent catalog materialization".into(), - ) - })?; - if binding.stored_output_reference.definition_id != identity.definition_id { - return Err(QueryPlanError::Invalid( - "read definition differs from installed output".into(), - )); - } - let _data = &catalog.data_descriptors[&identity.data_descriptor_id]; - if binding.window_ms == 0 { - return Err(QueryPlanError::Invalid( - "zero physical pane duration".into(), - )); - } - if binding.full_window_slide_ms == Some(0) { - return Err(QueryPlanError::Invalid( - "query full-window cadence must be nonzero".into(), - )); - } - } - for node in entry.nodes.values() { - let QueryPlanNode::ExactReadout { input, readout } = node else { - continue; - }; - if !matches!(readout, ExactReadout::Increase | ExactReadout::Rate) { - continue; - } - let Some(QueryPlanNode::ReadMaterialization { binding }) = entry.nodes.get(input) - else { - return Err(QueryPlanError::Invalid( - "counter readout must directly consume one catalog materialization".into(), - )); - }; - let identity = &catalog.outputs[&binding.materialization]; - let descriptor = &catalog.summary_descriptors[&identity.summary_descriptor_id]; - if !matches!( - descriptor.fidelity, - crate::sds::FidelityGuarantee::ExactCounter { - full_pane_coverage_required: true, - .. - } - ) { - return Err(QueryPlanError::Invalid( - "rate/increase binding does not reference an exact counter SDS".into(), - )); - } - } - } - Ok(()) - } - - pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { - if self.plan_id != 0 && self.plan_version == 0 { - return Err(QueryPlanError::Invalid( - "non-bootstrap QueryPlan has zero plan_version".into(), - )); - } - for (query_id, selected) in &self.selected_dags { - if query_id != &selected.query_id { - return Err(QueryPlanError::Invalid(format!( - "selected DAG map key `{query_id}` differs from document query ID `{}`", - selected.query_id - ))); - } - if selected.schema_version != crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION - { - return Err(QueryPlanError::Invalid(format!( - "selected DAG `{query_id}` has unsupported schema version {}", - selected.schema_version - ))); - } - selected.decode().map_err(|error| { - QueryPlanError::Invalid(format!("selected DAG `{query_id}` is invalid: {error}")) - })?; - let matching_entries = self - .entries - .values() - .filter(|entry| entry.query_id == *query_id) - .count(); - if matching_entries != 1 { - return Err(QueryPlanError::Invalid(format!( - "selected DAG `{query_id}` must correspond to exactly one query entry; found {matching_entries}" - ))); - } - } - if let Some(context) = &self.clickhouse_context { - for (template, identities) in &context.window_templates { - if !template.starts_with("moving-window-v1:") || identities.is_empty() { - return Err(QueryPlanError::Invalid( - "invalid SQL window template index".into(), - )); - } - let mut seen = BTreeSet::new(); - for identity in identities { - let entry = self.lookup_clickhouse(identity)?; - if !seen.insert(identity) - || entry - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::ExternalExact { .. })) - { - return Err(QueryPlanError::Invalid( - "SQL window template has duplicate or external bindings".into(), - )); - } - } - } - } - for (identity, entry) in &self.entries { - let expected = Self::catalog_key(entry.language, &entry.canonical_query); - if identity != &expected { - return Err(QueryPlanError::Invalid(format!( - "query map key `{identity}` differs from entry identity `{}`", - entry.canonical_query - ))); - } - match entry.language { - QueryLanguage::PromQl | QueryLanguage::MetricsQl - if entry.fixed_evaluation.is_some() => - { - return Err(QueryPlanError::Invalid( - "PromQL query entry carries a ClickHouse fixed evaluation range".into(), - )); - } - QueryLanguage::ClickHouseSql => { - if self.clickhouse_context.is_none() { - return Err(QueryPlanError::Invalid( - "ClickHouse query entry has no planning context".into(), - )); - } - let Some(range) = entry.fixed_evaluation else { - return Err(QueryPlanError::Invalid( - "ClickHouse query entry has no fixed evaluation range".into(), - )); - }; - if range.end_ms <= range.start_ms { - return Err(QueryPlanError::Invalid( - "ClickHouse query entry has an empty evaluation range".into(), - )); - } - } - QueryLanguage::PromQl | QueryLanguage::MetricsQl => {} - } - entry.validate(available)?; - } - Ok(()) - } -} - -pub use crate::executable_plan::QueryNodeId; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct QueryPlanEntry { - #[serde(default)] - pub language: QueryLanguage, - pub query_id: String, - pub canonical_query: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub fixed_evaluation: Option, - pub root: QueryNodeId, - pub nodes: BTreeMap, - pub instant: InstantExecution, - pub fallback: FallbackPolicy, -} - -fn topological_order( - root: QueryNodeId, - nodes: &BTreeMap, -) -> Result, QueryPlanError> { - fn visit( - id: QueryNodeId, - nodes: &BTreeMap, - visiting: &mut BTreeSet, - visited: &mut BTreeSet, - out: &mut Vec, - ) -> Result<(), QueryPlanError> { - if visited.contains(&id) { - return Ok(()); - } - if !visiting.insert(id) { - return Err(QueryPlanError::Invalid(format!( - "cycle detected at query node {}", - id.0 - ))); - } - let node = nodes - .get(&id) - .ok_or_else(|| QueryPlanError::Invalid(format!("missing query node {}", id.0)))?; - for input in node.inputs() { - visit(*input, nodes, visiting, visited, out)?; - } - visiting.remove(&id); - visited.insert(id); - out.push(id); - Ok(()) - } - let mut out = Vec::with_capacity(nodes.len()); - visit( - root, - nodes, - &mut BTreeSet::new(), - &mut BTreeSet::new(), - &mut out, - )?; - Ok(out) -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct InstantExecution { - pub lookback_ms: u64, - pub full_history: bool, - pub cumulative_readout: bool, -} - -impl QueryPlanEntry { - /// Materializations this executable DAG reads, in stable node order. - /// Serving uses this set for readiness accounting; it never performs a - /// catalog candidate search to reconstruct dependencies. - pub fn materialization_bindings(&self) -> Vec<&MaterializationBinding> { - self.nodes - .values() - .filter_map(|node| match node { - QueryPlanNode::ReadMaterialization { binding } => Some(binding), - _ => None, - }) - .collect() - } - - pub fn topological_order(&self) -> Result, QueryPlanError> { - topological_order(self.root, &self.nodes) - } - - pub fn topological_order_from( - &self, - root: QueryNodeId, - ) -> Result, QueryPlanError> { - topological_order(root, &self.nodes) - } - - /// The current external adapters provide evaluation time, not a snapshot - /// token compatible with local SDS or current-series revisions. - pub fn validate_snapshot_sources(&self) -> Result<(), QueryPlanError> { - use query_time::QueryTimeOperator; - let mut local = false; - let mut external = false; - for id in self.topological_order()? { - match &self.nodes[&id] { - QueryPlanNode::ReadMaterialization { .. } - | QueryPlanNode::Logical { - operator: QueryTimeOperator::CurrentSeries { .. }, - .. - } => local = true, - QueryPlanNode::ExternalExact { .. } - | QueryPlanNode::Logical { - operator: - QueryTimeOperator::ExactSubquery { .. } - | QueryTimeOperator::CandidateExactSubquery { .. }, - .. - } => external = true, - _ => {} - } - } - if local && external { - return Err(QueryPlanError::UnsupportedNode( - "local state and external exact input have no common snapshot proof".into(), - )); - } - Ok(()) - } - - /// Validate references, bindings, reachability, and cycles before activation. - pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { - self.validate_snapshot_sources()?; - if !self.nodes.contains_key(&self.root) { - return Err(QueryPlanError::Invalid(format!( - "query `{}` has missing root {}", - self.query_id, self.root.0 - ))); - } - for (id, node) in &self.nodes { - validate_native_relation(*id, node)?; - if let QueryPlanNode::PhysicalFragment { - inputs, - dag, - row_input, - pruning, - } = node - { - let compiled = - asap_physical_operators::physical_planner::CompiledPhysicalDag::decode(dag) - .map_err(|e| QueryPlanError::Invalid(e.to_string()))?; - for (_, contract) in compiled.input_contracts() { - let mut samples = 0; - for (index, field) in contract.schema.fields.iter().enumerate() { - use planner_types::{post_asap::SummaryFamilyType, pre_asap::DataType}; - match &field.dtype { - SummaryFamilyType::Plain(DataType::Float64 | DataType::Int64) => { - samples += 1 - } - SummaryFamilyType::Plain(DataType::Utf8) => {} - SummaryFamilyType::Plain(DataType::Timestamp) - if contract.schema.time_index == Some(index) => {} - _ => { - return Err(QueryPlanError::Invalid(format!( - "PromQL input binding cannot supply field {}", - field.name - ))) - } - } - } - if samples > 1 { - return Err(QueryPlanError::Invalid( - "PromQL vector input has only one numeric sample per row".into(), - )); - } - } - let source = compiled.input_contracts().nth(*row_input).map(|(id, _)| id); - if compiled.roots().len() != 1 - || source.is_none() - || compiled.row_source(compiled.roots()[0]) != source - { - return Err(QueryPlanError::Invalid( - "physical vector output must preserve its bound input rows".into(), - )); - } - if let Some(pruning) = pruning { - if matches!(&pruning.completeness, CandidateCompleteness::Certified { guarantee } - if guarantee.metric != planner_types::post_asap::ErrorMetric::TopKMembership || guarantee.bound.evaluate().is_none() || guarantee.failure_probability.evaluate().is_none()) - { - return Err(QueryPlanError::Invalid( - "invalid physical pruning certificate".into(), - )); - } - let contracts = compiled.input_contracts().collect::>(); - let left = contracts - .get(*row_input) - .ok_or_else(|| QueryPlanError::Invalid("invalid row input".into()))? - .1; - let right = contracts - .get(pruning.candidate_input) - .ok_or_else(|| QueryPlanError::Invalid("invalid candidate input".into()))? - .1; - asap_physical_operators::operators::Operator::semi_join( - left.schema.clone(), - right.schema.clone(), - pruning.keys.clone(), - ) - .map_err(|e| QueryPlanError::Invalid(e.to_string()))?; - } - if compiled.input_contracts().count() != inputs.len() - || compiled.roots().len() != 1 - || *row_input >= inputs.len() - { - return Err(QueryPlanError::Invalid( - "physical input/root binding mismatch".into(), - )); - } - } - - if let QueryPlanNode::Logical { operator, inputs } = node { - operator.validate(inputs.len())?; - } - if matches!(node, QueryPlanNode::Scalar { value } if !value.is_finite()) { - return Err(QueryPlanError::Invalid("non-finite scalar constant".into())); - } - if let QueryPlanNode::ExternalExact { request, inputs } = node { - if request.expression.trim().is_empty() { - return Err(QueryPlanError::Invalid( - "external exact expression must not be empty".into(), - )); - } - if request.input_contracts.len() != inputs.len() { - return Err(QueryPlanError::Invalid( - "external exact input contracts must match DAG inputs".into(), - )); - } - if request.input_contracts.iter().any(|contract| { - matches!(contract, ExternalExactInput::CandidateMembership { item_label } if item_label.is_empty()) - }) { - return Err(QueryPlanError::Invalid( - "external exact candidate item label must not be empty".into(), - )); - } - } - if let QueryPlanNode::RelationalJoin { - pruning: Some(completeness), - join_kind, - .. - } = node - { - if *join_kind != planner_types::pre_asap::JoinKind::Semi { - return Err(QueryPlanError::Invalid( - "pruning evidence requires a semi-join".into(), - )); - } - if matches!( - completeness, - CandidateCompleteness::Certified { guarantee } - if guarantee.metric - != planner_types::post_asap::ErrorMetric::TopKMembership - || guarantee.bound.evaluate().is_none() - || guarantee.failure_probability.evaluate().is_none() - ) { - return Err(QueryPlanError::Invalid( - "invalid semi-join pruning certificate".into(), - )); - } - } - for input in node.inputs() { - if !self.nodes.contains_key(input) { - return Err(QueryPlanError::Invalid(format!( - "query `{}` node {} references missing input {}", - self.query_id, id.0, input.0 - ))); - } - } - if let QueryPlanNode::ReadMaterialization { binding } = node { - if binding.stored_output_reference.stored_output_id != binding.materialization { - return Err(QueryPlanError::Invalid( - "read binding has invalid stored output or definition".into(), - )); - } - if binding.readout_lookback_ms == Some(0) { - return Err(QueryPlanError::Invalid( - "zero semantic readout lookback".into(), - )); - } - if !available.contains(&binding.materialization.fingerprint()) { - return Err(QueryPlanError::Invalid(format!( - "query `{}` node {} references absent materialization {}", - self.query_id, - id.0, - binding.materialization.as_u64() - ))); - } - } - } - let order = self.topological_order()?; - if order.len() != self.nodes.len() { - return Err(QueryPlanError::Invalid(format!( - "query `{}` contains unreachable nodes", - self.query_id - ))); - } - Ok(()) - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum FallbackPolicy { - ExactBackend, - Reject, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct MaterializationBinding { - pub stored_output_reference: crate::sds::StoredOutputReference, - /// Complete-window storage advances independently of its stored extent. - /// None denotes disjoint pane storage. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub full_window_slide_ms: Option, - pub materialization: StoredOutputId, - /// Query operator grouping applied while folding those SIDs. - pub output_grouping: PhysicalGrouping, - /// Labels whose values form an item identity inside a keyed sketch. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub item_labels: Vec, - pub window_ms: u64, - /// Unix millisecond timestamp on the materialized pane-boundary grid. - /// Legacy plans deserialize this as unknown and fall back at read time. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub pane_origin_ms: Option, - /// Semantic query lookback, independent of the physical pane duration. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub readout_lookback_ms: Option, -} - -impl MaterializationBinding { - /// Both range boundaries must identify complete stored state. Full windows - /// use a start grid; their end grid is displaced by the window width. - pub fn covers_range(&self, start_ms: u64, end_ms: u64) -> bool { - let Some(origin) = self.pane_origin_ms else { - return false; - }; - if self.window_ms == 0 || end_ms <= start_ms { - return false; - } - let start = i128::from(start_ms) - i128::from(origin); - match self.full_window_slide_ms { - Some(slide) => { - slide != 0 - && end_ms - start_ms == self.window_ms - && start.rem_euclid(i128::from(slide)) == 0 - } - None => { - start.rem_euclid(i128::from(self.window_ms)) == 0 - && (i128::from(end_ms) - i128::from(origin)) - .rem_euclid(i128::from(self.window_ms)) - == 0 - } - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "mode", content = "keys", rename_all = "snake_case")] -pub enum PhysicalGrouping { - PerEntity, - Reduce(Vec), -} - -/// Result shape promised by an external exact engine. The backend uses this -/// contract to type-check downstream DAG nodes without depending on an -/// engine-specific response envelope. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum ExternalExactOutput { - Scalar, - InstantVector, - RangeVector, - Relation { schema: serde_json::Value }, -} - -/// How an ordinary DAG input constrains an external exact evaluation. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum ExternalExactInput { - CandidateMembership { item_label: String }, -} - -/// Language-neutral request contract for an exact subtree. Evaluation time is -/// inherited from the containing QueryPlanEntry, avoiding a second time-range -/// envelope that could drift from the installed query plan. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct ExternalExactRequest { - pub language: QueryLanguage, - pub expression: String, - pub output: ExternalExactOutput, - /// Engine parameters forwarded without embedding transport details in the DAG. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub parameters: BTreeMap, - /// Optional parameter names populated from the query entry's evaluation range. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub start_parameter: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub end_parameter: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub input_contracts: Vec, -} - -/// Required candidate rows must have authoritative values at the bound source. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct PruningInputContract { - pub candidate_input: usize, - pub keys: Vec<(usize, usize)>, - pub completeness: CandidateCompleteness, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "op", rename_all = "snake_case", deny_unknown_fields)] -pub enum QueryPlanNode { - /// Planner-compiled computation. Input order follows the physical input contracts. - PhysicalFragment { - inputs: Vec, - dag: Vec, - row_input: usize, - pruning: Option, - }, - - RelationalJoin { - inputs: [QueryNodeId; 2], - join_kind: planner_types::pre_asap::JoinKind, - pruning: Option, - pred: serde_json::Value, - left_schema: planner_types::post_asap::SummarySchema, - right_schema: planner_types::post_asap::SummarySchema, - output_schema: planner_types::post_asap::SummarySchema, - }, - Relational { - input: QueryNodeId, - /// Serialized planner-owned operation. Keeping the wire form here makes - /// the published catalog Send + Sync even though the planner AST uses Rc. - operation: serde_json::Value, - input_schema: planner_types::post_asap::SummarySchema, - output_schema: planner_types::post_asap::SummarySchema, - }, - Logical { - operator: query_time::QueryTimeOperator, - inputs: Vec, - }, - Scalar { - value: f64, - }, - Binary { - inputs: [QueryNodeId; 2], - operator: planner_types::pre_asap::ArithmeticOpKind, - }, - ReduceSum { - input: QueryNodeId, - grouping: PhysicalGrouping, - }, - ReadMaterialization { - binding: MaterializationBinding, - }, - SummaryEstimate { - input: QueryNodeId, - query: QueryReadout, - }, - ExactReadout { - input: QueryNodeId, - readout: ExactReadout, - }, - SummaryMerge { - inputs: Vec, - }, - /// An exact subtree evaluated outside ASAP. Its results enter the query DAG - /// like any other node output and may depend on summary-produced inputs. - ExternalExact { - request: ExternalExactRequest, - inputs: Vec, - }, - ExactFallback { - reason: String, - }, -} - -impl QueryPlanNode { - pub fn inputs(&self) -> &[QueryNodeId] { - match self { - Self::Scalar { .. } | Self::ReadMaterialization { .. } | Self::ExactFallback { .. } => { - &[] - } - Self::Binary { inputs, .. } | Self::RelationalJoin { inputs, .. } => inputs, - Self::ReduceSum { input, .. } - | Self::Relational { input, .. } - | Self::SummaryEstimate { input, .. } - | Self::ExactReadout { input, .. } => std::slice::from_ref(input), - Self::PhysicalFragment { inputs, .. } - | Self::SummaryMerge { inputs } - | Self::Logical { inputs, .. } - | Self::ExternalExact { inputs, .. } => inputs, - } - } -} - -pub use planner_types::post_asap::CandidateCompleteness; - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum ExactReadout { - Sum, - Count, - Increase, - Rate, - Min, - Max, -} - -impl ExactReadout { - /// Planner family required by this installed DAG readout node. - pub fn planner_family(self) -> planner_types::post_asap::SummaryFamilyType { - use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; - let (kind, params) = match self { - Self::Sum => (ExactKind::Sum, ExactParams::Sum), - Self::Count => (ExactKind::Count, ExactParams::Count), - Self::Increase => (ExactKind::Increase, ExactParams::Increase), - Self::Rate => (ExactKind::Rate, ExactParams::Rate), - Self::Min => (ExactKind::Min, ExactParams::Min), - Self::Max => (ExactKind::Max, ExactParams::Max), - }; - SummaryFamilyType::ExactAggregate(kind, params) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum QueryReadout { - FrequencyL2, - FrequencyEntropy, - Quantile { - q: f64, - }, - PointCount { - key: planner_types::pre_asap::ColumnRef, - value: Option, - }, - Cardinality, - TopK { - k: usize, - }, -} - -impl From for QueryReadout { - fn from(query: SketchQuery) -> Self { - match query { - SketchQuery::FrequencyL2 => Self::FrequencyL2, - SketchQuery::FrequencyEntropy => Self::FrequencyEntropy, - SketchQuery::Quantile { q } => Self::Quantile { q }, - SketchQuery::PointCount { key, value } => Self::PointCount { key, value }, - SketchQuery::Cardinality => Self::Cardinality, - SketchQuery::TopK { k } => Self::TopK { k }, - } - } -} - -impl From for SketchQuery { - fn from(query: QueryReadout) -> Self { - match query { - QueryReadout::FrequencyL2 => Self::FrequencyL2, - QueryReadout::FrequencyEntropy => Self::FrequencyEntropy, - QueryReadout::Quantile { q } => Self::Quantile { q }, - QueryReadout::PointCount { key, value } => Self::PointCount { key, value }, - QueryReadout::Cardinality => Self::Cardinality, - QueryReadout::TopK { k } => Self::TopK { k }, - } - } -} - -#[derive(Debug, Error)] -pub enum QueryPlanError { - #[error("invalid PromQL query identity: {0}")] - InvalidPromql(String), - #[error("query is absent from the active QueryPlan: {0}")] - QueryNotPlanned(String), - #[error("post-ASAP DAG cannot be represented by the query executor: {0}")] - UnsupportedNode(String), - #[error("invalid QueryPlan: {0}")] - Invalid(String), -} - -pub fn canonical_promql(query: &str) -> Result { - promql_parser::parser::parse(query.trim()) - .map(|expr| expr.to_string()) - .map_err(|error| QueryPlanError::InvalidPromql(error.to_string())) -} - -#[cfg(test)] -mod contract_tests { - // Installed plans cross producer/query threads without Planner Rc state. - #[test] - fn installed_query_contract_is_send_sync() { - fn assert_send_sync() {} - assert_send_sync::(); - assert_send_sync::(); - } -} - -/// Bind portable relation semantics before an installed plan can access its sources. -fn validate_native_relation(id: QueryNodeId, node: &QueryPlanNode) -> Result<(), QueryPlanError> { - use planner_types::post_asap::{ - ExecutableDagNode, ExecutableOperatorPayload as Payload, ExecutionDataState, PostAsapNodeId, - }; - use std::sync::Arc; - let invalid = |error: String| QueryPlanError::Invalid(format!("query node {}: {error}", id.0)); - let (payload, inputs, output) = match node { - QueryPlanNode::Relational { - operation, - input_schema, - output_schema, - .. - } => ( - Payload::Value { - operation: serde_json::from_value(operation.clone()) - .map_err(|e| invalid(e.to_string()))?, - }, - vec![Arc::new(input_schema.clone())], - output_schema, - ), - QueryPlanNode::RelationalJoin { - join_kind, - pred, - left_schema, - right_schema, - output_schema, - pruning, - .. - } => ( - Payload::RelationalJoin { - join_kind: join_kind.clone(), - pred: serde_json::from_value(pred.clone()).map_err(|e| invalid(e.to_string()))?, - pruning: serde_json::from_value( - serde_json::to_value(pruning).map_err(|e| invalid(e.to_string()))?, - ) - .map_err(|e| invalid(e.to_string()))?, - }, - vec![ - Arc::new(left_schema.clone()), - Arc::new(right_schema.clone()), - ], - output_schema, - ), - _ => return Ok(()), - }; - let node = ExecutableDagNode { - id: PostAsapNodeId(0), - payload, - output_state: ExecutionDataState::QUERY_ROWS, - output_schema: output.clone(), - guarantee: None, - }; - asap_physical_operators::physical_planner::compile_node(&node, &inputs) - .map_err(|e| invalid(e.to_string()))?; - Ok(()) -} - -#[cfg(test)] -mod native_binding_tests { - use super::*; - use planner_types::{ - post_asap::{SummaryFamilyType, SummaryField, SummarySchema, ValueOperation}, - pre_asap::{DataType, Predicate, QueryExpr}, - }; - - // Unsupported expressions fail installation without evaluating any source. - #[test] - fn rejects_unimplemented_relation_predicate_before_execution() { - let schema = SummarySchema { - fields: vec![SummaryField { - name: "value".into(), - dtype: SummaryFamilyType::Plain(DataType::Float64), - nullable: false, - }], - time_index: None, - }; - let node = QueryPlanNode::Relational { - input: QueryNodeId(0), - operation: serde_json::to_value(ValueOperation::Filter { - pred: Predicate( - QueryExpr::FunctionCall { - name: "unimplemented_predicate".into(), - args: vec![], - } - .into(), - ), - }) - .unwrap(), - input_schema: schema.clone(), - output_schema: schema, - }; - assert!(validate_native_relation(QueryNodeId(1), &node).is_err()); - } -} diff --git a/crates/asap_types/src/query_plan/current_series.rs b/crates/asap_types/src/query_plan/current_series.rs deleted file mode 100644 index b109756b6..000000000 --- a/crates/asap_types/src/query_plan/current_series.rs +++ /dev/null @@ -1,88 +0,0 @@ -//! Maintained current-value populations, shared independently of q and k. -use super::{ - query_time::{Grouping, LabelMatcher}, - QueryPlanError, -}; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct SeriesPopulation { - pub metric: String, - pub matchers: Vec, - pub grouping: Grouping, - pub lookback_ms: u64, - pub max_input_lag_ms: u64, - /// Admitted historical distance behind ingestion; zero retains current state only. - #[serde(default)] - pub history_retention_ms: u64, - pub max_series: usize, - pub max_bytes: u64, - pub max_k: u64, - pub quantiles: bool, -} -impl SeriesPopulation { - pub fn key(&self) -> String { - serde_json::to_string(self).expect("serializable current-series population") - } - pub fn validate(&self) -> Result<(), QueryPlanError> { - if self.metric.is_empty() - || self.lookback_ms == 0 - || self.lookback_ms > i64::MAX as u64 - || self.max_input_lag_ms == 0 - || self.max_input_lag_ms > self.lookback_ms - || self.history_retention_ms > i64::MAX as u64 - || self.max_series == 0 - || self.max_series > 100_000 - || self.max_bytes == 0 - || self.max_bytes > 1_073_741_824 - || self.max_k > self.max_series as u64 - { - return Err(QueryPlanError::Invalid( - "invalid current-series population bounds".into(), - )); - } - Ok(()) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum SeriesReadout { - Quantile { q: f64 }, - TopK { k: u64 }, - Sum, - Count, - Average, -} - -#[cfg(test)] -mod tests { - use super::*; - // Planner-declared horizons are valid independently of the old five-minute default. - #[test] - fn accepts_declared_horizon_with_bounded_lag() { - let mut population = SeriesPopulation { - metric: "a".into(), - matchers: vec![], - grouping: Grouping { - labels: vec![], - without: false, - }, - lookback_ms: 1_000, - max_input_lag_ms: 1_000, - history_retention_ms: 0, - max_series: 100, - max_bytes: 1_000_000, - max_k: 3, - quantiles: true, - }; - population.validate().unwrap(); - population.lookback_ms = 0; - assert!(population.validate().is_err()); - population.lookback_ms = u64::MAX; - assert!(population.validate().is_err()); - population.lookback_ms = 999; - assert!(population.validate().is_err()); - } -} diff --git a/crates/asap_types/src/query_plan/query_time.rs b/crates/asap_types/src/query_plan/query_time.rs deleted file mode 100644 index a5b3fa9ef..000000000 --- a/crates/asap_types/src/query_plan/query_time.rs +++ /dev/null @@ -1,188 +0,0 @@ -//! Typed installed query-time operators; no Planner selection or AST lowering. -use super::QueryPlanError; -use promql_parser::parser::{self, Expr}; -use serde::{Deserialize, Serialize}; -fn invalid(message: impl Into) -> QueryPlanError { - QueryPlanError::Invalid(message.into()) -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum QueryTimeOperator { - /// Readout over a bounded current-value population maintained at ingest. - CurrentSeries { - population: super::current_series::SeriesPopulation, - readout: super::current_series::SeriesReadout, - }, - /// A maximal exact scalar/vector subtree evaluated by Prometheus. - ExactSubquery { - query: String, - }, - /// Prometheus exact subtree whose selectors are restricted at runtime by - /// the candidate vector produced by its single input. - CandidateExactSubquery { - query: String, - item_label: String, - }, - Scan { - metric: Option, - matchers: Vec, - range_ms: Option, - offset_ms: i64, - }, - UnaryNegate, - VectorToScalar, - Aggregate { - operation: Aggregation, - grouping: Grouping, - }, - /// Select an ordered slice independently within each group. - Limit { - n: u64, - offset: u64, - grouping: Grouping, - }, - Binary { - operation: BinaryOperation, - return_bool: bool, - }, - Temporal { - operation: TemporalOperation, - }, - Sort { - descending: bool, - grouping: Grouping, - }, - HistogramQuantile, - Subquery { - range_ms: u64, - step_ms: u64, - offset_ms: i64, - }, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct Grouping { - pub labels: Vec, - pub without: bool, -} -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct LabelMatcher { - pub name: String, - pub value: String, - pub operation: LabelMatch, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum LabelMatch { - Equal, - NotEqual, - Regex, - NotRegex, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum Aggregation { - Sum, - Max, - Min, - Avg, - Count, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum BinaryOperation { - Add, - Sub, - Mul, - Div, - /// Division with the Planner relative-value certificate domain checks. - CheckedDiv, - /// Division for conditional exact rewrites: finite inputs and finite output. - FiniteDiv, - Mod, - Pow, - Equal, - NotEqual, - Less, - LessEqual, - Greater, - GreaterEqual, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum TemporalOperation { - Rate, - Increase, - Avg, - Max, - Min, - Sum, - Count, -} - -impl QueryTimeOperator { - pub fn validate(&self, inputs: usize) -> Result<(), QueryPlanError> { - if let Self::CurrentSeries { - population, - readout, - } = self - { - population.validate()?; - match readout { - super::current_series::SeriesReadout::Quantile { q } - if !q.is_finite() || !population.quantiles => - { - return Err(invalid( - "quantile readout requires finite q and a quantile population", - )) - } - super::current_series::SeriesReadout::TopK { k } if *k > population.max_k => { - return Err(invalid("TopK readout exceeds shared population capacity")) - } - _ => {} - } - } - let expected = match self { - Self::Scan { .. } | Self::ExactSubquery { .. } | Self::CurrentSeries { .. } => 0, - Self::CandidateExactSubquery { .. } => 1, - Self::Binary { .. } | Self::HistogramQuantile => 2, - _ => 1, - }; - if inputs != expected { - return Err(invalid("logical operator input arity mismatch")); - } - if matches!( - self, - Self::Scan { - range_ms: Some(0), - .. - } - ) { - return Err(invalid("zero range")); - } - if let Self::ExactSubquery { query } | Self::CandidateExactSubquery { query, .. } = self { - let parsed = parser::parse(query).map_err(|e| invalid(e.to_string()))?; - if matches!(parsed, Expr::MatrixSelector(_) | Expr::Subquery(_)) { - return Err(invalid( - "exact subtree boundary must return scalar or instant vector", - )); - } - } - if let Self::Subquery { - range_ms, step_ms, .. - } = self - { - if *range_ms == 0 || *step_ms == 0 || range_ms / step_ms > 100_000 { - return Err(invalid("invalid or excessive subquery grid")); - } - } - Ok(()) - } -} - -// Compatibility imports; new callers use the domain names above. -#[deprecated(note = "Use QueryTimeOperator")] -pub use QueryTimeOperator as LogicalOperator; diff --git a/crates/asap_types/src/query_requirements.rs b/crates/asap_types/src/query_requirements.rs deleted file mode 100644 index f7a4b264e..000000000 --- a/crates/asap_types/src/query_requirements.rs +++ /dev/null @@ -1,21 +0,0 @@ -use crate::KeyByLabelNames; -use crate::Statistic; - -/// What a query needs in order to be answered by a stored aggregation. -#[derive(Debug, Clone)] -pub struct QueryRequirements { - /// Metric name (PromQL) or "table_name.value_column" (SQL). - pub metric: String, - /// One or more statistics needed. - /// For avg this is [Sum, Count]; for everything else it is a single element. - /// All statistics must be satisfied by aggregations sharing the same - /// window_size and grouping_labels. - pub statistics: Vec, - /// The span of historical data the query reads, in milliseconds. - /// None for spatial-only queries (no time range). - pub data_range_ms: Option, - /// GROUP BY labels expected in the query result. - pub grouping_labels: KeyByLabelNames, - /// Normalized label filter (produced by normalize_spatial_filter). - pub spatial_filter_normalized: String, -} diff --git a/crates/asap_types/src/routing_index.rs b/crates/asap_types/src/routing_index.rs deleted file mode 100644 index ec20b7ef2..000000000 --- a/crates/asap_types/src/routing_index.rs +++ /dev/null @@ -1,270 +0,0 @@ -//! `RoutingIndex` — a metric-bucketed structural index over a -//! [`PolicyRegistry`]. It is sourced from the content-addressed view over a -//! `InstalledPrecomputePlan`'s `PrecomputeMaterialization`s, so it represents planned policy -//! rather than a reconstruction from ingest side effects. -//! -//! **Tier 1** (exact `PolicyFingerprint` → config) is [`PolicyRegistry::get`] -//! itself — already O(1), nothing to add here. -//! -//! **Tier 2** (structural match: "every policy registered for this metric") -//! is what this type adds. Before this existed, -//! `control_plane::asap_tier_analysis::find_matching_policies` scanned -//! *every* policy in the registry for every candidate, checking each one's -//! metric name first — i.e. it paid for every OTHER metric's policies on -//! every lookup. `RoutingIndex` buckets by metric once, at construction -//! time, so a lookup only ever touches the policies that could possibly -//! match. -//! -//! ## Lifecycle -//! -//! Built fresh from a `PolicyRegistry` snapshot — construction is `O(N)` -//! over the registry, same order as `PolicyRegistry::from_configs` itself. -//! Callers that build one per query (mirroring today's -//! `streaming_snap.policy_registry()` call) still get the Tier-2 win for -//! any query with more than one candidate sharing the same snapshot -//! (composed PromQL shapes routinely do). Building it once per -//! `InstalledPrecomputePlan` hot-reload swap instead of once per query — the same -//! "cheap, but call at swap time not per query, if it shows up in -//! profiles" note `InstalledPrecomputePlan::policy_registry`'s own doc comment -//! already flags — is a further, larger change (it means threading a -//! cached derived value through `InstalledPrecomputePlanHandle`'s swap path) -//! and is not done by this type on its own. - -use std::collections::{BTreeSet, HashMap}; - -use crate::aggregation_config::PrecomputeMaterialization; -use crate::policy_fingerprint::PolicyFingerprint; -use crate::policy_registry::PolicyRegistry; - -/// See module docs. -#[derive(Debug, Clone, Default)] -pub struct RoutingIndex { - registry: PolicyRegistry, - by_metric: HashMap>, -} - -impl RoutingIndex { - /// Build from a `PolicyRegistry` snapshot. Takes ownership rather than - /// borrowing — callers that still need their own `PolicyRegistry` - /// handle after this should `.clone()` it first (cheap-ish, but real; - /// most callers don't need the raw registry once they have the index, - /// since [`Self::get`] delegates straight through). - pub fn build(registry: PolicyRegistry) -> Self { - let mut by_metric: HashMap> = HashMap::new(); - for (fp, cfg) in registry.iter() { - by_metric.entry(cfg.metric.clone()).or_default().push(*fp); - } - Self { - registry, - by_metric, - } - } - - /// Tier 1 — exact fingerprint lookup. Delegates to the underlying - /// registry; see [`PolicyRegistry::get`]. - pub fn get(&self, fp: PolicyFingerprint) -> Option<&PrecomputeMaterialization> { - self.registry.get(fp) - } - - /// Tier 2 — every policy fingerprint registered for `metric`, in - /// registration order. Empty slice (not an `Option`/error) when - /// nothing is registered for this metric — callers already treat "no - /// candidates" as a normal, expected outcome (capability miss → - /// archive fallback), not a failure to report. - pub fn candidates_for_metric(&self, metric: &str) -> &[PolicyFingerprint] { - self.by_metric.get(metric).map(Vec::as_slice).unwrap_or(&[]) - } - - /// Live policy count (same as the underlying registry's). - pub fn len(&self) -> usize { - self.registry.len() - } - - pub fn is_empty(&self) -> bool { - self.registry.is_empty() - } - - /// All fingerprints currently registered. Delegates to the underlying - /// registry — see [`PolicyRegistry::fingerprints`]. - pub fn fingerprints(&self) -> impl Iterator + '_ { - self.registry.fingerprints() - } - - /// Resolve an ingest-side sketch shape to exactly one configured policy. - /// Ambiguous or absent matches fail closed. - pub fn find_policy_by_content( - &self, - metric: &str, - group_by_keys: &BTreeSet, - agg_type: crate::AggregationType, - expected_params: &HashMap, - ) -> Option { - let mut hit = None; - for fp in self.candidates_for_metric(metric) { - let cfg = self.get(*fp)?; - let policy_keys: BTreeSet<_> = cfg.grouping_labels.iter().cloned().collect(); - if cfg.aggregation_type != agg_type - || &policy_keys != group_by_keys - || !cfg.spatial_filter_normalized.is_empty() - || !expected_params.iter().all(|(key, value)| { - cfg.parameters.get(key).or_else(|| match key.as_str() { - // The typed physical-plan compiler names this field - // after SummaryParams, while legacy collector YAML - // uses the equivalent runtime-facing name. - "relative_accuracy" => cfg.parameters.get("alpha"), - "alpha" => cfg.parameters.get("relative_accuracy"), - _ => None, - }) == Some(value) - }) - { - continue; - } - if hit.is_some() { - return None; - } - hit = Some(*fp); - } - hit - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::enums::WindowKind; - use crate::AggregationType; - use crate::KeyByLabelNames; - use std::collections::HashMap as StdHashMap; - - fn cfg(metric: &str) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - StdHashMap::new(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ) - } - - #[test] - fn buckets_by_metric() { - let a1 = cfg("http_lat"); - let a2 = cfg("cpu_pct"); - let fp_a1 = PolicyFingerprint::from_config(&a1); - let fp_a2 = PolicyFingerprint::from_config(&a2); - let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![a1, a2])); - - assert_eq!(idx.candidates_for_metric("http_lat"), &[fp_a1]); - assert_eq!(idx.candidates_for_metric("cpu_pct"), &[fp_a2]); - } - - #[test] - fn multiple_policies_for_the_same_metric_all_bucket_together() { - // Same metric, distinct group-by shapes -> distinct fingerprints, - // same bucket. - let a = PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - StdHashMap::new(), - KeyByLabelNames::new(vec!["zone".to_string()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - "http_lat".to_string(), - None, - None, - None, - ); - let b = cfg("http_lat"); - assert_ne!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![a, b])); - assert_eq!(idx.candidates_for_metric("http_lat").len(), 2); - } - - #[test] - fn unknown_metric_returns_empty_slice_not_missing() { - let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![cfg("http_lat")])); - assert!(idx.candidates_for_metric("no_such_metric").is_empty()); - } - - #[test] - fn get_delegates_to_underlying_registry() { - let a = cfg("http_lat"); - let fp = PolicyFingerprint::from_config(&a); - let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![a])); - assert_eq!( - idx.get(fp).map(|c| c.metric.clone()), - Some("http_lat".to_string()) - ); - assert!(idx - .get(PolicyFingerprint::from_config(&cfg("nope"))) - .is_none()); - } - - #[test] - fn len_and_is_empty_match_registry() { - let idx = RoutingIndex::build(PolicyRegistry::from_configs( - Vec::::new(), - )); - assert!(idx.is_empty()); - assert_eq!(idx.len(), 0); - - let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![cfg("m")])); - assert!(!idx.is_empty()); - assert_eq!(idx.len(), 1); - } - - #[test] - fn ddsketch_alpha_and_relative_accuracy_are_wire_compatible() { - let mut parameters = StdHashMap::new(); - parameters.insert("alpha".to_string(), serde_json::json!(0.01)); - let config = PrecomputeMaterialization::new( - AggregationType::DDSketch, - String::new(), - parameters, - KeyByLabelNames::new(vec!["service".to_string()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 5, - 5, - WindowKind::Tumbling, - String::new(), - "latency".to_string(), - None, - None, - None, - ); - let fingerprint = PolicyFingerprint::from_config(&config); - let index = RoutingIndex::build(PolicyRegistry::from_configs(vec![config])); - let expected = - StdHashMap::from([("relative_accuracy".to_string(), serde_json::json!(0.01))]); - - assert_eq!( - index.find_policy_by_content( - "latency", - &BTreeSet::from(["service".to_string()]), - AggregationType::DDSketch, - &expected, - ), - Some(fingerprint) - ); - } -} diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs deleted file mode 100644 index a6870ea20..000000000 --- a/crates/asap_types/src/sds.rs +++ /dev/null @@ -1,1876 +0,0 @@ -//! Shared contracts for summary definitions, stored-summary metadata, and plan -//! output bindings. Payload bytes remain storage-engine owned and logically -//! belong to the stored summary identified by this metadata. -pub const TIMESTAMPED_OBSERVATION_SEMANTICS: &str = "asap.timestamped-observations.v2"; - -use crate::{AggregationType, PrecomputeMaterialization}; -use planner_types::post_asap::{SketchAlgorithm, SketchParams, SummaryFamilyType}; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; -use std::collections::BTreeMap; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SdsError(pub String); -impl std::fmt::Display for SdsError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.0) - } -} -impl std::error::Error for SdsError {} - -macro_rules! descriptor_id { - ($name:ident) => { - #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] - #[serde(transparent)] - pub struct $name(String); - impl $name { - pub fn canonical(&self) -> &str { - &self.0 - } - } - }; -} -/// Identity of one deployed producer output. Runtime routing uses this identity, -/// never the semantic definition hash. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(transparent)] -pub struct StoredOutputId(pub u64); -impl StoredOutputId { - pub fn fingerprint(self) -> crate::PolicyFingerprint { - crate::PolicyFingerprint(self.0) - } - pub fn as_u64(self) -> u64 { - self.0 - } -} -impl From for StoredOutputId { - fn from(value: u64) -> Self { - Self(value) - } -} -impl From for StoredOutputId { - fn from(value: crate::PolicyFingerprint) -> Self { - Self(value.0) - } -} -impl From for crate::PolicyFingerprint { - fn from(value: StoredOutputId) -> Self { - Self(value.0) - } -} - -descriptor_id!(SummaryDefinitionId); -impl SummaryDefinitionId { - pub(crate) fn from_semantics(bytes: &[u8]) -> Self { - use sha2::{Digest, Sha256}; - Self(format!("sds-v1:{:x}", Sha256::digest(bytes))) - } - pub fn validate(&self) -> Result<(), SdsError> { - let hash = self.0.strip_prefix("sds-v1:").unwrap_or(""); - if hash.len() == 64 - && hash - .bytes() - .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) - { - Ok(()) - } else { - Err(SdsError("invalid semantic definition ID".into())) - } - } -} - -/// The installed writer/reader binding joins deployment identity and semantics. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct StoredOutputReference { - pub stored_output_id: StoredOutputId, - pub definition_id: SummaryDefinitionId, -} -impl StoredOutputReference { - /// Internal compilation placeholder. Catalog binding must replace its empty - /// semantic identity before installation; it cannot authorize a read/write. - pub fn for_output(stored_output_id: StoredOutputId) -> Self { - Self { - stored_output_id, - definition_id: SummaryDefinitionId(String::new()), - } - } - pub fn validate(&self) -> Result<(), SdsError> { - if self.stored_output_id.0 == 0 { - return Err(SdsError("stored output identity must be set".into())); - } - self.definition_id.validate() - } -} - -/// Canonical address of one stored DAG output for one population and window. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct StoredSummaryKey { - pub plan_id: u64, - pub plan_version: u64, - pub stored_output_id: StoredOutputId, - pub population: BTreeMap, - pub window: HalfOpenTimeRange, -} - -impl StoredSummaryKey { - pub fn validate(&self) -> Result<(), SdsError> { - if self.stored_output_id.0 == 0 { - return Err(SdsError("stored output identity must be set".into())); - } - if self.window.start_ms >= self.window.end_ms { - return Err(SdsError("stored summary requires a nonempty window".into())); - } - Ok(()) - } - - pub fn storage_key(&self) -> Result { - self.validate()?; - serde_json::to_string(self).map_err(|e| SdsError(e.to_string())) - } -} - -descriptor_id!(SummaryDescriptorId); -descriptor_id!(DataDescriptorId); - -descriptor_id!(SummaryInstanceId); - -impl SummaryInstanceId { - pub fn new(value: impl Into) -> Result { - let value = value.into(); - if value.trim().is_empty() { - return Err(SdsError("summary instance ID must not be empty".into())); - } - Ok(Self(value)) - } - - pub fn validate(&self) -> Result<(), SdsError> { - if self.0.trim().is_empty() { - Err(SdsError("summary instance ID must not be empty".into())) - } else { - Ok(()) - } - } -} - -/// Immutable identity of the desired catalog generation used to create state. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct CatalogGeneration { - pub schema_version: u32, - pub plan_id: u64, - pub plan_version: u64, - pub snapshot_sha256: String, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct HalfOpenTimeRange { - pub start_ms: i64, - pub end_ms: i64, -} - -/// One ordered producer partition. Completion and watermark claims are scoped -/// to this identity; a maximum timestamp observed by an unrelated worker is -/// never a source-completeness signal. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SummarySourcePartition { - pub producer_id: String, - pub partition_id: String, - /// Changes whenever a producer restarts or loses its sequence state. - pub producer_epoch: u64, -} - -/// A closed materialization bucket, including named group values so a -/// downstream maintenance DAG can project or shuffle groups without parsing a -/// routing key. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SummaryInstanceCoordinates { - pub stored_output_id: StoredOutputId, - pub time_range: HalfOpenTimeRange, - pub group_values: BTreeMap, -} - -impl SummaryInstanceCoordinates { - pub fn instance_id(&self) -> Result { - let bytes = - serde_json::to_vec(&self.group_values).map_err(|error| SdsError(error.to_string()))?; - SummaryInstanceId::new(format!( - "summary-instance:v1:{}:{}:{}:{}", - self.stored_output_id.as_u64(), - self.time_range.start_ms, - self.time_range.end_ms, - xxhash_rust::xxh64::xxh64(&bytes, 0) - )) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SummaryWindowCompletion { - pub catalog_generation: CatalogGeneration, - pub source: SummarySourcePartition, - /// Concrete SDS instance identity allocated by the storage engine. - pub instance_id: SummaryInstanceId, - pub coordinates: SummaryInstanceCoordinates, - /// Collision-free opaque producer lineage. Retries repeat these bytes. - pub input_lineage: Vec, -} - -impl HalfOpenTimeRange { - pub fn validate(self) -> Result<(), SdsError> { - if self.start_ms >= self.end_ms { - Err(SdsError( - "time range must be non-empty and half-open".into(), - )) - } else { - Ok(()) - } - } -} - -impl SummarySourcePartition { - pub fn validate(&self) -> Result<(), SdsError> { - if self.producer_id.trim().is_empty() || self.partition_id.trim().is_empty() { - return Err(SdsError( - "source producer and partition must be non-empty".into(), - )); - } - if self.producer_epoch == 0 { - return Err(SdsError("source producer epoch must be positive".into())); - } - Ok(()) - } -} - -impl SummaryWindowCompletion { - pub fn validate(&self) -> Result<(), SdsError> { - self.source.validate()?; - self.instance_id.validate()?; - self.coordinates.time_range.validate()?; - if self.input_lineage.is_empty() { - return Err(SdsError( - "summary completion lineage must not be empty".into(), - )); - } - validate_catalog_generation(&self.catalog_generation) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SummaryWatermarkBarrier { - pub catalog_generation: CatalogGeneration, - pub source: SummarySourcePartition, - /// Monotonic sequence within the producer partition. - pub sequence: u64, - /// Every event in this partition at or before this event-time watermark - /// was published before the barrier. - pub watermark_ms: i64, -} - -impl SummaryWatermarkBarrier { - pub fn validate(&self) -> Result<(), SdsError> { - self.source.validate()?; - if self.sequence == 0 { - return Err(SdsError("watermark sequence must be positive".into())); - } - validate_catalog_generation(&self.catalog_generation) - } -} - -fn validate_catalog_generation(generation: &CatalogGeneration) -> Result<(), SdsError> { - if generation.schema_version == 0 || generation.snapshot_sha256.trim().is_empty() { - Err(SdsError("catalog generation identity is invalid".into())) - } else { - Ok(()) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum InstanceCompleteness { - Complete, - Partial, - Unknown, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SummaryInstanceStatus { - Building, - Ready, - Retiring, - Failed, - MissingPayload, -} - -/// Logical producer and physical state location. Placement changes do not -/// change descriptor or materialization identity. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SummaryPlacement { - pub producer_id: String, - pub storage_node_id: String, -} - -/// Opaque storage-engine locator. `key` identifies payload stored elsewhere; -/// encoded summary state must never be placed in this metadata contract. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SummaryStateReference { - pub store: String, - pub key: String, - pub state_schema_version: u32, - pub generation: u64, - pub sequence: u64, - pub checksum: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct EphemeralLease { - pub lease_id: String, - pub owner_id: String, - pub issued_at_ms: i64, - pub expires_at_ms: i64, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum InstanceLifecycle { - Persistent, - Ephemeral { lease: EphemeralLease }, -} - -/// Observed metadata for one concrete SDS instance. Payload remains in the -/// storage engine and is reached only through `state_reference`. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SummaryInstance { - pub instance_id: SummaryInstanceId, - pub stored_output_id: StoredOutputId, - pub summary_definition_id: SummaryDefinitionId, - pub summary_descriptor_id: SummaryDescriptorId, - pub data_descriptor_id: DataDescriptorId, - pub time_range: HalfOpenTimeRange, - pub group_values: BTreeMap, - pub catalog_generation: CatalogGeneration, - pub placement: SummaryPlacement, - pub state_reference: SummaryStateReference, - pub status: SummaryInstanceStatus, - pub completeness: InstanceCompleteness, - pub lifecycle: InstanceLifecycle, - pub observed_at_ms: i64, -} - -impl SummaryInstance { - pub fn validate(&self) -> Result<(), SdsError> { - self.summary_definition_id.validate()?; - if self.time_range.start_ms >= self.time_range.end_ms { - return Err(SdsError( - "summary instance time range must be non-empty".into(), - )); - } - if self.catalog_generation.schema_version == 0 - || self.catalog_generation.snapshot_sha256.is_empty() - { - return Err(SdsError( - "summary instance has invalid catalog generation".into(), - )); - } - if self.placement.producer_id.is_empty() || self.placement.storage_node_id.is_empty() { - return Err(SdsError( - "summary instance placement must be resolved".into(), - )); - } - if self.state_reference.store.is_empty() - || self.state_reference.key.is_empty() - || self.state_reference.state_schema_version == 0 - || self.state_reference.generation != self.catalog_generation.plan_version - { - return Err(SdsError( - "summary instance has invalid state reference".into(), - )); - } - if let InstanceLifecycle::Ephemeral { lease } = &self.lifecycle { - if lease.lease_id.is_empty() - || lease.owner_id.is_empty() - || lease.issued_at_ms >= lease.expires_at_ms - { - return Err(SdsError( - "summary instance has invalid ephemeral lease".into(), - )); - } - } - Ok(()) - } -} - -/// Data-plane report of what is actually stored. This is observed state and is -/// deliberately separate from the control-plane desired SummaryCatalog. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ObservedSummaryInventory { - pub schema_version: u32, - pub reporter_id: String, - pub inventory_version: u64, - pub observed_at_ms: i64, - pub instances: BTreeMap, -} - -impl ObservedSummaryInventory { - pub fn validate(&self) -> Result<(), SdsError> { - if self.schema_version != 1 { - return Err(SdsError("unsupported observed inventory schema".into())); - } - if self.reporter_id.is_empty() { - return Err(SdsError("inventory reporter ID must not be empty".into())); - } - for (id, instance) in &self.instances { - instance.validate()?; - if id != &instance.instance_id { - return Err(SdsError("inventory key differs from instance ID".into())); - } - if instance.observed_at_ms > self.observed_at_ms { - return Err(SdsError( - "instance observation is newer than inventory".into(), - )); - } - } - Ok(()) - } - - pub fn validate_against_catalog( - &self, - catalog: &crate::summary_catalog::SummaryCatalog, - ) -> Result<(), SdsError> { - self.validate()?; - let generation = catalog - .reference() - .map_err(|error| SdsError(error.to_string()))?; - for instance in self.instances.values() { - if instance.catalog_generation != generation { - return Err(SdsError( - "summary instance belongs to another plan generation".into(), - )); - } - let definition = catalog - .outputs - .get(&instance.stored_output_id) - .ok_or_else(|| SdsError("summary instance has no catalog definition".into()))?; - if instance.summary_definition_id != definition.definition_id - || instance.summary_descriptor_id != definition.summary_descriptor_id - || instance.data_descriptor_id != definition.data_descriptor_id - || instance.state_reference.state_schema_version - != catalog.summary_descriptors[&definition.summary_descriptor_id] - .state_schema_version - { - return Err(SdsError( - "summary instance differs from its catalog definition".into(), - )); - } - } - Ok(()) - } -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum SummaryOperator { - /// Compatibility evidence from a legacy backend record, not a complete - /// configured contract. Its ID can never satisfy a Configured descriptor. - LegacyPartial { operator_canonical: String }, - Sketch { - algorithm: SketchAlgorithm, - parameters: BTreeMap, - }, - ExactAgg { - agg_type: AggregationType, - parameters_canonical: String, - }, - /// Complete planner materialization configuration, including heap/Hydra - /// dimensions and readout/update subtype. Never equal to a legacy projection. - Configured { - /// Planner-selected semantic family; grouping and pane layout live in - /// the data descriptor and summary definition, respectively. - family: planner_types::post_asap::SummaryFamilyType, - aggregation_type: AggregationType, - aggregation_sub_type: String, - parameters: BTreeMap, - }, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum FidelityGuarantee { - /// Total unit-frequency count is exact. Distinct, L2 and entropy require - /// readout-specific evidence; dimensions alone certify no error bound. - UnivMonFrequency { - heap_size: u32, - sketch_rows: u32, - sketch_cols: u32, - layers: u8, - }, - Exact, - /// Exact PromQL counter readout from fixed-size pane summaries. Each pane - /// stores only `(first value/time, last value/time, reset-corrected delta, - /// sample count)`. A query is eligible only when the selected panes fully - /// cover its range; partial boundary panes must fall back to Prometheus. - ExactCounter { - model: String, - full_pane_coverage_required: bool, - }, - KllRankError { - k: u32, - model: String, - }, - DdSketchRelativeError { - alpha: f64, - }, - HllCardinalityError { - precision: u32, - model: String, - }, - CmsFrequencyError { - width: u32, - depth: u32, - model: String, - }, - CountSketchFrequencyError { - width: u32, - depth: u32, - model: String, - }, - Unknown { - reason: String, - }, -} -impl FidelityGuarantee { - /// Model IDs name parameterized error families/scopes, not certified numeric - /// epsilon/confidence values. Heap membership and Hydra cross-cell readouts - /// need separate models; point-frequency/per-cell rank does not attest them. - pub fn from_config(config: &PrecomputeMaterialization) -> Self { - if config.aggregation_type == AggregationType::HLL { - let precision = config - .parameters - .get("precision") - .or_else(|| config.parameters.get("p")) - .and_then(Value::as_u64) - .and_then(|v| u32::try_from(v).ok()) - .unwrap_or(14); - return Self::HllCardinalityError { - precision, - model: "asap.hll.relative-cardinality.v1".into(), - }; - } - match config.accumulator_spec().map(|s| s.family) { - Ok(SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Increase - | planner_types::post_asap::ExactKind::Rate - | planner_types::post_asap::ExactKind::IRate, - _, - )) => Self::ExactCounter { - model: "prometheus.extrapolated-rate.v1".into(), - full_pane_coverage_required: true, - }, - Ok(SummaryFamilyType::ExactAggregate(..)) => Self::Exact, - Ok(SummaryFamilyType::Sketch(kind, _)) => match kind.params() { - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - } => Self::UnivMonFrequency { - heap_size: *heap_size, - sketch_rows: *sketch_rows, - sketch_cols: *sketch_cols, - layers: *layers, - }, - SketchParams::Kll { k } => Self::KllRankError { - k: *k, - model: if config.aggregation_type == AggregationType::HydraKLL { - "asap.hydra-kll.per-cell-rank.v1" - } else { - "asap.kll.normalized-rank.v1" - } - .into(), - }, - SketchParams::DDSketch { alpha } => Self::DdSketchRelativeError { alpha: *alpha }, - SketchParams::Hll { precision } => Self::HllCardinalityError { - precision: (*precision).into(), - model: "asap.hll.relative-cardinality.v1".into(), - }, - SketchParams::Cms { width, depth } - | SketchParams::CmsWithHeap { width, depth, .. } => Self::CmsFrequencyError { - width: *width, - depth: *depth, - model: "asap.cms.point-frequency.v1".into(), - }, - SketchParams::CountSketch { width, depth } - | SketchParams::CountSketchWithHeap { width, depth, .. } => { - Self::CountSketchFrequencyError { - width: *width, - depth: *depth, - model: "asap.count-sketch.point-frequency.v1".into(), - } - } - _ => Self::Unknown { - reason: "No shared parameterized error model for this sketch family".into(), - }, - }, - _ => Self::Unknown { - reason: "Physical accumulator family is unavailable".into(), - }, - } - } - fn validate(&self) -> Result<(), SdsError> { - let valid = match self { - Self::UnivMonFrequency { - heap_size, - sketch_rows, - sketch_cols, - layers, - } => { - *heap_size > 0 - && *sketch_cols > 0 - && (1..=20).contains(sketch_rows) - && (1..=64).contains(layers) - } - Self::Exact => true, - Self::ExactCounter { - model, - full_pane_coverage_required, - } => !model.is_empty() && *full_pane_coverage_required, - Self::KllRankError { k, model } => *k > 0 && !model.is_empty(), - Self::DdSketchRelativeError { alpha } => { - alpha.is_finite() && *alpha > 0.0 && *alpha < 1.0 - } - Self::HllCardinalityError { precision, model } => { - *precision > 0 && *precision < 64 && !model.is_empty() - } - Self::CmsFrequencyError { - width, - depth, - model, - } - | Self::CountSketchFrequencyError { - width, - depth, - model, - } => *width > 0 && *depth > 0 && !model.is_empty(), - Self::Unknown { reason } => !reason.is_empty(), - }; - if valid { - Ok(()) - } else { - Err(SdsError("invalid parameterized fidelity contract".into())) - } - } -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SummaryDescriptor { - pub id: SummaryDescriptorId, - pub operator: SummaryOperator, - pub fidelity: FidelityGuarantee, - pub state_schema_version: u32, -} - -/// Sort every JSON object, including nested configuration values. Arrays retain -/// order; opaque canonical strings are used verbatim, not reparsed as PromQL. -fn canonical(value: &Value) -> String { - match value { - Value::Object(map) => { - let sorted: BTreeMap<_, _> = map.iter().collect(); - format!( - "{{{}}}", - sorted - .into_iter() - .map(|(k, v)| format!("{}:{}", serde_json::to_string(k).unwrap(), canonical(v))) - .collect::>() - .join(",") - ) - } - Value::Array(values) => format!( - "[{}]", - values.iter().map(canonical).collect::>().join(",") - ), - _ => value.to_string(), - } -} -impl SummaryDescriptor { - pub fn new( - operator: SummaryOperator, - fidelity: FidelityGuarantee, - state_schema_version: u32, - ) -> Result { - if state_schema_version == 0 { - return Err(SdsError("state schema version must be positive".into())); - } - fidelity.validate()?; - if let SummaryOperator::Configured { - family, - aggregation_type, - .. - } = &operator - { - if let Some(expected) = aggregation_type.planner_exact_family() { - if family != &expected { - return Err(SdsError( - "configured storage type disagrees with Planner family".into(), - )); - } - } else { - use AggregationType as A; - let expected = match aggregation_type { - A::DatasketchesKLL | A::HydraKLL => Some(SketchAlgorithm::Kll), - A::CountMinSketch => Some(SketchAlgorithm::Cms), - A::CountMinSketchWithHeap => Some(SketchAlgorithm::CmsWithHeap), - A::CountSketch => Some(SketchAlgorithm::CountSketch), - A::CountSketchWithHeap => Some(SketchAlgorithm::CountSketchWithHeap), - A::DDSketch => Some(SketchAlgorithm::DDSketch), - A::HLL => Some(SketchAlgorithm::Hll), - A::UnivMon => Some(SketchAlgorithm::UnivMon), - _ => None, - }; - if let Some(expected) = expected { - if !matches!(family, planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &expected) - { - return Err(SdsError( - "configured sketch storage disagrees with Planner family".into(), - )); - } - } - } - } - if !fidelity.is_compatible_with(&operator) { - return Err(SdsError( - "summary operator and fidelity guarantee are incompatible".into(), - )); - } - let content = json!({"operator":operator,"fidelity":fidelity,"state_schema_version":state_schema_version}); - let id = SummaryDescriptorId(format!("summary:v3:{}", canonical(&content))); - Ok(Self { - id, - operator, - fidelity, - state_schema_version, - }) - } - pub fn id(&self) -> &SummaryDescriptorId { - &self.id - } - pub fn validate(&self) -> Result<(), SdsError> { - let rebuilt = Self::new( - self.operator.clone(), - self.fidelity.clone(), - self.state_schema_version, - )?; - if self.id != rebuilt.id { - return Err(SdsError("summary descriptor ID/content mismatch".into())); - } - Ok(()) - } - - /// Preserve every configured state/update parameter. Omitted defaults remain - /// distinct from explicit defaults (conservative identity, never false sharing). - /// Legacy AggKind projections intentionally have different operator variants: - /// they cannot attest heap, Hydra, or aggregation-subtype semantics they lost. - pub fn from_config(config: &PrecomputeMaterialization) -> Result { - let fidelity = FidelityGuarantee::from_config(config); - let state_schema_version = if matches!(fidelity, FidelityGuarantee::ExactCounter { .. }) { - 2 - } else { - 1 - }; - Self::new( - SummaryOperator::Configured { - family: config - .accumulator_spec() - .map_err(|error| SdsError(error.to_string()))? - .family, - aggregation_type: config.aggregation_type, - aggregation_sub_type: config.aggregation_sub_type.clone(), - parameters: config - .parameters - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(), - }, - fidelity, - state_schema_version, - ) - } -} - -impl FidelityGuarantee { - /// Reject a descriptor that advertises an error model belonging to a - /// different state family. This is part of the shared wire contract, so a - /// consumer must not need to trust the process that produced the catalog. - pub fn is_compatible_with(&self, operator: &SummaryOperator) -> bool { - use AggregationType as A; - use FidelityGuarantee::*; - use SketchAlgorithm as S; - - let configured = |aggregation_type| { - matches!( - (aggregation_type, self), - (A::UnivMon, UnivMonFrequency { .. }) - | (A::Sum | A::Count | A::Min | A::Max, Exact) - | (A::Increase | A::Rate, ExactCounter { .. }) - | (A::DatasketchesKLL | A::HydraKLL, KllRankError { .. }) - | (A::DDSketch, DdSketchRelativeError { .. }) - | (A::HLL, HllCardinalityError { .. }) - | ( - A::CountMinSketch | A::CountMinSketchWithHeap, - CmsFrequencyError { .. } - ) - | ( - A::CountSketch | A::CountSketchWithHeap, - CountSketchFrequencyError { .. } - ) - | ( - A::SingleSubpopulation | A::MultipleSubpopulation, - Unknown { .. } - ) - ) - }; - - match operator { - SummaryOperator::LegacyPartial { .. } => matches!(self, Unknown { .. }), - SummaryOperator::ExactAgg { agg_type, .. } => configured(*agg_type), - SummaryOperator::Configured { - aggregation_type, - parameters, - .. - } => configured(*aggregation_type) && self.matches_parameters(parameters), - SummaryOperator::Sketch { - algorithm, - parameters, - } => { - matches!( - (algorithm, self), - (S::UnivMon, UnivMonFrequency { .. }) - | (S::Kll, KllRankError { .. }) - | (S::DDSketch, DdSketchRelativeError { .. }) - | (S::Hll, HllCardinalityError { .. }) - | (S::Cms | S::CmsWithHeap, CmsFrequencyError { .. }) - | ( - S::CountSketch | S::CountSketchWithHeap, - CountSketchFrequencyError { .. } - ) - | (S::Kmv | S::Theta, Unknown { .. }) - ) && self.matches_parameters(parameters) - } - } - } - - fn matches_parameters(&self, parameters: &BTreeMap) -> bool { - let u32_parameter = |names: &[&str], expected: u32| { - names - .iter() - .filter_map(|name| parameters.get(*name)) - .all(|value| value.as_u64() == Some(u64::from(expected))) - }; - match self { - Self::UnivMonFrequency { - heap_size, - sketch_rows, - sketch_cols, - layers, - } => { - u32_parameter(&["heap_size"], *heap_size) - && u32_parameter(&["sketch_rows"], *sketch_rows) - && u32_parameter(&["sketch_cols"], *sketch_cols) - && u32_parameter(&["layers"], u32::from(*layers)) - } - Self::KllRankError { k, .. } => u32_parameter(&["k", "K"], *k), - Self::HllCardinalityError { precision, .. } => { - u32_parameter(&["precision", "p"], *precision) - } - Self::CmsFrequencyError { width, depth, .. } - | Self::CountSketchFrequencyError { width, depth, .. } => { - u32_parameter(&["w", "width"], *width) && u32_parameter(&["d", "depth"], *depth) - } - Self::DdSketchRelativeError { alpha } => ["alpha", "relative_accuracy"] - .iter() - .filter_map(|name| parameters.get(*name)) - .all(|value| value.as_f64() == Some(*alpha)), - _ => true, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "snake_case")] -pub enum DataSourceIdentity { - TimeSeries { - metric: String, - }, - Table { - table_ref: String, - }, - Derived { - input: crate::derived_input::DerivedInputIdentity, - }, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "snake_case")] -pub enum ValueProjectionIdentity { - SampleValue, - Column { - name: String, - }, - Constant { - value: planner_types::pre_asap::ScalarValue, - }, -} - -impl ValueProjectionIdentity { - pub fn column(&self) -> Option<&str> { - match self { - Self::Column { name } => Some(name), - _ => None, - } - } - - pub fn validate(&self) -> Result<(), String> { - use planner_types::pre_asap::ScalarValue; - match self { - Self::Column { name } => crate::table_population::validate_column_name(name), - Self::Constant { - value: ScalarValue::Int64(_), - } - | Self::SampleValue => Ok(()), - Self::Constant { - value: ScalarValue::Float64(value), - } if value.is_finite() => Ok(()), - Self::Constant { .. } => { - Err("summary value projection requires a finite numeric literal".into()) - } - } - } -} - -/// Whether a materialization preserves source entities or pools a population. -/// Grouped label names remain in `DataDescriptor::group_by_keys`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum PopulationPartitioning { - PerEntity, - Grouped, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct DataDescriptor { - pub id: DataDescriptorId, - pub source: DataSourceIdentity, - pub value_projection: ValueProjectionIdentity, - /// Table column containing Unix milliseconds. Absent for time-series sources. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub timestamp_column: Option, - pub population_filter_canonical: String, - pub group_by_keys: crate::GroupingProjection, - #[serde( - default, - skip_serializing_if = "crate::grouping_projection::PopulationKeyEncoding::is_legacy" - )] - pub population_key_encoding: crate::grouping_projection::PopulationKeyEncoding, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub partitioning: Option, - /// Versioned contract for timestamp interpretation and - /// missing/duplicate/invalid observation handling. - pub observation_semantics: String, -} -impl DataDescriptor { - pub fn time_series_metric(&self) -> Option<&str> { - match &self.source { - DataSourceIdentity::TimeSeries { metric } => Some(metric), - DataSourceIdentity::Table { .. } | DataSourceIdentity::Derived { .. } => None, - } - } - - pub fn new( - metric: impl Into, - filter: impl Into, - group_by: impl IntoIterator, - ) -> Self { - Self::new_with_semantics( - metric, - filter, - group_by, - "asap.timestamped-metric-samples.v1", - ) - } - - pub fn new_with_semantics( - metric: impl Into, - filter: impl Into, - group_by: impl IntoIterator, - observation_semantics: impl Into, - ) -> Self { - let source = DataSourceIdentity::TimeSeries { - metric: metric.into(), - }; - Self::new_typed( - source, - ValueProjectionIdentity::SampleValue, - filter, - group_by, - observation_semantics, - ) - } - - pub fn new_typed( - source: DataSourceIdentity, - value_projection: ValueProjectionIdentity, - filter: impl Into, - group_by: impl IntoIterator, - observation_semantics: impl Into, - ) -> Self { - let population_filter_canonical = filter.into(); - let group_by_keys = group_by.into_iter().collect(); - let observation_semantics = observation_semantics.into(); - let id = data_descriptor_id( - &source, - &value_projection, - &population_filter_canonical, - &group_by_keys, - &observation_semantics, - None, - None, - Default::default(), - ); - Self { - id, - source, - value_projection, - timestamp_column: None, - population_filter_canonical, - group_by_keys, - partitioning: None, - population_key_encoding: Default::default(), - observation_semantics, - } - } - pub fn with_grouping_projection(mut self, grouping: crate::GroupingProjection) -> Self { - self.group_by_keys = grouping; - self.id = data_descriptor_id( - &self.source, - &self.value_projection, - &self.population_filter_canonical, - &self.group_by_keys, - &self.observation_semantics, - self.partitioning, - self.timestamp_column.as_deref(), - self.population_key_encoding, - ); - self - } - pub fn with_partitioning(mut self, partitioning: Option) -> Self { - self.partitioning = partitioning; - self.id = data_descriptor_id( - &self.source, - &self.value_projection, - &self.population_filter_canonical, - &self.group_by_keys, - &self.observation_semantics, - partitioning, - self.timestamp_column.as_deref(), - self.population_key_encoding, - ); - self - } - pub fn with_population_key_encoding( - mut self, - encoding: crate::grouping_projection::PopulationKeyEncoding, - ) -> Self { - self.population_key_encoding = encoding; - let partitioning = self.partitioning; - self.with_partitioning(partitioning) - } - pub fn with_timestamp_column(mut self, column: Option) -> Self { - self.timestamp_column = column; - let partitioning = self.partitioning; - self.with_partitioning(partitioning) - } - pub fn id(&self) -> &DataDescriptorId { - &self.id - } - pub fn validate(&self) -> Result<(), SdsError> { - if let DataSourceIdentity::Derived { input } = &self.source { - input.validate().map_err(SdsError)?; - } - self.value_projection.validate().map_err(SdsError)?; - self.group_by_keys.validate().map_err(SdsError)?; - if matches!(self.source, DataSourceIdentity::TimeSeries { .. }) - && !self.group_by_keys.is_legacy_labels() - { - return Err(SdsError( - "time-series grouping requires non-null string labels".into(), - )); - } - - if matches!(self.source, DataSourceIdentity::Table { .. }) { - for column in self.group_by_keys.columns() { - crate::table_population::validate_column_name(&column.name).map_err(SdsError)?; - } - } - if let Some(column) = &self.timestamp_column { - if !matches!(self.source, DataSourceIdentity::Table { .. }) || column.is_empty() { - return Err(SdsError( - "table timestamp projection requires a table and a column".into(), - )); - } - crate::table_population::validate_column_name(column).map_err(SdsError)?; - } - if self.id - != data_descriptor_id( - &self.source, - &self.value_projection, - &self.population_filter_canonical, - &self.group_by_keys, - &self.observation_semantics, - self.partitioning, - self.timestamp_column.as_deref(), - self.population_key_encoding, - ) - { - return Err(SdsError("data descriptor ID/content mismatch".into())); - } - Ok(()) - } -} -#[allow(clippy::too_many_arguments)] -fn data_descriptor_id( - source: &DataSourceIdentity, - value_projection: &ValueProjectionIdentity, - filter: &str, - group_by: &crate::GroupingProjection, - observation_semantics: &str, - partitioning: Option, - timestamp_column: Option<&str>, - population_key_encoding: crate::grouping_projection::PopulationKeyEncoding, -) -> DataDescriptorId { - // Length framing keeps distinct typed sources, projections, predicates, - // and grouping keys collision-free in the content identity. - let source = canonical(&serde_json::to_value(source).expect("data source serializes")); - let projection = - canonical(&serde_json::to_value(value_projection).expect("value projection serializes")); - let mut key = format!( - "data:v2|{}:{source}|{}:{projection}|{}:{filter}", - source.len(), - projection.len(), - filter.len() - ); - if let Some(partitioning) = partitioning { - key.push_str(&format!("|partition:{partitioning:?}")); - } - if let Some(column) = timestamp_column { - key.push_str(&format!("|timestamp-ms:{}:{column}", column.len())); - } - for name in group_by.names() { - key.push_str(&format!("|{}:{name}", name.len())); - } - if !group_by.is_legacy_labels() { - let typed = - canonical(&serde_json::to_value(group_by).expect("group projection serializes")); - key.push_str(&format!("|group-types:{}:{typed}", typed.len())); - } - key.push_str(&format!( - "|{}:{observation_semantics}", - observation_semantics.len() - )); - if !population_key_encoding.is_legacy() { - key = format!("data:population-key:canonical-labels-v1|{key}"); - } - DataDescriptorId(key) -} - -#[cfg(test)] -mod partition_identity_tests { - use super::*; - #[test] - fn entity_and_global_population_have_distinct_identity() { - let legacy = DataDescriptor::new_typed( - DataSourceIdentity::TimeSeries { metric: "m".into() }, - ValueProjectionIdentity::SampleValue, - "", - Vec::::new(), - "v1", - ); - let entity = legacy - .clone() - .with_partitioning(Some(PopulationPartitioning::PerEntity)); - let grouped = legacy - .clone() - .with_partitioning(Some(PopulationPartitioning::Grouped)); - assert_ne!(entity.id, grouped.id); - assert_ne!(entity.id, legacy.id); - assert_eq!(legacy.clone().with_partitioning(None).id, legacy.id); - entity.validate().unwrap(); - grouped.validate().unwrap(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::BTreeSet; - - #[test] - fn population_encoding_is_part_of_data_identity_with_legacy_default() { - use crate::grouping_projection::PopulationKeyEncoding; - let legacy = DataDescriptor::new("m", "", ["host".to_string()]); - let wire = serde_json::to_value(&legacy).unwrap(); - assert!(wire.get("population_key_encoding").is_none()); - let decoded: DataDescriptor = serde_json::from_value(wire).unwrap(); - assert_eq!(legacy.id(), decoded.id()); - let canonical = legacy - .clone() - .with_population_key_encoding(PopulationKeyEncoding::CanonicalLabelsV1); - assert_ne!(legacy.id(), canonical.id()); - canonical.validate().unwrap(); - let reverted = - canonical.with_population_key_encoding(PopulationKeyEncoding::LegacyDelimited); - assert_eq!(legacy.id(), reverted.id()); - } - - fn completion(epoch: u64) -> SummaryWindowCompletion { - SummaryWindowCompletion { - catalog_generation: CatalogGeneration { - schema_version: 1, - plan_id: 4, - plan_version: 2, - snapshot_sha256: "snapshot".into(), - }, - source: SummarySourcePartition { - producer_id: "producer-a".into(), - partition_id: "partition-0".into(), - producer_epoch: epoch, - }, - instance_id: SummaryInstanceId::new("instance-1").unwrap(), - coordinates: SummaryInstanceCoordinates { - stored_output_id: StoredOutputId::from(crate::PolicyFingerprint(7)), - time_range: HalfOpenTimeRange { - start_ms: 1_000, - end_ms: 2_000, - }, - group_values: BTreeMap::from([("job".into(), "api".into())]), - }, - input_lineage: vec![1, 2, 3], - } - } - - #[test] - fn completion_contract_validates_identity_epoch_range_and_lineage() { - completion(1).validate().unwrap(); - - let mut invalid = completion(1); - invalid.coordinates.time_range.end_ms = invalid.coordinates.time_range.start_ms; - assert!(invalid.validate().is_err()); - invalid = completion(1); - invalid.source.producer_id.clear(); - assert!(invalid.validate().is_err()); - invalid = completion(0); - assert!(invalid.validate().is_err()); - invalid = completion(1); - invalid.input_lineage.clear(); - assert!(invalid.validate().is_err()); - - let mut wire = serde_json::to_value(completion(1)).unwrap(); - wire.as_object_mut() - .unwrap() - .insert("future_field".into(), json!(true)); - assert!(serde_json::from_value::(wire).is_err()); - } - - #[test] - fn producer_epoch_separates_restart_incarnations() { - let first = completion(1).source; - let second = completion(2).source; - assert_ne!(first, second); - assert_eq!(BTreeSet::from([first, second]).len(), 2); - } - - #[test] - fn watermark_contract_rejects_zero_sequence_and_unknown_fields() { - let completion = completion(3); - let mut barrier = SummaryWatermarkBarrier { - catalog_generation: completion.catalog_generation, - source: completion.source, - sequence: 1, - watermark_ms: 2_000, - }; - barrier.validate().unwrap(); - barrier.sequence = 0; - assert!(barrier.validate().is_err()); - - let mut wire = serde_json::to_value(&barrier).unwrap(); - wire.as_object_mut() - .unwrap() - .insert("ignored".into(), json!(1)); - assert!(serde_json::from_value::(wire).is_err()); - } - - fn observed_instance(lifecycle: InstanceLifecycle) -> SummaryInstance { - SummaryInstance { - instance_id: SummaryInstanceId::new("instance-1").unwrap(), - stored_output_id: StoredOutputId(7), - summary_definition_id: SummaryDefinitionId::from_semantics(b"fixture"), - summary_descriptor_id: descriptor( - 200, - FidelityGuarantee::KllRankError { - k: 200, - model: "rank.v1".into(), - }, - 1, - ) - .id, - data_descriptor_id: DataDescriptor::new("cpu", "", []).id, - time_range: HalfOpenTimeRange { - start_ms: 0, - end_ms: 10, - }, - group_values: BTreeMap::new(), - catalog_generation: CatalogGeneration { - schema_version: 1, - plan_id: 1, - plan_version: 2, - snapshot_sha256: "abc".into(), - }, - placement: SummaryPlacement { - producer_id: "producer".into(), - storage_node_id: "store".into(), - }, - state_reference: SummaryStateReference { - store: "summary-store".into(), - key: "state/1".into(), - state_schema_version: 1, - generation: 2, - sequence: 3, - checksum: None, - }, - status: SummaryInstanceStatus::Ready, - completeness: InstanceCompleteness::Complete, - lifecycle, - observed_at_ms: 10, - } - } - - #[test] - fn instance_inventory_has_metadata_reference_without_payload() { - let instance = observed_instance(InstanceLifecycle::Persistent); - instance.validate().unwrap(); - let encoded = serde_json::to_value(&instance).unwrap(); - assert!(encoded.get("state_reference").is_some()); - assert!(encoded.get("state").is_none()); - assert!(encoded.get("payload").is_none()); - let inventory = ObservedSummaryInventory { - schema_version: 1, - reporter_id: "store".into(), - inventory_version: 4, - observed_at_ms: 10, - instances: BTreeMap::from([(instance.instance_id.clone(), instance)]), - }; - inventory.validate().unwrap(); - } - - // Every component of the persisted output address participates in identity. - #[test] - fn stored_summary_key_binds_plan_output_population_and_window() { - let key = StoredSummaryKey { - plan_id: 1, - plan_version: 2, - stored_output_id: StoredOutputId(101), - population: BTreeMap::from([("service".into(), "api".into())]), - window: HalfOpenTimeRange { - start_ms: 0, - end_ms: 1000, - }, - }; - let mut variants = vec![key.clone(); 5]; - variants[0].plan_id += 1; - variants[1].plan_version += 1; - variants[2].stored_output_id.0 += 1; - variants[3].population.insert("service".into(), "db".into()); - variants[4].window.end_ms += 1; - let canonical = key.storage_key().unwrap(); - for other in variants { - assert_ne!(canonical, other.storage_key().unwrap()); - } - assert_eq!( - serde_json::from_str::(&canonical).unwrap(), - key - ); - } - - // A DAG output has its own identity even when its definition is shared. - #[test] - fn stored_output_identity_is_independent_of_definition() { - let definition_id = SummaryDefinitionId::from_semantics(b"fixture"); - for stored_output_id in [StoredOutputId(101), StoredOutputId(102)] { - StoredOutputReference { - stored_output_id, - definition_id: definition_id.clone(), - } - .validate() - .unwrap(); - } - } - - #[test] - fn stored_output_and_payload_version_must_match_instance_definition() { - let mut instance = observed_instance(InstanceLifecycle::Persistent); - instance.stored_output_id = StoredOutputId(8); - assert!(instance.validate().is_ok()); - instance.stored_output_id = StoredOutputId(7); - instance.state_reference.generation = 3; - assert!(instance.validate().is_err()); - let mut reference = StoredOutputReference::for_output(instance.stored_output_id); - reference.stored_output_id = StoredOutputId(8); - assert!(reference.validate().is_err()); - } - - // Old aliases and numeric semantic IDs must not authorize the new format. - #[test] - fn stored_output_reference_rejects_legacy_identity() { - assert!(serde_json::from_value::(json!({ - "state_slot_id": 7, "definition_id": 7 - })) - .is_err()); - let reference = StoredOutputReference { - stored_output_id: StoredOutputId(8), - definition_id: SummaryDefinitionId::from_semantics(b"fixture"), - }; - reference.validate().unwrap(); - assert_eq!( - serde_json::from_value::( - serde_json::to_value(&reference).unwrap() - ) - .unwrap(), - reference - ); - } - - #[test] - fn new_generation_rejects_cross_version_payload_adoption() { - let mut instance = observed_instance(InstanceLifecycle::Persistent); - let mut wire = serde_json::to_value(&instance).unwrap(); - wire["reused_from_generation"] = - serde_json::to_value(&instance.catalog_generation).unwrap(); - assert!(serde_json::from_value::(wire).is_err()); - instance.state_reference.generation -= 1; - assert!(instance.validate().is_err()); - } - - #[test] - fn invalid_range_and_lease_are_rejected() { - let mut instance = observed_instance(InstanceLifecycle::Ephemeral { - lease: EphemeralLease { - lease_id: "lease".into(), - owner_id: "fast-path".into(), - issued_at_ms: 10, - expires_at_ms: 20, - }, - }); - instance.validate().unwrap(); - instance.time_range.end_ms = instance.time_range.start_ms; - assert!(instance.validate().is_err()); - instance.time_range.end_ms = 10; - if let InstanceLifecycle::Ephemeral { lease } = &mut instance.lifecycle { - lease.expires_at_ms = lease.issued_at_ms; - } - assert!(instance.validate().is_err()); - } - fn descriptor(k: u32, fidelity: FidelityGuarantee, version: u32) -> SummaryDescriptor { - SummaryDescriptor::new( - SummaryOperator::Sketch { - algorithm: SketchAlgorithm::Kll, - parameters: BTreeMap::from([("k".into(), json!(k))]), - }, - fidelity, - version, - ) - .unwrap() - } - #[test] - fn identity_includes_configuration_fidelity_and_state_schema() { - let base = descriptor( - 200, - FidelityGuarantee::KllRankError { - k: 200, - model: "rank.v1".into(), - }, - 1, - ); - assert_ne!( - base.id, - descriptor( - 201, - FidelityGuarantee::KllRankError { - k: 201, - model: "rank.v1".into() - }, - 1 - ) - .id - ); - assert!(SummaryDescriptor::new( - SummaryOperator::Sketch { - algorithm: SketchAlgorithm::Kll, - parameters: BTreeMap::from([("k".into(), json!(200))]), - }, - FidelityGuarantee::Exact, - 1, - ) - .is_err()); - assert_ne!( - base.id, - descriptor( - 200, - FidelityGuarantee::KllRankError { - k: 200, - model: "rank.v1".into() - }, - 2 - ) - .id - ); - } - #[test] - fn group_order_and_duplicates_do_not_change_identity() { - let a = DataDescriptor::new("cpu", "{job=\"a\"}", ["z".into(), "a".into(), "a".into()]); - let b = DataDescriptor::new("cpu", "{job=\"a\"}", ["a".into(), "z".into()]); - assert_eq!(a, b); - assert_ne!( - a.id, - DataDescriptor::new("cpu", "{job=\"b\"}", ["a".into(), "z".into()]).id - ); - } - #[test] - fn length_framing_distinguishes_delimiters_and_unicode() { - assert_ne!( - DataDescriptor::new("a|b", "c", []).id, - DataDescriptor::new("a", "b|c", []).id - ); - assert_ne!( - DataDescriptor::new("π", "", ["x|y".into()]).id, - DataDescriptor::new("π", "", ["x".into(), "y".into()]).id - ); - } - #[test] - fn observation_semantics_is_part_of_data_identity() { - assert_ne!( - DataDescriptor::new_with_semantics("cpu", "", [], "samples.v1").id, - DataDescriptor::new_with_semantics("cpu", "", [], "samples.v2").id - ); - } - - #[test] - fn source_and_value_projection_are_part_of_data_identity() { - let metric = DataDescriptor::new("events", "", []); - let table_value = DataDescriptor::new_typed( - DataSourceIdentity::Table { - table_ref: "events".into(), - }, - ValueProjectionIdentity::Column { - name: "value".into(), - }, - "", - [], - "asap.timestamped-observations.v2", - ); - let table_cost = DataDescriptor::new_typed( - DataSourceIdentity::Table { - table_ref: "events".into(), - }, - ValueProjectionIdentity::Column { - name: "cost".into(), - }, - "", - [], - "asap.timestamped-observations.v2", - ); - assert_ne!(metric.id, table_value.id); - assert_ne!(table_value.id, table_cost.id); - assert_eq!(metric.time_series_metric(), Some("events")); - assert_eq!(table_value.time_series_metric(), None); - } - - #[test] - fn catalog_generation_rejects_legacy_digest_name() { - assert!(serde_json::from_value::(json!({ - "schema_version": 1, - "plan_id": 2, - "plan_version": 3, - "snapshot_digest": "abc" - })) - .is_err()); - } - #[test] - fn wire_roundtrip_and_tampered_id_validation() { - let original = descriptor( - 200, - FidelityGuarantee::KllRankError { - k: 200, - model: "rank.v1".into(), - }, - 1, - ); - let mut decoded: SummaryDescriptor = - serde_json::from_str(&serde_json::to_string(&original).unwrap()).unwrap(); - decoded.validate().unwrap(); - assert_eq!(decoded, original); - decoded.state_schema_version = 2; - assert!(decoded.validate().is_err()); - let mut data = DataDescriptor::new("cpu", "", []); - data.source = DataSourceIdentity::TimeSeries { - metric: "other".into(), - }; - assert!(data.validate().is_err()); - } - #[test] - fn invalid_fidelity_is_rejected() { - assert!(SummaryDescriptor::new( - SummaryOperator::ExactAgg { - agg_type: AggregationType::Sum, - parameters_canonical: String::new() - }, - FidelityGuarantee::DdSketchRelativeError { alpha: f64::NAN }, - 1 - ) - .is_err()); - assert!(SummaryDescriptor::new( - SummaryOperator::Configured { - family: AggregationType::Sum.planner_exact_family().unwrap(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: BTreeMap::new(), - }, - FidelityGuarantee::KllRankError { - k: 200, - model: "rank.v1".into(), - }, - 1, - ) - .is_err()); - assert!(SummaryDescriptor::new( - SummaryOperator::Sketch { - algorithm: SketchAlgorithm::Kll, - parameters: BTreeMap::from([("k".into(), json!(100))]), - }, - FidelityGuarantee::KllRankError { - k: 200, - model: "rank.v1".into(), - }, - 1, - ) - .is_err()); - } - - #[test] - fn configured_descriptor_rejects_family_storage_disagreement() { - assert!(SummaryDescriptor::new( - SummaryOperator::Configured { - family: AggregationType::Rate.planner_exact_family().unwrap(), - aggregation_type: AggregationType::Increase, - aggregation_sub_type: String::new(), - parameters: BTreeMap::new(), - }, - FidelityGuarantee::ExactCounter { - model: "prometheus.extrapolated-rate.v1".into(), - full_pane_coverage_required: true, - }, - 2, - ) - .is_err()); - } - #[test] - fn configured_identity_preserves_heap_hydra_and_subtype_and_excludes_population() { - let yaml:serde_yaml::Value=serde_yaml::from_str("aggregationType: DDSketch\naggregationSubType: ''\nmetric: m\nlabels:\n grouping: []\n rollup: []\n aggregated: []\nparameters:\n relative_accuracy: 0.01\nwindowSize: 30\nwindowType: tumbling\nspatialFilter: ''\n").unwrap(); - let mut config = - PrecomputeMaterialization::from_yaml_data(&yaml, None, crate::QueryLanguage::PromQl) - .unwrap(); - assert!(matches!( - SummaryDescriptor::from_config(&config).unwrap().fidelity, - FidelityGuarantee::DdSketchRelativeError { .. } - )); - for (kind, key) in [ - (AggregationType::CountMinSketchWithHeap, "heap_size"), - (AggregationType::HydraKLL, "row"), - (AggregationType::HydraKLL, "col"), - (AggregationType::HydraKLL, "k"), - ] { - config.aggregation_type = kind; - config.parameters.insert(key.into(), json!(10)); - let before = SummaryDescriptor::from_config(&config).unwrap(); - config.parameters.insert(key.into(), json!(11)); - let after = SummaryDescriptor::from_config(&config).unwrap(); - assert_ne!(before.id, after.id, "{key}"); - config.metric = "other".into(); - config.spatial_filter_normalized = "job=a".into(); - assert_eq!( - after.id, - SummaryDescriptor::from_config(&config).unwrap().id - ); - } - let before = SummaryDescriptor::from_config(&config).unwrap(); - config.aggregation_sub_type = "max".into(); - assert_ne!( - before.id, - SummaryDescriptor::from_config(&config).unwrap().id - ); - } - #[test] - fn increase_config_declares_prometheus_counter_fidelity() { - let yaml: serde_yaml::Value = serde_yaml::from_str( - "aggregationType: Increase\naggregationSubType: ''\nmetric: requests_total\nlabels:\n grouping: []\n rollup: []\n aggregated: []\nparameters: {}\nwindowSize: 60\nwindowType: tumbling\nspatialFilter: ''\n", - ) - .unwrap(); - let config = - PrecomputeMaterialization::from_yaml_data(&yaml, None, crate::QueryLanguage::PromQl) - .unwrap(); - assert!(matches!( - SummaryDescriptor::from_config(&config).unwrap().fidelity, - FidelityGuarantee::ExactCounter { - ref model, - full_pane_coverage_required: true - } if model == "prometheus.extrapolated-rate.v1" - )); - } - #[test] - fn canonical_nested_parameters_and_model_versions_are_identity() { - let a = SummaryOperator::Configured { - family: AggregationType::Sum.planner_exact_family().unwrap(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: BTreeMap::from([("nested".into(), json!({"z":1,"a":2}))]), - }; - let b = SummaryOperator::Configured { - family: AggregationType::Sum.planner_exact_family().unwrap(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: BTreeMap::from([("nested".into(), json!({"a":2,"z":1}))]), - }; - assert_eq!( - SummaryDescriptor::new(a.clone(), FidelityGuarantee::Exact, 1) - .unwrap() - .id, - SummaryDescriptor::new(b, FidelityGuarantee::Exact, 1) - .unwrap() - .id - ); - let kll = SummaryOperator::Sketch { - algorithm: SketchAlgorithm::Kll, - parameters: BTreeMap::from([("k".into(), json!(200))]), - }; - let first = SummaryDescriptor::new( - kll.clone(), - FidelityGuarantee::KllRankError { - k: 200, - model: "rank.v1".into(), - }, - 1, - ) - .unwrap(); - let second = SummaryDescriptor::new( - kll, - FidelityGuarantee::KllRankError { - k: 200, - model: "rank.v2".into(), - }, - 1, - ) - .unwrap(); - assert_ne!(first.id, second.id); - } - #[test] - fn stored_output_id_preserves_legacy_wire_identity() { - let fingerprint = crate::PolicyFingerprint(42); - let id = StoredOutputId::from(fingerprint); - assert_eq!(id.fingerprint(), fingerprint); - assert_eq!(id.as_u64(), 42); - assert_eq!(crate::PolicyFingerprint::from(id), fingerprint); - assert_eq!( - serde_json::to_value(id).unwrap(), - serde_json::to_value(fingerprint).unwrap() - ); - assert_eq!(serde_json::from_str::("42").unwrap(), id); - } - /// Every supplied alias must agree with the declared fidelity, including runtime w/d keys. - #[test] - fn fidelity_rejects_conflicting_parameter_aliases() { - use serde_json::json; - let cases = [ - ( - FidelityGuarantee::CmsFrequencyError { - width: 128, - depth: 5, - model: "cms".into(), - }, - json!({"w":128,"d":5,"width":128,"depth":5}), - "w", - json!(256), - ), - ( - FidelityGuarantee::CountSketchFrequencyError { - width: 128, - depth: 5, - model: "cs".into(), - }, - json!({"w":128,"d":5,"width":128,"depth":5}), - "d", - json!(4), - ), - ( - FidelityGuarantee::KllRankError { - k: 200, - model: "kll".into(), - }, - json!({"k":200,"K":200}), - "K", - json!(100), - ), - ( - FidelityGuarantee::HllCardinalityError { - precision: 12, - model: "hll".into(), - }, - json!({"precision":12,"p":12}), - "p", - json!(10), - ), - ( - FidelityGuarantee::DdSketchRelativeError { alpha: 0.01 }, - json!({"alpha":0.01,"relative_accuracy":0.01}), - "relative_accuracy", - json!(0.1), - ), - ]; - for (fidelity, values, alias, conflict) in cases { - let mut parameters: BTreeMap = serde_json::from_value(values).unwrap(); - assert!(fidelity.matches_parameters(¶meters)); - parameters.insert(alias.into(), conflict); - assert!(!fidelity.matches_parameters(¶meters)); - } - let cms = FidelityGuarantee::CmsFrequencyError { - width: 128, - depth: 5, - model: "cms".into(), - }; - assert!(!cms.matches_parameters(&BTreeMap::from([ - ("w".into(), json!(256)), - ("d".into(), json!(5)) - ]))); - } -} diff --git a/crates/asap_types/src/semantic_fragment.rs b/crates/asap_types/src/semantic_fragment.rs deleted file mode 100644 index 390354bd9..000000000 --- a/crates/asap_types/src/semantic_fragment.rs +++ /dev/null @@ -1,2 +0,0 @@ -//! Planner owns the versioned semantic description and its normalization. -pub use planner_types::post_asap::SummarySemanticFragment as SemanticFragment; diff --git a/crates/asap_types/src/storage_backend.rs b/crates/asap_types/src/storage_backend.rs deleted file mode 100644 index 68b98488b..000000000 --- a/crates/asap_types/src/storage_backend.rs +++ /dev/null @@ -1,33 +0,0 @@ -use serde::{Deserialize, Serialize}; - -pub const ENGINE_ID_ASAP_QUERY: &str = "asap_query"; -pub const CANONICAL_QUERY_ENGINE_IDS: &[&str] = &[ENGINE_ID_ASAP_QUERY]; - -/// Backend-owned physical storage target shared by both runtime planes. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum StorageBackend { - #[default] - SketchStore, - DoubleWrite, - PrometheusRemote, -} - -impl StorageBackend { - pub const fn data_source_id(self) -> &'static str { - match self { - Self::SketchStore => ENGINE_ID_ASAP_QUERY, - Self::DoubleWrite => "double_write", - Self::PrometheusRemote => "prometheus_remote", - } - } -} - -pub fn parse_storage_backend_engine_id(value: &str) -> Option { - match value { - ENGINE_ID_ASAP_QUERY => Some(StorageBackend::SketchStore), - "double_write" => Some(StorageBackend::DoubleWrite), - "prometheus_remote" => Some(StorageBackend::PrometheusRemote), - _ => None, - } -} diff --git a/crates/asap_types/src/summary_catalog.rs b/crates/asap_types/src/summary_catalog.rs deleted file mode 100644 index ec924e704..000000000 --- a/crates/asap_types/src/summary_catalog.rs +++ /dev/null @@ -1,638 +0,0 @@ -//! Authoritative descriptor snapshot for a compiled physical plan. -//! -//! The catalog owns semantic definitions; executable plans own writer and -//! reader bindings, while runtime inventory owns placement and pane state. - -use std::collections::BTreeMap; - -use crate::sds::{ - CatalogGeneration, DataDescriptor, DataDescriptorId, DataSourceIdentity, StoredOutputId, - SummaryDescriptor, SummaryDescriptorId, -}; -use crate::PolicyFingerprint; -use serde::{Deserialize, Serialize}; - -pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 6; - -/// Canonical definition binds operator and population descriptors. Writer -/// layout and concrete state belong to installed plans and runtime instances. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct StoredOutputDefinition { - pub definition_id: crate::sds::SummaryDefinitionId, - pub summary_descriptor_id: SummaryDescriptorId, - pub data_descriptor_id: DataDescriptorId, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct SummaryCatalog { - pub schema_version: u32, - pub plan_id: u64, - pub plan_version: u64, - pub summary_descriptors: BTreeMap, - pub data_descriptors: BTreeMap, - pub outputs: BTreeMap, - pub definitions: - BTreeMap, -} - -#[derive(Debug, thiserror::Error)] -pub enum SummaryCatalogError { - #[error("unsupported summary catalog schema version {0}")] - SchemaVersion(u32), - #[error("invalid summary catalog descriptor: {0}")] - Descriptor(String), - #[error("definition {0} has conflicting descriptor bindings")] - ConflictingDefinition(u64), - #[error("definition {0} references a missing descriptor")] - MissingDescriptor(u64), - #[error("catalog reference does not identify the supplied snapshot")] - ReferenceMismatch, -} - -impl CatalogGeneration { - #[cfg(test)] - /// Validate an untrusted wire reference against the installed immutable - /// snapshot and its enclosing plan generation. - pub fn validate_snapshot( - &self, - catalog: &SummaryCatalog, - plan_id: u64, - plan_version: u64, - ) -> Result<(), SummaryCatalogError> { - let expected = catalog.reference()?; - if self != &expected || (plan_id, plan_version) != (catalog.plan_id, catalog.plan_version) { - return Err(SummaryCatalogError::ReferenceMismatch); - } - Ok(()) - } -} - -impl SummaryCatalog { - pub fn output_reference( - &self, - id: StoredOutputId, - ) -> Result { - let output = self - .outputs - .get(&id) - .ok_or(SummaryCatalogError::MissingDescriptor(id.as_u64()))?; - Ok(crate::sds::StoredOutputReference { - stored_output_id: id, - definition_id: output.definition_id.clone(), - }) - } - - pub fn reference(&self) -> Result { - use sha2::{Digest, Sha256}; - self.validate()?; - // BTreeMap tables and typed descriptor fields serialize deterministically. - let bytes = serde_json::to_vec(self) - .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - Ok(CatalogGeneration { - schema_version: self.schema_version, - plan_id: self.plan_id, - plan_version: self.plan_version, - snapshot_sha256: format!("{:x}", Sha256::digest(bytes)), - }) - } - - pub fn from_materializations( - plan_id: u64, - plan_version: u64, - materializations: &[crate::PrecomputeMaterialization], - ) -> Result { - let entries = materializations - .iter() - .map(|config| { - let summary = SummaryDescriptor::from_config(config) - .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - let source = config.source_identity(); - let value_projection = config.effective_value_projection().clone(); - let data = DataDescriptor::new_typed( - source, - value_projection, - config - .population_filter_canonical() - .map_err(SummaryCatalogError::Descriptor)?, - config.grouping_labels.names(), - if config.table_name.is_some() && !config.grouping_labels.is_empty() { - crate::grouping_projection::TABLE_GROUP_OBSERVATION_SEMANTICS - } else { - crate::sds::TIMESTAMPED_OBSERVATION_SEMANTICS - }, - ) - .with_grouping_projection(config.grouping_labels.clone()) - .with_partitioning(config.partitioning) - .with_population_key_encoding(config.population_key_encoding) - .with_timestamp_column(config.table_timestamp_column.clone()); - Ok((config.policy_fingerprint(), summary, data)) - }) - .collect::, SummaryCatalogError>>()?; - let mut catalog = Self::build(plan_id, plan_version, entries)?; - let mut semantic_bindings = BTreeMap::new(); - for config in materializations { - let output = catalog - .outputs - .get_mut(&StoredOutputId::from(config.policy_fingerprint())) - .unwrap(); - let definition = crate::summary_semantics::SummaryDefinition::from_descriptors( - &catalog.summary_descriptors[&output.summary_descriptor_id], - &catalog.data_descriptors[&output.data_descriptor_id], - )? - .with_config(config); - let id = definition.id()?; - if semantic_bindings - .insert(config.policy_fingerprint(), id.clone()) - .is_some_and(|old| old != id) - { - return Err(SummaryCatalogError::ConflictingDefinition( - config.policy_fingerprint().0, - )); - } - output.definition_id = id.clone(); - catalog.definitions.insert(id, definition); - } - let used: std::collections::BTreeSet<_> = catalog - .outputs - .values() - .map(|o| o.definition_id.clone()) - .collect(); - catalog.definitions.retain(|id, _| used.contains(id)); - catalog.validate()?; - Ok(catalog) - } - - pub fn build( - plan_id: u64, - plan_version: u64, - entries: impl IntoIterator, - ) -> Result { - let mut catalog = Self { - schema_version: SUMMARY_CATALOG_SCHEMA_VERSION, - plan_id, - plan_version, - summary_descriptors: BTreeMap::new(), - data_descriptors: BTreeMap::new(), - outputs: BTreeMap::new(), - definitions: BTreeMap::new(), - }; - for (fingerprint, summary, data) in entries { - let definition = StoredOutputId::from(fingerprint); - summary - .validate() - .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - data.validate() - .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - let semantics = - crate::summary_semantics::SummaryDefinition::from_descriptors(&summary, &data)?; - let semantic_id = semantics.id()?; - catalog.definitions.insert(semantic_id.clone(), semantics); - let binding = StoredOutputDefinition { - definition_id: semantic_id, - summary_descriptor_id: summary.id().clone(), - data_descriptor_id: data.id().clone(), - }; - if catalog - .outputs - .get(&definition) - .is_some_and(|old| old != &binding) - { - return Err(SummaryCatalogError::ConflictingDefinition( - definition.as_u64(), - )); - } - catalog - .summary_descriptors - .insert(binding.summary_descriptor_id.clone(), summary); - catalog - .data_descriptors - .insert(binding.data_descriptor_id.clone(), data); - catalog.outputs.insert(definition, binding); - } - catalog.validate()?; - Ok(catalog) - } - - pub fn validate(&self) -> Result<(), SummaryCatalogError> { - if self.schema_version != SUMMARY_CATALOG_SCHEMA_VERSION { - return Err(SummaryCatalogError::SchemaVersion(self.schema_version)); - } - for (id, definition) in &self.definitions { - if &definition.id()? != id { - return Err(SummaryCatalogError::Descriptor( - "semantic definition content hash mismatch".into(), - )); - } - } - for (key, descriptor) in &self.summary_descriptors { - descriptor - .validate() - .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - if key != descriptor.id() { - return Err(SummaryCatalogError::Descriptor( - "summary table key differs from content identity".into(), - )); - } - } - for (key, descriptor) in &self.data_descriptors { - descriptor - .validate() - .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - if key != descriptor.id() { - return Err(SummaryCatalogError::Descriptor( - "data table key differs from content identity".into(), - )); - } - } - for (id, binding) in &self.outputs { - if !self.definitions.contains_key(&binding.definition_id) { - return Err(SummaryCatalogError::Descriptor( - "output references absent semantic definition".into(), - )); - } - if !self - .summary_descriptors - .contains_key(&binding.summary_descriptor_id) - || !self - .data_descriptors - .contains_key(&binding.data_descriptor_id) - { - return Err(SummaryCatalogError::MissingDescriptor(id.as_u64())); - } - } - for output in self.outputs.values() { - let expected = crate::summary_semantics::SummaryDefinition::from_descriptors( - &self.summary_descriptors[&output.summary_descriptor_id], - &self.data_descriptors[&output.data_descriptor_id], - )?; - if let ( - crate::summary_semantics::SummarySemantics::Configured { - computation: actual, - .. - }, - crate::summary_semantics::SummarySemantics::Configured { - computation: expected, - .. - }, - ) = ( - &self.definitions[&output.definition_id].semantics, - &expected.semantics, - ) { - if actual != expected { - return Err(SummaryCatalogError::Descriptor( - "output descriptors disagree with semantic definition".into(), - )); - } - } - } - if !self - .data_descriptors - .values() - .any(|data| matches!(data.source, DataSourceIdentity::Derived { .. })) - { - return Ok(()); - } - // Dependencies must refer to this snapshot and form an acyclic graph. - let mut pending = std::collections::BTreeMap::new(); - let mut consumers: std::collections::BTreeMap<_, Vec<_>> = - std::collections::BTreeMap::new(); - for (id, binding) in &self.outputs { - let dependencies = match &self.data_descriptors[&binding.data_descriptor_id].source { - DataSourceIdentity::Derived { input } => input.inputs.clone(), - _ => Default::default(), - }; - for source in &dependencies { - if !self.outputs.contains_key(source) { - return Err(SummaryCatalogError::Descriptor( - "derived input references missing summary".into(), - )); - } - consumers.entry(*source).or_default().push(*id); - } - pending.insert(*id, dependencies.len()); - } - let mut ready: Vec<_> = pending - .iter() - .filter_map(|(id, count)| (*count == 0).then_some(*id)) - .collect(); - let mut visited = 0; - while let Some(id) = ready.pop() { - visited += 1; - for consumer in consumers.get(&id).into_iter().flatten() { - let count = pending - .get_mut(consumer) - .expect("catalog dependency target"); - *count -= 1; - if *count == 0 { - ready.push(*consumer); - } - } - } - if visited != pending.len() { - return Err(SummaryCatalogError::Descriptor( - "derived summary dependencies have a cycle".into(), - )); - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::sds::ValueProjectionIdentity; - use crate::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - - fn config(metric: &str, filter: &str, window: u64) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), - KeyByLabelNames::new(vec!["job".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - window, - window, - WindowKind::Tumbling, - filter.into(), - metric.into(), - None, - None, - None, - ) - } - - // Content changes invalidate references even when plan/version are reused. - #[test] - fn table_populations_have_distinct_materialization_and_data_identities() { - use crate::table_population::{TableColumnPredicate, TablePopulation}; - use planner_types::pre_asap::{CompareOpKind, ScalarValue}; - let mut requests = config("raw_samples.value", "", 60); - requests.table_name = Some("raw_samples".into()); - requests.value_projection = Some(ValueProjectionIdentity::Column { - name: "value".into(), - }); - requests.table_population = Some(TablePopulation { - predicates: vec![TableColumnPredicate { - column: "metric".into(), - operator: CompareOpKind::Eq, - value: ScalarValue::Utf8("requests".into()), - }], - }); - let mut errors = requests.clone(); - errors.table_population.as_mut().unwrap().predicates[0].value = - ScalarValue::Utf8("errors".into()); - assert_ne!(requests.policy_fingerprint(), errors.policy_fingerprint()); - let mut other_table = requests.clone(); - other_table.table_name = Some("other_samples".into()); - assert_ne!( - requests.policy_fingerprint(), - other_table.policy_fingerprint() - ); - let mut other_value = requests.clone(); - other_value.value_projection = Some(ValueProjectionIdentity::Column { - name: "other_value".into(), - }); - assert_ne!( - requests.policy_fingerprint(), - other_value.policy_fingerprint() - ); - let mut other_time = requests.clone(); - other_time.table_timestamp_column = Some("event_time_ms".into()); - assert_ne!( - requests.policy_fingerprint(), - other_time.policy_fingerprint() - ); - let mut invalid_labels = requests.clone(); - invalid_labels.table_population = None; - invalid_labels.spatial_filter = "job=\"requests\"".into(); - assert!(SummaryCatalog::from_materializations(1, 1, &[invalid_labels]).is_err()); - let mut invalid_time = requests.clone(); - invalid_time.table_timestamp_column = Some("time; DROP TABLE samples".into()); - assert!(SummaryCatalog::from_materializations(1, 1, &[invalid_time]).is_err()); - let catalog = - SummaryCatalog::from_materializations(1, 1, &[requests, errors, other_time]).unwrap(); - assert_eq!(catalog.data_descriptors.len(), 3); - assert_eq!(catalog.outputs.len(), 3); - } - - #[test] - fn snapshot_reference_is_deterministic_and_content_sensitive() { - let a = config("requests", "", 60); - let b = config("errors", "", 60); - let left = SummaryCatalog::from_materializations(1, 2, &[a.clone(), b.clone()]).unwrap(); - let reordered = SummaryCatalog::from_materializations(1, 2, &[b, a.clone()]).unwrap(); - assert_eq!(left.reference().unwrap(), reordered.reference().unwrap()); - let changed = SummaryCatalog::from_materializations(1, 2, &[a]).unwrap(); - assert_ne!(left.reference().unwrap(), changed.reference().unwrap()); - left.reference() - .unwrap() - .validate_snapshot(&left, 1, 2) - .unwrap(); - assert!(left - .reference() - .unwrap() - .validate_snapshot(&left, 1, 3) - .is_err()); - } - - // Panes share definitions but retain distinct physical materialization IDs. - #[test] - fn shares_descriptors_across_windows_and_deduplicates_materializations() { - let one = config("requests", "", 60); - let two = config("requests", "", 120); - let catalog = - SummaryCatalog::from_materializations(7, 2, &[one.clone(), two, one]).unwrap(); - assert_eq!(catalog.summary_descriptors.len(), 1); - assert_eq!(catalog.data_descriptors.len(), 1); - assert_eq!(catalog.outputs.len(), 2); - assert_eq!((catalog.plan_id, catalog.plan_version), (7, 2)); - } - - // Stored pane duration identifies a deployment output, not the summary meaning. - #[test] - fn semantic_definition_is_shared_across_deployed_pane_outputs() { - let catalog = SummaryCatalog::from_materializations( - 1, - 1, - &[config("requests", "", 60), config("requests", "", 120)], - ) - .unwrap(); - assert_eq!(catalog.definitions.len(), 1); - assert_eq!(catalog.outputs.len(), 2); - } - - // A hot and rebuild output share meaning but remain independently bound. - #[test] - fn hot_and_rebuild_have_one_definition_and_two_bound_outputs() { - let mut hot = config("latency", "", 60); - hot.stored_output_id = Some(StoredOutputId(41)); - let mut rebuild = hot.clone(); - rebuild.stored_output_id = Some(StoredOutputId(42)); - let catalog = SummaryCatalog::from_materializations(7, 42, &[hot, rebuild]).unwrap(); - assert_eq!(catalog.definitions.len(), 1); - let hot = catalog.output_reference(StoredOutputId(41)).unwrap(); - let rebuild = catalog.output_reference(StoredOutputId(42)).unwrap(); - assert_eq!(hot.definition_id, rebuild.definition_id); - assert_ne!(hot, rebuild); - let mut forged = catalog.clone(); - let crate::summary_semantics::SummarySemantics::Configured { computation, .. } = - &mut forged.definitions.values_mut().next().unwrap().semantics - else { - panic!("fixture") - }; - computation.predicate = "service=other".into(); - assert!(forged.validate().is_err()); - let mut unknown = catalog.clone(); - unknown - .definitions - .values_mut() - .next() - .unwrap() - .semantic_format_version += 1; - assert!(unknown.validate().is_err()); - } - - // Source/population changes never alias, while the operator can be reused. - #[test] - fn separates_population_and_operator_identity() { - let catalog = SummaryCatalog::from_materializations( - 1, - 1, - &[ - config("requests", "", 60), - config("errors", "", 60), - config("requests", "job=user", 60), - ], - ) - .unwrap(); - assert_eq!(catalog.summary_descriptors.len(), 1); - assert_eq!(catalog.data_descriptors.len(), 3); - } - - // Operator changes share population metadata without sharing state identity. - #[test] - fn changing_operator_parameters_creates_a_new_summary_descriptor() { - let mut a = config("requests", "", 60); - a.aggregation_type = AggregationType::DatasketchesKLL; - a.parameters.insert("K".into(), serde_json::json!(100)); - let mut b = a.clone(); - b.parameters.insert("K".into(), serde_json::json!(200)); - let catalog = SummaryCatalog::from_materializations(1, 1, &[a, b]).unwrap(); - assert_eq!(catalog.summary_descriptors.len(), 2); - assert_eq!(catalog.data_descriptors.len(), 1); - assert_eq!(catalog.outputs.len(), 2); - } - - // Construction order cannot affect the published snapshot bytes. - #[test] - fn snapshot_is_deterministic_and_round_trips() { - let a = config("requests", "", 60); - let b = config("errors", "", 60); - let forward = SummaryCatalog::from_materializations(7, 2, &[a.clone(), b.clone()]).unwrap(); - let backward = SummaryCatalog::from_materializations(7, 2, &[b, a]).unwrap(); - let bytes = serde_json::to_vec(&forward).unwrap(); - assert_eq!(bytes, serde_json::to_vec(&backward).unwrap()); - let decoded: SummaryCatalog = serde_json::from_slice(&bytes).unwrap(); - decoded.validate().unwrap(); - assert_eq!(decoded, forward); - } - - #[test] - fn catalog_definitions_do_not_store_writer_layout() { - let mut materialization = config("requests", "", 60); - materialization.pane_origin_ms = Some(7_000); - let id = StoredOutputId::from(materialization.policy_fingerprint()); - let catalog = SummaryCatalog::from_materializations(7, 2, &[materialization]).unwrap(); - assert!(catalog.outputs.contains_key(&id)); - assert!( - !serde_json::to_value(&catalog).unwrap()["outputs"][id.as_u64().to_string()] - .as_object() - .unwrap() - .contains_key("pane_origin_ms") - ); - - let mut invalid = serde_json::to_value(&catalog).unwrap(); - invalid["outputs"][id.as_u64().to_string()]["pane_origin_ms"] = serde_json::json!(7_000); - assert!(serde_json::from_value::(invalid).is_err()); - } - - // The same materialization cannot silently rebind to another population. - #[test] - fn conflicting_materialization_is_rejected() { - let first = config("requests", "", 60); - let second = config("errors", "", 60); - let catalog = - SummaryCatalog::from_materializations(1, 1, &[first.clone(), second]).unwrap(); - let summary = catalog.summary_descriptors.values().next().unwrap().clone(); - let data = catalog - .data_descriptors - .values() - .cloned() - .collect::>(); - let error = SummaryCatalog::build( - 1, - 1, - [ - (first.policy_fingerprint(), summary.clone(), data[0].clone()), - (first.policy_fingerprint(), summary, data[1].clone()), - ], - ) - .unwrap_err(); - assert!(matches!( - error, - SummaryCatalogError::ConflictingDefinition(_) - )); - } - - // Imported catalogs must resolve every foreign key and content identity. - #[test] - fn rejects_dangling_refs_tampered_keys_and_unknown_schema() { - let catalog = - SummaryCatalog::from_materializations(1, 1, &[config("requests", "", 60)]).unwrap(); - let mut broken = catalog.clone(); - broken.data_descriptors.clear(); - assert!(matches!( - broken.validate(), - Err(SummaryCatalogError::MissingDescriptor(_)) - )); - let mut broken = catalog.clone(); - let (_, descriptor) = broken.summary_descriptors.pop_first().unwrap(); - broken.summary_descriptors.insert( - serde_json::from_value(serde_json::json!("forged")).unwrap(), - descriptor, - ); - assert!(matches!( - broken.validate(), - Err(SummaryCatalogError::Descriptor(_)) - )); - let mut broken = catalog.clone(); - broken - .summary_descriptors - .values_mut() - .next() - .unwrap() - .state_schema_version += 1; - assert!(matches!( - broken.validate(), - Err(SummaryCatalogError::Descriptor(_)) - )); - let mut broken = catalog; - broken.schema_version = SUMMARY_CATALOG_SCHEMA_VERSION + 1; - assert!(matches!( - broken.validate(), - Err(SummaryCatalogError::SchemaVersion(_)) - )); - } - - // Native exact-only plans have a valid empty catalog, not dummy state. - #[test] - fn empty_catalog_is_valid() { - let catalog = SummaryCatalog::from_materializations(1, 1, &[]).unwrap(); - assert!(catalog.outputs.is_empty()); - catalog.validate().unwrap(); - } -} diff --git a/crates/asap_types/src/summary_semantics.rs b/crates/asap_types/src/summary_semantics.rs deleted file mode 100644 index 757a7e15b..000000000 --- a/crates/asap_types/src/summary_semantics.rs +++ /dev/null @@ -1,129 +0,0 @@ -//! Versioned, content-addressed meaning of a stored result. Runtime routing, -//! pane placement, codecs, retention and plan generations are deliberately absent. -use crate::sds::{ - DataDescriptor, DataSourceIdentity, SummaryDescriptor, SummaryOperator, ValueProjectionIdentity, -}; -use crate::summary_catalog::SummaryCatalogError; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SummaryDefinition { - pub semantic_format_version: u32, - pub semantics: SummarySemantics, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum SummarySemantics { - Planner { - fragment: crate::semantic_fragment::SemanticFragment, - }, - /// Restricted raw-input adapter for explicit native summary configurations. - /// General expressions must use the Planner fragment variant. - Configured { - computation: Box, - value_source_column: Option, - aggregated_labels: crate::KeyByLabelNames, - rollup_labels: crate::KeyByLabelNames, - }, -} - -/// Typed semantic contract of the supported summary input. This is not a -/// deployment plan: it contains neither physical nodes nor storage references. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SummaryComputation { - pub operator: SummaryOperator, - pub source: DataSourceIdentity, - pub value: ValueProjectionIdentity, - pub predicate: String, - pub grouping: crate::GroupingProjection, - pub timestamp_column: Option, - pub observation_semantics: String, -} - -impl SummaryDefinition { - pub fn from_descriptors( - summary: &SummaryDescriptor, - data: &DataDescriptor, - ) -> Result { - summary - .validate() - .map_err(|e| SummaryCatalogError::Descriptor(e.to_string()))?; - data.validate() - .map_err(|e| SummaryCatalogError::Descriptor(e.to_string()))?; - Ok(Self { - semantic_format_version: 1, - semantics: SummarySemantics::Configured { - value_source_column: None, - aggregated_labels: crate::KeyByLabelNames::empty(), - rollup_labels: crate::KeyByLabelNames::empty(), - computation: Box::new(SummaryComputation { - operator: summary.operator.clone(), - source: data.source.clone(), - value: data.value_projection.clone(), - predicate: data.population_filter_canonical.clone(), - grouping: data.group_by_keys.clone(), - timestamp_column: data.timestamp_column.clone(), - observation_semantics: data.observation_semantics.clone(), - }), - }, - }) - } - - pub fn with_config(mut self, config: &crate::PrecomputeMaterialization) -> Self { - let fragment = config.semantic_fragment.as_ref(); - if let Some(fragment) = fragment { - self.semantics = SummarySemantics::Planner { - fragment: fragment.clone(), - }; - } else if let SummarySemantics::Configured { - value_source_column, - aggregated_labels, - rollup_labels, - .. - } = &mut self.semantics - { - *value_source_column = config.value_source_column.clone(); - *aggregated_labels = config.aggregated_labels.clone(); - *rollup_labels = config.rollup_labels.clone(); - } - self - } - - pub fn id(&self) -> Result { - if self.semantic_format_version != 1 { - return Err(SummaryCatalogError::Descriptor( - "unsupported semantic format version".into(), - )); - } - if let SummarySemantics::Planner { fragment } = &self.semantics { - fragment - .validate() - .map_err(SummaryCatalogError::Descriptor)?; - } - // Object keys are recursively sorted, independent of serde_json features. - fn canonical(value: serde_json::Value) -> serde_json::Value { - match value { - serde_json::Value::Object(values) => serde_json::Value::Object( - values - .into_iter() - .map(|(k, v)| (k, canonical(v))) - .collect::>() - .into_iter() - .collect(), - ), - serde_json::Value::Array(values) => { - serde_json::Value::Array(values.into_iter().map(canonical).collect()) - } - value => value, - } - } - let value = serde_json::to_value(self) - .map_err(|e| SummaryCatalogError::Descriptor(e.to_string()))?; - let bytes = serde_json::to_vec(&canonical(value)) - .map_err(|e| SummaryCatalogError::Descriptor(e.to_string()))?; - Ok(crate::sds::SummaryDefinitionId::from_semantics(&bytes)) - } -} diff --git a/crates/asap_types/src/table_population.rs b/crates/asap_types/src/table_population.rs deleted file mode 100644 index 4bb371387..000000000 --- a/crates/asap_types/src/table_population.rs +++ /dev/null @@ -1,133 +0,0 @@ -//! Typed table predicates shared by catalog identity and source readers. - -use planner_types::pre_asap::{CompareOpKind, ScalarValue}; -use serde::{Deserialize, Serialize}; - -/// A conjunction of column/literal comparisons. Column names are schema names, -/// not SQL fragments; readers must bind literal values as parameters. -#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] -pub struct TablePopulation { - pub predicates: Vec, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct TableColumnPredicate { - pub column: String, - pub operator: CompareOpKind, - pub value: ScalarValue, -} - -impl TablePopulation { - pub fn validate(&self) -> Result<(), String> { - for predicate in &self.predicates { - validate_column_name(&predicate.column)?; - if !matches!( - predicate.operator, - CompareOpKind::Eq - | CompareOpKind::Ne - | CompareOpKind::Lt - | CompareOpKind::Le - | CompareOpKind::Gt - | CompareOpKind::Ge - ) { - return Err("table population comparison is unsupported".into()); - } - if matches!( - predicate.value, - ScalarValue::Null | ScalarValue::Interval { .. } - ) || matches!(predicate.value, ScalarValue::Float64(value) if !value.is_finite()) - { - return Err("table population requires a finite non-null literal".into()); - } - } - Ok(()) - } - - /// Reordered or repeated conjuncts identify the same population. - pub fn canonical(&self) -> String { - if self.predicates.is_empty() { - return String::new(); - } - let mut predicates: Vec<_> = self - .predicates - .iter() - .map(|predicate| serde_json::to_string(predicate).expect("predicate serialization")) - .collect(); - predicates.sort(); - predicates.dedup(); - format!("sql.and.v1:[{}]", predicates.join(",")) - } -} - -pub(crate) fn validate_column_name(column: &str) -> Result<(), String> { - if column.is_empty() - || !column.as_bytes()[0].is_ascii_alphabetic() && !column.starts_with('_') - || !column - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') - { - return Err("table column is not an unqualified identifier".into()); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn metric(value: &str) -> TableColumnPredicate { - TableColumnPredicate { - column: "metric".into(), - operator: CompareOpKind::Eq, - value: ScalarValue::Utf8(value.into()), - } - } - - #[test] - fn population_identity_distinguishes_values_and_ignores_conjunct_order() { - let a = metric("requests"); - let b = TableColumnPredicate { - column: "status".into(), - operator: CompareOpKind::Ge, - value: ScalarValue::Int64(500), - }; - assert_eq!( - TablePopulation { - predicates: vec![a.clone(), b.clone(), a.clone()] - } - .canonical(), - TablePopulation { - predicates: vec![b, a.clone()] - } - .canonical() - ); - assert_ne!( - TablePopulation { - predicates: vec![a] - } - .canonical(), - TablePopulation { - predicates: vec![metric("errors")] - } - .canonical() - ); - } - - #[test] - fn rejects_sql_fragments_and_nonfinite_literals() { - let mut predicate = metric("requests"); - predicate.column = "metric OR 1=1".into(); - assert!(TablePopulation { - predicates: vec![predicate] - } - .validate() - .is_err()); - let mut predicate = metric("requests"); - predicate.value = ScalarValue::Float64(f64::NAN); - assert!(TablePopulation { - predicates: vec![predicate] - } - .validate() - .is_err()); - } -} diff --git a/crates/asap_types/src/traits.rs b/crates/asap_types/src/traits.rs deleted file mode 100644 index 51afd904f..000000000 --- a/crates/asap_types/src/traits.rs +++ /dev/null @@ -1 +0,0 @@ -pub use asap_physical_operators::SerializableToSink; diff --git a/crates/asap_types/src/utils.rs b/crates/asap_types/src/utils.rs deleted file mode 100644 index 025a5e78c..000000000 --- a/crates/asap_types/src/utils.rs +++ /dev/null @@ -1,56 +0,0 @@ -/// Normalize spatial filter for PromQL queries -pub fn normalize_spatial_filter(filter: &str) -> String { - if filter.is_empty() { - return String::new(); - } - - let trimmed = filter.trim().strip_prefix('{').unwrap_or(filter.trim()); - let trimmed = trimmed.strip_suffix('}').unwrap_or(trimmed); - let trimmed = trimmed.trim(); - - let mut parts = Vec::new(); - let mut start = 0; - let mut quoted = false; - let mut escaped = false; - for (index, ch) in trimmed.char_indices() { - if escaped { - escaped = false; - } else if ch == '\\' && quoted { - escaped = true; - } else if ch == '"' { - quoted = !quoted; - } else if ch == ',' && !quoted { - parts.push(trimmed[start..index].trim()); - start = index + 1; - } - } - parts.push(trimmed[start..].trim()); - parts.sort(); - - format!("{{{}}}", parts.join(",")) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_normalize_spatial_filter() { - assert_eq!(normalize_spatial_filter("").as_str(), ""); - - let result = normalize_spatial_filter("instance=\"localhost:9090\""); - assert_eq!(result, "{instance=\"localhost:9090\"}"); - - let result = normalize_spatial_filter("{instance=\"localhost:9090\"}"); - assert_eq!(result, "{instance=\"localhost:9090\"}"); - - let result = normalize_spatial_filter("{job=\"prometheus\",instance=\"localhost:9090\"}"); - assert_eq!(result, "{instance=\"localhost:9090\",job=\"prometheus\"}"); - - let result = normalize_spatial_filter("job=\"prometheus\",instance=\"localhost:9090\""); - assert_eq!(result, "{instance=\"localhost:9090\",job=\"prometheus\"}"); - - let result = normalize_spatial_filter(r#"{status=~"5(00,01)",job="api"}"#); - assert_eq!(result, r#"{job="api",status=~"5(00,01)"}"#); - } -} diff --git a/data_plane/.cargo/config.toml b/data_plane/.cargo/config.toml deleted file mode 100644 index c91c3f38b..000000000 --- a/data_plane/.cargo/config.toml +++ /dev/null @@ -1,2 +0,0 @@ -[net] -git-fetch-with-cli = true diff --git a/data_plane/.gitignore b/data_plane/.gitignore deleted file mode 100644 index eb5a316cb..000000000 --- a/data_plane/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index 223275418..d11d14796 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -4,103 +4,21 @@ version.workspace = true edition.workspace = true [dependencies] -# Internal crates (workspace) -asap_types.workspace = true -asap-physical-operators.workspace = true -asap_sketch_codec.workspace = true -# Phase 9: the control plane is now an in-process library inside the -# backend binary. Wiring up the in-process OpAMP server + capability-map -# exposure is a follow-up after Phase 4 (centralized series_id -# resolver). For now this dep just makes the control_plane crate -# compile in the workspace and importable from main.rs. -control_plane = { path = "../control_plane" } -sha2 = "0.10" - -# Use the same Planner IR as the control plane at compilation boundaries. -planner-types.workspace = true -asap-frontend-promql.workspace = true - -# Shared external (workspace) +control_plane.workspace = true serde.workspace = true serde_json.workspace = true -serde_yaml.workspace = true thiserror.workspace = true anyhow.workspace = true tracing.workspace = true tracing-subscriber.workspace = true clap.workspace = true -chrono.workspace = true -chrono-tz = "0.10" -promql-parser.workspace = true tokio.workspace = true -arc-swap.workspace = true - -# Crate-specific (keep version pinned here) -form_urlencoded = "1.2" -axum = "0.7" -bincode = "1.3" -dashmap = "5.5" -flate2 = "1.0" -async-trait = "0.1" -xxhash-rust = { version = "0.8", features = ["xxh32", "xxh64"] } -base64 = "0.21" -hex = "0.4" -arrow = "53.4.1" -futures = "0.3" +axum.workspace = true +reqwest.workspace = true prost = "0.13" snap = "1.1" -# Vendored modified opentelemetry-proto with first-class sketch metric -# variants from DataCollector (DDSketch / KLLSketch / CountSketch / -# CountMinSketch / HLLSketch). See docs/pipeline-query-catalog.md §5.4 in the -# DataCollector repo for the rationale. -asap_otel_proto.workspace = true -# `gzip` feature: enables CompressionEncoding::Gzip so the OTLP gRPC -# ingest server can `.accept_compressed()` gzip bodies from the -# `asap-gzip` arm's agent. control_plane already pulls tonic with this -# feature for its RuntimeSamples gRPC. -tonic = { version = "0.12", features = ["gzip"] } -tokio-stream = { version = "0.1", features = ["net"] } -regex = "1" -prometheus = "0.13" -lazy_static = "1.4" -reqwest.workspace = true -tracing-appender = "0.2" -asap_sketchlib = { git = "https://github.com/ProjectASAP/asap_sketchlib", rev = "5f03ccbd798ed5fec62bdd839bcb331123cab369" } -# Generic MessagePack codec for decoding the heap-bearing CountSketch -# DELTA-HEAP wire frame (`MSGPACK_DELTA`) directly in the backend, WITHOUT -# adding a delta API to the public `asap_sketchlib`: the frame is decoded -# into local serde structs here, the sparse matrix delta is applied onto the -# stored matrix, and the heap is replaced — all via `asap_sketchlib`'s -# existing PUBLIC `from_legacy_matrix`/`sketch_matrix`/`topk_heap_items` API. -# Already in the lock as a transitive dep of `asap_sketchlib`. -rmp-serde = "1.3" -# Persistence layer (SketchStore parts / manifest / Tier-2 cache) -moka = { version = "0.12", features = ["sync"] } -memmap2 = "0.9" -crc32fast = "1.4" -fs2 = "0.4" -# NOTE: the `asap-gorilla` path-dep + the `s3` (rust-s3) / `lru` -# crates were dropped when the superseded GORILLA1 read side -# (`gorilla_object_store`: GorillaS3Store + decode_block + the -# in-process GorillaQueryEngine) was deleted. The archive tier is -# served by the ThanosQueryEngine (Path A2) over HTTP, which needs -# none of them. [dev-dependencies] -asap-aware-mapping.workspace = true -tempfile = "3.20.0" -criterion = { version = "0.5", features = ["html_reports"] } +planner-types.workspace = true tower = { version = "0.5", features = ["util"] } http-body-util = "0.1" - -[[bench]] -name = "sketch_db" -harness = false - -[features] -#default = ["lock_profiling", "extra_debugging"] -default = [] -# Enable lock profiling instrumentation -lock_profiling = [] -# Enable extra debugging output -extra_debugging = ["asap-physical-operators/extra_debugging"] diff --git a/data_plane/Dockerfile b/data_plane/Dockerfile deleted file mode 100644 index 004c7947a..000000000 --- a/data_plane/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# Builds the data_plane (backend) binary → asap/data-plane:dev. -# -# Repo-ownership (2026-05): each ASAPQuery-backend crate owns its own image -# Dockerfile here — companion: control_plane/Dockerfile -# Dockerfile. ASAPCollector only orchestrates + references images by name; the -# old cross-repo combined builder (ASAPCollector deploy/docker/Dockerfile.backend) -# is retired. -# -# Build from the backend checkout; Cargo fetches public Git dependencies -# at the revisions recorded in Cargo.lock. -# -# docker build -f data_plane/Dockerfile -t asap/data-plane:dev . -FROM rust:1.90-bookworm AS build -WORKDIR /src - -# protoc for the backend's prost-build usages; git for any git-dep fetches. -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - protobuf-compiler git ca-certificates && \ - rm -rf /var/lib/apt/lists/* - -COPY . ASAPQuery-backend - -RUN cd ASAPQuery-backend && cargo build --locked --release --bin data_plane - -# Runtime image: binary + CA certs. -FROM debian:bookworm-slim -RUN apt-get update && \ - apt-get install -y --no-install-recommends ca-certificates && \ - rm -rf /var/lib/apt/lists/* -# Default --output-dir for the rolling log file. -RUN mkdir -p /var/log/asap -COPY --from=build /src/ASAPQuery-backend/target/release/data_plane \ - /usr/local/bin/data_plane - -ENV RUST_LOG=info \ - ASAP_BACKEND_PORT=9090 \ - ASAP_BACKEND_METRICS_PORT=9465 -# query 9091, OTLP ingest 4317/4318, backend 9090, metrics 9465. -EXPOSE 9090 9091 9465 4317 4318 -ENTRYPOINT ["/usr/local/bin/data_plane"] diff --git a/data_plane/LICENSE b/data_plane/LICENSE deleted file mode 100644 index 404d65798..000000000 --- a/data_plane/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2025 SketchDB - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/data_plane/benches/sketch_db.rs b/data_plane/benches/sketch_db.rs deleted file mode 100644 index de4a448cf..000000000 --- a/data_plane/benches/sketch_db.rs +++ /dev/null @@ -1,515 +0,0 @@ -//! Criterion benchmarks for `SketchStore` — the ASAP-tier sketch DB. -//! -//! Run with: `cargo bench -p data_plane --bench sketch_db` -//! -//! Covers: -//! * `append_sample` — sketch insert -//! * `append_precompute` — precompute insert -//! * `query_range` — per-sid range query -//! * `query_precomputes_by_agg` — cross-sid precompute scan -//! -//! Varied dimensions: sid cardinality, windows-per-sid (depth), -//! query-window width, and sketch kind (DDSketch / KLL / HLL). -//! -//! NOTE: the insert benches use `iter_batched_ref`, not `iter_batched`. -//! `iter_batched` (criterion-0.5.1 `src/bencher.rs:264-272`) consumes its -//! input by value, so the per-iteration `Drop` of the setup `SketchStore` -//! runs inside the timed region. The `SketchStore` Drop scales linearly -//! with `num_sids` (each `SummarySeriesMetadata` owns a `String`, -//! `BTreeSet`, `SketchConfig`, `Option` — roughly ~100 ns -//! to drop apiece). At 10k sids that's ~1 ms of pure drop work per -//! "append" — i.e. ~1000× the actual `append_sample` cost. See -//! `examples/sketch_db_diag.rs` for the breakdown. - -use std::collections::{BTreeMap, BTreeSet}; - -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; - -use asap_sketchlib::proto::sketchlib::{ - sketch_envelope, DdSketchState, HllVariant as ProtoVariant, HyperLogLogState, KllState, - SketchEnvelope, -}; -use asap_sketchlib::DdSketch; -use asap_sketchlib::{HllSketch, HllVariant}; -use prost::Message; - -use asap_physical_operators::summary_kernels::SumAccumulator; -use data_plane::storage_engines::sketch_db::data::{ - AccuracyBound, AggKind, AggregationType, Capability, SketchAlgorithm, SketchConfig, - SketchEncoding, -}; -use data_plane::storage_engines::sketch_db::index::{SketchStore, SummarySeriesMetadata}; -use data_plane::storage_engines::SketchSampleState; - -// ── Payload builders ──────────────────────────────────────────────────────── - -fn encode_ddsketch(values: &[f64], alpha: f64) -> Vec { - let mut sk = DdSketch::new(alpha); - for v in values { - sk.update(*v); - } - let state = DdSketchState { - alpha: sk.alpha, - store_counts: sk.store_counts.clone(), - store_offset: sk.store_offset, - ..Default::default() - }; - SketchEnvelope { - sketch_state: Some(sketch_envelope::SketchState::Ddsketch(state)), - ..Default::default() - } - .encode_to_vec() -} - -fn encode_kll(k: u32, items: &[f64]) -> Vec { - let state = KllState { - k, - items: items.to_vec(), - levels: vec![], - num_levels: 0, - ..Default::default() - }; - SketchEnvelope { - sketch_state: Some(sketch_envelope::SketchState::Kll(state)), - ..Default::default() - } - .encode_to_vec() -} - -fn encode_hll(distinct_items: usize, precision: u32) -> Vec { - let mut sk = HllSketch::new(HllVariant::Regular, precision); - for i in 0..distinct_items { - sk.update(format!("user-{i}").as_bytes()); - } - let state = HyperLogLogState { - variant: ProtoVariant::Regular as i32, - precision: sk.precision, - registers: sk.registers.clone(), - hip_kxq0: sk.hip_kxq0, - hip_kxq1: sk.hip_kxq1, - hip_est: sk.hip_est, - registers_sparse: None, - }; - SketchEnvelope { - sketch_state: Some(sketch_envelope::SketchState::Hll(state)), - ..Default::default() - } - .encode_to_vec() -} - -fn sample(bytes: Vec) -> SketchSampleState { - SketchSampleState { - bytes, - encoding: SketchEncoding::ProtoFull, - } -} - -// ── Metadata builders ─────────────────────────────────────────────────────── - -fn sketch_meta( - sid: u64, - algorithm: SketchAlgorithm, - config: SketchConfig, -) -> SummarySeriesMetadata { - SummarySeriesMetadata { - storage_handle: sid, - metric_name: "bench_metric".into(), - group_by_keys: BTreeSet::new(), - capability: Some(match algorithm { - SketchAlgorithm::Hll => Capability::CardinalityApprox, - _ => Capability::QuantileApprox(Some(algorithm.clone())), - }), - accuracy: Some(AccuracyBound::from_config(&config)), - agg_kind: AggKind::Sketch { - algorithm, - config, - spatial_filter_canonical: String::new(), - }, - first_seen_unix_ms: 0, - retired_at_ms: None, - expires_at_ms: None, - policy_fp: asap_types::PolicyFingerprint::UNSET, - } -} - -fn precompute_meta(sid: u64, metric: &str, agg_type: AggregationType) -> SummarySeriesMetadata { - SummarySeriesMetadata { - storage_handle: sid, - metric_name: metric.to_string(), - group_by_keys: BTreeSet::new(), - capability: None, - accuracy: None, - agg_kind: AggKind::ExactAgg { - agg_type, - parameters_canonical: String::new(), - spatial_filter_canonical: String::new(), - }, - first_seen_unix_ms: 0, - retired_at_ms: None, - expires_at_ms: None, - policy_fp: asap_types::PolicyFingerprint::UNSET, - } -} - -#[derive(Clone)] -struct KindCase { - name: &'static str, - algorithm: SketchAlgorithm, - build: fn() -> (SketchConfig, Vec), -} - -fn dd_small() -> (SketchConfig, Vec) { - let cfg = SketchConfig::DDSketch { - relative_accuracy: 0.01, - }; - let bytes = encode_ddsketch(&(1..=64).map(|i| i as f64).collect::>(), 0.01); - (cfg, bytes) -} - -fn kll_small() -> (SketchConfig, Vec) { - let cfg = SketchConfig::Kll { k: 200 }; - let items: Vec = (1..=64).map(|i| i as f64).collect(); - (cfg, encode_kll(200, &items)) -} - -fn hll_small() -> (SketchConfig, Vec) { - let cfg = SketchConfig::Hll { precision: 10 }; - let bytes = encode_hll(256, 10); - (cfg, bytes) -} - -const KINDS: &[KindCase] = &[ - KindCase { - name: "DDSketch", - algorithm: SketchAlgorithm::DDSketch, - build: dd_small, - }, - KindCase { - name: "KLL", - algorithm: SketchAlgorithm::Kll, - build: kll_small, - }, - KindCase { - name: "HLL", - algorithm: SketchAlgorithm::Hll, - build: hll_small, - }, -]; - -// ── append_sample ─────────────────────────────────────────────────────────── -// -// Pre-populates the store with `num_sids` registered metadata entries once -// per setup call; the timed routine performs ONE `append_sample`. Using -// `iter_batched_ref` so the setup store is dropped OUTSIDE the timed region -// (the original `iter_batched` formulation timed that drop, which scales -// linearly with `num_sids` and was masking the real per-op cost — see the -// module doc-comment). - -fn bench_append_sample(c: &mut Criterion) { - let mut g = c.benchmark_group("append_sample"); - g.sample_size(20); - - for kind in KINDS { - let (cfg, payload_bytes) = (kind.build)(); - for num_sids in [1usize, 100, 10_000] { - g.throughput(Throughput::Elements(1)); - g.bench_function( - BenchmarkId::new(kind.name, format!("sids={num_sids}")), - |b| { - b.iter_batched_ref( - || { - let store = SketchStore::new(); - for sid in 0..num_sids as u64 { - store.register(sketch_meta( - sid + 1, - kind.algorithm.clone(), - cfg.clone(), - )); - } - (store, 0u64) - }, - |(store, counter)| { - let c = *counter; - *counter = c.wrapping_add(1); - let sid = (c % num_sids as u64) + 1; - let win = (1_000 + c * 10, 1_000 + c * 10 + 10); - store.append_sample( - sid, - BTreeMap::new(), - win, - sample(payload_bytes.clone()), - ); - black_box(&*store); - }, - criterion::BatchSize::SmallInput, - ); - }, - ); - } - } - g.finish(); -} - -// ── append_precompute ─────────────────────────────────────────────────────── - -fn bench_append_precompute(c: &mut Criterion) { - let mut g = c.benchmark_group("append_precompute"); - g.sample_size(20); - - for num_sids in [1usize, 100, 10_000] { - g.throughput(Throughput::Elements(1)); - g.bench_function(BenchmarkId::new("Sum", format!("sids={num_sids}")), |b| { - b.iter_batched_ref( - || { - let store = SketchStore::new(); - for sid in 0..num_sids as u64 { - store.register(precompute_meta( - sid + 1, - "bench_metric", - AggregationType::Sum, - )); - } - (store, 0u64) - }, - |(store, counter)| { - let c = *counter; - *counter = c.wrapping_add(1); - let sid = (c % num_sids as u64) + 1; - let win = (1_000 + c * 10, 1_000 + c * 10 + 10); - store.append_precompute( - sid, - BTreeMap::new(), - win, - Box::new(SumAccumulator::with_sum(c as f64)), - ); - black_box(&*store); - }, - criterion::BatchSize::SmallInput, - ); - }); - } - g.finish(); -} - -// ── query_range ───────────────────────────────────────────────────────────── - -fn build_populated_store(depth: u64, kind: &KindCase, sid: u64) -> SketchStore { - let store = SketchStore::new(); - let (cfg, payload_bytes) = (kind.build)(); - store.register(sketch_meta(sid, kind.algorithm.clone(), cfg)); - for i in 0..depth { - let win = (1_000 + i * 10, 1_000 + i * 10 + 10); - store.append_sample(sid, BTreeMap::new(), win, sample(payload_bytes.clone())); - } - store -} - -fn bench_query_range(c: &mut Criterion) { - let mut g = c.benchmark_group("query_range"); - g.sample_size(20); - - let sid = 1u64; - for kind in KINDS { - for depth in [10u64, 100, 1_000] { - let store = build_populated_store(depth, kind, sid); - let full_end = 1_000 + depth * 10 + 10; - let cases = [ - ("w=1", (1_000u64, 1_010u64)), - ("w=half", (1_000u64, 1_000 + (depth / 2) * 10)), - ("w=full", (1_000u64, full_end)), - ]; - for (width_label, (start, end)) in cases { - g.throughput(Throughput::Elements(1)); - g.bench_function( - BenchmarkId::new(kind.name, format!("depth={depth}/{width_label}")), - |b| { - b.iter(|| { - let r = - store.query_range(black_box(sid), black_box(start), black_box(end)); - black_box(r); - }); - }, - ); - } - } - } - g.finish(); -} - -// ── query_precomputes_by_agg ─────────────────────────────────────────────── - -fn build_precompute_store(num_sids: usize, windows_per_sid: u64, metric: &str) -> SketchStore { - let store = SketchStore::new(); - for s in 0..num_sids as u64 { - let sid = s + 1; - store.register(precompute_meta(sid, metric, AggregationType::Sum)); - for i in 0..windows_per_sid { - let win = (1_000 + i * 10, 1_000 + i * 10 + 10); - store.append_precompute( - sid, - BTreeMap::new(), - win, - Box::new(SumAccumulator::with_sum((sid + i) as f64)), - ); - } - } - store -} - -fn bench_query_precomputes_by_agg(c: &mut Criterion) { - let mut g = c.benchmark_group("query_precomputes_by_agg"); - g.sample_size(15); - - let metric = "bench_metric"; - let cases: &[(usize, u64)] = &[(1, 100), (100, 10), (100, 100), (10_000, 10)]; - for (num_sids, depth) in cases { - let store = build_precompute_store(*num_sids, *depth, metric); - let full_end = 1_000 + depth * 10 + 10; - let widths = [ - ("w=1", (1_000u64, 1_010u64)), - ("w=full", (1_000u64, full_end)), - ]; - for (width_label, (start, end)) in widths { - g.throughput(Throughput::Elements(*num_sids as u64)); - g.bench_function( - BenchmarkId::new(format!("sids={num_sids}/depth={depth}"), width_label), - |b| { - b.iter(|| { - let r = store.query_precomputes_by_agg( - black_box(metric), - AggregationType::Sum, - black_box(start), - black_box(end), - ); - black_box(r); - }); - }, - ); - } - } - g.finish(); -} - -/// Build a `InstalledPrecomputePlan` whose single agg-config's content -/// signature matches every sid registered by `build_precompute_store` -/// (metric / `Sum` / no grouping / empty params+filter). With this -/// config the reconciler retires nothing — the steady-state ingest -/// case, where the per-batch reconcile is pure scan overhead. -fn matching_streaming_config( - metric: &str, -) -> data_plane::storage_engines::types::InstalledPrecomputePlan { - use asap_types::aggregation_config::PrecomputeMaterialization; - use asap_types::enums::WindowKind; - use asap_types::AggregationType as AT; - use asap_types::KeyByLabelNames; - use std::collections::HashMap; - - let cfg = PrecomputeMaterialization::new( - AT::Sum, - String::new(), - HashMap::new(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ); - let runtime = data_plane::storage_engines::types::InstalledPrecomputePlan::default(); - let plan = control_plane::physical::compiler::PrecomputePlan::build( - runtime.plan().envelope.clone(), - vec![cfg], - &[], - ) - .unwrap(); - data_plane::storage_engines::types::InstalledPrecomputePlan::from_precompute_plan(plan).unwrap() -} - -/// `reconcile_from_streaming_config` ran on EVERY ingest batch and, in -/// the pre-optimization code, deep-cloned every `SummarySeriesMetadata` -/// in the catalog (`snapshot_instances()`) — the dominant ingest-path -/// CPU cost in live `perf` profiling (BTreeMap/String clone + malloc -/// churn). This bench measures one un-gated reconcile against a -/// populated store at a few catalog sizes, so the before/after clone -/// elimination is directly visible. -fn bench_reconcile_per_batch(c: &mut Criterion) { - use std::time::Duration; - - let mut g = c.benchmark_group("reconcile_per_batch"); - g.sample_size(50); - - let metric = "bench_metric"; - for num_sids in [100usize, 1_000, 10_000] { - let store = build_precompute_store(num_sids, 1, metric); - let config = matching_streaming_config(metric); - g.throughput(Throughput::Elements(num_sids as u64)); - g.bench_function(BenchmarkId::new("full_scan", num_sids), |b| { - b.iter(|| { - let summary = - data_plane::storage_engines::sketch_db::lifecycle::reconcile_from_streaming_config( - black_box(&store), - black_box(&config), - Duration::from_secs(60), - ); - black_box(summary); - }); - }); - } - g.finish(); -} - -fn bench_group_key_projection(c: &mut Criterion) { - use data_plane::precompute_engine::group_key::intern_pairs; - - let cardinality = 100_000usize; - let labels = (0..cardinality) - .map(|index| (format!("region;{index}"), format!("service={index}"))) - .collect::>(); - let mut group = c.benchmark_group("group_key_projection"); - group.throughput(Throughput::Elements(cardinality as u64)); - group.bench_function("cold_high_cardinality", |b| { - b.iter(|| { - for (region, service) in &labels { - black_box(intern_pairs([ - ("region", region.as_str()), - ("service", service.as_str()), - ])); - } - }); - }); - // Warm the bounded interner, then measure shared-DAG reuse. - for (region, service) in &labels { - black_box(intern_pairs([ - ("region", region.as_str()), - ("service", service.as_str()), - ])); - } - group.bench_function("warm_shared_projection", |b| { - b.iter(|| { - for (region, service) in &labels { - black_box(intern_pairs([ - ("region", region.as_str()), - ("service", service.as_str()), - ])); - } - }); - }); - group.finish(); -} - -criterion_group!( - benches, - bench_append_sample, - bench_append_precompute, - bench_query_range, - bench_query_precomputes_by_agg, - bench_reconcile_per_batch, - bench_group_key_projection, -); -criterion_main!(benches); diff --git a/data_plane/docker-compose.yml.j2 b/data_plane/docker-compose.yml.j2 deleted file mode 100644 index cacf95efe..000000000 --- a/data_plane/docker-compose.yml.j2 +++ /dev/null @@ -1,43 +0,0 @@ -# QueryEngine Rust Docker Compose Template -# This template is rendered with Jinja2 to generate the final docker-compose.yml - -version: '3.8' - -services: - queryengine-rust: - image: sketchdb-queryengine-rust:latest - container_name: {{ container_name }} - environment: - - RUST_LOG={{ log_level }} - - RUST_BACKTRACE=1 - ports: - - "{{ http_port }}:8088" - network_mode: "host" - volumes: - # Mount output directory for experiment results - - "{{ experiment_output_dir }}:/app/outputs" - # Mount control plane output directory for configuration files (read-only) - - "{{ control_plane_remote_output_dir }}:/app/control_plane_output:ro" - command: [ - "--kafka-topic", "{{ kafka_topic }}", - "--kafka-broker", "{{ kafka_host }}:9092", - "--input-format", "{{ input_format }}", - "--config", "/app/control_plane_output/inference_config.yaml", - "--streaming-config", "/app/control_plane_output/streaming_config.yaml", - "--prometheus-server", "http://{{ prometheus_host }}:{{ prometheus_port }}", - "--prometheus-scrape-interval", "{{ prometheus_scrape_interval }}", - "--delete-existing-db", - "--log-level", "{{ log_level }}", - "--output-dir", "/app/outputs", - "--streaming-engine", "{{ streaming_engine }}", - "--query-language", "{{ query_language }}", - "--lock-strategy", "{{ lock_strategy }}"{% if compress_json %}, - "--decompress-json"{% endif %}{% if profile_query_engine %}, - "--do-profiling"{% endif %}{% if forward_unsupported_queries %}, - "--forward-unsupported-queries"{% endif %}{% if dump_precomputes %}, - "--dump-precomputes"{% endif %} - ] - extra_hosts: - - "kafka:{{ kafka_host }}" - - "prometheus:{{ prometheus_host }}" - restart: no diff --git a/data_plane/examples/audit_clickhouse_fallback.rs b/data_plane/examples/audit_clickhouse_fallback.rs deleted file mode 100644 index 02e2f085e..000000000 --- a/data_plane/examples/audit_clickhouse_fallback.rs +++ /dev/null @@ -1,200 +0,0 @@ -use axum::{ - body::Bytes, - http::{HeaderMap, Method}, -}; -use control_plane::{ - physical::compiler::{PlanEnvelope, PrecomputePlan, BACKEND_COMPAT, PLANNER_REVISION}, - query_plan::{ClickHousePlanningContext, QueryPlan}, -}; -use data_plane::{ - drivers::query::servers::http::{validate_and_build_runtime_plan, PhysicalPlanInstallRequest}, - query_engines::asap_clickhouse_query_engine::{ - accelerator::CatalogClickHouseAccelerator, ClickHouseAccelerationOutcome, - ClickHouseAccelerator, ClickHouseHttpFallback, ClickHouseHttpServer, - }, - storage_engines::{ - sketch_db::index::SketchStore, - types::{ActivePhysicalPlanHandle, BackendStorageRouting}, - }, -}; -use planner_types::{ - pre_asap::{Column, DataType, Schema}, - types::AccuracyTarget, -}; -use serde::Deserialize; -use serde_json::json; -use std::{ - collections::{BTreeMap, HashMap}, - fs, - path::PathBuf, - sync::Arc, -}; - -#[derive(Deserialize)] -struct Corpus { - queries: Vec, -} - -#[derive(Deserialize)] -struct Row { - id: String, - clickhouse_sql: Option, -} - -#[tokio::main] -async fn main() { - let path = std::env::args_os() - .nth(1) - .map(PathBuf::from) - .expect("usage: audit_clickhouse_fallback CORPUS.json"); - let corpus: Corpus = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); - let schema = Schema::with_time_index( - vec![ - Column::new("metric", DataType::Utf8, false), - Column::new("labels", DataType::Utf8, false), - Column::new("ts_ms", DataType::Timestamp, false), - Column::new("value", DataType::Float64, false), - ], - 2, - vec![], - ); - let envelope = PlanEnvelope { - plan_id: 27, - plan_version: 1, - generated_at_unix_ms: 1_788_891_296_000, - activation_unix_ms: 1_788_891_296_000, - expiry_unix_ms: None, - backend_compat: BACKEND_COMPAT.into(), - planner_revision: PLANNER_REVISION.into(), - capability_snapshot_id: "sql27-main-eval".into(), - }; - let catalog = - asap_types::summary_catalog::SummaryCatalog::from_materializations(27, 1, &[]).unwrap(); - let reference = catalog.reference().unwrap(); - let mut precompute_plan = - PrecomputePlan::build_backend_local(envelope.clone(), vec![]).unwrap(); - precompute_plan.bind_catalog(&catalog).unwrap(); - let mut transmission_plan = control_plane::physical::compiler::build_transmission_plan( - envelope, - &precompute_plan, - &BTreeMap::new(), - ) - .unwrap(); - transmission_plan.summary_catalog = Some(reference); - let query_plan = QueryPlan { - plan_id: 27, - plan_version: 1, - clickhouse_context: Some(ClickHousePlanningContext { - window_templates: Default::default(), - tables: HashMap::from([("raw_samples".into(), schema)]), - accuracy: AccuracyTarget::Epsilon(0.01), - }), - selected_dags: Default::default(), - entries: BTreeMap::new(), - }; - let active = validate_and_build_runtime_plan( - PhysicalPlanInstallRequest { - summary_catalog: catalog, - collector_plans: vec![], - precompute_plan, - transmission_plan, - query_plan, - storage_routing: None, - adaptation_evidence: vec![], - }, - Arc::new(BackendStorageRouting::empty()), - ) - .unwrap(); - let exact_url = - std::env::var("CLICKHOUSE_URL").unwrap_or_else(|_| "http://127.0.0.1:18123".into()); - let exact = Arc::new(ClickHouseHttpFallback::new( - exact_url.clone(), - std::env::var("CLICKHOUSE_DATABASE").unwrap_or_else(|_| "default".into()), - )); - let accelerator = Arc::new(CatalogClickHouseAccelerator::with_active_physical_plan( - Arc::new(SketchStore::new()), - ActivePhysicalPlanHandle::new(active), - )); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let app = ClickHouseHttpServer::router_with_accelerator(exact, accelerator.clone()); - let server = tokio::spawn(async move { axum::serve(listener, app).await }); - let client = reqwest::Client::new(); - let mut rows = Vec::new(); - for row in corpus.queries { - let sql = row - .clickhouse_sql - .unwrap_or_default() - .replace("{eval_ms}", "1788891296000"); - let request = data_plane::query_engines::asap_clickhouse_query_engine::request::ClickHouseQueryRequest { - method: Method::GET, - sql, - body: Bytes::new(), - parameters: BTreeMap::new(), - headers: HeaderMap::new(), - }; - let acceleration = accelerator.execute(&request).await; - let mut outbound = client - .post(format!("http://{address}/")) - .body(request.sql.clone()); - if let Ok(user) = std::env::var("CLICKHOUSE_USER") { - outbound = outbound.header("x-clickhouse-user", user); - } - if let Ok(password) = std::env::var("CLICKHOUSE_PASSWORD") { - outbound = outbound.header("x-clickhouse-key", password); - } - let response = outbound.send().await.unwrap(); - let status = response.status().as_u16(); - let headers = response.headers().clone(); - let body = response.text().await.unwrap(); - let mut direct = client.post(&exact_url).body(request.sql.clone()); - if let Ok(user) = std::env::var("CLICKHOUSE_USER") { - direct = direct.header("x-clickhouse-user", user); - } - if let Ok(password) = std::env::var("CLICKHOUSE_PASSWORD") { - direct = direct.header("x-clickhouse-key", password); - } - let direct = direct.send().await.unwrap(); - let direct_status = direct.status().as_u16(); - let direct_body = direct.text().await.unwrap(); - let matches_direct_exact = status == direct_status && body == direct_body; - rows.push(match acceleration { - ClickHouseAccelerationOutcome::Fallback(reason) => json!({ - "id": row.id, - "fallback_requested": true, - "acceleration_reason": format!("{reason:?}"), - "exact_executed": status != 502, - "exact_success": (200..300).contains(&status) && matches_direct_exact, - "exact_status": status, - "direct_exact_status": direct_status, - "matches_direct_exact": matches_direct_exact, - "clickhouse_summary": headers.get("x-clickhouse-summary").and_then(|v| v.to_str().ok()), - "clickhouse_exception_code": headers.get("x-clickhouse-exception-code").and_then(|v| v.to_str().ok()), - "result": body, - }), - ClickHouseAccelerationOutcome::Failed(error) => json!({ - "id": row.id, - "fallback_requested": false, - "exact_executed": false, - "exact_success": false, - "execution_error": error.to_string(), - "result": body, - }), - ClickHouseAccelerationOutcome::Accelerated(_) => json!({ - "id": row.id, - "fallback_requested": false, - "exact_executed": false, - "exact_success": false, - "exact_status": null, - "direct_exact_status": direct_status, - "matches_direct_exact": false, - "result": body, - }), - }); - } - println!( - "{}", - serde_json::to_string_pretty(&json!({"queries": rows})).unwrap() - ); - server.abort(); -} diff --git a/data_plane/examples/autosketch_comparison.rs b/data_plane/examples/autosketch_comparison.rs deleted file mode 100644 index 598e89e4d..000000000 --- a/data_plane/examples/autosketch_comparison.rs +++ /dev/null @@ -1,823 +0,0 @@ -//! Memory-constrained frequency-sketch selection; not a whole-system benchmark. -use asap_aware_mapping::erp::{ - AccuracyMode, ErpArtifact, ErpRecord, ErpResourceProfile, ErpSelectionRequest, - ERP_SCHEMA_VERSION, -}; -use asap_sketchlib::{Bloom, CountMinSketch, CountSketch, DataInput, DefaultXxHasher, RegularPath}; -use clap::{Parser, ValueEnum}; -use serde::Serialize; -use std::{ - collections::{BTreeMap, BTreeSet, VecDeque}, - time::Instant, -}; - -type MembershipBloom = Bloom; - -#[derive(Parser, Debug, Serialize)] -struct Args { - /// New JSON artifact path; existing files are never overwritten. - #[arg(long)] - output: std::path::PathBuf, - #[arg(long, default_value_t = 10000)] - events: usize, - /// Optional `pane_indexdense_key_index` trace. The first half is - /// calibration and the second half is held out from configuration search. - #[arg(long)] - input_tsv: Option, - #[arg(long, default_value_t = 1000)] - cardinality: usize, - #[arg(long, default_value_t = 10)] - runs: u64, - #[arg(long, default_value_t = 42)] - seed: u64, - #[arg(long, default_value_t = 0.01)] - epsilon: f64, - /// Query semantics determine which sketch families are legal. - #[arg(long, value_enum, default_value = "frequency")] - query: Query, - /// Zero is uniform; positive values generate a truncated Zipf distribution. - #[arg(long, default_value_t = 0.0)] - zipf: f64, - /// Required provenance, obtained with git rev-parse HEAD. - #[arg(long)] - backend_revision: String, - /// Hard cap on resident f64 counter payload per selected sketch (not RSS). - #[arg(long, default_value_t = 32768)] - memory_budget_bytes: usize, - #[arg( - long, - value_enum, - value_delimiter = ',', - default_value = "cms,count-sketch,bloom" - )] - sketches: Vec, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, ValueEnum)] -#[serde(rename_all = "snake_case")] -enum Family { - Cms, - CountSketch, - Bloom, -} -impl Family { - fn name(self) -> &'static str { - match self { - Self::Cms => "cms", - Self::CountSketch => "count_sketch", - Self::Bloom => "bloom", - } - } - - fn supports(self, query: Query) -> bool { - matches!( - (self, query), - (Self::Cms | Self::CountSketch, Query::Frequency) | (Self::Bloom, Query::Membership) - ) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, ValueEnum)] -#[serde(rename_all = "snake_case")] -enum Query { - Frequency, - Membership, -} -impl Query { - fn error_metric(self) -> &'static str { - match self { - Self::Frequency => "max_normalized_additive_error", - Self::Membership => "false_positive_rate", - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize)] -struct Config { - family: Family, - width: usize, - depth: usize, -} -impl Config { - fn bytes(self) -> usize { - match self.family { - Family::Cms | Family::CountSketch => { - self.width * self.depth * std::mem::size_of::() - } - Family::Bloom => { - MembershipBloom::with_dimensions(self.depth, self.width).size_in_bytes() - } - } - } - fn id(self) -> String { - format!("{}-{:05}-{:02}", self.family.name(), self.width, self.depth) - } - fn legal(self) -> bool { - // CMS selects Packed64/128/Rows automatically; portable CountSketch - // hardcodes Packed64 and needs room for every column/sign bit. - self.family == Family::Cms || self.depth * (self.width.ilog2() as usize + 1) <= 64 - } -} - -fn grid() -> Vec { - (0..7) - .flat_map(|i| { - (1..=8).map(move |depth| Config { - family: Family::Cms, - width: 64 << i, - depth, - }) - }) - .collect() -} - -fn candidate_grid(families: &[Family], query: Query, budget: usize) -> Vec { - families - .iter() - .flat_map(|&family| grid().into_iter().map(move |c| Config { family, ..c })) - .filter(|c| c.family.supports(query) && c.legal() && c.bytes() <= budget) - .collect::>() - .into_iter() - .collect() -} - -// Small deterministic PRNG for workload sampling and LHS permutations. CMS -// itself uses the library's fixed hash implementation, not this seed. -struct Rng(u64); -impl Rng { - fn next(&mut self) -> u64 { - self.0 = self.0.wrapping_add(0x9e3779b97f4a7c15); - let mut x = self.0; - x = (x ^ (x >> 30)).wrapping_mul(0xbf58476d1ce4e5b9); - x = (x ^ (x >> 27)).wrapping_mul(0x94d049bb133111eb); - x ^ (x >> 31) - } - fn shuffle(&mut self, values: &mut [T]) { - for i in (1..values.len()).rev() { - values.swap(i, self.next() as usize % (i + 1)); - } - } -} - -fn stream(events: usize, cardinality: usize, zipf: f64, seed: u64) -> Vec { - let mut cumulative = Vec::with_capacity(cardinality); - let mut sum = 0.0; - for i in 1..=cardinality { - sum += (i as f64).powf(-zipf); - cumulative.push(sum); - } - let mut rng = Rng(seed); - (0..events) - .map(|_| { - let u = (rng.next() >> 11) as f64 / ((1_u64 << 53) as f64) * sum; - cumulative.partition_point(|v| *v <= u).min(cardinality - 1) - }) - .collect() -} - -fn trace_split(path: &std::path::Path) -> Result<(Vec, Vec, usize), String> { - let contents = std::fs::read_to_string(path).map_err(|error| error.to_string())?; - let mut values = Vec::new(); - let mut max_key = 0; - for (line, row) in contents.lines().enumerate() { - let (_, key) = row - .split_once('\t') - .ok_or_else(|| format!("invalid trace TSV at line {}", line + 1))?; - let key: usize = key - .parse() - .map_err(|error| format!("line {}: {error}", line + 1))?; - max_key = max_key.max(key); - values.push(key); - } - if values.len() < 2 { - return Err("trace needs at least two events".into()); - } - let held_out = values.split_off(values.len() / 2); - Ok((values, held_out, max_key + 1)) -} - -#[derive(Debug, Serialize)] -struct Measurement { - config: Config, - error: f64, - payload_bytes: usize, - update_wall_seconds: f64, - query_wall_seconds: f64, -} - -fn measure(config: Config, query: Query, data: &[usize], keys: &[String]) -> Measurement { - assert!(!data.is_empty()); - let mut exact = vec![0_u64; keys.len()]; - for &key in data { - exact[key] += 1; - } - assert!(config.legal()); - let mut cms = - (config.family == Family::Cms).then(|| CountMinSketch::new(config.depth, config.width)); - let mut cs = (config.family == Family::CountSketch) - .then(|| CountSketch::new(config.depth, config.width)); - let mut bloom: Option = (config.family == Family::Bloom) - .then(|| Bloom::with_dimensions(config.depth, config.width)); - let started = Instant::now(); - for &key in data { - match (&mut cms, &mut cs, &mut bloom) { - (Some(sketch), _, _) => sketch.update(&keys[key], 1.0), - (_, Some(sketch), _) => sketch.update(&keys[key], 1.0), - (_, _, Some(sketch)) => sketch.insert(&DataInput::U64(key as u64)), - _ => unreachable!(), - } - } - let update_wall_seconds = started.elapsed().as_secs_f64(); - let started = Instant::now(); - let error = match query { - Query::Frequency => keys - .iter() - .zip(exact) - .map(|(key, truth)| { - let estimate = match (&cms, &cs) { - (Some(sketch), _) => sketch.estimate(key), - (_, Some(sketch)) => sketch.estimate(key), - _ => unreachable!(), - }; - assert!(estimate.is_finite()); - (estimate - truth as f64).abs() / data.len() as f64 - }) - .fold(0.0_f64, f64::max), - Query::Membership => { - let sketch = bloom.as_ref().expect("membership requires Bloom"); - assert!(data - .iter() - .all(|key| sketch.contains(&DataInput::U64(*key as u64)))); - let false_positives = (keys.len()..keys.len() * 2) - .filter(|key| sketch.contains(&DataInput::U64(*key as u64))) - .count(); - false_positives as f64 / keys.len() as f64 - } - }; - let query_wall_seconds = started.elapsed().as_secs_f64(); - Measurement { - config, - error, - payload_bytes: config.bytes(), - update_wall_seconds, - query_wall_seconds, - } -} - -fn lhs(seed: u64) -> Vec { - let mut rng = Rng(seed); - let mut widths: Vec<_> = (0..7).collect(); - let mut depths: Vec<_> = (1..=8).collect(); - rng.shuffle(&mut widths); - rng.shuffle(&mut depths); - widths - .into_iter() - .zip(depths) - .map(|(w, depth)| Config { - family: Family::Cms, - width: 64 << w, - depth, - }) - .collect() -} - -#[derive(Debug, Serialize)] -struct Search { - selected: Option, - visited: Vec, - selection_wall_seconds: f64, -} - -// Discrete software adaptation of Algorithm 4: depth +/-1, width x2 or /2. -// Each LHS path stops after crossing feasibility; duplicate evaluations are -// cached. Pruned feasible nodes may still lead to cheaper neighbors. -#[cfg(test)] -fn search(seed: u64, epsilon: f64, evaluate: impl FnMut(Config) -> f64) -> Search { - search_candidates( - seed, - epsilon, - &candidate_grid(&[Family::Cms], Query::Frequency, usize::MAX), - evaluate, - ) -} - -fn search_candidates( - seed: u64, - epsilon: f64, - candidates: &[Config], - mut evaluate: impl FnMut(Config) -> f64, -) -> Search { - let started = Instant::now(); - let families: BTreeSet<_> = candidates.iter().map(|c| c.family).collect(); - let mut pending = VecDeque::new(); - for family in families { - for point in lhs(seed) { - let c = Config { family, ..point }; - if candidates.contains(&c) { - pending.push_back((c, None)); - } - } - // A tight budget can exclude every LHS point. Keep a legal seed for - // every family rather than silently removing it from sketch selection. - if let Some(&minimum) = candidates - .iter() - .filter(|c| c.family == family) - .min_by_key(|c| (c.bytes(), **c)) - { - pending.push_back((minimum, None)); - } - } - let mut expanded = BTreeSet::new(); - let mut cache = BTreeMap::new(); - let mut visited = Vec::new(); - let mut best: Option = None; - while let Some((c, initial_feasible)) = pending.pop_front() { - if !expanded.insert((c, initial_feasible)) { - continue; - } - let error = *cache.entry(c).or_insert_with(|| { - visited.push(c); - evaluate(c) - }); - let feasible = error.is_finite() && error <= epsilon; - if feasible && best.is_none_or(|b| (c.bytes(), c) < (b.bytes(), b)) { - best = Some(c); - } - let direction = initial_feasible.unwrap_or(feasible); - if direction != feasible { - continue; - } - if !feasible && best.is_some_and(|b| c.bytes() >= b.bytes()) { - continue; - } - let neighbors = if feasible { - [ - c.width - .checked_div(2) - .filter(|w| *w >= 64) - .map(|width| Config { width, ..c }), - c.depth - .checked_sub(1) - .filter(|d| *d >= 1) - .map(|depth| Config { depth, ..c }), - ] - } else { - [ - (c.width < 4096).then_some(Config { - width: c.width * 2, - ..c - }), - (c.depth < 8).then_some(Config { - depth: c.depth + 1, - ..c - }), - ] - }; - for next in neighbors.into_iter().flatten() { - if candidates.contains(&next) { - pending.push_back((next, Some(direction))); - } - } - } - Search { - selected: best, - visited, - selection_wall_seconds: started.elapsed().as_secs_f64(), - } -} - -fn oracle(rows: &[Measurement], epsilon: f64) -> Option { - rows.iter() - .filter(|m| m.error <= epsilon) - .map(|m| m.config) - .min_by_key(|c| (c.bytes(), *c)) -} - -fn artifact( - rows: &[Measurement], - distribution: serde_json::Value, - revision: &str, - query: Query, -) -> ErpArtifact { - ErpArtifact { - schema_version: ERP_SCHEMA_VERSION, - producer_version: revision.into(), - records: rows - .iter() - .map(|m| ErpRecord { - id: m.config.id(), - sketch: m.config.family.name().into(), - implementation: "asap_sketchlib-portable".into(), - parameters: serde_json::json!({"width":m.config.width,"depth":m.config.depth}), - distribution: distribution.clone(), - trials: 1, - error_metrics: BTreeMap::from([(query.error_metric().into(), m.error)]), - // Memory-only experiment. Timings are wall measurements, not CPU - // profiles: deliberately do not mislabel them as ERP CPU evidence. - resources: ErpResourceProfile { - memory_bytes: m.payload_bytes as f64, - update_cpu_seconds: 0.0, - query_cpu_seconds: 0.0, - merge_cpu_seconds: 0.0, - }, - }) - .collect(), - } -} - -fn erp_select( - profile: &ErpArtifact, - distribution: serde_json::Value, - epsilon: f64, - query: Query, -) -> Option { - let request = ErpSelectionRequest { - distribution, - implementation: Some("asap_sketchlib-portable".into()), - allowed_sketches: match query { - Query::Frequency => vec!["cms".into(), "count_sketch".into()], - Query::Membership => vec!["bloom".into()], - }, - error_metric: query.error_metric().into(), - max_error: epsilon, - min_trials: 1, - expected_updates: 0.0, - expected_queries: 0.0, - expected_merges: 0.0, - retention_seconds: 1.0, - cpu_weight: 0.0, - byte_second_weight: 1.0, - mode: AccuracyMode::Empirical, - }; - match profile.select(&request) { - Ok(point) => Some(Config { - family: match point.record.sketch.as_str() { - "cms" => Family::Cms, - "count_sketch" => Family::CountSketch, - "bloom" => Family::Bloom, - _ => panic!("unknown family"), - }, - width: point.record.parameters["width"].as_u64().unwrap() as usize, - depth: point.record.parameters["depth"].as_u64().unwrap() as usize, - }), - Err(asap_aware_mapping::erp::ErpError::NoApplicableConfiguration) => None, - Err(error) => panic!("invalid experiment profile: {error}"), - } -} - -fn main() -> Result<(), Box> { - let mut args = Args::parse(); - let trace = args.input_tsv.as_deref().map(trace_split).transpose()?; - if let Some((calibration, _, cardinality)) = &trace { - args.events = calibration.len(); - args.cardinality = *cardinality; - } - if args.events == 0 - || args.cardinality == 0 - || args.runs == 0 - || !args.epsilon.is_finite() - || args.epsilon < 0.0 - || !args.zipf.is_finite() - || args.zipf < 0.0 - || args.backend_revision.trim().is_empty() - { - return Err("invalid experiment arguments".into()); - } - // Fail before benchmarking if the destination already exists. - let output = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&args.output)?; - let keys: Vec<_> = (0..args.cardinality).map(|i| format!("key-{i}")).collect(); - let mut runs = Vec::new(); - for run in 0..args.runs { - let calibration_seed = args.seed.wrapping_add(run.wrapping_mul(2)); - let test_seed = calibration_seed.wrapping_add(1); - let calibration = trace.as_ref().map_or_else( - || stream(args.events, args.cardinality, args.zipf, calibration_seed), - |(calibration, _, _)| calibration.clone(), - ); - let distribution = args.input_tsv.as_ref().map_or_else( - || serde_json::json!({"generator":"splitmix64-truncated-zipf-v1", - "events":args.events,"cardinality":args.cardinality,"zipf":args.zipf,"seed":calibration_seed}), - |path| serde_json::json!({"external":{"dataset":path.file_name().and_then(|name| name.to_str()).unwrap_or("trace.tsv"), - "split":"first_half_calibration"}}), - ); - let start = Instant::now(); - let mut candidates = candidate_grid(&args.sketches, args.query, args.memory_budget_bytes); - Rng(calibration_seed).shuffle(&mut candidates); - let rows: Vec<_> = candidates - .iter() - .copied() - .map(|c| measure(c, args.query, &calibration, &keys)) - .collect(); - let calibration_wall_seconds = start.elapsed().as_secs_f64(); - let profile = artifact( - &rows, - distribution.clone(), - &args.backend_revision, - args.query, - ); - let adapted = search_candidates(calibration_seed, args.epsilon, &candidates, |c| { - rows.iter().find(|m| m.config == c).unwrap().error - }); - let start = Instant::now(); - let erp = erp_select(&profile, distribution, args.epsilon, args.query); - let erp_selection_wall_seconds = start.elapsed().as_secs_f64(); - let start = Instant::now(); - let exhaustive = oracle(&rows, args.epsilon); - let oracle_selection_wall_seconds = start.elapsed().as_secs_f64(); - assert_eq!( - erp, exhaustive, - "same-table ERP must match the finite-grid oracle" - ); - // Test data is generated only after all configuration decisions are made. - let held_out = trace.as_ref().map_or_else( - || stream(args.events, args.cardinality, args.zipf, test_seed), - |(_, held_out, _)| held_out.clone(), - ); - let mut outcomes = Vec::new(); - for (name, selected) in [ - ("autosketch_adapted", adapted.selected), - ("asapplanner_erp_selector", erp), - ("grid_oracle", exhaustive), - ] { - let measured = selected.map(|c| measure(c, args.query, &held_out, &keys)); - let pass = measured.as_ref().map(|m| m.error <= args.epsilon); - outcomes.push(serde_json::json!({"method":name,"selected":selected, - "status":if selected.is_some(){"calibration_feasible"}else{"no_feasible_configuration"}, - "held_out_pass":pass,"held_out":measured})); - } - runs.push(serde_json::json!({"run":run,"calibration_seed":calibration_seed,"test_seed":test_seed, - "calibration_wall_seconds":calibration_wall_seconds,"calibration_grid":rows, - "erp_artifact":profile,"search":adapted,"erp_selection_wall_seconds":erp_selection_wall_seconds, - "oracle_selection_wall_seconds":oracle_selection_wall_seconds,"outcomes":outcomes})); - eprintln!( - "run {run}: ERP={erp:?}, adapted={:?}, oracle={exhaustive:?}", - adapted.selected - ); - } - serde_json::to_writer_pretty( - output, - &serde_json::json!({"schema_version":1,"args":args, - "debug_assertions":cfg!(debug_assertions), - "scope":"memory-constrained query-compatible sketch family and parameter selection; not full ASAPPlanner or system evaluation", - "memory_metric":"sketch payload bytes (f64 counters or packed Bloom bits); excludes allocator and object overhead", - "hash_seeds":"library fixed hash; only data/search seeds vary", - "timing_metric":"wall seconds, not CPU seconds; calibration cost reported separately", - "cargo_lock":include_str!("../../Cargo.lock"),"runs":runs}), - )?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn trace_calibration_and_held_out_halves_do_not_overlap() { - let path = - std::env::temp_dir().join(format!("autosketch-trace-split-{}.tsv", std::process::id())); - std::fs::write(&path, "0\t0\n0\t1\n1\t2\n1\t3\n").unwrap(); - let (calibration, held_out, cardinality) = trace_split(&path).unwrap(); - std::fs::remove_file(path).unwrap(); - assert_eq!(calibration, vec![0, 1]); - assert_eq!(held_out, vec![2, 3]); - assert_eq!(cardinality, 4); - } - #[test] - fn hard_budget_prunes_before_benchmark_and_empty_budget_fails_closed() { - for budget in [0, 511, 512, 2048] { - let candidates = candidate_grid( - &[Family::Cms, Family::CountSketch], - Query::Frequency, - budget, - ); - let result = search_candidates(42, 0.01, &candidates, |c| { - assert!(c.bytes() <= budget); - assert!(c.legal()); - 0.0 - }); - assert_eq!(result.selected.is_none(), budget < 512); - assert_eq!(candidates.is_empty(), budget < 512); - let rows: Vec<_> = candidates - .into_iter() - .map(|config| Measurement { - config, - error: 0.0, - payload_bytes: config.bytes(), - update_wall_seconds: 0.0, - query_wall_seconds: 0.0, - }) - .collect(); - let context = serde_json::json!({"test":"budget"}); - let selected = erp_select( - &artifact(&rows, context.clone(), "test", Query::Frequency), - context, - 0.01, - Query::Frequency, - ); - assert_eq!(selected, oracle(&rows, 0.01)); - assert_eq!(selected.is_none(), budget < 512); - assert!(selected.is_none_or(|c| c.bytes() <= budget)); - } - } - #[test] - fn sketch_selection_can_choose_either_family() { - let candidates = - candidate_grid(&[Family::Cms, Family::CountSketch], Query::Frequency, 2048); - for preferred in [Family::Cms, Family::CountSketch] { - let rows: Vec<_> = candidates - .iter() - .map(|&config| Measurement { - config, - error: if config.family == preferred { 0.0 } else { 1.0 }, - payload_bytes: config.bytes(), - update_wall_seconds: 0.0, - query_wall_seconds: 0.0, - }) - .collect(); - let result = search_candidates(42, 0.01, &candidates, |c| { - rows.iter().find(|r| r.config == c).unwrap().error - }); - let context = serde_json::json!({"test":"family"}); - let selected = erp_select( - &artifact(&rows, context.clone(), "test", Query::Frequency), - context, - 0.01, - Query::Frequency, - ); - assert_eq!(result.selected.unwrap().family, preferred); - assert_eq!(selected.unwrap().family, preferred); - assert_eq!(selected, oracle(&rows, 0.01)); - } - } - #[test] - fn query_semantics_filter_incompatible_families() { - let all = [Family::Cms, Family::CountSketch, Family::Bloom]; - let frequency = candidate_grid(&all, Query::Frequency, usize::MAX); - let membership = candidate_grid(&all, Query::Membership, usize::MAX); - assert!(frequency.iter().all(|c| c.family != Family::Bloom)); - assert!(membership.iter().all(|c| c.family == Family::Bloom)); - assert!(!membership.is_empty()); - } - #[test] - fn bloom_uses_sketchlib_packed_memory_and_has_no_false_negatives() { - let config = Config { - family: Family::Bloom, - width: 1024, - depth: 4, - }; - let keys: Vec<_> = (0..100).map(|i| format!("key-{i}")).collect(); - let data: Vec<_> = (0..100).collect(); - let measured = measure(config, Query::Membership, &data, &keys); - let library_bytes = - MembershipBloom::with_dimensions(config.depth, config.width).size_in_bytes(); - assert_eq!(measured.payload_bytes, library_bytes); - assert!((0.0..=1.0).contains(&measured.error)); - } - #[test] - fn bloom_autosketch_erp_and_oracle_share_budget_and_fpp_target() { - let candidates = candidate_grid(&[Family::Bloom], Query::Membership, 1024); - let keys: Vec<_> = (0..200).map(|i| format!("key-{i}")).collect(); - let data: Vec<_> = (0..200).collect(); - let rows: Vec<_> = candidates - .iter() - .copied() - .map(|c| measure(c, Query::Membership, &data, &keys)) - .collect(); - let adapted = search_candidates(42, 0.05, &candidates, |c| { - rows.iter().find(|m| m.config == c).unwrap().error - }); - let context = serde_json::json!({"test":"bloom"}); - let profile = artifact(&rows, context.clone(), "test", Query::Membership); - let erp = erp_select(&profile, context, 0.05, Query::Membership); - assert_eq!(erp, oracle(&rows, 0.05)); - assert!(adapted - .selected - .is_none_or(|c| { c.family == Family::Bloom && c.bytes() <= 1024 })); - } - #[test] - fn count_sketch_hash_limit_and_single_key_oracle() { - let candidates = candidate_grid(&[Family::CountSketch], Query::Frequency, usize::MAX); - assert!(candidates - .iter() - .all(|c| c.depth * (c.width.ilog2() as usize + 1) <= 64)); - assert!(!candidates.contains(&Config { - family: Family::CountSketch, - width: 4096, - depth: 8 - })); - for config in candidates { - assert_eq!( - measure(config, Query::Frequency, &[0; 100], &["only".into()]).error, - 0.0 - ); - } - } - #[test] - fn lhs_has_unique_coordinates_and_is_reproducible() { - let sample = lhs(42); - assert_eq!(sample, lhs(42)); - assert_eq!( - sample - .iter() - .map(|c| c.width) - .collect::>() - .len(), - 7 - ); - assert_eq!( - sample - .iter() - .map(|c| c.depth) - .collect::>() - .len(), - 7 - ); - assert!(sample.iter().all(|c| grid().contains(c))); - } - #[test] - fn search_deduplicates_and_stays_in_grid() { - let mut seen = BTreeSet::new(); - let result = search(42, 0.01, |c| { - assert!(seen.insert(c)); - assert!(grid().contains(&c)); - 1.0 / c.width as f64 - }); - assert!(result.selected.is_some()); - assert!(result.visited.len() <= 56); - } - #[test] - fn all_feasible_reaches_smallest_state() { - assert_eq!( - search(42, 0.01, |_| 0.0).selected, - Some(Config { - family: Family::Cms, - width: 64, - depth: 1 - }) - ); - } - #[test] - fn no_feasible_is_not_best_effort_success() { - assert_eq!(search(42, 0.01, |_| 1.0).selected, None); - assert_eq!(search(42, 0.01, |_| f64::NAN).selected, None); - } - #[test] - fn deterministic_streams_and_held_out_separation() { - assert_eq!(stream(100, 20, 1.2, 42), stream(100, 20, 1.2, 42)); - assert_ne!(stream(100, 20, 1.2, 42), stream(100, 20, 1.2, 43)); - } - #[test] - fn actual_cms_single_key_has_zero_error() { - let m = measure( - Config { - family: Family::Cms, - width: 64, - depth: 1, - }, - Query::Frequency, - &[0; 100], - &["only".into()], - ); - assert_eq!(m.error, 0.0); - assert_eq!(m.payload_bytes, 512); - } - #[test] - fn real_erp_matches_oracle_and_fails_closed_on_context_miss() { - let rows: Vec<_> = grid() - .into_iter() - .map(|config| Measurement { - config, - error: 1.0 / config.width as f64, - payload_bytes: config.bytes(), - update_wall_seconds: 0.0, - query_wall_seconds: 0.0, - }) - .collect(); - let distribution = serde_json::json!({"seed":42}); - let profile = artifact( - &rows, - distribution.clone(), - "test-revision", - Query::Frequency, - ); - assert_eq!( - erp_select(&profile, distribution.clone(), 0.01, Query::Frequency), - oracle(&rows, 0.01) - ); - assert_eq!( - erp_select( - &profile, - serde_json::json!({"seed":43}), - 0.01, - Query::Frequency, - ), - None - ); - assert_eq!( - erp_select(&profile, distribution, 0.0, Query::Frequency), - None - ); - } -} diff --git a/data_plane/examples/promql/streaming_config.yaml b/data_plane/examples/promql/streaming_config.yaml deleted file mode 100644 index 03001c95b..000000000 --- a/data_plane/examples/promql/streaming_config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -aggregations: -- aggregationId: 1 - aggregationType: DatasketchesKLL - aggregationSubType: '' - labels: - grouping: [label_0] - rollup: [instance, job, label_1] - aggregated: [] - metric: fake_metric - parameters: - K: 20 - windowSize: 1 - windowType: tumbling - spatialFilter: '' -metrics: - fake_metric: - - instance - - job - - label_0 - - label_1 diff --git a/data_plane/examples/repeated_window_comparison.rs b/data_plane/examples/repeated_window_comparison.rs deleted file mode 100644 index 5dca532bb..000000000 --- a/data_plane/examples/repeated_window_comparison.rs +++ /dev/null @@ -1,402 +0,0 @@ -//! Repeated-window execution comparison using the real ASAP CMS implementation. -use asap_aware_mapping::replacement::default_size_params; -use asap_sketchlib::CountMinSketch; -use clap::Parser; -use planner_types::{ - post_asap::{SketchAlgorithm, SketchParams}, - pre_asap::AggIntent, - types::AccuracyTarget, -}; -use serde::Serialize; -use std::{collections::VecDeque, hint::black_box, time::Instant}; - -#[derive(Clone, Debug, Parser, Serialize)] -struct Args { - #[arg(long)] - output: std::path::PathBuf, - #[arg(long)] - backend_revision: String, - /// Optional `pane_indexdense_key_index` replay file. - #[arg(long)] - input_tsv: Option, - #[arg(long, default_value_t = 120)] - panes: usize, - #[arg(long, default_value_t = 2000)] - events_per_pane: usize, - #[arg(long, default_value_t = 1000)] - cardinality: usize, - /// Zero is uniform; positive values use a truncated Zipf distribution. - #[arg(long, default_value_t = 0.0)] - zipf: f64, - #[arg(long, default_value_t = 256)] - width: usize, - #[arg(long, default_value_t = 4)] - depth: usize, - /// Accuracy target passed to ASAPPlanner's analytical sizing baseline. - #[arg(long, default_value_t = 0.01)] - epsilon: f64, - #[arg(long, default_value_t = 0.01)] - delta: f64, - /// Common candidate-space payload limit per sketch. - #[arg(long, default_value_t = 32768)] - memory_budget_bytes: usize, - #[arg(long, default_value_t = 5)] - trials: usize, - /// Repeat all four end-of-stream queries to model recurring executions. - #[arg(long, default_value_t = 10)] - query_repetitions: usize, - #[arg(long, default_value_t = 42)] - seed: u64, - #[arg(long, value_delimiter = ',', default_value = "1,5,15,60")] - window_panes: Vec, - #[arg(long, default_value_t = 60)] - pane_seconds: usize, -} - -struct Rng(u64); -impl Rng { - fn next(&mut self) -> u64 { - self.0 = self.0.wrapping_add(0x9e3779b97f4a7c15); - let mut x = self.0; - x = (x ^ (x >> 30)).wrapping_mul(0xbf58476d1ce4e5b9); - x = (x ^ (x >> 27)).wrapping_mul(0x94d049bb133111eb); - x ^ (x >> 31) - } -} - -fn workload(args: &Args, trial: usize) -> Vec> { - let mut rng = Rng(args.seed.wrapping_add(trial as u64)); - let mut cumulative = Vec::with_capacity(args.cardinality); - let mut total = 0.0; - for rank in 1..=args.cardinality { - total += (rank as f64).powf(-args.zipf); - cumulative.push(total); - } - (0..args.panes) - .map(|_| { - (0..args.events_per_pane) - .map(|_| { - let sample = (rng.next() >> 11) as f64 / ((1_u64 << 53) as f64) * total; - cumulative - .partition_point(|value| *value <= sample) - .min(args.cardinality - 1) - }) - .collect() - }) - .collect() -} - -#[allow(clippy::type_complexity)] -fn replay(path: &std::path::Path) -> Result<(Vec>, usize), Box> { - let contents = std::fs::read_to_string(path)?; - let mut rows = Vec::new(); - let mut max_pane = 0; - let mut max_key = 0; - for (line_number, line) in contents.lines().enumerate() { - let (pane, key) = line - .split_once('\t') - .ok_or_else(|| format!("invalid TSV at line {}", line_number + 1))?; - let pane: usize = pane.parse()?; - let key: usize = key.parse()?; - max_pane = max_pane.max(pane); - max_key = max_key.max(key); - rows.push((pane, key)); - } - if rows.is_empty() { - return Err("empty replay input".into()); - } - let mut panes = vec![Vec::new(); max_pane + 1]; - for (pane, key) in rows { - panes[pane].push(key); - } - Ok((panes, max_key + 1)) -} - -#[derive(Debug, Serialize)] -struct ResultRow { - method: &'static str, - update_wall_seconds: f64, - query_wall_seconds: f64, - maintenance_updates: usize, - retained_sketches: usize, - logical_payload_bytes: usize, - max_normalized_additive_error: f64, -} - -fn truth(data: &[Vec], window: usize, cardinality: usize) -> Vec { - let mut counts = vec![0; cardinality]; - for pane in data.iter().rev().take(window) { - for &key in pane { - counts[key] += 1; - } - } - counts -} - -fn evaluate( - method: &'static str, - states: &[VecDeque], - data: &[Vec], - args: &Args, - update_wall_seconds: f64, - maintenance_updates: usize, -) -> ResultRow { - let started = Instant::now(); - let mut max_error = 0.0_f64; - for _ in 0..args.query_repetitions { - for (query, &window) in args.window_panes.iter().enumerate() { - let exact = truth(data, window, args.cardinality); - let denominator = exact.iter().sum::().max(1) as f64; - for (key, exact) in exact.into_iter().enumerate() { - let name = format!("key-{key}"); - let estimate: f64 = states[query].iter().map(|s| s.estimate(&name)).sum(); - max_error = max_error.max((estimate - exact as f64).abs() / denominator); - black_box(estimate); - } - } - } - let retained_sketches = states.iter().map(VecDeque::len).sum(); - ResultRow { - method, - update_wall_seconds, - query_wall_seconds: started.elapsed().as_secs_f64(), - maintenance_updates, - retained_sketches, - logical_payload_bytes: retained_sketches * args.width * args.depth * 8, - max_normalized_additive_error: max_error, - } -} - -fn per_query(method: &'static str, data: &[Vec], args: &Args) -> ResultRow { - let mut states: Vec> = - args.window_panes.iter().map(|_| VecDeque::new()).collect(); - let started = Instant::now(); - for pane in data { - for (query, &retention) in args.window_panes.iter().enumerate() { - let mut sketch = CountMinSketch::new(args.depth, args.width); - for &key in pane { - sketch.update(&format!("key-{key}"), 1.0); - } - states[query].push_back(sketch); - if states[query].len() > retention { - states[query].pop_front(); - } - } - } - evaluate( - method, - &states, - data, - args, - started.elapsed().as_secs_f64(), - data.iter().map(Vec::len).sum::() * args.window_panes.len(), - ) -} - -fn shared(method: &'static str, data: &[Vec], args: &Args) -> ResultRow { - let mut panes = VecDeque::new(); - let started = Instant::now(); - for pane in data { - let mut sketch = CountMinSketch::new(args.depth, args.width); - for &key in pane { - sketch.update(&format!("key-{key}"), 1.0); - } - panes.push_back(sketch); - if panes.len() > *args.window_panes.last().unwrap() { - panes.pop_front(); - } - } - let update = started.elapsed().as_secs_f64(); - let states: Vec> = args - .window_panes - .iter() - .map(|&window| panes.iter().rev().take(window).rev().cloned().collect()) - .collect(); - // Report physical shared storage, not the temporary query views above. - let mut row = evaluate( - method, - &states, - data, - args, - update, - data.iter().map(Vec::len).sum(), - ); - row.retained_sketches = panes.len(); - row.logical_payload_bytes = panes.len() * args.width * args.depth * 8; - row -} - -fn analytical_args(args: &Args) -> Result> { - let intent = AggIntent::Count { - accuracy: AccuracyTarget::EpsilonDelta { - epsilon: args.epsilon, - delta: args.delta, - }, - }; - let SketchParams::Cms { width, depth } = - default_size_params(SketchAlgorithm::Cms, &intent, args.epsilon, args.delta) - else { - unreachable!("CMS analytical sizing must return CMS parameters") - }; - // The comparison uses the same power-of-two width/depth grid as the - // AutoSketch adaptation. Round theory upward so its guarantee is not weakened. - let mut analytical = args.clone(); - analytical.width = usize::try_from(width)?.max(64).next_power_of_two(); - analytical.depth = usize::try_from(depth)?; - if analytical.width * analytical.depth * 8 > args.memory_budget_bytes { - return Err("ASAPPlanner analytical CMS exceeds the common memory budget".into()); - } - Ok(analytical) -} - -fn exact(data: &[Vec], args: &Args) -> ResultRow { - let started = Instant::now(); - let retained: Vec<_> = data - .iter() - .rev() - .take(*args.window_panes.last().unwrap()) - .rev() - .flatten() - .copied() - .collect(); - let update = started.elapsed().as_secs_f64(); - let started = Instant::now(); - for _ in 0..args.query_repetitions { - for &window in &args.window_panes { - black_box(truth(data, window, args.cardinality)); - } - } - ResultRow { - method: "exact_raw", - update_wall_seconds: update, - query_wall_seconds: started.elapsed().as_secs_f64(), - maintenance_updates: data.iter().map(Vec::len).sum(), - retained_sketches: 0, - logical_payload_bytes: retained.len() * std::mem::size_of::(), - max_normalized_additive_error: 0.0, - } -} - -fn main() -> Result<(), Box> { - let mut args = Args::parse(); - let replayed = match &args.input_tsv { - Some(path) => { - let (data, cardinality) = replay(path)?; - args.panes = data.len(); - args.cardinality = cardinality; - Some(data) - } - None => None, - }; - if args.window_panes.is_empty() - || args.window_panes.contains(&0) - || !args.window_panes.windows(2).all(|pair| pair[0] < pair[1]) - || args.panes < *args.window_panes.last().unwrap() - || (args.input_tsv.is_none() && (args.events_per_pane == 0 || args.cardinality == 0)) - || args.width == 0 - || args.depth == 0 - || !args.zipf.is_finite() - || args.zipf < 0.0 - || !args.epsilon.is_finite() - || args.epsilon <= 0.0 - || !args.delta.is_finite() - || args.delta <= 0.0 - || args.delta >= 1.0 - || args.width * args.depth * 8 > args.memory_budget_bytes - || args.trials == 0 - || args.query_repetitions == 0 - || args.backend_revision.trim().is_empty() - { - return Err("invalid arguments (panes must cover increasing non-zero windows)".into()); - } - let output = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&args.output)?; - let mut trials = Vec::new(); - for trial in 0..args.trials { - let data = match &replayed { - Some(data) => data.clone(), - None => workload(&args, trial), - }; - if data.len() < *args.window_panes.last().unwrap() { - return Err("replay does not cover the largest window".into()); - } - let analytical = analytical_args(&args)?; - let rows = vec![ - per_query("autosketch_per_query", &data, &args), - per_query("asap_no_sharing", &data, &args), - shared("asapplanner_erp", &data, &args), - shared("asapplanner_analytical", &data, &analytical), - exact(&data, &args), - ]; - eprintln!("trial {trial} complete"); - trials.push(serde_json::json!({"trial":trial,"methods":rows})); - } - serde_json::to_writer_pretty( - output, - &serde_json::json!({ - "schema_version":1,"windows_minutes":args.window_panes.iter().map(|window| window * args.pane_seconds / 60).collect::>(), - "pane_seconds":args.pane_seconds,"args":args, - "timing_metric":"wall seconds; methods execute sequentially and are not CPU profiles", - "memory_metric":"logical retained payload; CMS counters or raw usize keys; excludes allocator overhead", - "adaptation":"AutoSketch is deployed independently per recurring query; width/depth are selected by the companion Rust AutoSketch/ERP calibration runner; ASAP-NoSharing isolates sharing; ASAPPlanner-ERP shares panes; ASAPPlanner-Analytical calls Planner default_size_params and rounds upward to the common candidate grid", - "trials":trials - }), - )?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - fn args() -> Args { - Args { - output: "unused".into(), - backend_revision: "test".into(), - input_tsv: None, - panes: 60, - events_per_pane: 10, - cardinality: 20, - zipf: 0.0, - width: 64, - depth: 2, - epsilon: 0.01, - delta: 0.01, - memory_budget_bytes: 32768, - trials: 1, - query_repetitions: 2, - seed: 42, - window_panes: vec![1, 5, 15, 60], - pane_seconds: 60, - } - } - #[test] - fn sharing_reduces_updates_and_storage_without_changing_answers() { - let args = args(); - let data = workload(&args, 0); - let per = per_query("autosketch_per_query", &data, &args); - let full = shared("asapplanner_erp", &data, &args); - assert_eq!(per.maintenance_updates, full.maintenance_updates * 4); - assert_eq!(per.retained_sketches, 81); - assert_eq!(full.retained_sketches, 60); - assert_eq!( - per.max_normalized_additive_error, - full.max_normalized_additive_error - ); - } - - #[test] - fn replay_preserves_empty_panes_and_derives_cardinality() { - let path = std::env::temp_dir().join(format!( - "asap-replay-{}-{}.tsv", - std::process::id(), - std::thread::current().name().unwrap_or("test") - )); - std::fs::write(&path, "0\t2\n2\t0\n2\t1\n").unwrap(); - let (panes, cardinality) = replay(&path).unwrap(); - std::fs::remove_file(path).unwrap(); - assert_eq!(panes, vec![vec![2], vec![], vec![0, 1]]); - assert_eq!(cardinality, 3); - } -} diff --git a/data_plane/examples/sketch_db_diag.rs b/data_plane/examples/sketch_db_diag.rs deleted file mode 100644 index 739842f49..000000000 --- a/data_plane/examples/sketch_db_diag.rs +++ /dev/null @@ -1,159 +0,0 @@ -//! Diagnostic for the 10k-sids slowdown observed in `benches/sketch_db.rs`. -//! -//! The original bench used `Bencher::iter_batched(setup, routine, SmallInput)`. -//! Criterion's `iter_batched` consumes the input by value and the input's -//! `Drop` runs *inside* the timed region (see criterion-0.5.1 -//! `src/bencher.rs:264-272`). Our `setup` builds a `SketchStore` with N -//! `SummarySeriesMetadata` entries registered; dropping that store at N=10_000 -//! pays for 10k Drops of `metric_name: String`, `BTreeSet`, `SketchConfig`, -//! `AccuracyBound`, etc. That drop, not `append_sample`, is what makes the -//! 10k case look ~90× slower than the 100-sid case. -//! -//! This diagnostic separates three quantities: -//! (A) rebuild-per-iter (matches the original bench): build + 1 append + drop -//! (B) isolated append: pre-built store, K appends -//! (C) isolated drop: build, time drop only -//! -//! Run: `cargo run --release --example sketch_db_diag` - -use std::collections::{BTreeMap, BTreeSet}; -use std::time::Instant; - -use asap_sketchlib::proto::sketchlib::{sketch_envelope, DdSketchState, SketchEnvelope}; -use asap_sketchlib::DdSketch; -use prost::Message; - -use data_plane::storage_engines::sketch_db::data::{ - AccuracyBound, AggKind, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, - SketchSampleState, -}; -use data_plane::storage_engines::sketch_db::index::{SketchStore, SummarySeriesMetadata}; - -fn ddsketch_payload() -> Vec { - let mut sk = DdSketch::new(0.01); - for v in 1..=64 { - sk.update(v as f64); - } - let state = DdSketchState { - alpha: sk.alpha, - store_counts: sk.store_counts.clone(), - store_offset: sk.store_offset, - ..Default::default() - }; - SketchEnvelope { - sketch_state: Some(sketch_envelope::SketchState::Ddsketch(state)), - ..Default::default() - } - .encode_to_vec() -} - -fn dd_meta(sid: u64) -> SummarySeriesMetadata { - let cfg = SketchConfig::DDSketch { - relative_accuracy: 0.01, - }; - SummarySeriesMetadata { - storage_handle: sid, - metric_name: "bench_metric".into(), - group_by_keys: BTreeSet::new(), - capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), - accuracy: Some(AccuracyBound::from_config(&cfg)), - agg_kind: AggKind::Sketch { - algorithm: SketchAlgorithm::DDSketch, - config: cfg, - spatial_filter_canonical: String::new(), - }, - first_seen_unix_ms: 0, - retired_at_ms: None, - expires_at_ms: None, - policy_fp: asap_types::PolicyFingerprint::UNSET, - } -} - -fn build_store(num_sids: u64) -> SketchStore { - let store = SketchStore::new(); - for sid in 1..=num_sids { - store.register(dd_meta(sid)); - } - store -} - -fn sample(bytes: Vec) -> SketchSampleState { - SketchSampleState { - bytes, - encoding: SketchEncoding::ProtoFull, - } -} - -fn bench_a_rebuild_per_iter(num_sids: u64, iters: u64, payload: &[u8]) -> f64 { - // (A) reproduces what `iter_batched` actually measured: build store, one - // append, then drop the store — all timed. - let t0 = Instant::now(); - for i in 0..iters { - let store = build_store(num_sids); - let win = (1_000 + i * 10, 1_000 + i * 10 + 10); - store.append_sample(1, BTreeMap::new(), win, sample(payload.to_vec())); - // drop happens at end of loop body — inside the timed region. - } - t0.elapsed().as_nanos() as f64 / iters as f64 -} - -fn bench_b_isolated_append(num_sids: u64, iters: u64, payload: &[u8]) -> f64 { - // (B) pre-build the store ONCE; loop pure `append_sample` calls. - // The store accumulates depth over the run, but DashMap entry + intern - // + columnar push is O(1) amortized. - let store = build_store(num_sids); - let t0 = Instant::now(); - for i in 0..iters { - let sid = (i % num_sids) + 1; - let win = (1_000 + i * 10, 1_000 + i * 10 + 10); - store.append_sample(sid, BTreeMap::new(), win, sample(payload.to_vec())); - } - let avg = t0.elapsed().as_nanos() as f64 / iters as f64; - drop(store); - avg -} - -fn bench_c_isolated_drop(num_sids: u64, iters: u64) -> f64 { - // (C) drop-only cost. Build outside, drop inside, time the drop. - let mut stores: Vec = (0..iters).map(|_| build_store(num_sids)).collect(); - let t0 = Instant::now(); - while let Some(s) = stores.pop() { - drop(s); - } - t0.elapsed().as_nanos() as f64 / iters as f64 -} - -fn main() { - let payload = ddsketch_payload(); - println!("payload bytes: {}", payload.len()); - println!(); - println!( - "{:>10} | {:>14} | {:>14} | {:>14} | {:>14}", - "num_sids", "(A) rebuild+1", "(B) append", "(C) drop", "A - B - C" - ); - println!("{}", "-".repeat(80)); - - for (num_sids, iters_a, iters_b, iters_c) in [ - (1u64, 50_000u64, 200_000u64, 50_000u64), - (100, 5_000, 200_000, 5_000), - (10_000, 500, 200_000, 500), - ] { - let a = bench_a_rebuild_per_iter(num_sids, iters_a, &payload); - let b = bench_b_isolated_append(num_sids, iters_b, &payload); - let c = bench_c_isolated_drop(num_sids, iters_c); - let residual = a - b - c; - println!( - "{:>10} | {:>12.0} ns | {:>12.0} ns | {:>12.0} ns | {:>12.0} ns", - num_sids, a, b, c, residual - ); - } - - println!(); - println!("Legend:"); - println!( - " (A) per-iter store rebuild + 1 append + drop (≈ what the criterion bench reported)" - ); - println!(" (B) pure append_sample on a pre-built store (the actual per-op cost)"); - println!(" (C) drop of a freshly-built store (the accounting term we missed)"); - println!(" A - B - C should be ≈ build_store cost (register() x num_sids)."); -} diff --git a/data_plane/examples/univmon_erp_artifact.rs b/data_plane/examples/univmon_erp_artifact.rs deleted file mode 100644 index c20687a1b..000000000 --- a/data_plane/examples/univmon_erp_artifact.rs +++ /dev/null @@ -1,204 +0,0 @@ -//! Measure readout-specific ERP evidence from finite JSONL evaluation data. -//! This offline tool retains samples; the production backend does not. -use asap_physical_operators::summary_kernels::hll_sketch::HllSketchAccumulator; -use asap_physical_operators::summary_kernels::univmon::UnivMonAccumulator; -use data_plane::storage_engines::types::{AggregateCore, SerializableToSink}; -use serde_json::{json, Value}; -use std::collections::{BTreeMap, HashMap}; -use std::io::{BufRead, BufReader}; - -fn main() -> Result<(), Box> { - let path = std::env::args() - .nth(1) - .ok_or("usage: univmon_erp_artifact samples.jsonl")?; - let mut series = BTreeMap::>::new(); - for line in BufReader::new(std::fs::File::open(&path)?).lines() { - let row: Value = serde_json::from_str(&line?)?; - let key = serde_json::to_string(&row["labels"])?; - series.entry(key).or_default().push(( - row["ts_ms"].as_u64().ok_or("timestamp")?, - row["value"].as_f64().ok_or("value")?, - )); - } - if series - .values() - .any(|samples| samples.windows(2).any(|pair| pair[0].0 >= pair[1].0)) - { - return Err("each series must have strictly increasing timestamps".into()); - } - let first = series.values().next().ok_or("empty dataset")?; - let start = first.first().ok_or("empty series")?.0; - let end = first.last().ok_or("empty series")?.0; - let populations: Vec<_> = series - .iter() - .take(10) - .map(|(key, samples)| { - ( - key, - samples - .iter() - .filter(|(ts, _)| *ts > start && *ts <= end) - .map(|(_, v)| *v + 1_000_000_000_000.0) - .collect::>(), - ) - }) - .collect(); - if populations.len() < 2 { - return Err("need at least two independent populations".into()); - } - // Calibration hashes a disjoint numeric key namespace. The shift preserves - // frequencies/cardinality for this integer-valued finite fixture; reject - // inputs for which floating-point rounding could silently merge values. - if series.values().flatten().any(|(_, value)| { - !value.is_finite() || value.fract() != 0.0 || !(0.0..1_000_000_000_000.0).contains(value) - }) { - return Err( - "calibration namespace requires finite nonnegative integer values below 1e12".into(), - ); - } - let mut records = Vec::new(); - let mut observed_shape = None; - for (heap, cols, layers) in [(64, 512, 8), (512, 2048, 8), (4096, 8192, 16)] { - let mut max_errors = [0.0f64; 3]; - let mut bytes = 0; - let mut observed = None; - for (_, raw) in &populations { - let mut counts = HashMap::::new(); - for v in raw { - *counts - .entry(if *v == 0.0 { 0 } else { v.to_bits() }) - .or_default() += 1; - } - let truths = [ - counts.len() as f64, - counts - .values() - .map(|c| (*c as f64).powi(2)) - .sum::() - .sqrt(), - counts - .values() - .map(|c| { - let p = *c as f64 / raw.len() as f64; - -p * p.log2() - }) - .sum(), - ]; - let mut observer = control_plane::physical::erp::ErpShapeObserver::new(raw.len())?; - for (i, v) in raw.iter().enumerate() { - observer.observe(&v.to_string(), i / 1000)?; - } - let next = observer.snapshot().ok_or("invalid observation")?; - if observed.as_ref().is_some_and( - |prior: &control_plane::physical::erp::ErpObservedShape| { - prior.observation != next.observation - }, - ) { - return Err( - "training populations have different shapes; calibrate separate shape strata" - .into(), - ); - } - observed = Some(next); - let mut left = - UnivMonAccumulator::new(heap, 5, cols, layers).map_err(|e| e.to_string())?; - let mut right = left.clone(); - for (i, v) in raw.iter().enumerate() { - if i % 2 == 0 { &mut left } else { &mut right } - .insert_sample(*v) - .map_err(|e| e.to_string())?; - } - left.merge_in_place(&right).map_err(|e| e.to_string())?; - bytes = bytes.max(left.serialize_to_bytes().len()); - for (i, stat) in [ - asap_types::Statistic::Cardinality, - asap_types::Statistic::FrequencyL2, - asap_types::Statistic::FrequencyEntropy, - ] - .into_iter() - .enumerate() - { - let answer = left - .query_statistic(stat, &None, &Default::default()) - .map_err(|e| e.to_string())?; - if !answer.is_finite() { - return Err("nonfinite estimate".into()); - } - max_errors[i] = max_errors[i] - .max((answer - truths[i]).abs() / if i == 2 { 1.0 } else { truths[i] }); - } - } - let observed = observed.unwrap(); - observed_shape = Some(observed.observation.clone()); - let fit = observed - .observation - .fits - .iter() - .min_by(|a, b| a.goodness_of_fit.total_cmp(&b.goodness_of_fit)) - .ok_or("no shape")?; - records.push(json!({"id":format!("univmon-h{heap}-c{cols}-l{layers}"),"sketch":"univmon","implementation":"asap-sketchlib-univmon-standard-v1", - "parameters":{"heap_size":heap,"sketch_rows":5,"sketch_cols":cols,"layers":layers},"trials":populations.len(), - "distribution":{"erp_shape":{"family":fit.family,"parameters":fit.parameters,"cardinality":observed.observation.cardinality,"benchmark_events":observed.observation.observed_events}}, - "error_metrics":{"max_cardinality_relative_error":max_errors[0],"max_frequency_l2_relative_error":max_errors[1],"max_frequency_entropy_absolute_bits_error":max_errors[2]}, - "resources":{"memory_bytes":bytes,"update_cpu_seconds":0.0,"merge_cpu_seconds":0.0,"query_cpu_seconds":0.0}})); - eprintln!("measured heap={heap} cols={cols} layers={layers}: {max_errors:?}"); - } - // Predeclared precision grid; validation uses disjoint source populations. - let observation = observed_shape.as_ref().ok_or("no observation")?; - let fit = observation - .fits - .iter() - .min_by(|a, b| a.goodness_of_fit.total_cmp(&b.goodness_of_fit)) - .ok_or("no fit")?; - for precision in [10, 12, 14] { - let mut max_error = 0.0f64; - let mut bytes = 0; - for (_, raw) in &populations { - let mut left = - HllSketchAccumulator::new(asap_sketchlib::HllVariant::Regular, precision); - let mut right = left.clone(); - let mut distinct = std::collections::HashSet::new(); - for (i, value) in raw.iter().enumerate() { - let bits = if *value == 0.0 { 0 } else { value.to_bits() }; - distinct.insert(bits); - if i % 2 == 0 { &mut left } else { &mut right } - .inner - .update(&bits.to_le_bytes()); - } - let merged = left.merge_with(&right).map_err(|e| e.to_string())?; - let merged = merged - .as_any() - .downcast_ref::() - .ok_or("HLL merge type")?; - let estimate = merged - .query_statistic( - asap_types::Statistic::Cardinality, - &None, - &Default::default(), - ) - .map_err(|e| e.to_string())?; - max_error = - max_error.max((estimate - distinct.len() as f64).abs() / distinct.len() as f64); - bytes = bytes.max(merged.serialize_to_bytes().len()); - } - records.push(json!({"id":format!("hll-p{precision}"),"sketch":"hll","implementation":"asap-sketchlib-hll-regular-v1", - "parameters":{"precision":precision},"trials":populations.len(), - "distribution":{"erp_shape":{"family":fit.family,"parameters":fit.parameters,"cardinality":observation.cardinality,"benchmark_events":observation.observed_events}}, - "error_metrics":{"max_cardinality_relative_error":max_error}, - "resources":{"memory_bytes":bytes,"update_cpu_seconds":0.0,"merge_cpu_seconds":0.0,"query_cpu_seconds":0.0}})); - eprintln!("measured HLL precision={precision}: max relative cardinality error={max_error}"); - } - if let Some(path) = std::env::args().nth(2) { - std::fs::write( - path, - serde_json::to_vec_pretty(&observed_shape.ok_or("no observation")?)?, - )?; - } - println!( - "{}", - serde_json::to_string_pretty( - &json!({"schema_version":1,"producer_version":"backend-standard-unit-frequency-merged-two-pane-data-calibration-cpu-excluded","records":records}) - )? - ); - Ok(()) -} diff --git a/data_plane/query-engine-rust-cli-compose.yml.j2 b/data_plane/query-engine-rust-cli-compose.yml.j2 deleted file mode 100644 index 32fe8f962..000000000 --- a/data_plane/query-engine-rust-cli-compose.yml.j2 +++ /dev/null @@ -1,50 +0,0 @@ -# QueryEngine Rust Docker Compose Template -# This template is rendered with Jinja2 to generate the final docker-compose.yml - -services: - queryengine-rust: - image: sketchdb-queryengine-rust:latest # Need to change image name to 'asap' prefix - container_name: asap-queryengine-rust - hostname: queryengine-rust - networks: - - asap-network - ports: - - "{{ http_port | default('8088') }}:8088" - environment: - - RUST_LOG={{ log_level | default('info') }} - - RUST_BACKTRACE=1 - volumes: - - "{{ experiment_output_dir }}:/app/outputs" - - "{{ control_plane_remote_output_dir }}:/app/control_plane_output:ro" - command: [ - "--kafka-topic", "{{ kafka_topic }}", - "--kafka-broker", "kafka:9092", # Uses kafka container service name instead of IP - "--input-format", "{{ input_format }}", - "--config", "/app/control_plane_output/inference_config.yaml", - "--streaming-config", "/app/control_plane_output/streaming_config.yaml", - "--prometheus-server", "http://prometheus:9090", - "--prometheus-scrape-interval", "{{ prometheus_scrape_interval }}", - "--delete-existing-db", - "--log-level", "{{ log_level }}", - "--output-dir", "/app/outputs", - "--streaming-engine", "{{ streaming_engine }}", - "--query-language", "{{ query_language }}", - "--lock-strategy", "{{ lock_strategy }}"{% if compress_json %}, - "--decompress-json"{% endif %}{% if profile_query_engine %}, - "--do-profiling"{% endif %}{% if forward_unsupported_queries %}, - "--forward-unsupported-queries"{% endif %}{% if dump_precomputes %}, - "--dump-precomputes"{% endif %} - ] - depends_on: - kafka: - condition: service_healthy - kafka-init: - condition: service_completed_successfully # Wait for Kafka topics to be created - prometheus: - condition: service_healthy - control_plane: - condition: service_completed_successfully # Wait for control plane to generate configs - arroyosketch: - condition: service_completed_successfully # Wait for pipeline configuration - restart: no - # Add healthcheck diff --git a/data_plane/rustfmt.toml b/data_plane/rustfmt.toml deleted file mode 100644 index 88c5c0f00..000000000 --- a/data_plane/rustfmt.toml +++ /dev/null @@ -1,4 +0,0 @@ -# Optional: customize rustfmt behavior -max_width = 100 -hard_tabs = false -tab_spaces = 4 diff --git a/data_plane/src/bin/monitor_coordinator_harness.rs b/data_plane/src/bin/monitor_coordinator_harness.rs deleted file mode 100644 index 3f993ed2a..000000000 --- a/data_plane/src/bin/monitor_coordinator_harness.rs +++ /dev/null @@ -1,84 +0,0 @@ -//! Minimal CDM monitor-coordinator harness for the cross-language e2e -//! (deploy/mvp-multinode/scripts/monitor_e2e.sh in ASAPCollector). Runs the -//! tonic MonitorService with a single monitor and prints a line to stdout once -//! the coordinator has computed a REAL coordinated-sampling grant for the -//! connecting edge (i.e. the edge's periodic rate report reached the -//! coordinator and a non-trivial sample_p came back), then exits 0 — so a -//! driver script can assert the Go edge ⇄ Rust coordinator wire path -//! end-to-end without standing up the full data-plane. -//! -//! Global-threshold alerting is retired (see `data_plane::monitor` module -//! docs) — this harness no longer waits for or reports an alert. -//! -//! Usage: monitor_coordinator_harness [timeout_secs] [key] -//! -//! `tau` is accepted for CLI/back-compat but unused. `key` is the monitor's -//! group key (the edge's canonical `k=v;k2=v2` label encoding). Scalar (Sum) -//! monitors register under their series-group key, so the harness config must -//! carry the same key or registrations are rejected as unconfigured. Empty -//! (default) = ungrouped. - -use data_plane::update_sampling::{MonitorConfig, MonitorCoordinator, MonitorServiceImpl}; - -/// The edge_id the Go e2edriver connects as (asap-precompute-go/monitor/ -/// grpcclient/cmd/e2edriver/main.go: `monitor.NewEngine("e2e-edge", ...)`). -const E2E_EDGE_ID: &str = "e2e-edge"; - -#[tokio::main] -async fn main() -> Result<(), Box> { - let mut args = std::env::args().skip(1); - let port: u16 = args.next().and_then(|s| s.parse().ok()).unwrap_or(4319); - let agg_id: u64 = args.next().and_then(|s| s.parse().ok()).unwrap_or(1); - let tau: f64 = args.next().and_then(|s| s.parse().ok()).unwrap_or(100.0); - let window_ms: u64 = args - .next() - .and_then(|s| s.parse().ok()) - .unwrap_or(3_600_000); - let timeout_secs: u64 = args.next().and_then(|s| s.parse().ok()).unwrap_or(30); - let key: Vec = args.next().map(|s| s.into_bytes()).unwrap_or_default(); - - let cfg = MonitorConfig { - agg_id, - key: key.clone(), - tau, - epsilon: 0.05, - window_ms, - ..Default::default() - }; - let coord = MonitorCoordinator::new(vec![cfg]); - let svc = MonitorServiceImpl::new(coord.clone()).into_server(); - let addr = format!("0.0.0.0:{port}").parse()?; - eprintln!("harness: serving MonitorService on {addr} (agg_id={agg_id}, window_ms={window_ms})"); - println!("HARNESS_READY {port}"); - - // Poll for a real coordinated-sampling grant reaching this edge — the - // proof that the edge's periodic rate report made the live gRPC round - // trip and the coordinator answered with a computed sample_p. - let poll = { - let coord = coord.clone(); - let key = key.clone(); - async move { - loop { - if let Some(p) = coord.granted_sample_p(agg_id, &key, E2E_EDGE_ID).await { - println!("MONITOR_GRANT edge={E2E_EDGE_ID} agg_id={agg_id} sample_p={p}"); - return; - } - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } - } - }; - - tokio::select! { - r = tonic::transport::Server::builder().add_service(svc).serve(addr) => { - r?; - } - _ = poll => { - eprintln!("harness: coordinated-sampling grant observed — exiting"); - } - _ = tokio::time::sleep(std::time::Duration::from_secs(timeout_secs)) => { - eprintln!("harness: timeout after {timeout_secs}s with no grant"); - std::process::exit(2); - } - } - Ok(()) -} diff --git a/data_plane/src/drivers/ingest/mod.rs b/data_plane/src/drivers/ingest/mod.rs deleted file mode 100644 index 89d1915d0..000000000 --- a/data_plane/src/drivers/ingest/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -pub mod otel; -pub mod prometheus_remote_write; -pub mod series_resolver; - -pub use otel::{OtlpReceiver, OtlpReceiverConfig}; -pub use prometheus_remote_write::{ - PrometheusRemoteWriteConfig, PrometheusRemoteWriteReceiver, RemoteWriteStats, -}; -pub use series_resolver::{ - canonical_attrs_fingerprint, population_attrs_fingerprint, SeriesIdResolver, -}; diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs deleted file mode 100644 index 81925b1a7..000000000 --- a/data_plane/src/drivers/ingest/otel.rs +++ /dev/null @@ -1,4766 +0,0 @@ -//! OTLP ingest driver. -//! -//! Accepts OTLP metrics via gRPC (4317) and HTTP (4318, POST /v1/metrics). -//! Parses `ExportMetricsServiceRequest`, and — when wired to a precompute -//! engine via [`OtlpReceiver::with_ingest_state`] — routes both raw metric -//! points and pre-built sketches through the precompute engine's worker -//! pool. The precompute engine then performs window-aligned aggregation -//! per `InstalledPrecomputePlan` and writes results to `SketchStore`. -//! -//! Architectural flow: -//! ```text -//! DataCollector OTel collector -//! → OTLP gRPC/HTTP (this receiver) -//! → precompute engine ingest router -//! → workers (per (agg_id, group_key) panes) -//! → StoreOutputSink → SketchStore -//! → query engine -//! ``` -//! -//! Labels from the OTLP wire format are preserved all the way into -//! `KeyByLabelValues` via the standard `series_key` → grouping-label -//! extraction used by the Prometheus/VictoriaMetrics ingest paths. - -use std::collections::HashMap; -use std::io::Read; - -use crate::precompute_engine::series_router::WorkerMessage; -use crate::precompute_engine::IngestState; -use crate::query_engines::routing::FreshnessProbeCache; -use crate::storage_engines::types::AggregateCore; -use asap_otel_proto::tonic::collector::metrics::v1::{ - metrics_service_server::MetricsService, ExportMetricsServiceRequest, - ExportMetricsServiceResponse, -}; -use asap_otel_proto::tonic::common::v1::any_value::Value as AnyValueVariant; -use asap_otel_proto::tonic::metrics::v1::number_data_point::Value as NumberValue; -use asap_physical_operators::summary_kernels::sketch_envelope::SketchEnvelopeAccumulator; -use asap_sketchlib::proto::sketchlib::{sketch_envelope, SketchEnvelope}; -use asap_sketchlib::MessagePackCodec; -use axum::{body::Bytes, extract::State, routing::post, Json, Router}; -use flate2::read::GzDecoder; -use planner_types::post_asap::SketchAlgorithm; -use prost::Message; -use std::sync::Arc; -use std::time::Instant; -use tonic::{Request, Response, Status}; -use tracing::{debug, error, info, warn}; - -/// Configuration for the OTLP receiver. -#[derive(Debug, Clone)] -pub struct OtlpReceiverConfig { - pub grpc_port: u16, - pub http_port: u16, -} - -/// Shared state accessible by both gRPC and HTTP handlers. -#[derive(Clone)] -struct OtlpSharedState { - /// Handle into the precompute engine's worker pool. When `Some`, - /// OTLP metrics and sketches are routed through the engine; when - /// `None` the receiver accepts data but only logs it (no storage). - ingest_state: Option>, - /// Freshness-probe last-value cache (issue #46 ⑥). When `Some`, - /// every received `http_freshness_probe_*` data point updates the - /// cache so the HTTP query handler can answer - /// `last_over_time([])` from RAM with sub-second - /// freshness — bypassing the 60–90 s cold-tier flush gap that - /// would otherwise leave a 10 s lookback window empty. - probe_cache: Option>, -} - -/// OTLP receiver that accepts metrics via gRPC and HTTP. -pub struct OtlpReceiver { - config: OtlpReceiverConfig, - ingest_state: Option>, - probe_cache: Option>, -} - -impl OtlpReceiver { - /// Construct a receiver without a backend. Metrics are parsed and - /// logged but not stored — useful for diagnosing the OTLP pipe. - pub fn new(config: OtlpReceiverConfig) -> Self { - Self { - config, - ingest_state: None, - probe_cache: None, - } - } - - /// Construct a receiver wired to a precompute engine's ingest state. - /// Incoming metrics and sketches are routed through the engine's - /// worker pool, where they are merged into per-`(agg_id, group_key)` - /// panes and eventually emitted to the store. - pub fn with_ingest_state(config: OtlpReceiverConfig, ingest_state: Arc) -> Self { - Self { - config, - ingest_state: Some(ingest_state), - probe_cache: None, - } - } - - /// Attach a [`FreshnessProbeCache`] so the receiver captures the - /// latest sample for every `http_freshness_probe_*` metric it - /// sees. Builder-style; chains with [`Self::with_ingest_state`] - /// at the binary's wiring site. Without this call, probe-shaped - /// metrics still reach the precompute engine and the cold-tier - /// TSDB write path — only the in-memory query short-circuit is - /// disabled. - pub fn with_probe_cache(mut self, cache: Arc) -> Self { - debug!( - "OTLP receiver attached freshness-probe cache for issue #46 \ - criterion ⑥ short-circuit" - ); - self.probe_cache = Some(cache); - self - } - - pub async fn run(&self) -> Result<(), Box> { - let grpc_addr = std::net::SocketAddr::from(([0, 0, 0, 0], self.config.grpc_port)); - let http_addr = std::net::SocketAddr::from(([0, 0, 0, 0], self.config.http_port)); - - let shared = Arc::new(OtlpSharedState { - ingest_state: self.ingest_state.clone(), - probe_cache: self.probe_cache.clone(), - }); - - let grpc_svc = MetricsServiceImpl { - shared: shared.clone(), - }; - // Bump tonic's default 4 MiB receive cap. A single agent - // window emits ~1000 series, each carrying a typed - // DDSketch / KLLSketch / ... state — the full-state - // payloads run 17+ MiB at the cardinalities the e2e - // harness uses. With the default cap, the gateway's - // OTLP exporter retries forever with - // `decoded message length too large`. Match the - // gateway/agent receiver caps (`max_recv_msg_size_mib: 64` - // in their YAMLs) so all three tiers agree. - let grpc_svc = - asap_otel_proto::tonic::collector::metrics::v1::metrics_service_server::MetricsServiceServer::new( - grpc_svc, - ) - .max_decoding_message_size(64 * 1024 * 1024) - // Accept gzip-compressed request bodies so the `asap-gzip` - // arm's agent (`otlp/backend: compression: gzip`) can - // shrink the per-window sketch/aggregate payload on the - // wire. This is the matched-codec partner of the OTLP→VM - // `b1` baseline (also gzip): comparing b1 vs asap-gzip - // isolates the aggregation gain at a fixed codec. - // tonic negotiates per-RPC via the grpc-encoding header, so - // uncompressed agents (the plain `asap` arm, - // `compression: none`, matched against the `b0` baseline) - // are unaffected — this only enables the server to decode - // gzip when an agent chooses to send it. The control_plane - // RuntimeSamples server already does the same - // (control_plane/src/runtime_samples.rs). - .accept_compressed(tonic::codec::CompressionEncoding::Gzip); - - let app = Router::new() - .route("/v1/metrics", post(handle_otlp_http)) - .with_state(shared); - - let grpc_listener = tokio::net::TcpListener::bind(grpc_addr).await?; - let http_listener = tokio::net::TcpListener::bind(http_addr).await?; - - info!("OTLP gRPC listening on {}", grpc_addr); - info!("OTLP HTTP listening on {} (POST /v1/metrics)", http_addr); - - tokio::select! { - r = tonic::transport::Server::builder() - .add_service(grpc_svc) - .serve_with_incoming(tokio_stream::wrappers::TcpListenerStream::new(grpc_listener)) => { - if let Err(e) = r { - error!("OTLP gRPC server error: {}", e); - } - } - r = axum::serve(http_listener, app) => { - if let Err(e) = r { - error!("OTLP HTTP server error: {}", e); - } - } - } - - Ok(()) - } -} - -struct MetricsServiceImpl { - shared: Arc, -} - -#[tonic::async_trait] -impl MetricsService for MetricsServiceImpl { - async fn export( - &self, - request: Request, - ) -> Result, Status> { - debug!("OTLP received request via gRPC"); - let req = request.into_inner(); - // PERF-1 — parse the OTLP request ONCE here and hand the parsed - // points/sketches to the raw-path consumers - // (process_otlp_request / capture_freshness / route_otlp_to_precompute) - // instead of each re-walking the protobuf. The first-class - // sketch path (`route_modified_otlp_sketches_to_precompute`) - // still walks the raw request because it reads the sketch - // DataPoint variants this parse intentionally skips. - let (points, sketch_payloads) = otlp_to_metric_points_and_sketches(&req); - process_otlp_request(&req, &points, &sketch_payloads, "gRPC"); - if let Some(cache) = &self.shared.probe_cache { - capture_freshness_probe_samples(&points, cache); - } - let mut outcome = IngestOutcome::default(); - if let Some(state) = &self.shared.ingest_state { - route_otlp_to_precompute(&points, &sketch_payloads, state).await; - outcome = route_modified_otlp_sketches_to_precompute(&req, state) - .await - .map_err(Status::invalid_argument)?; - } - debug!("OTLP sending response via gRPC"); - Ok(Response::new(ExportMetricsServiceResponse { - // A successful RPC is the transport ACK: every summary frame - // passed the active-plan gate and was applied. Contract failures - // return a non-OK gRPC status before any frame is written. - partial_success: None, - // Sid bindings the sender should cache. Each entry maps an - // `attributes_fingerprint` to the canonical sid the backend's - // `SeriesIdResolver` minted (or returned from its cache). The - // sender's local dictionary keys on `fingerprint`, so it can - // refresh stale entries and pick up brand-new ones from this - // field without a separate `ResolveSeriesIDs` round-trip. - series_assignments: outcome.series_assignments, - // backend signals senders to evict cached sids here - // when this Export carried a sid the resolver does not - // recognize (sid-cache divergence — e.g. after a backend - // restart without persistence, or when the sender's sid - // disagrees with the resolved sid for the same attrs). - unknown_series_ids: outcome.unknown_series_ids, - })) - } - - async fn resolve_series_i_ds( - &self, - request: Request, - ) -> Result< - Response, - Status, - > { - use asap_otel_proto::tonic::collector::metrics::v1::ResolveSeriesIDsResponse; - - // Pre-resolve handshake is structurally redundant with the - // canonical Export path under Interpretation B (sid identity is - // `(metric, attrs_fingerprint, agg_kind_canonical)` and - // `agg_kind` isn't carried in `SeriesQuery`). On the first - // Export with attrs, the backend resolves the correct sid and - // echoes it back via `ExportMetricsServiceResponse.series_assignments` - // — that's the canonical channel, and it also handles every - // cache-divergence failure mode (proto comment at - // `ExportMetricsServiceResponse.unknown_series_ids:100-104`). - // - // The RPC is kept as a stable surface so older patched - // exporters that still call it don't see `Unimplemented`. The - // returned `assignments` vec is empty; the agent's local cache - // stays cold and the first real Export populates it. - // - // Follow-up: drop the RPC entirely (proto change), OR extend - // `SeriesQuery` to carry `agg_kind_canonical` so this can - // perform a real pre-resolve. Today's stub is the no-harm path. - let req = request.into_inner(); - if !req.queries.is_empty() { - warn!( - queries = req.queries.len(), - "ResolveSeriesIDs RPC called with non-empty batch; \ - returning empty assignments — sid identity now \ - includes agg_kind which this RPC doesn't carry. \ - Agent will mint on first Export with attrs.", - ); - } - Ok(Response::new(ResolveSeriesIDsResponse { - assignments: Vec::new(), - })) - } -} - -async fn handle_otlp_http( - headers: axum::http::HeaderMap, - State(shared): State>, - body: Bytes, -) -> Result, (axum::http::StatusCode, String)> { - debug!("OTLP received request via HTTP, body_bytes={}", body.len()); - let body = if let Some(enc) = headers.get(axum::http::header::CONTENT_ENCODING) { - let enc = enc.to_str().unwrap_or("").trim().to_ascii_lowercase(); - if enc == "gzip" { - let mut decoder = GzDecoder::new(body.as_ref()); - let mut out = Vec::new(); - decoder.read_to_end(&mut out).map_err(|e| { - ( - axum::http::StatusCode::BAD_REQUEST, - format!("Gzip decode error: {}", e), - ) - })?; - Bytes::from(out) - } else { - body - } - } else { - body - }; - - let req = ExportMetricsServiceRequest::decode(body.as_ref()).map_err(|e| { - ( - axum::http::StatusCode::BAD_REQUEST, - format!("Protobuf decode error: {}", e), - ) - })?; - // PERF-1 — parse once, share the result across the raw-path - // consumers (see the gRPC `export` path for the rationale). - let (points, sketch_payloads) = otlp_to_metric_points_and_sketches(&req); - process_otlp_request(&req, &points, &sketch_payloads, "HTTP"); - if let Some(cache) = &shared.probe_cache { - capture_freshness_probe_samples(&points, cache); - } - let mut outcome = IngestOutcome::default(); - if let Some(state) = &shared.ingest_state { - route_otlp_to_precompute(&points, &sketch_payloads, state).await; - outcome = route_modified_otlp_sketches_to_precompute(&req, state) - .await - .map_err(|error| (axum::http::StatusCode::UNPROCESSABLE_ENTITY, error))?; - } - debug!("OTLP sending response via HTTP"); - // HTTP OTLP exporters don't generally read `series_assignments` - // back the way the gRPC path does (the OTLP/HTTP spec keeps the - // response shape minimal), so we surface assignments only as a - // count for observability. Senders that need the bindings should - // use the gRPC transport. Eviction signals stay first-class — they - // are the universal recovery primitive (see proto comment on - // `ExportMetricsServiceResponse.unknown_series_ids`). - Ok(Json(serde_json::json!({ - "rejected": 0, - "unknown_series_ids": outcome.unknown_series_ids, - "series_assignments_count": outcome.series_assignments.len(), - }))) -} - -/// A parsed metric data point: name, labels, timestamp (nanos), and numeric value. -#[derive(Debug)] -pub struct MetricPoint { - pub name: String, - pub labels: HashMap, - pub timestamp_nanos: u64, - pub value: f64, -} - -/// A parsed sketch payload extracted from OTLP attributes. Carries the -/// metric name, attribute name (identifies the sketch kind on the wire), -/// labels (preserved from the OTLP DataPoint attributes + resource/scope), -/// wire timestamp, and the opaque `SketchEnvelope` protobuf bytes. -#[derive(Debug)] -pub struct SketchPoint { - pub name: String, - pub attr_name: String, - pub labels: HashMap, - pub timestamp_nanos: u64, - pub payload: Vec, -} - -type OtlpParseResult = (Vec, Vec); - -/// Render OTLP `(name, labels)` into the canonical PromQL-style series -/// key `metric{k1="v1",k2="v2"}` used everywhere in the data plane. -/// -/// Values are wrapped in double quotes and embedded `"`, `\`, `\n` are -/// escaped per the PromQL lexer rules. This matches: -/// -/// * `parse_labels_from_series_key` in `precompute_engine/worker.rs` -/// (the inverse — expects `key="value"`) -/// * `render_series_key` in `storage_engines/sketch_db/backfill/ -/// prometheus_reader.rs` (the other producer of this shape) -/// * `RawSample.labels`' documented shape (`metric{k="v",...}`) -/// * `sample_matches` in the backfill raw reader (strips `"` when -/// parsing) -/// -/// Pre-fix this helper emitted **unquoted** values (`k=v`), which the -/// parser silently rejected → `IngestState::extract_group_key_for` -/// returned `""` for every OTLP wire-format input, and the keyed -/// dispatch path inside `apply_sample` saw empty aggregated keys for -/// every OTLP sample. The bug was discovered while implementing PR -/// #284 (B7.6 ingest sid rekey); that PR side-stepped it by reading -/// `point.labels` directly, but emit-time `KeyByLabelValues` -/// content elsewhere depended on the roundtrip working — hence this -/// fix. See the regression test -/// `format_series_key_roundtrips_through_parse_labels` in -/// `precompute_engine/worker.rs`. -fn format_series_key(name: &str, labels: &HashMap) -> String { - let mut pairs: Vec<_> = labels.iter().collect(); - pairs.sort_by_key(|(k, _)| *k); - let labels_str = pairs - .iter() - .map(|(k, v)| format!("{}=\"{}\"", k, escape_label_value(v))) - .collect::>() - .join(","); - format!("{}{{{}}}", name, labels_str) -} - -/// Escape a label value for the PromQL series-key format. Mirrors -/// `storage_engines::sketch_db::backfill::prometheus_reader:: -/// escape_label_value` — both producers must stay in lockstep so the -/// parser in `precompute_engine::worker::parse_labels_from_series_key` -/// sees a consistent shape regardless of which ingest path emitted -/// the series key. -fn escape_label_value(v: &str) -> String { - let mut out = String::with_capacity(v.len()); - for c in v.chars() { - match c { - '\\' => out.push_str("\\\\"), - '"' => out.push_str("\\\""), - '\n' => out.push_str("\\n"), - other => out.push(other), - } - } - out -} - -fn get_sketch_payload_from_attrs( - attrs: &[asap_otel_proto::tonic::common::v1::KeyValue], -) -> Option<(String, Vec)> { - for kv in attrs { - match kv.key.as_str() { - "kll.sketch_payload" | "cms.sketch_payload" | "countsketch.sketch_payload" => { - if let Some(value) = &kv.value { - if let Some(AnyValueVariant::BytesValue(bytes)) = &value.value { - return Some((kv.key.clone(), bytes.clone())); - } - } - } - _ => {} - } - } - None -} - -fn log_sketch_envelope_type(attr_name: &str, payload: &[u8], metric_name: &str) { - match SketchEnvelope::decode(payload) { - Ok(env) => { - let sketch_type = match env.sketch_state { - Some(sketch_envelope::SketchState::Kll(_)) => "KLL", - Some(sketch_envelope::SketchState::CountMin(_)) => "CountMin", - Some(sketch_envelope::SketchState::CountSketch(_)) => "CountSketch", - Some(_) => "Other", - None => "Unknown", - }; - debug!( - "OTLP Sketches: metric='{}' attr='{}' payload_bytes={} sketch_type={}", - metric_name, - attr_name, - payload.len(), - sketch_type - ); - } - Err(e) => { - debug!( - "OTLP Sketches: metric='{}' attr='{}' payload_bytes={} decode_error='{}'", - metric_name, - attr_name, - payload.len(), - e - ); - } - } -} - -/// Capture every `http_freshness_probe_*` data point in the request -/// into the [`FreshnessProbeCache`]. Walks the parsed `MetricPoint`s -/// and only stores those whose metric name matches the probe prefix -/// — the cache itself enforces the prefix check via -/// [`FreshnessProbeCache::record`], so non-probe points are -/// short-circuited cheaply. -/// -/// Issue #46 ⑥ — without this hook, the cold-tier flush latency -/// (gorillas3 → 60 s TSDB block → Thanos sync) leaves the -/// `last_over_time(probe[10s])` query empty for the entire MVP demo -/// run. The cache lets the HTTP query handler answer the same query -/// from RAM with sub-second freshness. -fn capture_freshness_probe_samples(points: &[MetricPoint], cache: &FreshnessProbeCache) { - let mut updated = 0usize; - for point in points { - // The cache filters by metric-name prefix internally; calling - // `record` for every point is fine — non-probes are cheap - // string-prefix rejections and do not touch the lock. - let ts_ms = (point.timestamp_nanos / 1_000_000) as i64; - if cache.record(&point.name, ts_ms, point.value) { - updated += 1; - } - } - if updated > 0 { - debug!( - updated_probes = updated, - cache_size = cache.len(), - "freshness-probe cache updated" - ); - } -} - -fn process_otlp_request( - request: &ExportMetricsServiceRequest, - points: &[MetricPoint], - sketch_payloads: &[SketchPoint], - transport: &str, -) { - let resource_count = request.resource_metrics.len(); - let total_points = otlp_to_record_count(request); - if resource_count > 0 || total_points > 0 { - debug!( - "OTLP ingest: received {} resource metrics, {} total data points (transport={})", - resource_count, total_points, transport - ); - } - - for sketch in sketch_payloads { - log_sketch_envelope_type(&sketch.attr_name, &sketch.payload, &sketch.name); - } - if !sketch_payloads.is_empty() { - debug!( - "OTLP Sketch Payload Flow: received {} sketch payload(s), decoded successfully", - sketch_payloads.len() - ); - } - - let mut by_series: HashMap = HashMap::new(); - for point in points { - let key = format_series_key(&point.name, &point.labels); - *by_series.entry(key).or_insert(0) += 1; - } - if !by_series.is_empty() { - debug!( - "OTLP Raw Metrics Flow: received {} raw metric series", - by_series.len() - ); - for (series, count) in by_series { - debug!("OTLP Raw Metrics Flow: series {} count={}", series, count); - } - } - if let Some(first) = points.first() { - debug!( - "OTLP parse example: {} {:?} @{}ns = {}", - first.name, first.labels, first.timestamp_nanos, first.value - ); - } -} - -/// Route parsed OTLP data through the precompute engine's worker pool. -/// -/// Both raw metric points and pre-built sketch payloads are dispatched via -/// `WorkerMessage::GroupSamples` / `WorkerMessage::AccumulatorInput`, with -/// `(agg_id, group_key)` derived from the wire labels using the same logic -/// as the Prometheus/VictoriaMetrics paths. This preserves full label -/// semantics and lets the precompute engine perform config-driven window -/// aggregation over both streams. -/// -/// Metrics whose name does not match any aggregation in the streaming -/// config are dropped with a debug log — the precompute engine only -/// maintains state for configured metrics. -/// -/// Log a per-driver `HashMap` of §6.3 write-barrier -/// drops. Post-schema-retirement the agg_id-keyed -/// `IngestState::record_barrier_drop` counter is gone — the -/// sid-level barrier inside `SketchStore::ingest_precompute_for_agg_config` -/// silently rejects retired-sid writes without crossing this -/// observer. The function is kept (callers still hand it an empty -/// map) so the call shape doesn't churn; if the map is non-empty -/// it emits a single debug log for forensic visibility. -fn flush_barrier_drops(_state: &IngestState, drops: &HashMap, driver_tag: &'static str) { - if drops.is_empty() { - return; - } - let total: u64 = drops.values().sum(); - debug!( - driver = driver_tag, - total_dropped = total, - by_agg_id = ?drops, - "§6.3 write barrier dropped OTLP samples (agg is retired/expired)" - ); -} - -/// Resolve the bucket sid (and `policy_fp`) for a single data point -/// against a single matching `PrecomputeMaterialization`. -/// -/// B7.6 — sid is the bucket identity in the precompute engine; this -/// helper folds `(config, grouping-label-values)` into a single u64 via -/// `SeriesIdResolver`. Same `(metric, grouping-label-values, agg_kind)` -/// always returns the same sid, so distinct data points that share the -/// same group bucket land in the same `GroupSamples` / -/// `AccumulatorInput` message — the GROUP-BY semantics the legacy -/// `(agg_id, group_key)` tuple expressed. -/// -/// The "attrs" passed to the resolver are the GROUPING-LABEL projection -/// of the wire labels (NOT the full label set) — otherwise every -/// distinct `(rack, node, pod)` tuple under a `grouping_labels=[zone]` -/// policy would mint its own sid and never roll up. -/// -/// Configured ingest shares the policy-aware physical identity used by the -/// live storage sink and backfill. Unbound modified-OTLP sketches retain -/// their separate wire-level identity protocol. -fn resolve_bucket_sid_for_agg_config( - ingest_state: &Arc, - config: &asap_types::aggregation_config::PrecomputeMaterialization, - point_labels: &HashMap, - captured_generation: Option<&asap_types::sds::CatalogGeneration>, -) -> Result<(u64, asap_types::PolicyFingerprint), String> { - if !config.population_key_encoding.is_legacy() { - return Err("canonical population requires typed OTLP label propagation".into()); - } - let grouping_pairs: Vec<(&str, &str)> = config - .grouping_labels - .iter() - .map(|name| { - let v = point_labels.get(name).map(|s| s.as_str()).unwrap_or(""); - (name.as_str(), v) - }) - .collect(); - let fp = crate::drivers::ingest::population_attrs_fingerprint( - config.population_key_encoding, - &grouping_pairs, - )?; - let sid = ingest_state.summary_store.resolve_output_storage_handle( - &ingest_state.series_resolver, - config.policy_fingerprint().into(), - &fp, - captured_generation, - )?; - let policy_fp = asap_types::PolicyFingerprint(config.policy_fp_u64()); - Ok((sid, policy_fp)) -} - -async fn route_otlp_to_precompute( - points: &[MetricPoint], - sketch_payloads: &[SketchPoint], - ingest_state: &Arc, -) { - let ingest_received_at = Instant::now(); - - // Snapshot the latest agg_configs from the hot-reload handle so - // new aggregations are visible without restart. - let active_physical_plan_snapshot = ingest_state.active_physical_plan_snapshot(); - let catalog_generation = active_physical_plan_snapshot - .as_ref() - .and_then(|plan| plan.precompute_plan.summary_catalog.clone()) - .map(Arc::new); - let snap = active_physical_plan_snapshot - .as_ref() - .map(|plan| plan.installed_precompute_plan.clone()) - .unwrap_or_else(|| ingest_state.config_snapshot()); - let agg_configs = snap.materializations(); - // Reconcile sid lifecycle using the current streaming-config snapshot. - // The store enforces the write barrier for retired and expired instances. - // A config-Arc identity check skips catalog scans while the config is unchanged. - let _ = crate::storage_engines::sketch_db::lifecycle::reconcile_if_config_changed( - ingest_state.summary_store.as_ref(), - &snap, - crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, - ); - - // B7.6 — bucket by `sid` instead of `(agg_id, group_key)`. The - // grouping label values fold into the sid via the - // `(metric, attrs_fp, agg_kind_canonical)` identity contract on - // `SeriesIdResolver`: same (config, grouping-label-values) → same - // sid → same bucket. We still carry `policy_fp` and `group_key` - // alongside the sid in the WorkerMessage so the worker can resolve - // the source config and render emit-time labels without consulting - // the sid → attrs reverse mapping. - type BucketTuple = ( - u64, - asap_types::PolicyFingerprint, - Arc, - ); // (sid, policy_fp, group_key) - type SampleTuple = (String, i64, f64); - let mut by_bucket: HashMap)> = HashMap::new(); - let mut raw_matched = 0usize; - let mut raw_unmatched = 0usize; - // The store enforces the sid-level write barrier. This empty map preserves - // the barrier-log interface without duplicating that check. - let raw_barrier_drops: HashMap = HashMap::new(); - - for point in points { - let series_key = format_series_key(&point.name, &point.labels); - let ts_ms = (point.timestamp_nanos / 1_000_000) as i64; - let mut matched = false; - for config in agg_configs.values() { - if config.metric != point.name - && config.spatial_filter_normalized != point.name - && config.spatial_filter != point.name - { - continue; - } - let group_key = IngestState::extract_group_key_for(&series_key, config); - let (sid, policy_fp) = match resolve_bucket_sid_for_agg_config( - ingest_state, - config, - &point.labels, - catalog_generation.as_deref(), - ) { - Ok(binding) => binding, - Err(error) => { - warn!(%error, "configured ingest series reactivation rejected"); - continue; - } - }; - by_bucket - .entry(sid) - .or_insert_with(|| ((sid, policy_fp, group_key.clone()), Vec::new())) - .1 - .push((series_key.clone(), ts_ms, point.value)); - matched = true; - } - if matched { - raw_matched += 1; - } else { - raw_unmatched += 1; - } - } - flush_barrier_drops(ingest_state, &raw_barrier_drops, "otlp-raw"); - - let raw_messages: Vec = by_bucket - .into_iter() - .map( - |(_sid, ((sid, policy_fp, group_key), samples))| WorkerMessage::GroupSamples { - sid, - policy_fp, - group_key, - samples, - ingest_received_at, - }, - ) - .collect(); - - if !raw_messages.is_empty() { - if let Err(e) = ingest_state - .router - .route_group_batch(raw_messages, ingest_received_at, catalog_generation.clone()) - .await - { - warn!("OTLP raw-sample routing error: {}", e); - } - } - - // Build AccumulatorInput messages for pre-built sketches. - // Each sketch payload carries a metric name that must match an - // aggregation config; labels come from the point's attributes. The - // payload is decoded enough to identify the sketch type for logging; - // the accumulator the worker receives is a conservative placeholder - // until per-variant `SketchEnvelope → concrete accumulator` decoders - // are wired up (see TODO below). - let mut sketch_messages: Vec = Vec::new(); - let mut sketch_matched = 0usize; - let mut sketch_unmatched = 0usize; - // See `raw_barrier_drops` above — schema-keyed barrier dropped; - // sid-level barrier in `SketchStore` enforces §6.3 going - // forward. - let sketch_barrier_drops: HashMap = HashMap::new(); - for point in sketch_payloads { - let series_key = format_series_key(&point.name, &point.labels); - let ts_ms = (point.timestamp_nanos / 1_000_000) as i64; - let sketch_type = identify_sketch_type(&point.payload); - let mut matched = false; - for config in agg_configs.values() { - if config.metric != point.name - && config.spatial_filter_normalized != point.name - && config.spatial_filter != point.name - { - continue; - } - let group_key = IngestState::extract_group_key_for(&series_key, config); - // Wrap the raw SketchEnvelope bytes in a SketchEnvelopeAccumulator - // so the precompute engine receives the opaque sketch as-is. This - // preserves all sketch state end-to-end; per-variant decoding - // (e.g. CountMin → CountMinSketchAccumulator) can layer on top - // later without changing the routing contract. - let accumulator: Box = - match SketchEnvelopeAccumulator::from_proto_bytes(point.payload.clone()) { - Ok(acc) => Box::new(acc), - Err(e) => { - warn!( - "OTLP sketch decode failed for metric='{}' attr='{}': {}", - point.name, point.attr_name, e - ); - continue; - } - }; - // B7.6 — same sid-resolution scheme as the raw path: bucket - // identity is (metric, grouping-label-values, agg_kind). - // The opaque SketchEnvelope path predates per-variant - // sketch-kind plumbing here; treat the bucket as ExactAgg - // so the resolver key matches what - // `reconcile_from_streaming_config` derives from the same - // config (otherwise the bucket would be reachable but never - // reconciled). - let (sid, policy_fp) = match resolve_bucket_sid_for_agg_config( - ingest_state, - config, - &point.labels, - catalog_generation.as_deref(), - ) { - Ok(binding) => binding, - Err(error) => { - warn!(%error, "configured ingest series reactivation rejected"); - continue; - } - }; - sketch_messages.push(WorkerMessage::AccumulatorInput { - sid, - policy_fp, - group_key, - timestamp_ms: ts_ms, - accumulator, - ingest_received_at, - }); - matched = true; - } - if matched { - sketch_matched += 1; - debug!( - "OTLP sketch routed to precompute engine: metric='{}' attr='{}' type={} bytes={}", - point.name, - point.attr_name, - sketch_type, - point.payload.len() - ); - } else { - sketch_unmatched += 1; - } - } - flush_barrier_drops(ingest_state, &sketch_barrier_drops, "otlp-sketch-envelope"); - - if !sketch_messages.is_empty() { - if let Err(e) = ingest_state - .router - .route_group_batch( - sketch_messages, - ingest_received_at, - catalog_generation.clone(), - ) - .await - { - warn!("OTLP sketch routing error: {}", e); - } - } - - if raw_unmatched > 0 || sketch_unmatched > 0 { - debug!( - "OTLP ingest: {} raw samples + {} sketches dropped (no matching aggregation config); \ - {} raw + {} sketches routed to precompute engine", - raw_unmatched, sketch_unmatched, raw_matched, sketch_matched - ); - } -} - -/// Walk the modified-OTLP first-class sketch metric variants -/// (`DDSketch` / `KLLSketch` / `CountSketch` / `CountMinSketch` / -/// `HLLSketch` on `Metric.data` tags 13–17) and dispatch each -/// `*SketchDataPoint` through the precompute engine via -/// `WorkerMessage::AccumulatorInput`. -/// -/// Per-variant decoding of the typed `sketch` bytes is delegated to -/// `decode_modified_otlp_sketch_bytes`, which in turn calls into the -/// matching concrete accumulator's `from_sketchlib_proto_bytes` -/// constructor when one exists. Variants without a concrete decoder -/// today (KLL / DDSketch / CountSketch / HLL) fall through to the -/// §5.2 fallback path so the user still gets a correct answer; PR C -/// (task #8) will close those decoder gaps. -/// Per-Export outcome of the modified-OTLP sketch ingest path. Carries -/// both halves of the Phase-4/B round trip: -/// -/// - `unknown_series_ids` → sids the receiver could not satisfy this -/// Export (sender's cache is stale; sender must evict + re-emit with -/// attrs). -/// - `series_assignments` → newly-minted or cache-resolved sid bindings -/// the receiver wants the sender to cache. Sent back in -/// `ExportMetricsServiceResponse.series_assignments` so the sender -/// omits attrs on subsequent emits keyed by these sids. -/// -/// Empty `series_assignments` is the no-op case (every DP arrived with -/// the right sid already, or every DP was rejected to `unknown_series_ids`). -#[derive(Debug, Default)] -pub(crate) struct IngestOutcome { - pub unknown_series_ids: Vec, - pub series_assignments: Vec, -} - -async fn route_modified_otlp_sketches_to_precompute( - request: &ExportMetricsServiceRequest, - ingest_state: &Arc, -) -> Result { - use asap_otel_proto::tonic::metrics::v1::metric::Data; - - let ingest_received_at = Instant::now(); - // Load the generation exactly once. Deriving both the runtime config and - // transmission contract from this Arc prevents an activation between two - // independent ArcSwap loads from producing a torn ingest view. - let active_physical_plan_snapshot = ingest_state.active_physical_plan_snapshot(); - let snap = active_physical_plan_snapshot - .as_ref() - .map(|plan| plan.installed_precompute_plan.clone()) - .unwrap_or_else(|| ingest_state.config_snapshot()); - let catalog_generation = active_physical_plan_snapshot - .as_ref() - .and_then(|plan| plan.precompute_plan.summary_catalog.clone()) - .map(Arc::new); - let active_physical_plan = active_physical_plan_snapshot.filter(|plan| plan.plan_id() != 0); - let lineage_batch_guard = active_physical_plan - .as_ref() - .map(|_| ingest_state.observability.frame_lineage.lock_batch()); - if let Some(active) = active_physical_plan.as_ref() { - // Validate the complete request before mutating the SID registry, - // sketch store, snapshot cache, or worker queues. This makes the - // OTLP request the atomic full-frame publication unit: 2xx/OK means - // the batch was accepted, while a contract error applies none of it. - preflight_summary_frames(request, ingest_state, active)?; - } - let agg_configs = snap.materializations(); - // Reconcile sid lifecycle only when the configuration snapshot changes. - let _ = crate::storage_engines::sketch_db::lifecycle::reconcile_if_config_changed( - ingest_state.summary_store.as_ref(), - &snap, - crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, - ); - let mut messages: Vec = Vec::new(); - let mut routed = 0usize; - let mut decoded_failed = 0usize; - let mut unconfigured = 0usize; - // CQ-2 — the legacy routing-side WorkerMessage push (the DEPRECATED - // dual-write that clones the accumulator into the worker under a - // bucket-sid, in tandem with the Phase-5 SketchStore append above) is - // gated OFF by default. The SketchStore `append_sample` path is the - // live write; the worker push only matters until the ASAP-tier query - // reducer is validated end-to-end, so re-enable it with - // `ASAP_LEGACY_DUAL_WRITE=1` rather than paying the clone + double - // store on the default path. Read once per Export (cheap), not per DP. - let legacy_dual_write = std::env::var("ASAP_LEGACY_DUAL_WRITE") - .map(|v| { - let v = v.trim(); - v == "1" || v.eq_ignore_ascii_case("true") || v.eq_ignore_ascii_case("on") - }) - .unwrap_or(false); - // Schema-keyed barrier dropped (see `raw_barrier_drops` above); - // sid-level barrier in `SketchStore` carries §6.3 going forward. - let barrier_drops: HashMap = HashMap::new(); - // sids the receiver did not recognize this Export. Returned - // to the caller so the gRPC / HTTP handler can stamp them into - // `ExportMetricsServiceResponse.unknown_series_ids`. Senders evict - // these sids and re-emit with attributes; backend re-resolves and - // returns fresh `series_assignments`. - let mut unknown_sids: Vec = Vec::new(); - // Sid bindings the receiver wants the sender to cache. Populated - // every time an attrs-bearing DP is resolved by the - // `SeriesIdResolver` (either fresh mint or cache hit). Returned - // alongside `unknown_sids` so the gRPC / HTTP handler can stamp - // them into `ExportMetricsServiceResponse.series_assignments`. - let mut new_assignments: Vec = - Vec::new(); - - for resource_metrics in &request.resource_metrics { - let resource_attrs = resource_metrics - .resource - .as_ref() - .map(|r| attributes_to_map(&r.attributes)) - .unwrap_or_default(); - - for scope_metrics in &resource_metrics.scope_metrics { - let scope_attrs = scope_metrics - .scope - .as_ref() - .map(|s| attributes_to_map(&s.attributes)) - .unwrap_or_default(); - - for metric in &scope_metrics.metrics { - if metric.name.is_empty() { - continue; - } - - let base_labels: HashMap = scope_attrs - .iter() - .chain(resource_attrs.iter()) - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - // Each branch yields an iterator-like slice of (kind, attrs, - // time_unix_nano, sketch_bytes, encoding_i32) tuples. We - // then route each tuple through the same dispatcher. - let dps: Vec = match &metric.data { - Some(Data::Ddsketch(d)) => { - let cfg = - crate::storage_engines::sketch_db::index::SketchConfig::DDSketch { - relative_accuracy: d.relative_accuracy, - }; - d.data_points - .iter() - .map(|dp| ModifiedOtlpSketchDp { - algorithm: SketchAlgorithm::DDSketch, - attrs: merge_point_attributes(&base_labels, &dp.attributes), - time_unix_nano: dp.time_unix_nano, - sketch: dp.sketch.clone(), - encoding: dp.encoding, - series_id: dp.series_id, - start_time_unix_nano: dp.start_time_unix_nano, - container_config: cfg.clone(), - }) - .collect() - } - Some(Data::Kllsketch(k)) => { - let cfg = - crate::storage_engines::sketch_db::index::SketchConfig::Kll { k: k.k }; - k.data_points - .iter() - .map(|dp| ModifiedOtlpSketchDp { - algorithm: SketchAlgorithm::Kll, - attrs: merge_point_attributes(&base_labels, &dp.attributes), - time_unix_nano: dp.time_unix_nano, - sketch: dp.sketch.clone(), - encoding: dp.encoding, - series_id: dp.series_id, - start_time_unix_nano: dp.start_time_unix_nano, - container_config: cfg.clone(), - }) - .collect() - } - Some(Data::Countsketch(c)) => { - let cfg = - crate::storage_engines::sketch_db::index::SketchConfig::CountSketch { - rows: c.rows, - cols: c.cols, - }; - c.data_points - .iter() - .map(|dp| ModifiedOtlpSketchDp { - algorithm: SketchAlgorithm::CountSketch, - attrs: merge_point_attributes(&base_labels, &dp.attributes), - time_unix_nano: dp.time_unix_nano, - sketch: dp.sketch.clone(), - encoding: dp.encoding, - series_id: dp.series_id, - start_time_unix_nano: dp.start_time_unix_nano, - container_config: cfg.clone(), - }) - .collect() - } - Some(Data::Countminsketch(c)) => { - let cfg = - crate::storage_engines::sketch_db::index::SketchConfig::CountMin { - rows: c.rows, - cols: c.cols, - }; - c.data_points - .iter() - .map(|dp| ModifiedOtlpSketchDp { - algorithm: SketchAlgorithm::Cms, - attrs: merge_point_attributes(&base_labels, &dp.attributes), - time_unix_nano: dp.time_unix_nano, - sketch: dp.sketch.clone(), - encoding: dp.encoding, - series_id: dp.series_id, - start_time_unix_nano: dp.start_time_unix_nano, - container_config: cfg.clone(), - }) - .collect() - } - Some(Data::Hllsketch(h)) => { - let cfg = crate::storage_engines::sketch_db::index::SketchConfig::Hll { - precision: h.precision, - }; - h.data_points - .iter() - .map(|dp| ModifiedOtlpSketchDp { - algorithm: SketchAlgorithm::Hll, - attrs: merge_point_attributes(&base_labels, &dp.attributes), - time_unix_nano: dp.time_unix_nano, - sketch: dp.sketch.clone(), - encoding: dp.encoding, - series_id: dp.series_id, - start_time_unix_nano: dp.start_time_unix_nano, - container_config: cfg.clone(), - }) - .collect() - } - _ => continue, - }; - - // Canonicalize the metric name once per metric: strip the - // agent-side sketch-family suffix (`_kll`, `_hll`, …) so - // the whole ingest pipeline — sid resolution, series-key - // snapshot cache, `SummarySeriesMetadata.metric_name`, - // `derive_sketch_policy_fp`, and the legacy precompute - // router match below — keys on the RAW metric name that - // the controller's streaming-config and the query - // analyzer speak. Without this the warm-tier sketch - // queries can never resolve (see - // `canonical_sketch_metric_name` for the full rationale). - // All datapoints in one OTLP metric share the same family - // (the `metric.data` variant), so the first dp's `kind` - // determines the suffix for the whole metric. - let canonical_name: String = match dps.first() { - Some(first) => { - canonical_sketch_metric_name(&metric.name, first.algorithm.clone()) - .to_string() - } - None => metric.name.clone(), - }; - - for mut dp in dps { - let frame_identity = if let Some(active) = active_physical_plan.as_ref() { - match take_summary_frame_identity( - &mut dp.attrs, - dp.start_time_unix_nano, - dp.time_unix_nano, - ) - .and_then(|frame| { - if state_encoding_for_wire(dp.encoding) != Some(frame.encoding.clone()) - { - return Err("wire encoding differs from frame identity".into()); - } - active - .transmission_plan - .validate_frame(&frame) - .map_err(|error| error.to_string())?; - Ok(frame) - }) { - Ok(frame) => Some(frame), - Err(error) => unreachable!( - "summary frame changed after successful request preflight: {error}" - ), - } - } else { - None - }; - let series_key = bound_sketch_series_key( - &canonical_name, - &dp.attrs, - frame_identity.as_ref(), - ); - let ts_ms = (dp.time_unix_nano / 1_000_000) as i64; - - // Sid resolution — registry-allocated, NOT content- - // addressed. The `SeriesIdResolver` is the single - // authoritative mint for sids in the pipeline: same - // `(metric_name, attrs_fingerprint)` always returns - // the same sid for the lifetime of the resolver's - // cache. Uniqueness is by construction - // (`AtomicU64::fetch_add`); two different identities - // CANNOT share a sid. The content-addressed - // `compute_sketch_sid` path was retired here because - // u64 xxhash gives only probabilistic uniqueness, - // and "unique sid per series" is a contract this - // wire shape needs (the agent omits attrs on - // subsequent emits; the receiver must be able to - // disambiguate `sid → (metric, attrs)`). - // - // Determinism trade-off: sids are NOT stable across - // independent backends, and (without persistence) - // not across backend restarts either. PR-2 of this - // chain wires a WAL-backed `SeriesResolverPersistence` - // trait so the mapping survives restart; without it, - // restart-recovery still works via the - // `unknown_series_ids` eviction primitive (agents - // observe their cached sid is unknown, evict, re-emit - // with attrs, get a fresh assignment). - // - // Four wire cases: - // (sid=0, attrs) → resolve (mint or cache - // hit); always emit a - // `SeriesAssignment` so the - // sender caches the binding - // (sid!=0, attrs) → resolve; if the resolver's - // sid disagrees with the - // sender's, push sender's - // sid to `unknown_sids` - // (sender's cache was stale, - // e.g. after a backend - // restart without - // persistence); always emit a - // fresh assignment - // (sid!=0, no attrs) → can't resolve without - // attrs; accept the sid iff - // the SketchStore has it - // registered. Unknown → push - // to `unknown_sids` and drop - // this DP (sender will - // re-emit with attrs next - // pass) - // (sid=0, no attrs) → invalid wire shape, drop - let attrs_pairs: Vec<(&str, &str)> = dp - .attrs - .iter() - .map(|(k, v)| (k.as_str(), v.as_str())) - .collect(); - // `canonical_attrs_fingerprint(&[])` is `""` — a valid, - // stable key. The empty-attrs case is therefore NOT a - // reason to skip the resolver (P1-5): a globally- - // aggregated sketch (no resource/scope/DP attrs at all) - // resolves to the stable `(metric, "", agg_kind)` sid - // just like any other series. - let fp = crate::drivers::ingest::canonical_attrs_fingerprint(&attrs_pairs); - // Store-lookup-ONLY path is reserved for the bandwidth- - // saving case: the sender already holds a cached sid and - // deliberately omits attrs on this emit. We cannot - // re-derive its identity (no attrs to fingerprint AND a - // non-zero sid the sender expects us to honor), so we - // accept the sid iff the SketchStore has it registered. - // - // Every OTHER case — including (sid=0, empty attrs), - // i.e. global aggregation — goes through the resolver, - // because `fp=""` is a perfectly good mint/lookup key. - let resolved_sid: Option = if dp.series_id != 0 && attrs_pairs.is_empty() { - let sid = dp.series_id; - if ingest_state.summary_store.is_current_storage_handle(sid) { - Some(sid) - } else { - unknown_sids.push(sid); - None - } - } else { - // Build the canonical AggKind string for this DP so - // the resolver's cache key is `(metric, fp, agg_kind)`. - // Two DPs over the same (metric, attrs) but different - // sketch algorithms/configs (e.g. DDSketch vs Kll, or two - // DDSketches at different relative_accuracy) get - // SEPARATE sids — matching the identity model the - // retired `compute_sketch_sid` hashed over. For the - // empty-attrs global-aggregation case `fp` is `""`, - // so the resolver mints a single stable sid for - // `(metric, "", agg_kind)`. - // - // P1-4 — sid identity uses the BASE (heap-less) - // family so a heap-bearing frame and a heap-LESS - // frame for the same series share ONE sid. The heap - // is an additive enrichment on the same sketch - // substrate, not a different series; the capability - // UPGRADE below promotes that shared sid's metadata - // when a heap arrives. Without this collapse the two - // frames would mint distinct sids and the upgrade - // could never fire (the analyzer would also see two - // candidates for one logical series). - let definition = frame_identity - .as_ref() - .map(|frame| frame.materialization) - .unwrap_or_else(|| asap_types::PolicyFingerprint(0).into()); - let resolved = ingest_state.summary_store.resolve_output_storage_handle( - &ingest_state.series_resolver, - definition, - &fp, - catalog_generation.as_deref(), - ); - // Codec-only unit fixtures have no installed plan. Production - // accepts stored summaries only through an installed output. - #[cfg(test)] - let resolved = if catalog_generation.is_none() - && definition.fingerprint().is_unset() - { - let kind = crate::storage_engines::sketch_db::data::AggKind::Sketch { - algorithm: base_sketch_algorithm(sketch_algorithm_for(&dp)), - config: dp.container_config.clone(), - spatial_filter_canonical: String::new(), - }; - Ok(ingest_state.series_resolver.resolve( - &canonical_name, - &fp, - &kind.canonical_string(), - )) - } else { - resolved - }; - let assigned = match resolved { - Ok(sid) => sid, - Err(error) => { - if dp.series_id != 0 { - unknown_sids.push(dp.series_id); - } - warn!(%error, "modified OTLP series reactivation rejected"); - continue; - } - }; - if dp.series_id != 0 && dp.series_id != assigned { - // Sender's cached sid disagrees with the - // resolver's binding — sender's cache is - // stale, signal eviction. - unknown_sids.push(dp.series_id); - } - // Always echo the canonical binding back in - // `series_assignments` so the sender caches it - // (or refreshes a stale entry). The dictionary - // bookkeeping fields beyond - // `(attributes_fingerprint, series_id)` are - // optional today — the patched OTel-Go exporter - // keys its local cache on the fingerprint, not on - // the dictionary metadata. The empty-attrs global - // series echoes an assignment with an empty - // `attributes_fingerprint`, which the exporter - // caches like any other binding. - new_assignments.push( - asap_otel_proto::tonic::collector::metrics::v1::SeriesAssignment { - attributes_fingerprint: fp.as_bytes().to_vec(), - series_id: assigned, - metric_name: metric.name.clone(), - ..Default::default() - }, - ); - Some(assigned) - }; - let Some(sid) = resolved_sid else { - continue; - }; - - if let Some(frame) = frame_identity.as_ref() { - let observed_policy = ingest_state - .summary_store - .instance(sid) - .map(|metadata| metadata.policy_fp) - .unwrap_or_else(|| { - derive_sketch_policy_fp( - ingest_state, - &canonical_name, - sketch_algorithm_for(&dp), - &dp.container_config, - &dp.attrs.keys().cloned().collect(), - Some(frame.materialization), - ) - }); - if observed_policy != frame.materialization.fingerprint() { - unreachable!( - "materialization changed after successful request preflight" - ); - } - } - - // register a `SummarySeriesMetadata` on - // first sight of `sid` and append this DP's sketch - // state to the per-sid columnar storage. The instance - // is keyed by sid, so subsequent DPs on the same sid - // skip the register step. `group_by_keys` is - // `dp.attrs.keys()` — after the agent's `AggregateBy` - // rollup, `attributes` is the group-by VALUES vector, - // and its key set IS the group-by KEY set. - { - use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchAlgorithm, SummarySeriesMetadata, - }; - use std::collections::BTreeSet; - - if ingest_state.summary_store.instance(sid).is_none() { - let algorithm = sketch_algorithm_for(&dp); - let cap = match algorithm { - SketchAlgorithm::DDSketch | SketchAlgorithm::Kll => { - Capability::QuantileApprox(Some(algorithm.clone())) - } - SketchAlgorithm::Hll | SketchAlgorithm::UnivMon => { - Capability::CardinalityApprox - } - // Heap-LESS frequency sketches answer bare - // frequency point queries (no top-k); index - // them as FrequencyEstimate so a `topk(...)` - // query routes to archive (or to a different - // sid that carries a heap-bearing variant). - SketchAlgorithm::CountSketch | SketchAlgorithm::Cms => { - Capability::FrequencyEstimate(Some(algorithm.clone())) - } - // Heap-BEARING frequency sketches answer - // both point-frequency AND top-k. We register - // them under FrequencyTopk (top-k is the - // strongest claim); the analyzer-side - // `is_satisfied_by` for FrequencyEstimate - // explicitly accepts heap-bearing variants, - // so bare-frequency queries still route here. - SketchAlgorithm::CmsWithHeap - | SketchAlgorithm::CountSketchWithHeap => { - Capability::FrequencyTopk(Some(algorithm.clone())) - } - SketchAlgorithm::Kmv | SketchAlgorithm::Theta => { - Capability::CardinalityApprox - } - }; - let group_by_keys: BTreeSet = - dp.attrs.keys().cloned().collect(); - let cfg = dp.container_config.clone(); - // Derive the policy fingerprint by content- - // matching the OTLP DP's shape against the - // streaming-config registry. Sketches arrive - // with `(kind, config)` embedded but no policy - // reference; we find the policy whose contents - // produce the same shape. Lookup returns - // `Some(fp)` on a unique match, `None` - // when zero policies match (sketch ingested - // before the streaming-config caught up) or - // when multiple policies match the same shape - // (would have been a sid-collision bug — - // surfaces as an UNSET registration so the - // legacy `instances_matching` walk still - // covers it). - let policy_fp = derive_sketch_policy_fp( - ingest_state, - &canonical_name, - algorithm.clone(), - &cfg, - &group_by_keys, - frame_identity.as_ref().map(|frame| frame.materialization), - ); - // Per-item dimension (item_label) the controller threaded - // into the matched policy's parameters — recorded on the sid - // below so the query engine can answer per-item estimate(key) - // (the CMS/CountSketch FrequencyEstimate gate consults it). - let item_label_for_sid: Option = { - snap.get_aggregation_config(policy_fp.into()) - .or_else(|| { - snap.materializations() - .values() - .find(|c| c.metric == canonical_name) - }) - .and_then(|c| c.parameters.get("item_label")) - .and_then(|v| v.as_str()) - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()) - }; - ingest_state.summary_store.register(SummarySeriesMetadata { - storage_handle: sid, - metric_name: canonical_name.clone(), - group_by_keys, - capability: Some(cap), - agg_kind: - crate::storage_engines::sketch_db::index::AggKind::Sketch { - algorithm, - config: cfg.clone(), - spatial_filter_canonical: String::new(), - }, - accuracy: Some(AccuracyBound::from_config(&cfg)), - first_seen_unix_ms: ts_ms, - retired_at_ms: None, - expires_at_ms: None, - policy_fp, - }); - if let Some(label) = &item_label_for_sid { - ingest_state.summary_store.set_item_label(sid, label); - } - } else if let Some(existing) = ingest_state.summary_store.instance(sid) { - // P1-4 (a) — one-way capability UPGRADE. The sid - // was first registered from a non-heap frame - // (PROTO, a delta, or a heap-LESS MSGPACK), so it - // carries `FrequencyEstimate(CountMin|CountSketch)` - // and can never answer `topk(...)`. If a later - // heap-bearing frame arrives for the SAME sid, - // promote it to `FrequencyTopk(*WithHeap)` and - // upgrade its kind handle so the analyzer routes - // top-k queries here. Never downgrades: we only - // act when the current cap is heap-LESS frequency - // and the incoming frame actually carries a heap. - let incoming_algorithm = sketch_algorithm_for(&dp); - let upgrade_to = match (&existing.capability, incoming_algorithm) { - ( - Some(Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))), - SketchAlgorithm::CmsWithHeap, - ) => Some(SketchAlgorithm::CmsWithHeap), - ( - Some(Capability::FrequencyEstimate(Some( - SketchAlgorithm::CountSketch, - ))), - SketchAlgorithm::CountSketchWithHeap, - ) => Some(SketchAlgorithm::CountSketchWithHeap), - _ => None, - }; - if let Some(new_algorithm) = upgrade_to { - let mut upgraded = (*existing).clone(); - upgraded.capability = - Some(Capability::FrequencyTopk(Some(new_algorithm.clone()))); - upgraded.agg_kind = - crate::storage_engines::sketch_db::index::AggKind::Sketch { - algorithm: new_algorithm.clone(), - config: dp.container_config.clone(), - spatial_filter_canonical: String::new(), - }; - // `register` overwrites the sid-keyed entry - // in place (same sid → same policy/metric - // index slots), so this is an atomic swap to - // the stronger capability. - ingest_state.summary_store.register(upgraded); - debug!( - "OTLP sketch sid {} upgraded {:?} -> FrequencyTopk({:?}) \ - on heap-bearing frame (metric={}, encoding={})", - sid, - SketchAlgorithm::Cms, - new_algorithm, - metric.name, - dp.encoding - ); - } - } - } - - // Encoding dispatch: full frames (PROTO / - // MSGPACK) decode standalone and refresh the - // per-series snapshot cache; delta frames - // (PROTO_DELTA) look up the cached base and - // apply the diff in place. The cache key is the - // series_key — agent-side windowing guarantees - // one in-flight delta per (metric, labels) so - // the next full snapshot replaces the current - // cache entry cleanly. - // - // Per-window base rotation - // (`docs/delta-baseline-contract.md` §3): the edge - // tumbling window resets per-series sketch state - // every window, so each window's delta is that - // window's marginal against an empty base. The - // backend therefore must NOT accumulate forever - // (`state(N) = state(N-1) ⊕ delta(N)`), which would - // over-count across windows. Instead we detect a - // window boundary per series — a change in the data - // point's window start (`start_time_unix_nano`) - // versus the `window_start` stored with the cached - // base — and reset the cached base to empty before - // applying the new window's delta. Within a window - // deltas still accumulate; at a new window the base - // starts fresh, so the reconstructed `state(N)` is - // window N only. Sketch-agnostic: the reset is the - // additive families' (DDSketch / CMS / CountSketch / - // HLL) `AggregateCore::reset_to_empty`; KLL never - // deltas. Full frames keep REPLACE semantics and set - // the stored `window_start`. - let accumulator: Box = if dp.encoding == ENCODING_PROTO_DELTA - || dp.encoding == ENCODING_MSGPACK_DELTA - { - let (mut merged, base_window_start) = match ingest_state - .sketch_snapshots - .get(&series_key) - .map(|e| (e.core.clone_boxed_core(), e.window_start)) - { - Some(pair) => pair, - None => { - // P1-1/P1-2 — no cached base. Under the - // per-window-reset contract each delta - // reconstructs onto an EMPTY base, so for the - // ADDITIVE families (CMS / CountSketch / HLL) - // we bootstrap an empty accumulator of the - // frame's (kind, config) and apply the delta - // onto it — mirroring the warm-read tier so - // the worker tier agrees and recovers after a - // backend restart (which drops this in-memory - // cache). DDSketch / KLL can't reconstruct - // from a bare delta, so they keep the - // unchanged "drop until the next full frame" - // behavior. - match empty_accumulator_for_delta_bootstrap( - dp.algorithm.clone(), - &dp.container_config, - dp.encoding, - ) { - Some(empty) => { - debug!( - "OTLP delta-sketch with no base \ - bootstrapped onto an empty \ - accumulator (metric={}, \ - series_key={}, kind={:?}, \ - encoding={})", - metric.name, series_key, dp.algorithm, dp.encoding - ); - // Treat the freshly-minted empty base - // as belonging to THIS delta's window - // so the window-boundary reset below - // is a no-op (the base is already - // empty for this window). - (empty, dp.start_time_unix_nano) - } - None => { - // DD/KLL — unchanged drop. Count as a - // distinct "no base" drop reason. - ingest_state - .observability - .dropped_no_base - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - decoded_failed += 1; - debug!( - "OTLP delta-sketch arrived before any base \ - snapshot and is not bootstrappable \ - (metric={}, series_key={}, kind={:?}); \ - dropping — agent must resend the next full \ - frame", - metric.name, series_key, dp.algorithm - ); - continue; - } - } - } - }; - // Window boundary: the incoming delta opens a new - // window for this series. Rotate the base to empty - // so the new window starts fresh (state == this - // window only), keeping the sketch's shape/config - // intact for the additive apply below. - if dp.start_time_unix_nano != base_window_start { - debug!( - "OTLP delta-sketch window boundary (metric={}, \ - series_key={}, prev_window_start={}, \ - new_window_start={}); rotating per-series base", - metric.name, series_key, base_window_start, dp.start_time_unix_nano - ); - merged.reset_to_empty(); - } - if let Err(e) = apply_modified_otlp_delta_bytes( - dp.algorithm.clone(), - dp.encoding, - &mut merged, - &dp.sketch, - ) { - ingest_state - .observability - .dropped_decode_fail - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - decoded_failed += 1; - debug!( - "OTLP delta-sketch apply failed \ - (metric={}, kind={:?}, encoding={}, \ - bytes={}): {} — falling through to §5.2 \ - fallback", - metric.name, - dp.algorithm, - dp.encoding, - dp.sketch.len(), - e - ); - continue; - } - merged - } else { - match decode_modified_otlp_sketch_bytes( - dp.algorithm.clone(), - dp.encoding, - &dp.sketch, - ) { - Ok(acc) => acc, - Err(e) => { - ingest_state - .observability - .dropped_decode_fail - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - decoded_failed += 1; - let msg = e.to_string(); - // Defensive dim-validation rejections - // (malformed / degenerate CMS / CountSketch - // dims — see `validate_sketch_dims`) signal a - // misbehaving producer, so surface them at - // WARN; ordinary decode fallbacks stay at - // DEBUG to avoid log spam. - if msg.contains("rejecting") { - warn!( - "OTLP modified-proto sketch dropped on dim \ - validation (metric={}, kind={:?}, \ - encoding={}, bytes={}): {}", - metric.name, - dp.algorithm, - dp.encoding, - dp.sketch.len(), - msg - ); - } else { - debug!( - "OTLP modified-proto sketch decode failed \ - (metric={}, kind={:?}, encoding={}, \ - bytes={}): {} — falling through to §5.2 \ - fallback", - metric.name, - dp.algorithm, - dp.encoding, - dp.sketch.len(), - msg - ); - } - continue; - } - } - }; - - // Stateful lineage is committed only after the payload - // has decoded successfully. Everything after this gate - // (snapshot replacement and SketchStore insertion) is - // synchronous/infallible, so an HTTP/gRPC success cannot - // acknowledge a sequence whose payload was never applied. - if let Some(frame) = frame_identity.as_ref() { - match ingest_state.observability.frame_lineage.observe(frame) { - Ok( - crate::precompute_engine::frame_lineage::FrameLineageDecision::Apply, - ) => { - if frame.kind - == asap_types::producer_plan::SummaryFrameKind::Full - { - ingest_state - .summary_store - .clear_summary_lineage_incomplete(sid, frame); - } - } - Ok( - crate::precompute_engine::frame_lineage::FrameLineageDecision::Duplicate, - ) => { - debug!( - plan_id = frame.plan_id, - plan_version = frame.plan_version, - materialization = frame.materialization.as_u64(), - producer = %frame.producer_id, - producer_epoch = %frame.producer_epoch, - sequence = frame.sequence, - "ignored duplicate summary frame" - ); - continue; - } - Err(error) => { - ingest_state - .summary_store - .mark_summary_lineage_incomplete(sid, frame); - warn!( - plan_id = frame.plan_id, - plan_version = frame.plan_version, - materialization = frame.materialization.as_u64(), - producer = %frame.producer_id, - producer_epoch = %frame.producer_epoch, - sequence = frame.sequence, - %error, - "rejected summary frame lineage" - ); - return Err(error.to_string()); - } - } - } - - use crate::storage_engines::sketch_db::index::{ - SketchEncoding, SketchSampleState, - }; - use std::collections::BTreeMap; - let label_values: BTreeMap = dp - .attrs - .iter() - .map(|(key, value)| (key.clone(), value.clone())) - .collect(); - let window: crate::storage_engines::sketch_db::index::epoch_columnar::TimestampRange = ( - dp.start_time_unix_nano / 1_000_000, - dp.time_unix_nano / 1_000_000, - ); - let encoding = - encoding_to_handle(dp.encoding).unwrap_or(SketchEncoding::ProtoFull); - if !ingest_state.summary_store.append_sample( - sid, - label_values, - window, - SketchSampleState { - bytes: dp.sketch.clone(), - encoding, - }, - ) { - return Err("summary window is immutable after completion".into()); - } - - ingest_state.sketch_snapshots.insert( - series_key.clone(), - crate::precompute_engine::ingest_handler::SnapshotCacheEntry { - core: accumulator.clone_boxed_core(), - window_start: dp.start_time_unix_nano, - }, - ); - ingest_state.note_window_and_sweep(dp.start_time_unix_nano); - - // Collect the configs whose metric matches this DP. - // Detection is independent of the legacy dual-write - // (it only drives the routed/unconfigured accounting), - // so we walk it whether or not the worker push fires. - let matching_configs: Vec< - &asap_types::aggregation_config::PrecomputeMaterialization, - > = agg_configs - .values() - .filter(|config| { - config.metric == canonical_name - || config.spatial_filter_normalized == canonical_name - || config.spatial_filter == canonical_name - }) - .collect(); - let matched_any = !matching_configs.is_empty(); - - // CQ-2 — only pay the worker push (and the per-config - // sid resolution + accumulator clone) when the legacy - // dual-write is explicitly enabled. The SketchStore - // `append_sample` above is the live write either way. - if legacy_dual_write { - // B7.6 — bucket key is the per-config bucket sid - // (folds in `(metric, grouping-label-values, - // ExactAgg-of-config)`), NOT the per-DP `sid` - // resolved above. The per-DP `sid` keys the - // `SketchStore::register/append_sample` lane - // (which uses `AggKind::Sketch` to distinguish - // sketch shapes); the worker's group_states are - // keyed per-aggregation-policy bucket, which - // matches what `reconcile_from_streaming_config` - // derives from the same config (so retirement / - // orphan eviction stays consistent). - // - // DEPRECATED routing-side write — kept behind the - // `ASAP_LEGACY_DUAL_WRITE` gate until the ASAP-tier - // query reducer is validated end-to-end. - let n = matching_configs.len(); - // PERF-2 — carry the owned accumulator so the LAST - // matching config can MOVE it into its message - // instead of cloning. `Some` until consumed; the - // common single-match case pays zero clones. - let mut owned_acc: Option> = Some(accumulator); - for (i, config) in matching_configs.iter().enumerate() { - // PERF-4 — group key straight from `dp.attrs` - // (no format_series_key → parse round-trip). - let group_key = - IngestState::extract_group_key_from_labels(&dp.attrs, config); - // PERF-3 — `dp.attrs` is already a - // `HashMap`; pass it directly - // instead of rebuilding `attrs_map` per config. - let (bucket_sid, policy_fp) = match resolve_bucket_sid_for_agg_config( - ingest_state, - config, - &dp.attrs, - catalog_generation.as_deref(), - ) { - Ok(binding) => binding, - Err(error) => { - warn!(%error, "configured ingest series reactivation rejected"); - continue; - } - }; - let acc_for_msg = if i + 1 == n { - // Last (or only) match — move the owned - // accumulator out, no clone. - owned_acc.take().expect("owned_acc present on last match") - } else { - // 2nd..Nth match — clone from the still-owned - // accumulator (cloning a sketch is a msgpack - // round-trip, so we only pay it when a DP - // genuinely fans out to multiple policies). - owned_acc - .as_ref() - .expect("owned_acc present before last match") - .clone_boxed_core() - }; - messages.push(WorkerMessage::AccumulatorInput { - sid: bucket_sid, - policy_fp, - group_key, - timestamp_ms: ts_ms, - accumulator: acc_for_msg, - ingest_received_at, - }); - } - } - - if matched_any { - routed += 1; - } else { - // CQ-6 — a decoded sketch that matched no - // PrecomputeMaterialization in the running streaming config. - ingest_state - .observability - .dropped_unconfigured - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - unconfigured += 1; - } - } - } - } - } - - flush_barrier_drops(ingest_state, &barrier_drops, "otlp-modified-proto"); - - // No lineage-protected store mutation occurs after this point. Do not - // carry a synchronous mutex guard across the async worker-queue flush. - drop(lineage_batch_guard); - - if !messages.is_empty() { - if let Err(e) = ingest_state - .router - .route_group_batch(messages, ingest_received_at, catalog_generation.clone()) - .await - { - warn!("OTLP modified-proto sketch routing error: {}", e); - } - } - - if routed + decoded_failed + unconfigured > 0 { - debug!( - "OTLP modified-proto sketch ingest: {} routed, {} decode-failed (fallback), {} unconfigured", - routed, decoded_failed, unconfigured - ); - } - - Ok(IngestOutcome { - unknown_series_ids: unknown_sids, - series_assignments: new_assignments, - }) -} - -/// Map `SketchAlgorithm` to the corresponding wire-format -/// `AggregationType`. Inverse direction is in -/// `sketch_algorithm_for` above. Used by -/// [`derive_sketch_policy_fp`] to find the policy whose -/// `PrecomputeMaterialization.aggregation_type` matches a freshly-ingested -/// sketch. -/// -/// `Any` is a control-plane analysis-time wildcard — it doesn't -/// appear on the ingest path. Returns `None` so the policy lookup -/// fails the (rare) defensive path explicitly. -fn aggregation_type_for_sketch_algorithm( - handle: crate::storage_engines::sketch_db::index::SketchAlgorithm, -) -> Option { - use crate::storage_engines::sketch_db::index::SketchAlgorithm; - use asap_types::AggregationType; - match handle { - SketchAlgorithm::DDSketch => Some(AggregationType::DDSketch), - SketchAlgorithm::Kll => Some(AggregationType::DatasketchesKLL), - SketchAlgorithm::Hll => Some(AggregationType::HLL), - SketchAlgorithm::UnivMon => Some(AggregationType::UnivMon), - SketchAlgorithm::CountSketch => Some(AggregationType::CountSketch), - SketchAlgorithm::CountSketchWithHeap => Some(AggregationType::CountSketchWithHeap), - SketchAlgorithm::Cms => Some(AggregationType::CountMinSketch), - SketchAlgorithm::CmsWithHeap => Some(AggregationType::CountMinSketchWithHeap), - SketchAlgorithm::Kmv | SketchAlgorithm::Theta => None, - } -} - -/// Render a `SketchConfig` into the param map the streaming-config -/// stores. The control plane authors these as -/// `parameters: {: }` JSON; the data plane has the -/// parameters typed in `SketchConfig`. This function converts. -/// -/// Keys MUST match what the control plane emits (see -/// `crates/asap_types/src/aggregation_config.rs::from_yaml_data` for -/// the canonical names). Drift here surfaces as policy lookups that -/// silently miss. -fn sketch_config_to_params( - cfg: &crate::storage_engines::sketch_db::data::SketchConfig, -) -> std::collections::HashMap { - use crate::storage_engines::sketch_db::data::SketchConfig; - let mut params = std::collections::HashMap::new(); - match cfg { - SketchConfig::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - } => { - params.insert("heap_size".into(), serde_json::json!(heap_size)); - params.insert("sketch_rows".into(), serde_json::json!(sketch_rows)); - params.insert("sketch_cols".into(), serde_json::json!(sketch_cols)); - params.insert("layers".into(), serde_json::json!(layers)); - } - SketchConfig::DDSketch { relative_accuracy } => { - params.insert( - "relative_accuracy".to_string(), - serde_json::json!(*relative_accuracy), - ); - } - SketchConfig::Kll { k } => { - params.insert("k".to_string(), serde_json::json!(*k)); - } - SketchConfig::Hll { precision } => { - params.insert("precision".to_string(), serde_json::json!(*precision)); - } - SketchConfig::CountSketch { rows, cols } | SketchConfig::CountMin { rows, cols } => { - // Canonical key mapping (matches the controller's - // `sketch_params_to_json` in - // `control_plane::emit::stage_config`): `w` is the - // matrix width (=cols), `d` is the depth (=rows). The - // controller writes `{w, d}` into the streaming-config - // `parameters`, so the policy_fp content match has to - // probe the same keys. - params.insert("w".to_string(), serde_json::json!(*cols)); - params.insert("d".to_string(), serde_json::json!(*rows)); - } - } - params -} - -// Snapshot and delta bases are scoped to the producer, never only its semantics. -fn bound_sketch_series_key( - name: &str, - labels: &HashMap, - frame: Option<&asap_types::producer_plan::SummaryFrameIdentity>, -) -> String { - let key = format_series_key(name, labels); - match frame { - Some(frame) => format!( - "{}:{}:{}:{key}", - frame.plan_id, - frame.plan_version, - frame.materialization.as_u64() - ), - None => key, - } -} - -/// Resolve the framed deployed output, then verify its content contract. -/// Unframed input uses legacy content matching and must have exactly one match. -/// An absent or ambiguous match returns UNSET and cannot authorize bound writes. -fn derive_sketch_policy_fp( - ingest_state: &IngestState, - metric: &str, - kind: crate::storage_engines::sketch_db::index::SketchAlgorithm, - cfg: &crate::storage_engines::sketch_db::data::SketchConfig, - group_by_keys: &std::collections::BTreeSet, - bound_output: Option, -) -> asap_types::PolicyFingerprint { - let Some(agg_type) = aggregation_type_for_sketch_algorithm(kind) else { - return asap_types::PolicyFingerprint::UNSET; - }; - let params = sketch_config_to_params(cfg); - let snap = ingest_state.config_snapshot(); - let registry = snap.policy_registry(); - let registry = if let Some(output) = bound_output { - asap_types::PolicyRegistry::from_configs(registry.get(output.fingerprint()).cloned()) - } else { - registry - }; - let index = asap_types::RoutingIndex::build(registry); - index - .find_policy_by_content(metric, group_by_keys, agg_type, ¶ms) - .unwrap_or(asap_types::PolicyFingerprint::UNSET) -} - -/// Phase 5 helper — map a `ModifiedOtlpSketchDp` to the matching -/// `SketchAlgorithm` so registration and capability classification -/// share one source of truth. -/// -/// CMS-with-heap detection: the OTLP `CountMinSketch` wire struct -/// itself doesn't carry a top-k heap field (see metrics.proto -/// `CountMinSketch`/`CountMinSketchDataPoint`). The heap is embedded -/// inside the msgpack-encoded `CountMinSketchWithHeapSerialized` -/// payload (an outer `{sketch, topk_heap, heap_size}` wrapper). When -/// the encoding is MSGPACK and the bytes round-trip via -/// `CountMinSketchWithHeap::deserialize_msgpack`, we classify the sid -/// as `CmsWithHeap` so the ASAP-tier reducer can later read the heap -/// directly for `topk` / `topk_over_time` queries. -/// Strip the agent-side sketch-family name suffix from an OTLP sketch -/// metric name, returning the *raw* metric name the controller's -/// streaming-config policies and user PromQL are keyed on. -/// -/// ## Why this exists -/// -/// The ASAPCollector edge pipeline's fused `asapedgeprocessor` -/// (`processor/asapedgeprocessor/sketch.go`) sets -/// `MetricSuffix: "_" + family` on every sketch it emits, so a KLL -/// sketch over `request_size_bytes` arrives on the wire named -/// `request_size_bytes_kll`, an HLL over `unique_users_per_min` arrives -/// as `unique_users_per_min_hll`, and so on. -/// -/// Both the controller (which plans + pushes streaming-config policies -/// keyed on the *bare* metric `request_size_bytes`) and the query -/// analyzer (`control_plane::asap_tier_analysis`, which lifts the bare -/// metric name out of the PromQL selector) speak the bare name. With -/// the suffix left on, `SummarySeriesMetadata.metric_name` is the -/// suffixed form, so `SketchIndex::instances_matching(bare, …)` and -/// `find_matching_policies` / `find_policy_by_content` (all of which -/// compare `metric_name` for equality) never match — every warm sketch -/// query (`quantile_over_time`, `count`/HLL, `topk`) capability-misses -/// and the user sees `data_source: asap_query, "No result"`. -/// -/// Per `docs/design_docs/series-identity.md`, the summary *family* -/// is a wire-level attribute (carried here in `agg_kind` / -/// [`SketchAlgorithm`]), NOT a name suffix; storage + query must be -/// keyed on the raw SDK metric name. This helper applies that -/// canonicalization at the ingest seam so the backend resolves -/// correctly regardless of whether the deployed agent still suffixes. -/// -/// The strip is gated on the suffix matching the datapoint's *actual* -/// sketch kind, so a metric a user legitimately named `foo_hll` that -/// arrives as a KLL sketch is left untouched, and the operation is a -/// no-op (and therefore safe / idempotent) once agents stop suffixing. -fn canonical_sketch_metric_name<'a>(name: &'a str, algorithm: SketchAlgorithm) -> &'a str { - let suffix: &str = match algorithm { - SketchAlgorithm::DDSketch => "_ddsketch", - SketchAlgorithm::Kll => "_kll", - SketchAlgorithm::Hll => "_hll", - SketchAlgorithm::CountSketch => "_countsketch", - SketchAlgorithm::Cms => "_countminsketch", - // These algorithms do not currently have modified-OTLP - // containers in this receiver, so there is no suffix to strip. - _ => return name, - }; - // Only strip when there's a non-empty base left over (so a metric - // literally named `_kll` is never collapsed to the empty string). - match name.strip_suffix(suffix) { - Some(base) if !base.is_empty() => base, - _ => name, - } -} - -/// P1-4 — does this frequency-sketch DataPoint carry a non-empty top-k -/// heap? Detects the heap on BOTH heap-bearing wire encodings: -/// -/// * `ENCODING_MSGPACK` — full heap-bearing frame -/// (`{sketch, topk_heap, heap_size}`); decoded with -/// `CountMinSketchWithHeap::from_msgpack`. -/// * `ENCODING_MSGPACK_DELTA` — DELTA-HEAP frame (sparse matrix delta + -/// the FULL top-k heap); decoded via the heap accumulator's -/// `from_msgpack_heap_delta_bytes` (the same generic `rmp_serde` -/// reader the apply path uses). This lets a sid whose FIRST frame is -/// a delta still be detected as heap-bearing and registered/upgraded -/// to a top-k capability — part (b) of P1-4. -/// -/// PROTO / PROTO_DELTA frequency frames don't carry a heap on the wire, -/// so they always read as heap-LESS here. -fn dp_carries_heap(dp: &ModifiedOtlpSketchDp) -> bool { - match dp.encoding { - ENCODING_MSGPACK => { - use asap_sketchlib::CountMinSketchWithHeap; - CountMinSketchWithHeap::from_msgpack(&dp.sketch) - .map(|cms| !cms.topk_heap_items().is_empty()) - .unwrap_or(false) - } - ENCODING_MSGPACK_DELTA => { - use asap_physical_operators::summary_kernels::CountMinSketchWithHeapAccumulator; - CountMinSketchWithHeapAccumulator::from_msgpack_heap_delta_bytes(&dp.sketch) - .map(|acc| !acc.inner.topk_heap_items().is_empty()) - .unwrap_or(false) - } - _ => false, - } -} - -/// P1-4 — collapse a heap-BEARING frequency handle to its heap-LESS base -/// family. Used for SID IDENTITY so a heap-bearing frame and a heap-less -/// frame for the same `(metric, attrs, config)` resolve to ONE sid (the -/// heap is enrichment on the same substrate, not a different series). The -/// CAPABILITY still tracks the heap via the metadata upgrade path. All -/// other handles pass through unchanged. -fn base_sketch_algorithm( - kind: crate::storage_engines::sketch_db::index::SketchAlgorithm, -) -> crate::storage_engines::sketch_db::index::SketchAlgorithm { - use crate::storage_engines::sketch_db::index::SketchAlgorithm; - match kind { - SketchAlgorithm::CmsWithHeap => SketchAlgorithm::Cms, - SketchAlgorithm::CountSketchWithHeap => SketchAlgorithm::CountSketch, - other => other, - } -} - -fn sketch_algorithm_for( - dp: &ModifiedOtlpSketchDp, -) -> crate::storage_engines::sketch_db::index::SketchAlgorithm { - use crate::storage_engines::sketch_db::index::SketchAlgorithm; - match dp.algorithm.clone() { - SketchAlgorithm::DDSketch => SketchAlgorithm::DDSketch, - SketchAlgorithm::Kll => SketchAlgorithm::Kll, - SketchAlgorithm::Hll => SketchAlgorithm::Hll, - SketchAlgorithm::CountSketch => { - // Mirror the CountMin branch: CountSketch-with-heap - // payloads share the same outer msgpack envelope - // (`CountMinSketchWithHeapSerialized` — see the comment - // in `sketch_reducer.rs` at the dispatch site, which - // notes both heap-bearing variants reuse this wire - // shape since the heap is the distinguishing payload). - // Auto-promote to `CountSketchWithHeap` when the bytes - // decode AND the heap is non-empty; otherwise stay with - // vanilla `CountSketch`. - if dp_carries_heap(dp) { - return SketchAlgorithm::CountSketchWithHeap; - } - SketchAlgorithm::CountSketch - } - SketchAlgorithm::Cms => { - // Try a no-cost peek: msgpack-encoded CMS-with-heap payloads - // round-trip through asap_sketchlib's - // `CountMinSketchWithHeap::deserialize_msgpack`. If the - // sketch bytes decode against that wrapper *and* the - // resulting heap is non-empty, treat the sid as - // CmsWithHeap so ASAP-tier `topk` can read the heap. - // Otherwise stay with vanilla `CountMin`. - if dp_carries_heap(dp) { - return SketchAlgorithm::CmsWithHeap; - } - SketchAlgorithm::Cms - } - other => other, - } -} - -/// Phase 5 helper — translate the wire-format `encoding` integer to the -/// SketchStore's `SketchEncoding` enum. Returns `None` for the unset -/// (0) encoding so callers can default to `ProtoFull` (the dominant -/// case for full-state frames). -fn encoding_to_handle( - encoding: i32, -) -> Option { - use crate::storage_engines::sketch_db::index::SketchEncoding; - match encoding { - ENCODING_PROTO => Some(SketchEncoding::ProtoFull), - ENCODING_PROTO_DELTA => Some(SketchEncoding::ProtoDelta), - ENCODING_MSGPACK => Some(SketchEncoding::MsgpackFull), - ENCODING_MSGPACK_DELTA => Some(SketchEncoding::MsgpackDelta), - _ => None, - } -} - -/// A single modified-OTLP sketch data point flattened across the five -/// per-variant data-point types so the routing loop can treat them -/// uniformly. -struct ModifiedOtlpSketchDp { - algorithm: SketchAlgorithm, - attrs: HashMap, - time_unix_nano: u64, - sketch: Vec, - encoding: i32, - /// sender-supplied series_id, 0 when unset / first emit. - /// Backend's resolver mints a fresh sid when this is 0 with attrs - /// populated; pushes the sid into `unknown_series_ids` when this is - /// non-zero with empty attrs and the resolver doesn't recognize it. - series_id: u64, - /// DataPoint-level start of the sketch window. Combined - /// with `time_unix_nano` to form the `(start_ms, end_ms)` window - /// the SketchStore's columnar storage keys on. - start_time_unix_nano: u64, - /// sketch-instance configuration lifted off the parent - /// container. Drives `SummarySeriesMetadata.sketch_config` and the - /// derived `AccuracyBound`. - container_config: crate::storage_engines::sketch_db::index::SketchConfig, -} - -/// Validate every first-class summary frame in an OTLP request before the -/// ingest loop performs any externally visible mutation. The transport -/// response is therefore the acknowledgement boundary; there is no second -/// application-level ACK protocol. -fn preflight_summary_frames( - request: &ExportMetricsServiceRequest, - ingest_state: &IngestState, - active: &crate::storage_engines::types::RuntimePhysicalPlan, -) -> Result<(), String> { - use asap_otel_proto::tonic::metrics::v1::metric::Data; - - fn validate_one( - metric_name: &str, - mut dp: ModifiedOtlpSketchDp, - ingest_state: &IngestState, - active: &crate::storage_engines::types::RuntimePhysicalPlan, - ) -> Result { - let canonical_name = canonical_sketch_metric_name(metric_name, dp.algorithm.clone()); - let frame = - take_summary_frame_identity(&mut dp.attrs, dp.start_time_unix_nano, dp.time_unix_nano)?; - if state_encoding_for_wire(dp.encoding) != Some(frame.encoding.clone()) { - return Err(format!( - "summary frame for {metric_name} declares an encoding different from its payload" - )); - } - active - .transmission_plan - .validate_frame(&frame) - .map_err(|error| error.to_string())?; - - if active - .precompute_plan - .materializations - .iter() - .any(|config| { - config.policy_fingerprint() == frame.materialization.fingerprint() - && !config.population_key_encoding.is_legacy() - }) - { - return Err( - "canonical population is not supported by modified-OTLP summary routing".into(), - ); - } - - let schema = active - .precompute_plan - .schemas - .iter() - .find(|schema| schema.materialization == frame.materialization) - .ok_or_else(|| format!("summary frame for {metric_name} has no active schema"))?; - if dp.attrs.is_empty() && !schema.group_by.is_empty() { - return Err(format!( - "summary frame for {metric_name} omits labels required by its grouped schema" - )); - } - let observed_series = if dp.attrs.is_empty() { - "".to_string() - } else { - let pairs: Vec<(&str, &str)> = dp - .attrs - .iter() - .map(|(key, value)| (key.as_str(), value.as_str())) - .collect(); - crate::drivers::ingest::canonical_attrs_fingerprint(&pairs) - }; - if observed_series != frame.series_identity { - return Err(format!( - "summary frame for {metric_name} declares a series identity different from its labels" - )); - } - - // A malformed full snapshot must not be discovered after an earlier - // frame in the request has already reached SketchStore. - if frame.kind == asap_types::producer_plan::SummaryFrameKind::Full { - decode_modified_otlp_sketch_bytes(dp.algorithm.clone(), dp.encoding, &dp.sketch) - .map_err(|error| format!("invalid full frame for {metric_name}: {error}"))?; - } else { - let series_key = bound_sketch_series_key(canonical_name, &dp.attrs, Some(&frame)); - let (mut base, base_window_start) = ingest_state - .sketch_snapshots - .get(&series_key) - .map(|entry| (entry.core.clone_boxed_core(), entry.window_start)) - .or_else(|| { - empty_accumulator_for_delta_bootstrap( - dp.algorithm.clone(), - &dp.container_config, - dp.encoding, - ) - .map(|base| (base, dp.start_time_unix_nano)) - }) - .ok_or_else(|| { - format!("delta frame for {metric_name} has no reconstructable base") - })?; - if base_window_start != dp.start_time_unix_nano { - base.reset_to_empty(); - } - apply_modified_otlp_delta_bytes( - dp.algorithm.clone(), - dp.encoding, - &mut base, - &dp.sketch, - ) - .map_err(|error| format!("invalid delta frame for {metric_name}: {error}"))?; - } - - // Attribute-elided retries can recover the policy from their known - // SID. Attribute-bearing frames derive it from the active physical - // schema. Either route must agree with the declared materialization. - let observed = if dp.series_id != 0 && dp.attrs.is_empty() { - ingest_state - .summary_store - .instance(dp.series_id) - .map(|metadata| metadata.policy_fp) - .ok_or_else(|| { - format!( - "summary frame for {metric_name} references unknown sid {} without labels", - dp.series_id - ) - })? - } else { - derive_sketch_policy_fp( - ingest_state, - canonical_name, - sketch_algorithm_for(&dp), - &dp.container_config, - &dp.attrs.keys().cloned().collect(), - Some(frame.materialization), - ) - }; - if observed != frame.materialization.fingerprint() { - return Err(format!( - "summary frame for {metric_name} declares materialization {} but active schema resolves {}", - frame.materialization.as_u64(), observed.0 - )); - } - Ok(frame) - } - - let mut frames = Vec::new(); - for resource_metrics in &request.resource_metrics { - let resource_attrs = resource_metrics - .resource - .as_ref() - .map(|resource| attributes_to_map(&resource.attributes)) - .unwrap_or_default(); - for scope_metrics in &resource_metrics.scope_metrics { - let scope_attrs = scope_metrics - .scope - .as_ref() - .map(|scope| attributes_to_map(&scope.attributes)) - .unwrap_or_default(); - for metric in &scope_metrics.metrics { - let base_labels: HashMap = scope_attrs - .iter() - .chain(resource_attrs.iter()) - .map(|(key, value)| (key.clone(), value.clone())) - .collect(); - macro_rules! validate_points { - ($points:expr, $algorithm:expr, $config:expr) => {{ - let config = $config; - for point in &$points { - frames.push(validate_one( - &metric.name, - ModifiedOtlpSketchDp { - algorithm: $algorithm, - attrs: merge_point_attributes(&base_labels, &point.attributes), - time_unix_nano: point.time_unix_nano, - sketch: point.sketch.clone(), - encoding: point.encoding, - series_id: point.series_id, - start_time_unix_nano: point.start_time_unix_nano, - container_config: config.clone(), - }, - ingest_state, - active, - )?); - } - }}; - } - match &metric.data { - Some(Data::Ddsketch(data)) => validate_points!( - data.data_points, - SketchAlgorithm::DDSketch, - crate::storage_engines::sketch_db::index::SketchConfig::DDSketch { - relative_accuracy: data.relative_accuracy, - } - ), - Some(Data::Kllsketch(data)) => validate_points!( - data.data_points, - SketchAlgorithm::Kll, - crate::storage_engines::sketch_db::index::SketchConfig::Kll { k: data.k } - ), - Some(Data::Countsketch(data)) => validate_points!( - data.data_points, - SketchAlgorithm::CountSketch, - crate::storage_engines::sketch_db::index::SketchConfig::CountSketch { - rows: data.rows, - cols: data.cols, - } - ), - Some(Data::Countminsketch(data)) => validate_points!( - data.data_points, - SketchAlgorithm::Cms, - crate::storage_engines::sketch_db::index::SketchConfig::CountMin { - rows: data.rows, - cols: data.cols, - } - ), - Some(Data::Hllsketch(data)) => validate_points!( - data.data_points, - SketchAlgorithm::Hll, - crate::storage_engines::sketch_db::index::SketchConfig::Hll { - precision: data.precision, - } - ), - _ => {} - } - } - } - } - ingest_state - .observability - .frame_lineage - .validate_batch(frames.iter()) - .map_err(|error| error.to_string())?; - Ok(()) -} - -fn take_summary_frame_identity( - attrs: &mut HashMap, - window_start_unix_nano: u64, - window_end_unix_nano: u64, -) -> Result { - use asap_types::producer_plan::{SummaryFrameIdentity, SummaryFrameKind}; - use control_plane::physical::compiler::StateEncoding; - - fn required(attrs: &mut HashMap, key: &str) -> Result { - attrs - .remove(key) - .filter(|value| !value.is_empty()) - .ok_or_else(|| format!("missing {key}")) - } - fn number(attrs: &mut HashMap, key: &str) -> Result { - required(attrs, key)? - .parse() - .map_err(|_| format!("invalid {key}")) - } - - let identity_version = u32::try_from(number(attrs, "asap.frame.identity_version")?) - .map_err(|_| "invalid asap.frame.identity_version".to_string())?; - let plan_id = number(attrs, "asap.frame.plan_id")?; - let plan_version = number(attrs, "asap.frame.plan_version")?; - let backend_compat = required(attrs, "asap.frame.backend_compat")?; - let materialization = - asap_types::PolicyFingerprint(number(attrs, "asap.frame.materialization")?); - let series_identity = required(attrs, "asap.frame.series_identity")?; - let schema_id = required(attrs, "asap.frame.schema_id")?; - let producer_id = required(attrs, "asap.frame.producer_id")?; - let producer_epoch = required(attrs, "asap.frame.producer_epoch")?; - let sequence = number(attrs, "asap.frame.sequence")?; - let kind = match required(attrs, "asap.frame.kind")?.as_str() { - "full" => SummaryFrameKind::Full, - "delta" => SummaryFrameKind::Delta, - _ => return Err("invalid asap.frame.kind".into()), - }; - let encoding = match required(attrs, "asap.frame.encoding")?.as_str() { - "sketchlib_protobuf_v1" => StateEncoding::SketchlibProtobufV1, - "sketch_core_msgpack_v1" => StateEncoding::SketchCoreMsgpackV1, - "exact_accumulator_v1" => StateEncoding::ExactAccumulatorV1, - _ => return Err("invalid asap.frame.encoding".into()), - }; - let checkpoint_id = attrs - .remove("asap.frame.checkpoint_id") - .filter(|value| !value.is_empty()); - let base_checkpoint_id = attrs - .remove("asap.frame.base_checkpoint_id") - .filter(|value| !value.is_empty()); - - Ok(SummaryFrameIdentity { - identity_version, - plan_id, - plan_version, - backend_compat, - materialization: materialization.into(), - series_identity, - schema_id, - producer_id, - producer_epoch, - window_start_unix_nano, - window_end_unix_nano, - sequence, - kind, - encoding, - checkpoint_id, - base_checkpoint_id, - }) -} - -fn state_encoding_for_wire(encoding: i32) -> Option { - use asap_types::precompute_plan::StateEncoding; - match encoding { - ENCODING_PROTO | ENCODING_PROTO_DELTA => Some(StateEncoding::SketchlibProtobufV1), - ENCODING_MSGPACK | ENCODING_MSGPACK_DELTA => Some(StateEncoding::SketchCoreMsgpackV1), - _ => None, - } -} - -/// Decode the typed `sketch` bytes from a modified-OTLP -/// `*SketchDataPoint` into a concrete `AggregateCore`. -/// -/// Dispatches on the `(SketchAlgorithm, encoding)` pair. For each -/// `(algorithm, _ENCODING_PROTO)` pair we call the matching accumulator's -/// `from_sketchlib_proto_bytes` constructor. Variants without a -/// constructor today return `Err`; the caller falls through to §5.2 -/// fallback so the user still gets a correct answer. Per-variant -/// decoders are tracked in PR C (task #8) and PR I (task #14, for -/// `_ENCODING_MSGPACK` parity). -fn decode_modified_otlp_sketch_bytes( - algorithm: SketchAlgorithm, - encoding: i32, - bytes: &[u8], -) -> Result, Box> { - use asap_physical_operators::summary_kernels::{ - CountMinSketchAccumulator, CountSketchAccumulator, DDSketchAccumulator, - DatasketchesKLLAccumulator, HllSketchAccumulator, - }; - - // The encoding value is the raw i32 from the per-sketch encoding - // enum. All five sketch variants share the same wire tag layout: - // - // 1 — ENCODING_PROTO (full sketchlib proto state) - // 2 — ENCODING_PROTO_DELTA (sparse diff vs caller's base - // snapshot — apply via - // `apply_modified_otlp_delta_bytes`; - // not standalone-decodable) - // 3 — ENCODING_MSGPACK (full sketch-core msgpack state) - // 4 — ENCODING_MSGPACK_DELTA (MSGPACK diff; not yet wired) - - match encoding { - ENCODING_PROTO => match algorithm { - // The neutral codec accepts both full envelopes and supported bare - // states. Query accumulators retain their family-specific readouts. - SketchAlgorithm::DDSketch => { - let (inner, sample_p) = asap_sketch_codec::reconstruct_ddsketch(bytes)?; - let sample_p = if sample_p.is_finite() && sample_p > 0.0 && sample_p < 1.0 { - sample_p - } else { - 1.0 - }; - Ok(Box::new(DDSketchAccumulator { inner, sample_p })) - } - SketchAlgorithm::Kll => Ok(Box::new( - DatasketchesKLLAccumulator::from_sketchlib_proto_bytes(bytes)?, - )), - SketchAlgorithm::Cms => Ok(Box::new( - CountMinSketchAccumulator::from_sketchlib_proto_bytes(bytes)?, - )), - SketchAlgorithm::CountSketch => Ok(Box::new( - CountSketchAccumulator::from_sketchlib_proto_bytes(bytes)?, - )), - SketchAlgorithm::Hll => Ok(Box::new(HllSketchAccumulator::from_sketchlib_proto_bytes( - bytes, - )?)), - other => { - Err(format!("modified-OTLP PROTO decoding is not implemented for {other:?}").into()) - } - }, - ENCODING_MSGPACK => match algorithm { - SketchAlgorithm::Cms => Ok(Box::new(CountMinSketchAccumulator::from_msgpack_bytes( - bytes, - )?)), - SketchAlgorithm::CountSketch => { - // Heap-bearing CountSketch full frame: the bytes are the - // `{sketch,topk_heap,heap_size}` envelope (a DIFFERENT inner - // field order than the plain CountSketch msgpack), so - // `CountSketch::from_msgpack` can't parse it. Try the heap - // decode FIRST when the heap is non-empty (the same promotion - // gate `sketch_algorithm_for` uses); cache THAT heap - // accumulator as the per-series base so a later MSGPACK_DELTA - // frame applies its matrix delta + heap onto a heap - // accumulator. Fall back to the plain CountSketch decode for - // heap-less msgpack frames (byte-parity path, PR I). - // - // Uses the real `CountSketchWithHeap` (median-of-signed-rows), - // NOT `CountMinSketchWithHeap` — the two share the same wire - // envelope shape (structural peek only), but decoding a real - // CountSketch's matrix through the CMS wrapper would silently - // apply CMS's min-of-rows math to CountSketch data forever - // after (the same conflation bug fixed on the write side in - // `accumulator_factory.rs`). - use asap_sketchlib::CountSketchWithHeap; - if let Ok(heap) = CountSketchWithHeap::from_msgpack(bytes) { - if !heap.topk_heap_items().is_empty() { - use asap_physical_operators::summary_kernels::CountSketchWithHeapAccumulator; - return Ok(Box::new( - CountSketchWithHeapAccumulator::from_msgpack_with_heap_bytes(bytes)?, - )); - } - } - Ok(Box::new(CountSketchAccumulator::from_msgpack_bytes(bytes)?)) - } - SketchAlgorithm::Kll => Ok(Box::new(DatasketchesKLLAccumulator::from_msgpack_bytes( - bytes, - )?)), - SketchAlgorithm::DDSketch => { - Ok(Box::new(DDSketchAccumulator::from_msgpack_bytes(bytes)?)) - } - SketchAlgorithm::Hll => Ok(Box::new(HllSketchAccumulator::from_msgpack_bytes(bytes)?)), - other => Err(format!( - "modified-OTLP MSGPACK decoding is not implemented for {other:?}" - ) - .into()), - }, - ENCODING_PROTO_DELTA => Err(format!( - "sketch encoding PROTO_DELTA (2) is not standalone-decodable — \ - it carries only a diff against the caller's base snapshot. \ - Caller must route these through \ - `apply_modified_otlp_delta_bytes` with a cached accumulator; \ - this decoder is for full-state frames only." - ) - .into()), - ENCODING_MSGPACK_DELTA => Err(format!( - "sketch encoding MSGPACK_DELTA (4) deferred — PR G wires \ - PROTO_DELTA only; msgpack delta is a follow-up." - ) - .into()), - _ => Err(format!( - "unknown modified-OTLP sketch encoding {encoding} \ - (expected 1 / 2 / 3 / 4)" - ) - .into()), - } -} - -/// P1-1/P1-2 — construct an EMPTY accumulator matching a sketch -/// DataPoint's `(kind, container_config)`, for bootstrapping a delta -/// frame that arrives with no cached base. -/// -/// Under the per-window-reset contract (`docs/delta-baseline-contract.md` -/// §3) each window's delta encodes that window's own state against an -/// EMPTY base. So a leading delta (no prior full frame — e.g. the very -/// first frame for a sid, or the first frame after a backend restart -/// dropped the in-memory snapshot cache) is reconstructable for the -/// ADDITIVE families by `empty(dims) + apply(delta)`. This mirrors the -/// warm-read tier's standalone delta reconstruction so the worker tier -/// agrees with it and recovers after restart, instead of dropping the -/// delta. -/// -/// Returns `None` for families that CANNOT reconstruct from a bare delta -/// (DDSketch and KLL): DDSketch deltas are bucket-index diffs whose -/// absolute store layout depends on the base's offset, and KLL never -/// deltas. Those keep the unchanged "drop until the next full frame" -/// behavior. -fn empty_accumulator_for_delta_bootstrap( - algorithm: SketchAlgorithm, - config: &crate::storage_engines::sketch_db::index::SketchConfig, - encoding: i32, -) -> Option> { - use crate::storage_engines::sketch_db::index::SketchConfig; - use asap_physical_operators::summary_kernels::{ - CountMinSketchAccumulator, CountSketchAccumulator, CountSketchWithHeapAccumulator, - HllSketchAccumulator, - }; - - match (algorithm, config) { - (SketchAlgorithm::Hll, SketchConfig::Hll { precision }) => { - use asap_sketchlib::HllVariant; - // Regular is the default agent variant; HLL's additive delta - // merge tolerates an empty same-precision base. - Some(Box::new(HllSketchAccumulator::new( - HllVariant::Regular, - *precision, - ))) - } - (SketchAlgorithm::Cms, SketchConfig::CountMin { rows, cols }) => Some(Box::new( - CountMinSketchAccumulator::new(*rows as usize, *cols as usize), - )), - (SketchAlgorithm::CountSketch, SketchConfig::CountSketch { rows, cols }) => { - // A heap-bearing DELTA-HEAP frame must reconstruct onto a heap - // accumulator (the apply path downcasts to - // `CountSketchWithHeapAccumulator`); a plain matrix delta - // reconstructs onto a vanilla CountSketch. Pick the base shape - // from the encoding so the subsequent - // `apply_modified_otlp_delta_bytes` downcast succeeds. - if encoding == ENCODING_MSGPACK_DELTA { - // heap_size 0 is fine — the DELTA-HEAP apply REPLACES the - // heap wholesale from the frame's full heap. - Some(Box::new(CountSketchWithHeapAccumulator::new( - *rows as usize, - *cols as usize, - 0, - ))) - } else { - Some(Box::new(CountSketchAccumulator::new( - *rows as usize, - *cols as usize, - ))) - } - } - // DDSketch / KLL (and any config/kind mismatch) — not - // reconstructable from a bare delta. Caller keeps the unchanged - // drop behavior. - _ => None, - } -} - -// Shared constants — exposed at module scope so both the full-state -// decoder and the delta applier match on the same values. -const ENCODING_PROTO: i32 = 1; -const ENCODING_PROTO_DELTA: i32 = 2; -const ENCODING_MSGPACK: i32 = 3; -const ENCODING_MSGPACK_DELTA: i32 = 4; - -/// Apply a modified-OTLP `*SketchDataPoint.sketch` delta frame onto an -/// existing accumulator. -/// -/// Paper §6.2 B3 / B4 sketch delta-transmission: the agent sends a -/// sparse diff against its last-flushed snapshot. The backend keeps a -/// per-series accumulator around, and on arrival of a delta frame -/// dispatches here to merge the diff in place. -/// -/// The caller owns the per-series snapshot cache — this dispatcher is -/// stateless. Today wires `PROTO_DELTA` for DDSketch + HLL (the two -/// delta-capable sketches in `sketchlib-go`); `MSGPACK_DELTA` and the -/// KLL/CountSketch/CountMinSketch deltas are deferred to follow-ups -/// as their delta codecs land. -pub(crate) fn apply_modified_otlp_delta_bytes( - algorithm: SketchAlgorithm, - encoding: i32, - existing: &mut Box, - bytes: &[u8], -) -> Result<(), Box> { - use asap_physical_operators::summary_kernels::{ - CountMinSketchAccumulator, CountSketchAccumulator, CountSketchWithHeapAccumulator, - DDSketchAccumulator, HllSketchAccumulator, - }; - - match (encoding, algorithm) { - (ENCODING_PROTO_DELTA, SketchAlgorithm::DDSketch) => { - let dd = existing - .as_any_mut() - .downcast_mut::() - .ok_or( - "apply_modified_otlp_delta_bytes: existing accumulator is \ - not a DDSketchAccumulator", - )?; - dd.apply_proto_delta_bytes(bytes) - } - (ENCODING_PROTO_DELTA, SketchAlgorithm::Hll) => { - let hll = existing - .as_any_mut() - .downcast_mut::() - .ok_or( - "apply_modified_otlp_delta_bytes: existing accumulator is \ - not an HllSketchAccumulator", - )?; - hll.apply_proto_delta_bytes(bytes) - } - (ENCODING_PROTO_DELTA, SketchAlgorithm::CountSketch) => { - let cs = existing - .as_any_mut() - .downcast_mut::() - .ok_or( - "apply_modified_otlp_delta_bytes: existing accumulator is \ - not a CountSketchAccumulator", - )?; - cs.apply_proto_delta_bytes(bytes) - } - (ENCODING_PROTO_DELTA, SketchAlgorithm::Cms) => { - let cms = existing - .as_any_mut() - .downcast_mut::() - .ok_or( - "apply_modified_otlp_delta_bytes: existing accumulator is \ - not a CountMinSketchAccumulator", - )?; - cms.apply_proto_delta_bytes(bytes) - } - (ENCODING_PROTO_DELTA, other) => Err(format!( - "PROTO_DELTA for sketch kind {other:?} is not yet supported; \ - DDSketch / HLL / CountSketch / CountMin are wired" - ) - .into()), - (ENCODING_MSGPACK_DELTA, SketchAlgorithm::CountSketch) => { - // DELTA-HEAP frame for the heap-bearing CountSketch: a sparse - // signed matrix delta + the full top-k heap. The cached base is - // a heap accumulator (window-1 full frame decoded via - // `from_msgpack_with_heap_bytes`); under the per-window-reset - // model the ingest caller has already reset it to empty at a - // window boundary, so applying the delta reconstructs the - // window's own matrix and replaces the heap. Decoded generically - // in `apply_msgpack_heap_delta_bytes` (rmp_serde, no - // `asap_sketchlib` delta API). Real `CountSketchWithHeapAccumulator` - // (median-of-signed-rows), not the CMS-family wrapper. - let heap = existing - .as_any_mut() - .downcast_mut::() - .ok_or( - "apply_modified_otlp_delta_bytes: existing accumulator is \ - not a CountSketchWithHeapAccumulator (heap-bearing \ - CountSketch delta requires a heap base — the window-1 \ - full frame must have promoted the sid)", - )?; - heap.apply_msgpack_heap_delta_bytes(bytes) - } - (ENCODING_MSGPACK_DELTA, other) => Err(format!( - "MSGPACK_DELTA for sketch kind {other:?} is not yet wired; only \ - the heap-bearing CountSketch DELTA-HEAP frame is supported" - ) - .into()), - (ENCODING_PROTO, _) | (ENCODING_MSGPACK, _) => Err(format!( - "encoding {encoding} is a full-state frame — route through \ - `decode_modified_otlp_sketch_bytes` and replace the cached \ - accumulator, not through the delta applier" - ) - .into()), - (other, _) => Err(format!("unknown modified-OTLP sketch encoding {other}").into()), - } -} - -/// Identify the concrete sketch type inside a `SketchEnvelope` payload, -/// returning a human-readable name for logging. Returns `"Unknown"` if the -/// payload does not decode or the `sketch_state` variant is unset. -fn identify_sketch_type(payload: &[u8]) -> &'static str { - match SketchEnvelope::decode(payload) { - Ok(env) => match env.sketch_state { - Some(sketch_envelope::SketchState::Kll(_)) => "KLL", - Some(sketch_envelope::SketchState::CountMin(_)) => "CountMin", - Some(sketch_envelope::SketchState::CountSketch(_)) => "CountSketch", - Some(_) => "Other", - None => "Unknown", - }, - Err(_) => "Unknown", - } -} - -/// Count total data points by traversing resource_metrics -> scope_metrics -> metrics. -/// Reuses the same conversion traversal as asap-otel-ingest (Gauge, Sum, Histogram, etc.). -fn otlp_to_record_count(request: &ExportMetricsServiceRequest) -> usize { - let mut count = 0; - for resource_metrics in &request.resource_metrics { - for scope_metrics in &resource_metrics.scope_metrics { - for metric in &scope_metrics.metrics { - if metric.name.is_empty() { - continue; - } - - use asap_otel_proto::tonic::metrics::v1::metric::Data; - match &metric.data { - Some(Data::Gauge(g)) => count += g.data_points.len(), - Some(Data::Sum(s)) => count += s.data_points.len(), - Some(Data::Histogram(hist)) => { - for dp in &hist.data_points { - if dp.sum.is_some() { - count += 1; - } - count += 1; // _count - count += dp.bucket_counts.len(); // _bucket per le - } - } - Some(Data::ExponentialHistogram(eh)) => { - for dp in &eh.data_points { - if dp.sum.is_some() { - count += 1; - } - count += 1; // _count - count += 1; // _scale - } - } - Some(Data::Summary(summary)) => { - for dp in &summary.data_points { - count += 1; // _sum - count += 1; // _count - count += dp.quantile_values.len(); - } - } - Some(Data::Ddsketch(d)) => count += d.data_points.len(), - Some(Data::Kllsketch(k)) => count += k.data_points.len(), - Some(Data::Countsketch(c)) => count += c.data_points.len(), - Some(Data::Countminsketch(c)) => count += c.data_points.len(), - Some(Data::Hllsketch(h)) => count += h.data_points.len(), - Some(Data::SumAgg(sa)) => count += sa.data_points.len(), - None => {} - } - } - } - } - count -} - -/// Parse OTLP request and convert to metric data points (name, labels, timestamp, value). -/// Data points with sketch payloads in attributes are excluded from points and returned -/// separately for Sketch Payload Flow processing. -fn otlp_to_metric_points_and_sketches(request: &ExportMetricsServiceRequest) -> OtlpParseResult { - let mut points = Vec::new(); - let mut sketch_payloads = Vec::new(); - for resource_metrics in &request.resource_metrics { - let resource_attrs = resource_metrics - .resource - .as_ref() - .map(|r| attributes_to_map(&r.attributes)) - .unwrap_or_default(); - - for scope_metrics in &resource_metrics.scope_metrics { - let scope_attrs = scope_metrics - .scope - .as_ref() - .map(|s| attributes_to_map(&s.attributes)) - .unwrap_or_default(); - - for metric in &scope_metrics.metrics { - if metric.name.is_empty() { - continue; - } - - let base_labels: HashMap = scope_attrs - .iter() - .chain(resource_attrs.iter()) - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - use asap_otel_proto::tonic::metrics::v1::metric::Data; - match &metric.data { - Some(Data::Gauge(g)) => { - for dp in &g.data_points { - if let Some((attr_name, payload)) = - get_sketch_payload_from_attrs(&dp.attributes) - { - sketch_payloads.push(SketchPoint { - name: metric.name.clone(), - attr_name, - labels: merge_point_attributes(&base_labels, &dp.attributes), - timestamp_nanos: dp.time_unix_nano, - payload, - }); - continue; - } - let labels = merge_point_attributes(&base_labels, &dp.attributes); - let value = number_value_to_f64(&dp.value); - points.push(MetricPoint { - name: metric.name.clone(), - labels, - timestamp_nanos: dp.time_unix_nano, - value, - }); - } - } - Some(Data::Sum(s)) => { - for dp in &s.data_points { - if let Some((attr_name, payload)) = - get_sketch_payload_from_attrs(&dp.attributes) - { - sketch_payloads.push(SketchPoint { - name: metric.name.clone(), - attr_name, - labels: merge_point_attributes(&base_labels, &dp.attributes), - timestamp_nanos: dp.time_unix_nano, - payload, - }); - continue; - } - let labels = merge_point_attributes(&base_labels, &dp.attributes); - let value = number_value_to_f64(&dp.value); - points.push(MetricPoint { - name: metric.name.clone(), - labels, - timestamp_nanos: dp.time_unix_nano, - value, - }); - } - } - Some(Data::SumAgg(sa)) => { - // First-class Sum AggregationType: each data point carries - // a SumState envelope ({sum,count}) in `sketch`. Decode it - // and feed the sum as a MetricPoint into the SAME - // ExactAgg(Sum) path as a plain delta Sum — the backend sums - // the per-window/per-shard partials for the same sid. - for dp in &sa.data_points { - let value = match asap_physical_operators::summary_kernels::sum::SumAccumulator::from_sum_bytes(&dp.sketch) { - Ok(acc) => acc.sum, - Err(e) => { - debug!("asap_edge: SumAgg data point decode failed (skipping): {e}"); - continue; - } - }; - let labels = merge_point_attributes(&base_labels, &dp.attributes); - points.push(MetricPoint { - name: metric.name.clone(), - labels, - timestamp_nanos: dp.time_unix_nano, - value, - }); - } - } - Some(Data::Histogram(hist)) => { - for dp in &hist.data_points { - if let Some((attr_name, payload)) = - get_sketch_payload_from_attrs(&dp.attributes) - { - sketch_payloads.push(SketchPoint { - name: metric.name.clone(), - attr_name, - labels: merge_point_attributes(&base_labels, &dp.attributes), - timestamp_nanos: dp.time_unix_nano, - payload, - }); - continue; - } - let labels = merge_point_attributes(&base_labels, &dp.attributes); - if let Some(sum) = dp.sum { - points.push(MetricPoint { - name: format!("{}_sum", metric.name), - labels: labels.clone(), - timestamp_nanos: dp.time_unix_nano, - value: sum, - }); - } - points.push(MetricPoint { - name: format!("{}_count", metric.name), - labels: labels.clone(), - timestamp_nanos: dp.time_unix_nano, - value: dp.count as f64, - }); - } - } - Some(Data::ExponentialHistogram(eh)) => { - for dp in &eh.data_points { - if let Some((attr_name, payload)) = - get_sketch_payload_from_attrs(&dp.attributes) - { - sketch_payloads.push(SketchPoint { - name: metric.name.clone(), - attr_name, - labels: merge_point_attributes(&base_labels, &dp.attributes), - timestamp_nanos: dp.time_unix_nano, - payload, - }); - continue; - } - let labels = merge_point_attributes(&base_labels, &dp.attributes); - if let Some(sum) = dp.sum { - points.push(MetricPoint { - name: format!("{}_sum", metric.name), - labels: labels.clone(), - timestamp_nanos: dp.time_unix_nano, - value: sum, - }); - } - points.push(MetricPoint { - name: format!("{}_count", metric.name), - labels: labels.clone(), - timestamp_nanos: dp.time_unix_nano, - value: dp.count as f64, - }); - } - } - Some(Data::Summary(sm)) => { - for dp in &sm.data_points { - if let Some((attr_name, payload)) = - get_sketch_payload_from_attrs(&dp.attributes) - { - sketch_payloads.push(SketchPoint { - name: metric.name.clone(), - attr_name, - labels: merge_point_attributes(&base_labels, &dp.attributes), - timestamp_nanos: dp.time_unix_nano, - payload, - }); - continue; - } - let labels = merge_point_attributes(&base_labels, &dp.attributes); - points.push(MetricPoint { - name: format!("{}_sum", metric.name), - labels: labels.clone(), - timestamp_nanos: dp.time_unix_nano, - value: dp.sum, - }); - points.push(MetricPoint { - name: format!("{}_count", metric.name), - labels: labels.clone(), - timestamp_nanos: dp.time_unix_nano, - value: dp.count as f64, - }); - } - } - // Modified-OTLP first-class sketch metric variants - // (Ddsketch / Kllsketch / Countsketch / Countminsketch / - // Hllsketch) and `None` are not handled here: this - // function only parses raw scalar metric points and - // attribute-embedded `SketchEnvelope` payloads. The - // first-class sketch DataPoints are decoded + routed by - // `route_modified_otlp_sketches_to_precompute`, so we - // intentionally ignore them in this parse pass. - _ => {} - } - } - } - } - (points, sketch_payloads) -} - -fn merge_point_attributes( - base: &HashMap, - attrs: &[asap_otel_proto::tonic::common::v1::KeyValue], -) -> HashMap { - let mut m = base.clone(); - for (k, v) in attributes_to_map(attrs) { - m.insert(k, v); - } - m -} - -fn number_value_to_f64(v: &Option) -> f64 { - match v { - Some(NumberValue::AsDouble(x)) => *x, - Some(NumberValue::AsInt(x)) => *x as f64, - None => 0.0, - } -} - -fn any_value_to_string(v: &asap_otel_proto::tonic::common::v1::AnyValue) -> String { - use asap_otel_proto::tonic::common::v1::any_value::Value as AnyValueVariant; - match &v.value { - Some(AnyValueVariant::StringValue(s)) => s.clone(), - Some(AnyValueVariant::IntValue(i)) => i.to_string(), - Some(AnyValueVariant::DoubleValue(d)) => d.to_string(), - Some(AnyValueVariant::BoolValue(b)) => b.to_string(), - Some(AnyValueVariant::BytesValue(bytes)) => format!("{:?}", bytes), - _ => String::new(), - } -} - -fn attributes_to_map( - attrs: &[asap_otel_proto::tonic::common::v1::KeyValue], -) -> HashMap { - let mut m = HashMap::new(); - for kv in attrs { - let key = kv.key.clone(); - let value = kv - .value - .as_ref() - .map(any_value_to_string) - .unwrap_or_default(); - if !key.is_empty() { - m.insert(key, value); - } - } - m -} - -#[cfg(test)] -mod canonical_metric_name_tests { - //! Coverage for `canonical_sketch_metric_name` — the ingest-seam - //! strip of the agent's `_` suffix (ASAPCollector - //! `asapedgeprocessor` sets `MetricSuffix: "_" + family`). Without - //! it, warm-tier sketch queries against the raw metric name - //! capability-miss because `SummarySeriesMetadata.metric_name` and - //! the controller's streaming-config policy `metric` never line up. - use super::*; - - #[test] - fn strips_matching_family_suffix() { - assert_eq!( - canonical_sketch_metric_name("request_size_bytes_kll", SketchAlgorithm::Kll), - "request_size_bytes" - ); - assert_eq!( - canonical_sketch_metric_name( - "http_requests_total_latency_ms_kll", - SketchAlgorithm::Kll - ), - "http_requests_total_latency_ms" - ); - assert_eq!( - canonical_sketch_metric_name("unique_users_per_min_hll", SketchAlgorithm::Hll), - "unique_users_per_min" - ); - assert_eq!( - canonical_sketch_metric_name( - "top_endpoint_qps_countsketch", - SketchAlgorithm::CountSketch - ), - "top_endpoint_qps" - ); - assert_eq!( - canonical_sketch_metric_name( - "endpoint_request_freq_countminsketch", - SketchAlgorithm::Cms - ), - "endpoint_request_freq" - ); - assert_eq!( - canonical_sketch_metric_name("latency_ddsketch", SketchAlgorithm::DDSketch), - "latency" - ); - } - - #[test] - fn leaves_bare_name_untouched_idempotent() { - // Once agents stop suffixing (series-identity - // Phase 1), the strip must be a no-op. - assert_eq!( - canonical_sketch_metric_name("request_size_bytes", SketchAlgorithm::Kll), - "request_size_bytes" - ); - assert_eq!( - canonical_sketch_metric_name("unique_users_per_min", SketchAlgorithm::Hll), - "unique_users_per_min" - ); - } - - #[test] - fn does_not_strip_suffix_of_a_different_family() { - // A metric whose name happens to end in `_hll` but arrives as a - // KLL sketch keeps its name — the strip is gated on the dp's - // actual sketch kind, so we never collapse a legitimately-named - // metric onto a different one. - assert_eq!( - canonical_sketch_metric_name("my_metric_hll", SketchAlgorithm::Kll), - "my_metric_hll" - ); - } - - #[test] - fn never_collapses_to_empty_string() { - // A metric literally named `_kll` (base would be empty) is left - // intact rather than emptied. - assert_eq!( - canonical_sketch_metric_name("_kll", SketchAlgorithm::Kll), - "_kll" - ); - } -} - -#[cfg(test)] -mod series_key_roundtrip_tests { - //! Regression coverage for the `format_series_key` ↔ - //! `parse_labels_from_series_key` roundtrip bug discovered while - //! shipping PR #284 (B7.6 ingest sid rekey). The formatter - //! emitted unquoted `k=v` pairs but the parser required - //! `k="v"`, which silently produced empty group keys for every - //! OTLP wire-format input. These tests pin the canonical - //! PromQL-style quoted format the data plane now uses - //! throughout (see also `render_series_key` in - //! `storage_engines/sketch_db/backfill/prometheus_reader.rs`). - use super::*; - use crate::precompute_engine::worker::{decode_label_value, parse_labels_from_series_key}; - - fn roundtrip(name: &str, input: &[(&str, &str)]) { - let labels: HashMap = input - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(); - let key = format_series_key(name, &labels); - let parsed = parse_labels_from_series_key(&key); - for (k, v) in input { - let got = parsed - .get(*k) - .map(|raw| decode_label_value(raw).into_owned()) - .unwrap_or_else(|| panic!("label '{}' missing after roundtrip; key={}", k, key)); - assert_eq!( - got, *v, - "label '{}' value mismatch after roundtrip; key={}", - k, key - ); - } - assert_eq!( - parsed.len(), - input.len(), - "label count mismatch after roundtrip; key={} parsed={:?}", - key, - parsed - ); - } - - #[test] - fn format_series_key_emits_promql_quoted_form() { - // The canonical shape every downstream parser - // (`parse_labels_from_series_key`, `sample_matches`) expects. - let mut labels = HashMap::new(); - labels.insert("svc".to_string(), "auth".to_string()); - labels.insert("env".to_string(), "prod".to_string()); - let key = format_series_key("latency", &labels); - // Keys are sorted lexicographically so the formatted output - // is deterministic regardless of HashMap iteration order. - assert_eq!(key, r#"latency{env="prod",svc="auth"}"#); - } - - #[test] - fn roundtrip_simple_alphanumeric() { - roundtrip("metric", &[("svc", "auth"), ("env", "prod")]); - } - - #[test] - fn roundtrip_value_with_comma() { - // Comma is the pair delimiter — quoting must keep it inside - // the value. Pre-fix the unquoted format would split mid-value. - roundtrip("metric", &[("tag", "a,b,c"), ("svc", "auth")]); - } - - #[test] - fn roundtrip_value_with_equals() { - // Equals is the key/value delimiter — quoting must protect it. - roundtrip("metric", &[("expr", "x=y"), ("svc", "auth")]); - } - - #[test] - fn roundtrip_value_with_embedded_quote() { - // `"` must be escaped as `\"` on emit and decoded back on - // read. The parser's closing-quote scan must walk past the - // escape; `decode_label_value` un-escapes the slice. - roundtrip("metric", &[("msg", r#"hello "world""#), ("svc", "auth")]); - } - - #[test] - fn roundtrip_value_with_backslash() { - roundtrip("metric", &[("path", r"C:\Users\app"), ("svc", "auth")]); - } - - #[test] - fn roundtrip_value_with_newline() { - // `\n` round-trips through the `\n` escape; verifies the - // decoder handles all three escape body variants. - roundtrip("metric", &[("multi", "line1\nline2"), ("svc", "auth")]); - } - - #[test] - fn roundtrip_value_with_all_metacharacters() { - // One stress case combining every escape body and every - // pair-delimiter character in a single value. - roundtrip("metric", &[("payload", "a,b=c\"d\\e\nf"), ("svc", "auth")]); - } - - #[test] - fn empty_labels_yield_bare_braces() { - let labels: HashMap = HashMap::new(); - let key = format_series_key("metric", &labels); - assert_eq!(key, "metric{}"); - let parsed = parse_labels_from_series_key(&key); - assert!(parsed.is_empty()); - } -} - -#[cfg(test)] -mod policy_fp_lookup_tests { - use super::*; - use crate::storage_engines::sketch_db::data::SketchConfig; - use crate::storage_engines::sketch_db::index::SketchAlgorithm; - use asap_types::AggregationType; - - #[test] - fn handle_to_agg_type_round_trips_canonical_kinds() { - // Locks in the data-plane → control-plane name mapping. - // Drift surfaces as policy lookups that silently miss because - // the handle resolves to an `AggregationType` no policy uses. - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::DDSketch), - Some(AggregationType::DDSketch) - ); - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::Kll), - Some(AggregationType::DatasketchesKLL) - ); - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::Hll), - Some(AggregationType::HLL) - ); - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::Cms), - Some(AggregationType::CountMinSketch) - ); - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::CmsWithHeap), - Some(AggregationType::CountMinSketchWithHeap) - ); - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::CountSketch), - Some(AggregationType::CountSketch) - ); - // `Any` is a control-plane wildcard, not a real DP shape. - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::Kmv), - None - ); - } - - #[test] - fn sketch_config_to_params_uses_canonical_keys() { - // The param-name vocabulary must match what the control plane - // writes in streaming-config YAML (see - // `asap_types::aggregation_config::PrecomputeMaterialization::from_yaml_data`). - // Drift surfaces as `find_policy_by_content` missing matches. - let dd = sketch_config_to_params(&SketchConfig::DDSketch { - relative_accuracy: 0.01, - }); - assert_eq!(dd.get("relative_accuracy"), Some(&serde_json::json!(0.01))); - - let kll = sketch_config_to_params(&SketchConfig::Kll { k: 200 }); - assert_eq!(kll.get("k"), Some(&serde_json::json!(200))); - - let hll = sketch_config_to_params(&SketchConfig::Hll { precision: 14 }); - assert_eq!(hll.get("precision"), Some(&serde_json::json!(14))); - - let cs = sketch_config_to_params(&SketchConfig::CountSketch { rows: 4, cols: 256 }); - // Canonical keys: w (=cols, width) and d (=rows, depth) — - // matches `control_plane::emit::stage_config::sketch_params_to_json`. - assert_eq!(cs.get("w"), Some(&serde_json::json!(256))); - assert_eq!(cs.get("d"), Some(&serde_json::json!(4))); - - let cm = sketch_config_to_params(&SketchConfig::CountMin { rows: 4, cols: 256 }); - assert_eq!(cm.get("w"), Some(&serde_json::json!(256))); - assert_eq!(cm.get("d"), Some(&serde_json::json!(4))); - } -} - -#[cfg(test)] -mod dispatcher_tests { - use super::*; - use crate::storage_engines::types::AggregateCore; - use asap_physical_operators::summary_kernels::{DDSketchAccumulator, HllSketchAccumulator}; - use asap_sketchlib::DdSketch; - use asap_sketchlib::HllVariant; - - #[test] - fn apply_modified_otlp_delta_bytes_ddsketch_round_trip() { - use asap_otel_proto::sketchlib::v1::{DdSketchBucketDelta, DdSketchDelta as PbDelta}; - use prost::Message; - - // Base sketch represents the last full snapshot the agent sent. - let mut acc: Box = Box::new(DDSketchAccumulator { - inner: DdSketch::from_raw(0.01, vec![1, 2, 3], 0), - sample_p: 1.0, - }); - - // The wire delta now carries only bucket deltas (tags 2-7 - // reserved post ProjectASAP/sketchlib-go#243 / asap_sketchlib#57). - let bytes = PbDelta { - buckets: vec![ - DdSketchBucketDelta { - index: 0, - d_count: 10, - }, - DdSketchBucketDelta { - index: 2, - d_count: 20, - }, - ], - } - .encode_to_vec(); - - apply_modified_otlp_delta_bytes( - SketchAlgorithm::DDSketch, - ENCODING_PROTO_DELTA, - &mut acc, - &bytes, - ) - .expect("apply ok"); - - let dd = acc.as_any().downcast_ref::().unwrap(); - assert_eq!(dd.inner.store_counts, vec![11, 2, 23]); - // `count` recomputed from the merged buckets: 11 + 2 + 23 = 36. - assert_eq!(dd.inner.total_count(), 36); - } - - #[test] - fn apply_modified_otlp_delta_bytes_hll_round_trip() { - use asap_otel_proto::sketchlib::v1::HllDelta as PbDelta; - use prost::Message; - - let mut acc: Box = - Box::new(HllSketchAccumulator::new(HllVariant::Regular, 2)); - acc.as_any_mut() - .downcast_mut::() - .unwrap() - .inner - .registers = vec![1, 5, 3, 7]; - - // Packed (index_delta, value) blob for updates {0:4, 2:6}. - let bytes = PbDelta { - packed_updates: vec![0, 4, 2, 6], - } - .encode_to_vec(); - - apply_modified_otlp_delta_bytes( - SketchAlgorithm::Hll, - ENCODING_PROTO_DELTA, - &mut acc, - &bytes, - ) - .expect("apply ok"); - - let hll = acc.as_any().downcast_ref::().unwrap(); - assert_eq!(hll.inner.registers, vec![4, 5, 6, 7]); - } - - #[test] - fn apply_rejects_wrong_accumulator_type() { - let mut acc: Box = - Box::new(HllSketchAccumulator::new(HllVariant::Regular, 2)); - let err = apply_modified_otlp_delta_bytes( - SketchAlgorithm::DDSketch, - ENCODING_PROTO_DELTA, - &mut acc, - &[0u8; 4], - ) - .expect_err("expected type-mismatch error") - .to_string(); - assert!(err.contains("not a DDSketchAccumulator")); - } - - #[test] - fn apply_rejects_full_state_encoding() { - let mut acc: Box = Box::new(DDSketchAccumulator::new(0.01)); - let err = apply_modified_otlp_delta_bytes( - SketchAlgorithm::DDSketch, - ENCODING_PROTO, - &mut acc, - &[], - ) - .expect_err("expected full-state-rejection error") - .to_string(); - assert!(err.contains("full-state frame")); - } - - #[test] - fn decode_rejects_delta_encoding_with_helpful_message() { - let err = match decode_modified_otlp_sketch_bytes( - SketchAlgorithm::DDSketch, - ENCODING_PROTO_DELTA, - &[], - ) { - Ok(_) => panic!("expected PROTO_DELTA to be rejected by full-state decoder"), - Err(e) => e.to_string(), - }; - assert!(err.contains("apply_modified_otlp_delta_bytes")); - } -} - -/// sid-resolution gate tests. Construct an OTLP DDSketch -/// Export with one DataPoint per scenario, run it through -/// `route_modified_otlp_sketches_to_precompute`, and assert on the -/// returned `unknown_series_ids` plus the SeriesIdResolver / SketchStore -/// state on the shared IngestState. -#[cfg(test)] -mod sid_resolution_tests { - use super::*; - use crate::drivers::ingest::series_resolver::SeriesIdResolver; - use crate::precompute_engine::series_router::SeriesRouter; - use crate::storage_engines::sketch_db::index::SketchStore; - use crate::storage_engines::types::{InstalledPrecomputePlan, InstalledPrecomputePlanHandle}; - use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; - use asap_otel_proto::tonic::common::v1::{any_value::Value as AnyVal, AnyValue, KeyValue}; - use asap_otel_proto::tonic::metrics::v1::{ - metric::Data, DdSketch as PbDDSketch, DdSketchDataPoint, Metric as PbMetric, - ResourceMetrics, ScopeMetrics, - }; - use std::sync::Arc; - use tokio::sync::mpsc; - - async fn make_state() -> (Arc, tokio::task::JoinHandle<()>) { - let (tx, mut rx) = mpsc::channel(1024); - let router = SeriesRouter::new(vec![tx]); - let streaming = InstalledPrecomputePlan::new(std::collections::HashMap::new()); - let hot_reload = InstalledPrecomputePlanHandle::new(streaming.clone()); - let state = Arc::new(IngestState { - router, - samples_ingested: std::sync::atomic::AtomicU64::new(0), - samples_blocked_by_schema_barrier: std::sync::atomic::AtomicU64::new(0), - hot_reload_config: hot_reload, - pass_raw_samples: false, - sketch_snapshots: dashmap::DashMap::new(), - series_resolver: Arc::new(SeriesIdResolver::new()), - summary_store: Arc::new(SketchStore::new()), - observability: crate::precompute_engine::ingest_handler::IngestObservability::default(), - }); - let drain = tokio::spawn(async move { while rx.recv().await.is_some() {} }); - (state, drain) - } - - fn kv(k: &str, v: &str) -> KeyValue { - KeyValue { - key: k.to_string(), - value: Some(AnyValue { - value: Some(AnyVal::StringValue(v.to_string())), - }), - } - } - - fn build_request(metric_name: &str, dp: DdSketchDataPoint) -> ExportMetricsServiceRequest { - ExportMetricsServiceRequest { - resource_metrics: vec![ResourceMetrics { - resource: None, - scope_metrics: vec![ScopeMetrics { - scope: None, - metrics: vec![PbMetric { - name: metric_name.to_string(), - description: String::new(), - unit: String::new(), - metadata: Vec::new(), - data: Some(Data::Ddsketch(PbDDSketch { - data_points: vec![dp], - aggregation_temporality: 0, - relative_accuracy: 0.01, - })), - }], - schema_url: String::new(), - }], - schema_url: String::new(), - }], - } - } - - #[tokio::test] - async fn fresh_sid_minted_when_sender_supplies_zero_with_attrs() { - let (state, drain) = make_state().await; - let dp = DdSketchDataPoint { - attributes: vec![kv("zone", "z0")], - start_time_unix_nano: 1_000_000, - time_unix_nano: 11_000_000, - sketch: vec![1, 2, 3], - encoding: 1, - exemplars: Vec::new(), - flags: 0, - series_id: 0, - }; - let req = build_request("http_latency_ms", dp); - - let outcome = route_modified_otlp_sketches_to_precompute(&req, &state) - .await - .expect("ingest succeeds"); - assert!( - outcome.unknown_series_ids.is_empty(), - "no unknown sids on a fresh-attrs DP" - ); - // Option B — sid is resolver-allocated; every attrs-bearing DP - // gets a SeriesAssignment echoed back so the sender caches it. - assert_eq!( - outcome.series_assignments.len(), - 1, - "one series_assignment returned for one fresh DP" - ); - let assigned = &outcome.series_assignments[0]; - assert_eq!(assigned.metric_name, "http_latency_ms"); - assert_ne!( - assigned.series_id, 0, - "resolver mints a non-zero sid (zero is reserved on the wire)" - ); - assert_eq!( - state.summary_store.instance_count(), - 1, - "SketchStore registered one instance under the resolver-minted sid" - ); - assert!(state.summary_store.instance(assigned.series_id).is_some()); - - drop(state); - let _ = drain.await; - } - - #[tokio::test] - async fn unknown_sid_with_empty_attrs_is_returned_in_response() { - let (state, drain) = make_state().await; - // sid != 0, no attrs — SketchStore doesn't know it; should land - // in unknown_sids and the DP must be dropped (no instance - // registered). The resolver can't be consulted without attrs. - let dp = DdSketchDataPoint { - attributes: Vec::new(), - start_time_unix_nano: 0, - time_unix_nano: 5_000_000, - sketch: vec![9], - encoding: 1, - exemplars: Vec::new(), - flags: 0, - series_id: 7777, - }; - let req = build_request("http_latency_ms", dp); - - let outcome = route_modified_otlp_sketches_to_precompute(&req, &state) - .await - .expect("ingest succeeds"); - assert_eq!(outcome.unknown_series_ids, vec![7777]); - assert!( - outcome.series_assignments.is_empty(), - "no assignment when attrs are missing" - ); - assert_eq!(state.summary_store.instance_count(), 0); - - drop(state); - let _ = drain.await; - } - - #[tokio::test] - async fn sid_attrs_disagreement_signals_stale_sid_but_uses_resolved_value() { - let (state, drain) = make_state().await; - // First, register the sid by sending sid=0 with attrs. The - // resolver mints a fresh u64 (sequential, NOT content-addressed). - let dp_seed = DdSketchDataPoint { - attributes: vec![kv("zone", "z0")], - start_time_unix_nano: 1_000_000, - time_unix_nano: 11_000_000, - sketch: vec![1], - encoding: 1, - exemplars: Vec::new(), - flags: 0, - series_id: 0, - }; - let seed_outcome = route_modified_otlp_sketches_to_precompute( - &build_request("http_latency_ms", dp_seed), - &state, - ) - .await - .expect("seed ingest succeeds"); - let assigned_sid = seed_outcome.series_assignments[0].series_id; - assert!( - state.summary_store.instance(assigned_sid).is_some(), - "seed registers the resolver-minted sid" - ); - - // Now arrive with the same attrs but a STALE sid (sender's - // local cache was wrong, e.g. survived a backend restart without - // persistence). - let stale = assigned_sid.wrapping_add(123); - let dp_disagree = DdSketchDataPoint { - attributes: vec![kv("zone", "z0")], - start_time_unix_nano: 1_000_000, - time_unix_nano: 12_000_000, - sketch: vec![2], - encoding: 1, - exemplars: Vec::new(), - flags: 0, - series_id: stale, - }; - let outcome = route_modified_otlp_sketches_to_precompute( - &build_request("http_latency_ms", dp_disagree), - &state, - ) - .await - .expect("ingest succeeds"); - assert_eq!( - outcome.unknown_series_ids, - vec![stale], - "stale sid should be signalled for eviction" - ); - assert_eq!( - outcome.series_assignments.len(), - 1, - "fresh assignment echoed so sender can refresh its cache" - ); - assert_eq!( - outcome.series_assignments[0].series_id, assigned_sid, - "resolver returns the canonical sid for this (metric, attrs) — same as seed" - ); - // The assigned sid stays registered — the second DP routed to - // it via the resolver's cache hit. - assert!(state.summary_store.instance(assigned_sid).is_some()); - - drop(state); - let _ = drain.await; - } - - /// Per-window base rotation (`docs/delta-baseline-contract.md` §3): - /// the backend must NOT accumulate deltas across windows. For one - /// series, a full frame opens window 1, a delta in window 1 (same - /// `start_time_unix_nano`) accumulates onto it, then a delta in - /// window 2 (a NEW `start_time_unix_nano`) must reset the cached base - /// to empty first — so the reconstructed state is window 2's delta - /// only, NOT window1 + window2. - /// - /// Uses DDSketch (an additive family) so accumulation vs. reset is - /// directly observable on the bucket counts. - #[tokio::test] - async fn delta_apply_rotates_per_series_base_at_window_boundary() { - use asap_otel_proto::sketchlib::v1::{DdSketchBucketDelta, DdSketchDelta as PbDelta}; - use asap_physical_operators::summary_kernels::DDSketchAccumulator; - use asap_sketchlib::proto::sketchlib::{sketch_envelope, DdSketchState, SketchEnvelope}; - use prost::Message; - - let (state, drain) = make_state().await; - - const WIN1_START: u64 = 1_000_000; - const WIN2_START: u64 = 2_000_000; - - // Build a DDSketch DataPoint with explicit encoding / window-start - // / payload so we can stage a full frame then per-window deltas. - let make_dp = |start: u64, ts: u64, encoding: i32, sketch: Vec| DdSketchDataPoint { - attributes: vec![kv("zone", "z0")], - start_time_unix_nano: start, - time_unix_nano: ts, - sketch, - encoding, - exemplars: Vec::new(), - flags: 0, - series_id: 0, - }; - - // The cache key (series_key) is derived from the canonical metric - // name + attrs; recompute it the same way the ingest loop does so - // we can read the reconstructed base back out. - let mut attrs = HashMap::new(); - attrs.insert("zone".to_string(), "z0".to_string()); - let series_key = format_series_key( - canonical_sketch_metric_name("http_latency_ms", SketchAlgorithm::DDSketch), - &attrs, - ); - - // ── Window 1: full frame. Base buckets [10, 0, 5]. ── - let full_w1 = SketchEnvelope { - sketch_state: Some(sketch_envelope::SketchState::Ddsketch(DdSketchState { - alpha: 0.01, - store_counts: vec![10, 0, 5], - store_offset: 0, - ..Default::default() - })), - ..Default::default() - } - .encode_to_vec(); - route_modified_otlp_sketches_to_precompute( - &build_request( - "http_latency_ms", - make_dp(WIN1_START, 11_000_000, 1, full_w1), - ), - &state, - ) - .await - .expect("first ingest succeeds"); - - // ── Window 1: delta (SAME window_start). Adds +3 to bucket 0, - // +7 to bucket 1. Within the window this accumulates onto the - // full frame → [13, 7, 5]. ── - let delta_w1 = PbDelta { - buckets: vec![ - DdSketchBucketDelta { - index: 0, - d_count: 3, - }, - DdSketchBucketDelta { - index: 1, - d_count: 7, - }, - ], - } - .encode_to_vec(); - route_modified_otlp_sketches_to_precompute( - &build_request( - "http_latency_ms", - make_dp(WIN1_START, 12_000_000, 2, delta_w1), - ), - &state, - ) - .await - .expect("second ingest succeeds"); - - { - let entry = state - .sketch_snapshots - .get(&series_key) - .expect("base cached after full + delta in window 1"); - let dd = entry - .core - .as_any() - .downcast_ref::() - .expect("DDSketch base"); - assert_eq!( - dd.inner.store_counts, - vec![13, 7, 5], - "within window 1 the delta accumulates onto the full frame" - ); - assert_eq!( - entry.window_start, WIN1_START, - "cached window_start tracks window 1" - ); - } - - // ── Window 2: delta with a NEW window_start. Adds +20 to bucket - // 2. With per-window base rotation the base is reset to empty - // BEFORE this delta is applied, so the reconstructed state is - // window 2 ONLY: count 20 — NOT window1 + window2 (count 45). - let delta_w2 = PbDelta { - buckets: vec![DdSketchBucketDelta { - index: 2, - d_count: 20, - }], - } - .encode_to_vec(); - route_modified_otlp_sketches_to_precompute( - &build_request( - "http_latency_ms", - make_dp(WIN2_START, 21_000_000, 2, delta_w2), - ), - &state, - ) - .await - .expect("ingest succeeds"); - - { - let entry = state - .sketch_snapshots - .get(&series_key) - .expect("base still cached after window 2 delta"); - let dd = entry - .core - .as_any() - .downcast_ref::() - .expect("DDSketch base"); - // The base was rotated to empty before the window-2 delta, so - // it holds window 2 ONLY: total count 20 (the +20 on bucket 2), - // NOT window1 + window2 (which would be 13 + 7 + 5 + 20 = 45). - // Asserting on `total_count` keeps the check independent of the - // empty-sketch's store offset/layout (a fresh sketch re-bases - // its store offset around the first touched bucket). - assert_eq!( - dd.inner.total_count(), - 20, - "new window_start rotates the base to empty: state == window 2 only \ - (count 20), NOT the all-time accumulation (45)" - ); - // Only bucket 2 carries mass; the window-1 buckets (0 and 1) - // were dropped by the rotation. - let bucket_count = |abs_idx: i32| -> u64 { - let i = abs_idx - dd.inner.store_offset; - if i >= 0 && (i as usize) < dd.inner.store_counts.len() { - dd.inner.store_counts[i as usize] - } else { - 0 - } - }; - assert_eq!(bucket_count(2), 20, "window 2's +20 lands on bucket 2"); - assert_eq!( - bucket_count(0), - 0, - "window 1's bucket 0 mass was rotated away" - ); - assert_eq!( - bucket_count(1), - 0, - "window 1's bucket 1 mass was rotated away" - ); - assert_eq!( - entry.window_start, WIN2_START, - "cached window_start advanced to window 2" - ); - } - - drop(state); - let _ = drain.await; - } - - #[tokio::test] - async fn second_emit_with_cached_sid_and_no_attrs_hits_same_instance() { - // Round-trip: emit DP with attrs → cache the assignment → - // re-emit with (sid, no attrs). Second emit must NOT push - // anything to unknown_sids and MUST land in the same SketchStore - // instance. This is the bandwidth-saving path the registry - // approach is built for. - let (state, drain) = make_state().await; - - let dp_first = DdSketchDataPoint { - attributes: vec![kv("zone", "z0")], - start_time_unix_nano: 1_000_000, - time_unix_nano: 11_000_000, - sketch: vec![1], - encoding: 1, - exemplars: Vec::new(), - flags: 0, - series_id: 0, - }; - let first = route_modified_otlp_sketches_to_precompute( - &build_request("http_latency_ms", dp_first), - &state, - ) - .await - .expect("first ingest succeeds"); - let cached_sid = first.series_assignments[0].series_id; - - let dp_second = DdSketchDataPoint { - attributes: Vec::new(), // sender omits attrs now that it has the sid - start_time_unix_nano: 1_000_000, - time_unix_nano: 12_000_000, - sketch: vec![2], - encoding: 1, - exemplars: Vec::new(), - flags: 0, - series_id: cached_sid, - }; - let second = route_modified_otlp_sketches_to_precompute( - &build_request("http_latency_ms", dp_second), - &state, - ) - .await - .expect("second ingest succeeds"); - assert!( - second.unknown_series_ids.is_empty(), - "cached sid + no attrs hits the same SketchStore instance" - ); - assert!( - second.series_assignments.is_empty(), - "no fresh assignment when sender already had a valid binding" - ); - // Both DPs landed against the same sid — no proliferation. - assert_eq!(state.summary_store.instance_count(), 1); - - drop(state); - let _ = drain.await; - } - - // ── P1-1/P1-2: leading delta (no prior full) for additive families ── - - /// Build an export request carrying one `CountMinSketch` DataPoint. - fn build_cms_request( - metric: &str, - rows: i32, - cols: i32, - encoding: i32, - sketch: Vec, - start_ns: u64, - ts_ns: u64, - ) -> ExportMetricsServiceRequest { - use asap_otel_proto::tonic::metrics::v1::{ - CountMinSketch as PbCms, CountMinSketchDataPoint as PbCmsDp, - }; - let dp = PbCmsDp { - attributes: vec![kv("svc", "auth")], - start_time_unix_nano: start_ns, - time_unix_nano: ts_ns, - sketch, - encoding, - flags: 0, - series_id: 0, - }; - ExportMetricsServiceRequest { - resource_metrics: vec![ResourceMetrics { - resource: None, - scope_metrics: vec![ScopeMetrics { - scope: None, - metrics: vec![PbMetric { - name: metric.to_string(), - description: String::new(), - unit: String::new(), - metadata: Vec::new(), - data: Some(Data::Countminsketch(PbCms { - data_points: vec![dp], - aggregation_temporality: 0, - rows, - cols, - })), - }], - schema_url: String::new(), - }], - schema_url: String::new(), - }], - } - } - - /// Build an export request carrying one `HllSketch` DataPoint. - fn build_hll_request( - metric: &str, - precision: u32, - encoding: i32, - sketch: Vec, - start_ns: u64, - ts_ns: u64, - ) -> ExportMetricsServiceRequest { - use asap_otel_proto::tonic::metrics::v1::{ - HllSketch as PbHll, HllSketchDataPoint as PbHllDp, - }; - let dp = PbHllDp { - attributes: vec![kv("svc", "auth")], - start_time_unix_nano: start_ns, - time_unix_nano: ts_ns, - sketch, - encoding, - flags: 0, - series_id: 0, - }; - ExportMetricsServiceRequest { - resource_metrics: vec![ResourceMetrics { - resource: None, - scope_metrics: vec![ScopeMetrics { - scope: None, - metrics: vec![PbMetric { - name: metric.to_string(), - description: String::new(), - unit: String::new(), - metadata: Vec::new(), - data: Some(Data::Hllsketch(PbHll { - data_points: vec![dp], - aggregation_temporality: 0, - precision, - })), - }], - schema_url: String::new(), - }], - schema_url: String::new(), - }], - } - } - - /// P1-1/P1-2 — a CMS PROTO_DELTA frame that is the FIRST frame for its - /// series (no prior full snapshot) must NOT be dropped: the ingest - /// path bootstraps an empty CMS of the frame's (rows, cols) and - /// applies the delta onto it, recovering the window's state. This is - /// what makes the worker tier agree with the warm-read tier and - /// recover after a backend restart. - #[tokio::test] - async fn leading_cms_delta_bootstraps_onto_empty_base() { - use asap_otel_proto::sketchlib::v1::CountMinDelta as PbDelta; - use asap_physical_operators::summary_kernels::CountMinSketchAccumulator; - use prost::Message; - - let (state, drain) = make_state().await; - - const ROWS: i32 = 4; - const COLS: i32 = 8; - const WIN_START: u64 = 1_000_000; - - // Sparse delta: +5 at (0,1), +9 at (2,3). All cells within 4x8. - let delta = PbDelta { - rows: ROWS as u32, - cols: COLS as u32, - cell_rows: vec![0u32, 2u32], - cell_cols: vec![1u32, 3u32], - d_counts: vec![5i64, 9i64], - l1: vec![5.0, 0.0, 9.0, 0.0], - l2: vec![25.0, 0.0, 81.0, 0.0], - } - .encode_to_vec(); - - let req = build_cms_request( - "frequency_metric", - ROWS, - COLS, - ENCODING_PROTO_DELTA, - delta, - WIN_START, - 11_000_000, - ); - // No prior full frame for this series — pre-fix this DP was - // dropped (decoded_failed). Post-fix it bootstraps + applies. - route_modified_otlp_sketches_to_precompute(&req, &state) - .await - .expect("ingest succeeds"); - - // The per-series base is now cached, holding the window's - // reconstructed matrix. - let mut attrs = HashMap::new(); - attrs.insert("svc".to_string(), "auth".to_string()); - let series_key = format_series_key( - canonical_sketch_metric_name("frequency_metric", SketchAlgorithm::Cms), - &attrs, - ); - { - let entry = state - .sketch_snapshots - .get(&series_key) - .expect("leading CMS delta bootstrapped + cached a base"); - let cms = entry - .core - .as_any() - .downcast_ref::() - .expect("base is a CountMinSketchAccumulator"); - let matrix = cms.inner.sketch(); - assert_eq!( - matrix[0][1], 5.0, - "delta cell (0,1) applied onto empty base" - ); - assert_eq!( - matrix[2][3], 9.0, - "delta cell (2,3) applied onto empty base" - ); - assert_eq!(matrix[0][0], 0.0, "untouched cell stays empty"); - } - // The DD/KLL "no base" drop counter must NOT have ticked — CMS is - // bootstrappable. - assert_eq!( - state - .observability - .dropped_no_base - .load(std::sync::atomic::Ordering::Relaxed), - 0, - "CMS leading delta is bootstrapped, not dropped as no-base" - ); - - drop(state); - let _ = drain.await; - } - - /// P1-1/P1-2 — same as above but for HLL: a leading PROTO_DELTA frame - /// bootstraps onto an empty HLL of the frame's precision and applies - /// the register-max updates. - #[tokio::test] - async fn leading_hll_delta_bootstraps_onto_empty_base() { - use asap_otel_proto::sketchlib::v1::HllDelta as PbDelta; - use asap_physical_operators::summary_kernels::HllSketchAccumulator; - use prost::Message; - - let (state, drain) = make_state().await; - - const PRECISION: u32 = 2; // 2^2 = 4 registers - const WIN_START: u64 = 1_000_000; - - // Packed (index, value) updates: set register 0 -> 4, register 2 -> 6. - let delta = PbDelta { - packed_updates: vec![0, 4, 2, 6], - } - .encode_to_vec(); - - let req = build_hll_request( - "cardinality_metric", - PRECISION, - ENCODING_PROTO_DELTA, - delta, - WIN_START, - 11_000_000, - ); - route_modified_otlp_sketches_to_precompute(&req, &state) - .await - .expect("ingest succeeds"); - - let mut attrs = HashMap::new(); - attrs.insert("svc".to_string(), "auth".to_string()); - let series_key = format_series_key( - canonical_sketch_metric_name("cardinality_metric", SketchAlgorithm::Hll), - &attrs, - ); - { - let entry = state - .sketch_snapshots - .get(&series_key) - .expect("leading HLL delta bootstrapped + cached a base"); - let hll = entry - .core - .as_any() - .downcast_ref::() - .expect("base is an HllSketchAccumulator"); - // Empty base registers are all 0; the delta sets max(0,4)=4 and - // max(0,6)=6 on registers 0 and 2. - assert_eq!(hll.inner.registers[0], 4, "register 0 set to 4"); - assert_eq!(hll.inner.registers[2], 6, "register 2 set to 6"); - } - assert_eq!( - state - .observability - .dropped_no_base - .load(std::sync::atomic::Ordering::Relaxed), - 0, - "HLL leading delta is bootstrapped, not dropped as no-base" - ); - - drop(state); - let _ = drain.await; - } - - /// P1-1/P1-2 — DDSketch is NOT bootstrappable from a bare delta, so a - /// leading DDSketch delta is still dropped and counted under - /// `dropped_no_base`. Confirms the fix is scoped to the additive - /// families and leaves DD/KLL behavior unchanged. - #[tokio::test] - async fn leading_ddsketch_delta_still_dropped_as_no_base() { - use asap_otel_proto::sketchlib::v1::{DdSketchBucketDelta, DdSketchDelta as PbDelta}; - use prost::Message; - - let (state, drain) = make_state().await; - let delta = PbDelta { - buckets: vec![DdSketchBucketDelta { - index: 0, - d_count: 10, - }], - } - .encode_to_vec(); - let dp = DdSketchDataPoint { - attributes: vec![kv("zone", "z0")], - start_time_unix_nano: 1_000_000, - time_unix_nano: 11_000_000, - sketch: delta, - encoding: ENCODING_PROTO_DELTA, - exemplars: Vec::new(), - flags: 0, - series_id: 0, - }; - route_modified_otlp_sketches_to_precompute(&build_request("dd_latency_ms", dp), &state) - .await - .expect("ingest succeeds"); - - let mut attrs = HashMap::new(); - attrs.insert("zone".to_string(), "z0".to_string()); - let series_key = format_series_key( - canonical_sketch_metric_name("dd_latency_ms", SketchAlgorithm::DDSketch), - &attrs, - ); - assert!( - state.sketch_snapshots.get(&series_key).is_none(), - "DDSketch leading delta is dropped (no bootstrap), so nothing cached" - ); - assert_eq!( - state - .observability - .dropped_no_base - .load(std::sync::atomic::Ordering::Relaxed), - 1, - "DDSketch leading delta counted as a no-base drop" - ); - - drop(state); - let _ = drain.await; - } - - // ── P1-4: heap capability upgrade ── - - /// P1-4 (a) — a CMS sid first registered from a heap-LESS frame - /// (`FrequencyEstimate(CountMin)`) must UPGRADE to - /// `FrequencyTopk(CmsWithHeap)` when a later heap-bearing MSGPACK - /// frame arrives for the same sid. One-way; never downgrades. - #[tokio::test] - async fn heap_bearing_frame_upgrades_cms_sid_capability() { - use crate::storage_engines::sketch_db::index::{Capability, SketchAlgorithm}; - use asap_sketchlib::{CountMinSketchWithHeap, MessagePackCodec}; - - let (state, drain) = make_state().await; - - const ROWS: i32 = 4; - const COLS: i32 = 8; - - // ── Frame 1: heap-LESS plain CMS msgpack. Registers the sid as - // FrequencyEstimate(CountMin). ── - let plain = asap_sketchlib::CountMinSketch::new(ROWS as usize, COLS as usize); - let plain_bytes = plain.to_msgpack().expect("serialize plain CMS msgpack"); - let req1 = build_cms_request( - "topk_metric", - ROWS, - COLS, - ENCODING_MSGPACK, - plain_bytes, - 1_000_000, - 11_000_000, - ); - let out1 = route_modified_otlp_sketches_to_precompute(&req1, &state) - .await - .expect("ingest succeeds"); - let sid = out1.series_assignments[0].series_id; - let meta1 = state.summary_store.instance(sid).expect("sid registered"); - assert_eq!( - meta1.capability, - Some(Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))), - "heap-less first frame registers FrequencyEstimate(CountMin)" - ); - - // ── Frame 2: heap-BEARING CMS-with-heap msgpack for the SAME - // (metric, attrs) → same sid. Must upgrade the capability. ── - let mut heap = CountMinSketchWithHeap::new(ROWS as usize, COLS as usize, 4); - heap.update("hot_key", 100.0); - heap.update("hot_key", 50.0); - assert!( - !heap.topk_heap_items().is_empty(), - "heap frame carries a non-empty top-k heap" - ); - let heap_bytes = heap.to_msgpack().expect("serialize CMS-with-heap msgpack"); - let req2 = build_cms_request( - "topk_metric", - ROWS, - COLS, - ENCODING_MSGPACK, - heap_bytes, - 1_000_000, - 12_000_000, - ); - route_modified_otlp_sketches_to_precompute(&req2, &state) - .await - .expect("ingest succeeds"); - - let meta2 = state - .summary_store - .instance(sid) - .expect("sid still registered"); - assert_eq!( - meta2.capability, - Some(Capability::FrequencyTopk(Some( - SketchAlgorithm::CmsWithHeap - ))), - "heap-bearing frame upgrades the sid to FrequencyTopk(CmsWithHeap)" - ); - // Still one instance — the upgrade is an in-place overwrite, not a - // new sid. - assert_eq!(state.summary_store.instance_count(), 1); - - // ── Frame 3: a later heap-LESS frame must NOT downgrade. ── - let plain2 = asap_sketchlib::CountMinSketch::new(ROWS as usize, COLS as usize); - let plain2_bytes = plain2.to_msgpack().expect("serialize plain CMS msgpack"); - let req3 = build_cms_request( - "topk_metric", - ROWS, - COLS, - ENCODING_MSGPACK, - plain2_bytes, - 1_000_000, - 13_000_000, - ); - route_modified_otlp_sketches_to_precompute(&req3, &state) - .await - .expect("ingest succeeds"); - let meta3 = state - .summary_store - .instance(sid) - .expect("sid still registered"); - assert_eq!( - meta3.capability, - Some(Capability::FrequencyTopk(Some( - SketchAlgorithm::CmsWithHeap - ))), - "a later heap-less frame never downgrades the capability" - ); - - drop(state); - let _ = drain.await; - } - - // ── P1-5: empty-attrs (global-aggregation) series can mint a sid ── - - /// P1-5 — a sketch DataPoint with NO attributes (global aggregation, - /// no resource/scope/DP attrs) must still resolve to a sid via the - /// resolver (fingerprint `""`) and register a SketchStore instance, - /// instead of being dropped by the store-lookup-only branch. - #[tokio::test] - async fn attr_less_sketch_dp_mints_a_sid() { - let (state, drain) = make_state().await; - - let dp = DdSketchDataPoint { - attributes: Vec::new(), // global aggregation — no attrs at all - start_time_unix_nano: 1_000_000, - time_unix_nano: 11_000_000, - sketch: vec![1, 2, 3], - encoding: ENCODING_PROTO, - exemplars: Vec::new(), - flags: 0, - series_id: 0, // first emit, no cached sid - }; - let out = route_modified_otlp_sketches_to_precompute( - &build_request("global_latency_ms", dp), - &state, - ) - .await - .expect("ingest succeeds"); - - // Pre-fix: dropped (sid=0 + no attrs → None). Post-fix: resolver - // mints a stable sid for (metric, "", agg_kind) and echoes an - // assignment with an empty fingerprint. - assert_eq!( - out.series_assignments.len(), - 1, - "attr-less DP resolves and echoes one assignment" - ); - let assigned = &out.series_assignments[0]; - assert_ne!(assigned.series_id, 0, "resolver mints a non-zero sid"); - assert!( - assigned.attributes_fingerprint.is_empty(), - "global series carries the empty fingerprint" - ); - assert!( - out.unknown_series_ids.is_empty(), - "no unknown sids — the DP was ingestable, not dropped" - ); - assert_eq!( - state.summary_store.instance_count(), - 1, - "SketchStore registered the global-aggregation instance" - ); - assert!(state.summary_store.instance(assigned.series_id).is_some()); - - drop(state); - let _ = drain.await; - } -} - -// ── B7.6 regression: ingest bucketing is keyed by sid, not (agg_id, group_key) ── -#[cfg(test)] -mod sid_bucketing_tests { - //! Pin the B7.6 contract: ingest dispatches to the precompute engine - //! with `(sid, policy_fp, group_key)` on every `GroupSamples` / - //! `AccumulatorInput`, and distinct group_key values mint distinct - //! sids that round-trip through `SeriesIdResolver::lookup`. Tests - //! the actual `route_otlp_to_precompute` path end-to-end so a - //! refactor that drops the per-DP sid-resolve call (or routes by - //! agg_id) breaks here. - use super::*; - use crate::drivers::ingest::series_resolver::SeriesIdResolver; - use crate::precompute_engine::series_router::{SeriesRouter, WorkerMessage}; - use crate::storage_engines::sketch_db::index::SketchStore; - use crate::storage_engines::types::{InstalledPrecomputePlan, InstalledPrecomputePlanHandle}; - use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; - use asap_otel_proto::tonic::common::v1::{any_value::Value as AnyVal, AnyValue, KeyValue}; - use asap_otel_proto::tonic::metrics::v1::{ - metric::Data, number_data_point::Value as NumberValue, Gauge as PbGauge, - Metric as PbMetric, NumberDataPoint, ResourceMetrics, ScopeMetrics, - }; - use asap_types::aggregation_config::PrecomputeMaterialization; - use asap_types::enums::WindowKind; - use asap_types::AggregationType; - use asap_types::KeyByLabelNames; - use std::collections::HashMap; - use std::sync::Arc; - use tokio::sync::mpsc; - - fn kv(k: &str, v: &str) -> KeyValue { - KeyValue { - key: k.to_string(), - value: Some(AnyValue { - value: Some(AnyVal::StringValue(v.to_string())), - }), - } - } - - fn sum_agg_config(metric: &str, grouping: &[&str]) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::SingleSubpopulation, - "Sum".to_string(), - HashMap::new(), - KeyByLabelNames::new(grouping.iter().map(|s| s.to_string()).collect()), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 10, - 10, - WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ) - } - - /// Build a Gauge request with one DataPoint per (zone, value) entry. - /// Distinct `zone` values are the two grouping-label buckets the - /// test inspects. - fn build_gauge_request(metric: &str, points: &[(&str, f64)]) -> ExportMetricsServiceRequest { - let data_points = points - .iter() - .map(|(zone, val)| NumberDataPoint { - attributes: vec![kv("zone", zone)], - start_time_unix_nano: 1_000_000, - time_unix_nano: 11_000_000, - value: Some(NumberValue::AsDouble(*val)), - exemplars: Vec::new(), - flags: 0, - series_id: 0, - }) - .collect(); - ExportMetricsServiceRequest { - resource_metrics: vec![ResourceMetrics { - resource: None, - scope_metrics: vec![ScopeMetrics { - scope: None, - metrics: vec![PbMetric { - name: metric.to_string(), - description: String::new(), - unit: String::new(), - metadata: Vec::new(), - data: Some(Data::Gauge(PbGauge { data_points })), - }], - schema_url: String::new(), - }], - schema_url: String::new(), - }], - } - } - - /// Two distinct `zone` values under one policy → two distinct - /// `GroupSamples` messages, each keyed by the sid the resolver - /// minted for the corresponding `(metric, zone-only-attrs, - /// ExactAgg-of-config)` tuple. - /// - /// Pre-B7.6 this path dispatched `WorkerMessage::GroupSamples { - /// agg_id, group_key, ... }` and the worker bucketed by - /// `(agg_id, group_key)`. The contract this test pins is: - /// - exactly two `WorkerMessage::GroupSamples` are emitted - /// - their sids are non-zero and distinct - /// - each sid equals what `SeriesIdResolver::lookup` records for - /// `(metric, "zone=;", ExactAgg-canonical)` — i.e. the - /// bucket identity is folded into sid via the resolver - /// - policy_fp = config.policy_fp_u64() on every message - /// - samples in each bucket are exactly the DPs whose `zone` - /// attribute matches that bucket (the GROUP-BY semantic) - /// - /// Note: the `group_key` field on the message comes from - /// `IngestState::extract_group_key_for(series_key, config)`. Pre- - /// PR-after-#284 a quoting mismatch between `format_series_key` - /// (unquoted) and `parse_labels_from_series_key` (quoted) caused - /// this to return the empty string for OTLP wire inputs; the - /// follow-up PR fixed the formatter to emit the canonical - /// PromQL `k="v"` form and added a roundtrip regression in - /// `series_key_roundtrip_tests`. Bucketing was always correct - /// here because B7.6 routes by sid (read directly from - /// `point.labels`, not the joined series_key) — the - /// group_key value is informational only for this test. - #[tokio::test] - async fn raw_otlp_buckets_by_sid_with_distinct_group_keys() { - // Channel large enough to capture all routed messages without - // blocking the dispatch loop. - let (tx, mut rx) = mpsc::channel::(64); - let router = SeriesRouter::new(vec![tx]); - - let metric = "cpu_seconds"; - let cfg = sum_agg_config(metric, &["zone"]); - let policy_fp = asap_types::PolicyFingerprint(cfg.policy_fp_u64()); - let mut configs = HashMap::new(); - configs.insert(cfg.policy_fp_u64(), cfg.clone()); - let streaming = InstalledPrecomputePlan::from_raw_ids(configs); - let hot_reload = InstalledPrecomputePlanHandle::new(streaming); - - let resolver = Arc::new(SeriesIdResolver::new()); - let state = Arc::new(IngestState { - router, - samples_ingested: std::sync::atomic::AtomicU64::new(0), - samples_blocked_by_schema_barrier: std::sync::atomic::AtomicU64::new(0), - hot_reload_config: hot_reload, - pass_raw_samples: false, - sketch_snapshots: dashmap::DashMap::new(), - series_resolver: resolver.clone(), - summary_store: Arc::new(SketchStore::new()), - observability: crate::precompute_engine::ingest_handler::IngestObservability::default(), - }); - - // Two zones × two DPs each. The two zones must produce two - // separate buckets; the two DPs within a zone must accumulate - // into the same bucket. - let req = build_gauge_request( - metric, - &[("z0", 1.0), ("z0", 2.0), ("z1", 10.0), ("z1", 20.0)], - ); - - // PERF-1 — parse once and pass the slices, matching the receiver - // path's new shape. - let (points, sketch_payloads) = otlp_to_metric_points_and_sketches(&req); - route_otlp_to_precompute(&points, &sketch_payloads, &state).await; - - // Drain the messages the dispatcher emitted (one per bucket). - let mut messages: Vec = Vec::new(); - while let Ok(msg) = rx.try_recv() { - messages.push(msg); - } - - // Filter to GroupSamples — the only variant raw OTLP emits. - let groups: Vec<( - u64, - asap_types::PolicyFingerprint, - Arc, - Vec<(String, i64, f64)>, - )> = messages - .into_iter() - .filter_map(|m| match m { - WorkerMessage::GroupSamples { - sid, - policy_fp, - group_key, - samples, - .. - } => Some((sid, policy_fp, group_key, samples)), - _ => None, - }) - .collect(); - - assert_eq!( - groups.len(), - 2, - "exactly two buckets (one per zone) — observed {} messages", - groups.len() - ); - - // Both buckets carry the same policy_fp (one source config). - for (_, pf, _, _) in &groups { - assert_eq!( - *pf, policy_fp, - "policy_fp must equal config.policy_fp_u64()" - ); - } - - // sids must be non-zero (zero is reserved on the wire) and distinct. - let mut sids: Vec = groups.iter().map(|(s, _, _, _)| *s).collect(); - sids.sort(); - sids.dedup(); - assert_eq!(sids.len(), 2, "two distinct sids — one per group_key"); - assert!(sids.iter().all(|&s| s != 0), "sid 0 is reserved"); - - // Each sid must match what the resolver records for its bucket - // identity: (metric, "zone=;", ExactAgg-canonical). Use - // the bucket's sample values to identify which zone it - // represents (sample-value-based identification is robust - // regardless of group_key shape — the test pin is on sid - // assignment, not on group_key content), then verify the - // sid matches the resolver mint for THAT zone. - let agg_kind_canonical = - crate::storage_engines::sketch_db::data::materialization_kind_for_config(&cfg); - for (sid, _, _, samples) in &groups { - let mut vals: Vec = samples.iter().map(|(_, _, v)| *v).collect(); - vals.sort_by(|a, b| a.partial_cmp(b).unwrap()); - let zone_for_bucket: &str = match vals.as_slice() { - [1.0, 2.0] => "z0", - [10.0, 20.0] => "z1", - other => panic!("unexpected bucket sample values: {other:?}"), - }; - let fp = - crate::drivers::ingest::canonical_attrs_fingerprint(&[("zone", zone_for_bucket)]); - let resolved = state - .summary_store - .resolve_output_storage_handle(&resolver, policy_fp.into(), &fp, None) - .ok(); - assert_eq!( - resolved, - Some(*sid), - "sid for zone={zone_for_bucket} (inferred from sample values) must equal \ - resolver mint for (metric={metric}, fp={fp}, agg_kind={agg_kind_canonical})", - ); - } - } - - #[test] - fn summary_frame_identity_is_parsed_and_removed_from_series_labels() { - let mut attrs = HashMap::from([ - ("service".into(), "api".into()), - ("asap.frame.identity_version".into(), "1".into()), - ("asap.frame.plan_id".into(), "42".into()), - ("asap.frame.plan_version".into(), "3".into()), - ( - "asap.frame.backend_compat".into(), - "asap-query-backend.v1".into(), - ), - ("asap.frame.materialization".into(), "99".into()), - ( - "asap.frame.series_identity".into(), - "service=checkout,zone=a".into(), - ), - ("asap.frame.schema_id".into(), "schema-99".into()), - ("asap.frame.producer_id".into(), "edge-a".into()), - ("asap.frame.producer_epoch".into(), "boot-7".into()), - ("asap.frame.sequence".into(), "8".into()), - ("asap.frame.kind".into(), "full".into()), - ("asap.frame.encoding".into(), "sketchlib_protobuf_v1".into()), - ("asap.frame.checkpoint_id".into(), "cp-8".into()), - ]); - let frame = take_summary_frame_identity(&mut attrs, 100, 200).expect("valid identity"); - assert_eq!(frame.plan_id, 42); - assert_eq!(frame.plan_version, 3); - assert_eq!( - frame.materialization, - asap_types::PolicyFingerprint(99).into() - ); - assert_eq!(attrs, HashMap::from([("service".into(), "api".into())])); - } -} diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs deleted file mode 100644 index 6f2c2ba2a..000000000 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ /dev/null @@ -1,1983 +0,0 @@ -//! Prometheus Remote Write v1 adapter for the backend-local precompute path. -//! -//! This module deliberately stops at wire validation, canonical series -//! identity, retry deduplication, and config-driven routing. Aggregation-family -//! selection remains owned by the installed precompute plan. - -use crate::precompute_engine::ingest_handler::IngestState; -use crate::precompute_engine::series_router::{TryRouteError, WorkerMessage}; -use prost::Message; -use std::collections::{BTreeMap, HashMap}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; - -pub const STALE_NAN_BITS: u64 = 0x7ff0_0000_0000_0002; - -#[derive(Clone, PartialEq, Message)] -pub struct WriteRequest { - #[prost(message, repeated, tag = "1")] - pub timeseries: Vec, -} - -#[derive(Clone, PartialEq, Message)] -pub struct TimeSeries { - #[prost(message, repeated, tag = "1")] - pub labels: Vec