diff --git a/.github/workflows/correctness.yml b/.github/workflows/correctness.yml index 4e6f6ab3..d7e6f83d 100644 --- a/.github/workflows/correctness.yml +++ b/.github/workflows/correctness.yml @@ -110,10 +110,10 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable - - name: Install protoc + - name: Install protoc and HiGHS build deps run: | sudo apt-get update -qq - sudo apt-get install -y protobuf-compiler + sudo apt-get install -y protobuf-compiler cmake libclang-dev - name: Run sccache uses: mozilla-actions/sccache-action@v0.0.4 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 01049098..53daaf2d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -46,10 +46,10 @@ jobs: toolchain: stable components: rustfmt, clippy - - name: Install protoc + - name: Install protoc and HiGHS build deps run: | sudo apt-get update - sudo apt-get install -y protobuf-compiler + sudo apt-get install -y protobuf-compiler cmake libclang-dev - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.4 @@ -82,10 +82,10 @@ jobs: with: toolchain: stable - - name: Install protoc + - name: Install protoc and HiGHS build deps run: | sudo apt-get update - sudo apt-get install -y protobuf-compiler + sudo apt-get install -y protobuf-compiler cmake libclang-dev - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.4 diff --git a/Cargo.lock b/Cargo.lock index dc602750..b11704f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,6 +109,36 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "aqpbm-core" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/sketch-bench?branch=main#964ccb5811440245a7d75f93e88649e8463eba86" +dependencies = [ + "anyhow", + "aqpbm-datagen", + "bitflags 2.11.1", + "chrono", + "libc", + "rand 0.9.4", + "rand_xoshiro", + "serde", + "serde_json", +] + +[[package]] +name = "aqpbm-datagen" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/sketch-bench?branch=main#964ccb5811440245a7d75f93e88649e8463eba86" +dependencies = [ + "rand 0.9.4", + "rand_distr", + "rand_xoshiro", + "serde", + "serde_json", + "serde_norway", + "thiserror 2.0.18", +] + [[package]] name = "ar_archive_writer" version = "0.5.1" @@ -141,6 +171,7 @@ dependencies = [ "promql-parser", "promql_utilities", "reqwest", + "rqe-optimizer", "serde", "serde_json", "serde_yaml", @@ -326,6 +357,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.11.1", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.117", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -374,6 +425,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -409,6 +469,17 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "2.34.0" @@ -464,6 +535,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -873,6 +953,16 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "good_lp" +version = "1.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4255eb1d65eea322f73a326038f2bcec19dae13ca50fa557da19b03274b4d56" +dependencies = [ + "fnv", + "highs", +] + [[package]] name = "h2" version = "0.3.27" @@ -980,6 +1070,26 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "highs" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfc791427e78fdc1a0a37983f79c20f86a0b7b7dfdb5d6a57a6f72d77cd9074" +dependencies = [ + "highs-sys", + "log", +] + +[[package]] +name = "highs-sys" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4da0c52322648e2360a5a8896d23301fa5ed99ed7607c5faf229bd0bbb30d3f" +dependencies = [ + "bindgen", + "cmake", +] + [[package]] name = "http" version = "0.2.12" @@ -1402,6 +1512,22 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libsqlite3-sys" version = "0.28.0" @@ -1521,6 +1647,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1559,6 +1691,16 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1581,6 +1723,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -2106,6 +2249,25 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.4", +] + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "recursive" version = "0.1.1" @@ -2223,6 +2385,16 @@ dependencies = [ "serde", ] +[[package]] +name = "rqe-optimizer" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/sketch-bench?branch=main#964ccb5811440245a7d75f93e88649e8463eba86" +dependencies = [ + "aqpbm-core", + "good_lp", + "serde_json", +] + [[package]] name = "rusqlite" version = "0.31.0" @@ -2237,6 +2409,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustix" version = "1.1.4" @@ -2377,6 +2555,19 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_norway" +version = "0.9.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e408f29489b5fd500fab51ff1484fc859bb655f32c671f307dcd733b72e8168c" +dependencies = [ + "indexmap 2.14.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml-norway", +] + [[package]] name = "serde_path_to_error" version = "0.1.20" @@ -3067,6 +3258,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "unsafe-libyaml-norway" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39abd59bf32521c7f2301b52d05a6a2c975b6003521cbd0c6dc1582f0a22104" + [[package]] name = "url" version = "2.5.8" diff --git a/asap-planner-rs/Cargo.toml b/asap-planner-rs/Cargo.toml index a94e78d2..3918f0b1 100644 --- a/asap-planner-rs/Cargo.toml +++ b/asap-planner-rs/Cargo.toml @@ -41,6 +41,7 @@ indexmap.workspace = true chrono.workspace = true promql-parser = "0.5.0" reqwest = { version = "0.11", features = ["blocking", "json"] } +rqe-optimizer = { git = "https://github.com/ProjectASAP/sketch-bench", branch = "main" } [dev-dependencies] tempfile = "3.20" diff --git a/asap-planner-rs/Dockerfile b/asap-planner-rs/Dockerfile index 438955ee..b040253f 100644 --- a/asap-planner-rs/Dockerfile +++ b/asap-planner-rs/Dockerfile @@ -5,9 +5,11 @@ LABEL description="ASAP Planner (Rust) for SketchDB" WORKDIR /code -# Required by prost-build dependencies during cargo build +# protobuf-compiler: prost-build. cmake, libclang-dev: HiGHS (rqe-optimizer's MILP solver). RUN apt-get update && apt-get install -y --no-install-recommends \ protobuf-compiler \ + cmake \ + libclang-dev \ && rm -rf /var/lib/apt/lists/* COPY asap-common ./asap-common @@ -42,6 +44,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ libssl3 \ zlib1g \ + libstdc++6 \ && rm -rf /var/lib/apt/lists/* RUN mkdir -p /app/input /app/output diff --git a/asap-planner-rs/tests/rqe_optimizer_smoke.rs b/asap-planner-rs/tests/rqe_optimizer_smoke.rs new file mode 100644 index 00000000..03858aa5 --- /dev/null +++ b/asap-planner-rs/tests/rqe_optimizer_smoke.rs @@ -0,0 +1,66 @@ +//! Links sketch-bench's rqe-optimizer and solves a one-query MILP with HiGHS. + +use std::collections::BTreeMap; + +use rqe_optimizer::milp::{minimize_cost, MilpBounds}; +use rqe_optimizer::objectives::MachineFamily; +use rqe_optimizer::{ + AccuracyDirection, AtomicCostEntry, Capability, Deployment, LabelSet, LabelSetInfo, Rqe, +}; + +fn deployment(insert_cpu_secs: f64) -> Deployment { + Deployment { + capability: Capability::Quantile, + labels: LabelSet::new(), + config: AtomicCostEntry { + sketch: "kll-percall".into(), + sketch_config: serde_json::json!(null), + mem_bytes_per_instance: 1024.0, + insert_cpu_secs, + merge_cpu_secs: 1.0, + query_cpu_secs: 1.0, + query_accuracy: BTreeMap::from([("err".into(), 0.0)]), + }, + window_secs: 60, + slide_secs: 60, + } +} + +#[test] +fn minimize_cost_picks_the_cheaper_deployment() { + let rqes = vec![Rqe { + id: "q".into(), + capability: Capability::Quantile, + lookback_secs: 60, + interval_secs: 60, + labels: LabelSet::new(), + accuracy_metric: "err".into(), + accuracy_tolerance: 1.0, + accuracy_direction: AccuracyDirection::LowerIsBetter, + }]; + let deployments = vec![deployment(2.0), deployment(1.0)]; + let label_sets = BTreeMap::from([( + LabelSet::new(), + LabelSetInfo { + cardinality: 1, + arrival_rate_per_sec: 1.0, + }, + )]); + let family = MachineFamily { + family: "test".into(), + vcpu: 4.0, + memory_gib: 16.0, + usd_per_hour: 1.0, + }; + + let solution = minimize_cost( + &rqes, + &deployments, + &label_sets, + &MilpBounds::default(), + &family, + ) + .expect("feasible MILP"); + + assert_eq!(solution.mapping, vec![1]); +} diff --git a/asap-query-engine/Dockerfile b/asap-query-engine/Dockerfile index bc9a13bc..ea778113 100644 --- a/asap-query-engine/Dockerfile +++ b/asap-query-engine/Dockerfile @@ -8,9 +8,12 @@ LABEL description="QueryEngine Rust service for SketchDB" WORKDIR /code -# Required by prost-build dependencies (e.g., asap_sketchlib) during cargo build +# protobuf-compiler: prost-build (e.g., asap_sketchlib). cmake, libclang-dev: +# HiGHS, built through the asap_planner dependency. RUN apt-get update && apt-get install -y --no-install-recommends \ protobuf-compiler \ + cmake \ + libclang-dev \ && rm -rf /var/lib/apt/lists/* # Copy the asap-common directory