diff --git a/asap-query-engine/src/utils/file_io.rs b/asap-query-engine/src/utils/file_io.rs index dd129d49..f057f71a 100644 --- a/asap-query-engine/src/utils/file_io.rs +++ b/asap-query-engine/src/utils/file_io.rs @@ -41,6 +41,43 @@ mod tests { use std::io::Write; use tempfile::NamedTempFile; + /// The PromQL compliance roll-up suite ships hand-written planner output; + /// parse it the way the engine does so it can't drift from the format. + #[test] + fn compliance_rollup_planner_output_loads_as_engine_config() { + let directory = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../promql-compliance/suites/rollup-planner-output" + ); + let inference = read_inference_config( + &format!("{directory}/inference_config.yaml"), + QueryLanguage::promql, + ) + .unwrap(); + let streaming = + read_streaming_config(&format!("{directory}/streaming_config.yaml"), &inference) + .unwrap(); + + assert!(!inference.query_configs.is_empty()); + for query_config in &inference.query_configs { + assert!( + !query_config.aggregations.is_empty(), + "{}", + query_config.query + ); + for reference in &query_config.aggregations { + assert!( + streaming + .get_aggregation_config(reference.aggregation_id) + .is_some(), + "{} names missing aggregation {}", + query_config.query, + reference.aggregation_id + ); + } + } + } + #[test] fn test_read_streaming_config() { let streaming_yaml_content = r#" diff --git a/promql-compliance/ARCHITECTURE.md b/promql-compliance/ARCHITECTURE.md index e7da232a..23998fad 100644 --- a/promql-compliance/ARCHITECTURE.md +++ b/promql-compliance/ARCHITECTURE.md @@ -54,7 +54,9 @@ The runner performs these stages: are relative to this timestamp. 3. Generate temporary planner and engine configuration. 4. Start Prometheus and wait for its health check. -5. Run the planner. Its output is placed in a named shared volume. +5. Run the planner. Its output is placed in a named shared volume. A suite + with `planner_output_dir` skips the planner and mounts that directory + instead. 6. Start the ASAPQuery query engine after the planner completes. 7. Send the same encoded remote-write batches to Prometheus and ASAPQuery. 8. Wait for both query APIs to be ready and for the first probe query to return diff --git a/promql-compliance/HOW_TO.md b/promql-compliance/HOW_TO.md index 71bacc33..d3a99a1f 100644 --- a/promql-compliance/HOW_TO.md +++ b/promql-compliance/HOW_TO.md @@ -128,6 +128,27 @@ To change the five-minute cap or one-second ingestion assumption, change the runner’s generated-config logic in `runner/run.go`. Do not edit the generated configuration: it is temporary and is deleted after the run. +## Use fixed planner output + +By default the planner generates the engine's streaming and inference +configs from the suite. To test configs the planner does not produce, such as +roll-ups from a finer grouping, check the configs in next to the suite and +point the suite at their directory: + +```yaml +name: rollup +planner_output_dir: rollup-planner-output +``` + +The path is relative to the suite file. The directory must contain +`streaming_config.yaml` and `inference_config.yaml` in the planner's output +format; the runner fails at load time if either is missing. The planner +service then does not run, and the query engine reads the directory in place +of the planner's output volume. This needs Docker Compose v2.24.4 or later +(for `!override`); the runner checks the version first. With +already-running targets the field has no effect, and the planner input is +generated and validated as usual. `suites/rollup.yaml` is an example. + ## Configure tolerance Set a suite-wide default: diff --git a/promql-compliance/datasets/rollup.yaml b/promql-compliance/datasets/rollup.yaml new file mode 100644 index 00000000..9091e645 --- /dev/null +++ b/promql-compliance/datasets/rollup.yaml @@ -0,0 +1,569 @@ +name: rollup +series: + # rollup_latency{service, endpoint}: 2 services x 10 endpoints at 60s + # cadence, plus endpoint e-late on service a that starts at offset 600 so + # roll-ups see a subpopulation appear mid-range. + # + # With i = endpoint*2 + service (e-late is i = 20), value = 1000 + 2*i + + # 10*(t/60). Within one service adjacent values are 4 apart (under 0.5%), + # which bounds the gap between ASAPQuery's nearest-rank quantile and + # Prometheus' interpolated one well inside the suite's sketch tolerance. + # + # Samples run to offset 1260, past the last evaluated offset (1200). + - metric: rollup_latency + labels: + service: a + endpoint: e00 + samples: + - {offset_seconds: 0, value: 1000} + - {offset_seconds: 60, value: 1010} + - {offset_seconds: 120, value: 1020} + - {offset_seconds: 180, value: 1030} + - {offset_seconds: 240, value: 1040} + - {offset_seconds: 300, value: 1050} + - {offset_seconds: 360, value: 1060} + - {offset_seconds: 420, value: 1070} + - {offset_seconds: 480, value: 1080} + - {offset_seconds: 540, value: 1090} + - {offset_seconds: 600, value: 1100} + - {offset_seconds: 660, value: 1110} + - {offset_seconds: 720, value: 1120} + - {offset_seconds: 780, value: 1130} + - {offset_seconds: 840, value: 1140} + - {offset_seconds: 900, value: 1150} + - {offset_seconds: 960, value: 1160} + - {offset_seconds: 1020, value: 1170} + - {offset_seconds: 1080, value: 1180} + - {offset_seconds: 1140, value: 1190} + - {offset_seconds: 1200, value: 1200} + - {offset_seconds: 1260, value: 1210} + - metric: rollup_latency + labels: + service: b + endpoint: e00 + samples: + - {offset_seconds: 0, value: 1002} + - {offset_seconds: 60, value: 1012} + - {offset_seconds: 120, value: 1022} + - {offset_seconds: 180, value: 1032} + - {offset_seconds: 240, value: 1042} + - {offset_seconds: 300, value: 1052} + - {offset_seconds: 360, value: 1062} + - {offset_seconds: 420, value: 1072} + - {offset_seconds: 480, value: 1082} + - {offset_seconds: 540, value: 1092} + - {offset_seconds: 600, value: 1102} + - {offset_seconds: 660, value: 1112} + - {offset_seconds: 720, value: 1122} + - {offset_seconds: 780, value: 1132} + - {offset_seconds: 840, value: 1142} + - {offset_seconds: 900, value: 1152} + - {offset_seconds: 960, value: 1162} + - {offset_seconds: 1020, value: 1172} + - {offset_seconds: 1080, value: 1182} + - {offset_seconds: 1140, value: 1192} + - {offset_seconds: 1200, value: 1202} + - {offset_seconds: 1260, value: 1212} + - metric: rollup_latency + labels: + service: a + endpoint: e01 + samples: + - {offset_seconds: 0, value: 1004} + - {offset_seconds: 60, value: 1014} + - {offset_seconds: 120, value: 1024} + - {offset_seconds: 180, value: 1034} + - {offset_seconds: 240, value: 1044} + - {offset_seconds: 300, value: 1054} + - {offset_seconds: 360, value: 1064} + - {offset_seconds: 420, value: 1074} + - {offset_seconds: 480, value: 1084} + - {offset_seconds: 540, value: 1094} + - {offset_seconds: 600, value: 1104} + - {offset_seconds: 660, value: 1114} + - {offset_seconds: 720, value: 1124} + - {offset_seconds: 780, value: 1134} + - {offset_seconds: 840, value: 1144} + - {offset_seconds: 900, value: 1154} + - {offset_seconds: 960, value: 1164} + - {offset_seconds: 1020, value: 1174} + - {offset_seconds: 1080, value: 1184} + - {offset_seconds: 1140, value: 1194} + - {offset_seconds: 1200, value: 1204} + - {offset_seconds: 1260, value: 1214} + - metric: rollup_latency + labels: + service: b + endpoint: e01 + samples: + - {offset_seconds: 0, value: 1006} + - {offset_seconds: 60, value: 1016} + - {offset_seconds: 120, value: 1026} + - {offset_seconds: 180, value: 1036} + - {offset_seconds: 240, value: 1046} + - {offset_seconds: 300, value: 1056} + - {offset_seconds: 360, value: 1066} + - {offset_seconds: 420, value: 1076} + - {offset_seconds: 480, value: 1086} + - {offset_seconds: 540, value: 1096} + - {offset_seconds: 600, value: 1106} + - {offset_seconds: 660, value: 1116} + - {offset_seconds: 720, value: 1126} + - {offset_seconds: 780, value: 1136} + - {offset_seconds: 840, value: 1146} + - {offset_seconds: 900, value: 1156} + - {offset_seconds: 960, value: 1166} + - {offset_seconds: 1020, value: 1176} + - {offset_seconds: 1080, value: 1186} + - {offset_seconds: 1140, value: 1196} + - {offset_seconds: 1200, value: 1206} + - {offset_seconds: 1260, value: 1216} + - metric: rollup_latency + labels: + service: a + endpoint: e02 + samples: + - {offset_seconds: 0, value: 1008} + - {offset_seconds: 60, value: 1018} + - {offset_seconds: 120, value: 1028} + - {offset_seconds: 180, value: 1038} + - {offset_seconds: 240, value: 1048} + - {offset_seconds: 300, value: 1058} + - {offset_seconds: 360, value: 1068} + - {offset_seconds: 420, value: 1078} + - {offset_seconds: 480, value: 1088} + - {offset_seconds: 540, value: 1098} + - {offset_seconds: 600, value: 1108} + - {offset_seconds: 660, value: 1118} + - {offset_seconds: 720, value: 1128} + - {offset_seconds: 780, value: 1138} + - {offset_seconds: 840, value: 1148} + - {offset_seconds: 900, value: 1158} + - {offset_seconds: 960, value: 1168} + - {offset_seconds: 1020, value: 1178} + - {offset_seconds: 1080, value: 1188} + - {offset_seconds: 1140, value: 1198} + - {offset_seconds: 1200, value: 1208} + - {offset_seconds: 1260, value: 1218} + - metric: rollup_latency + labels: + service: b + endpoint: e02 + samples: + - {offset_seconds: 0, value: 1010} + - {offset_seconds: 60, value: 1020} + - {offset_seconds: 120, value: 1030} + - {offset_seconds: 180, value: 1040} + - {offset_seconds: 240, value: 1050} + - {offset_seconds: 300, value: 1060} + - {offset_seconds: 360, value: 1070} + - {offset_seconds: 420, value: 1080} + - {offset_seconds: 480, value: 1090} + - {offset_seconds: 540, value: 1100} + - {offset_seconds: 600, value: 1110} + - {offset_seconds: 660, value: 1120} + - {offset_seconds: 720, value: 1130} + - {offset_seconds: 780, value: 1140} + - {offset_seconds: 840, value: 1150} + - {offset_seconds: 900, value: 1160} + - {offset_seconds: 960, value: 1170} + - {offset_seconds: 1020, value: 1180} + - {offset_seconds: 1080, value: 1190} + - {offset_seconds: 1140, value: 1200} + - {offset_seconds: 1200, value: 1210} + - {offset_seconds: 1260, value: 1220} + - metric: rollup_latency + labels: + service: a + endpoint: e03 + samples: + - {offset_seconds: 0, value: 1012} + - {offset_seconds: 60, value: 1022} + - {offset_seconds: 120, value: 1032} + - {offset_seconds: 180, value: 1042} + - {offset_seconds: 240, value: 1052} + - {offset_seconds: 300, value: 1062} + - {offset_seconds: 360, value: 1072} + - {offset_seconds: 420, value: 1082} + - {offset_seconds: 480, value: 1092} + - {offset_seconds: 540, value: 1102} + - {offset_seconds: 600, value: 1112} + - {offset_seconds: 660, value: 1122} + - {offset_seconds: 720, value: 1132} + - {offset_seconds: 780, value: 1142} + - {offset_seconds: 840, value: 1152} + - {offset_seconds: 900, value: 1162} + - {offset_seconds: 960, value: 1172} + - {offset_seconds: 1020, value: 1182} + - {offset_seconds: 1080, value: 1192} + - {offset_seconds: 1140, value: 1202} + - {offset_seconds: 1200, value: 1212} + - {offset_seconds: 1260, value: 1222} + - metric: rollup_latency + labels: + service: b + endpoint: e03 + samples: + - {offset_seconds: 0, value: 1014} + - {offset_seconds: 60, value: 1024} + - {offset_seconds: 120, value: 1034} + - {offset_seconds: 180, value: 1044} + - {offset_seconds: 240, value: 1054} + - {offset_seconds: 300, value: 1064} + - {offset_seconds: 360, value: 1074} + - {offset_seconds: 420, value: 1084} + - {offset_seconds: 480, value: 1094} + - {offset_seconds: 540, value: 1104} + - {offset_seconds: 600, value: 1114} + - {offset_seconds: 660, value: 1124} + - {offset_seconds: 720, value: 1134} + - {offset_seconds: 780, value: 1144} + - {offset_seconds: 840, value: 1154} + - {offset_seconds: 900, value: 1164} + - {offset_seconds: 960, value: 1174} + - {offset_seconds: 1020, value: 1184} + - {offset_seconds: 1080, value: 1194} + - {offset_seconds: 1140, value: 1204} + - {offset_seconds: 1200, value: 1214} + - {offset_seconds: 1260, value: 1224} + - metric: rollup_latency + labels: + service: a + endpoint: e04 + samples: + - {offset_seconds: 0, value: 1016} + - {offset_seconds: 60, value: 1026} + - {offset_seconds: 120, value: 1036} + - {offset_seconds: 180, value: 1046} + - {offset_seconds: 240, value: 1056} + - {offset_seconds: 300, value: 1066} + - {offset_seconds: 360, value: 1076} + - {offset_seconds: 420, value: 1086} + - {offset_seconds: 480, value: 1096} + - {offset_seconds: 540, value: 1106} + - {offset_seconds: 600, value: 1116} + - {offset_seconds: 660, value: 1126} + - {offset_seconds: 720, value: 1136} + - {offset_seconds: 780, value: 1146} + - {offset_seconds: 840, value: 1156} + - {offset_seconds: 900, value: 1166} + - {offset_seconds: 960, value: 1176} + - {offset_seconds: 1020, value: 1186} + - {offset_seconds: 1080, value: 1196} + - {offset_seconds: 1140, value: 1206} + - {offset_seconds: 1200, value: 1216} + - {offset_seconds: 1260, value: 1226} + - metric: rollup_latency + labels: + service: b + endpoint: e04 + samples: + - {offset_seconds: 0, value: 1018} + - {offset_seconds: 60, value: 1028} + - {offset_seconds: 120, value: 1038} + - {offset_seconds: 180, value: 1048} + - {offset_seconds: 240, value: 1058} + - {offset_seconds: 300, value: 1068} + - {offset_seconds: 360, value: 1078} + - {offset_seconds: 420, value: 1088} + - {offset_seconds: 480, value: 1098} + - {offset_seconds: 540, value: 1108} + - {offset_seconds: 600, value: 1118} + - {offset_seconds: 660, value: 1128} + - {offset_seconds: 720, value: 1138} + - {offset_seconds: 780, value: 1148} + - {offset_seconds: 840, value: 1158} + - {offset_seconds: 900, value: 1168} + - {offset_seconds: 960, value: 1178} + - {offset_seconds: 1020, value: 1188} + - {offset_seconds: 1080, value: 1198} + - {offset_seconds: 1140, value: 1208} + - {offset_seconds: 1200, value: 1218} + - {offset_seconds: 1260, value: 1228} + - metric: rollup_latency + labels: + service: a + endpoint: e05 + samples: + - {offset_seconds: 0, value: 1020} + - {offset_seconds: 60, value: 1030} + - {offset_seconds: 120, value: 1040} + - {offset_seconds: 180, value: 1050} + - {offset_seconds: 240, value: 1060} + - {offset_seconds: 300, value: 1070} + - {offset_seconds: 360, value: 1080} + - {offset_seconds: 420, value: 1090} + - {offset_seconds: 480, value: 1100} + - {offset_seconds: 540, value: 1110} + - {offset_seconds: 600, value: 1120} + - {offset_seconds: 660, value: 1130} + - {offset_seconds: 720, value: 1140} + - {offset_seconds: 780, value: 1150} + - {offset_seconds: 840, value: 1160} + - {offset_seconds: 900, value: 1170} + - {offset_seconds: 960, value: 1180} + - {offset_seconds: 1020, value: 1190} + - {offset_seconds: 1080, value: 1200} + - {offset_seconds: 1140, value: 1210} + - {offset_seconds: 1200, value: 1220} + - {offset_seconds: 1260, value: 1230} + - metric: rollup_latency + labels: + service: b + endpoint: e05 + samples: + - {offset_seconds: 0, value: 1022} + - {offset_seconds: 60, value: 1032} + - {offset_seconds: 120, value: 1042} + - {offset_seconds: 180, value: 1052} + - {offset_seconds: 240, value: 1062} + - {offset_seconds: 300, value: 1072} + - {offset_seconds: 360, value: 1082} + - {offset_seconds: 420, value: 1092} + - {offset_seconds: 480, value: 1102} + - {offset_seconds: 540, value: 1112} + - {offset_seconds: 600, value: 1122} + - {offset_seconds: 660, value: 1132} + - {offset_seconds: 720, value: 1142} + - {offset_seconds: 780, value: 1152} + - {offset_seconds: 840, value: 1162} + - {offset_seconds: 900, value: 1172} + - {offset_seconds: 960, value: 1182} + - {offset_seconds: 1020, value: 1192} + - {offset_seconds: 1080, value: 1202} + - {offset_seconds: 1140, value: 1212} + - {offset_seconds: 1200, value: 1222} + - {offset_seconds: 1260, value: 1232} + - metric: rollup_latency + labels: + service: a + endpoint: e06 + samples: + - {offset_seconds: 0, value: 1024} + - {offset_seconds: 60, value: 1034} + - {offset_seconds: 120, value: 1044} + - {offset_seconds: 180, value: 1054} + - {offset_seconds: 240, value: 1064} + - {offset_seconds: 300, value: 1074} + - {offset_seconds: 360, value: 1084} + - {offset_seconds: 420, value: 1094} + - {offset_seconds: 480, value: 1104} + - {offset_seconds: 540, value: 1114} + - {offset_seconds: 600, value: 1124} + - {offset_seconds: 660, value: 1134} + - {offset_seconds: 720, value: 1144} + - {offset_seconds: 780, value: 1154} + - {offset_seconds: 840, value: 1164} + - {offset_seconds: 900, value: 1174} + - {offset_seconds: 960, value: 1184} + - {offset_seconds: 1020, value: 1194} + - {offset_seconds: 1080, value: 1204} + - {offset_seconds: 1140, value: 1214} + - {offset_seconds: 1200, value: 1224} + - {offset_seconds: 1260, value: 1234} + - metric: rollup_latency + labels: + service: b + endpoint: e06 + samples: + - {offset_seconds: 0, value: 1026} + - {offset_seconds: 60, value: 1036} + - {offset_seconds: 120, value: 1046} + - {offset_seconds: 180, value: 1056} + - {offset_seconds: 240, value: 1066} + - {offset_seconds: 300, value: 1076} + - {offset_seconds: 360, value: 1086} + - {offset_seconds: 420, value: 1096} + - {offset_seconds: 480, value: 1106} + - {offset_seconds: 540, value: 1116} + - {offset_seconds: 600, value: 1126} + - {offset_seconds: 660, value: 1136} + - {offset_seconds: 720, value: 1146} + - {offset_seconds: 780, value: 1156} + - {offset_seconds: 840, value: 1166} + - {offset_seconds: 900, value: 1176} + - {offset_seconds: 960, value: 1186} + - {offset_seconds: 1020, value: 1196} + - {offset_seconds: 1080, value: 1206} + - {offset_seconds: 1140, value: 1216} + - {offset_seconds: 1200, value: 1226} + - {offset_seconds: 1260, value: 1236} + - metric: rollup_latency + labels: + service: a + endpoint: e07 + samples: + - {offset_seconds: 0, value: 1028} + - {offset_seconds: 60, value: 1038} + - {offset_seconds: 120, value: 1048} + - {offset_seconds: 180, value: 1058} + - {offset_seconds: 240, value: 1068} + - {offset_seconds: 300, value: 1078} + - {offset_seconds: 360, value: 1088} + - {offset_seconds: 420, value: 1098} + - {offset_seconds: 480, value: 1108} + - {offset_seconds: 540, value: 1118} + - {offset_seconds: 600, value: 1128} + - {offset_seconds: 660, value: 1138} + - {offset_seconds: 720, value: 1148} + - {offset_seconds: 780, value: 1158} + - {offset_seconds: 840, value: 1168} + - {offset_seconds: 900, value: 1178} + - {offset_seconds: 960, value: 1188} + - {offset_seconds: 1020, value: 1198} + - {offset_seconds: 1080, value: 1208} + - {offset_seconds: 1140, value: 1218} + - {offset_seconds: 1200, value: 1228} + - {offset_seconds: 1260, value: 1238} + - metric: rollup_latency + labels: + service: b + endpoint: e07 + samples: + - {offset_seconds: 0, value: 1030} + - {offset_seconds: 60, value: 1040} + - {offset_seconds: 120, value: 1050} + - {offset_seconds: 180, value: 1060} + - {offset_seconds: 240, value: 1070} + - {offset_seconds: 300, value: 1080} + - {offset_seconds: 360, value: 1090} + - {offset_seconds: 420, value: 1100} + - {offset_seconds: 480, value: 1110} + - {offset_seconds: 540, value: 1120} + - {offset_seconds: 600, value: 1130} + - {offset_seconds: 660, value: 1140} + - {offset_seconds: 720, value: 1150} + - {offset_seconds: 780, value: 1160} + - {offset_seconds: 840, value: 1170} + - {offset_seconds: 900, value: 1180} + - {offset_seconds: 960, value: 1190} + - {offset_seconds: 1020, value: 1200} + - {offset_seconds: 1080, value: 1210} + - {offset_seconds: 1140, value: 1220} + - {offset_seconds: 1200, value: 1230} + - {offset_seconds: 1260, value: 1240} + - metric: rollup_latency + labels: + service: a + endpoint: e08 + samples: + - {offset_seconds: 0, value: 1032} + - {offset_seconds: 60, value: 1042} + - {offset_seconds: 120, value: 1052} + - {offset_seconds: 180, value: 1062} + - {offset_seconds: 240, value: 1072} + - {offset_seconds: 300, value: 1082} + - {offset_seconds: 360, value: 1092} + - {offset_seconds: 420, value: 1102} + - {offset_seconds: 480, value: 1112} + - {offset_seconds: 540, value: 1122} + - {offset_seconds: 600, value: 1132} + - {offset_seconds: 660, value: 1142} + - {offset_seconds: 720, value: 1152} + - {offset_seconds: 780, value: 1162} + - {offset_seconds: 840, value: 1172} + - {offset_seconds: 900, value: 1182} + - {offset_seconds: 960, value: 1192} + - {offset_seconds: 1020, value: 1202} + - {offset_seconds: 1080, value: 1212} + - {offset_seconds: 1140, value: 1222} + - {offset_seconds: 1200, value: 1232} + - {offset_seconds: 1260, value: 1242} + - metric: rollup_latency + labels: + service: b + endpoint: e08 + samples: + - {offset_seconds: 0, value: 1034} + - {offset_seconds: 60, value: 1044} + - {offset_seconds: 120, value: 1054} + - {offset_seconds: 180, value: 1064} + - {offset_seconds: 240, value: 1074} + - {offset_seconds: 300, value: 1084} + - {offset_seconds: 360, value: 1094} + - {offset_seconds: 420, value: 1104} + - {offset_seconds: 480, value: 1114} + - {offset_seconds: 540, value: 1124} + - {offset_seconds: 600, value: 1134} + - {offset_seconds: 660, value: 1144} + - {offset_seconds: 720, value: 1154} + - {offset_seconds: 780, value: 1164} + - {offset_seconds: 840, value: 1174} + - {offset_seconds: 900, value: 1184} + - {offset_seconds: 960, value: 1194} + - {offset_seconds: 1020, value: 1204} + - {offset_seconds: 1080, value: 1214} + - {offset_seconds: 1140, value: 1224} + - {offset_seconds: 1200, value: 1234} + - {offset_seconds: 1260, value: 1244} + - metric: rollup_latency + labels: + service: a + endpoint: e09 + samples: + - {offset_seconds: 0, value: 1036} + - {offset_seconds: 60, value: 1046} + - {offset_seconds: 120, value: 1056} + - {offset_seconds: 180, value: 1066} + - {offset_seconds: 240, value: 1076} + - {offset_seconds: 300, value: 1086} + - {offset_seconds: 360, value: 1096} + - {offset_seconds: 420, value: 1106} + - {offset_seconds: 480, value: 1116} + - {offset_seconds: 540, value: 1126} + - {offset_seconds: 600, value: 1136} + - {offset_seconds: 660, value: 1146} + - {offset_seconds: 720, value: 1156} + - {offset_seconds: 780, value: 1166} + - {offset_seconds: 840, value: 1176} + - {offset_seconds: 900, value: 1186} + - {offset_seconds: 960, value: 1196} + - {offset_seconds: 1020, value: 1206} + - {offset_seconds: 1080, value: 1216} + - {offset_seconds: 1140, value: 1226} + - {offset_seconds: 1200, value: 1236} + - {offset_seconds: 1260, value: 1246} + - metric: rollup_latency + labels: + service: b + endpoint: e09 + samples: + - {offset_seconds: 0, value: 1038} + - {offset_seconds: 60, value: 1048} + - {offset_seconds: 120, value: 1058} + - {offset_seconds: 180, value: 1068} + - {offset_seconds: 240, value: 1078} + - {offset_seconds: 300, value: 1088} + - {offset_seconds: 360, value: 1098} + - {offset_seconds: 420, value: 1108} + - {offset_seconds: 480, value: 1118} + - {offset_seconds: 540, value: 1128} + - {offset_seconds: 600, value: 1138} + - {offset_seconds: 660, value: 1148} + - {offset_seconds: 720, value: 1158} + - {offset_seconds: 780, value: 1168} + - {offset_seconds: 840, value: 1178} + - {offset_seconds: 900, value: 1188} + - {offset_seconds: 960, value: 1198} + - {offset_seconds: 1020, value: 1208} + - {offset_seconds: 1080, value: 1218} + - {offset_seconds: 1140, value: 1228} + - {offset_seconds: 1200, value: 1238} + - {offset_seconds: 1260, value: 1248} + - metric: rollup_latency + labels: + service: a + endpoint: e-late + samples: + - {offset_seconds: 600, value: 1140} + - {offset_seconds: 660, value: 1150} + - {offset_seconds: 720, value: 1160} + - {offset_seconds: 780, value: 1170} + - {offset_seconds: 840, value: 1180} + - {offset_seconds: 900, value: 1190} + - {offset_seconds: 960, value: 1200} + - {offset_seconds: 1020, value: 1210} + - {offset_seconds: 1080, value: 1220} + - {offset_seconds: 1140, value: 1230} + - {offset_seconds: 1200, value: 1240} + - {offset_seconds: 1260, value: 1250} diff --git a/promql-compliance/runner/Makefile b/promql-compliance/runner/Makefile index 0461ac2e..3f501175 100644 --- a/promql-compliance/runner/Makefile +++ b/promql-compliance/runner/Makefile @@ -13,7 +13,8 @@ CI_CASES := \ aggregations-native-dag:../datasets/aggregations.yaml:../suites/native-dag-aggregations.yaml \ aggregations:../datasets/aggregations.yaml:../suites/aggregations.yaml \ topk-ordering:../datasets/topk-ordering.yaml:../suites/topk-ordering.yaml \ - quantiles:../datasets/quantiles.yaml:../suites/quantiles.yaml + quantiles:../datasets/quantiles.yaml:../suites/quantiles.yaml \ + rollup:../datasets/rollup.yaml:../suites/rollup.yaml # Tracks planner coverage (#738); most queries are expected to fail. NON_CI_CASES := \ diff --git a/promql-compliance/runner/checked_in_fixture_validation_test.go b/promql-compliance/runner/checked_in_fixture_validation_test.go index dea0a89c..18b56400 100644 --- a/promql-compliance/runner/checked_in_fixture_validation_test.go +++ b/promql-compliance/runner/checked_in_fixture_validation_test.go @@ -1,10 +1,13 @@ package runner import ( + "os" + "path/filepath" "testing" "github.com/ProjectASAP/ASAPQuery/promql-compliance/seeder" promqlparser "github.com/prometheus/prometheus/promql/parser" + "gopkg.in/yaml.v3" ) func TestCheckedInFixturesAndSuites(t *testing.T) { @@ -80,3 +83,97 @@ func TestCheckedInNativeDagSuites(t *testing.T) { } } } + +// Every rollup suite query must have a query config in the fixed planner +// output, naming aggregations that exist, on a metric and labels the dataset +// has; a mismatch would otherwise surface only as a slow Docker-suite failure. +func TestCheckedInRollupSuiteMatchesFixedPlannerOutput(t *testing.T) { + fixture, err := seeder.LoadFixture("../datasets/rollup.yaml") + if err != nil { + t.Fatalf("LoadFixture: %v", err) + } + suite, err := LoadSuiteFile("../suites/rollup.yaml") + if err != nil { + t.Fatalf("LoadSuiteFile: %v", err) + } + if suite.PlannerOutputDir == "" { + t.Fatal("rollup suite has no planner_output_dir") + } + contents, err := os.ReadFile(filepath.Join(suite.PlannerOutputDir, plannerInferenceConfigFile)) + if err != nil { + t.Fatalf("read inference config: %v", err) + } + var inference struct { + Queries []struct { + Query string `yaml:"query"` + Aggregations []struct { + AggregationID int `yaml:"aggregation_id"` + } `yaml:"aggregations"` + } `yaml:"queries"` + } + if err := yaml.Unmarshal(contents, &inference); err != nil { + t.Fatalf("parse inference config: %v", err) + } + streamingContents, err := os.ReadFile(filepath.Join(suite.PlannerOutputDir, plannerStreamingConfigFile)) + if err != nil { + t.Fatalf("read streaming config: %v", err) + } + var streaming struct { + Aggregations []struct { + AggregationID int `yaml:"aggregationId"` + Metric string `yaml:"metric"` + Labels struct { + Grouping []string `yaml:"grouping"` + } `yaml:"labels"` + } `yaml:"aggregations"` + } + if err := yaml.Unmarshal(streamingContents, &streaming); err != nil { + t.Fatalf("parse streaming config: %v", err) + } + datasetLabels := make(map[string]map[string]struct{}) + for _, series := range fixture.Series { + if datasetLabels[series.Metric] == nil { + datasetLabels[series.Metric] = make(map[string]struct{}) + } + for label := range series.Labels { + datasetLabels[series.Metric][label] = struct{}{} + } + } + aggregationIDs := make(map[int]struct{}, len(streaming.Aggregations)) + for _, aggregation := range streaming.Aggregations { + aggregationIDs[aggregation.AggregationID] = struct{}{} + labels, ok := datasetLabels[aggregation.Metric] + if !ok { + t.Errorf("aggregation %d is on metric %q, which the dataset lacks", aggregation.AggregationID, aggregation.Metric) + continue + } + for _, label := range aggregation.Labels.Grouping { + if _, ok := labels[label]; !ok { + t.Errorf("aggregation %d groups by %q, which %q series lack", aggregation.AggregationID, label, aggregation.Metric) + } + } + } + for _, query := range inference.Queries { + for _, reference := range query.Aggregations { + if _, ok := aggregationIDs[reference.AggregationID]; !ok { + t.Errorf("query %q names aggregation %d, which the streaming config lacks", query.Query, reference.AggregationID) + } + } + } + configured := make(map[string]struct{}, len(inference.Queries)) + for _, query := range inference.Queries { + configured[query.Query] = struct{}{} + } + parser := promqlparser.NewParser(promqlparser.Options{}) + for _, query := range suite.Queries { + if _, err := parser.ParseExpr(query.Expr); err != nil { + t.Fatalf("parse %q: %v", query.Expr, err) + } + if _, ok := configured[query.Expr]; !ok { + t.Errorf("query %q has no query config in the fixed planner output", query.Expr) + } + } + if got, want := len(inference.Queries), len(suite.Queries); got != want { + t.Errorf("fixed planner output has %d query configs, suite has %d queries", got, want) + } +} diff --git a/promql-compliance/runner/config.go b/promql-compliance/runner/config.go index a0857636..5eee6309 100644 --- a/promql-compliance/runner/config.go +++ b/promql-compliance/runner/config.go @@ -5,6 +5,7 @@ import ( "fmt" "math" "os" + "path/filepath" "time" "gopkg.in/yaml.v3" @@ -15,6 +16,10 @@ const ( instantOrderDescending = "descending" orderGroupingBy = "by" orderGroupingWithout = "without" + + // File names the planner writes and the query engine reads. + plannerStreamingConfigFile = "streaming_config.yaml" + plannerInferenceConfigFile = "inference_config.yaml" ) // Suite is a data-independent collection of PromQL cases. All timestamps are @@ -23,6 +28,11 @@ type Suite struct { Name string `yaml:"name" json:"name"` ComparisonDefaults ComparisonPolicy `yaml:"comparison_defaults" json:"comparisonDefaults"` Queries []QueryCase `yaml:"queries" json:"queries"` + // PlannerOutputDir, when set, holds fixed streaming and inference configs + // that the query engine uses instead of running the planner. It is + // relative to the suite file; LoadSuiteFile resolves it to an absolute + // path. + PlannerOutputDir string `yaml:"planner_output_dir" json:"plannerOutputDir,omitempty"` } type QueryCase struct { @@ -64,8 +74,30 @@ type Tolerance struct { } // LoadSuite parses and validates a query suite. It rejects incomplete cases -// rather than silently selecting wall-clock defaults. +// rather than silently selecting wall-clock defaults. A planner_output_dir +// must be absolute here; a relative one needs the suite file's location, so +// use LoadSuiteFile. func LoadSuite(contents []byte) (Suite, error) { + suite, err := parseSuite(contents) + if err != nil { + return Suite{}, err + } + if suite.PlannerOutputDir != "" { + if !filepath.IsAbs(suite.PlannerOutputDir) { + return Suite{}, fmt.Errorf( + "query suite %q: planner_output_dir %q is relative; load the suite with LoadSuiteFile", + suite.Name, + suite.PlannerOutputDir, + ) + } + if suite.PlannerOutputDir, err = resolvePlannerOutputDir("", suite.PlannerOutputDir); err != nil { + return Suite{}, fmt.Errorf("query suite %q: %w", suite.Name, err) + } + } + return suite, nil +} + +func parseSuite(contents []byte) (Suite, error) { var suite Suite decoder := yaml.NewDecoder(bytes.NewReader(contents)) decoder.KnownFields(true) @@ -123,13 +155,47 @@ func LoadSuite(contents []byte) (Suite, error) { return suite, nil } -// LoadSuiteFile reads a query suite from disk. +// LoadSuiteFile reads a query suite from disk and resolves its +// planner_output_dir against the suite file's directory. func LoadSuiteFile(path string) (Suite, error) { contents, err := os.ReadFile(path) if err != nil { return Suite{}, fmt.Errorf("read query suite %q: %w", path, err) } - return LoadSuite(contents) + suite, err := parseSuite(contents) + if err != nil { + return Suite{}, err + } + if suite.PlannerOutputDir != "" { + resolved, err := resolvePlannerOutputDir(filepath.Dir(path), suite.PlannerOutputDir) + if err != nil { + return Suite{}, fmt.Errorf("query suite %q: %w", suite.Name, err) + } + suite.PlannerOutputDir = resolved + } + return suite, nil +} + +func resolvePlannerOutputDir(suiteDirectory, configured string) (string, error) { + directory := configured + if !filepath.IsAbs(directory) { + directory = filepath.Join(suiteDirectory, directory) + } + directory, err := filepath.Abs(directory) + if err != nil { + return "", fmt.Errorf("resolve planner_output_dir %q: %w", configured, err) + } + for _, name := range []string{plannerStreamingConfigFile, plannerInferenceConfigFile} { + file := filepath.Join(directory, name) + info, err := os.Stat(file) + if err != nil { + return "", fmt.Errorf("planner_output_dir %q: %w", configured, err) + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("planner_output_dir %q: %s is not a regular file", configured, file) + } + } + return directory, nil } func (q QueryCase) InstantTimes(base time.Time) []time.Time { diff --git a/promql-compliance/runner/config_test.go b/promql-compliance/runner/config_test.go index d38d8a2d..2e346699 100644 --- a/promql-compliance/runner/config_test.go +++ b/promql-compliance/runner/config_test.go @@ -1,6 +1,10 @@ package runner import ( + "fmt" + "os" + "path/filepath" + "strings" "testing" "time" ) @@ -171,3 +175,141 @@ queries: t.Fatal("LoadSuite accepted an invalid instant vector order direction") } } + +const plannerOutputSuite = `name: fixed-output +planner_output_dir: %s +queries: + - name: up + expr: up + instant_offsets_seconds: [60] +` + +// writePlannerOutputSuite writes a suite under root/suites whose +// planner_output_dir is configured, plus the named files under +// root/suites/fixed. +func writePlannerOutputSuite(t *testing.T, configured string, files ...string) (string, string) { + t.Helper() + root := t.TempDir() + suiteDirectory := filepath.Join(root, "suites") + outputDirectory := filepath.Join(suiteDirectory, "fixed") + if err := os.MkdirAll(outputDirectory, 0o755); err != nil { + t.Fatalf("create planner output directory: %v", err) + } + writePlannerOutputFiles(t, outputDirectory, files...) + suitePath := filepath.Join(suiteDirectory, "suite.yaml") + if err := os.WriteFile(suitePath, []byte(fmt.Sprintf(plannerOutputSuite, configured)), 0o600); err != nil { + t.Fatalf("write suite: %v", err) + } + return suitePath, outputDirectory +} + +func writePlannerOutputFiles(t *testing.T, directory string, files ...string) { + t.Helper() + for _, name := range files { + if err := os.WriteFile(filepath.Join(directory, name), []byte("{}\n"), 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } +} + +func TestLoadSuiteFileResolvesPlannerOutputDirAgainstSuiteFile(t *testing.T) { + suitePath, outputDirectory := writePlannerOutputSuite(t, "fixed", plannerStreamingConfigFile, plannerInferenceConfigFile) + + // Load through a relative suite path so resolution cannot depend on the + // caller's working directory matching the suite's directory. + workingDirectory, err := os.Getwd() + if err != nil { + t.Fatalf("Getwd: %v", err) + } + relativeSuitePath, err := filepath.Rel(workingDirectory, suitePath) + if err != nil { + t.Fatalf("Rel: %v", err) + } + suite, err := LoadSuiteFile(relativeSuitePath) + if err != nil { + t.Fatalf("LoadSuiteFile: %v", err) + } + if got, want := suite.PlannerOutputDir, outputDirectory; got != want { + t.Fatalf("planner output dir = %q, want %q", got, want) + } +} + +func TestLoadSuiteFileAcceptsAbsolutePlannerOutputDir(t *testing.T) { + outputDirectory := t.TempDir() + writePlannerOutputFiles(t, outputDirectory, plannerStreamingConfigFile, plannerInferenceConfigFile) + suitePath, _ := writePlannerOutputSuite(t, outputDirectory) + + suite, err := LoadSuiteFile(suitePath) + if err != nil { + t.Fatalf("LoadSuiteFile: %v", err) + } + if got, want := suite.PlannerOutputDir, outputDirectory; got != want { + t.Fatalf("planner output dir = %q, want %q", got, want) + } +} + +func TestLoadSuiteFileLeavesPlannerOutputDirUnsetByDefault(t *testing.T) { + suite, err := LoadSuiteFile("../suites/temporal.yaml") + if err != nil { + t.Fatalf("LoadSuiteFile: %v", err) + } + if suite.PlannerOutputDir != "" { + t.Fatalf("planner output dir = %q, want empty", suite.PlannerOutputDir) + } +} + +func TestLoadSuiteFileRejectsIncompletePlannerOutputDir(t *testing.T) { + for _, test := range []struct { + name string + configured string + files []string + missing string + }{ + {"missing inference config", "fixed", []string{plannerStreamingConfigFile}, plannerInferenceConfigFile}, + {"missing streaming config", "fixed", []string{plannerInferenceConfigFile}, plannerStreamingConfigFile}, + {"missing directory", "absent", nil, plannerStreamingConfigFile}, + } { + t.Run(test.name, func(t *testing.T) { + suitePath, _ := writePlannerOutputSuite(t, test.configured, test.files...) + _, err := LoadSuiteFile(suitePath) + if err == nil { + t.Fatal("LoadSuiteFile accepted a planner_output_dir without both config files") + } + if !strings.Contains(err.Error(), test.missing) { + t.Fatalf("error %q does not name the missing file %q", err, test.missing) + } + }) + } +} + +func TestLoadSuiteFileRejectsPlannerOutputDirectoryInPlaceOfFile(t *testing.T) { + suitePath, outputDirectory := writePlannerOutputSuite(t, "fixed", plannerStreamingConfigFile) + if err := os.Mkdir(filepath.Join(outputDirectory, plannerInferenceConfigFile), 0o755); err != nil { + t.Fatalf("create directory: %v", err) + } + if _, err := LoadSuiteFile(suitePath); err == nil { + t.Fatal("LoadSuiteFile accepted a directory as a planner config file") + } +} + +// LoadSuite has no suite file to resolve a relative planner_output_dir +// against, so every Suite it returns already holds an absolute, validated one. +func TestLoadSuiteRequiresAnAbsoluteValidPlannerOutputDir(t *testing.T) { + if _, err := LoadSuite([]byte(fmt.Sprintf(plannerOutputSuite, "fixed"))); err == nil || + !strings.Contains(err.Error(), "LoadSuiteFile") { + t.Fatalf("LoadSuite accepted a relative planner_output_dir (err: %v)", err) + } + + outputDirectory := t.TempDir() + if _, err := LoadSuite([]byte(fmt.Sprintf(plannerOutputSuite, outputDirectory))); err == nil { + t.Fatal("LoadSuite accepted a planner_output_dir without its config files") + } + writePlannerOutputFiles(t, outputDirectory, plannerStreamingConfigFile, plannerInferenceConfigFile) + suite, err := LoadSuite([]byte(fmt.Sprintf(plannerOutputSuite, outputDirectory))) + if err != nil { + t.Fatalf("LoadSuite: %v", err) + } + if got := suite.PlannerOutputDir; got != outputDirectory { + t.Fatalf("planner output dir = %q, want %q", got, outputDirectory) + } +} diff --git a/promql-compliance/runner/run.go b/promql-compliance/runner/run.go index c229a4eb..c5c03ad4 100644 --- a/promql-compliance/runner/run.go +++ b/promql-compliance/runner/run.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "sort" + "strconv" "strings" "time" @@ -27,8 +28,35 @@ const ( dataReadyTimeout = 3 * time.Minute readinessPollInterval = time.Second composeShutdownTimeout = time.Minute + + plannerOutputMountPath = "/asap-planner-output" + fixedPlannerOutputComposeFile = "docker-compose.fixed-planner-output.yml" + fixedPlannerOutputDirEnv = "ASAP_FIXED_PLANNER_OUTPUT_DIR" ) +// fixedPlannerOutputCompose overrides the base Compose file for suites with a +// planner_output_dir: the planner sits behind a never-activated profile so it +// does not run, and the query engine mounts the fixed directory where the +// planner's output volume would be. The bind uses long syntax so a host path +// containing ':' stays one source. +const fixedPlannerOutputCompose = `services: + planner: + profiles: ["generated-planner-output"] + queryengine: + depends_on: !override + prometheus: + condition: service_healthy + volumes: + - type: bind + source: "${` + fixedPlannerOutputDirEnv + `:?` + fixedPlannerOutputDirEnv + ` is required}" + target: ` + plannerOutputMountPath + ` + read_only: true +` + +// The override's `!override` tag needs this Compose release or later; older +// ones merge depends_on and fail on the disabled planner. +var minComposeVersionForOverride = [3]int{2, 24, 4} + type RunOptions struct { DatasetPath string SuitePath string @@ -70,7 +98,15 @@ func Run(ctx context.Context, options RunOptions) (Report, error) { return Report{}, fmt.Errorf("create run directory: %w", err) } defer os.RemoveAll(runDirectory) - if err := writeGeneratedConfigs(runDirectory, fixture, suite); err != nil { + // Fixed planner output only replaces the planner inside a Compose project; + // with caller-owned targets the planner input is still generated. + fixedPlannerOutput := suite.PlannerOutputDir != "" && len(options.ComposeFiles) > 0 + if fixedPlannerOutput { + if err := requireComposeOverrideSupport(ctx); err != nil { + return Report{}, err + } + } + if err := writeGeneratedConfigs(runDirectory, fixture, suite, fixedPlannerOutput); err != nil { return Report{}, err } @@ -79,6 +115,18 @@ func Run(ctx context.Context, options RunOptions) (Report, error) { project: options.ComposeProject, env: []string{"ASAP_TEST_CONFIG_DIR=" + runDirectory}, } + if fixedPlannerOutput { + lifecycle.files = append( + append([]string(nil), options.ComposeFiles...), + filepath.Join(runDirectory, fixedPlannerOutputComposeFile), + ) + lifecycle.env = append( + lifecycle.env, + fixedPlannerOutputDirEnv+"="+suite.PlannerOutputDir, + // An inherited COMPOSE_PROFILES could re-enable the planner. + "COMPOSE_PROFILES=", + ) + } if len(options.ComposeFiles) > 0 { if err := lifecycle.Start(ctx); err != nil { return Report{}, err @@ -345,7 +393,72 @@ type plannerCleanup struct { Policy string `yaml:"policy"` } -func writeGeneratedConfigs(directory string, fixture seeder.Fixture, suite Suite) error { +// writeGeneratedConfigs writes the engine config plus either the planner input +// or, with fixed planner output, the Compose override that skips the planner. +// The planner input is not generated in the fixed case, so its timing +// derivation cannot reject queries the planner never sees. +func writeGeneratedConfigs(directory string, fixture seeder.Fixture, suite Suite, fixedPlannerOutput bool) error { + if err := writeEngineConfig(directory); err != nil { + return err + } + if fixedPlannerOutput { + if err := os.WriteFile(filepath.Join(directory, fixedPlannerOutputComposeFile), []byte(fixedPlannerOutputCompose), 0o600); err != nil { + return fmt.Errorf("write fixed planner output compose override: %w", err) + } + return nil + } + return writePlannerConfig(directory, fixture, suite) +} + +// requireComposeOverrideSupport fails unless `docker compose` understands the +// fixed-output override's `!override` tag. +func requireComposeOverrideSupport(ctx context.Context) error { + output, err := exec.CommandContext(ctx, "docker", "compose", "version", "--short").Output() + if err != nil { + return fmt.Errorf("check docker compose version for planner_output_dir: %w", err) + } + version := strings.TrimSpace(string(output)) + ok, err := composeVersionAtLeast(version, minComposeVersionForOverride) + if err != nil { + return err + } + if !ok { + return fmt.Errorf( + "planner_output_dir needs docker compose >= %d.%d.%d, found %s", + minComposeVersionForOverride[0], + minComposeVersionForOverride[1], + minComposeVersionForOverride[2], + version, + ) + } + return nil +} + +// composeVersionAtLeast compares a `docker compose version --short` string +// such as "2.29.1" or "v2.24.4-desktop.1" against minimum. +func composeVersionAtLeast(version string, minimum [3]int) (bool, error) { + core := strings.SplitN(strings.TrimPrefix(version, "v"), "-", 2)[0] + parts := strings.Split(core, ".") + if len(parts) != 3 { + return false, fmt.Errorf("unrecognized docker compose version %q", version) + } + var got [3]int + for i, part := range parts { + number, err := strconv.Atoi(part) + if err != nil { + return false, fmt.Errorf("unrecognized docker compose version %q", version) + } + got[i] = number + } + for i := range got { + if got[i] != minimum[i] { + return got[i] > minimum[i], nil + } + } + return true, nil +} + +func writePlannerConfig(directory string, fixture seeder.Fixture, suite Suite) error { metrics := make(map[string]map[string]struct{}) for _, series := range fixture.Series { if metrics[series.Metric] == nil { @@ -399,6 +512,10 @@ func writeGeneratedConfigs(directory string, fixture seeder.Fixture, suite Suite if err := os.WriteFile(filepath.Join(directory, "controller-config.yaml"), contents, 0o600); err != nil { return fmt.Errorf("write generated planner config: %w", err) } + return nil +} + +func writeEngineConfig(directory string) error { engineConfig := []byte(`output_dir: "/app/outputs" log_level: "INFO" data_ingestion_interval_ms: 1000 @@ -414,8 +531,8 @@ store: ingest: type: "http_remote_write" port: 9091 -inference_config: "/asap-planner-output/inference_config.yaml" -streaming_config: "/asap-planner-output/streaming_config.yaml" +inference_config: "` + plannerOutputMountPath + `/` + plannerInferenceConfigFile + `" +streaming_config: "` + plannerOutputMountPath + `/` + plannerStreamingConfigFile + `" `) if err := os.WriteFile(filepath.Join(directory, "engine_config.yaml"), engineConfig, 0o600); err != nil { return fmt.Errorf("write generated engine config: %w", err) diff --git a/promql-compliance/runner/run_test.go b/promql-compliance/runner/run_test.go index 825efc8e..735cb34c 100644 --- a/promql-compliance/runner/run_test.go +++ b/promql-compliance/runner/run_test.go @@ -66,7 +66,7 @@ func TestGeneratedConfigsUseCurrentPlannerAndEngineSchema(t *testing.T) { Queries: []QueryCase{{Name: "up", Expr: "up"}}, } - if err := writeGeneratedConfigs(directory, fixture, suite); err != nil { + if err := writeGeneratedConfigs(directory, fixture, suite, false); err != nil { t.Fatalf("writeGeneratedConfigs: %v", err) } @@ -124,7 +124,7 @@ func TestGeneratedPlannerGroupsUseCompatibleQueryTiming(t *testing.T) { }, } - if err := writeGeneratedConfigs(directory, fixture, suite); err != nil { + if err := writeGeneratedConfigs(directory, fixture, suite, false); err != nil { t.Fatalf("writeGeneratedConfigs: %v", err) } @@ -159,3 +159,147 @@ func TestGeneratedPlannerGroupsUseCompatibleQueryTiming(t *testing.T) { t.Fatalf("short-rate range_duration_ms = %v, want 60000", planner.QueryGroups[1].RangeDurationMS) } } + +func TestGeneratedConfigsWithFixedPlannerOutputSkipPlanner(t *testing.T) { + directory := t.TempDir() + fixture := seeder.Fixture{ + Name: "single-series", + Series: []seeder.FixtureSeries{{ + Metric: "up", + Labels: map[string]string{"job": "test"}, + Samples: []seeder.FixtureSample{{OffsetSeconds: 0, Value: 1}}, + }}, + } + suite := Suite{ + Name: "fixed-output", + Queries: []QueryCase{{Name: "up", Expr: "up"}}, + PlannerOutputDir: "/fixed/planner/output", + } + + if err := writeGeneratedConfigs(directory, fixture, suite, true); err != nil { + t.Fatalf("writeGeneratedConfigs: %v", err) + } + + if _, err := os.Stat(filepath.Join(directory, "controller-config.yaml")); !os.IsNotExist(err) { + t.Fatalf("planner config was generated for a suite with fixed planner output (stat err: %v)", err) + } + + overrideContents, err := os.ReadFile(filepath.Join(directory, fixedPlannerOutputComposeFile)) + if err != nil { + t.Fatalf("read compose override: %v", err) + } + var override struct { + Services struct { + Planner struct { + Profiles []string `yaml:"profiles"` + } `yaml:"planner"` + QueryEngine struct { + DependsOn yaml.Node `yaml:"depends_on"` + Volumes []struct { + Type string `yaml:"type"` + Source string `yaml:"source"` + Target string `yaml:"target"` + ReadOnly bool `yaml:"read_only"` + } `yaml:"volumes"` + } `yaml:"queryengine"` + } `yaml:"services"` + } + if err := yaml.Unmarshal(overrideContents, &override); err != nil { + t.Fatalf("parse compose override: %v", err) + } + if len(override.Services.Planner.Profiles) == 0 { + t.Fatal("compose override leaves the planner service active") + } + dependsOn := override.Services.QueryEngine.DependsOn + if dependsOn.Tag != "!override" { + t.Fatalf("queryengine depends_on tag = %q, want !override so the planner dependency is dropped", dependsOn.Tag) + } + var dependencies map[string]any + if err := dependsOn.Decode(&dependencies); err != nil { + t.Fatalf("decode queryengine depends_on: %v", err) + } + if _, ok := dependencies["planner"]; ok { + t.Fatal("queryengine still depends on the planner") + } + volumes := override.Services.QueryEngine.Volumes + wantSource := "${" + fixedPlannerOutputDirEnv + ":?" + fixedPlannerOutputDirEnv + " is required}" + if len(volumes) != 1 || volumes[0].Type != "bind" || volumes[0].Source != wantSource || + volumes[0].Target != plannerOutputMountPath || !volumes[0].ReadOnly { + t.Fatalf("queryengine volumes = %+v, want one read-only bind of %s at %s", volumes, wantSource, plannerOutputMountPath) + } + + engineContents, err := os.ReadFile(filepath.Join(directory, "engine_config.yaml")) + if err != nil { + t.Fatalf("read engine config: %v", err) + } + var engine struct { + InferenceConfig string `yaml:"inference_config"` + StreamingConfig string `yaml:"streaming_config"` + } + if err := yaml.Unmarshal(engineContents, &engine); err != nil { + t.Fatalf("parse engine config: %v", err) + } + if got, want := engine.InferenceConfig, plannerOutputMountPath+"/"+plannerInferenceConfigFile; got != want { + t.Fatalf("engine inference_config = %q, want %q", got, want) + } + if got, want := engine.StreamingConfig, plannerOutputMountPath+"/"+plannerStreamingConfigFile; got != want { + t.Fatalf("engine streaming_config = %q, want %q", got, want) + } +} + +// With caller-owned targets there is no Compose project to swap the planner +// out of, so the planner input is still generated and validated. +func TestGeneratedConfigsWithoutComposeStillWritePlannerInput(t *testing.T) { + directory := t.TempDir() + fixture := seeder.Fixture{ + Name: "single-series", + Series: []seeder.FixtureSeries{{ + Metric: "up", + Labels: map[string]string{"job": "test"}, + Samples: []seeder.FixtureSample{{OffsetSeconds: 0, Value: 1}}, + }}, + } + suite := Suite{ + Name: "fixed-output", + Queries: []QueryCase{{Name: "up", Expr: "up"}}, + PlannerOutputDir: "/fixed/planner/output", + } + + if err := writeGeneratedConfigs(directory, fixture, suite, false); err != nil { + t.Fatalf("writeGeneratedConfigs: %v", err) + } + + if _, err := os.Stat(filepath.Join(directory, "controller-config.yaml")); err != nil { + t.Fatalf("planner config not generated without Compose: %v", err) + } + if _, err := os.Stat(filepath.Join(directory, fixedPlannerOutputComposeFile)); !os.IsNotExist(err) { + t.Fatalf("compose override written without Compose (stat err: %v)", err) + } +} + +func TestComposeVersionAtLeast(t *testing.T) { + minimum := [3]int{2, 24, 4} + for _, test := range []struct { + version string + want bool + }{ + {"2.24.4", true}, + {"v2.29.1", true}, + {"2.24.4-desktop.1", true}, + {"3.0.0", true}, + {"2.24.3", false}, + {"2.9.10", false}, + {"1.29.2", false}, + } { + got, err := composeVersionAtLeast(test.version, minimum) + if err != nil { + t.Fatalf("composeVersionAtLeast(%q): %v", test.version, err) + } + if got != test.want { + t.Fatalf("composeVersionAtLeast(%q) = %v, want %v", test.version, got, test.want) + } + } + if _, err := composeVersionAtLeast("not-a-version", minimum); err == nil { + t.Fatal("an unrecognized version must be an error") + } +} diff --git a/promql-compliance/suites/rollup-planner-output/inference_config.yaml b/promql-compliance/suites/rollup-planner-output/inference_config.yaml new file mode 100644 index 00000000..6a7f7006 --- /dev/null +++ b/promql-compliance/suites/rollup-planner-output/inference_config.yaml @@ -0,0 +1,117 @@ +# Hand-written in the planner's output format. Queries grouped by a subset of +# an aggregation's grouping labels point at that aggregation, so the engine +# answers them by rolling the per-(service, endpoint) aggregates up. +# no_cleanup: several queries read the same windows, so a per-query read +# count threshold would evict windows other queries still need. +cleanup_policy: + name: no_cleanup +metrics: + rollup_latency: + - endpoint + - service +queries: +- aggregations: + - aggregation_id: 1 + query: quantile by (service, endpoint) (0.5, rollup_latency) + planned_subquery: quantile by (service, endpoint) (0.5, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + query: quantile by (service) (0.5, rollup_latency) + planned_subquery: quantile by (service) (0.5, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + query: quantile(0.5, rollup_latency) + planned_subquery: quantile(0.5, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + query: quantile by (service, endpoint) (0.9, rollup_latency) + planned_subquery: quantile by (service, endpoint) (0.9, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + query: quantile by (service) (0.9, rollup_latency) + planned_subquery: quantile by (service) (0.9, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + query: quantile(0.9, rollup_latency) + planned_subquery: quantile(0.9, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 2 + query: quantile by (service, endpoint) (0.75, rollup_latency) + planned_subquery: quantile by (service, endpoint) (0.75, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 2 + query: quantile by (service) (0.75, rollup_latency) + planned_subquery: quantile by (service) (0.75, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 2 + query: quantile(0.75, rollup_latency) + planned_subquery: quantile(0.75, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + query: sum by (service, endpoint) (rollup_latency) + planned_subquery: sum by (service, endpoint) (rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + query: sum by (service) (rollup_latency) + planned_subquery: sum by (service) (rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + query: sum(rollup_latency) + planned_subquery: sum(rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + query: sum by (service, endpoint) (sum_over_time(rollup_latency[5m])) + planned_subquery: sum by (service, endpoint) (sum_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + query: sum by (service) (sum_over_time(rollup_latency[5m])) + planned_subquery: sum by (service) (sum_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + query: sum(sum_over_time(rollup_latency[5m])) + planned_subquery: sum(sum_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 5 + query: min by (service, endpoint) (min_over_time(rollup_latency[5m])) + planned_subquery: min by (service, endpoint) (min_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 5 + query: min by (service) (min_over_time(rollup_latency[5m])) + planned_subquery: min by (service) (min_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 5 + query: min(min_over_time(rollup_latency[5m])) + planned_subquery: min(min_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 6 + query: max by (service, endpoint) (max_over_time(rollup_latency[5m])) + planned_subquery: max by (service, endpoint) (max_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 6 + query: max by (service) (max_over_time(rollup_latency[5m])) + planned_subquery: max by (service) (max_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 6 + query: max(max_over_time(rollup_latency[5m])) + planned_subquery: max(max_over_time(rollup_latency[5m])) + query_time_aggregations: [] diff --git a/promql-compliance/suites/rollup-planner-output/streaming_config.yaml b/promql-compliance/suites/rollup-planner-output/streaming_config.yaml new file mode 100644 index 00000000..afb0dc73 --- /dev/null +++ b/promql-compliance/suites/rollup-planner-output/streaming_config.yaml @@ -0,0 +1,113 @@ +# Hand-written in the planner's output format. Every aggregation is a +# single-subpopulation type grouped by (service, endpoint); the tumbling +# window sizes match what the planner picks for each query shape, and ids 4 +# and 6 are sliding to cover sliding-window roll-ups. +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DatasketchesKLL + labels: + aggregated: [] + grouping: + - endpoint + - service + rollup: [] + metric: rollup_latency + parameters: + K: 500 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 1000 + windowType: tumbling +- aggregationId: 2 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - endpoint + - service + rollup: [] + metric: rollup_latency + parameters: + alpha: 0.01 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 1000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: Sum + labels: + aggregated: [] + grouping: + - endpoint + - service + rollup: [] + metric: rollup_latency + parameters: {} + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 1000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: Sum + labels: + aggregated: [] + grouping: + - endpoint + - service + rollup: [] + metric: rollup_latency + parameters: {} + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 300000 + windowType: sliding +- aggregationId: 5 + aggregationSubType: min + aggregationType: MinMax + labels: + aggregated: [] + grouping: + - endpoint + - service + rollup: [] + metric: rollup_latency + parameters: {} + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 60000 + windowType: tumbling +- aggregationId: 6 + aggregationSubType: max + aggregationType: MinMax + labels: + aggregated: [] + grouping: + - endpoint + - service + rollup: [] + metric: rollup_latency + parameters: {} + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 300000 + windowType: sliding +metrics: + rollup_latency: + - endpoint + - service diff --git a/promql-compliance/suites/rollup.yaml b/promql-compliance/suites/rollup.yaml new file mode 100644 index 00000000..e595cca3 --- /dev/null +++ b/promql-compliance/suites/rollup.yaml @@ -0,0 +1,125 @@ +name: rollup +# Every aggregation in the fixed planner output is grouped by (service, +# endpoint). Each query appears grouped by (service, endpoint), which reads +# the aggregation as-is, then by (service) and with no grouping, which roll +# the per-endpoint aggregates up. The first query is the readiness probe, so +# it is a non-rolled-up one. +planner_output_dir: rollup-planner-output +comparison_defaults: + value_tolerance: + relative: 0.01 + absolute: 0.000001 + +queries: + # DatasketchesKLL, 1s tumbling. + - name: kll-quantile-0.5-by-service-endpoint + expr: "quantile by (service, endpoint) (0.5, rollup_latency)" + instant_offsets_seconds: &evaluation_offsets [300, 600, 900, 1200] + range: &evaluation_range + start_offset_seconds: 300 + end_offset_seconds: 1200 + step_seconds: 60 + - name: kll-quantile-0.5-by-service + expr: "quantile by (service) (0.5, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + - name: kll-quantile-0.5-all + expr: "quantile(0.5, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + - name: kll-quantile-0.9-by-service-endpoint + expr: "quantile by (service, endpoint) (0.9, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + - name: kll-quantile-0.9-by-service + expr: "quantile by (service) (0.9, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + - name: kll-quantile-0.9-all + expr: "quantile(0.9, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + # DDSketch (alpha 0.01), 1s tumbling. Its relative error adds to the + # nearest-rank gap, hence the wider tolerance. + - name: ddsketch-quantile-0.75-by-service-endpoint + expr: "quantile by (service, endpoint) (0.75, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: &ddsketch_tolerance + value_tolerance: + relative: 0.02 + - name: ddsketch-quantile-0.75-by-service + expr: "quantile by (service) (0.75, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *ddsketch_tolerance + - name: ddsketch-quantile-0.75-all + expr: "quantile(0.75, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *ddsketch_tolerance + # Sum and MinMax merge exactly, so they are compared tightly. Sum is 1s + # tumbling for the instant-vector sum and 5m/1m sliding for sum_over_time; + # min is 1m tumbling and max is 5m/1m sliding. + - name: sum-by-service-endpoint + expr: "sum by (service, endpoint) (rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: &exact + value_tolerance: + relative: 0 + - name: sum-by-service + expr: "sum by (service) (rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: sum-all + expr: "sum(rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: sum-sum-over-time-by-service-endpoint + expr: "sum by (service, endpoint) (sum_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: sum-sum-over-time-by-service + expr: "sum by (service) (sum_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: sum-sum-over-time-all + expr: "sum(sum_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: min-min-over-time-by-service-endpoint + expr: "min by (service, endpoint) (min_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: min-min-over-time-by-service + expr: "min by (service) (min_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: min-min-over-time-all + expr: "min(min_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: max-max-over-time-by-service-endpoint + expr: "max by (service, endpoint) (max_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: max-max-over-time-by-service + expr: "max by (service) (max_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: max-max-over-time-all + expr: "max(max_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact