From 2f70f7bcd9fa666bb8da0fe1855e1ed05731a749 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Mon, 12 Oct 2026 22:53:53 -0400 Subject: [PATCH 1/3] feat(promql-compliance): let a suite supply fixed planner output Add an optional suite-level planner_output_dir holding streaming_config.yaml and inference_config.yaml. When set, the runner validates both files at load time, skips generating the planner input, and adds a Compose override that parks the planner behind an inactive profile and mounts the directory where the query engine reads planner output. This lets the harness exercise engine configs the planner does not produce yet. Co-Authored-By: Claude Opus 5.5 --- promql-compliance/ARCHITECTURE.md | 4 +- promql-compliance/HOW_TO.md | 19 ++++ promql-compliance/runner/config.go | 48 +++++++++- promql-compliance/runner/config_test.go | 119 ++++++++++++++++++++++++ promql-compliance/runner/run.go | 51 +++++++++- promql-compliance/runner/run_test.go | 80 ++++++++++++++++ 6 files changed, 316 insertions(+), 5 deletions(-) diff --git a/promql-compliance/ARCHITECTURE.md b/promql-compliance/ARCHITECTURE.md index e7da232a..23998fad 100644 --- a/promql-compliance/ARCHITECTURE.md +++ b/promql-compliance/ARCHITECTURE.md @@ -54,7 +54,9 @@ The runner performs these stages: are relative to this timestamp. 3. Generate temporary planner and engine configuration. 4. Start Prometheus and wait for its health check. -5. Run the planner. Its output is placed in a named shared volume. +5. Run the planner. Its output is placed in a named shared volume. A suite + with `planner_output_dir` skips the planner and mounts that directory + instead. 6. Start the ASAPQuery query engine after the planner completes. 7. Send the same encoded remote-write batches to Prometheus and ASAPQuery. 8. Wait for both query APIs to be ready and for the first probe query to return diff --git a/promql-compliance/HOW_TO.md b/promql-compliance/HOW_TO.md index 71bacc33..e05602c3 100644 --- a/promql-compliance/HOW_TO.md +++ b/promql-compliance/HOW_TO.md @@ -128,6 +128,25 @@ To change the five-minute cap or one-second ingestion assumption, change the runner’s generated-config logic in `runner/run.go`. Do not edit the generated configuration: it is temporary and is deleted after the run. +## Use fixed planner output + +By default the planner generates the engine's streaming and inference +configs from the suite. To test configs the planner does not produce, such as +roll-ups from a finer grouping, check the configs in next to the suite and +point the suite at their directory: + +```yaml +name: rollup +planner_output_dir: rollup-planner-output +``` + +The path is relative to the suite file. The directory must contain +`streaming_config.yaml` and `inference_config.yaml` in the planner's output +format; the runner fails at load time if either is missing. The planner +service then does not run, and the query engine reads the directory in place +of the planner's output volume. With already-running targets the field has no +effect. `suites/rollup.yaml` is an example. + ## Configure tolerance Set a suite-wide default: diff --git a/promql-compliance/runner/config.go b/promql-compliance/runner/config.go index a0857636..0c197a40 100644 --- a/promql-compliance/runner/config.go +++ b/promql-compliance/runner/config.go @@ -5,6 +5,7 @@ import ( "fmt" "math" "os" + "path/filepath" "time" "gopkg.in/yaml.v3" @@ -15,6 +16,10 @@ const ( instantOrderDescending = "descending" orderGroupingBy = "by" orderGroupingWithout = "without" + + // File names the planner writes and the query engine reads. + plannerStreamingConfigFile = "streaming_config.yaml" + plannerInferenceConfigFile = "inference_config.yaml" ) // Suite is a data-independent collection of PromQL cases. All timestamps are @@ -23,6 +28,11 @@ type Suite struct { Name string `yaml:"name" json:"name"` ComparisonDefaults ComparisonPolicy `yaml:"comparison_defaults" json:"comparisonDefaults"` Queries []QueryCase `yaml:"queries" json:"queries"` + // PlannerOutputDir, when set, holds fixed streaming and inference configs + // that the query engine uses instead of running the planner. It is + // relative to the suite file; LoadSuiteFile resolves it to an absolute + // path. + PlannerOutputDir string `yaml:"planner_output_dir" json:"plannerOutputDir,omitempty"` } type QueryCase struct { @@ -123,13 +133,47 @@ func LoadSuite(contents []byte) (Suite, error) { return suite, nil } -// LoadSuiteFile reads a query suite from disk. +// LoadSuiteFile reads a query suite from disk and resolves its +// planner_output_dir against the suite file's directory. func LoadSuiteFile(path string) (Suite, error) { contents, err := os.ReadFile(path) if err != nil { return Suite{}, fmt.Errorf("read query suite %q: %w", path, err) } - return LoadSuite(contents) + suite, err := LoadSuite(contents) + if err != nil { + return Suite{}, err + } + if suite.PlannerOutputDir != "" { + resolved, err := resolvePlannerOutputDir(filepath.Dir(path), suite.PlannerOutputDir) + if err != nil { + return Suite{}, fmt.Errorf("query suite %q: %w", suite.Name, err) + } + suite.PlannerOutputDir = resolved + } + return suite, nil +} + +func resolvePlannerOutputDir(suiteDirectory, configured string) (string, error) { + directory := configured + if !filepath.IsAbs(directory) { + directory = filepath.Join(suiteDirectory, directory) + } + directory, err := filepath.Abs(directory) + if err != nil { + return "", fmt.Errorf("resolve planner_output_dir %q: %w", configured, err) + } + for _, name := range []string{plannerStreamingConfigFile, plannerInferenceConfigFile} { + file := filepath.Join(directory, name) + info, err := os.Stat(file) + if err != nil { + return "", fmt.Errorf("planner_output_dir %q: %w", configured, err) + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("planner_output_dir %q: %s is not a regular file", configured, file) + } + } + return directory, nil } func (q QueryCase) InstantTimes(base time.Time) []time.Time { diff --git a/promql-compliance/runner/config_test.go b/promql-compliance/runner/config_test.go index d38d8a2d..8aa0a234 100644 --- a/promql-compliance/runner/config_test.go +++ b/promql-compliance/runner/config_test.go @@ -1,6 +1,10 @@ package runner import ( + "fmt" + "os" + "path/filepath" + "strings" "testing" "time" ) @@ -171,3 +175,118 @@ queries: t.Fatal("LoadSuite accepted an invalid instant vector order direction") } } + +const plannerOutputSuite = `name: fixed-output +planner_output_dir: %s +queries: + - name: up + expr: up + instant_offsets_seconds: [60] +` + +// writePlannerOutputSuite writes a suite under root/suites whose +// planner_output_dir is configured, plus the named files under +// root/suites/fixed. +func writePlannerOutputSuite(t *testing.T, configured string, files ...string) (string, string) { + t.Helper() + root := t.TempDir() + suiteDirectory := filepath.Join(root, "suites") + outputDirectory := filepath.Join(suiteDirectory, "fixed") + if err := os.MkdirAll(outputDirectory, 0o755); err != nil { + t.Fatalf("create planner output directory: %v", err) + } + for _, name := range files { + if err := os.WriteFile(filepath.Join(outputDirectory, name), []byte("{}\n"), 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + suitePath := filepath.Join(suiteDirectory, "suite.yaml") + if err := os.WriteFile(suitePath, []byte(fmt.Sprintf(plannerOutputSuite, configured)), 0o600); err != nil { + t.Fatalf("write suite: %v", err) + } + return suitePath, outputDirectory +} + +func TestLoadSuiteFileResolvesPlannerOutputDirAgainstSuiteFile(t *testing.T) { + suitePath, outputDirectory := writePlannerOutputSuite(t, "fixed", plannerStreamingConfigFile, plannerInferenceConfigFile) + + // Load through a relative suite path so resolution cannot depend on the + // caller's working directory matching the suite's directory. + workingDirectory, err := os.Getwd() + if err != nil { + t.Fatalf("Getwd: %v", err) + } + relativeSuitePath, err := filepath.Rel(workingDirectory, suitePath) + if err != nil { + t.Fatalf("Rel: %v", err) + } + suite, err := LoadSuiteFile(relativeSuitePath) + if err != nil { + t.Fatalf("LoadSuiteFile: %v", err) + } + if got, want := suite.PlannerOutputDir, outputDirectory; got != want { + t.Fatalf("planner output dir = %q, want %q", got, want) + } +} + +func TestLoadSuiteFileAcceptsAbsolutePlannerOutputDir(t *testing.T) { + outputDirectory := t.TempDir() + for _, name := range []string{plannerStreamingConfigFile, plannerInferenceConfigFile} { + if err := os.WriteFile(filepath.Join(outputDirectory, name), []byte("{}\n"), 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + suitePath, _ := writePlannerOutputSuite(t, outputDirectory) + + suite, err := LoadSuiteFile(suitePath) + if err != nil { + t.Fatalf("LoadSuiteFile: %v", err) + } + if got, want := suite.PlannerOutputDir, outputDirectory; got != want { + t.Fatalf("planner output dir = %q, want %q", got, want) + } +} + +func TestLoadSuiteFileLeavesPlannerOutputDirUnsetByDefault(t *testing.T) { + suite, err := LoadSuiteFile("../suites/temporal.yaml") + if err != nil { + t.Fatalf("LoadSuiteFile: %v", err) + } + if suite.PlannerOutputDir != "" { + t.Fatalf("planner output dir = %q, want empty", suite.PlannerOutputDir) + } +} + +func TestLoadSuiteFileRejectsIncompletePlannerOutputDir(t *testing.T) { + for _, test := range []struct { + name string + configured string + files []string + missing string + }{ + {"missing inference config", "fixed", []string{plannerStreamingConfigFile}, plannerInferenceConfigFile}, + {"missing streaming config", "fixed", []string{plannerInferenceConfigFile}, plannerStreamingConfigFile}, + {"missing directory", "absent", nil, plannerStreamingConfigFile}, + } { + t.Run(test.name, func(t *testing.T) { + suitePath, _ := writePlannerOutputSuite(t, test.configured, test.files...) + _, err := LoadSuiteFile(suitePath) + if err == nil { + t.Fatal("LoadSuiteFile accepted a planner_output_dir without both config files") + } + if !strings.Contains(err.Error(), test.missing) { + t.Fatalf("error %q does not name the missing file %q", err, test.missing) + } + }) + } +} + +func TestLoadSuiteFileRejectsPlannerOutputDirectoryInPlaceOfFile(t *testing.T) { + suitePath, outputDirectory := writePlannerOutputSuite(t, "fixed", plannerStreamingConfigFile) + if err := os.Mkdir(filepath.Join(outputDirectory, plannerInferenceConfigFile), 0o755); err != nil { + t.Fatalf("create directory: %v", err) + } + if _, err := LoadSuiteFile(suitePath); err == nil { + t.Fatal("LoadSuiteFile accepted a directory as a planner config file") + } +} diff --git a/promql-compliance/runner/run.go b/promql-compliance/runner/run.go index c229a4eb..595e4af3 100644 --- a/promql-compliance/runner/run.go +++ b/promql-compliance/runner/run.go @@ -27,8 +27,27 @@ const ( dataReadyTimeout = 3 * time.Minute readinessPollInterval = time.Second composeShutdownTimeout = time.Minute + + plannerOutputMountPath = "/asap-planner-output" + fixedPlannerOutputComposeFile = "docker-compose.fixed-planner-output.yml" + fixedPlannerOutputDirEnv = "ASAP_FIXED_PLANNER_OUTPUT_DIR" ) +// fixedPlannerOutputCompose overrides the base Compose file for suites with a +// planner_output_dir: the planner sits behind a never-activated profile so it +// does not run, and the query engine mounts the fixed directory where the +// planner's output volume would be. +const fixedPlannerOutputCompose = `services: + planner: + profiles: ["generated-planner-output"] + queryengine: + depends_on: !override + prometheus: + condition: service_healthy + volumes: + - "${` + fixedPlannerOutputDirEnv + `:?` + fixedPlannerOutputDirEnv + ` is required}:` + plannerOutputMountPath + `:ro" +` + type RunOptions struct { DatasetPath string SuitePath string @@ -79,6 +98,13 @@ func Run(ctx context.Context, options RunOptions) (Report, error) { project: options.ComposeProject, env: []string{"ASAP_TEST_CONFIG_DIR=" + runDirectory}, } + if suite.PlannerOutputDir != "" { + lifecycle.files = append( + append([]string(nil), options.ComposeFiles...), + filepath.Join(runDirectory, fixedPlannerOutputComposeFile), + ) + lifecycle.env = append(lifecycle.env, fixedPlannerOutputDirEnv+"="+suite.PlannerOutputDir) + } if len(options.ComposeFiles) > 0 { if err := lifecycle.Start(ctx); err != nil { return Report{}, err @@ -345,7 +371,24 @@ type plannerCleanup struct { Policy string `yaml:"policy"` } +// writeGeneratedConfigs writes the engine config plus either the planner input +// or, for a suite with fixed planner output, the Compose override that skips +// the planner. The planner input is not generated in the fixed case, so its +// timing derivation cannot reject queries the planner never sees. func writeGeneratedConfigs(directory string, fixture seeder.Fixture, suite Suite) error { + if err := writeEngineConfig(directory); err != nil { + return err + } + if suite.PlannerOutputDir != "" { + if err := os.WriteFile(filepath.Join(directory, fixedPlannerOutputComposeFile), []byte(fixedPlannerOutputCompose), 0o600); err != nil { + return fmt.Errorf("write fixed planner output compose override: %w", err) + } + return nil + } + return writePlannerConfig(directory, fixture, suite) +} + +func writePlannerConfig(directory string, fixture seeder.Fixture, suite Suite) error { metrics := make(map[string]map[string]struct{}) for _, series := range fixture.Series { if metrics[series.Metric] == nil { @@ -399,6 +442,10 @@ func writeGeneratedConfigs(directory string, fixture seeder.Fixture, suite Suite if err := os.WriteFile(filepath.Join(directory, "controller-config.yaml"), contents, 0o600); err != nil { return fmt.Errorf("write generated planner config: %w", err) } + return nil +} + +func writeEngineConfig(directory string) error { engineConfig := []byte(`output_dir: "/app/outputs" log_level: "INFO" data_ingestion_interval_ms: 1000 @@ -414,8 +461,8 @@ store: ingest: type: "http_remote_write" port: 9091 -inference_config: "/asap-planner-output/inference_config.yaml" -streaming_config: "/asap-planner-output/streaming_config.yaml" +inference_config: "` + plannerOutputMountPath + `/` + plannerInferenceConfigFile + `" +streaming_config: "` + plannerOutputMountPath + `/` + plannerStreamingConfigFile + `" `) if err := os.WriteFile(filepath.Join(directory, "engine_config.yaml"), engineConfig, 0o600); err != nil { return fmt.Errorf("write generated engine config: %w", err) diff --git a/promql-compliance/runner/run_test.go b/promql-compliance/runner/run_test.go index 825efc8e..a7bcc188 100644 --- a/promql-compliance/runner/run_test.go +++ b/promql-compliance/runner/run_test.go @@ -159,3 +159,83 @@ func TestGeneratedPlannerGroupsUseCompatibleQueryTiming(t *testing.T) { t.Fatalf("short-rate range_duration_ms = %v, want 60000", planner.QueryGroups[1].RangeDurationMS) } } + +func TestGeneratedConfigsWithFixedPlannerOutputSkipPlanner(t *testing.T) { + directory := t.TempDir() + fixture := seeder.Fixture{ + Name: "single-series", + Series: []seeder.FixtureSeries{{ + Metric: "up", + Labels: map[string]string{"job": "test"}, + Samples: []seeder.FixtureSample{{OffsetSeconds: 0, Value: 1}}, + }}, + } + suite := Suite{ + Name: "fixed-output", + Queries: []QueryCase{{Name: "up", Expr: "up"}}, + PlannerOutputDir: "/fixed/planner/output", + } + + if err := writeGeneratedConfigs(directory, fixture, suite); err != nil { + t.Fatalf("writeGeneratedConfigs: %v", err) + } + + if _, err := os.Stat(filepath.Join(directory, "controller-config.yaml")); !os.IsNotExist(err) { + t.Fatalf("planner config was generated for a suite with fixed planner output (stat err: %v)", err) + } + + overrideContents, err := os.ReadFile(filepath.Join(directory, fixedPlannerOutputComposeFile)) + if err != nil { + t.Fatalf("read compose override: %v", err) + } + var override struct { + Services struct { + Planner struct { + Profiles []string `yaml:"profiles"` + } `yaml:"planner"` + QueryEngine struct { + DependsOn yaml.Node `yaml:"depends_on"` + Volumes []string `yaml:"volumes"` + } `yaml:"queryengine"` + } `yaml:"services"` + } + if err := yaml.Unmarshal(overrideContents, &override); err != nil { + t.Fatalf("parse compose override: %v", err) + } + if len(override.Services.Planner.Profiles) == 0 { + t.Fatal("compose override leaves the planner service active") + } + dependsOn := override.Services.QueryEngine.DependsOn + if dependsOn.Tag != "!override" { + t.Fatalf("queryengine depends_on tag = %q, want !override so the planner dependency is dropped", dependsOn.Tag) + } + var dependencies map[string]any + if err := dependsOn.Decode(&dependencies); err != nil { + t.Fatalf("decode queryengine depends_on: %v", err) + } + if _, ok := dependencies["planner"]; ok { + t.Fatal("queryengine still depends on the planner") + } + wantVolume := "${" + fixedPlannerOutputDirEnv + ":?" + fixedPlannerOutputDirEnv + " is required}:" + plannerOutputMountPath + ":ro" + if got := override.Services.QueryEngine.Volumes; len(got) != 1 || got[0] != wantVolume { + t.Fatalf("queryengine volumes = %v, want [%s]", got, wantVolume) + } + + engineContents, err := os.ReadFile(filepath.Join(directory, "engine_config.yaml")) + if err != nil { + t.Fatalf("read engine config: %v", err) + } + var engine struct { + InferenceConfig string `yaml:"inference_config"` + StreamingConfig string `yaml:"streaming_config"` + } + if err := yaml.Unmarshal(engineContents, &engine); err != nil { + t.Fatalf("parse engine config: %v", err) + } + if got, want := engine.InferenceConfig, plannerOutputMountPath+"/"+plannerInferenceConfigFile; got != want { + t.Fatalf("engine inference_config = %q, want %q", got, want) + } + if got, want := engine.StreamingConfig, plannerOutputMountPath+"/"+plannerStreamingConfigFile; got != want { + t.Fatalf("engine streaming_config = %q, want %q", got, want) + } +} From 901cf843c2fbfdf1bbb9517a459acd85f7ad1a76 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Mon, 12 Oct 2026 22:53:55 -0400 Subject: [PATCH 2/3] test(promql-compliance): add roll-up suite with fixed planner output The rollup suite reads KLL, DDSketch, Sum and MinMax aggregations grouped by (service, endpoint), over tumbling and sliding windows, through queries grouped by (service, endpoint), by (service) and with no grouping. The dataset includes an endpoint that starts mid-range. Co-Authored-By: Claude Opus 5.5 --- promql-compliance/datasets/rollup.yaml | 569 ++++++++++++++++++ promql-compliance/runner/Makefile | 3 +- .../checked_in_fixture_validation_test.go | 47 ++ .../inference_config.yaml | 117 ++++ .../streaming_config.yaml | 113 ++++ promql-compliance/suites/rollup.yaml | 125 ++++ 6 files changed, 973 insertions(+), 1 deletion(-) create mode 100644 promql-compliance/datasets/rollup.yaml create mode 100644 promql-compliance/suites/rollup-planner-output/inference_config.yaml create mode 100644 promql-compliance/suites/rollup-planner-output/streaming_config.yaml create mode 100644 promql-compliance/suites/rollup.yaml diff --git a/promql-compliance/datasets/rollup.yaml b/promql-compliance/datasets/rollup.yaml new file mode 100644 index 00000000..9091e645 --- /dev/null +++ b/promql-compliance/datasets/rollup.yaml @@ -0,0 +1,569 @@ +name: rollup +series: + # rollup_latency{service, endpoint}: 2 services x 10 endpoints at 60s + # cadence, plus endpoint e-late on service a that starts at offset 600 so + # roll-ups see a subpopulation appear mid-range. + # + # With i = endpoint*2 + service (e-late is i = 20), value = 1000 + 2*i + + # 10*(t/60). Within one service adjacent values are 4 apart (under 0.5%), + # which bounds the gap between ASAPQuery's nearest-rank quantile and + # Prometheus' interpolated one well inside the suite's sketch tolerance. + # + # Samples run to offset 1260, past the last evaluated offset (1200). + - metric: rollup_latency + labels: + service: a + endpoint: e00 + samples: + - {offset_seconds: 0, value: 1000} + - {offset_seconds: 60, value: 1010} + - {offset_seconds: 120, value: 1020} + - {offset_seconds: 180, value: 1030} + - {offset_seconds: 240, value: 1040} + - {offset_seconds: 300, value: 1050} + - {offset_seconds: 360, value: 1060} + - {offset_seconds: 420, value: 1070} + - {offset_seconds: 480, value: 1080} + - {offset_seconds: 540, value: 1090} + - {offset_seconds: 600, value: 1100} + - {offset_seconds: 660, value: 1110} + - {offset_seconds: 720, value: 1120} + - {offset_seconds: 780, value: 1130} + - {offset_seconds: 840, value: 1140} + - {offset_seconds: 900, value: 1150} + - {offset_seconds: 960, value: 1160} + - {offset_seconds: 1020, value: 1170} + - {offset_seconds: 1080, value: 1180} + - {offset_seconds: 1140, value: 1190} + - {offset_seconds: 1200, value: 1200} + - {offset_seconds: 1260, value: 1210} + - metric: rollup_latency + labels: + service: b + endpoint: e00 + samples: + - {offset_seconds: 0, value: 1002} + - {offset_seconds: 60, value: 1012} + - {offset_seconds: 120, value: 1022} + - {offset_seconds: 180, value: 1032} + - {offset_seconds: 240, value: 1042} + - {offset_seconds: 300, value: 1052} + - {offset_seconds: 360, value: 1062} + - {offset_seconds: 420, value: 1072} + - {offset_seconds: 480, value: 1082} + - {offset_seconds: 540, value: 1092} + - {offset_seconds: 600, value: 1102} + - {offset_seconds: 660, value: 1112} + - {offset_seconds: 720, value: 1122} + - {offset_seconds: 780, value: 1132} + - {offset_seconds: 840, value: 1142} + - {offset_seconds: 900, value: 1152} + - {offset_seconds: 960, value: 1162} + - {offset_seconds: 1020, value: 1172} + - {offset_seconds: 1080, value: 1182} + - {offset_seconds: 1140, value: 1192} + - {offset_seconds: 1200, value: 1202} + - {offset_seconds: 1260, value: 1212} + - metric: rollup_latency + labels: + service: a + endpoint: e01 + samples: + - {offset_seconds: 0, value: 1004} + - {offset_seconds: 60, value: 1014} + - {offset_seconds: 120, value: 1024} + - {offset_seconds: 180, value: 1034} + - {offset_seconds: 240, value: 1044} + - {offset_seconds: 300, value: 1054} + - {offset_seconds: 360, value: 1064} + - {offset_seconds: 420, value: 1074} + - {offset_seconds: 480, value: 1084} + - {offset_seconds: 540, value: 1094} + - {offset_seconds: 600, value: 1104} + - {offset_seconds: 660, value: 1114} + - {offset_seconds: 720, value: 1124} + - {offset_seconds: 780, value: 1134} + - {offset_seconds: 840, value: 1144} + - {offset_seconds: 900, value: 1154} + - {offset_seconds: 960, value: 1164} + - {offset_seconds: 1020, value: 1174} + - {offset_seconds: 1080, value: 1184} + - {offset_seconds: 1140, value: 1194} + - {offset_seconds: 1200, value: 1204} + - {offset_seconds: 1260, value: 1214} + - metric: rollup_latency + labels: + service: b + endpoint: e01 + samples: + - {offset_seconds: 0, value: 1006} + - {offset_seconds: 60, value: 1016} + - {offset_seconds: 120, value: 1026} + - {offset_seconds: 180, value: 1036} + - {offset_seconds: 240, value: 1046} + - {offset_seconds: 300, value: 1056} + - {offset_seconds: 360, value: 1066} + - {offset_seconds: 420, value: 1076} + - {offset_seconds: 480, value: 1086} + - {offset_seconds: 540, value: 1096} + - {offset_seconds: 600, value: 1106} + - {offset_seconds: 660, value: 1116} + - {offset_seconds: 720, value: 1126} + - {offset_seconds: 780, value: 1136} + - {offset_seconds: 840, value: 1146} + - {offset_seconds: 900, value: 1156} + - {offset_seconds: 960, value: 1166} + - {offset_seconds: 1020, value: 1176} + - {offset_seconds: 1080, value: 1186} + - {offset_seconds: 1140, value: 1196} + - {offset_seconds: 1200, value: 1206} + - {offset_seconds: 1260, value: 1216} + - metric: rollup_latency + labels: + service: a + endpoint: e02 + samples: + - {offset_seconds: 0, value: 1008} + - {offset_seconds: 60, value: 1018} + - {offset_seconds: 120, value: 1028} + - {offset_seconds: 180, value: 1038} + - {offset_seconds: 240, value: 1048} + - {offset_seconds: 300, value: 1058} + - {offset_seconds: 360, value: 1068} + - {offset_seconds: 420, value: 1078} + - {offset_seconds: 480, value: 1088} + - {offset_seconds: 540, value: 1098} + - {offset_seconds: 600, value: 1108} + - {offset_seconds: 660, value: 1118} + - {offset_seconds: 720, value: 1128} + - {offset_seconds: 780, value: 1138} + - {offset_seconds: 840, value: 1148} + - {offset_seconds: 900, value: 1158} + - {offset_seconds: 960, value: 1168} + - {offset_seconds: 1020, value: 1178} + - {offset_seconds: 1080, value: 1188} + - {offset_seconds: 1140, value: 1198} + - {offset_seconds: 1200, value: 1208} + - {offset_seconds: 1260, value: 1218} + - metric: rollup_latency + labels: + service: b + endpoint: e02 + samples: + - {offset_seconds: 0, value: 1010} + - {offset_seconds: 60, value: 1020} + - {offset_seconds: 120, value: 1030} + - {offset_seconds: 180, value: 1040} + - {offset_seconds: 240, value: 1050} + - {offset_seconds: 300, value: 1060} + - {offset_seconds: 360, value: 1070} + - {offset_seconds: 420, value: 1080} + - {offset_seconds: 480, value: 1090} + - {offset_seconds: 540, value: 1100} + - {offset_seconds: 600, value: 1110} + - {offset_seconds: 660, value: 1120} + - {offset_seconds: 720, value: 1130} + - {offset_seconds: 780, value: 1140} + - {offset_seconds: 840, value: 1150} + - {offset_seconds: 900, value: 1160} + - {offset_seconds: 960, value: 1170} + - {offset_seconds: 1020, value: 1180} + - {offset_seconds: 1080, value: 1190} + - {offset_seconds: 1140, value: 1200} + - {offset_seconds: 1200, value: 1210} + - {offset_seconds: 1260, value: 1220} + - metric: rollup_latency + labels: + service: a + endpoint: e03 + samples: + - {offset_seconds: 0, value: 1012} + - {offset_seconds: 60, value: 1022} + - {offset_seconds: 120, value: 1032} + - {offset_seconds: 180, value: 1042} + - {offset_seconds: 240, value: 1052} + - {offset_seconds: 300, value: 1062} + - {offset_seconds: 360, value: 1072} + - {offset_seconds: 420, value: 1082} + - {offset_seconds: 480, value: 1092} + - {offset_seconds: 540, value: 1102} + - {offset_seconds: 600, value: 1112} + - {offset_seconds: 660, value: 1122} + - {offset_seconds: 720, value: 1132} + - {offset_seconds: 780, value: 1142} + - {offset_seconds: 840, value: 1152} + - {offset_seconds: 900, value: 1162} + - {offset_seconds: 960, value: 1172} + - {offset_seconds: 1020, value: 1182} + - {offset_seconds: 1080, value: 1192} + - {offset_seconds: 1140, value: 1202} + - {offset_seconds: 1200, value: 1212} + - {offset_seconds: 1260, value: 1222} + - metric: rollup_latency + labels: + service: b + endpoint: e03 + samples: + - {offset_seconds: 0, value: 1014} + - {offset_seconds: 60, value: 1024} + - {offset_seconds: 120, value: 1034} + - {offset_seconds: 180, value: 1044} + - {offset_seconds: 240, value: 1054} + - {offset_seconds: 300, value: 1064} + - {offset_seconds: 360, value: 1074} + - {offset_seconds: 420, value: 1084} + - {offset_seconds: 480, value: 1094} + - {offset_seconds: 540, value: 1104} + - {offset_seconds: 600, value: 1114} + - {offset_seconds: 660, value: 1124} + - {offset_seconds: 720, value: 1134} + - {offset_seconds: 780, value: 1144} + - {offset_seconds: 840, value: 1154} + - {offset_seconds: 900, value: 1164} + - {offset_seconds: 960, value: 1174} + - {offset_seconds: 1020, value: 1184} + - {offset_seconds: 1080, value: 1194} + - {offset_seconds: 1140, value: 1204} + - {offset_seconds: 1200, value: 1214} + - {offset_seconds: 1260, value: 1224} + - metric: rollup_latency + labels: + service: a + endpoint: e04 + samples: + - {offset_seconds: 0, value: 1016} + - {offset_seconds: 60, value: 1026} + - {offset_seconds: 120, value: 1036} + - {offset_seconds: 180, value: 1046} + - {offset_seconds: 240, value: 1056} + - {offset_seconds: 300, value: 1066} + - {offset_seconds: 360, value: 1076} + - {offset_seconds: 420, value: 1086} + - {offset_seconds: 480, value: 1096} + - {offset_seconds: 540, value: 1106} + - {offset_seconds: 600, value: 1116} + - {offset_seconds: 660, value: 1126} + - {offset_seconds: 720, value: 1136} + - {offset_seconds: 780, value: 1146} + - {offset_seconds: 840, value: 1156} + - {offset_seconds: 900, value: 1166} + - {offset_seconds: 960, value: 1176} + - {offset_seconds: 1020, value: 1186} + - {offset_seconds: 1080, value: 1196} + - {offset_seconds: 1140, value: 1206} + - {offset_seconds: 1200, value: 1216} + - {offset_seconds: 1260, value: 1226} + - metric: rollup_latency + labels: + service: b + endpoint: e04 + samples: + - {offset_seconds: 0, value: 1018} + - {offset_seconds: 60, value: 1028} + - {offset_seconds: 120, value: 1038} + - {offset_seconds: 180, value: 1048} + - {offset_seconds: 240, value: 1058} + - {offset_seconds: 300, value: 1068} + - {offset_seconds: 360, value: 1078} + - {offset_seconds: 420, value: 1088} + - {offset_seconds: 480, value: 1098} + - {offset_seconds: 540, value: 1108} + - {offset_seconds: 600, value: 1118} + - {offset_seconds: 660, value: 1128} + - {offset_seconds: 720, value: 1138} + - {offset_seconds: 780, value: 1148} + - {offset_seconds: 840, value: 1158} + - {offset_seconds: 900, value: 1168} + - {offset_seconds: 960, value: 1178} + - {offset_seconds: 1020, value: 1188} + - {offset_seconds: 1080, value: 1198} + - {offset_seconds: 1140, value: 1208} + - {offset_seconds: 1200, value: 1218} + - {offset_seconds: 1260, value: 1228} + - metric: rollup_latency + labels: + service: a + endpoint: e05 + samples: + - {offset_seconds: 0, value: 1020} + - {offset_seconds: 60, value: 1030} + - {offset_seconds: 120, value: 1040} + - {offset_seconds: 180, value: 1050} + - {offset_seconds: 240, value: 1060} + - {offset_seconds: 300, value: 1070} + - {offset_seconds: 360, value: 1080} + - {offset_seconds: 420, value: 1090} + - {offset_seconds: 480, value: 1100} + - {offset_seconds: 540, value: 1110} + - {offset_seconds: 600, value: 1120} + - {offset_seconds: 660, value: 1130} + - {offset_seconds: 720, value: 1140} + - {offset_seconds: 780, value: 1150} + - {offset_seconds: 840, value: 1160} + - {offset_seconds: 900, value: 1170} + - {offset_seconds: 960, value: 1180} + - {offset_seconds: 1020, value: 1190} + - {offset_seconds: 1080, value: 1200} + - {offset_seconds: 1140, value: 1210} + - {offset_seconds: 1200, value: 1220} + - {offset_seconds: 1260, value: 1230} + - metric: rollup_latency + labels: + service: b + endpoint: e05 + samples: + - {offset_seconds: 0, value: 1022} + - {offset_seconds: 60, value: 1032} + - {offset_seconds: 120, value: 1042} + - {offset_seconds: 180, value: 1052} + - {offset_seconds: 240, value: 1062} + - {offset_seconds: 300, value: 1072} + - {offset_seconds: 360, value: 1082} + - {offset_seconds: 420, value: 1092} + - {offset_seconds: 480, value: 1102} + - {offset_seconds: 540, value: 1112} + - {offset_seconds: 600, value: 1122} + - {offset_seconds: 660, value: 1132} + - {offset_seconds: 720, value: 1142} + - {offset_seconds: 780, value: 1152} + - {offset_seconds: 840, value: 1162} + - {offset_seconds: 900, value: 1172} + - {offset_seconds: 960, value: 1182} + - {offset_seconds: 1020, value: 1192} + - {offset_seconds: 1080, value: 1202} + - {offset_seconds: 1140, value: 1212} + - {offset_seconds: 1200, value: 1222} + - {offset_seconds: 1260, value: 1232} + - metric: rollup_latency + labels: + service: a + endpoint: e06 + samples: + - {offset_seconds: 0, value: 1024} + - {offset_seconds: 60, value: 1034} + - {offset_seconds: 120, value: 1044} + - {offset_seconds: 180, value: 1054} + - {offset_seconds: 240, value: 1064} + - {offset_seconds: 300, value: 1074} + - {offset_seconds: 360, value: 1084} + - {offset_seconds: 420, value: 1094} + - {offset_seconds: 480, value: 1104} + - {offset_seconds: 540, value: 1114} + - {offset_seconds: 600, value: 1124} + - {offset_seconds: 660, value: 1134} + - {offset_seconds: 720, value: 1144} + - {offset_seconds: 780, value: 1154} + - {offset_seconds: 840, value: 1164} + - {offset_seconds: 900, value: 1174} + - {offset_seconds: 960, value: 1184} + - {offset_seconds: 1020, value: 1194} + - {offset_seconds: 1080, value: 1204} + - {offset_seconds: 1140, value: 1214} + - {offset_seconds: 1200, value: 1224} + - {offset_seconds: 1260, value: 1234} + - metric: rollup_latency + labels: + service: b + endpoint: e06 + samples: + - {offset_seconds: 0, value: 1026} + - {offset_seconds: 60, value: 1036} + - {offset_seconds: 120, value: 1046} + - {offset_seconds: 180, value: 1056} + - {offset_seconds: 240, value: 1066} + - {offset_seconds: 300, value: 1076} + - {offset_seconds: 360, value: 1086} + - {offset_seconds: 420, value: 1096} + - {offset_seconds: 480, value: 1106} + - {offset_seconds: 540, value: 1116} + - {offset_seconds: 600, value: 1126} + - {offset_seconds: 660, value: 1136} + - {offset_seconds: 720, value: 1146} + - {offset_seconds: 780, value: 1156} + - {offset_seconds: 840, value: 1166} + - {offset_seconds: 900, value: 1176} + - {offset_seconds: 960, value: 1186} + - {offset_seconds: 1020, value: 1196} + - {offset_seconds: 1080, value: 1206} + - {offset_seconds: 1140, value: 1216} + - {offset_seconds: 1200, value: 1226} + - {offset_seconds: 1260, value: 1236} + - metric: rollup_latency + labels: + service: a + endpoint: e07 + samples: + - {offset_seconds: 0, value: 1028} + - {offset_seconds: 60, value: 1038} + - {offset_seconds: 120, value: 1048} + - {offset_seconds: 180, value: 1058} + - {offset_seconds: 240, value: 1068} + - {offset_seconds: 300, value: 1078} + - {offset_seconds: 360, value: 1088} + - {offset_seconds: 420, value: 1098} + - {offset_seconds: 480, value: 1108} + - {offset_seconds: 540, value: 1118} + - {offset_seconds: 600, value: 1128} + - {offset_seconds: 660, value: 1138} + - {offset_seconds: 720, value: 1148} + - {offset_seconds: 780, value: 1158} + - {offset_seconds: 840, value: 1168} + - {offset_seconds: 900, value: 1178} + - {offset_seconds: 960, value: 1188} + - {offset_seconds: 1020, value: 1198} + - {offset_seconds: 1080, value: 1208} + - {offset_seconds: 1140, value: 1218} + - {offset_seconds: 1200, value: 1228} + - {offset_seconds: 1260, value: 1238} + - metric: rollup_latency + labels: + service: b + endpoint: e07 + samples: + - {offset_seconds: 0, value: 1030} + - {offset_seconds: 60, value: 1040} + - {offset_seconds: 120, value: 1050} + - {offset_seconds: 180, value: 1060} + - {offset_seconds: 240, value: 1070} + - {offset_seconds: 300, value: 1080} + - {offset_seconds: 360, value: 1090} + - {offset_seconds: 420, value: 1100} + - {offset_seconds: 480, value: 1110} + - {offset_seconds: 540, value: 1120} + - {offset_seconds: 600, value: 1130} + - {offset_seconds: 660, value: 1140} + - {offset_seconds: 720, value: 1150} + - {offset_seconds: 780, value: 1160} + - {offset_seconds: 840, value: 1170} + - {offset_seconds: 900, value: 1180} + - {offset_seconds: 960, value: 1190} + - {offset_seconds: 1020, value: 1200} + - {offset_seconds: 1080, value: 1210} + - {offset_seconds: 1140, value: 1220} + - {offset_seconds: 1200, value: 1230} + - {offset_seconds: 1260, value: 1240} + - metric: rollup_latency + labels: + service: a + endpoint: e08 + samples: + - {offset_seconds: 0, value: 1032} + - {offset_seconds: 60, value: 1042} + - {offset_seconds: 120, value: 1052} + - {offset_seconds: 180, value: 1062} + - {offset_seconds: 240, value: 1072} + - {offset_seconds: 300, value: 1082} + - {offset_seconds: 360, value: 1092} + - {offset_seconds: 420, value: 1102} + - {offset_seconds: 480, value: 1112} + - {offset_seconds: 540, value: 1122} + - {offset_seconds: 600, value: 1132} + - {offset_seconds: 660, value: 1142} + - {offset_seconds: 720, value: 1152} + - {offset_seconds: 780, value: 1162} + - {offset_seconds: 840, value: 1172} + - {offset_seconds: 900, value: 1182} + - {offset_seconds: 960, value: 1192} + - {offset_seconds: 1020, value: 1202} + - {offset_seconds: 1080, value: 1212} + - {offset_seconds: 1140, value: 1222} + - {offset_seconds: 1200, value: 1232} + - {offset_seconds: 1260, value: 1242} + - metric: rollup_latency + labels: + service: b + endpoint: e08 + samples: + - {offset_seconds: 0, value: 1034} + - {offset_seconds: 60, value: 1044} + - {offset_seconds: 120, value: 1054} + - {offset_seconds: 180, value: 1064} + - {offset_seconds: 240, value: 1074} + - {offset_seconds: 300, value: 1084} + - {offset_seconds: 360, value: 1094} + - {offset_seconds: 420, value: 1104} + - {offset_seconds: 480, value: 1114} + - {offset_seconds: 540, value: 1124} + - {offset_seconds: 600, value: 1134} + - {offset_seconds: 660, value: 1144} + - {offset_seconds: 720, value: 1154} + - {offset_seconds: 780, value: 1164} + - {offset_seconds: 840, value: 1174} + - {offset_seconds: 900, value: 1184} + - {offset_seconds: 960, value: 1194} + - {offset_seconds: 1020, value: 1204} + - {offset_seconds: 1080, value: 1214} + - {offset_seconds: 1140, value: 1224} + - {offset_seconds: 1200, value: 1234} + - {offset_seconds: 1260, value: 1244} + - metric: rollup_latency + labels: + service: a + endpoint: e09 + samples: + - {offset_seconds: 0, value: 1036} + - {offset_seconds: 60, value: 1046} + - {offset_seconds: 120, value: 1056} + - {offset_seconds: 180, value: 1066} + - {offset_seconds: 240, value: 1076} + - {offset_seconds: 300, value: 1086} + - {offset_seconds: 360, value: 1096} + - {offset_seconds: 420, value: 1106} + - {offset_seconds: 480, value: 1116} + - {offset_seconds: 540, value: 1126} + - {offset_seconds: 600, value: 1136} + - {offset_seconds: 660, value: 1146} + - {offset_seconds: 720, value: 1156} + - {offset_seconds: 780, value: 1166} + - {offset_seconds: 840, value: 1176} + - {offset_seconds: 900, value: 1186} + - {offset_seconds: 960, value: 1196} + - {offset_seconds: 1020, value: 1206} + - {offset_seconds: 1080, value: 1216} + - {offset_seconds: 1140, value: 1226} + - {offset_seconds: 1200, value: 1236} + - {offset_seconds: 1260, value: 1246} + - metric: rollup_latency + labels: + service: b + endpoint: e09 + samples: + - {offset_seconds: 0, value: 1038} + - {offset_seconds: 60, value: 1048} + - {offset_seconds: 120, value: 1058} + - {offset_seconds: 180, value: 1068} + - {offset_seconds: 240, value: 1078} + - {offset_seconds: 300, value: 1088} + - {offset_seconds: 360, value: 1098} + - {offset_seconds: 420, value: 1108} + - {offset_seconds: 480, value: 1118} + - {offset_seconds: 540, value: 1128} + - {offset_seconds: 600, value: 1138} + - {offset_seconds: 660, value: 1148} + - {offset_seconds: 720, value: 1158} + - {offset_seconds: 780, value: 1168} + - {offset_seconds: 840, value: 1178} + - {offset_seconds: 900, value: 1188} + - {offset_seconds: 960, value: 1198} + - {offset_seconds: 1020, value: 1208} + - {offset_seconds: 1080, value: 1218} + - {offset_seconds: 1140, value: 1228} + - {offset_seconds: 1200, value: 1238} + - {offset_seconds: 1260, value: 1248} + - metric: rollup_latency + labels: + service: a + endpoint: e-late + samples: + - {offset_seconds: 600, value: 1140} + - {offset_seconds: 660, value: 1150} + - {offset_seconds: 720, value: 1160} + - {offset_seconds: 780, value: 1170} + - {offset_seconds: 840, value: 1180} + - {offset_seconds: 900, value: 1190} + - {offset_seconds: 960, value: 1200} + - {offset_seconds: 1020, value: 1210} + - {offset_seconds: 1080, value: 1220} + - {offset_seconds: 1140, value: 1230} + - {offset_seconds: 1200, value: 1240} + - {offset_seconds: 1260, value: 1250} diff --git a/promql-compliance/runner/Makefile b/promql-compliance/runner/Makefile index 0461ac2e..3f501175 100644 --- a/promql-compliance/runner/Makefile +++ b/promql-compliance/runner/Makefile @@ -13,7 +13,8 @@ CI_CASES := \ aggregations-native-dag:../datasets/aggregations.yaml:../suites/native-dag-aggregations.yaml \ aggregations:../datasets/aggregations.yaml:../suites/aggregations.yaml \ topk-ordering:../datasets/topk-ordering.yaml:../suites/topk-ordering.yaml \ - quantiles:../datasets/quantiles.yaml:../suites/quantiles.yaml + quantiles:../datasets/quantiles.yaml:../suites/quantiles.yaml \ + rollup:../datasets/rollup.yaml:../suites/rollup.yaml # Tracks planner coverage (#738); most queries are expected to fail. NON_CI_CASES := \ diff --git a/promql-compliance/runner/checked_in_fixture_validation_test.go b/promql-compliance/runner/checked_in_fixture_validation_test.go index dea0a89c..8eb38c00 100644 --- a/promql-compliance/runner/checked_in_fixture_validation_test.go +++ b/promql-compliance/runner/checked_in_fixture_validation_test.go @@ -1,10 +1,13 @@ package runner import ( + "os" + "path/filepath" "testing" "github.com/ProjectASAP/ASAPQuery/promql-compliance/seeder" promqlparser "github.com/prometheus/prometheus/promql/parser" + "gopkg.in/yaml.v3" ) func TestCheckedInFixturesAndSuites(t *testing.T) { @@ -80,3 +83,47 @@ func TestCheckedInNativeDagSuites(t *testing.T) { } } } + +// Every rollup suite query must have a query config in the fixed planner +// output; a mismatched string would make the engine reject it rather than +// exercise a roll-up. +func TestCheckedInRollupSuiteMatchesFixedPlannerOutput(t *testing.T) { + if _, err := seeder.LoadFixture("../datasets/rollup.yaml"); err != nil { + t.Fatalf("LoadFixture: %v", err) + } + suite, err := LoadSuiteFile("../suites/rollup.yaml") + if err != nil { + t.Fatalf("LoadSuiteFile: %v", err) + } + if suite.PlannerOutputDir == "" { + t.Fatal("rollup suite has no planner_output_dir") + } + contents, err := os.ReadFile(filepath.Join(suite.PlannerOutputDir, plannerInferenceConfigFile)) + if err != nil { + t.Fatalf("read inference config: %v", err) + } + var inference struct { + Queries []struct { + Query string `yaml:"query"` + } `yaml:"queries"` + } + if err := yaml.Unmarshal(contents, &inference); err != nil { + t.Fatalf("parse inference config: %v", err) + } + configured := make(map[string]struct{}, len(inference.Queries)) + for _, query := range inference.Queries { + configured[query.Query] = struct{}{} + } + parser := promqlparser.NewParser(promqlparser.Options{}) + for _, query := range suite.Queries { + if _, err := parser.ParseExpr(query.Expr); err != nil { + t.Fatalf("parse %q: %v", query.Expr, err) + } + if _, ok := configured[query.Expr]; !ok { + t.Errorf("query %q has no query config in the fixed planner output", query.Expr) + } + } + if got, want := len(inference.Queries), len(suite.Queries); got != want { + t.Errorf("fixed planner output has %d query configs, suite has %d queries", got, want) + } +} diff --git a/promql-compliance/suites/rollup-planner-output/inference_config.yaml b/promql-compliance/suites/rollup-planner-output/inference_config.yaml new file mode 100644 index 00000000..6a7f7006 --- /dev/null +++ b/promql-compliance/suites/rollup-planner-output/inference_config.yaml @@ -0,0 +1,117 @@ +# Hand-written in the planner's output format. Queries grouped by a subset of +# an aggregation's grouping labels point at that aggregation, so the engine +# answers them by rolling the per-(service, endpoint) aggregates up. +# no_cleanup: several queries read the same windows, so a per-query read +# count threshold would evict windows other queries still need. +cleanup_policy: + name: no_cleanup +metrics: + rollup_latency: + - endpoint + - service +queries: +- aggregations: + - aggregation_id: 1 + query: quantile by (service, endpoint) (0.5, rollup_latency) + planned_subquery: quantile by (service, endpoint) (0.5, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + query: quantile by (service) (0.5, rollup_latency) + planned_subquery: quantile by (service) (0.5, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + query: quantile(0.5, rollup_latency) + planned_subquery: quantile(0.5, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + query: quantile by (service, endpoint) (0.9, rollup_latency) + planned_subquery: quantile by (service, endpoint) (0.9, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + query: quantile by (service) (0.9, rollup_latency) + planned_subquery: quantile by (service) (0.9, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + query: quantile(0.9, rollup_latency) + planned_subquery: quantile(0.9, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 2 + query: quantile by (service, endpoint) (0.75, rollup_latency) + planned_subquery: quantile by (service, endpoint) (0.75, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 2 + query: quantile by (service) (0.75, rollup_latency) + planned_subquery: quantile by (service) (0.75, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 2 + query: quantile(0.75, rollup_latency) + planned_subquery: quantile(0.75, rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + query: sum by (service, endpoint) (rollup_latency) + planned_subquery: sum by (service, endpoint) (rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + query: sum by (service) (rollup_latency) + planned_subquery: sum by (service) (rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + query: sum(rollup_latency) + planned_subquery: sum(rollup_latency) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + query: sum by (service, endpoint) (sum_over_time(rollup_latency[5m])) + planned_subquery: sum by (service, endpoint) (sum_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + query: sum by (service) (sum_over_time(rollup_latency[5m])) + planned_subquery: sum by (service) (sum_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + query: sum(sum_over_time(rollup_latency[5m])) + planned_subquery: sum(sum_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 5 + query: min by (service, endpoint) (min_over_time(rollup_latency[5m])) + planned_subquery: min by (service, endpoint) (min_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 5 + query: min by (service) (min_over_time(rollup_latency[5m])) + planned_subquery: min by (service) (min_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 5 + query: min(min_over_time(rollup_latency[5m])) + planned_subquery: min(min_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 6 + query: max by (service, endpoint) (max_over_time(rollup_latency[5m])) + planned_subquery: max by (service, endpoint) (max_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 6 + query: max by (service) (max_over_time(rollup_latency[5m])) + planned_subquery: max by (service) (max_over_time(rollup_latency[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 6 + query: max(max_over_time(rollup_latency[5m])) + planned_subquery: max(max_over_time(rollup_latency[5m])) + query_time_aggregations: [] diff --git a/promql-compliance/suites/rollup-planner-output/streaming_config.yaml b/promql-compliance/suites/rollup-planner-output/streaming_config.yaml new file mode 100644 index 00000000..afb0dc73 --- /dev/null +++ b/promql-compliance/suites/rollup-planner-output/streaming_config.yaml @@ -0,0 +1,113 @@ +# Hand-written in the planner's output format. Every aggregation is a +# single-subpopulation type grouped by (service, endpoint); the tumbling +# window sizes match what the planner picks for each query shape, and ids 4 +# and 6 are sliding to cover sliding-window roll-ups. +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DatasketchesKLL + labels: + aggregated: [] + grouping: + - endpoint + - service + rollup: [] + metric: rollup_latency + parameters: + K: 500 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 1000 + windowType: tumbling +- aggregationId: 2 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - endpoint + - service + rollup: [] + metric: rollup_latency + parameters: + alpha: 0.01 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 1000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: Sum + labels: + aggregated: [] + grouping: + - endpoint + - service + rollup: [] + metric: rollup_latency + parameters: {} + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 1000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: Sum + labels: + aggregated: [] + grouping: + - endpoint + - service + rollup: [] + metric: rollup_latency + parameters: {} + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 300000 + windowType: sliding +- aggregationId: 5 + aggregationSubType: min + aggregationType: MinMax + labels: + aggregated: [] + grouping: + - endpoint + - service + rollup: [] + metric: rollup_latency + parameters: {} + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 60000 + windowType: tumbling +- aggregationId: 6 + aggregationSubType: max + aggregationType: MinMax + labels: + aggregated: [] + grouping: + - endpoint + - service + rollup: [] + metric: rollup_latency + parameters: {} + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 300000 + windowType: sliding +metrics: + rollup_latency: + - endpoint + - service diff --git a/promql-compliance/suites/rollup.yaml b/promql-compliance/suites/rollup.yaml new file mode 100644 index 00000000..e595cca3 --- /dev/null +++ b/promql-compliance/suites/rollup.yaml @@ -0,0 +1,125 @@ +name: rollup +# Every aggregation in the fixed planner output is grouped by (service, +# endpoint). Each query appears grouped by (service, endpoint), which reads +# the aggregation as-is, then by (service) and with no grouping, which roll +# the per-endpoint aggregates up. The first query is the readiness probe, so +# it is a non-rolled-up one. +planner_output_dir: rollup-planner-output +comparison_defaults: + value_tolerance: + relative: 0.01 + absolute: 0.000001 + +queries: + # DatasketchesKLL, 1s tumbling. + - name: kll-quantile-0.5-by-service-endpoint + expr: "quantile by (service, endpoint) (0.5, rollup_latency)" + instant_offsets_seconds: &evaluation_offsets [300, 600, 900, 1200] + range: &evaluation_range + start_offset_seconds: 300 + end_offset_seconds: 1200 + step_seconds: 60 + - name: kll-quantile-0.5-by-service + expr: "quantile by (service) (0.5, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + - name: kll-quantile-0.5-all + expr: "quantile(0.5, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + - name: kll-quantile-0.9-by-service-endpoint + expr: "quantile by (service, endpoint) (0.9, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + - name: kll-quantile-0.9-by-service + expr: "quantile by (service) (0.9, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + - name: kll-quantile-0.9-all + expr: "quantile(0.9, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + # DDSketch (alpha 0.01), 1s tumbling. Its relative error adds to the + # nearest-rank gap, hence the wider tolerance. + - name: ddsketch-quantile-0.75-by-service-endpoint + expr: "quantile by (service, endpoint) (0.75, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: &ddsketch_tolerance + value_tolerance: + relative: 0.02 + - name: ddsketch-quantile-0.75-by-service + expr: "quantile by (service) (0.75, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *ddsketch_tolerance + - name: ddsketch-quantile-0.75-all + expr: "quantile(0.75, rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *ddsketch_tolerance + # Sum and MinMax merge exactly, so they are compared tightly. Sum is 1s + # tumbling for the instant-vector sum and 5m/1m sliding for sum_over_time; + # min is 1m tumbling and max is 5m/1m sliding. + - name: sum-by-service-endpoint + expr: "sum by (service, endpoint) (rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: &exact + value_tolerance: + relative: 0 + - name: sum-by-service + expr: "sum by (service) (rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: sum-all + expr: "sum(rollup_latency)" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: sum-sum-over-time-by-service-endpoint + expr: "sum by (service, endpoint) (sum_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: sum-sum-over-time-by-service + expr: "sum by (service) (sum_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: sum-sum-over-time-all + expr: "sum(sum_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: min-min-over-time-by-service-endpoint + expr: "min by (service, endpoint) (min_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: min-min-over-time-by-service + expr: "min by (service) (min_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: min-min-over-time-all + expr: "min(min_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: max-max-over-time-by-service-endpoint + expr: "max by (service, endpoint) (max_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: max-max-over-time-by-service + expr: "max by (service) (max_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact + - name: max-max-over-time-all + expr: "max(max_over_time(rollup_latency[5m]))" + instant_offsets_seconds: *evaluation_offsets + range: *evaluation_range + comparison: *exact From ba348ee22894008644182b0196ba53cab8cf8a1b Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Tue, 13 Oct 2026 10:33:36 -0400 Subject: [PATCH 3/3] fix(promql-compliance): harden the fixed planner output override - Check docker compose >= 2.24.4 (needed for !override) before using it, and document it. - Clear COMPOSE_PROFILES so an inherited value can't re-enable the planner. - Bind the fixed directory with long syntax, so a ':' in its path is safe. - Only skip planner-input generation when a Compose project is used. - LoadSuite rejects a relative planner_output_dir and validates an absolute one, so every loaded suite holds an absolute, checked path. - The checked-in fixture test also checks aggregation ids, metrics and grouping labels; an engine test parses the fixed configs with the engine's own loaders. Co-Authored-By: Claude Opus 5.5 --- asap-query-engine/src/utils/file_io.rs | 37 ++++++++ promql-compliance/HOW_TO.md | 6 +- .../checked_in_fixture_validation_test.go | 58 +++++++++++- promql-compliance/runner/config.go | 26 +++++- promql-compliance/runner/config_test.go | 43 ++++++--- promql-compliance/runner/run.go | 90 ++++++++++++++++--- promql-compliance/runner/run_test.go | 78 ++++++++++++++-- 7 files changed, 303 insertions(+), 35 deletions(-) diff --git a/asap-query-engine/src/utils/file_io.rs b/asap-query-engine/src/utils/file_io.rs index dd129d49..f057f71a 100644 --- a/asap-query-engine/src/utils/file_io.rs +++ b/asap-query-engine/src/utils/file_io.rs @@ -41,6 +41,43 @@ mod tests { use std::io::Write; use tempfile::NamedTempFile; + /// The PromQL compliance roll-up suite ships hand-written planner output; + /// parse it the way the engine does so it can't drift from the format. + #[test] + fn compliance_rollup_planner_output_loads_as_engine_config() { + let directory = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../promql-compliance/suites/rollup-planner-output" + ); + let inference = read_inference_config( + &format!("{directory}/inference_config.yaml"), + QueryLanguage::promql, + ) + .unwrap(); + let streaming = + read_streaming_config(&format!("{directory}/streaming_config.yaml"), &inference) + .unwrap(); + + assert!(!inference.query_configs.is_empty()); + for query_config in &inference.query_configs { + assert!( + !query_config.aggregations.is_empty(), + "{}", + query_config.query + ); + for reference in &query_config.aggregations { + assert!( + streaming + .get_aggregation_config(reference.aggregation_id) + .is_some(), + "{} names missing aggregation {}", + query_config.query, + reference.aggregation_id + ); + } + } + } + #[test] fn test_read_streaming_config() { let streaming_yaml_content = r#" diff --git a/promql-compliance/HOW_TO.md b/promql-compliance/HOW_TO.md index e05602c3..d3a99a1f 100644 --- a/promql-compliance/HOW_TO.md +++ b/promql-compliance/HOW_TO.md @@ -144,8 +144,10 @@ The path is relative to the suite file. The directory must contain `streaming_config.yaml` and `inference_config.yaml` in the planner's output format; the runner fails at load time if either is missing. The planner service then does not run, and the query engine reads the directory in place -of the planner's output volume. With already-running targets the field has no -effect. `suites/rollup.yaml` is an example. +of the planner's output volume. This needs Docker Compose v2.24.4 or later +(for `!override`); the runner checks the version first. With +already-running targets the field has no effect, and the planner input is +generated and validated as usual. `suites/rollup.yaml` is an example. ## Configure tolerance diff --git a/promql-compliance/runner/checked_in_fixture_validation_test.go b/promql-compliance/runner/checked_in_fixture_validation_test.go index 8eb38c00..18b56400 100644 --- a/promql-compliance/runner/checked_in_fixture_validation_test.go +++ b/promql-compliance/runner/checked_in_fixture_validation_test.go @@ -85,10 +85,11 @@ func TestCheckedInNativeDagSuites(t *testing.T) { } // Every rollup suite query must have a query config in the fixed planner -// output; a mismatched string would make the engine reject it rather than -// exercise a roll-up. +// output, naming aggregations that exist, on a metric and labels the dataset +// has; a mismatch would otherwise surface only as a slow Docker-suite failure. func TestCheckedInRollupSuiteMatchesFixedPlannerOutput(t *testing.T) { - if _, err := seeder.LoadFixture("../datasets/rollup.yaml"); err != nil { + fixture, err := seeder.LoadFixture("../datasets/rollup.yaml") + if err != nil { t.Fatalf("LoadFixture: %v", err) } suite, err := LoadSuiteFile("../suites/rollup.yaml") @@ -104,12 +105,61 @@ func TestCheckedInRollupSuiteMatchesFixedPlannerOutput(t *testing.T) { } var inference struct { Queries []struct { - Query string `yaml:"query"` + Query string `yaml:"query"` + Aggregations []struct { + AggregationID int `yaml:"aggregation_id"` + } `yaml:"aggregations"` } `yaml:"queries"` } if err := yaml.Unmarshal(contents, &inference); err != nil { t.Fatalf("parse inference config: %v", err) } + streamingContents, err := os.ReadFile(filepath.Join(suite.PlannerOutputDir, plannerStreamingConfigFile)) + if err != nil { + t.Fatalf("read streaming config: %v", err) + } + var streaming struct { + Aggregations []struct { + AggregationID int `yaml:"aggregationId"` + Metric string `yaml:"metric"` + Labels struct { + Grouping []string `yaml:"grouping"` + } `yaml:"labels"` + } `yaml:"aggregations"` + } + if err := yaml.Unmarshal(streamingContents, &streaming); err != nil { + t.Fatalf("parse streaming config: %v", err) + } + datasetLabels := make(map[string]map[string]struct{}) + for _, series := range fixture.Series { + if datasetLabels[series.Metric] == nil { + datasetLabels[series.Metric] = make(map[string]struct{}) + } + for label := range series.Labels { + datasetLabels[series.Metric][label] = struct{}{} + } + } + aggregationIDs := make(map[int]struct{}, len(streaming.Aggregations)) + for _, aggregation := range streaming.Aggregations { + aggregationIDs[aggregation.AggregationID] = struct{}{} + labels, ok := datasetLabels[aggregation.Metric] + if !ok { + t.Errorf("aggregation %d is on metric %q, which the dataset lacks", aggregation.AggregationID, aggregation.Metric) + continue + } + for _, label := range aggregation.Labels.Grouping { + if _, ok := labels[label]; !ok { + t.Errorf("aggregation %d groups by %q, which %q series lack", aggregation.AggregationID, label, aggregation.Metric) + } + } + } + for _, query := range inference.Queries { + for _, reference := range query.Aggregations { + if _, ok := aggregationIDs[reference.AggregationID]; !ok { + t.Errorf("query %q names aggregation %d, which the streaming config lacks", query.Query, reference.AggregationID) + } + } + } configured := make(map[string]struct{}, len(inference.Queries)) for _, query := range inference.Queries { configured[query.Query] = struct{}{} diff --git a/promql-compliance/runner/config.go b/promql-compliance/runner/config.go index 0c197a40..5eee6309 100644 --- a/promql-compliance/runner/config.go +++ b/promql-compliance/runner/config.go @@ -74,8 +74,30 @@ type Tolerance struct { } // LoadSuite parses and validates a query suite. It rejects incomplete cases -// rather than silently selecting wall-clock defaults. +// rather than silently selecting wall-clock defaults. A planner_output_dir +// must be absolute here; a relative one needs the suite file's location, so +// use LoadSuiteFile. func LoadSuite(contents []byte) (Suite, error) { + suite, err := parseSuite(contents) + if err != nil { + return Suite{}, err + } + if suite.PlannerOutputDir != "" { + if !filepath.IsAbs(suite.PlannerOutputDir) { + return Suite{}, fmt.Errorf( + "query suite %q: planner_output_dir %q is relative; load the suite with LoadSuiteFile", + suite.Name, + suite.PlannerOutputDir, + ) + } + if suite.PlannerOutputDir, err = resolvePlannerOutputDir("", suite.PlannerOutputDir); err != nil { + return Suite{}, fmt.Errorf("query suite %q: %w", suite.Name, err) + } + } + return suite, nil +} + +func parseSuite(contents []byte) (Suite, error) { var suite Suite decoder := yaml.NewDecoder(bytes.NewReader(contents)) decoder.KnownFields(true) @@ -140,7 +162,7 @@ func LoadSuiteFile(path string) (Suite, error) { if err != nil { return Suite{}, fmt.Errorf("read query suite %q: %w", path, err) } - suite, err := LoadSuite(contents) + suite, err := parseSuite(contents) if err != nil { return Suite{}, err } diff --git a/promql-compliance/runner/config_test.go b/promql-compliance/runner/config_test.go index 8aa0a234..2e346699 100644 --- a/promql-compliance/runner/config_test.go +++ b/promql-compliance/runner/config_test.go @@ -195,11 +195,7 @@ func writePlannerOutputSuite(t *testing.T, configured string, files ...string) ( if err := os.MkdirAll(outputDirectory, 0o755); err != nil { t.Fatalf("create planner output directory: %v", err) } - for _, name := range files { - if err := os.WriteFile(filepath.Join(outputDirectory, name), []byte("{}\n"), 0o600); err != nil { - t.Fatalf("write %s: %v", name, err) - } - } + writePlannerOutputFiles(t, outputDirectory, files...) suitePath := filepath.Join(suiteDirectory, "suite.yaml") if err := os.WriteFile(suitePath, []byte(fmt.Sprintf(plannerOutputSuite, configured)), 0o600); err != nil { t.Fatalf("write suite: %v", err) @@ -207,6 +203,15 @@ func writePlannerOutputSuite(t *testing.T, configured string, files ...string) ( return suitePath, outputDirectory } +func writePlannerOutputFiles(t *testing.T, directory string, files ...string) { + t.Helper() + for _, name := range files { + if err := os.WriteFile(filepath.Join(directory, name), []byte("{}\n"), 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } +} + func TestLoadSuiteFileResolvesPlannerOutputDirAgainstSuiteFile(t *testing.T) { suitePath, outputDirectory := writePlannerOutputSuite(t, "fixed", plannerStreamingConfigFile, plannerInferenceConfigFile) @@ -231,11 +236,7 @@ func TestLoadSuiteFileResolvesPlannerOutputDirAgainstSuiteFile(t *testing.T) { func TestLoadSuiteFileAcceptsAbsolutePlannerOutputDir(t *testing.T) { outputDirectory := t.TempDir() - for _, name := range []string{plannerStreamingConfigFile, plannerInferenceConfigFile} { - if err := os.WriteFile(filepath.Join(outputDirectory, name), []byte("{}\n"), 0o600); err != nil { - t.Fatalf("write %s: %v", name, err) - } - } + writePlannerOutputFiles(t, outputDirectory, plannerStreamingConfigFile, plannerInferenceConfigFile) suitePath, _ := writePlannerOutputSuite(t, outputDirectory) suite, err := LoadSuiteFile(suitePath) @@ -290,3 +291,25 @@ func TestLoadSuiteFileRejectsPlannerOutputDirectoryInPlaceOfFile(t *testing.T) { t.Fatal("LoadSuiteFile accepted a directory as a planner config file") } } + +// LoadSuite has no suite file to resolve a relative planner_output_dir +// against, so every Suite it returns already holds an absolute, validated one. +func TestLoadSuiteRequiresAnAbsoluteValidPlannerOutputDir(t *testing.T) { + if _, err := LoadSuite([]byte(fmt.Sprintf(plannerOutputSuite, "fixed"))); err == nil || + !strings.Contains(err.Error(), "LoadSuiteFile") { + t.Fatalf("LoadSuite accepted a relative planner_output_dir (err: %v)", err) + } + + outputDirectory := t.TempDir() + if _, err := LoadSuite([]byte(fmt.Sprintf(plannerOutputSuite, outputDirectory))); err == nil { + t.Fatal("LoadSuite accepted a planner_output_dir without its config files") + } + writePlannerOutputFiles(t, outputDirectory, plannerStreamingConfigFile, plannerInferenceConfigFile) + suite, err := LoadSuite([]byte(fmt.Sprintf(plannerOutputSuite, outputDirectory))) + if err != nil { + t.Fatalf("LoadSuite: %v", err) + } + if got := suite.PlannerOutputDir; got != outputDirectory { + t.Fatalf("planner output dir = %q, want %q", got, outputDirectory) + } +} diff --git a/promql-compliance/runner/run.go b/promql-compliance/runner/run.go index 595e4af3..c5c03ad4 100644 --- a/promql-compliance/runner/run.go +++ b/promql-compliance/runner/run.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "sort" + "strconv" "strings" "time" @@ -36,7 +37,8 @@ const ( // fixedPlannerOutputCompose overrides the base Compose file for suites with a // planner_output_dir: the planner sits behind a never-activated profile so it // does not run, and the query engine mounts the fixed directory where the -// planner's output volume would be. +// planner's output volume would be. The bind uses long syntax so a host path +// containing ':' stays one source. const fixedPlannerOutputCompose = `services: planner: profiles: ["generated-planner-output"] @@ -45,9 +47,16 @@ const fixedPlannerOutputCompose = `services: prometheus: condition: service_healthy volumes: - - "${` + fixedPlannerOutputDirEnv + `:?` + fixedPlannerOutputDirEnv + ` is required}:` + plannerOutputMountPath + `:ro" + - type: bind + source: "${` + fixedPlannerOutputDirEnv + `:?` + fixedPlannerOutputDirEnv + ` is required}" + target: ` + plannerOutputMountPath + ` + read_only: true ` +// The override's `!override` tag needs this Compose release or later; older +// ones merge depends_on and fail on the disabled planner. +var minComposeVersionForOverride = [3]int{2, 24, 4} + type RunOptions struct { DatasetPath string SuitePath string @@ -89,7 +98,15 @@ func Run(ctx context.Context, options RunOptions) (Report, error) { return Report{}, fmt.Errorf("create run directory: %w", err) } defer os.RemoveAll(runDirectory) - if err := writeGeneratedConfigs(runDirectory, fixture, suite); err != nil { + // Fixed planner output only replaces the planner inside a Compose project; + // with caller-owned targets the planner input is still generated. + fixedPlannerOutput := suite.PlannerOutputDir != "" && len(options.ComposeFiles) > 0 + if fixedPlannerOutput { + if err := requireComposeOverrideSupport(ctx); err != nil { + return Report{}, err + } + } + if err := writeGeneratedConfigs(runDirectory, fixture, suite, fixedPlannerOutput); err != nil { return Report{}, err } @@ -98,12 +115,17 @@ func Run(ctx context.Context, options RunOptions) (Report, error) { project: options.ComposeProject, env: []string{"ASAP_TEST_CONFIG_DIR=" + runDirectory}, } - if suite.PlannerOutputDir != "" { + if fixedPlannerOutput { lifecycle.files = append( append([]string(nil), options.ComposeFiles...), filepath.Join(runDirectory, fixedPlannerOutputComposeFile), ) - lifecycle.env = append(lifecycle.env, fixedPlannerOutputDirEnv+"="+suite.PlannerOutputDir) + lifecycle.env = append( + lifecycle.env, + fixedPlannerOutputDirEnv+"="+suite.PlannerOutputDir, + // An inherited COMPOSE_PROFILES could re-enable the planner. + "COMPOSE_PROFILES=", + ) } if len(options.ComposeFiles) > 0 { if err := lifecycle.Start(ctx); err != nil { @@ -372,14 +394,14 @@ type plannerCleanup struct { } // writeGeneratedConfigs writes the engine config plus either the planner input -// or, for a suite with fixed planner output, the Compose override that skips -// the planner. The planner input is not generated in the fixed case, so its -// timing derivation cannot reject queries the planner never sees. -func writeGeneratedConfigs(directory string, fixture seeder.Fixture, suite Suite) error { +// or, with fixed planner output, the Compose override that skips the planner. +// The planner input is not generated in the fixed case, so its timing +// derivation cannot reject queries the planner never sees. +func writeGeneratedConfigs(directory string, fixture seeder.Fixture, suite Suite, fixedPlannerOutput bool) error { if err := writeEngineConfig(directory); err != nil { return err } - if suite.PlannerOutputDir != "" { + if fixedPlannerOutput { if err := os.WriteFile(filepath.Join(directory, fixedPlannerOutputComposeFile), []byte(fixedPlannerOutputCompose), 0o600); err != nil { return fmt.Errorf("write fixed planner output compose override: %w", err) } @@ -388,6 +410,54 @@ func writeGeneratedConfigs(directory string, fixture seeder.Fixture, suite Suite return writePlannerConfig(directory, fixture, suite) } +// requireComposeOverrideSupport fails unless `docker compose` understands the +// fixed-output override's `!override` tag. +func requireComposeOverrideSupport(ctx context.Context) error { + output, err := exec.CommandContext(ctx, "docker", "compose", "version", "--short").Output() + if err != nil { + return fmt.Errorf("check docker compose version for planner_output_dir: %w", err) + } + version := strings.TrimSpace(string(output)) + ok, err := composeVersionAtLeast(version, minComposeVersionForOverride) + if err != nil { + return err + } + if !ok { + return fmt.Errorf( + "planner_output_dir needs docker compose >= %d.%d.%d, found %s", + minComposeVersionForOverride[0], + minComposeVersionForOverride[1], + minComposeVersionForOverride[2], + version, + ) + } + return nil +} + +// composeVersionAtLeast compares a `docker compose version --short` string +// such as "2.29.1" or "v2.24.4-desktop.1" against minimum. +func composeVersionAtLeast(version string, minimum [3]int) (bool, error) { + core := strings.SplitN(strings.TrimPrefix(version, "v"), "-", 2)[0] + parts := strings.Split(core, ".") + if len(parts) != 3 { + return false, fmt.Errorf("unrecognized docker compose version %q", version) + } + var got [3]int + for i, part := range parts { + number, err := strconv.Atoi(part) + if err != nil { + return false, fmt.Errorf("unrecognized docker compose version %q", version) + } + got[i] = number + } + for i := range got { + if got[i] != minimum[i] { + return got[i] > minimum[i], nil + } + } + return true, nil +} + func writePlannerConfig(directory string, fixture seeder.Fixture, suite Suite) error { metrics := make(map[string]map[string]struct{}) for _, series := range fixture.Series { diff --git a/promql-compliance/runner/run_test.go b/promql-compliance/runner/run_test.go index a7bcc188..735cb34c 100644 --- a/promql-compliance/runner/run_test.go +++ b/promql-compliance/runner/run_test.go @@ -66,7 +66,7 @@ func TestGeneratedConfigsUseCurrentPlannerAndEngineSchema(t *testing.T) { Queries: []QueryCase{{Name: "up", Expr: "up"}}, } - if err := writeGeneratedConfigs(directory, fixture, suite); err != nil { + if err := writeGeneratedConfigs(directory, fixture, suite, false); err != nil { t.Fatalf("writeGeneratedConfigs: %v", err) } @@ -124,7 +124,7 @@ func TestGeneratedPlannerGroupsUseCompatibleQueryTiming(t *testing.T) { }, } - if err := writeGeneratedConfigs(directory, fixture, suite); err != nil { + if err := writeGeneratedConfigs(directory, fixture, suite, false); err != nil { t.Fatalf("writeGeneratedConfigs: %v", err) } @@ -176,7 +176,7 @@ func TestGeneratedConfigsWithFixedPlannerOutputSkipPlanner(t *testing.T) { PlannerOutputDir: "/fixed/planner/output", } - if err := writeGeneratedConfigs(directory, fixture, suite); err != nil { + if err := writeGeneratedConfigs(directory, fixture, suite, true); err != nil { t.Fatalf("writeGeneratedConfigs: %v", err) } @@ -195,7 +195,12 @@ func TestGeneratedConfigsWithFixedPlannerOutputSkipPlanner(t *testing.T) { } `yaml:"planner"` QueryEngine struct { DependsOn yaml.Node `yaml:"depends_on"` - Volumes []string `yaml:"volumes"` + Volumes []struct { + Type string `yaml:"type"` + Source string `yaml:"source"` + Target string `yaml:"target"` + ReadOnly bool `yaml:"read_only"` + } `yaml:"volumes"` } `yaml:"queryengine"` } `yaml:"services"` } @@ -216,9 +221,11 @@ func TestGeneratedConfigsWithFixedPlannerOutputSkipPlanner(t *testing.T) { if _, ok := dependencies["planner"]; ok { t.Fatal("queryengine still depends on the planner") } - wantVolume := "${" + fixedPlannerOutputDirEnv + ":?" + fixedPlannerOutputDirEnv + " is required}:" + plannerOutputMountPath + ":ro" - if got := override.Services.QueryEngine.Volumes; len(got) != 1 || got[0] != wantVolume { - t.Fatalf("queryengine volumes = %v, want [%s]", got, wantVolume) + volumes := override.Services.QueryEngine.Volumes + wantSource := "${" + fixedPlannerOutputDirEnv + ":?" + fixedPlannerOutputDirEnv + " is required}" + if len(volumes) != 1 || volumes[0].Type != "bind" || volumes[0].Source != wantSource || + volumes[0].Target != plannerOutputMountPath || !volumes[0].ReadOnly { + t.Fatalf("queryengine volumes = %+v, want one read-only bind of %s at %s", volumes, wantSource, plannerOutputMountPath) } engineContents, err := os.ReadFile(filepath.Join(directory, "engine_config.yaml")) @@ -239,3 +246,60 @@ func TestGeneratedConfigsWithFixedPlannerOutputSkipPlanner(t *testing.T) { t.Fatalf("engine streaming_config = %q, want %q", got, want) } } + +// With caller-owned targets there is no Compose project to swap the planner +// out of, so the planner input is still generated and validated. +func TestGeneratedConfigsWithoutComposeStillWritePlannerInput(t *testing.T) { + directory := t.TempDir() + fixture := seeder.Fixture{ + Name: "single-series", + Series: []seeder.FixtureSeries{{ + Metric: "up", + Labels: map[string]string{"job": "test"}, + Samples: []seeder.FixtureSample{{OffsetSeconds: 0, Value: 1}}, + }}, + } + suite := Suite{ + Name: "fixed-output", + Queries: []QueryCase{{Name: "up", Expr: "up"}}, + PlannerOutputDir: "/fixed/planner/output", + } + + if err := writeGeneratedConfigs(directory, fixture, suite, false); err != nil { + t.Fatalf("writeGeneratedConfigs: %v", err) + } + + if _, err := os.Stat(filepath.Join(directory, "controller-config.yaml")); err != nil { + t.Fatalf("planner config not generated without Compose: %v", err) + } + if _, err := os.Stat(filepath.Join(directory, fixedPlannerOutputComposeFile)); !os.IsNotExist(err) { + t.Fatalf("compose override written without Compose (stat err: %v)", err) + } +} + +func TestComposeVersionAtLeast(t *testing.T) { + minimum := [3]int{2, 24, 4} + for _, test := range []struct { + version string + want bool + }{ + {"2.24.4", true}, + {"v2.29.1", true}, + {"2.24.4-desktop.1", true}, + {"3.0.0", true}, + {"2.24.3", false}, + {"2.9.10", false}, + {"1.29.2", false}, + } { + got, err := composeVersionAtLeast(test.version, minimum) + if err != nil { + t.Fatalf("composeVersionAtLeast(%q): %v", test.version, err) + } + if got != test.want { + t.Fatalf("composeVersionAtLeast(%q) = %v, want %v", test.version, got, test.want) + } + } + if _, err := composeVersionAtLeast("not-a-version", minimum); err == nil { + t.Fatal("an unrecognized version must be an error") + } +}