test: add runnable PromQL differential compliance suite

.github/workflows/promql-compliance.yml

@@ -0,0 +1,37 @@
+name: PromQL compliance
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  differential:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: ASAPQuery-backend
+      - uses: actions/checkout@v4
+        with:
+          repository: ProjectASAP/ASAPCollector
+          path: ASAPCollector
+      - uses: actions/checkout@v4
+        with:
+          repository: ProjectASAP/asap_sketchlib
+          path: asap_sketchlib
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.25.8"
+      - name: Run every differential corpus
+        working-directory: ASAPQuery-backend/promql-compliance/runner
+        run: make run-all REPORT_DIR="$GITHUB_WORKSPACE/artifacts/reports" LOGS_DIR="$GITHUB_WORKSPACE/artifacts/logs"
+      - name: Upload reports and service logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: promql-compliance-evidence
+          path: artifacts
+          if-no-files-found: warn

control_plane/Dockerfile

@@ -15,6 +15,7 @@
 #     --build-context asap-gorilla-rust=/path/ASAPCollector/asap-gorilla-rust \
 #     -t asap/control-plane:dev .
 FROM rust:1.90-bookworm AS build
+ARG LOCAL_DEP_OVERRIDES=false
 WORKDIR /src
 
 # protoc for control_plane/build.rs (prost-build/tonic-build compile
@@ -27,11 +28,26 @@ RUN apt-get update && \
 # Same sibling-checkout layout as data_plane/Dockerfile so workspace path-deps
 # resolve during the build.
 COPY . ASAPQuery-backend
+# hadolint ignore=DL3022
 COPY --from=asap-precompute-rs . ASAPCollector/asap-precompute-rs
+# hadolint ignore=DL3022
 COPY --from=asap-sketchlib     . asap_sketchlib
+# hadolint ignore=DL3022
 COPY --from=asap-gorilla-rust  . ASAPCollector/asap-gorilla-rust
 
-RUN cd ASAPQuery-backend && cargo build --release --bin control_plane
+# See data_plane/Dockerfile: resolve private sibling dependencies from the
+# build contexts instead of requiring Docker to authenticate to GitHub.
+RUN if [ "$LOCAL_DEP_OVERRIDES" = true ]; then mkdir -p ASAPQuery-backend/.cargo && \
+    printf '%s\n' \
+      '[patch."https://github.com/ProjectASAP/ASAPCollector"]' \
+      'asap-precompute-rs = { path = "/src/ASAPCollector/asap-precompute-rs" }' \
+      '' \
+      '[patch."https://github.com/ProjectASAP/asap_sketchlib"]' \
+      'asap_sketchlib = { path = "/src/asap_sketchlib" }' \
+      >> ASAPQuery-backend/.cargo/config.toml; fi
+
+WORKDIR /src/ASAPQuery-backend
+RUN cargo build --release --bin control_plane --bin control_plane_quote_snapshot
 
 # Runtime image: binary + CA certs.
 FROM debian:bookworm-slim
@@ -40,6 +56,8 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/*
 COPY --from=build /src/ASAPQuery-backend/target/release/control_plane \
     /usr/local/bin/control_plane
+COPY --from=build /src/ASAPQuery-backend/target/release/control_plane_quote_snapshot \
+    /usr/local/bin/control_plane_quote_snapshot
 
 ENV RUST_LOG=info
 # OpAMP ws 4320, controller gRPC 4321, controller HTTP 8080.

control_plane/src/bin/control_plane_quote_snapshot.rs

@@ -0,0 +1,95 @@
+//! Complete a backend-local planning snapshot with deterministic unit-cost
+//! quotes for the PromQL compliance harness. Candidate identities and demand
+//! still come from the production planner and physical compiler.
+
+use control_plane::physical::{
+    compiler::{
+        BackendLocalPlanningInput, PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION,
+    },
+    workload_cost::{
+        enumerate_exact_and_materialized_candidates, manifest, WorkloadCostEvidence, WorkloadQuote,
+    },
+};
+use std::{collections::BTreeMap, path::Path};
+
+fn quote_snapshot(
+    mut snapshot: BackendLocalPlanningInput,
+) -> Result<BackendLocalPlanningInput, String> {
+    if snapshot.workload_cost_evidence.is_some() {
+        return Err("planning snapshot already contains workload_cost_evidence".into());
+    }
+    let observed_at_unix_ms = snapshot.environment.observed_at_unix_ms;
+    let valid_for_ms = snapshot.environment.max_evidence_age_ms;
+    let (request, environment) = snapshot
+        .clone()
+        .into_physical_compilation_request()
+        .map_err(|error| error.to_string())?;
+    let quotes = enumerate_exact_and_materialized_candidates(request)
+        .map_err(|error| error.to_string())?
+        .into_iter()
+        .filter_map(|candidate| {
+            let plan = PhysicalPlanCompiler
+                .compile_promql(candidate.clone(), environment.clone())
+                .ok()?;
+            let manifest = manifest(&plan, &candidate.queries).ok()?;
+            Some(WorkloadQuote {
+                unit_costs: manifest
+                    .components
+                    .keys()
+                    .map(|key| (key.clone(), 1.0))
+                    .collect::<BTreeMap<_, _>>(),
+                manifest,
+                executable: true,
+            })
+        })
+        .collect();
+    snapshot.workload_cost_evidence = Some(WorkloadCostEvidence {
+        backend_revision: BACKEND_REVISION.into(),
+        planner_revision: PLANNER_REVISION.into(),
+        data_snapshot_id: "promql-compliance".into(),
+        model_version: "promql-compliance-unit-costs".into(),
+        observed_at_unix_ms,
+        valid_for_ms,
+        quotes,
+    });
+    Ok(snapshot)
+}
+
+fn main() -> Result<(), String> {
+    let mut args = std::env::args_os().skip(1);
+    let input = args
+        .next()
+        .ok_or("usage: control_plane_quote_snapshot INPUT OUTPUT")?;
+    let output = args
+        .next()
+        .ok_or("usage: control_plane_quote_snapshot INPUT OUTPUT")?;
+    if args.next().is_some() {
+        return Err("usage: control_plane_quote_snapshot INPUT OUTPUT".into());
+    }
+    let snapshot = std::fs::read(&input)
+        .map_err(|error| format!("read {}: {error}", Path::new(&input).display()))?;
+    let snapshot =
+        serde_json::from_slice(&snapshot).map_err(|error| format!("decode snapshot: {error}"))?;
+    let quoted = quote_snapshot(snapshot)?;
+    std::fs::write(
+        &output,
+        serde_json::to_vec_pretty(&quoted).map_err(|error| error.to_string())?,
+    )
+    .map_err(|error| format!("write {}: {error}", Path::new(&output).display()))?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn adds_complete_candidate_quotes_to_a_snapshot() {
+        let fixture =
+            include_str!("../../../docs/examples/asapquery-compatibility-demo-snapshot.json");
+        let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(fixture).unwrap();
+        snapshot.workload_cost_evidence = None;
+        let quoted = quote_snapshot(snapshot).unwrap();
+        assert!(!quoted.workload_cost_evidence.unwrap().quotes.is_empty());
+    }
+}

data_plane/Dockerfile

@@ -19,6 +19,7 @@
 #     --build-context asap-gorilla-rust=/path/ASAPCollector/asap-gorilla-rust \
 #     -t asap/data-plane:dev .
 FROM rust:1.90-bookworm AS build
+ARG LOCAL_DEP_OVERRIDES=false
 WORKDIR /src
 
 # protoc for the backend's prost-build usages; git for any git-dep fetches.
@@ -31,11 +32,27 @@ RUN apt-get update && \
 #   /src/ASAPQuery-backend, /src/ASAPCollector/asap-precompute-rs,
 #   /src/asap_sketchlib, /src/ASAPCollector/asap-gorilla-rust
 COPY . ASAPQuery-backend
+# hadolint ignore=DL3022
 COPY --from=asap-precompute-rs . ASAPCollector/asap-precompute-rs
+# hadolint ignore=DL3022
 COPY --from=asap-sketchlib     . asap_sketchlib
+# hadolint ignore=DL3022
 COPY --from=asap-gorilla-rust  . ASAPCollector/asap-gorilla-rust
 
-RUN cd ASAPQuery-backend && cargo build --release --bin data_plane
+# Build from the supplied sibling checkouts. CI and local Compose builds do
+# not need credentials for ASAPCollector or Sketchlib while this dependency
+# remains private.
+RUN if [ "$LOCAL_DEP_OVERRIDES" = true ]; then mkdir -p ASAPQuery-backend/.cargo && \
+    printf '%s\n' \
+      '[patch."https://github.com/ProjectASAP/ASAPCollector"]' \
+      'asap-precompute-rs = { path = "/src/ASAPCollector/asap-precompute-rs" }' \
+      '' \
+      '[patch."https://github.com/ProjectASAP/asap_sketchlib"]' \
+      'asap_sketchlib = { path = "/src/asap_sketchlib" }' \
+      >> ASAPQuery-backend/.cargo/config.toml; fi
+
+WORKDIR /src/ASAPQuery-backend
+RUN cargo build --release --bin data_plane
 
 # Runtime image: binary + CA certs.
 FROM debian:bookworm-slim

data_plane/src/main.rs

@@ -430,11 +430,6 @@ fn validate_profile(args: &Args) -> Result<()> {
         )
         .into());
     }
-    if !args.forward_unsupported_queries {
-        return Err(
-            "--profile asapquery requires --forward-unsupported-queries for exact fallback".into(),
-        );
-    }
     let required_horizon = (args.precompute_allowed_lateness_ms.max(0) as u64)
         .saturating_add(args.remote_write_expected_retry_interval_ms);
     if args.remote_write_dedup_horizon_ms < required_horizon {
@@ -1503,6 +1498,19 @@ mod tests {
         .unwrap();
         assert!(validate_profile(&valid).is_ok());
 
+        let no_fallback = Args::try_parse_from([
+            "data_plane",
+            "--profile",
+            "asapquery",
+            "--physical-plan",
+            "plan.json",
+        ])
+        .unwrap();
+        assert!(
+            validate_profile(&no_fallback).is_ok(),
+            "a backend-local plan must be allowed to reject unsupported queries"
+        );
+
         let legacy = Args::try_parse_from([
             "data_plane",
             "--profile",

docs/design_docs/promql-compliance-grill.md

@@ -0,0 +1,79 @@
+# PromQL compliance design Q&A
+
+> **Question:** should v1 be a strict differential suite for a deliberately supported, locally answered subset of PromQL—with unsupported queries required to fail—or should it allow Prometheus fallback?
+>
+> **Recommendation:** require local answers and reject fallback in v1. Otherwise Prometheus can answer on the backend’s behalf, yielding a green comparison that proves neither backend ingestion nor backend query semantics.
+
+**Answer:** unsupported queries required to fail, yes
+
+> **Question:** should the suite compare approximate sketch-backed values to Prometheus within an explicit per-query tolerance, or start with only result shapes/data sizes where equality is exact?
+>
+> **Recommendation:** support explicit per-query tolerances from day one, defaulting to exact equality. The backend is designed to return approximate sketch results, so pretending all valid local answers are exact would either constrain the suite to an unrepresentative corpus or create noisy failures.
+
+**Answer:** yes
+
+> **Question:** should v1 use a finite, timestamp-pinned fixture and explicitly call the backend’s `/api/v1/precompute/drain` before querying?
+>
+> **Recommendation:** yes. It makes ingestion completion deterministic and avoids flaky “did the asynchronous precompute path catch up?” failures. Live-scrape/streaming behavior should be a separate suite later, because `drain` deliberately closes the backend’s Remote Write input.
+
+**Answer:** yes
+
+> **Question:** before the backend can accept Remote Write, it needs a configuration called a PhysicalPlan that says what metrics/queries it should accelerate. Should this new test suite: (1) give the backend a small, fixed test configuration; or (2) start the planner/control plane too and have it create that configuration?
+>
+> **Recommendation:** option 1 for this issue. It keeps failures clear: if the test fails, it is likely ingestion or query behavior in the backend, not plan generation.
+
+**Answer:** i wanna do option 2 actually. we can add option 1 later
+
+> **Question:** should “planner/control plane” mean the backend repository’s `control_plane` binary and its normal plan-publication protocol, rather than importing ASAPQuery’s older `asap-planner-rs` service?
+>
+> **Recommendation:** use the backend’s own `control_plane`. It tests the deployment users of `ASAPQuery-backend` actually run and avoids coupling this repository’s suite to a separate repository’s planner implementation.
+
+**Answer:** yes
+
+> **Question:** should we port the existing Go-based seeder/comparator and Docker Compose runner essentially intact, adapting only the backend service wiring, or rewrite it as Rust process tests?
+>
+> **Recommendation:** port the Go tooling and Compose workflow. It already provides deterministic Remote Write encoding, structured result comparison, fixtures, tolerances, reports, and lifecycle handling; a rewrite would add risk without improving the test’s signal.
+
+**Answer:** yes
+
+> **Question:** what should be the first required green corpus?
+>
+> **Recommendation:** begin with one small fixture and suite covering a selector, `rate`, `increase`, one aggregation, and both instant/range evaluations—plus one intentionally unsupported query that must fail locally. Add sparse/staleness, counter resets, and larger aggregation matrices after this vertical slice is stable.
+
+**Answer:** Let's just add all corpora. I think we may not get any green righ tnow
+
+> **Question:** when those corpora find differences, should the runner exit non-zero from day one?
+>
+> **Recommendation:** yes—never encode known failures as passes. Keep the full workflow manual/non-required initially, preserve its JSON reports as evidence, and only make a stable subset required in PR CI after the gaps are fixed.
+
+**Answer:** yes
+
+> **Question:** should every successful backend response be required to carry and pass a local-execution provenance check (for example, its existing ASAPQuery data-source marker), in addition to disabling fallback?
+>
+> **Recommendation:** yes. Disabling fallback catches most masking, but provenance makes the test’s claim explicit and will catch accidental routing changes that still return a successful response.
+
+**Answer:** no, it's fine. just disable fallback
+
+> **Question:** should the runner wait until the control plane has published and the backend reports an active Remote Write-ready plan before it sends any fixture data?
+>
+> **Recommendation:** yes. Then seed both targets, call the backend’s finite-input `drain`, and only then execute the fixed-time queries. This makes plan activation and ingestion completion explicit rather than timing-dependent.
+
+**Answer:** yes
+
+> **Question:** should the runner derive the control-plane workload/configuration from the same query-suite YAML it executes, rather than maintain a second hand-written plan configuration per corpus?
+>
+> **Recommendation:** yes. One source of truth prevents a test from querying expressions that the control plane was never asked to plan, and it makes adding a corpus a fixture-only change.
+
+**Answer:** yes
+
+> **Question:** on failure, should the runner retain the JSON report and collect service logs, while still tearing down containers by default?
+>
+> **Recommendation:** yes. CI should upload the report and logs as artifacts; local runs should offer `--keep-services` for interactive debugging. Default cleanup prevents stale volumes/ports from contaminating the next run.
+
+**Answer:** yes
+
+> **Question:** should `ASAPQuery-backend` own a copied/adapted version of the harness and fixtures, rather than invoke `ASAPQuery/promql-compliance` across repositories?
+>
+> **Recommendation:** own it in the backend repository. The backend needs different service wiring—its `control_plane`, PhysicalPlan lifecycle, and one HTTP listener—and an external cross-repo dependency would make local and CI runs less reproducible.
+
+**Answer:** yes