diff --git a/product/skills/ci-job-timing/SKILL.md b/product/skills/ci-job-timing/SKILL.md index c79b42611..4f21a0dae 100644 --- a/product/skills/ci-job-timing/SKILL.md +++ b/product/skills/ci-job-timing/SKILL.md @@ -21,7 +21,18 @@ fields, so the script reads the message text ("took N seconds") instead. ## Steps -1. Build the query and its link (`` is this skill's folder): +0. Context first: why does the job start late? What does it wait for, and which + prerequisite finished last? (`` is this skill's folder.) + `python3 /scripts/ci_job_timing.py context --job "" --pipelines 10 --job-ids ` + Get `` (comma-separated) from Datadog first: `search_datadog_ci_pipeline_events`, + `ci_level=job`, query `@ci.job.name:"" @ci.status:success`, `limit=10`; use the + `job_id` column. Scanning GitLab's pipeline list instead can miss the job on a busy project. + This reads GitLab (read-only) with the token in `GITLAB_TOKEN`; without it the + script prints an error and exits 2. Timing comes from GitLab, not Datadog, + because Datadog job records have no pipeline number. Needs absent from a + pipeline (optional needs) are skipped. Host and project default to + `https://gitlab.rivianvw.io` and `core/firmware` (`--gitlab-host`, `--project`). +1. Build the query and its link: `python3 /scripts/ci_job_timing.py queries --job "" --days 3` 2. Run the printed steps query with the Datadog `ddsql_run_query` tool (from `now-d`). 3. Save the TSV it returns to a file and render the table: diff --git a/product/skills/ci-job-timing/scripts/ci_job_timing.py b/product/skills/ci-job-timing/scripts/ci_job_timing.py index 33e6256f3..c942c224d 100644 --- a/product/skills/ci-job-timing/scripts/ci_job_timing.py +++ b/product/skills/ci-job-timing/scripts/ci_job_timing.py @@ -3,18 +3,28 @@ The Datadog connector is only reachable from the agent, so this script does the two parts that must be identical on every run: write the query text (`queries`, -`timeline`) and turn the query result into a table (`report`). +`timeline`) and turn the query result into a table (`report`). `context` reads +GitLab (read-only) to show what a job waits for and which prerequisite finished last. """ from __future__ import annotations import argparse import csv import io +import json +import os import re +import statistics import sys +import urllib.error +import urllib.request +from datetime import datetime from urllib.parse import quote DEFAULT_HOST = "rivianvw.datadoghq.com" +DEFAULT_GITLAB = "https://gitlab.rivianvw.io" +DEFAULT_PROJECT = "core/firmware" +TOKEN_ENV = "GITLAB_TOKEN" _JOB_RE = re.compile(r"^[A-Za-z0-9 ()._:/\-]+$") @@ -104,6 +114,179 @@ def report(job: str, days: int, rows: list[dict], url: str) -> str: ] return "\n".join(out) +NEEDS_QUERY = ( + "query($path: ID!, $iid: ID!) { project(fullPath: $path) { pipeline(iid: $iid) { " + "stages { nodes { groups { nodes { jobs { nodes { name needs { nodes { name } } } } } } } } } } }" +) + + +def parse_needs(graphql_json: dict, job: str) -> list[str]: + """Names the job needs, from the stages/groups/jobs GraphQL shape. Job absent -> error.""" + if graphql_json.get("errors"): + raise ValueError(f"GitLab GraphQL returned errors: {graphql_json['errors']}") + pipeline = ((graphql_json.get("data") or {}).get("project") or {}).get("pipeline") + if not pipeline: + raise ValueError("GraphQL reply has no project.pipeline (wrong project or pipeline number)") + for stage in (pipeline.get("stages") or {}).get("nodes") or []: + for group in (stage.get("groups") or {}).get("nodes") or []: + for j in (group.get("jobs") or {}).get("nodes") or []: + if j.get("name") == job: + names = [n["name"] for n in (j.get("needs") or {}).get("nodes") or []] + return list(dict.fromkeys(names)) + raise ValueError(f"job {job!r} not found in the pipeline's stages") + + +def _ts(value: str | None) -> datetime | None: + if not value: + return None + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def _latest(jobs: list[dict], name: str, key: str) -> dict | None: + """Latest attempt of one job name by `key` (retries share a name).""" + best, best_t = None, None + for j in jobs: + t = _ts(j.get(key)) if j.get("name") == name else None + if t is not None and (best_t is None or t > best_t): + best, best_t = j, t + return best + + +def summarize(job: str, needs: list[str], jobs_by_pipeline: dict) -> dict: + """Per pipeline: which needed job finished last and how long the job then waited. + + A needed job that is absent or unfinished in a pipeline is skipped (optional needs). + A pipeline where the job never started, or no need finished, yields no row. + """ + rows, last_counts, skipped = [], {}, {} + for pid, jobs in jobs_by_pipeline.items(): + me = _latest(jobs, job, "started_at") + if me is None: + continue + finishers = [] + for need in needs: + n = _latest(jobs, need, "finished_at") + if n is None: + skipped[need] = skipped.get(need, 0) + 1 + else: + finishers.append((_ts(n["finished_at"]), need)) + if not finishers: + continue + fin_t, fin_name = max(finishers) + gap = (_ts(me["started_at"]) - fin_t).total_seconds() + rows.append({ + "pipeline": pid, + "last_finisher": fin_name, + "gap_s": gap, + "queued_s": me.get("queued_duration"), + }) + last_counts[fin_name] = last_counts.get(fin_name, 0) + 1 + gaps = [r["gap_s"] for r in rows] + return { + "job": job, + "needs": list(needs), + "rows": rows, + "last_counts": dict(sorted(last_counts.items(), key=lambda kv: (-kv[1], kv[0]))), + "skipped": skipped, + "median_gap_s": statistics.median(gaps) if gaps else None, + "max_gap_s": max(gaps) if gaps else None, + } + + +def render_context(summary: dict, base_url: str = DEFAULT_GITLAB, project: str = DEFAULT_PROJECT) -> str: + if not summary["rows"]: + raise ValueError("no pipeline had both the job started and a needed job finished") + out = [f"## {summary['job']}: what it waits for", ""] + out.append("Needs: " + (", ".join(f"`{n}`" for n in summary["needs"]) or "none listed")) + n = len(summary["rows"]) + out += ["", f"Last needed job to finish, over {n} pipeline(s):", "", + "| Needed job | Finished last | Share |", "|---|---|---|"] + for name, cnt in summary["last_counts"].items(): + out.append(f"| {name} | {cnt} | {cnt / n:.0%} |") + out += ["", "Gap = last needed job finished -> this job started.", + f"- Median gap: {summary['median_gap_s']:.0f} s", + f"- Maximum gap: {summary['max_gap_s']:.0f} s", "", + "| Pipeline | Last finisher | Gap (s) | Queued (s) |", "|---|---|---|---|"] + for r in summary["rows"]: + url = f"{base_url}/{project}/-/pipelines/{r['pipeline']}" + q = "" if r["queued_s"] is None else f"{float(r['queued_s']):.0f}" + out.append(f"| [{r['pipeline']}]({url}) | {r['last_finisher']} | {r['gap_s']:.0f} | {q} |") + out += ["", "Needs listed but absent (or unfinished) in a pipeline are skipped; they may be optional needs."] + if summary["skipped"]: + out.append("Skipped: " + ", ".join(f"{k} ({v}x)" for k, v in sorted(summary["skipped"].items()))) + return "\n".join(out) + + + + +def http_get_json(url: str, token: str, body: dict | None = None): + """GET (or POST for GraphQL, which is a read-only query). Errors never echo the token.""" + headers = {"PRIVATE-TOKEN": token, "Content-Type": "application/json"} + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(url, data=data, headers=headers) + try: + with urllib.request.urlopen(req, timeout=60) as resp: + return json.load(resp) + except urllib.error.HTTPError as e: + raise OSError(f"GitLab request failed: HTTP {e.code} for {url}") from None + except urllib.error.URLError as e: + raise OSError(f"GitLab request failed: {e.reason} for {url}") from None + + +def _paged(url: str, token: str, get=http_get_json): + page = 1 + while True: + batch = get(f"{url}{'&' if '?' in url else '?'}per_page=100&page={page}", token) + yield from batch + if len(batch) < 100: + return + page += 1 + + +def _pipelines_from_job_ids(api, job, job_ids, token, get): + """Resolve known job numbers to their pipelines. A job number whose name differs is an error.""" + found = {} + for jid in job_ids: + j = get(f"{api}/jobs/{int(jid)}", token) + if j.get("name") != job: + raise ValueError(f"job {int(jid)} is named {j.get('name')!r}, not {job!r}") + pl = j["pipeline"] + found[pl["id"]] = pl["iid"] + return found + + +def fetch_context_inputs(host, project, job, n, token, scan_limit=200, get=http_get_json, job_ids=None): + """Return (needs, jobs_by_pipeline) for pipelines containing `job`. + + With job_ids, those jobs' pipelines are used directly. Without, the newest finished pipelines + are scanned, which can miss the job on a busy project.""" + api = f"{host}/api/v4/projects/{quote(project, safe='')}" + if job_ids: + pipes = _pipelines_from_job_ids(api, job, job_ids, token, get) + jobs_by_pipeline = {pid: list(_paged(f"{api}/pipelines/{pid}/jobs", token, get)) for pid in pipes} + newest_iid = max(pipes.values()) + reply = get(f"{host}/api/graphql", token, + {"query": NEEDS_QUERY, "variables": {"path": project, "iid": str(newest_iid)}}) + return parse_needs(reply, job), jobs_by_pipeline + jobs_by_pipeline, newest_iid, scanned = {}, None, 0 + for p in _paged(f"{api}/pipelines?scope=finished&order_by=id&sort=desc", token, get): + scanned += 1 + if scanned > scan_limit: + break + jobs = list(_paged(f"{api}/pipelines/{p['id']}/jobs", token, get)) + if not any(j.get("name") == job for j in jobs): + continue + jobs_by_pipeline[p["id"]] = jobs + if newest_iid is None: + newest_iid = p["iid"] + if len(jobs_by_pipeline) >= n: + break + if not jobs_by_pipeline: + raise ValueError(f"no job named {job!r} in the {scanned} newest finished pipelines; pass --job-ids with recent job numbers from Datadog") + reply = get(f"{host}/api/graphql", token, + {"query": NEEDS_QUERY, "variables": {"path": project, "iid": str(newest_iid)}}) + return parse_needs(reply, job), jobs_by_pipeline + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description=__doc__) @@ -117,7 +300,30 @@ def main(argv: list[str] | None = None) -> int: p.add_argument("--job-id", type=int, required=True) if name == "report": p.add_argument("--tsv", required=True, help="file holding the steps query result") + pc = sub.add_parser("context", help="what the job waits for and which need finished last (GitLab, read-only)") + pc.add_argument("--job", required=True) + pc.add_argument("--pipelines", type=int, default=10) + pc.add_argument("--gitlab-host", default=DEFAULT_GITLAB) + pc.add_argument("--project", default=DEFAULT_PROJECT) + pc.add_argument("--job-ids", default="", help="comma-separated recent job numbers for this job (from Datadog); preferred") a = ap.parse_args(argv) + if a.cmd == "context": + token = os.environ.get(TOKEN_ENV) + if not token: + print(f"error: {TOKEN_ENV} is not set; export a GitLab read_api token", file=sys.stderr) + return 2 + try: + check_job(a.job) + if a.pipelines < 1: + raise ValueError("--pipelines must be at least 1") + host = a.gitlab_host.rstrip("/") + ids = [int(x) for x in a.job_ids.split(",") if x.strip()] + needs, by_pipeline = fetch_context_inputs(host, a.project, a.job, a.pipelines, token, job_ids=ids) + print(render_context(summarize(a.job, needs, by_pipeline), host, a.project)) + except (ValueError, OSError) as e: + print(f"error: {e}", file=sys.stderr) + return 2 + return 0 try: if a.cmd == "queries": sql = steps_sql(a.job, a.days) diff --git a/product/skills/ci-job-timing/tests/test_ci_job_timing.py b/product/skills/ci-job-timing/tests/test_ci_job_timing.py index 348fd368d..fac3e6aca 100644 --- a/product/skills/ci-job-timing/tests/test_ci_job_timing.py +++ b/product/skills/ci-job-timing/tests/test_ci_job_timing.py @@ -1,7 +1,11 @@ from __future__ import annotations +import contextlib +import io +import os import sys import unittest +from unittest import mock from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) @@ -53,5 +57,115 @@ def test_timeline_uses_job_id(self): self.assertIn("[job_id=253261575]", c.timeline_sql("Build LAGVM (dev)", 253261575, 3)) +def _gql(jobs): + return {"data": {"project": {"pipeline": {"stages": {"nodes": [ + {"groups": {"nodes": [{"jobs": {"nodes": [ + {"name": n, "needs": {"nodes": [{"name": x} for x in needs]}}]}} for n, needs in jobs]}}]}}}}} + + +def _j(name, start, end, queued=5): + return {"name": name, "status": "success", "started_at": start, "finished_at": end, "queued_duration": queued} + + +class Needs(unittest.TestCase): + def test_parse_needs_reads_stages_groups_jobs(self): + g = _gql([("a", []), ("pkg", ["a", "b", "a"])]) + self.assertEqual(c.parse_needs(g, "pkg"), ["a", "b"]) + + def test_missing_job_is_an_error(self): + with self.assertRaises(ValueError): + c.parse_needs(_gql([("a", [])]), "zzz") + + +class Summarize(unittest.TestCase): + P = { + 1: [_j("a", "2026-01-01T00:00:00Z", "2026-01-01T00:10:00Z"), + _j("b", "2026-01-01T00:00:00Z", "2026-01-01T00:20:00Z"), + _j("pkg", "2026-01-01T00:21:00Z", "2026-01-01T00:30:00Z", 7)], + 2: [_j("a", "2026-01-02T00:00:00Z", "2026-01-02T00:30:00Z"), + _j("b", "2026-01-02T00:00:00Z", "2026-01-02T00:20:00Z"), + _j("pkg", "2026-01-02T00:30:10Z", "2026-01-02T00:40:00Z")], + } + + def test_last_finisher_and_gap(self): + s = c.summarize("pkg", ["a", "b"], self.P) + self.assertEqual([r["last_finisher"] for r in s["rows"]], ["b", "a"]) + self.assertEqual([r["gap_s"] for r in s["rows"]], [60.0, 10.0]) + self.assertEqual(s["last_counts"], {"a": 1, "b": 1}) + self.assertEqual(s["median_gap_s"], 35.0) + self.assertEqual(s["max_gap_s"], 60.0) + + def test_absent_need_is_skipped_not_an_error(self): + s = c.summarize("pkg", ["a", "b", "optional"], self.P) + self.assertEqual(s["skipped"], {"optional": 2}) + self.assertEqual(len(s["rows"]), 2) + self.assertIn("optional needs", c.render_context(s)) + + +class ContextCommand(unittest.TestCase): + def _run(self, argv, env): + out, err = io.StringIO(), io.StringIO() + with mock.patch.dict(os.environ, env, clear=False), \ + contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + rc = c.main(argv) + return rc, out.getvalue(), err.getvalue() + + def test_missing_token_exits_2_with_no_report(self): + env = dict(os.environ) + env.pop("GITLAB_TOKEN", None) + out, err = io.StringIO(), io.StringIO() + with mock.patch.dict(os.environ, env, clear=True), \ + contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + rc = c.main(["context", "--job", "pkg", "--pipelines", "2"]) + self.assertEqual(rc, 2) + self.assertEqual(out.getvalue(), "") + self.assertIn("GITLAB_TOKEN", err.getvalue()) + + def test_job_ids_resolve_pipelines_without_scanning(self): + calls = [] + + def fake(url, token, body=None): + calls.append(url) + if url.endswith("/graphql"): + return _gql([("pkg", ["a", "b"])]) + if "/jobs/" in url and "/pipelines/" not in url: + jid = int(url.rsplit("/", 1)[1]) + return {"name": "pkg", "pipeline": {"id": {101: 1, 102: 2}[jid], "iid": {101: 11, 102: 12}[jid]}} + if "/pipelines/" in url: + return Summarize.P[int(url.split("/pipelines/")[1].split("/")[0])] + raise AssertionError(f"unexpected scan call: {url}") + + needs, by_p = c.fetch_context_inputs("https://h", "core/firmware", "pkg", 2, "tok", get=fake, job_ids=[101, 102]) + self.assertEqual(sorted(by_p), [1, 2]) + self.assertEqual(needs, ["a", "b"]) + self.assertFalse(any("order_by" in u for u in calls)) + + def test_job_id_with_a_different_name_is_an_error(self): + def fake(url, token, body=None): + return {"name": "other", "pipeline": {"id": 1, "iid": 11}} + + with self.assertRaises(ValueError): + c.fetch_context_inputs("https://h", "core/firmware", "pkg", 1, "tok", get=fake, job_ids=[101]) + + def test_token_value_never_appears_in_output(self): + secret = "glpat-SECRET-VALUE-123" + + def fake(url, token, body=None): + self.assertEqual(token, secret) + if url.endswith("/graphql"): + return _gql([("pkg", ["a", "b"])]) + if "/pipelines/" in url: + return Summarize.P[int(url.split("/pipelines/")[1].split("/")[0])] + return [{"id": 1, "iid": 11}, {"id": 2, "iid": 12}] + + needs, by_p = c.fetch_context_inputs("https://h", "core/firmware", "pkg", 2, secret, get=fake) + text = c.render_context(c.summarize("pkg", needs, by_p)) + self.assertNotIn(secret, text) + with mock.patch.object(c, "fetch_context_inputs", side_effect=OSError("boom")): + rc, out, err = self._run(["context", "--job", "pkg"], {"GITLAB_TOKEN": secret}) + self.assertEqual(rc, 2) + self.assertNotIn(secret, out + err) + + if __name__ == "__main__": unittest.main() diff --git a/product/skills/land-stack/SKILL.md b/product/skills/land-stack/SKILL.md index 938dcb440..4a55352ad 100644 --- a/product/skills/land-stack/SKILL.md +++ b/product/skills/land-stack/SKILL.md @@ -100,6 +100,21 @@ guard before any write (label, thread-resolve, queue, merge). Mergify reports `Merge queue is ready` until a `@mergifyio queue` comment (or the dashboard) actually queues the PR. + Queued PRs can still be rejected later, and the queue says why. Read its + own comment instead of guessing: + + ```sh + python3 scripts/verify_stack.py --repo --queue-feedback ... + ``` + + Exit 0 means each PR is merged or in the queue. Exit 1 names each rejected + PR with its kind (`timeout`, `conflict`, `checks-failed`, `other`), the + reason Mergify gave, the checks that failed or were still running, and the + next step. Repair from that, not from the PR's own CI: a `conflict` needs a + rebase, a `checks-failed` names the check to fix, and a `timeout` is not a + defect in the PR, so requeue it and, if it repeats, compare the queue's + `checks_timeout` with how long the named check normally runs. + 4. **Never batch merges without checking each result.** A merge command can look silent on both success and some failure paths; a silent-looking run is not proof of a merge. diff --git a/product/skills/land-stack/scripts/verify_stack.py b/product/skills/land-stack/scripts/verify_stack.py index c20cf48d8..5040ee2a4 100644 --- a/product/skills/land-stack/scripts/verify_stack.py +++ b/product/skills/land-stack/scripts/verify_stack.py @@ -23,6 +23,16 @@ Queue-state exit 0: every PR merged or queued. Exit 1: a PR is not queued. Exit 2: a PR's queue state could not be read. +After a queue write, `--queue-feedback` reads Mergify's own status comment on +each PR and says in-queue, rejected, merged, or unchecked. A rejection names +its kind (timeout, conflict, checks-failed, other), the reason Mergify gave, +the checks that were failing or still running, and what to do next. + + verify_stack.py --repo owner/name --queue-feedback 7006 7007 + verify_stack.py --feedback-json comments.json --queue-feedback 7006 + +Queue-feedback exit 0: every PR merged, in the queue, or waiting to enter it. Exit 1: a PR was +rejected. Exit 2: a PR's queue comment could not be read. Never uses `gh pr list --head ` -- discovering by branch name is the unsafe path the skill exists to prevent. """ @@ -30,6 +40,7 @@ import argparse import json +import re import subprocess import sys from typing import Callable @@ -151,6 +162,135 @@ def report_queue_state(keys: list[str], entries: dict) -> int: return 2 return 0 +STATUS_MARK = "# Merge Queue Status" +PAYLOAD_RE = re.compile(r"Mergify Payload -\*-\s*(\{.*\})\s*-\*- Mergify Payload End", re.S) +HTML_COMMENT_RE = re.compile(r"", re.S) +REASON_RE = re.compile(r"## Reason\s*\n(.*?)(?=\n## |\Z)", re.S) +FAILING_CHECK_RE = re.compile(r"^- \u274c \[`([^`]+)`\]", re.M) +STILL_RUNNING_RE = re.compile(r"These checks were still running:\s*\n((?:- `[^`]+`\s*\n?)+)") + +FEEDBACK_ACTIONS = { + "timeout": "A check was still running when the queue's checks_timeout ran out. Requeue with `@mergifyio queue`; if it keeps timing out, the timeout in .mergify.yml is shorter than the check's normal run time.", + "conflict": "GitHub refused the merge. Rebase onto the trunk and push; Mergify requeues on the update.", + "checks-failed": "Read the failing check named here, fix the PR, and push; Mergify requeues on the update.", + "ahead-failed": "An earlier batch failed and this PR is being checked again. No action unless it is rejected next.", + "other": "Read the reason, then fix the PR or requeue with `@mergifyio queue`.", +} + + +def _unchecked(reason: str) -> dict: + return {"state": "unchecked", "kind": "", "reason": reason, "action": "", "failing_checks": [], + "still_running": [], "rule": "", "eta": ""} + + +def _feedback_kind(reason: str) -> str: + low = reason.lower() + if "ahead in the queue failed" in low: + return "ahead-failed" + if "merge conflicts" in low: + return "conflict" + if "checks timeout" in low: + return "timeout" + if "failing checks" in low: + return "checks-failed" + return "other" + + +def queue_feedback(comments: list[dict], head_at: str | None = None) -> dict: + status = [c for c in comments if STATUS_MARK in (c.get("body") or "")] + if not status: + return _unchecked(f"no {STATUS_MARK[2:]} comment from Mergify") + latest = max(status, key=lambda c: c.get("id", 0)) + body = latest.get("body") or "" + found = PAYLOAD_RE.search(body) + if not found: + return _unchecked("Merge Queue Status comment has no readable Mergify payload") + try: + payload = json.loads(found.group(1)) + except json.JSONDecodeError as exc: + return _unchecked(f"Mergify payload is not valid JSON: {exc}") + state = payload.get("state") + text = HTML_COMMENT_RE.sub("", body) + reason_match = REASON_RE.search(text) + section = reason_match.group(1).strip() if reason_match else "" + paragraphs = [para.strip() for para in section.split("\n\n") if para.strip()] + informative = [para for para in paragraphs if not para.endswith("has been dequeued")] + reason = (informative or paragraphs or [""])[0] + running = STILL_RUNNING_RE.search(text) + still_running = re.findall(r"`([^`]+)`", running.group(1)) if running else [] + result = { + "kind": _feedback_kind(section) if section else "", + "reason": reason, + "failing_checks": FAILING_CHECK_RE.findall(text), + "still_running": still_running, + "rule": payload.get("queue_rule_name") or "", + "eta": payload.get("estimated_time_of_merge") or "", + } + if state == "dequeued": + if head_at and (latest.get("updated_at") or "") and head_at > latest["updated_at"]: + result.update(state="waiting", kind="", reason="the head commit is newer than the queue's last status; Mergify will check the new head", action="") + return result + kind = result["kind"] or "other" + result.update(state="rejected", kind=kind, action=FEEDBACK_ACTIONS[kind]) + return result + if state == "checking": + result.update(state="in-queue", action="") + return result + if state == "waiting": + result.update(state="waiting", reason="waiting for queue conditions (for example the new head's checks)", action="") + return result + return _unchecked(f"unrecognised Mergify queue state: {state!r}") + + +def fetch_queue_comments(repo: str | None, number: int) -> dict: + prefix = f"repos/{repo}" if repo else "repos/{owner}/{repo}" + pr = json.loads(subprocess.run(["gh", "api", f"{prefix}/pulls/{number}"], + capture_output=True, text=True, check=True).stdout) + pages = subprocess.run( + ["gh", "api", f"{prefix}/issues/{number}/comments", "--paginate", "--jq", + '[.[]|select(.user.login=="mergify[bot]")|{id,created_at,updated_at,body}]'], + capture_output=True, text=True, check=True).stdout + comments = [row for line in pages.splitlines() if line.strip() for row in json.loads(line)] + head = subprocess.run(["gh", "api", f"{prefix}/commits/{pr['head']['sha']}", "--jq", ".commit.committer.date"], + capture_output=True, text=True, check=True).stdout.strip() + return {"merged": bool(pr.get("merged")), "comments": comments, "head_at": head} + + +def report_queue_feedback(keys: list[str], entries: dict) -> int: + worst = 0 + for key in keys: + entry = entries.get(key) + if entry is None: + fb, merged = _unchecked("no payload for this PR"), False + else: + merged = bool(entry.get("merged")) if isinstance(entry, dict) else False + comments = entry.get("comments", []) if isinstance(entry, dict) else entry + head_at = entry.get("head_at") if isinstance(entry, dict) else None + fb = queue_feedback(comments, head_at=head_at) + if merged: + print(f"{'merged':<10} #{key}") + continue + if fb["state"] == "in-queue": + extra = f" (rule {fb['rule']}, ETA {fb['eta']})" if fb["rule"] else "" + note = f" -- {fb['reason']}" if fb["kind"] == "ahead-failed" else "" + print(f"{'in-queue':<10} #{key}{extra}{note}") + elif fb["state"] == "waiting": + print(f"{'waiting':<10} #{key} {fb['reason']}") + elif fb["state"] == "rejected": + print(f"{'rejected':<10} #{key} {fb['kind']}: {fb['reason']}") + if fb["failing_checks"]: + print(f"{'':<10} failing checks: {', '.join(fb['failing_checks'])}") + if fb["still_running"]: + print(f"{'':<10} still running at the timeout: {', '.join(fb['still_running'])}") + print(f"{'':<10} next: {fb['action']}") + worst = max(worst, 1) + else: + print(f"{'unchecked':<10} #{key} {fb['reason']}") + if worst == 0: + worst = 2 + return worst + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("numbers", nargs="*", help="PR numbers, bottom of stack first") @@ -163,8 +303,26 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--json", action="store_true", help="machine-readable output") ap.add_argument("--queue-state", action="store_true", help="after a queue write: is each PR merged or queued?") ap.add_argument("--queue-json", help="replay saved pull + check-run payloads keyed by PR for --queue-state") + ap.add_argument("--queue-feedback", action="store_true", help="read Mergify's status comment: in-queue, rejected (with reason), merged") + ap.add_argument("--feedback-json", help="replay saved Mergify comments keyed by PR for --queue-feedback") args = ap.parse_args(argv) + if args.queue_feedback: + keys = args.numbers + if not keys: + ap.error("give PR numbers for --queue-feedback") + if args.feedback_json: + with open(args.feedback_json, encoding="utf-8") as fh: + entries = json.load(fh) + else: + entries = {} + for key in keys: + try: + entries[key] = fetch_queue_comments(args.repo, int(key)) + except (subprocess.CalledProcessError, ValueError, KeyError, json.JSONDecodeError) as exc: + print(f"unchecked #{key} could not read queue comments: {exc}", file=sys.stderr) + return report_queue_feedback(keys, entries) + if args.queue_state: keys = args.numbers if not keys: diff --git a/product/skills/land-stack/tests/fixtures/mergify_queue_comments_2026-10-12.json b/product/skills/land-stack/tests/fixtures/mergify_queue_comments_2026-10-12.json new file mode 100644 index 000000000..f32680498 --- /dev/null +++ b/product/skills/land-stack/tests/fixtures/mergify_queue_comments_2026-10-12.json @@ -0,0 +1,106 @@ +{ + "catstack#1333": [ + { + "body": "Queued \u2014 the merge queue status continues in [this comment \u2193](https://github.com/EdbertChan/catstack/pull/1333#issuecomment-6126434423).", + "created_at": "2026-10-11T19:10:58Z", + "id": 6112652141, + "updated_at": "2026-10-12T14:36:29Z" + }, + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 14:36 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1333#issuecomment-6126433073)\n- \u274c **Checks timed out** \u00b7 on draft #1341\n- \ud83d\udeab **Left the queue** \u2014 `2026-10-12 15:37 UTC` \u00b7 at `9787af8f07826c77ba0072030a241cf253e6a08c`\n\nThis pull request spent **1 hour 1 minute 3 seconds** in the queue, including **1 hour 31 seconds** running CI.\n\n## Reason\n\nThe merge conditions cannot be satisfied: the checks did not pass within the checks timeout of 1 hour\n\nThese checks were still running:\n- `test`\n\nFailing checks:\n- \ud83d\udfe0 [`test`](https://github.com/EdbertChan/catstack/actions/runs/38264185548/job/114862046006) ([job log](https://github.com/EdbertChan/catstack/actions/runs/38264185548/job/114862046006))\n\n## Hint\n\nYou may have to fix your CI before adding the pull request to the queue again.\n\nIf you update this pull request, to fix the CI, it will automatically be requeued once the queue conditions match again.\nIf you think this was a flaky issue instead, you can requeue the pull request, without updating it, by posting a `@mergifyio queue` comment.\n\n
\nWaiting for\n\n- [ ] `check-success = test`\n\n
\n
\nAll conditions\n\n- [ ] `check-success = test`\n- [X] `check-success = lint`\n- [X] `check-success = validate`\n\n
\n\nRequeued \u2014 the merge queue status continues in [this comment \u2193](https://github.com/EdbertChan/catstack/pull/1333#issuecomment-6127751086).", + "created_at": "2026-10-12T14:36:28Z", + "id": 6126434423, + "updated_at": "2026-10-12T15:52:17Z" + }, + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 15:52 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1333#issuecomment-6127749387)\n- \u274c **Checks timed out** \u2014 `2026-10-12 18:22 UTC` \u00b7 on draft #1351\n- \u2702\ufe0f Bisecting to identify the failing PR (round 1/1)\n- \ud83d\udfe0 **Checks running** \u00b7 on draft #1353 \u00b7 [dashboard](https://dashboard.mergify.com/orgs/EdbertChan/repos/catstack/queues/status?branch=main&pull=1333)\n- \u23f3 Merge \u00b7 ETA: `2026-10-12 20:23 UTC` \ud83d\ude80\n\n
\nWaiting for\n\n- [ ] `check-success = test`\n\n
\n
\nAll merge conditions\n\n- [ ] `check-success = test`\n- [X] `check-success = lint`\n- [X] `check-success = validate`\n\n
\n\n
\nRequired conditions to stay in the queue\n\n- [X] `-closed` [\ud83d\udccc queue requirement]\n- [X] `-conflict` [\ud83d\udccc queue requirement]\n- [X] `-draft` [\ud83d\udccc queue requirement]\n- [X] any of [\ud83d\udccc queue -> configuration change requirements]:\n - [X] `-mergify-configuration-changed`\n - [ ] `check-success = @mergify/Configuration changed`\n - [ ] `check-success = @mergify/Configuration has been deleted`\n- [X] any of [\ud83d\udd00 queue conditions]:\n - [X] all of [\ud83d\udccc queue conditions of queue rule `admin-bypass`]:\n - [X] `-draft`\n - [X] `base=main`\n - [X] `label=admin-bypass`\n\n
", + "created_at": "2026-10-12T15:52:16Z", + "id": 6127751086, + "updated_at": "2026-10-12T19:24:32Z" + } + ], + "catstack#1332": [ + { + "body": "Queued \u2014 the merge queue status continues in [this comment \u2193](https://github.com/EdbertChan/catstack/pull/1332#issuecomment-6112771226).", + "created_at": "2026-10-11T19:10:53Z", + "id": 6112651269, + "updated_at": "2026-10-11T19:22:07Z" + }, + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-11 19:22 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1332#issuecomment-6112764720)\n- \u274c **Checks timed out** \u00b7 on draft #1339\n- \ud83d\udeab **Left the queue** \u2014 `2026-10-11 20:23 UTC` \u00b7 at `0e6a50c53c329843afa3886cf73cbe244fc3dfa3`\n\nThis pull request spent **1 hour 1 minute 19 seconds** in the queue, including **1 hour 20 seconds** running CI.\n\n## Reason\n\nThe merge conditions cannot be satisfied: the checks did not pass within the checks timeout of 1 hour\n\nThese checks were still running:\n- `test`\n\nFailing checks:\n- \ud83d\udfe0 [`test`](https://github.com/EdbertChan/catstack/actions/runs/38167777445/job/114553826149) ([job log](https://github.com/EdbertChan/catstack/actions/runs/38167777445/job/114553826149))\n\n## Hint\n\nYou may have to fix your CI before adding the pull request to the queue again.\n\nIf you update this pull request, to fix the CI, it will automatically be requeued once the queue conditions match again.\nIf you think this was a flaky issue instead, you can requeue the pull request, without updating it, by posting a `@mergifyio queue` comment.\n\n
\nWaiting for\n\n- [ ] `check-success = test`\n\n
\n
\nAll conditions\n\n- [ ] `check-success = test`\n- [X] `check-success = lint`\n- [X] `check-success = validate`\n\n
\n\nRequeued \u2014 the merge queue status continues in [this comment \u2193](https://github.com/EdbertChan/catstack/pull/1332#issuecomment-6126529630).", + "created_at": "2026-10-11T19:22:06Z", + "id": 6112771226, + "updated_at": "2026-10-12T14:42:12Z" + }, + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 14:42 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1332#issuecomment-6126526873)\n- \u274c **Checks timed out** \u2014 `2026-10-12 15:37 UTC` \u00b7 on draft #1345\n- \u2702\ufe0f Bisecting to identify the failing PR (round 1/1)\n- \u274c **Checks timed out** \u00b7 on draft #1348 \u00b7 identified as the failing PR\n- \ud83d\udeab **Left the queue** \u2014 `2026-10-12 17:39 UTC` \u00b7 at `0e6a50c53c329843afa3886cf73cbe244fc3dfa3`\n\nThis pull request spent **2 hours 57 minutes 4 seconds** in the queue, including **2 hours 54 seconds** running CI.\n\n## Reason\n\nThe merge conditions cannot be satisfied: the checks did not pass within the checks timeout of 1 hour\n\nThese checks were still running:\n- `test`\n\nFailing checks:\n- \ud83d\udfe0 [`test`](https://github.com/EdbertChan/catstack/actions/runs/38278827982/job/114910815884) ([job log](https://github.com/EdbertChan/catstack/actions/runs/38278827982/job/114910815884))\n\n## Hint\n\nYou may have to fix your CI before adding the pull request to the queue again.\n\nIf you update this pull request, to fix the CI, it will automatically be requeued once the queue conditions match again.\nIf you think this was a flaky issue instead, you can requeue the pull request, without updating it, by posting a `@mergifyio queue` comment.\n\n
\nWaiting for\n\n- [ ] `check-success = test`\n\n
\n
\nAll conditions\n\n- [ ] `check-success = test`\n- [X] `check-success = lint`\n- [X] `check-success = validate`\n\n
\n\nRequeued \u2014 the merge queue status continues in [this comment \u2193](https://github.com/EdbertChan/catstack/pull/1332#issuecomment-6130960344).", + "created_at": "2026-10-12T14:42:11Z", + "id": 6126529630, + "updated_at": "2026-10-12T19:16:26Z" + }, + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 19:16 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1332#issuecomment-6130958496)\n- \ud83d\udfe0 **Checks running** \u00b7 on draft #1352\n- \ud83d\udd04 **Checks restarted** \u00b7 will be requeued automatically\n\n## Reason\n\nPull requests batch (#1333, #1336) which was ahead in the queue failed", + "created_at": "2026-10-12T19:16:25Z", + "id": 6130960344, + "updated_at": "2026-10-12T19:23:13Z" + } + ], + "catstack#1309": [ + { + "body": "Queued \u2014 the merge queue status continues in [this comment \u2193](https://github.com/EdbertChan/catstack/pull/1309#issuecomment-6126707738).", + "created_at": "2026-10-11T19:11:56Z", + "id": 6112662978, + "updated_at": "2026-10-12T14:52:33Z" + }, + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 14:52 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1309#issuecomment-6126705187)\n- \u274c **Checks timed out** \u2014 `2026-10-12 15:37 UTC` \u00b7 on draft #1345\n- \u2702\ufe0f Bisecting to identify the failing PR (round 1/1)\n- \u274c **Checks failed** \u00b7 on draft #1350 \u00b7 identified as the failing PR\n- \ud83d\udeab **Left the queue** \u2014 `2026-10-12 18:22 UTC` \u00b7 at `737c66df5221d80ff48c8308067840366155180e`\n\nThis pull request spent **3 hours 29 minutes 55 seconds** in the queue, including **1 hour 43 minutes** running CI.\n\n## Reason\n\nPull request #1309 has been dequeued\n\nGitHub refused to merge the pull request. Pull Request has merge conflicts. This is usually enforced by a [branch protection](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-protected-branches/about-protected-branches) or [ruleset rule](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-rulesets/about-rulesets).\n\n## Hint\n\nYou should look at the reason for the failure and decide if the pull request needs to be fixed or if you want to requeue it.\nIf you do update this pull request, it will automatically be requeued once the queue conditions match again.\nIf you think this was a flaky issue instead, you can requeue the pull request, without updating it, by posting a `@mergifyio queue` comment.\n\n
\nRequired conditions to merge\n\n- [X] `check-success = lint`\n- [X] `check-success = test`\n- [X] `check-success = validate`\n\n
\n\nTick the box to put this pull request back in the merge queue (same as `@mergifyio queue`).\n\n- [ ] Requeue this pull request ", + "created_at": "2026-10-12T14:52:32Z", + "id": 6126707738, + "updated_at": "2026-10-12T18:22:27Z" + } + ], + "catstack#1354": [ + { + "body": "Queued \u2014 the merge queue status continues in [this comment \u2193](https://github.com/EdbertChan/catstack/pull/1354#issuecomment-6131697266).", + "created_at": "2026-10-12T20:00:48Z", + "id": 6131654915, + "updated_at": "2026-10-12T20:03:38Z" + }, + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 20:03 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1354#issuecomment-6131696378)\n- \ud83d\udfe0 **Preparing checks**\n- \u23f3 Merge \u00b7 ETA: `2026-10-12 21:23 UTC` \ud83d\ude80", + "created_at": "2026-10-12T20:03:38Z", + "id": 6131697266, + "updated_at": "2026-10-12T20:03:42Z" + } + ], + "Invoker#15193": [ + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 16:58 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by rule `autoqueue admin-bypass PRs to master`\n- \u274c **Checks failed** \u00b7 on draft #15210\n- \ud83d\udeab **Left the queue** \u2014 `2026-10-12 17:05 UTC` \u00b7 at `31fa323940264c601c16f192c8a22497f6684235`\n\nThis pull request spent **6 minutes 53 seconds** in the queue, including **2 minutes 42 seconds** running CI.\n\n## Reason\n\nThe merge conditions cannot be satisfied due to failing checks\n\n- `PR Body`\n\nFailing checks:\n- \u274c [`PR Body`](https://github.com/Neko-Catpital-Labs/Invoker/actions/runs/38281741556/job/114920522730) ([job log](https://github.com/Neko-Catpital-Labs/Invoker/actions/runs/38281741556/job/114920522730))\n\n## Hint\n\nYou may have to fix your CI before adding the pull request to the queue again.\nIf you update this pull request, to fix the CI, it will automatically be requeued once the queue conditions match again.\nIf you think this was a flaky issue instead, you can requeue the pull request, without updating it, by posting a `@mergifyio queue` comment.\n\n
\nWaiting for\n\n- [ ] `check-success = PR Body`\n- [ ] `check-success = UI Vitest`\n- [ ] `check-success = required-fast / Guardrails`\n- [ ] `check-success = required-fast / Launch Dispatch Queue Repro`\n- [ ] `check-success = required-fast / Start Running MECE Repros`\n- [ ] `check-success = required-fast / Submit Workflow Chain`\n\n
\n
\nAll conditions\n\n- [ ] `check-success = PR Body`\n- [ ] `check-success = UI Vitest`\n- [ ] `check-success = required-fast / Guardrails`\n- [ ] `check-success = required-fast / Launch Dispatch Queue Repro`\n- [ ] `check-success = required-fast / Start Running MECE Repros`\n- [ ] `check-success = required-fast / Submit Workflow Chain`\n- `#review-threads-unresolved = 0`\n - [X] #15193\n- [X] `check-success = build-artifacts`\n- [X] `check-success = quality / Dependency Cruise`\n- [X] `check-success = quality / TypeScript Types`\n- [X] `check-success = required-fast / Branch Carry Forward`\n- [X] `check-success = required-fast / Merge Gate Concurrency Repro`\n\n
\n\nTick the box to put this pull request back in the merge queue (same as `@mergifyio queue`).\n\n- [ ] Requeue this pull request ", + "created_at": "2026-10-12T16:58:35Z", + "id": 6128803276, + "updated_at": "2026-10-12T17:05:21Z" + } + ], + "Invoker#15249": [ + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 20:06 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by rule `autoqueue admin-bypass PRs to master`\n- \ud83d\udfe0 **Checks running** \u00b7 on draft #15253 \u00b7 [dashboard](https://dashboard.mergify.com/orgs/Neko-Catpital-Labs/repos/Invoker/queues/status?branch=master&pull=15249)\n- \u23f3 Merge \u00b7 ETA: `2026-10-12 20:17 UTC` \ud83d\ude80\n\n
\nWaiting for\n\n- [ ] `check-success = PR Body`\n- [ ] `check-success = UI Vitest`\n\n
\n
\nAll merge conditions\n\n- [ ] `check-success = PR Body`\n- [ ] `check-success = UI Vitest`\n- `#review-threads-unresolved = 0`\n - [X] #15249\n- [X] `check-success = build-artifacts`\n- [X] `check-success = quality / Dependency Cruise`\n- [X] `check-success = quality / TypeScript Types`\n- [X] `check-success = required-fast / Branch Carry Forward`\n- [X] `check-success = required-fast / Guardrails`\n- [X] `check-success = required-fast / Launch Dispatch Queue Repro`\n- [X] `check-success = required-fast / Merge Gate Concurrency Repro`\n- [X] `check-success = required-fast / Start Running MECE Repros`\n- [X] `check-success = required-fast / Submit Workflow Chain`\n\n
\n\n
\nRequired conditions to stay in the queue\n\n- [X] `-closed` [\ud83d\udccc queue requirement]\n- [X] `-conflict` [\ud83d\udccc queue requirement]\n- [X] `-draft` [\ud83d\udccc queue requirement]\n- [X] any of [\ud83d\udccc queue -> configuration change requirements]:\n - [X] `-mergify-configuration-changed`\n - [ ] `check-success = @mergify/Configuration changed`\n - [ ] `check-success = @mergify/Configuration has been deleted`\n- [X] any of [\ud83d\udd00 queue conditions]:\n - [X] all of [\ud83d\udccc queue conditions of queue rule `admin-bypass`]:\n - [X] `#review-threads-unresolved = 0`\n - [X] `-draft`\n - [X] `-label=merge-hold`\n - [X] `base=master`\n - [X] `label=admin-bypass`\n - [ ] all of [\ud83d\udccc queue conditions of queue rule `default`]:\n - [ ] `#approved-reviews-by >= 1`\n - [X] `#review-threads-unresolved = 0`\n - [X] `-draft`\n - [X] `-label=merge-hold`\n - [X] `base=master`\n - [ ] all of [\ud83d\udccc queue conditions of queue rule `fast-lane`]:\n - [ ] `-label=admin-bypass`\n - [ ] `label=fast-lane`\n - [X] `#review-threads-unresolved = 0`\n - [X] `-draft`\n - [X] `-label=merge-hold`\n - [X] `base=master`\n\n
", + "created_at": "2026-10-12T20:06:17Z", + "id": 6131736089, + "updated_at": "2026-10-12T20:11:23Z" + }, + { + "body": "> `queue`\n\n#### \u2611\ufe0f This pull request is already queued\n\n\n\n", + "created_at": "2026-10-12T20:06:34Z", + "id": 6131740612, + "updated_at": "2026-10-12T20:06:34Z" + } + ], + "derived-waiting-from-catstack#1354": [ + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-13 04:08 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1354#issuecomment-6138020313)\n- \ud83d\udfe0 **Checks running** \u00b7 on draft #1376 \u00b7 [dashboard](https://dashboard.mergify.com/orgs/EdbertChan/repos/catstack/queues/status?branch=main&pull=1354)\n- \u23f3 Merge \u00b7 ETA: `2026-10-13 05:08 UTC` \ud83d\ude80\n\n
\nWaiting for\n\n- [ ] `check-success = lint`\n- [ ] `check-success = test`\n- [ ] `check-success = validate`\n\n
\n
\nAll merge conditions\n\n- [ ] `check-success = lint`\n- [ ] `check-success = test`\n- [ ] `check-success = validate`\n\n
\n\n
\nRequired conditions to stay in the queue\n\n- [X] `-closed` [\ud83d\udccc queue requirement]\n- [X] `-conflict` [\ud83d\udccc queue requirement]\n- [X] `-draft` [\ud83d\udccc queue requirement]\n- [X] any of [\ud83d\udccc queue -> configuration change requirements]:\n - [X] `-mergify-configuration-changed`\n - [ ] `check-success = @mergify/Configuration changed`\n - [ ] `check-success = @mergify/Configuration has been deleted`\n- [X] any of [\ud83d\udd00 queue conditions]:\n - [X] all of [\ud83d\udccc queue conditions of queue rule `admin-bypass`]:\n - [X] `-draft`\n - [X] `base=main`\n - [X] `label=admin-bypass`\n\n
", + "created_at": "2026-10-13T04:08:08Z", + "id": 6138021515, + "updated_at": "2026-10-13T04:08:32Z" + } + ] +} \ No newline at end of file diff --git a/product/skills/land-stack/tests/test_queue_feedback.py b/product/skills/land-stack/tests/test_queue_feedback.py new file mode 100644 index 000000000..3e9ab78a9 --- /dev/null +++ b/product/skills/land-stack/tests/test_queue_feedback.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Tests for verify_stack.py --queue-feedback against real Mergify comments. + +fixtures/mergify_queue_comments_2026-10-12.json holds every comment Mergify +posted on six real PRs, as `gh api issues//comments` returned them: +"1309" was dequeued for merge conflicts after a timed-out batch; "1333" and +"1332" were dequeued by timeouts and then requeued; "1354" is queued and +preparing checks; Invoker "15193" was dequeued because the PR Body check +failed; Invoker "15249" is running queue checks. + +"derived-waiting-from-catstack#1354" is the real "checking" comment from #1354 +with its payload state changed to "waiting", the value Mergify reported +("Waiting for queue conditions") on #1354 after a head update. Mergify edits +that comment in place, so the original could not be saved. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(os.path.dirname(HERE), "scripts")) +import verify_stack as vs # noqa: E402 + +FIXTURE = os.path.join(HERE, "fixtures", "mergify_queue_comments_2026-10-12.json") +SCRIPT = os.path.join(os.path.dirname(HERE), "scripts", "verify_stack.py") + + +def load(): + with open(FIXTURE, encoding="utf-8") as fh: + return json.load(fh) + + +def status_comments(rows): + return [r for r in rows if "# Merge Queue Status" in r["body"]] + + +class TestQueueFeedback(unittest.TestCase): + def test_conflict_dequeue_is_rejected_with_a_rebase_action(self): + fb = vs.queue_feedback(load()["catstack#1309"]) + self.assertEqual(fb["state"], "rejected") + self.assertEqual(fb["kind"], "conflict") + self.assertIn("merge conflicts", fb["reason"]) + self.assertIn("rebase", fb["action"].lower()) + + def test_timeout_dequeue_is_rejected_as_timeout(self): + only_timeout = [status_comments(load()["catstack#1333"])[0]] + fb = vs.queue_feedback(only_timeout) + self.assertEqual(fb["state"], "rejected") + self.assertEqual(fb["kind"], "timeout") + self.assertIn("checks timeout", fb["reason"]) + self.assertIn("checks_timeout", fb["action"]) + + def test_batch_ahead_failure_is_a_recheck_not_a_rejection(self): + rows = [r for r in status_comments(load()["catstack#1332"]) + if "which was ahead in the queue failed" in r["body"]] + self.assertTrue(rows, "fixture lost the batch-ahead comment") + fb = vs.queue_feedback(rows[:1]) + self.assertEqual(fb["state"], "in-queue") + self.assertEqual(fb["kind"], "ahead-failed") + + def test_timeout_names_the_check_that_was_still_running(self): + only_timeout = [status_comments(load()["catstack#1333"])[0]] + self.assertEqual(vs.queue_feedback(only_timeout)["still_running"], ["test"]) + + def test_failed_check_names_the_check(self): + fb = vs.queue_feedback(load()["Invoker#15193"]) + self.assertEqual(fb["state"], "rejected") + self.assertEqual(fb["kind"], "checks-failed") + self.assertIn("PR Body", fb["failing_checks"]) + + def test_requeued_pr_reports_the_latest_comment_not_the_old_dequeue(self): + rows = load()["catstack#1333"] + self.assertTrue(any('"state": "dequeued"' in r["body"] for r in rows)) + fb = vs.queue_feedback(rows) + self.assertEqual(fb["state"], "in-queue") + + def test_queued_pr_is_in_queue_with_rule(self): + fb = vs.queue_feedback(load()["catstack#1354"]) + self.assertEqual(fb["state"], "in-queue") + self.assertEqual(fb["rule"], "admin-bypass") + + def test_waiting_for_conditions_is_waiting_not_unchecked(self): + fb = vs.queue_feedback(load()["derived-waiting-from-catstack#1354"]) + self.assertEqual(fb["state"], "waiting") + self.assertIn("conditions", fb["reason"]) + + def test_rejection_older_than_the_head_commit_is_waiting(self): + rows = load()["catstack#1309"] + stale = vs.queue_feedback(rows, head_at="2026-10-12T19:00:00Z") + self.assertEqual(stale["state"], "waiting") + self.assertIn("head", stale["reason"]) + + def test_rejection_newer_than_the_head_commit_stays_rejected(self): + rows = load()["catstack#1309"] + fb = vs.queue_feedback(rows, head_at="2026-10-12T14:00:00Z") + self.assertEqual(fb["state"], "rejected") + + def test_no_status_comment_is_unchecked(self): + fb = vs.queue_feedback([]) + self.assertEqual(fb["state"], "unchecked") + self.assertIn("no Merge Queue Status comment", fb["reason"]) + + def test_unreadable_payload_is_unchecked_not_in_queue(self): + broken = [{"id": 1, "body": "# Merge Queue Status\n\n- something new Mergify added\n"}] + fb = vs.queue_feedback(broken) + self.assertEqual(fb["state"], "unchecked") + + def test_cli_exit_codes_replay_json(self): + data = load() + path = os.path.join(os.environ.get("TMPDIR", "/tmp"), "queue_feedback_cli_test.json") + with open(path, "w", encoding="utf-8") as fh: + json.dump({"1309": data["catstack#1309"], "1354": data["catstack#1354"]}, fh) + try: + rejected = subprocess.run([sys.executable, SCRIPT, "--queue-feedback", "--feedback-json", path, "1309"], + capture_output=True, text=True) + ok = subprocess.run([sys.executable, SCRIPT, "--queue-feedback", "--feedback-json", path, "1354"], + capture_output=True, text=True) + finally: + os.unlink(path) + self.assertEqual(rejected.returncode, 1, rejected.stdout + rejected.stderr) + self.assertIn("rejected", rejected.stdout) + self.assertIn("#1309", rejected.stdout) + self.assertEqual(ok.returncode, 0, ok.stdout + ok.stderr) + self.assertIn("in-queue", ok.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cat_mode.py b/tests/test_cat_mode.py index c5c065d30..7a8228880 100644 --- a/tests/test_cat_mode.py +++ b/tests/test_cat_mode.py @@ -489,6 +489,7 @@ def test_every_referenced_skill_still_exists(self): "ls", "owner-serve", "strace", + "intervention-must-automate", # Absolute-negative words in backticks, not skill names. "only", "never",