From d1c6b2425009f1da0e78057fd3b50b5128824e86 Mon Sep 17 00:00:00 2001 From: Edbert Chan <1916223+EdbertChan@users.noreply.github.com> Date: Mon, 12 Oct 2026 21:02:21 -0700 Subject: [PATCH] Read the merge queue's reason when it rejects a pull request Co-Authored-By: Claude Sonnet 5.5 Change-Id: Ic68644735337fb896acc5f3a44eb1815c5d57b60 --- product/skills/land-stack/SKILL.md | 15 ++ .../skills/land-stack/scripts/verify_stack.py | 158 ++++++++++++++++++ .../mergify_queue_comments_2026-10-12.json | 106 ++++++++++++ .../land-stack/tests/test_queue_feedback.py | 132 +++++++++++++++ 4 files changed, 411 insertions(+) create mode 100644 product/skills/land-stack/tests/fixtures/mergify_queue_comments_2026-10-12.json create mode 100644 product/skills/land-stack/tests/test_queue_feedback.py diff --git a/product/skills/land-stack/SKILL.md b/product/skills/land-stack/SKILL.md index 938dcb440..4a55352ad 100644 --- a/product/skills/land-stack/SKILL.md +++ b/product/skills/land-stack/SKILL.md @@ -100,6 +100,21 @@ guard before any write (label, thread-resolve, queue, merge). Mergify reports `Merge queue is ready` until a `@mergifyio queue` comment (or the dashboard) actually queues the PR. + Queued PRs can still be rejected later, and the queue says why. Read its + own comment instead of guessing: + + ```sh + python3 scripts/verify_stack.py --repo --queue-feedback ... + ``` + + Exit 0 means each PR is merged or in the queue. Exit 1 names each rejected + PR with its kind (`timeout`, `conflict`, `checks-failed`, `other`), the + reason Mergify gave, the checks that failed or were still running, and the + next step. Repair from that, not from the PR's own CI: a `conflict` needs a + rebase, a `checks-failed` names the check to fix, and a `timeout` is not a + defect in the PR, so requeue it and, if it repeats, compare the queue's + `checks_timeout` with how long the named check normally runs. + 4. **Never batch merges without checking each result.** A merge command can look silent on both success and some failure paths; a silent-looking run is not proof of a merge. diff --git a/product/skills/land-stack/scripts/verify_stack.py b/product/skills/land-stack/scripts/verify_stack.py index c20cf48d8..5040ee2a4 100644 --- a/product/skills/land-stack/scripts/verify_stack.py +++ b/product/skills/land-stack/scripts/verify_stack.py @@ -23,6 +23,16 @@ Queue-state exit 0: every PR merged or queued. Exit 1: a PR is not queued. Exit 2: a PR's queue state could not be read. +After a queue write, `--queue-feedback` reads Mergify's own status comment on +each PR and says in-queue, rejected, merged, or unchecked. A rejection names +its kind (timeout, conflict, checks-failed, other), the reason Mergify gave, +the checks that were failing or still running, and what to do next. + + verify_stack.py --repo owner/name --queue-feedback 7006 7007 + verify_stack.py --feedback-json comments.json --queue-feedback 7006 + +Queue-feedback exit 0: every PR merged, in the queue, or waiting to enter it. Exit 1: a PR was +rejected. Exit 2: a PR's queue comment could not be read. Never uses `gh pr list --head ` -- discovering by branch name is the unsafe path the skill exists to prevent. """ @@ -30,6 +40,7 @@ import argparse import json +import re import subprocess import sys from typing import Callable @@ -151,6 +162,135 @@ def report_queue_state(keys: list[str], entries: dict) -> int: return 2 return 0 +STATUS_MARK = "# Merge Queue Status" +PAYLOAD_RE = re.compile(r"Mergify Payload -\*-\s*(\{.*\})\s*-\*- Mergify Payload End", re.S) +HTML_COMMENT_RE = re.compile(r"", re.S) +REASON_RE = re.compile(r"## Reason\s*\n(.*?)(?=\n## |\Z)", re.S) +FAILING_CHECK_RE = re.compile(r"^- \u274c \[`([^`]+)`\]", re.M) +STILL_RUNNING_RE = re.compile(r"These checks were still running:\s*\n((?:- `[^`]+`\s*\n?)+)") + +FEEDBACK_ACTIONS = { + "timeout": "A check was still running when the queue's checks_timeout ran out. Requeue with `@mergifyio queue`; if it keeps timing out, the timeout in .mergify.yml is shorter than the check's normal run time.", + "conflict": "GitHub refused the merge. Rebase onto the trunk and push; Mergify requeues on the update.", + "checks-failed": "Read the failing check named here, fix the PR, and push; Mergify requeues on the update.", + "ahead-failed": "An earlier batch failed and this PR is being checked again. No action unless it is rejected next.", + "other": "Read the reason, then fix the PR or requeue with `@mergifyio queue`.", +} + + +def _unchecked(reason: str) -> dict: + return {"state": "unchecked", "kind": "", "reason": reason, "action": "", "failing_checks": [], + "still_running": [], "rule": "", "eta": ""} + + +def _feedback_kind(reason: str) -> str: + low = reason.lower() + if "ahead in the queue failed" in low: + return "ahead-failed" + if "merge conflicts" in low: + return "conflict" + if "checks timeout" in low: + return "timeout" + if "failing checks" in low: + return "checks-failed" + return "other" + + +def queue_feedback(comments: list[dict], head_at: str | None = None) -> dict: + status = [c for c in comments if STATUS_MARK in (c.get("body") or "")] + if not status: + return _unchecked(f"no {STATUS_MARK[2:]} comment from Mergify") + latest = max(status, key=lambda c: c.get("id", 0)) + body = latest.get("body") or "" + found = PAYLOAD_RE.search(body) + if not found: + return _unchecked("Merge Queue Status comment has no readable Mergify payload") + try: + payload = json.loads(found.group(1)) + except json.JSONDecodeError as exc: + return _unchecked(f"Mergify payload is not valid JSON: {exc}") + state = payload.get("state") + text = HTML_COMMENT_RE.sub("", body) + reason_match = REASON_RE.search(text) + section = reason_match.group(1).strip() if reason_match else "" + paragraphs = [para.strip() for para in section.split("\n\n") if para.strip()] + informative = [para for para in paragraphs if not para.endswith("has been dequeued")] + reason = (informative or paragraphs or [""])[0] + running = STILL_RUNNING_RE.search(text) + still_running = re.findall(r"`([^`]+)`", running.group(1)) if running else [] + result = { + "kind": _feedback_kind(section) if section else "", + "reason": reason, + "failing_checks": FAILING_CHECK_RE.findall(text), + "still_running": still_running, + "rule": payload.get("queue_rule_name") or "", + "eta": payload.get("estimated_time_of_merge") or "", + } + if state == "dequeued": + if head_at and (latest.get("updated_at") or "") and head_at > latest["updated_at"]: + result.update(state="waiting", kind="", reason="the head commit is newer than the queue's last status; Mergify will check the new head", action="") + return result + kind = result["kind"] or "other" + result.update(state="rejected", kind=kind, action=FEEDBACK_ACTIONS[kind]) + return result + if state == "checking": + result.update(state="in-queue", action="") + return result + if state == "waiting": + result.update(state="waiting", reason="waiting for queue conditions (for example the new head's checks)", action="") + return result + return _unchecked(f"unrecognised Mergify queue state: {state!r}") + + +def fetch_queue_comments(repo: str | None, number: int) -> dict: + prefix = f"repos/{repo}" if repo else "repos/{owner}/{repo}" + pr = json.loads(subprocess.run(["gh", "api", f"{prefix}/pulls/{number}"], + capture_output=True, text=True, check=True).stdout) + pages = subprocess.run( + ["gh", "api", f"{prefix}/issues/{number}/comments", "--paginate", "--jq", + '[.[]|select(.user.login=="mergify[bot]")|{id,created_at,updated_at,body}]'], + capture_output=True, text=True, check=True).stdout + comments = [row for line in pages.splitlines() if line.strip() for row in json.loads(line)] + head = subprocess.run(["gh", "api", f"{prefix}/commits/{pr['head']['sha']}", "--jq", ".commit.committer.date"], + capture_output=True, text=True, check=True).stdout.strip() + return {"merged": bool(pr.get("merged")), "comments": comments, "head_at": head} + + +def report_queue_feedback(keys: list[str], entries: dict) -> int: + worst = 0 + for key in keys: + entry = entries.get(key) + if entry is None: + fb, merged = _unchecked("no payload for this PR"), False + else: + merged = bool(entry.get("merged")) if isinstance(entry, dict) else False + comments = entry.get("comments", []) if isinstance(entry, dict) else entry + head_at = entry.get("head_at") if isinstance(entry, dict) else None + fb = queue_feedback(comments, head_at=head_at) + if merged: + print(f"{'merged':<10} #{key}") + continue + if fb["state"] == "in-queue": + extra = f" (rule {fb['rule']}, ETA {fb['eta']})" if fb["rule"] else "" + note = f" -- {fb['reason']}" if fb["kind"] == "ahead-failed" else "" + print(f"{'in-queue':<10} #{key}{extra}{note}") + elif fb["state"] == "waiting": + print(f"{'waiting':<10} #{key} {fb['reason']}") + elif fb["state"] == "rejected": + print(f"{'rejected':<10} #{key} {fb['kind']}: {fb['reason']}") + if fb["failing_checks"]: + print(f"{'':<10} failing checks: {', '.join(fb['failing_checks'])}") + if fb["still_running"]: + print(f"{'':<10} still running at the timeout: {', '.join(fb['still_running'])}") + print(f"{'':<10} next: {fb['action']}") + worst = max(worst, 1) + else: + print(f"{'unchecked':<10} #{key} {fb['reason']}") + if worst == 0: + worst = 2 + return worst + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("numbers", nargs="*", help="PR numbers, bottom of stack first") @@ -163,8 +303,26 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--json", action="store_true", help="machine-readable output") ap.add_argument("--queue-state", action="store_true", help="after a queue write: is each PR merged or queued?") ap.add_argument("--queue-json", help="replay saved pull + check-run payloads keyed by PR for --queue-state") + ap.add_argument("--queue-feedback", action="store_true", help="read Mergify's status comment: in-queue, rejected (with reason), merged") + ap.add_argument("--feedback-json", help="replay saved Mergify comments keyed by PR for --queue-feedback") args = ap.parse_args(argv) + if args.queue_feedback: + keys = args.numbers + if not keys: + ap.error("give PR numbers for --queue-feedback") + if args.feedback_json: + with open(args.feedback_json, encoding="utf-8") as fh: + entries = json.load(fh) + else: + entries = {} + for key in keys: + try: + entries[key] = fetch_queue_comments(args.repo, int(key)) + except (subprocess.CalledProcessError, ValueError, KeyError, json.JSONDecodeError) as exc: + print(f"unchecked #{key} could not read queue comments: {exc}", file=sys.stderr) + return report_queue_feedback(keys, entries) + if args.queue_state: keys = args.numbers if not keys: diff --git a/product/skills/land-stack/tests/fixtures/mergify_queue_comments_2026-10-12.json b/product/skills/land-stack/tests/fixtures/mergify_queue_comments_2026-10-12.json new file mode 100644 index 000000000..f32680498 --- /dev/null +++ b/product/skills/land-stack/tests/fixtures/mergify_queue_comments_2026-10-12.json @@ -0,0 +1,106 @@ +{ + "catstack#1333": [ + { + "body": "Queued \u2014 the merge queue status continues in [this comment \u2193](https://github.com/EdbertChan/catstack/pull/1333#issuecomment-6126434423).", + "created_at": "2026-10-11T19:10:58Z", + "id": 6112652141, + "updated_at": "2026-10-12T14:36:29Z" + }, + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 14:36 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1333#issuecomment-6126433073)\n- \u274c **Checks timed out** \u00b7 on draft #1341\n- \ud83d\udeab **Left the queue** \u2014 `2026-10-12 15:37 UTC` \u00b7 at `9787af8f07826c77ba0072030a241cf253e6a08c`\n\nThis pull request spent **1 hour 1 minute 3 seconds** in the queue, including **1 hour 31 seconds** running CI.\n\n## Reason\n\nThe merge conditions cannot be satisfied: the checks did not pass within the checks timeout of 1 hour\n\nThese checks were still running:\n- `test`\n\nFailing checks:\n- \ud83d\udfe0 [`test`](https://github.com/EdbertChan/catstack/actions/runs/38264185548/job/114862046006) ([job log](https://github.com/EdbertChan/catstack/actions/runs/38264185548/job/114862046006))\n\n## Hint\n\nYou may have to fix your CI before adding the pull request to the queue again.\n\nIf you update this pull request, to fix the CI, it will automatically be requeued once the queue conditions match again.\nIf you think this was a flaky issue instead, you can requeue the pull request, without updating it, by posting a `@mergifyio queue` comment.\n\n
\nWaiting for\n\n- [ ] `check-success = test`\n\n
\n
\nAll conditions\n\n- [ ] `check-success = test`\n- [X] `check-success = lint`\n- [X] `check-success = validate`\n\n
\n\nRequeued \u2014 the merge queue status continues in [this comment \u2193](https://github.com/EdbertChan/catstack/pull/1333#issuecomment-6127751086).", + "created_at": "2026-10-12T14:36:28Z", + "id": 6126434423, + "updated_at": "2026-10-12T15:52:17Z" + }, + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 15:52 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1333#issuecomment-6127749387)\n- \u274c **Checks timed out** \u2014 `2026-10-12 18:22 UTC` \u00b7 on draft #1351\n- \u2702\ufe0f Bisecting to identify the failing PR (round 1/1)\n- \ud83d\udfe0 **Checks running** \u00b7 on draft #1353 \u00b7 [dashboard](https://dashboard.mergify.com/orgs/EdbertChan/repos/catstack/queues/status?branch=main&pull=1333)\n- \u23f3 Merge \u00b7 ETA: `2026-10-12 20:23 UTC` \ud83d\ude80\n\n
\nWaiting for\n\n- [ ] `check-success = test`\n\n
\n
\nAll merge conditions\n\n- [ ] `check-success = test`\n- [X] `check-success = lint`\n- [X] `check-success = validate`\n\n
\n\n
\nRequired conditions to stay in the queue\n\n- [X] `-closed` [\ud83d\udccc queue requirement]\n- [X] `-conflict` [\ud83d\udccc queue requirement]\n- [X] `-draft` [\ud83d\udccc queue requirement]\n- [X] any of [\ud83d\udccc queue -> configuration change requirements]:\n - [X] `-mergify-configuration-changed`\n - [ ] `check-success = @mergify/Configuration changed`\n - [ ] `check-success = @mergify/Configuration has been deleted`\n- [X] any of [\ud83d\udd00 queue conditions]:\n - [X] all of [\ud83d\udccc queue conditions of queue rule `admin-bypass`]:\n - [X] `-draft`\n - [X] `base=main`\n - [X] `label=admin-bypass`\n\n
", + "created_at": "2026-10-12T15:52:16Z", + "id": 6127751086, + "updated_at": "2026-10-12T19:24:32Z" + } + ], + "catstack#1332": [ + { + "body": "Queued \u2014 the merge queue status continues in [this comment \u2193](https://github.com/EdbertChan/catstack/pull/1332#issuecomment-6112771226).", + "created_at": "2026-10-11T19:10:53Z", + "id": 6112651269, + "updated_at": "2026-10-11T19:22:07Z" + }, + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-11 19:22 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1332#issuecomment-6112764720)\n- \u274c **Checks timed out** \u00b7 on draft #1339\n- \ud83d\udeab **Left the queue** \u2014 `2026-10-11 20:23 UTC` \u00b7 at `0e6a50c53c329843afa3886cf73cbe244fc3dfa3`\n\nThis pull request spent **1 hour 1 minute 19 seconds** in the queue, including **1 hour 20 seconds** running CI.\n\n## Reason\n\nThe merge conditions cannot be satisfied: the checks did not pass within the checks timeout of 1 hour\n\nThese checks were still running:\n- `test`\n\nFailing checks:\n- \ud83d\udfe0 [`test`](https://github.com/EdbertChan/catstack/actions/runs/38167777445/job/114553826149) ([job log](https://github.com/EdbertChan/catstack/actions/runs/38167777445/job/114553826149))\n\n## Hint\n\nYou may have to fix your CI before adding the pull request to the queue again.\n\nIf you update this pull request, to fix the CI, it will automatically be requeued once the queue conditions match again.\nIf you think this was a flaky issue instead, you can requeue the pull request, without updating it, by posting a `@mergifyio queue` comment.\n\n
\nWaiting for\n\n- [ ] `check-success = test`\n\n
\n
\nAll conditions\n\n- [ ] `check-success = test`\n- [X] `check-success = lint`\n- [X] `check-success = validate`\n\n
\n\nRequeued \u2014 the merge queue status continues in [this comment \u2193](https://github.com/EdbertChan/catstack/pull/1332#issuecomment-6126529630).", + "created_at": "2026-10-11T19:22:06Z", + "id": 6112771226, + "updated_at": "2026-10-12T14:42:12Z" + }, + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 14:42 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1332#issuecomment-6126526873)\n- \u274c **Checks timed out** \u2014 `2026-10-12 15:37 UTC` \u00b7 on draft #1345\n- \u2702\ufe0f Bisecting to identify the failing PR (round 1/1)\n- \u274c **Checks timed out** \u00b7 on draft #1348 \u00b7 identified as the failing PR\n- \ud83d\udeab **Left the queue** \u2014 `2026-10-12 17:39 UTC` \u00b7 at `0e6a50c53c329843afa3886cf73cbe244fc3dfa3`\n\nThis pull request spent **2 hours 57 minutes 4 seconds** in the queue, including **2 hours 54 seconds** running CI.\n\n## Reason\n\nThe merge conditions cannot be satisfied: the checks did not pass within the checks timeout of 1 hour\n\nThese checks were still running:\n- `test`\n\nFailing checks:\n- \ud83d\udfe0 [`test`](https://github.com/EdbertChan/catstack/actions/runs/38278827982/job/114910815884) ([job log](https://github.com/EdbertChan/catstack/actions/runs/38278827982/job/114910815884))\n\n## Hint\n\nYou may have to fix your CI before adding the pull request to the queue again.\n\nIf you update this pull request, to fix the CI, it will automatically be requeued once the queue conditions match again.\nIf you think this was a flaky issue instead, you can requeue the pull request, without updating it, by posting a `@mergifyio queue` comment.\n\n
\nWaiting for\n\n- [ ] `check-success = test`\n\n
\n
\nAll conditions\n\n- [ ] `check-success = test`\n- [X] `check-success = lint`\n- [X] `check-success = validate`\n\n
\n\nRequeued \u2014 the merge queue status continues in [this comment \u2193](https://github.com/EdbertChan/catstack/pull/1332#issuecomment-6130960344).", + "created_at": "2026-10-12T14:42:11Z", + "id": 6126529630, + "updated_at": "2026-10-12T19:16:26Z" + }, + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 19:16 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1332#issuecomment-6130958496)\n- \ud83d\udfe0 **Checks running** \u00b7 on draft #1352\n- \ud83d\udd04 **Checks restarted** \u00b7 will be requeued automatically\n\n## Reason\n\nPull requests batch (#1333, #1336) which was ahead in the queue failed", + "created_at": "2026-10-12T19:16:25Z", + "id": 6130960344, + "updated_at": "2026-10-12T19:23:13Z" + } + ], + "catstack#1309": [ + { + "body": "Queued \u2014 the merge queue status continues in [this comment \u2193](https://github.com/EdbertChan/catstack/pull/1309#issuecomment-6126707738).", + "created_at": "2026-10-11T19:11:56Z", + "id": 6112662978, + "updated_at": "2026-10-12T14:52:33Z" + }, + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 14:52 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1309#issuecomment-6126705187)\n- \u274c **Checks timed out** \u2014 `2026-10-12 15:37 UTC` \u00b7 on draft #1345\n- \u2702\ufe0f Bisecting to identify the failing PR (round 1/1)\n- \u274c **Checks failed** \u00b7 on draft #1350 \u00b7 identified as the failing PR\n- \ud83d\udeab **Left the queue** \u2014 `2026-10-12 18:22 UTC` \u00b7 at `737c66df5221d80ff48c8308067840366155180e`\n\nThis pull request spent **3 hours 29 minutes 55 seconds** in the queue, including **1 hour 43 minutes** running CI.\n\n## Reason\n\nPull request #1309 has been dequeued\n\nGitHub refused to merge the pull request. Pull Request has merge conflicts. This is usually enforced by a [branch protection](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-protected-branches/about-protected-branches) or [ruleset rule](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-rulesets/about-rulesets).\n\n## Hint\n\nYou should look at the reason for the failure and decide if the pull request needs to be fixed or if you want to requeue it.\nIf you do update this pull request, it will automatically be requeued once the queue conditions match again.\nIf you think this was a flaky issue instead, you can requeue the pull request, without updating it, by posting a `@mergifyio queue` comment.\n\n
\nRequired conditions to merge\n\n- [X] `check-success = lint`\n- [X] `check-success = test`\n- [X] `check-success = validate`\n\n
\n\nTick the box to put this pull request back in the merge queue (same as `@mergifyio queue`).\n\n- [ ] Requeue this pull request ", + "created_at": "2026-10-12T14:52:32Z", + "id": 6126707738, + "updated_at": "2026-10-12T18:22:27Z" + } + ], + "catstack#1354": [ + { + "body": "Queued \u2014 the merge queue status continues in [this comment \u2193](https://github.com/EdbertChan/catstack/pull/1354#issuecomment-6131697266).", + "created_at": "2026-10-12T20:00:48Z", + "id": 6131654915, + "updated_at": "2026-10-12T20:03:38Z" + }, + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 20:03 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1354#issuecomment-6131696378)\n- \ud83d\udfe0 **Preparing checks**\n- \u23f3 Merge \u00b7 ETA: `2026-10-12 21:23 UTC` \ud83d\ude80", + "created_at": "2026-10-12T20:03:38Z", + "id": 6131697266, + "updated_at": "2026-10-12T20:03:42Z" + } + ], + "Invoker#15193": [ + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 16:58 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by rule `autoqueue admin-bypass PRs to master`\n- \u274c **Checks failed** \u00b7 on draft #15210\n- \ud83d\udeab **Left the queue** \u2014 `2026-10-12 17:05 UTC` \u00b7 at `31fa323940264c601c16f192c8a22497f6684235`\n\nThis pull request spent **6 minutes 53 seconds** in the queue, including **2 minutes 42 seconds** running CI.\n\n## Reason\n\nThe merge conditions cannot be satisfied due to failing checks\n\n- `PR Body`\n\nFailing checks:\n- \u274c [`PR Body`](https://github.com/Neko-Catpital-Labs/Invoker/actions/runs/38281741556/job/114920522730) ([job log](https://github.com/Neko-Catpital-Labs/Invoker/actions/runs/38281741556/job/114920522730))\n\n## Hint\n\nYou may have to fix your CI before adding the pull request to the queue again.\nIf you update this pull request, to fix the CI, it will automatically be requeued once the queue conditions match again.\nIf you think this was a flaky issue instead, you can requeue the pull request, without updating it, by posting a `@mergifyio queue` comment.\n\n
\nWaiting for\n\n- [ ] `check-success = PR Body`\n- [ ] `check-success = UI Vitest`\n- [ ] `check-success = required-fast / Guardrails`\n- [ ] `check-success = required-fast / Launch Dispatch Queue Repro`\n- [ ] `check-success = required-fast / Start Running MECE Repros`\n- [ ] `check-success = required-fast / Submit Workflow Chain`\n\n
\n
\nAll conditions\n\n- [ ] `check-success = PR Body`\n- [ ] `check-success = UI Vitest`\n- [ ] `check-success = required-fast / Guardrails`\n- [ ] `check-success = required-fast / Launch Dispatch Queue Repro`\n- [ ] `check-success = required-fast / Start Running MECE Repros`\n- [ ] `check-success = required-fast / Submit Workflow Chain`\n- `#review-threads-unresolved = 0`\n - [X] #15193\n- [X] `check-success = build-artifacts`\n- [X] `check-success = quality / Dependency Cruise`\n- [X] `check-success = quality / TypeScript Types`\n- [X] `check-success = required-fast / Branch Carry Forward`\n- [X] `check-success = required-fast / Merge Gate Concurrency Repro`\n\n
\n\nTick the box to put this pull request back in the merge queue (same as `@mergifyio queue`).\n\n- [ ] Requeue this pull request ", + "created_at": "2026-10-12T16:58:35Z", + "id": 6128803276, + "updated_at": "2026-10-12T17:05:21Z" + } + ], + "Invoker#15249": [ + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-12 20:06 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by rule `autoqueue admin-bypass PRs to master`\n- \ud83d\udfe0 **Checks running** \u00b7 on draft #15253 \u00b7 [dashboard](https://dashboard.mergify.com/orgs/Neko-Catpital-Labs/repos/Invoker/queues/status?branch=master&pull=15249)\n- \u23f3 Merge \u00b7 ETA: `2026-10-12 20:17 UTC` \ud83d\ude80\n\n
\nWaiting for\n\n- [ ] `check-success = PR Body`\n- [ ] `check-success = UI Vitest`\n\n
\n
\nAll merge conditions\n\n- [ ] `check-success = PR Body`\n- [ ] `check-success = UI Vitest`\n- `#review-threads-unresolved = 0`\n - [X] #15249\n- [X] `check-success = build-artifacts`\n- [X] `check-success = quality / Dependency Cruise`\n- [X] `check-success = quality / TypeScript Types`\n- [X] `check-success = required-fast / Branch Carry Forward`\n- [X] `check-success = required-fast / Guardrails`\n- [X] `check-success = required-fast / Launch Dispatch Queue Repro`\n- [X] `check-success = required-fast / Merge Gate Concurrency Repro`\n- [X] `check-success = required-fast / Start Running MECE Repros`\n- [X] `check-success = required-fast / Submit Workflow Chain`\n\n
\n\n
\nRequired conditions to stay in the queue\n\n- [X] `-closed` [\ud83d\udccc queue requirement]\n- [X] `-conflict` [\ud83d\udccc queue requirement]\n- [X] `-draft` [\ud83d\udccc queue requirement]\n- [X] any of [\ud83d\udccc queue -> configuration change requirements]:\n - [X] `-mergify-configuration-changed`\n - [ ] `check-success = @mergify/Configuration changed`\n - [ ] `check-success = @mergify/Configuration has been deleted`\n- [X] any of [\ud83d\udd00 queue conditions]:\n - [X] all of [\ud83d\udccc queue conditions of queue rule `admin-bypass`]:\n - [X] `#review-threads-unresolved = 0`\n - [X] `-draft`\n - [X] `-label=merge-hold`\n - [X] `base=master`\n - [X] `label=admin-bypass`\n - [ ] all of [\ud83d\udccc queue conditions of queue rule `default`]:\n - [ ] `#approved-reviews-by >= 1`\n - [X] `#review-threads-unresolved = 0`\n - [X] `-draft`\n - [X] `-label=merge-hold`\n - [X] `base=master`\n - [ ] all of [\ud83d\udccc queue conditions of queue rule `fast-lane`]:\n - [ ] `-label=admin-bypass`\n - [ ] `label=fast-lane`\n - [X] `#review-threads-unresolved = 0`\n - [X] `-draft`\n - [X] `-label=merge-hold`\n - [X] `base=master`\n\n
", + "created_at": "2026-10-12T20:06:17Z", + "id": 6131736089, + "updated_at": "2026-10-12T20:11:23Z" + }, + { + "body": "> `queue`\n\n#### \u2611\ufe0f This pull request is already queued\n\n\n\n", + "created_at": "2026-10-12T20:06:34Z", + "id": 6131740612, + "updated_at": "2026-10-12T20:06:34Z" + } + ], + "derived-waiting-from-catstack#1354": [ + { + "body": "\n\n# Merge Queue Status\n\n- \u2705 **Entered queue** \u2014 `2026-10-13 04:08 UTC` \u00b7 Rule: `admin-bypass` \u00b7 triggered by @EdbertChan with the [`@mergifyio queue` command](https://github.com/EdbertChan/catstack/pull/1354#issuecomment-6138020313)\n- \ud83d\udfe0 **Checks running** \u00b7 on draft #1376 \u00b7 [dashboard](https://dashboard.mergify.com/orgs/EdbertChan/repos/catstack/queues/status?branch=main&pull=1354)\n- \u23f3 Merge \u00b7 ETA: `2026-10-13 05:08 UTC` \ud83d\ude80\n\n
\nWaiting for\n\n- [ ] `check-success = lint`\n- [ ] `check-success = test`\n- [ ] `check-success = validate`\n\n
\n
\nAll merge conditions\n\n- [ ] `check-success = lint`\n- [ ] `check-success = test`\n- [ ] `check-success = validate`\n\n
\n\n
\nRequired conditions to stay in the queue\n\n- [X] `-closed` [\ud83d\udccc queue requirement]\n- [X] `-conflict` [\ud83d\udccc queue requirement]\n- [X] `-draft` [\ud83d\udccc queue requirement]\n- [X] any of [\ud83d\udccc queue -> configuration change requirements]:\n - [X] `-mergify-configuration-changed`\n - [ ] `check-success = @mergify/Configuration changed`\n - [ ] `check-success = @mergify/Configuration has been deleted`\n- [X] any of [\ud83d\udd00 queue conditions]:\n - [X] all of [\ud83d\udccc queue conditions of queue rule `admin-bypass`]:\n - [X] `-draft`\n - [X] `base=main`\n - [X] `label=admin-bypass`\n\n
", + "created_at": "2026-10-13T04:08:08Z", + "id": 6138021515, + "updated_at": "2026-10-13T04:08:32Z" + } + ] +} \ No newline at end of file diff --git a/product/skills/land-stack/tests/test_queue_feedback.py b/product/skills/land-stack/tests/test_queue_feedback.py new file mode 100644 index 000000000..3e9ab78a9 --- /dev/null +++ b/product/skills/land-stack/tests/test_queue_feedback.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Tests for verify_stack.py --queue-feedback against real Mergify comments. + +fixtures/mergify_queue_comments_2026-10-12.json holds every comment Mergify +posted on six real PRs, as `gh api issues//comments` returned them: +"1309" was dequeued for merge conflicts after a timed-out batch; "1333" and +"1332" were dequeued by timeouts and then requeued; "1354" is queued and +preparing checks; Invoker "15193" was dequeued because the PR Body check +failed; Invoker "15249" is running queue checks. + +"derived-waiting-from-catstack#1354" is the real "checking" comment from #1354 +with its payload state changed to "waiting", the value Mergify reported +("Waiting for queue conditions") on #1354 after a head update. Mergify edits +that comment in place, so the original could not be saved. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(os.path.dirname(HERE), "scripts")) +import verify_stack as vs # noqa: E402 + +FIXTURE = os.path.join(HERE, "fixtures", "mergify_queue_comments_2026-10-12.json") +SCRIPT = os.path.join(os.path.dirname(HERE), "scripts", "verify_stack.py") + + +def load(): + with open(FIXTURE, encoding="utf-8") as fh: + return json.load(fh) + + +def status_comments(rows): + return [r for r in rows if "# Merge Queue Status" in r["body"]] + + +class TestQueueFeedback(unittest.TestCase): + def test_conflict_dequeue_is_rejected_with_a_rebase_action(self): + fb = vs.queue_feedback(load()["catstack#1309"]) + self.assertEqual(fb["state"], "rejected") + self.assertEqual(fb["kind"], "conflict") + self.assertIn("merge conflicts", fb["reason"]) + self.assertIn("rebase", fb["action"].lower()) + + def test_timeout_dequeue_is_rejected_as_timeout(self): + only_timeout = [status_comments(load()["catstack#1333"])[0]] + fb = vs.queue_feedback(only_timeout) + self.assertEqual(fb["state"], "rejected") + self.assertEqual(fb["kind"], "timeout") + self.assertIn("checks timeout", fb["reason"]) + self.assertIn("checks_timeout", fb["action"]) + + def test_batch_ahead_failure_is_a_recheck_not_a_rejection(self): + rows = [r for r in status_comments(load()["catstack#1332"]) + if "which was ahead in the queue failed" in r["body"]] + self.assertTrue(rows, "fixture lost the batch-ahead comment") + fb = vs.queue_feedback(rows[:1]) + self.assertEqual(fb["state"], "in-queue") + self.assertEqual(fb["kind"], "ahead-failed") + + def test_timeout_names_the_check_that_was_still_running(self): + only_timeout = [status_comments(load()["catstack#1333"])[0]] + self.assertEqual(vs.queue_feedback(only_timeout)["still_running"], ["test"]) + + def test_failed_check_names_the_check(self): + fb = vs.queue_feedback(load()["Invoker#15193"]) + self.assertEqual(fb["state"], "rejected") + self.assertEqual(fb["kind"], "checks-failed") + self.assertIn("PR Body", fb["failing_checks"]) + + def test_requeued_pr_reports_the_latest_comment_not_the_old_dequeue(self): + rows = load()["catstack#1333"] + self.assertTrue(any('"state": "dequeued"' in r["body"] for r in rows)) + fb = vs.queue_feedback(rows) + self.assertEqual(fb["state"], "in-queue") + + def test_queued_pr_is_in_queue_with_rule(self): + fb = vs.queue_feedback(load()["catstack#1354"]) + self.assertEqual(fb["state"], "in-queue") + self.assertEqual(fb["rule"], "admin-bypass") + + def test_waiting_for_conditions_is_waiting_not_unchecked(self): + fb = vs.queue_feedback(load()["derived-waiting-from-catstack#1354"]) + self.assertEqual(fb["state"], "waiting") + self.assertIn("conditions", fb["reason"]) + + def test_rejection_older_than_the_head_commit_is_waiting(self): + rows = load()["catstack#1309"] + stale = vs.queue_feedback(rows, head_at="2026-10-12T19:00:00Z") + self.assertEqual(stale["state"], "waiting") + self.assertIn("head", stale["reason"]) + + def test_rejection_newer_than_the_head_commit_stays_rejected(self): + rows = load()["catstack#1309"] + fb = vs.queue_feedback(rows, head_at="2026-10-12T14:00:00Z") + self.assertEqual(fb["state"], "rejected") + + def test_no_status_comment_is_unchecked(self): + fb = vs.queue_feedback([]) + self.assertEqual(fb["state"], "unchecked") + self.assertIn("no Merge Queue Status comment", fb["reason"]) + + def test_unreadable_payload_is_unchecked_not_in_queue(self): + broken = [{"id": 1, "body": "# Merge Queue Status\n\n- something new Mergify added\n"}] + fb = vs.queue_feedback(broken) + self.assertEqual(fb["state"], "unchecked") + + def test_cli_exit_codes_replay_json(self): + data = load() + path = os.path.join(os.environ.get("TMPDIR", "/tmp"), "queue_feedback_cli_test.json") + with open(path, "w", encoding="utf-8") as fh: + json.dump({"1309": data["catstack#1309"], "1354": data["catstack#1354"]}, fh) + try: + rejected = subprocess.run([sys.executable, SCRIPT, "--queue-feedback", "--feedback-json", path, "1309"], + capture_output=True, text=True) + ok = subprocess.run([sys.executable, SCRIPT, "--queue-feedback", "--feedback-json", path, "1354"], + capture_output=True, text=True) + finally: + os.unlink(path) + self.assertEqual(rejected.returncode, 1, rejected.stdout + rejected.stderr) + self.assertIn("rejected", rejected.stdout) + self.assertIn("#1309", rejected.stdout) + self.assertEqual(ok.returncode, 0, ok.stdout + ok.stderr) + self.assertIn("in-queue", ok.stdout) + + +if __name__ == "__main__": + unittest.main()