diff --git a/.github/workflows/catalog.yml b/.github/workflows/catalog.yml index c5e1bba..b73f397 100644 --- a/.github/workflows/catalog.yml +++ b/.github/workflows/catalog.yml @@ -29,19 +29,31 @@ jobs: QUALIFIED_COMMIT: ${{ inputs.qualified_commit }} QUALIFICATION_URL: ${{ inputs.qualification_url }} CARGO_BUILD_BUILD_DIR: target/build + TOOLING_COMMIT: ${{ github.sha }} steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: ref: refs/tags/${{ inputs.tag }} fetch-depth: 0 persist-credentials: false + - name: Checkout reviewed catalog tooling separately from release source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ github.sha }} + path: catalog-tools + persist-credentials: false + - uses: pnpm/setup@5d160c5bc68a09337ad0d5654e237e03253b5879 # v1 + with: + version: 11.28.5 + runtime: node@24.21.0 + install: false - run: bash scripts/ci.sh - name: Fail closed on source, native qualification, policy and raw LSP downloads run: python3 scripts/marketplace.py --catalog --extension-path catalog/terraphim-lsp --tag "$RELEASE_TAG" --repo "$SOURCE_REPO" --qualified-commit "$QUALIFIED_COMMIT" --qualification-url "$QUALIFICATION_URL" --out public-preflight - name: Prepare the patch for human submission under upstream AI policy env: GH_TOKEN: ${{ github.token }} - run: python3 scripts/prepare_catalog.py --tag "$RELEASE_TAG" --preflight public-preflight/preflight.json --out catalog-submission + run: python3 catalog-tools/scripts/prepare_catalog.py --tag "$RELEASE_TAG" --source-root . --tooling-commit "$TOOLING_COMMIT" --preflight public-preflight/preflight.json --out catalog-submission - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: zed-catalog-${{ inputs.tag }} diff --git a/docs/marketplace.md b/docs/marketplace.md index 41b3755..4315332 100644 --- a/docs/marketplace.md +++ b/docs/marketplace.md @@ -117,15 +117,23 @@ and release CI remain automated; the upstream submission is a human action. 1. Publish reviewed source and the exact immutable dependency release. Test that exact source manually in stock Zed and save the output evidence described in [qualification.md](qualification.md). -2. Dispatch `catalog.yml` with its existing stable tag, the exact native-qualified - source SHA and an HTTPS evidence URL. No cross-repository token is needed. +2. Dispatch `catalog.yml` from reviewed `main` with its existing stable tag, the + exact native-qualified source SHA and an immutable evidence URL. The workflow + keeps its tooling commit separate from the clean tagged release checkout, so + a tooling repair does not move an existing tag or rebuild a published package. + No cross-repository token is needed. 3. CI repeats tests and public download checks, then `prepare_catalog.py` checks out the official catalog, updates only this extension's gitlink, `.gitmodules` and `extensions.toml` (with the catalog subdirectory), and runs upstream sorting. - Package install hooks are disabled; package tooling receives no GitHub token. + It independently checks the published archive/checksum/qualification files + against the tagged source. Upstream dependencies use its frozen pnpm lockfile, + pnpm 11.28.5 and Node 24.21.0; runtime changes require tooling review. The exact + upstream manifest and lock bytes must remain unchanged. Package install hooks + are disabled; package tooling receives no GitHub token. 4. The artifact contains `catalog.patch`, `submission.json` and human handoff instructions. It records `HUMAN_SUBMISSION_REQUIRED`, exact source/tag/base, - evidence URL and patch hash. It creates no fork, branch, PR or comment. + evidence URL and patch hash, plus the separate tooling commit, runtime/lock + identities and published package hashes. It creates no fork, branch, PR or comment. 5. A human maintainer reviews the outputs, applies the patch to their personal catalog fork, writes the PR explanation in their own words and submits it. Keep one extension per PR and no more than three open PRs per submitter. diff --git a/scripts/prepare_catalog.py b/scripts/prepare_catalog.py index 64828aa..e04d2e4 100644 --- a/scripts/prepare_catalog.py +++ b/scripts/prepare_catalog.py @@ -12,11 +12,94 @@ import catalog import marketplace +import package +import package_qualification from publish_github import verify_remote_tag UPSTREAM = "zed-industries/extensions" POLICY = "https://github.com/zed-industries/extensions/blob/main/AI_POLICY.md" +# Match upstream's Node runtime and a reviewed exact pnpm 11 release. +NODE_VERSION = "24.21.0" +PNPM_VERSION = "11.28.5" +LOCKED_FILES = ("package.json", "pnpm-lock.yaml") + + +def validate_release_source(root, receipt, tag): + """Bind release evidence to a clean tagged checkout, separate from tooling.""" + source = catalog.git(root, "rev-parse", "HEAD") + tagged = catalog.git(root, "rev-parse", f"refs/tags/{tag}^{{commit}}") + if (source != tagged + or catalog.git(root, "status", "--porcelain", "--untracked-files=no")): + raise marketplace.GateError("Release source must be the clean tagged checkout") + marketplace.validate_receipt(receipt, source, tag, catalog=True) + return source + + +def validate_released_package(root, receipt, tag, archive, sums, qualification): + """Require actual published package bytes to name the exact release source.""" + value = json.loads(qualification) + name = f"terraphim-lsp-{receipt['version']}.tar.gz" + archive_record = package_qualification.record(archive) + expected = {"status": "PACKAGE_BYTES_VERIFIED", "source": receipt["source"], + "sourceTreeDirty": False, "tag": tag, + "repository": receipt["repository"], "version": receipt["version"], + "extensionId": "terraphim-lsp", "extensionPath": package.CATALOG_PATH} + if (any(value.get(key) != item for key, item in expected.items()) + or value.get("archive") != {"name": name, **archive_record} + or marketplace.checksums(sums.decode()) != {name: archive_record["sha256"]} + or value.get("checksums") != {"name": "SHA256SUMS", + **package_qualification.record(sums)}): + raise marketplace.GateError("Published package identity or checksum differs") + files = package_qualification.inspect_archive(archive, receipt["version"], + package.CATALOG_PATH) + if value.get("entries") != {name: package_qualification.record(data) + for name, data in files.items()}: + raise marketplace.GateError("Published package entries differ") + for source_file in package.FILES: + if files[source_file] != (root / package.CATALOG_PATH / source_file).read_bytes(): + raise marketplace.GateError("Published package differs from tagged source") + return {"archive": {"name": name, **archive_record}, + "qualification": package_qualification.record(qualification)} + + +def locked_inputs(root): + """Require upstream's regular manifest/lock files and reviewed runtime.""" + if any(not (root / name).exists() for name in LOCKED_FILES): + raise marketplace.GateError("Upstream manifest and pnpm lockfile are required") + if any((root / name).is_symlink() or not (root / name).is_file() + for name in LOCKED_FILES): + raise marketplace.GateError("Upstream manifest and lockfile must be regular files") + saved = {name: (root / name).read_bytes() for name in LOCKED_FILES} + if json.loads(saved["package.json"]).get("volta", {}).get("node") != NODE_VERSION: + raise marketplace.GateError("Upstream runtime changed; review catalog tooling") + return saved + + +def verify_locked_inputs(root, saved): + """Refuse a dependency installer or sorter that rewrites upstream inputs.""" + if any((root / name).read_bytes() != data for name, data in saved.items()): + raise marketplace.GateError("Upstream manifest or lockfile changed") + + +def install_and_sort(root, env): + """Run upstream's real sorter using its frozen production dependencies.""" + saved = locked_inputs(root) + node = subprocess.check_output(["node", "--version"], text=True, env=env).strip() + pnpm = subprocess.check_output(["pnpm", "--version"], text=True, env=env).strip() + if node != f"v{NODE_VERSION}" or pnpm != PNPM_VERSION: + raise marketplace.GateError("Catalog requires the reviewed exact Node/pnpm versions") + subprocess.run(["pnpm", "install", "--frozen-lockfile", "--ignore-scripts", "--prod"], + cwd=root, check=True, env=env) + verify_locked_inputs(root, saved) + subprocess.run(["pnpm", "run", "sort-extensions"], cwd=root, check=True, env=env) + verify_locked_inputs(root, saved) + return {"node": node, "pnpm": pnpm, + "upstreamInputs": {name: hashlib.sha256(data).hexdigest() + for name, data in saved.items()}, + "frozenLockfile": True, "inputBytesUnchanged": True} + + def write_handoff(root, receipt, output): """Save the actual catalog diff and factual evidence without authoring a PR.""" @@ -68,13 +151,28 @@ def main(): parser.add_argument("--tag", required=True) parser.add_argument("--preflight", type=Path, required=True) parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--source-root", type=Path, required=True) + parser.add_argument("--tooling-commit", required=True) args = parser.parse_args() try: receipt = json.loads(args.preflight.read_text()) - source = catalog.git(marketplace.ROOT, "rev-parse", "HEAD") - marketplace.validate_receipt(receipt, source, args.tag, catalog=True) + tooling = catalog.git(marketplace.ROOT, "rev-parse", "HEAD") + if (not marketplace.SHA_RE.fullmatch(args.tooling_commit) + or tooling != args.tooling_commit + or catalog.git(marketplace.ROOT, "status", "--porcelain", + "--untracked-files=no")): + raise marketplace.GateError("Catalog tooling differs from reviewed dispatch commit") + source = validate_release_source(args.source_root, receipt, args.tag) verify_remote_tag(receipt["repository"], args.tag, source) with tempfile.TemporaryDirectory() as folder: + downloads = Path(folder) / "released" + downloads.mkdir() + names = [f"terraphim-lsp-{receipt['version']}.tar.gz", + "SHA256SUMS", "package-qualification.json"] + data = [marketplace.anonymous_download( + f"https://github.com/{receipt['repository']}/releases/download/{args.tag}/{name}", + downloads / name) for name in names] + released = validate_released_package(args.source_root, receipt, args.tag, *data) root = Path(folder) / "catalog" subprocess.run( [ @@ -101,25 +199,14 @@ def main(): for k, v in os.environ.items() if k not in ["GH_TOKEN", "GITHUB_TOKEN"] } - subprocess.run( - [ - "npm", - "install", - "--package-lock=false", - "--ignore-scripts", - "--omit=dev", - "--no-audit", - "--no-fund", - ], - cwd=root, - check=True, - env=clean_env, - ) - subprocess.run( - ["npm", "run", "sort-extensions"], cwd=root, check=True, env=clean_env - ) + toolchain = install_and_sort(root, clean_env) catalog.git(root, "add", ".gitmodules", "extensions.toml") - print(json.dumps(write_handoff(root, receipt, args.out))) + result = write_handoff(root, receipt, args.out) + result["releasedPackage"] = released + result["toolingSource"] = tooling + result["toolchain"] = toolchain + (args.out / "submission.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result)) except ( marketplace.GateError, KeyError, diff --git a/scripts/test_catalog_handoff.py b/scripts/test_catalog_handoff.py index 5d46657..517b42b 100644 --- a/scripts/test_catalog_handoff.py +++ b/scripts/test_catalog_handoff.py @@ -10,6 +10,7 @@ import catalog import marketplace import prepare_catalog +from test_package_qualification import Fixture class CatalogHandoffTests(unittest.TestCase): @@ -36,6 +37,8 @@ def setUp(self): ) self.source = self.git("rev-parse", "HEAD") self.receipt = { + "status": "PASS", + "catalog": True, "source": self.source, "tag": "v0.2.0", "extensionId": "terraphim-lsp", @@ -86,6 +89,63 @@ def test_actual_patch_applies_to_clean_catalog_without_any_remote_mutation(self) ) self.assertFalse((output / "pr-body.md").exists()) + def test_release_identity_uses_explicit_tagged_source_not_tooling_checkout(self): + self.git("tag", "v0.2.0") + self.assertEqual(prepare_catalog.validate_release_source( + self.root, self.receipt, "v0.2.0"), self.source) + (self.root / "extensions.toml").write_text("changed source") + with self.assertRaisesRegex(marketplace.GateError, "clean tagged"): + prepare_catalog.validate_release_source(self.root, self.receipt, "v0.2.0") + self.git("checkout", "--", "extensions.toml") + (self.root / "later.txt").write_text("tooling may follow release") + self.git("add", "later.txt") + self.git("-c", "user.name=Test", "-c", "user.email=test@example.invalid", + "-c", "commit.gpgsign=false", "commit", "-qm", "later tooling") + with self.assertRaisesRegex(marketplace.GateError, "tagged"): + prepare_catalog.validate_release_source(self.root, self.receipt, "v0.2.0") + + def test_locked_upstream_inputs_fail_closed_and_preserve_exact_bytes(self): + with self.assertRaisesRegex(marketplace.GateError, "lockfile"): + prepare_catalog.locked_inputs(self.root) + manifest = self.root / "package.json" + lock = self.root / "pnpm-lock.yaml" + manifest.write_text(json.dumps({"volta": {"node": "24.21.0"}})) + lock.write_text("lockfileVersion: '9.0'\n") + saved = prepare_catalog.locked_inputs(self.root) + prepare_catalog.verify_locked_inputs(self.root, saved) + lock.write_text("lockfileVersion: '10.0'\n") + with self.assertRaisesRegex(marketplace.GateError, "changed"): + prepare_catalog.verify_locked_inputs(self.root, saved) + lock.unlink() + lock.symlink_to(manifest) + with self.assertRaisesRegex(marketplace.GateError, "regular"): + prepare_catalog.locked_inputs(self.root) + + def test_actual_release_archive_and_receipt_are_required(self): + fixture = Fixture(self, extension_path="catalog/terraphim-lsp") + fixture.qualify() + data = [fixture.packaged.tarball.read_bytes(), fixture.packaged.sums.read_bytes(), + fixture.receipt.read_bytes()] + result = prepare_catalog.validate_released_package( + fixture.root, fixture.proof, fixture.tag, *data) + self.assertEqual(result["archive"]["name"], fixture.packaged.tarball.name) + for index in range(3): + changed = list(data) + if index == 2: + value = json.loads(changed[index]); value["source"] = "f" * 40 + changed[index] = json.dumps(value).encode() + else: + changed[index] += b"changed" + with self.subTest(index=index), self.assertRaises(marketplace.GateError): + prepare_catalog.validate_released_package( + fixture.root, fixture.proof, fixture.tag, *changed) + + def test_runtime_declaration_change_requires_tooling_review(self): + (self.root / "package.json").write_text(json.dumps({"volta": {"node": "25.0.0"}})) + (self.root / "pnpm-lock.yaml").write_text("lockfileVersion: '9.0'\n") + with self.assertRaisesRegex(marketplace.GateError, "runtime"): + prepare_catalog.locked_inputs(self.root) + def test_unrelated_staged_changes_are_refused(self): self.prepare() (self.root / "unrelated.txt").write_text("must not submit")