diff --git a/.github/workflows/tests-pr.yml b/.github/workflows/tests-pr.yml index 880eca5f8b1..3165c915e66 100644 --- a/.github/workflows/tests-pr.yml +++ b/.github/workflows/tests-pr.yml @@ -133,6 +133,8 @@ jobs: node-version: ${{ env.DEFAULT_NODE_VERSION }} - name: Build run: pnpm build --output-style=stream + - name: Check commands snapshot + run: node bin/check-commands-snapshot.js - name: Refresh manifests run: pnpm refresh-manifests - name: Check if there are changes @@ -222,6 +224,9 @@ jobs: e2e-tests: name: "E2E tests (shard ${{ matrix.shard }})" if: github.event_name == 'merge_group' || github.event.pull_request.head.repo.full_name == github.repository + # Gate on build and type-check so a PR that doesn't compile never reaches + # Playwright — those failures belong to the build lane, not to E2E. + needs: [type-check, bundle] runs-on: ubuntu-latest timeout-minutes: 20 continue-on-error: true diff --git a/bin/check-commands-snapshot.js b/bin/check-commands-snapshot.js new file mode 100644 index 00000000000..33a334596c6 --- /dev/null +++ b/bin/check-commands-snapshot.js @@ -0,0 +1,56 @@ +import {spawnSync} from 'node:child_process' +import {mkdtempSync, readFileSync, writeFileSync} from 'node:fs' +import {tmpdir} from 'node:os' +import * as path from 'node:path' +import {fileURLToPath} from 'node:url' + +// Verifies that `shopify commands --tree` matches the committed snapshot. +// This proves every command can load, and catches command additions/removals +// that were not regenerated. It used to run inside the Playwright E2E suite, +// where its failures were misclassified as E2E flake. + +const repoRoot = path.dirname(path.dirname(fileURLToPath(import.meta.url))) +const cliBin = path.join(repoRoot, 'packages/cli/bin/run.js') +const snapshotPath = path.join(repoRoot, 'packages/e2e/data/snapshots/commands.txt') + +const errorMessage = ` +SNAPSHOT TEST FAILED! + +The result of 'shopify commands --tree' has changed! We run this to check that +all commands can load successfully. + +It's normal to see this test fail when you add or remove a command in the CLI. +In this case you can run this command to regenerate the snapshot file: + +$ pnpm test:regenerate-snapshots + +Then you can commit this change and this test will pass. + +If instead you didn't mean to change a command, UH OH. Check the commands in +the diff below and figure out what is broken. +` + +const normalize = (value) => value.replace(/\r\n/g, '\n').trimEnd() + +const env = {...process.env, FORCE_COLOR: '0'} +delete env.DEBUG + +const result = spawnSync('node', [cliBin, 'commands', '--tree'], {encoding: 'utf8', env}) + +if (result.status !== 0) { + console.error(`::error::commands --tree failed (exit ${result.status})\nstdout: ${result.stdout}\nstderr: ${result.stderr}`) + process.exit(1) +} + +const actual = normalize(result.stdout) +const expected = normalize(readFileSync(snapshotPath, 'utf8')) + +if (actual !== expected) { + console.error(errorMessage) + const actualPath = path.join(mkdtempSync(path.join(tmpdir(), 'commands-snapshot-')), 'commands.txt') + writeFileSync(actualPath, `${actual}\n`) + spawnSync('git', ['diff', '--no-index', snapshotPath, actualPath], {stdio: 'inherit'}) + process.exit(1) +} + +console.log('commands --tree matches snapshot') diff --git a/packages/e2e/tests/commands.spec.ts b/packages/e2e/tests/commands.spec.ts deleted file mode 100644 index 9171bd2ee14..00000000000 --- a/packages/e2e/tests/commands.spec.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* eslint-disable no-restricted-imports */ -import {cliFixture as test} from '../setup/cli.js' -import {expect} from '@playwright/test' -import * as fs from 'fs/promises' -import * as path from 'path' -import {fileURLToPath} from 'url' - -const __dirname = path.dirname(fileURLToPath(import.meta.url)) -const snapshotPath = path.join(__dirname, '../data/snapshots/commands.txt') - -const errorMessage = ` -SNAPSHOT TEST FAILED! - -The result of 'shopify commands --tree' has changed! We run this to check that -all commands can load successfully. - -It's normal to see this test fail when you add or remove a command in the CLI. -In this case you can run this command to regenerate the snapshot file: - -$ pnpm test:regenerate-snapshots - -Then you can commit this change and this test will pass. - -If instead you didn't mean to change a command, UH OH. Check the commands in -the diff below and figure out what is broken. -` - -const normalize = (value: string) => value.replace(/\r\n/g, '\n').trimEnd() - -test.describe('Command snapshot', () => { - test('shopify commands --tree matches snapshot', async ({cli}) => { - const result = await cli.exec(['commands', '--tree']) - expect(result.exitCode, `commands --tree failed:\nstdout: ${result.stdout}\nstderr: ${result.stderr}`).toBe(0) - - const snapshot = await fs.readFile(snapshotPath, {encoding: 'utf8'}) - expect(normalize(result.stdout), errorMessage).toBe(normalize(snapshot)) - }) -})