diff --git a/.github/actions/uitests/action.yml b/.github/actions/uitests/action.yml index 256dea058..2f9e77a7d 100644 --- a/.github/actions/uitests/action.yml +++ b/.github/actions/uitests/action.yml @@ -169,6 +169,7 @@ runs: continue-on-error: true shell: bash env: + TEST_DESTINATION: ${{ inputs.destination }} UITEST_ONLY_TESTING: ${{ inputs.only-testing }} TEST_RUNNER_SNAPSHOT_REFERENCE_DIR: ${{ steps.reference.outputs.root }} run: | @@ -261,12 +262,16 @@ runs: done trap remove_current_lock EXIT INT TERM + test_destination="$TEST_DESTINATION" + if [[ "${{ inputs.platform }}" == "ios" ]]; then + test_destination="$(ruby ../Scripts/CI/test_simulator.rb create "${{ steps.paths.outputs.root }}/record-simulator.json" "$TEST_DESTINATION")" + fi result_bundle="${{ steps.paths.outputs.artifacts }}/SUI_${{ inputs.test-suite }}.xcresult" test_log="${{ steps.paths.outputs.artifacts }}/SUI_${{ inputs.test-suite }}.log" record_status=0 NSUnbufferedIO=YES tuist_xcodebuild "$result_bundle" test \ -scheme "SUI_${{ inputs.test-suite }}" \ - -destination "${{ inputs.destination }}" \ + -destination "$test_destination" \ -skipMacroValidation \ -skipPackagePluginValidation \ -derivedDataPath "${{ steps.paths.outputs.derived-data-path }}" \ @@ -320,6 +325,7 @@ runs: continue-on-error: true shell: bash env: + TEST_DESTINATION: ${{ inputs.destination }} UITEST_ONLY_TESTING: ${{ inputs.only-testing }} TEST_RUNNER_SNAPSHOT_REFERENCE_DIR: ${{ steps.reference.outputs.root }} run: | @@ -333,12 +339,16 @@ runs: if [[ -n "$UITEST_ONLY_TESTING" ]]; then test_arguments+=("-only-testing:$UITEST_ONLY_TESTING") fi + test_destination="$TEST_DESTINATION" + if [[ "${{ inputs.platform }}" == "ios" ]]; then + test_destination="$(ruby ../Scripts/CI/test_simulator.rb create "${{ steps.paths.outputs.root }}/verify-simulator.json" "$TEST_DESTINATION")" + fi result_bundle="${{ steps.paths.outputs.artifacts }}/OSUI_${{ inputs.test-suite }}.xcresult" test_log="${{ steps.paths.outputs.artifacts }}/OSUI_${{ inputs.test-suite }}.log" test_status=0 NSUnbufferedIO=YES tuist_xcodebuild "$result_bundle" test \ -scheme "OSUI_${{ inputs.test-suite }}" \ - -destination "${{ inputs.destination }}" \ + -destination "$test_destination" \ -skipMacroValidation \ -skipPackagePluginValidation \ -derivedDataPath "${{ steps.paths.outputs.derived-data-path }}" \ @@ -377,8 +387,21 @@ runs: path: ${{ steps.paths.outputs.artifacts }} retention-days: 7 + - name: Clean up test simulators + id: simulator-cleanup + if: always() && steps.paths.outputs.root != '' && inputs.platform == 'ios' + shell: bash + env: + TEST_ROOT: ${{ steps.paths.outputs.root }} + run: | + cleanup_status=0 + for phase in record verify; do + ruby Scripts/CI/test_simulator.rb cleanup "$TEST_ROOT/$phase-simulator.json" || cleanup_status=1 + done + exit "$cleanup_status" + - name: Clean up isolated test directories - if: always() && steps.paths.outputs.root != '' + if: always() && steps.paths.outputs.root != '' && steps.simulator-cleanup.outcome != 'failure' shell: bash env: TEST_ROOT: ${{ steps.paths.outputs.root }} diff --git a/Docs/CI/UITest.md b/Docs/CI/UITest.md index 0b0d62316..6413f321a 100644 --- a/Docs/CI/UITest.md +++ b/Docs/CI/UITest.md @@ -8,6 +8,19 @@ UI tests run only through manual dispatch or a trusted PR comment. Pushes and pu The workflow uses the same Example setup entry point as local development. Non-Compute configurations run `Example/setup.sh`, and Compute configurations run `Example/setup.sh --compute`. +Each iOS recording or verification invocation creates a new simulator with the +requested device type and OS version. The action passes its UDID to Xcode without +booting it first. UI tests retain parallel testing, so Xcode can clone a source +that has no Spotlight index protected by the host's locked keybag. Recording and +verification use separate source devices, even when they run in the same job. +Interaction tests keep their existing serial execution setting. + +An `always()` step removes the source devices and their named parallel workers +from the default and testing device sets. It does not reset shared simulators or +change data protection. If cleanup fails, the action retains the temporary test +directory and simulator ownership files for diagnosis. A stopped runner or a +force-terminated job can prevent cleanup from running. + ## Default Matrix Requests without a platform or configuration selection run the default matrix on both iOS Simulator and macOS: diff --git a/Scripts/CI/Tests/interaction-test-action-tests.mjs b/Scripts/CI/Tests/interaction-test-action-tests.mjs index 13c7bf3c4..ed4b9eabb 100644 --- a/Scripts/CI/Tests/interaction-test-action-tests.mjs +++ b/Scripts/CI/Tests/interaction-test-action-tests.mjs @@ -212,6 +212,10 @@ async function runTests(t, fixtures = {}, options = {}) { const bin = path.join(directory, 'bin'); const referenceRoot = path.join(directory, 'Persistent references'); const commandLog = path.join(directory, 'commands.jsonl'); + const simulatorLog = path.join(directory, 'simulator-commands.jsonl'); + const simulatorState = path.join(directory, 'simulators.json'); + await fs.writeFile(simulatorLog, ''); + await fs.writeFile(simulatorState, '[]'); const suite = options.suite ?? 'InteractionTests'; const platform = options.platform ?? 'ios'; const snapshotPlatform = platform === 'ios' ? 'iOS_Simulator' : 'macOS'; @@ -267,6 +271,39 @@ process.exit(fixture.status ?? (recording ? 65 : 0)); const fs = require('node:fs'); const path = require('node:path'); const args = process.argv.slice(2); +if (args[0] === 'simctl') { + fs.appendFileSync(process.env.SIMULATOR_LOG, JSON.stringify(args) + '\\n'); + const statePath = process.env.SIMULATOR_STATE; + const devices = JSON.parse(fs.readFileSync(statePath)); + const runtime = 'com.apple.CoreSimulator.SimRuntime.iOS-18-5'; + const testing = args[1] === '--set'; + const command = args[testing ? 3 : 1]; + const parameters = args.slice(testing ? 4 : 2); + if (command === 'list' && parameters[0] === 'runtimes') { + console.log(JSON.stringify({ runtimes: [{ identifier: runtime, version: '18.5', isAvailable: true, + supportedDeviceTypes: [{ name: 'iPhone 16 Pro', identifier: 'phone-type' }] }] })); + } else if (command === 'create') { + const udid = require('node:crypto').randomUUID(); + devices.push({ name: parameters[0], udid, state: 'Shutdown', testing: false }); + // Model a worker left by Xcode and an unrelated device that cleanup must preserve. + devices.push({ name: 'Clone 1 of ' + parameters[0], udid: require('node:crypto').randomUUID(), state: 'Booted', testing: true }); + devices.push({ name: 'Unrelated iPhone', udid: require('node:crypto').randomUUID(), state: 'Booted', testing: true }); + fs.writeFileSync(statePath, JSON.stringify(devices)); + if (process.env.SIMULATOR_CREATE_FAILURE === 'true') process.exit(1); + console.log(udid); + } else if (command === 'list' && parameters[0] === 'devices') { + console.log(JSON.stringify({ devices: { [runtime]: devices.filter(device => device.testing === testing) } })); + } else if (command === 'delete' || command === 'shutdown') { + const device = devices.find(device => device.udid === parameters[0] && device.testing === testing); + if (!device || device.name === 'Unrelated iPhone') process.exit(2); + if (command === 'delete') { + if (process.env.SIMULATOR_DELETE_FAILURE === 'true') process.exit(1); + devices.splice(devices.indexOf(device), 1); + } else device.state = 'Shutdown'; + fs.writeFileSync(statePath, JSON.stringify(devices)); + } else process.exit(2); + process.exit(0); +} if (args.slice(0, 3).join(' ') !== 'xcresulttool get test-results' || !['summary', 'tests'].includes(args[3])) process.exit(2); process.stdout.write(fs.readFileSync(path.join(args[args.indexOf('--path') + 1], args[3] + '.json'))); `, { mode: 0o755 }); @@ -274,7 +311,7 @@ process.stdout.write(fs.readFileSync(path.join(args[args.indexOf('--path') + 1], let inputs = { ...defaults(suite === 'InteractionTests' ? interactionAction : action), platform, - destination: platform === 'ios' ? destination : 'platform=macOS', + destination: platform === 'ios' ? (options.destination ?? destination) : 'platform=macOS', 'artifact-name': 'snapshot-tests', 'reference-root': referenceRoot, 'only-testing': options.onlyTesting ?? '', @@ -292,7 +329,7 @@ process.stdout.write(fs.readFileSync(path.join(args[args.indexOf('--path') + 1], let failed = false; const selectedSteps = new Set(['reference', 'paths', 'record-baseline', 'uitest']); for (const step of action.runs.steps) { - if (!selectedSteps.has(step.id) && !['Clean up snapshot reference lock', 'Fail if baseline recording failed'].includes(step.name) && !(options.collectArtifacts && step.name === 'Collect failure artifacts')) continue; + if (!selectedSteps.has(step.id) && !['Clean up snapshot reference lock', 'Fail if baseline recording failed', 'Clean up test simulators'].includes(step.name) && !(options.collectArtifacts && step.name === 'Collect failure artifacts')) continue; const bindings = { inputs, steps }; // Steps without always() only run while no earlier step has failed. if (failed && !/always\(\)/.test(step.if ?? '')) continue; @@ -311,6 +348,9 @@ process.stdout.write(fs.readFileSync(path.join(args[args.indexOf('--path') + 1], GITHUB_OUTPUT: output, GITHUB_RUN_ID: '123', GITHUB_JOB: 'snapshot-tests', GITHUB_RUN_ATTEMPT: '1', INTERACTION_TEST_COMMAND_LOG: commandLog, + SIMULATOR_LOG: simulatorLog, SIMULATOR_STATE: simulatorState, + SIMULATOR_CREATE_FAILURE: String(options.simulatorCreateFailure ?? false), + SIMULATOR_DELETE_FAILURE: String(options.simulatorDeleteFailure ?? false), INTERACTION_TEST_FIXTURES: JSON.stringify(fixtures), INTERACTION_TEST_RECORDING_SUMMARY: JSON.stringify(recordingSummary), SNAPSHOT_PLATFORM: snapshotPlatform, SNAPSHOT_VERSION: snapshotVersion, TEST_SUITE: suite, @@ -334,6 +374,8 @@ process.stdout.write(fs.readFileSync(path.join(args[args.indexOf('--path') + 1], const commands = (await fs.readFile(commandLog, 'utf8')).trim().split('\n').filter(Boolean).map(line => JSON.parse(line)); const verified = inputs.mode === 'record' ? steps['record-baseline'].outcome === 'success' : steps.uitest.outcome === 'success'; return { + simulatorCommands: (await fs.readFile(simulatorLog, 'utf8')).trim().split('\n').filter(Boolean).map(line => JSON.parse(line)), + remainingSimulators: JSON.parse(await fs.readFile(simulatorState, 'utf8')), status: failed || !verified ? 1 : 0, stdout, stderr, commands, steps, directory, referenceRoot, referenceDirectory, artifacts: steps.paths.outputs.artifacts, @@ -347,7 +389,7 @@ test('interaction action records SwiftUI references before OpenSwiftUI verificat await assert.rejects(fs.access(result.steps.reference.outputs.lockfile), { code: 'ENOENT' }); for (const [index, args] of result.commands.entries()) { assert.deepEqual(args.slice(0, 7), ['--env', 'compute', 'exec', '--', 'tuist', 'xcodebuild', 'test']); - assert.equal(args[args.indexOf('-destination') + 1], destination); + assert.match(args[args.indexOf('-destination') + 1], /^platform=iOS Simulator,id=[0-9a-f-]{36}$/); assert.equal(args[args.indexOf('-parallel-testing-enabled') + 1], 'NO'); assert.equal(args[args.indexOf('-derivedDataPath') + 1], result.derivedData); assert.equal(args[args.indexOf('-resultBundlePath') + 1], path.join(result.artifacts, `${schemes[index]}.xcresult`)); @@ -481,3 +523,63 @@ for (const scheme of schemes) { assert.match(await fs.readFile(path.join(result.artifacts, scheme, 'logs/test_output.log'), 'utf8'), /Executed/); }); } + +for (const mode of ['record', 'test']) { + test(`iOS UI ${mode} uses fresh destinations and cleans up only its devices`, async t => { + const result = await runTests(t, {}, { suite: 'UITests', update: true, mode }); + assert.equal(result.status, 0, result.stdout + result.stderr); + const destinations = result.commands.map(args => args[args.indexOf('-destination') + 1]); + assert.equal(new Set(destinations).size, mode === 'record' ? 1 : 2); + for (const args of result.commands) { + assert.match(args[args.indexOf('-destination') + 1], /^platform=iOS Simulator,id=[0-9a-f-]{36}$/); + assert.ok(!args.includes('-parallel-testing-enabled')); + } + assert.ok(!result.simulatorCommands.some(args => args.includes('boot') || args.includes('erase'))); + assert.ok(result.remainingSimulators.length > 0); + assert.ok(result.remainingSimulators.every(device => device.name === 'Unrelated iPhone' && device.state === 'Booted')); + await assert.rejects(fs.access(path.join(result.steps.paths.outputs.root, 'record-simulator.json')), { code: 'ENOENT' }); + }); +} + +test('simulator cleanup runs after test failure and partial creation failure', async t => { + for (const creationFails of [false, true]) { + const result = await runTests(t, { OSUI_InteractionTests: { status: 65 } }, { simulatorCreateFailure: creationFails }); + assert.equal(result.status, 1); + if (creationFails) assert.deepEqual(result.commands, []); + assert.ok(result.remainingSimulators.every(device => device.name === 'Unrelated iPhone')); + } +}); + +test('simulator cleanup reports deletion failure and retains ownership state', async t => { + const result = await runTests(t, {}, { simulatorDeleteFailure: true }); + assert.equal(result.status, 1); + assert.match(result.stderr, /Simulator cleanup failed/); + await fs.access(path.join(result.steps.paths.outputs.root, 'verify-simulator.json')); + const cleanup = action.runs.steps.find(step => step.name === 'Clean up isolated test directories'); + assert.equal(evaluate(cleanup.if, { steps: result.steps }), false); +}); + +test('macOS keeps its destination and does not manage simulators', async t => { + const result = await runTests(t, {}, { platform: 'macos' }); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.equal(result.commands[0][result.commands[0].indexOf('-destination') + 1], 'platform=macOS'); + assert.deepEqual(result.simulatorCommands, []); +}); + +test('simulator selection rejects a missing runtime or device type before running tests', async t => { + for (const destination of [ + 'platform=iOS Simulator,OS=18.6,name=iPhone 16 Pro', + 'platform=iOS Simulator,OS=18.5,name=Unknown iPhone', + ]) { + const result = await runTests(t, {}, { destination }); + assert.equal(result.status, 1); + assert.deepEqual(result.commands, []); + assert.ok(!result.simulatorCommands.some(args => args.includes('create'))); + assert.deepEqual(result.remainingSimulators, []); + } +}); + +test('simulator selection accepts an explicit zero patch version', async t => { + const result = await runTests(t, {}, { destination: 'platform=iOS Simulator,OS=18.5.0,name=iPhone 16 Pro' }); + assert.equal(result.status, 0, result.stdout + result.stderr); +}); diff --git a/Scripts/CI/test_simulator.rb b/Scripts/CI/test_simulator.rb new file mode 100644 index 000000000..f23a768d4 --- /dev/null +++ b/Scripts/CI/test_simulator.rb @@ -0,0 +1,61 @@ +#!/usr/bin/env ruby + +require 'json' +require 'open3' +require 'securerandom' + +def simctl(*arguments) + output, error, status = Open3.capture3('xcrun', 'simctl', *arguments) + abort("::error::simctl #{arguments.join(' ')} failed: #{error}") unless status.success? + output +end + +command, state_path, destination = ARGV +case command +when 'create' + abort('::error::Simulator state already exists.') if File.exist?(state_path) + fields = destination.split(',').to_h { |field| field.strip.split('=', 2) } + unless fields.keys.sort == %w[OS name platform] && fields['platform'] == 'iOS Simulator' && fields['OS'].match?(/\A\d+(?:\.\d+){1,2}\z/) + abort('::error::Expected an iOS Simulator destination with an explicit OS version and device name.') + end + version = ->(value) { value.split('.').map(&:to_i).values_at(0, 1, 2).map { |part| part || 0 } } + runtimes = JSON.parse(simctl('list', 'runtimes', '--json')).fetch('runtimes') + runtime = runtimes.find do |item| + item['isAvailable'] && item['identifier'].start_with?('com.apple.CoreSimulator.SimRuntime.iOS-') && version.call(item['version']) == version.call(fields['OS']) + end + abort("::error::No available iOS #{fields['OS']} runtime.") unless runtime + device_type = runtime.fetch('supportedDeviceTypes').find { |item| item['name'] == fields['name'] } + abort("::error::#{fields['name']} is not supported by iOS #{fields['OS']}.") unless device_type + + # Persist ownership before creation so cleanup also handles partial failures. + state = { 'name' => "OpenSwiftUI-#{SecureRandom.uuid}", 'runtime' => runtime.fetch('identifier') } + File.write(state_path, JSON.generate(state), mode: 'wx') + udid = simctl('create', state.fetch('name'), device_type.fetch('identifier'), state.fetch('runtime')).strip + abort('::error::simctl returned an invalid device UDID.') unless udid.match?(/\A[0-9A-Fa-f-]{36}\z/) + warn "Created test simulator #{state.fetch('name')} (#{udid})." + # Do not boot the source: Xcode must clone it before Spotlight creates protected data. + puts "platform=iOS Simulator,id=#{udid}" +when 'cleanup' + exit unless File.exist?(state_path) + state = JSON.parse(File.read(state_path)) + name = state.fetch('name') + abort('::error::Invalid simulator ownership state.') unless name.match?(/\AOpenSwiftUI-[0-9a-f-]{36}\z/) + owned_name = /\A(?:Clone \d+ of )?#{Regexp.escape(name)}\z/ + failures = [] + # Xcode keeps parallel workers in a separate device set. + [['--set', 'testing'], []].each do |device_set| + devices = JSON.parse(simctl(*device_set, 'list', 'devices', '--json')).fetch('devices') + devices.fetch(state.fetch('runtime'), []).each do |device| + next unless owned_name.match?(device.fetch('name')) + udid = device.fetch('udid') + Open3.capture3('xcrun', 'simctl', *device_set, 'shutdown', udid) unless device['state'] == 'Shutdown' + _, error, status = Open3.capture3('xcrun', 'simctl', *device_set, 'delete', udid) + failures << "#{udid}: #{error}" unless status.success? + warn "Removed test simulator #{device.fetch('name')} (#{udid})." if status.success? + end + end + abort("::error::Simulator cleanup failed: #{failures.join('; ')}") unless failures.empty? + File.delete(state_path) +else + abort('Usage: test_simulator.rb create|cleanup STATE_PATH [DESTINATION]') +end