diff --git a/.github/workflows/configurator-release.yml b/.github/workflows/configurator-release.yml index 55fb947..c5c38a1 100644 --- a/.github/workflows/configurator-release.yml +++ b/.github/workflows/configurator-release.yml @@ -120,6 +120,8 @@ jobs: with: tofu_version: '1.12.6' tofu_wrapper: false + - name: Test isolated native Plan hold and interruption + run: LZC_TEST_TOFU_BIN="$(command -v tofu)" npx vitest run tests/worker-recovery-hold.test.ts - name: Test real OpenTofu contract and package isolated runner run: | npm run test:common-contract diff --git a/.github/workflows/validate-configurator.yml b/.github/workflows/validate-configurator.yml index dbd9d38..fadebc9 100644 --- a/.github/workflows/validate-configurator.yml +++ b/.github/workflows/validate-configurator.yml @@ -111,6 +111,8 @@ jobs: run: npm run test:common-contract - name: Test native Plan, Application and governance contracts run: LZC_TEST_TOFU_BIN="$(command -v tofu)" npm run test:plan + - name: Test isolated native Plan hold and interruption + run: LZC_TEST_TOFU_BIN="$(command -v tofu)" npx vitest run tests/worker-recovery-hold.test.ts - name: Test database migrations and tenant isolation run: npm run test:database -- --maxWorkers=2 --testTimeout=30000 env: diff --git a/landing-zone-configurator/app/apps/worker/src/observation.ts b/landing-zone-configurator/app/apps/worker/src/observation.ts index 4c47c26..c962ff5 100644 --- a/landing-zone-configurator/app/apps/worker/src/observation.ts +++ b/landing-zone-configurator/app/apps/worker/src/observation.ts @@ -71,7 +71,11 @@ export function watchPlanVersion( export function applicationPlanObservation( jobId: unknown, instanceId: unknown, - phase: "native-plan-observed" | "plan-command-finished", + phase: + | "native-plan-observed" + | "plan-command-finished" + | "native-plan-held" + | "native-plan-released", ) { const job = applicationPlanJobSchema.shape.id.safeParse(jobId); const instance = diff --git a/landing-zone-configurator/app/apps/worker/src/recovery-hold.ts b/landing-zone-configurator/app/apps/worker/src/recovery-hold.ts new file mode 100644 index 0000000..eec7b13 --- /dev/null +++ b/landing-zone-configurator/app/apps/worker/src/recovery-hold.ts @@ -0,0 +1,156 @@ +import { readFile } from "node:fs/promises"; +import { basename } from "node:path"; +import { z } from "zod"; + +export const devPlanHoldMilliseconds = 30_000; +const api = "https://api.system.01.cf.eu01.stackit.cloud"; +const origin = "https://lzc-dev-configurator.apps.01.cf.eu01.stackit.cloud"; +const configuration = z.strictObject({ + version: z.literal(1), + instanceId: z.uuid(), + appId: z.uuid(), + spaceId: z.uuid(), + expiresAt: z.iso.datetime(), +}); +const context = z.object({ + cf_api: z.literal(api), + organization_name: z.literal("lzc-dev-runners"), + space_name: z.literal("plans"), + application_id: z.uuid(), + application_name: z.string().regex(/^lzc-warm-[a-f0-9-]{36}$/), + space_id: z.uuid(), + process_type: z.literal("web"), + application_uris: z.array(z.string()).length(0), +}); + +// Only an operator-prepared, one-shot Dev warm app can opt into this drill. +export function devPlanHoldConfiguration( + job: { + mode: unknown; + instanceId?: string | undefined; + purpose?: string | undefined; + origin?: string | undefined; + local?: boolean | undefined; + }, + env: NodeJS.ProcessEnv = process.env, + now = Date.now(), +) { + if ( + job.mode !== "application-plan" || + job.local || + job.origin !== origin || + (job.purpose !== undefined && job.purpose !== "standard") || + !env.LZC_RECOVERY_PLAN_HOLD || + !z.uuid().safeParse(env.CF_INSTANCE_GUID).success + ) + return undefined; + try { + const config = configuration.parse(JSON.parse(env.LZC_RECOVERY_PLAN_HOLD)); + const cf = context.parse(JSON.parse(env.VCAP_APPLICATION ?? "")); + const expiresAt = Date.parse(config.expiresAt); + if ( + config.instanceId !== job.instanceId || + cf.application_id !== config.appId || + cf.space_id !== config.spaceId || + expiresAt <= now + devPlanHoldMilliseconds || + expiresAt > now + 15 * 60_000 + ) + return undefined; + return config; + } catch { + return undefined; + } +} + +async function stoppedNativePlan(group: number) { + const children = await readFile( + `/proc/${group}/task/${group}/children`, + "utf8", + ).catch(() => ""); + for (const pid of [ + group, + ...children.trim().split(/\s+/).filter(Boolean).map(Number), + ]) { + const [stat, command] = await Promise.all([ + readFile(`/proc/${pid}/stat`, "utf8").catch(() => ""), + readFile(`/proc/${pid}/cmdline`, "utf8").catch(() => ""), + ]); + const fields = stat.slice(stat.lastIndexOf(") ") + 2).split(" "); + const args = command.split("\0"); + if ( + fields[0] === "T" && + Number(fields[2]) === group && + basename(args[0] ?? "") === "tofu" && + args[1] === "plan" + ) + return true; + } + return false; +} + +export function createDevPlanHold( + enabled: ReturnType, + observation: (phase: "native-plan-held" | "native-plan-released") => void, +) { + let attempted = false; + let pid: number | undefined; + let confirmed = false; + let timer: ReturnType | undefined; + const release = () => { + if (timer) clearTimeout(timer); + timer = undefined; + if (pid === undefined) return; + const group = pid; + pid = undefined; + try { + process.kill(-group, "SIGCONT"); + } catch { + /* The child may already be gone. */ + } + if (confirmed) observation("native-plan-released"); + }; + return { + hold(childPid: number | undefined, deadline: number) { + if ( + !enabled || + attempted || + !childPid || + !Number.isInteger(childPid) || + childPid <= 1 || + Date.parse(enabled.expiresAt) <= Date.now() + devPlanHoldMilliseconds || + deadline <= Date.now() + devPlanHoldMilliseconds + 10_000 + ) + return; + attempted = true; + try { + process.kill(-childPid, "SIGSTOP"); + pid = childPid; + timer = setTimeout(release, devPlanHoldMilliseconds); + } catch { + return; + } + // Confirm both the shell leader and its native Plan are stopped, not merely a finished shell. + void (async () => { + for (let attempt = 0; attempt < 50 && pid === childPid; attempt++) { + const status = await readFile( + `/proc/${childPid}/status`, + "utf8", + ).catch(() => ""); + if (pid !== childPid) return; + if ( + /^State:\s+T\b/m.test(status) && + (await stoppedNativePlan(childPid)) && + pid === childPid + ) { + confirmed = true; + observation("native-plan-held"); + return; + } + await new Promise((done) => setTimeout(done, 10)); + } + release(); + })().catch(release); + }, + close: release, + }; +} diff --git a/landing-zone-configurator/app/apps/worker/src/runner.ts b/landing-zone-configurator/app/apps/worker/src/runner.ts index 4ffd6b1..64682c8 100644 --- a/landing-zone-configurator/app/apps/worker/src/runner.ts +++ b/landing-zone-configurator/app/apps/worker/src/runner.ts @@ -30,6 +30,10 @@ import { summarizePlan, } from "@lzc/contracts"; import { applicationPlanObservation, watchPlanVersion } from "./observation.js"; +import { + createDevPlanHold, + devPlanHoldConfiguration, +} from "./recovery-hold.js"; export const brokerOrigin = "https://lzc-dev-configurator.apps.01.cf.eu01.stackit.cloud"; @@ -444,8 +448,13 @@ export async function runWorker( ); let activeCommand: string | undefined; let planObserved = false; + let holdNativePlan: (() => void) | undefined; const observePlan = ( - phase: "native-plan-observed" | "plan-command-finished", + phase: + | "native-plan-observed" + | "plan-command-finished" + | "native-plan-held" + | "native-plan-released", ) => { if ( mode !== "application-plan" || @@ -545,6 +554,7 @@ export async function runWorker( ) { planObserved = true; observePlan("native-plan-observed"); + holdNativePlan?.(); } content = progress.text; if (progress.invalid && !invalidProgressReported) { @@ -610,6 +620,18 @@ export async function runWorker( outputTimer = setTimeout(pollOutput, 500); try { activeCommand = phase; + const recoveryHold = createDevPlanHold( + phase === "planning" + ? devPlanHoldConfiguration({ + mode, + instanceId: applicationBinding?.instanceId, + purpose: applicationBinding?.purpose, + origin: options.brokerOrigin, + local: options.local, + }) + : undefined, + observePlan, + ); const stopObservation = phase === "planning" && mode === "application-plan" ? watchPlanVersion( @@ -618,6 +640,7 @@ export async function runWorker( () => { planObserved = true; observePlan("native-plan-observed"); + holdNativePlan?.(); }, ) : () => {}; @@ -645,6 +668,7 @@ export async function runWorker( detached: true, }, ); + holdNativePlan = () => recoveryHold.hold(child.pid, deadline); let killTimer: ReturnType | undefined; const stop = (signal: NodeJS.Signals) => { try { @@ -680,6 +704,8 @@ export async function runWorker( }); } finally { stopObservation(); + holdNativePlan = undefined; + recoveryHold.close(); } } finally { activeCommand = undefined; diff --git a/landing-zone-configurator/app/integration/runner-pool.test.ts b/landing-zone-configurator/app/integration/runner-pool.test.ts index fadcb1c..6e86f2a 100644 --- a/landing-zone-configurator/app/integration/runner-pool.test.ts +++ b/landing-zone-configurator/app/integration/runner-pool.test.ts @@ -1,4 +1,5 @@ import { randomBytes, randomUUID } from "node:crypto"; +import { readFile } from "node:fs/promises"; import { fileURLToPath } from "node:url"; import pg from "pg"; import { afterAll, beforeAll, beforeEach, expect, it, vi } from "vitest"; @@ -518,3 +519,55 @@ it("hands a real database lease through the HTTP broker to exactly one waiting w await app.close(); } }); + +it("atomically rewarms only a fresh unassigned exact-source lease and rotates its capability", async () => { + const pool = create(); + const slot = await ready(pool); + const sql = await readFile( + new URL("../../infra/recovery-lab/hold_lease.sql", import.meta.url), + "utf8", + ); + const ticket = randomBytes(32).toString("base64url"); + const { createHash } = await import("node:crypto"); + const hash = createHash("sha256").update(ticket).digest("hex"); + for (const [space, source] of [ + [randomUUID(), config.dropletId], + [config.spaceId, randomUUID()], + ]) + expect( + (await database.query(sql, [space, config.origin, source, hash])) + .rowCount, + ).toBe(0); + const leased = await database.query(sql, [ + config.spaceId, + config.origin, + config.dropletId, + hash, + ]); + expect(leased.rows).toEqual([ + { id: slot.id, app_id: slot.appId, source_droplet_id: config.dropletId }, + ]); + await expect(pool.claim(slot.id, slot.ticket)).rejects.toThrow(); + await pool.maintain(); + expect(driver.prepareWarm).toHaveBeenCalledTimes(1); + expect(await pool.claim(slot.id, ticket)).toBeNull(); + const assignment = job(); + await pool.start( + assignment.id, + assignment.ticket, + config.origin, + "application", + async () => {}, + ); + expect( + ( + await database.query(sql, [ + config.spaceId, + config.origin, + config.dropletId, + hash, + ]) + ).rowCount, + ).toBe(0); + expect((await pool.claim(slot.id, ticket))?.id).toBe(assignment.id); +}); diff --git a/landing-zone-configurator/app/tests/worker-recovery-hold.test.ts b/landing-zone-configurator/app/tests/worker-recovery-hold.test.ts new file mode 100644 index 0000000..a6b6eaf --- /dev/null +++ b/landing-zone-configurator/app/tests/worker-recovery-hold.test.ts @@ -0,0 +1,295 @@ +import { execFile, spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, it, vi } from "vitest"; +import { + createDevPlanHold, + devPlanHoldConfiguration, + devPlanHoldMilliseconds, +} from "../apps/worker/src/recovery-hold.js"; + +const instanceId = randomUUID(); +const appId = randomUUID(); +const spaceId = randomUUID(); +const job = { + mode: "application-plan", + instanceId, + origin: "https://lzc-dev-configurator.apps.01.cf.eu01.stackit.cloud", +}; +const make = () => ({ + CF_INSTANCE_GUID: randomUUID(), + LZC_RECOVERY_PLAN_HOLD: JSON.stringify({ + version: 1, + instanceId, + appId, + spaceId, + expiresAt: new Date(Date.now() + 600_000).toISOString(), + }), + VCAP_APPLICATION: JSON.stringify({ + cf_api: "https://api.system.01.cf.eu01.stackit.cloud", + organization_name: "lzc-dev-runners", + space_name: "plans", + application_id: appId, + application_name: `lzc-warm-${randomUUID()}`, + space_id: spaceId, + process_type: "web", + application_uris: [], + }), +}); +afterEach(() => vi.restoreAllMocks()); +it("requires explicit short-lived exact-instance opt-in in the route-free Dev warm app", () => { + const env = make(); + expect(devPlanHoldConfiguration(job, env)?.instanceId).toBe(instanceId); + expect(devPlanHoldConfiguration(job, {})).toBeUndefined(); + for (const changes of [ + { mode: "application-apply" }, + { mode: "platform-plan" }, + { local: true }, + { purpose: "drift" }, + { purpose: "destroy" }, + { instanceId: randomUUID() }, + { origin: "https://production.example" }, + ]) + expect( + devPlanHoldConfiguration({ ...job, ...changes }, env), + ).toBeUndefined(); + for (const key of [ + "CF_INSTANCE_GUID", + "LZC_RECOVERY_PLAN_HOLD", + "VCAP_APPLICATION", + ]) + expect( + devPlanHoldConfiguration(job, { ...env, [key]: "invalid" }), + ).toBeUndefined(); + for (const changes of [ + { organization_name: "production" }, + { space_name: "production" }, + { application_id: randomUUID() }, + { space_id: randomUUID() }, + { application_name: "lzc-plan-template" }, + { process_type: "task" }, + { application_uris: ["public.example"] }, + ]) + expect( + devPlanHoldConfiguration(job, { + ...env, + VCAP_APPLICATION: JSON.stringify({ + ...JSON.parse(env.VCAP_APPLICATION), + ...changes, + }), + }), + ).toBeUndefined(); + for (const changes of [ + { expiresAt: new Date(Date.now() - 1).toISOString() }, + { expiresAt: new Date(Date.now() + 3_600_000).toISOString() }, + { command: "arbitrary" }, + { instanceId: randomUUID() }, + ]) + expect( + devPlanHoldConfiguration(job, { + ...env, + LZC_RECOVERY_PLAN_HOLD: JSON.stringify({ + ...JSON.parse(env.LZC_RECOVERY_PLAN_HOLD), + ...changes, + }), + }), + ).toBeUndefined(); +}); +it("does not signal any process without opt-in, a safe child PID and remaining deadline", () => { + const kill = vi.spyOn(process, "kill"); + const disabled = createDevPlanHold(undefined, vi.fn()); + disabled.hold(999999, Date.now() + 60_000); + disabled.close(); + const hold = createDevPlanHold( + devPlanHoldConfiguration(job, make()), + vi.fn(), + ); + for (const pid of [undefined, 0, -1, 1, 1.5]) + hold.hold(pid, Date.now() + 60_000); + hold.hold(999999, Date.now() + 1000); + hold.close(); + expect(kill).not.toHaveBeenCalled(); +}); +it("does not claim a native Plan hold for a different stopped process", async () => { + const child = spawn("/bin/sleep", ["60"], { + detached: true, + stdio: "ignore", + }); + const exited = new Promise((done) => child.once("exit", () => done())); + await new Promise((done, fail) => { + child.once("spawn", done); + child.once("error", fail); + }); + const emit = vi.fn(); + const hold = createDevPlanHold(devPlanHoldConfiguration(job, make()), emit); + try { + hold.hold(child.pid, Date.now() + 60_000); + await vi.waitFor( + async () => { + expect(await readFile(`/proc/${child.pid}/status`, "utf8")).not.toMatch( + /^State:\s+T\b/m, + ); + }, + { timeout: 2000 }, + ); + expect(emit).not.toHaveBeenCalledWith("native-plan-held"); + } finally { + hold.close(); + if (child.pid) process.kill(-child.pid, "SIGKILL"); + await exited; + } +}); + +it + .skipIf(!process.env.LZC_TEST_TOFU_BIN) + .each(["resume", "kill", "auto"] as const)( + "holds a real pinned OpenTofu Plan with synthetic loopback State (%s)", + async (outcome) => { + const binary = process.env.LZC_TEST_TOFU_BIN as string; + const realTimeout = setTimeout; + const timer = vi + .spyOn(globalThis, "setTimeout") + .mockImplementation(((fn: () => void, ms?: number, ...args: unknown[]) => + realTimeout( + fn, + outcome === "auto" && ms === devPlanHoldMilliseconds ? 800 : ms, + ...args, + )) as typeof setTimeout); + const root = await mkdtemp(join(tmpdir(), "lzc-native-plan-hold-")); + let released = false; + const waiting: (() => void)[] = []; + const server = createServer((_req, res) => { + const send = () => { + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + version: 4, + terraform_version: "1.12.6", + serial: 1, + lineage: randomUUID(), + outputs: { marker: { value: "synthetic", type: "string" } }, + resources: [], + }), + ); + }; + if (released) send(); + else waiting.push(send); + }); + await new Promise((done) => server.listen(0, "127.0.0.1", done)); + const address = server.address(); + if (!address || typeof address === "string") + throw Error("fixture address missing"); + const env = { + PATH: "/usr/bin:/bin", + HOME: root, + TF_IN_AUTOMATION: "true", + CHECKPOINT_DISABLE: "1", + TF_CLI_CONFIG_FILE: join(root, "fixture.tfrc"), + }; + let child: ReturnType | undefined; + let exit: Promise | undefined; + const emit = vi.fn(); + const hold = createDevPlanHold(devPlanHoldConfiguration(job, make()), emit); + try { + await writeFile( + join(root, "fixture.tfrc"), + "disable_checkpoint = true\n", + ); + await writeFile( + join(root, "main.tf"), + `data "terraform_remote_state" "fixture" {\n backend = "http"\n config = { address = "http://127.0.0.1:${address.port}/state" }\n}\noutput "marker" { value = data.terraform_remote_state.fixture.outputs.marker }\n`, + ); + const version = await promisify(execFile)(binary, ["version", "-json"], { + env, + cwd: root, + }); + expect(JSON.parse(version.stdout).terraform_version).toBe("1.12.6"); + await promisify(execFile)( + binary, + ["init", "-backend=false", "-input=false", "-no-color"], + { env, cwd: root }, + ); + child = spawn( + "/bin/bash", + [ + "-c", + '"$1" plan -json -input=false -out=fixture.plan; result=$?; exit "$result"', + "fixture", + binary, + ], + { env, cwd: root, detached: true, stdio: ["ignore", "pipe", "ignore"] }, + ); + const running = child; + exit = new Promise((done, fail) => { + running.once("exit", done); + running.once("error", fail); + }); + let output = ""; + running.stdout?.on("data", (chunk) => { + output += chunk.toString(); + if ( + output.includes('"type":"version"') || + output.includes('"type": "version"') + ) + hold.hold(running.pid, Date.now() + 60_000); + }); + await vi.waitFor(() => + expect(emit).toHaveBeenCalledWith("native-plan-held"), + ); + expect(await readFile(`/proc/${running.pid}/status`, "utf8")).toMatch( + /^State:\s+T\b/m, + ); + expect(running.exitCode).toBeNull(); + if (outcome === "kill") { + if (!running.pid) throw Error("fixture PID missing"); + process.kill(-running.pid, "SIGKILL"); + expect(await exit).toBeNull(); + expect(running.signalCode).toBe("SIGKILL"); + await expect( + promisify(execFile)(binary, ["show", "-json", "fixture.plan"], { + env, + cwd: root, + }), + ).rejects.toBeDefined(); + return; + } + released = true; + waiting.splice(0).forEach((send) => { + send(); + }); + if (outcome !== "auto") hold.close(); + await vi.waitFor( + () => expect(emit).toHaveBeenCalledWith("native-plan-released"), + { timeout: 2000 }, + ); + expect(timer.mock.calls.some((x) => x[1] === 30_000)).toBe(true); + expect(await exit).toBe(0); + const shown = await promisify(execFile)( + binary, + ["show", "-json", "fixture.plan"], + { env, cwd: root }, + ); + expect(JSON.parse(shown.stdout).planned_values.outputs.marker.value).toBe( + "synthetic", + ); + } finally { + hold.close(); + released = true; + waiting.splice(0).forEach((send) => { + send(); + }); + if (child?.pid && child.exitCode === null) + try { + process.kill(-child.pid, "SIGKILL"); + } catch {} + await exit?.catch(() => {}); + await new Promise((done) => server.close(() => done())); + await rm(root, { recursive: true, force: true }); + } + }, + 15_000, +); diff --git a/landing-zone-configurator/infra/recovery-lab/README.de.md b/landing-zone-configurator/infra/recovery-lab/README.de.md index 17e72ad..02ff3e2 100644 --- a/landing-zone-configurator/infra/recovery-lab/README.de.md +++ b/landing-zone-configurator/infra/recovery-lab/README.de.md @@ -187,6 +187,95 @@ verarbeitete. Einen abgeschlossenen Plan oder eine bloße Stop-Anforderung niema als erfolgreichen Abbruchtest zählen. Apply-/Checkpoint-Tests benötigen einen separaten Controller und werden mit diesem Tool nicht aktiviert. +## Expliziter nativer Plan-Wartepunkt nur für Dev + +Für Plans, die schneller als Log-Zustellung und CF-Abbruch sind, bereitet +`held_plan.py` einen entbehrlichen warmen Runner mit begrenztem Prozess-Wartepunkt +vor. Dies ist ein instrumentierter Abbruchtest, keine Messung normaler +Plan-Laufzeiten. Er umgeht keine UI/API-Ausführungsberechtigung und verändert +weder native Eingaben noch Backend oder gespeichertes Planformat. Normale Jobs +haben keine Wartekonfiguration und laufen unverändert. + +Der Worker akzeptiert `LZC_RECOVERY_PLAN_HOLD` ausschließlich in der festen +Dev-Foundation, im Web-Prozess von `lzc-dev-runners/plans`, ohne öffentliche +Routen, für die exakte CF-App, den Space und die Testinstanz sowie mit einem +Ablauf innerhalb von 15 Minuten. Apply, Plattform, Drift, Destroy, lokale +Ausführung und andere Instanzen können den Wartepunkt nicht aktivieren. Beim +ersten nativen Versionsereignis sendet er `SIGSTOP` nur an seine eigene gestartete +Prozessgruppe, prüft den von Linux gemeldeten Halt sowohl des Shell-Gruppenleiters +als auch seines echten nativen `tofu plan` in derselben Gruppe und meldet +`native-plan-held`. Nach 30 Sekunden oder beim Aufräumen des Befehls setzt +er automatisch fort. Frei einstellbare Verzögerungen oder Shell-Befehle werden +nicht angenommen. Siehe die offiziellen +[CF-Laufzeitmetadaten](https://docs.cloudfoundry.org/devguide/deploy-apps/environment-variable.html). + +Die Vorbereitung benötigt zwei getrennte private CF-Anmeldungen: den bestehenden +begrenzten Runner-Zugang und den Dev-App-Manager mit Ziel `lzc-dev/configurator`. +Eine eigene private JSON-Datei mit Modus `0600` enthält ausschließlich +`LZC_DATABASE_HOST`, `LZC_DATABASE_PORT`, `LZC_DATABASE_NAME`, `LZC_DATABASE_USER` +und `LZC_DATABASE_PASSWORD` der bestehenden Dev-Migrationsidentität. Das Werkzeug +prüft das Ziel gegen die laufende API. Zugangsdaten gehören nicht in Chat, +Issues oder Shell-Historie. Beide CF_HOME-Verzeichnisse müssen dem Operator +gehören und privat sein (`0700`). + +```sh +python3 landing-zone-configurator/infra/recovery-lab/held_plan.py \ + --cf /absolute/path/to/cf --cf-home /private/runner-cf-home \ + --app-cf-home /private/app-manager-cf-home \ + --migration-vars /private/dev-migration.json \ + --runner-space --runner-template \ + --instance \ + --confirm-disposable-hold \ + --receipt /private/drill-evidence/new-preparation.json +``` + +Das Werkzeug prüft zuerst die installierte Wartepunkt-Unterstützung. Eine +temporäre Task-App ohne Route nutzt die Migrationsidentität, um atomar einen +frischen, unzugeordneten, bereiten Pool-Slot der exakten Quelle zu reservieren +und seinen Ticket-Hash zu wechseln. Der alte Worker kann ihn nicht erneut +beanspruchen. Erst dann stoppt es diese unzugeordnete App, ergänzt das neue Ticket +und die ablaufende Wartekonfiguration in ihrer Umgebung und startet sie. Bevor es +Erfolg meldet, muss der normale Broker mit dem neuen Ticket Bereitschaft +bestätigen. CF-Umgebungsänderungen gelten nach dem Neustart; die übrige Umgebung +bleibt durch die +[CAPI-Zusammenführung](https://v3-apidocs.cloudfoundry.org/version/3.226.0/index.html#update-environment-variables) +erhalten. Die temporäre privilegierte Task-App wird beim Verlassen gelöscht. +Fehlende Bereitschaft bleibt ungeklärt; die zweiminütige Warming-Lease läuft +über die normale Wartung ab. Nach einem unklaren Nachweis keine Mutation +wiederholen: zuerst aufgezeichnete Phase und exakte App/Lease prüfen. +Ein Vorbereitungsnachweis allein autorisiert keinen Plan. + +Nach erfolgreicher Vorbereitung den Beobachter vor dem Plan scharfschalten: + +```sh +python3 landing-zone-configurator/infra/recovery-lab/hosted_plan.py \ + --cf /absolute/path/to/cf --cf-home /private/runner-cf-home \ + --runner-space --runner-template \ + --instance --require-held-plan \ + --confirm-disposable-stop \ + --receipt /private/drill-evidence/new-stop.json +``` + +Auf `observer-ready` warten, dann genau einen neuen Plan über die authentifizierte +UI starten. Andere Jobs sollten den vorbereiteten Standby nicht belegen: Eine +fremde Instanz läuft normal; danach muss dieser Test erneut vorbereitet werden. +Der Held-Modus ignoriert gewöhnliche native Startmeldungen, akzeptiert einen +bestätigten Halt höchstens 15 Sekunden lang und verweigert freigegebene oder +beendete Plans. Ein verpasstes Zeitfenster bleibt ungeklärt. Vor einer +Recovery-Qualifikation tatsächlichen Prozessverlust, dauerhaften Jobstatus, +State-/Lock-Zustand und einen frischen Plan derselben Bestellung getrennt prüfen. +Ein Halt beim Versionsereignis beweist insbesondere weder Backend-Sperre noch +Wiederherstellung einer verwaisten Sperre. Darauf niemals ein Force-Unlock oder +die Wiederholung eines unklaren Apply stützen. + +Lokale Tests verwenden das echte festgelegte OpenTofu mit einer synthetischen +Loopback-State-Lesebarriere, um bestätigten Prozesshalt, automatisches/manuelles +Fortsetzen und harten Abbruch ohne Cloud-Zugangsdaten nachzuweisen. Release- und +Validierungs-CI führen sie explizit mit `LZC_TEST_TOFU_BIN` aus; normale Unit-Läufe +überspringen die nativen Fälle ohne diese Variable. PostgreSQL-Integrationstests +prüfen das echte Lease-SQL, den Ticketwechsel sowie die Ablehnung zugeordneter +Slots und fremder Quellen. + ## CI Der Job `Isolated recovery lab` in **Validate Configurator** führt Isolationstests, diff --git a/landing-zone-configurator/infra/recovery-lab/README.md b/landing-zone-configurator/infra/recovery-lab/README.md index e2f98a1..5e7f0e7 100644 --- a/landing-zone-configurator/infra/recovery-lab/README.md +++ b/landing-zone-configurator/infra/recovery-lab/README.md @@ -171,6 +171,85 @@ engine was still running when CF processed cancellation. Do not count a complete Plan or a mere stop request as a successful interruption. Apply/checkpoint drills need a separate controller and are not enabled by this tool. +## Explicit Dev-only native Plan hold + +For Plans too fast for log delivery and CF cancellation, `held_plan.py` prepares +one disposable warm Runner with a bounded native-process hold. This is an +instrumented interruption drill, not a measurement of ordinary Plan latency. +It adds no UI/API execution bypass and changes no native inputs, backend or saved +Plan format. Ordinary jobs have no hold configuration and run unchanged. + +The worker accepts `LZC_RECOVERY_PLAN_HOLD` only in the fixed Dev foundation, +`lzc-dev-runners/plans` web process, with no public routes, the exact CF app/space +and disposable instance, and an expiry within 15 minutes. Apply, platform, drift, +destroy, local execution and foreign instances cannot activate it. At the first +native version observation it sends `SIGSTOP` only to its spawned process group, +checks Linux reports both the shell leader and its actual native `tofu plan` +process stopped in that group, and emits `native-plan-held`. +It resumes automatically after 30 seconds or when command cleanup runs. No +operator-set arbitrary delay or shell command is accepted. See the official +[CF runtime metadata](https://docs.cloudfoundry.org/devguide/deploy-apps/environment-variable.html). + +Preparation requires two separate private CF logins: the existing confined Runner +login and the Dev application manager login targeted at `lzc-dev/configurator`. +Provide a private, owned `0600` JSON file with only `LZC_DATABASE_HOST`, +`LZC_DATABASE_PORT`, `LZC_DATABASE_NAME`, `LZC_DATABASE_USER` and +`LZC_DATABASE_PASSWORD` for the existing Dev migration identity. The tool checks +the target against the running API; do not retrieve or paste credentials in chat, +issues or shell history. Both CF_HOME directories must be owned/private (`0700`). + +```sh +python3 landing-zone-configurator/infra/recovery-lab/held_plan.py \ + --cf /absolute/path/to/cf --cf-home /private/runner-cf-home \ + --app-cf-home /private/app-manager-cf-home \ + --migration-vars /private/dev-migration.json \ + --runner-space --runner-template \ + --instance \ + --confirm-disposable-hold \ + --receipt /private/drill-evidence/new-preparation.json +``` + +The tool first verifies deployed hold support. A temporary route-free task app +uses the migration identity to atomically lease one fresh, unassigned, exact-source +ready pool slot and rotate its ticket hash. The old worker cannot reclaim it. +Only then does it stop that unassigned app, merge the new ticket and expiring hold +configuration into its environment, and start it. It requires the normal broker to +confirm readiness with the new ticket before reporting success. CF environment +changes take effect after restart; the existing environment is retained through +[the CAPI merge operation](https://v3-apidocs.cloudfoundry.org/version/3.226.0/index.html#update-environment-variables). +The temporary privileged task app is deleted on exit. Failed readiness is +inconclusive; the two-minute warming lease expires through normal maintenance. +Do not replay a mutation after an ambiguous receipt. Inspect its recorded phase +and exact app/lease first. A preparation receipt alone never authorizes a Plan. + +After successful preparation, arm the observer before starting a Plan: + +```sh +python3 landing-zone-configurator/infra/recovery-lab/hosted_plan.py \ + --cf /absolute/path/to/cf --cf-home /private/runner-cf-home \ + --runner-space --runner-template \ + --instance --require-held-plan \ + --confirm-disposable-stop \ + --receipt /private/drill-evidence/new-stop.json +``` + +Wait for `observer-ready`, then start exactly one new Plan through the authenticated +UI. Avoid unrelated jobs consuming the prepared standby: a foreign instance runs +normally, and this drill must then be prepared again. Held-only observation ignores +an ordinary native-start event, accepts a confirmed hold no older than 15 seconds, +and refuses a released/finished Plan. A missed window remains inconclusive. +Verify actual process loss, durable job status, State/lock disposition and a fresh +Plan on the same order separately before qualifying recovery. In particular, a +hold at the version event does not prove backend lock acquisition or orphan-lock +recovery. Never force-unlock or retry an ambiguous Apply based on this test. + +Local tests use the real pinned OpenTofu with a synthetic loopback State-read +barrier to prove stopped-process confirmation, automatic/manual continuation and +hard interruption without cloud credentials. Both release and validation CI run +them explicitly with `LZC_TEST_TOFU_BIN`; ordinary unit runs skip native cases when +that variable is absent. PostgreSQL integration tests exercise the actual lease +SQL, ticket rotation and refusal of assigned or foreign-source slots. + ## CI The `Isolated recovery lab` job in **Validate Configurator** runs the isolation diff --git a/landing-zone-configurator/infra/recovery-lab/held_plan.py b/landing-zone-configurator/infra/recovery-lab/held_plan.py new file mode 100644 index 0000000..8ed377a --- /dev/null +++ b/landing-zone-configurator/infra/recovery-lab/held_plan.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Operator-only preparation of one expiring native Plan hold in isolated Dev. + +No customer job creation, Apply, State access or recovery clearance. +""" +import argparse +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import secrets +import shlex +import subprocess +import tempfile +import time +import uuid + +from hosted_plan import API, Controller, guid +ORIGIN = "https://lzc-dev-configurator.apps.01.cf.eu01.stackit.cloud" + + +def private_file(path): + path = Path(path) + if path.is_symlink() or not path.is_file() or path.stat().st_uid != os.getuid() or path.stat().st_mode & 0o077: + raise ValueError("Owned private file required") + return path + + +class Client: + def __init__(self, cf, home): + home = Path(home) + if home.is_symlink() or not home.is_dir() or home.stat().st_uid != os.getuid() or home.stat().st_mode & 0o077: + raise ValueError("Owned private CF_HOME required") + if (home / ".cf").is_symlink(): raise ValueError("Symlinked CF configuration directory") + config = json.loads(private_file(home / ".cf" / "config.json").read_text()) + if config.get("Target") != API: + raise ValueError("Unexpected CF foundation") + self.cf = str(Path(cf).resolve(strict=True)) + self.env = {"PATH": "/usr/bin:/bin", "LANG": "C.UTF-8", "CF_HOME": str(home.resolve())} + self.space = guid(config["SpaceFields"]["GUID"]) + + def call(self, args, timeout=180): + result = subprocess.run([self.cf, *args], env=self.env, capture_output=True, timeout=timeout) + if result.returncode: + raise RuntimeError("Scoped CF operation failed; private response withheld") + return result.stdout.decode() + + def api(self, path, method="GET", body=None): + with tempfile.TemporaryDirectory(prefix="lzc-drill-request-") as folder: + args = ["curl", path, "--fail"] + if method != "GET": + request = Path(folder) / "body.json" + request.write_text(json.dumps({} if body is None else body)) + request.chmod(0o600) + args += ["-X", method, "-d", "@" + str(request)] + raw = self.call(args) + return json.loads(raw) if raw.strip() else {} + + def boundary(self, organization, name): + space = self.api("/v3/spaces/" + self.space) + org = self.api("/v3/organizations/" + guid(space["relationships"]["organization"]["data"]["guid"])) + if space["name"] != name or org["name"] != organization: + raise ValueError("Unexpected Dev organization/space") + + +class OperatorTask: + def __init__(self, client, app, variables): + self.client, self.source, self.variables = client, app, variables + self.id = None + self.name = "lzc-recovery-drill-operator-" + uuid.uuid4().hex[:12] + + def __enter__(self): + try: + droplet = self.client.api(f"/v3/apps/{self.source}/droplets/current") + if droplet["state"] != "STAGED": raise RuntimeError("API source not staged") + created = self.client.api("/v3/apps", "POST", { + "name": self.name, + "lifecycle": {"type": "buildpack", "data": {"buildpacks": ["binary_buildpack"], "stack": "cflinuxfs4"}}, + "relationships": {"space": {"data": {"guid": self.client.space}}}, + "environment_variables": self.variables, + }) + self.id = guid(created["guid"]) + copy = self.client.api("/v3/droplets?source_guid=" + guid(droplet["guid"]), "POST", {"relationships": {"app": {"data": {"guid": self.id}}}}) + copied = guid(copy["guid"]) + for _ in range(90): + state = self.client.api("/v3/droplets/" + copied)["state"] + if state == "STAGED": break + if state != "COPYING": raise RuntimeError("Operator droplet copy failed") + time.sleep(1) + else: raise RuntimeError("Operator droplet copy timed out") + self.client.api(f"/v3/apps/{self.id}/relationships/current_droplet", "PATCH", {"data": {"guid": copied}}) + return self + except BaseException: + self.close() + raise + + def close(self): + if self.id: + self.client.api("/v3/apps/" + self.id, "DELETE") + self.id = None + + def __exit__(self, *args): self.close() + + def query(self, sql, values, wait_ready=False): + query = "await c.query(" + json.dumps(sql) + "," + json.dumps(values) + ")" + polling = "for(let i=0;i<120&&r.rows[0]?.status==='warming';i++){await new Promise(done=>setTimeout(done,1000));r=" + query + ";}" if wait_ready else "" + script = 'import pg from "pg";import {databaseConfig} from "./apps/api/dist/storage/database.js";const c=new pg.Client(databaseConfig());await c.connect();try{let r=' + query + ';' + polling + 'console.log("DRILL_QUERY="+JSON.stringify(r.rows));}finally{await c.end();}' + task = "q-" + uuid.uuid4().hex[:12] + command = "./runtime/bin/node --input-type=module -e " + shlex.quote(script) + self.client.call(["run-task", self.name, "--command", command, "--name", task, "-m", "128M", "-k", "6144M", "--wait"]) + # Poll only logs: the SQL task, including lease acquisition, must never be replayed. + for _ in range(10): + logs = self.client.call(["logs", self.name, "--recent"]) + hits = [] + for line in logs.splitlines(): + if f"[APP/TASK/{task}/" in line and "DRILL_QUERY=" in line: + hits.append(json.loads(line.split("DRILL_QUERY=", 1)[1])) + if len(hits) == 1: return hits[0] + if len(hits) > 1: break + time.sleep(1) + raise RuntimeError("Exact operator task receipt unavailable") + + +def arm(runner, manager, task, instance, space, template, source, record): + ticket = secrets.token_urlsafe(32) + record({"phase": "lease-intent", "instanceId": instance, "sourceDropletId": source, "qualifiesRecovery": False}) + rows = task.query(Path(__file__).with_name("hold_lease.sql").read_text(), [space, ORIGIN, source, hashlib.sha256(ticket.encode()).hexdigest()]) + if len(rows) != 1: raise RuntimeError("No fresh unassigned source-bound standby; no Runner stopped") + slot = rows[0] + slot_id, app_id = guid(slot["id"]), guid(slot["app_id"]) + record({"phase": "lease-rewarming", "instanceId": instance, "slotId": slot_id, "appId": app_id, "sourceDropletId": source, "qualifiesRecovery": False}) + app = runner.api("/v3/apps/" + app_id) + if app_id == template or app.get("name") != "lzc-warm-" + slot_id or app["relationships"]["space"]["data"]["guid"] != space: + raise RuntimeError("Exact unassigned warm app not established; lease will expire") + if runner.api(f"/v3/apps/{app_id}/routes")["resources"]: + raise RuntimeError("Routed app cannot be used for a drill") + deployed = runner.api(f"/v3/apps/{app_id}/droplets/current") + if deployed.get("guid") != source: raise RuntimeError("Warm app source mismatch") + current = runner.api(f"/v3/apps/{template}/droplets/current") + if current.get("guid") != source: raise RuntimeError("Runner source changed during arming") + record({"phase": "stop-unassigned-intent", "appId": app_id, "qualifiesRecovery": False}) + runner.api(f"/v3/apps/{app_id}/actions/stop", "POST") + expires = (datetime.now(timezone.utc) + timedelta(minutes=15)).isoformat(timespec="milliseconds").replace("+00:00", "Z") + config = {"version": 1, "instanceId": instance, "appId": app_id, "spaceId": space, "expiresAt": expires} + record({"phase": "configure-hold-intent", "appId": app_id, "qualifiesRecovery": False}) + runner.api(f"/v3/apps/{app_id}/environment_variables", "PATCH", {"var": {"LZC_POOL_TICKET": ticket, "LZC_RECOVERY_PLAN_HOLD": json.dumps(config, separators=(",", ":"))}}) + record({"phase": "start-prepared-intent", "appId": app_id, "qualifiesRecovery": False}) + runner.api(f"/v3/apps/{app_id}/actions/start", "POST") + ready = task.query("SELECT status,app_id,source_droplet_id,last_seen_at>now()-interval '10 seconds' AS fresh FROM lzc.runner_pool_slots WHERE id=$1", [slot_id], wait_ready=True) + if len(ready) != 1 or ready[0]["status"] != "ready" or not ready[0]["fresh"] or ready[0]["app_id"] != app_id or ready[0]["source_droplet_id"] != source: + raise RuntimeError("Broker readiness not established; no Plan may be started") + record({"phase": "ready", **config, "slotId": slot_id, "sourceDropletId": source, "qualifiesRecovery": False}) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + for name in ["cf", "cf-home", "app-cf-home", "migration-vars", "receipt"]: p.add_argument("--" + name, required=True) + for name in ["runner-space", "runner-template", "instance", "confirm-disposable-hold"]: p.add_argument("--" + name, required=True, type=guid) + args = p.parse_args() + os.umask(0o077) + try: + if args.instance != args.confirm_disposable_hold: raise ValueError("Exact disposable-instance confirmation required") + # Reuse the stop controller's private login and fixed Runner boundary checks. + guard = Controller(args.cf, args.cf_home, args.runner_space, args.runner_template, args.instance) + guard.boundary() + runner, manager = Client(args.cf, args.cf_home), Client(args.cf, args.app_cf_home) + if runner.space != args.runner_space: raise ValueError("Runner space mismatch") + manager.boundary("lzc-dev", "configurator") + found = manager.api(f"/v3/apps?space_guids={manager.space}&names=landing-zone-configurator")["resources"] + if len(found) != 1: raise ValueError("Exact Dev API app unavailable") + app = guid(found[0]["guid"]) + runtime = manager.api(f"/v3/apps/{app}/environment_variables")["var"] + if runtime.get("LZC_RUNNER_POOL_SIZE") != "1" or runtime.get("LZC_RUNNER_SPACE_ID") != args.runner_space or runtime.get("LZC_RUNNER_TEMPLATE_ID") != args.runner_template: + raise ValueError("Expected single-standby Dev runtime not established") + source = guid(runtime["LZC_RUNNER_DROPLET_ID"]) + variables = json.loads(private_file(args.migration_vars).read_text()) + keys = {"LZC_DATABASE_HOST", "LZC_DATABASE_PORT", "LZC_DATABASE_NAME", "LZC_DATABASE_USER", "LZC_DATABASE_PASSWORD"} + if set(variables) != keys or variables.get("LZC_DATABASE_USER") != "configurator_migration" or variables.get("LZC_DATABASE_NAME") != "configurator": + raise ValueError("Isolated migration identity required") + for key in ["LZC_DATABASE_HOST", "LZC_DATABASE_PORT", "LZC_DATABASE_NAME"]: + if variables[key] != runtime[key]: raise ValueError("Dev database target mismatch") + # Capability preflight has no job ticket, native process or State access. + code = 'import {devPlanHoldMilliseconds} from "./apps/worker/dist/recovery-hold.js";if(devPlanHoldMilliseconds!==30000)throw Error("hold-capability-mismatch");' + runner.call(["run-task", "lzc-plan-template", "--command", "./runtime/bin/node --input-type=module -e " + shlex.quote(code), "--name", "hold-capability-" + uuid.uuid4().hex[:8], "-m", "128M", "-k", "4096M", "--wait"]) + from hosted_plan import Receipt + with Receipt(args.receipt) as receipt, OperatorTask(manager, app, variables) as task: + arm(runner, manager, task, args.instance, args.runner_space, args.runner_template, source, receipt.write) + print(json.dumps({"event": "held-plan-runner-ready", "qualifiesRecovery": False}), flush=True) + except (ValueError, RuntimeError, KeyError, OSError, subprocess.SubprocessError): + print(json.dumps({"event": "hold-arming-inconclusive", "qualifiesRecovery": False}), flush=True) + raise SystemExit(1) + + +if __name__ == "__main__": main() diff --git a/landing-zone-configurator/infra/recovery-lab/hold_lease.sql b/landing-zone-configurator/infra/recovery-lab/hold_lease.sql new file mode 100644 index 0000000..4f4289c --- /dev/null +++ b/landing-zone-configurator/infra/recovery-lab/hold_lease.sql @@ -0,0 +1,11 @@ +WITH candidate AS ( + SELECT id FROM lzc.runner_pool_slots + WHERE space_id=$1 AND origin=$2 AND source_droplet_id=$3 AND status='ready' + AND job_id IS NULL AND app_id IS NOT NULL AND NOT is_probe + AND last_seen_at>now()-interval '10 seconds' AND expires_at>now()+interval '30 seconds' + ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 1 +) +UPDATE lzc.runner_pool_slots slot + SET status='warming',token_hash=$4,last_seen_at=NULL,expires_at=now()+interval '2 minutes' + FROM candidate WHERE slot.id=candidate.id + RETURNING slot.id,slot.app_id,slot.source_droplet_id; diff --git a/landing-zone-configurator/infra/recovery-lab/hosted_plan.py b/landing-zone-configurator/infra/recovery-lab/hosted_plan.py index 972b644..62005d4 100644 --- a/landing-zone-configurator/infra/recovery-lab/hosted_plan.py +++ b/landing-zone-configurator/infra/recovery-lab/hosted_plan.py @@ -37,12 +37,13 @@ def observation(line, instance, armed_at, now): guid(event["jobId"]) if event["instanceId"] != instance or event["event"] != "application_plan_observation": return None - if event["observation"] not in {"native-plan-observed", "plan-command-finished"}: + if event["observation"] not in {"native-plan-observed", "plan-command-finished", "native-plan-held", "native-plan-released"}: return None if not isinstance(event["observedAt"], str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z", event["observedAt"]): return None stamp = datetime.fromisoformat(event["observedAt"].replace("Z", "+00:00")).timestamp() - if not armed_at <= stamp <= now + 1 or now - stamp > 3: + max_age = 15 if event["observation"] == "native-plan-held" else 3 + if not armed_at <= stamp <= now + 1 or now - stamp > max_age: return None return event except (ValueError, TypeError, KeyError): @@ -50,7 +51,7 @@ def observation(line, instance, armed_at, now): class Controller: - def __init__(self, cf, cf_home, space, template, instance, confirm=None, receipt=None): + def __init__(self, cf, cf_home, space, template, instance, confirm=None, receipt=None, require_held=False): self.cf = str(Path(cf).resolve(strict=True)) self.space, self.template, self.instance = map(guid, (space, template, instance)) if confirm is not None and confirm != instance: @@ -68,6 +69,7 @@ def __init__(self, cf, cf_home, space, template, instance, confirm=None, receipt if config.get("Target") != API or config.get("SpaceFields", {}).get("GUID") != space: raise ValueError("CF target does not match the selected Dev Runner space") self.env = {"PATH": "/usr/bin:/bin", "LANG": "C.UTF-8", "CF_HOME": str(home.resolve())} + self.require_held = require_held self.confirm = confirm self.stopped = False self.boundary_verified = False @@ -109,7 +111,7 @@ def stop(self, app_id, event, finished): guid(app_id) if not self.confirm or self.stopped or not self.boundary_verified or event["jobId"] in finished: raise RuntimeError("No stop authorization or observation is already finished") - if observation(json.dumps(event, separators=(",", ":")), self.instance, self.armed_at, time.time()) is None or event["observation"] != "native-plan-observed": + if observation(json.dumps(event, separators=(",", ":")), self.instance, self.armed_at, time.time()) is None or event["observation"] != ("native-plan-held" if self.require_held else "native-plan-observed"): raise RuntimeError("Fresh matching native Plan observation required") app = self.api(f"/v3/apps/{app_id}") if app_id == self.template or app.get("guid") != app_id or app.get("relationships", {}).get("space", {}).get("data", {}).get("guid") != self.space: @@ -122,7 +124,7 @@ def fresh(): def request(path, result): fresh() with Receipt(self.receipt_path) as receipt: - record = {"phase": "stop-intent", "instanceId": self.instance, "jobId": event["jobId"], "observedAt": event["observedAt"], "intentAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), "qualifiesRecovery": False, **result} + record = {"phase": "stop-intent", "instanceId": self.instance, "jobId": event["jobId"], "observedAt": event["observedAt"], "intentAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), "qualifiesRecovery": False, "observation": event["observation"], **result} receipt.write(record) fresh() self.api(path, "POST") @@ -176,9 +178,9 @@ def observe(self, seconds): buffer[:] = remainder event = observation(line.decode(errors="replace"), self.instance, self.armed_at, time.time()) if event: - if event["observation"] == "plan-command-finished": + if event["observation"] in {"plan-command-finished", "native-plan-released"}: finished.add(event["jobId"]) - else: + elif event["observation"] == ("native-plan-held" if self.require_held else "native-plan-observed"): candidates.append((app_id, event)) if len({(a, e["jobId"]) for a, e in candidates}) > 1: raise RuntimeError("Ambiguous concurrent target observations") @@ -241,11 +243,12 @@ def main(): parser.add_argument("--runner-template", required=True, type=guid) parser.add_argument("--instance", required=True, type=guid) parser.add_argument("--confirm-disposable-stop", type=guid) + parser.add_argument("--require-held-plan", action="store_true", help="Stop only a confirmed operator-armed 30-second native Plan hold") parser.add_argument("--receipt", help="New file in an owned private directory; required for stop mode") parser.add_argument("--seconds", type=int, default=300, choices=range(10, 901)) args = parser.parse_args() try: - Controller(args.cf, args.cf_home, args.runner_space, args.runner_template, args.instance, args.confirm_disposable_stop, args.receipt).observe(args.seconds) + Controller(args.cf, args.cf_home, args.runner_space, args.runner_template, args.instance, args.confirm_disposable_stop, args.receipt, args.require_held_plan).observe(args.seconds) except (RuntimeError, ValueError, KeyError, OSError, subprocess.SubprocessError): print(json.dumps({"event": "observer-inconclusive", "qualifiesRecovery": False}), flush=True) raise SystemExit(1) diff --git a/landing-zone-configurator/infra/recovery-lab/test_held_plan.py b/landing-zone-configurator/infra/recovery-lab/test_held_plan.py new file mode 100644 index 0000000..0a2b188 --- /dev/null +++ b/landing-zone-configurator/infra/recovery-lab/test_held_plan.py @@ -0,0 +1,104 @@ +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch + +import held_plan as held +from test_hosted_plan import INSTANCE, APP, SPACE, TEMPLATE, OTHER +SOURCE = "77777777-7777-4777-8777-777777777777" +SLOT = "88888888-8888-4888-8888-888888888888" + + +class HeldPlanArming(unittest.TestCase): + def runner(self): + runner = Mock() + def api(path, method="GET", body=None): + if path == "/v3/apps/" + APP: + return {"name":"lzc-warm-"+SLOT,"relationships":{"space":{"data":{"guid":SPACE}}}} + if path.endswith("/routes"): return {"resources":[]} + if path.endswith("/droplets/current"): return {"guid":SOURCE} + return {} + runner.api.side_effect = api + return runner + + def task(self): + task = Mock() + task.query.side_effect = [ + [{"id":SLOT,"app_id":APP,"source_droplet_id":SOURCE}], + [{"status":"ready","app_id":APP,"source_droplet_id":SOURCE,"fresh":True}], + ] + return task + + def test_busy_or_absent_standby_never_stops_a_runner(self): + runner, task = self.runner(), self.task() + task.query.side_effect = [[]] + with self.assertRaises(RuntimeError): held.arm(runner,None,task,INSTANCE,SPACE,TEMPLATE,SOURCE,Mock()) + runner.api.assert_not_called() + + def test_source_or_space_mismatch_never_stops_a_runner(self): + for data in [ + {"name":"lzc-warm-"+SLOT,"relationships":{"space":{"data":{"guid":OTHER}}}}, + {"name":"landing-zone-configurator","relationships":{"space":{"data":{"guid":SPACE}}}}, + ]: + runner=self.runner();runner.api.side_effect=lambda *args:data + with self.assertRaises(RuntimeError): held.arm(runner,None,self.task(),INSTANCE,SPACE,TEMPLATE,SOURCE,Mock()) + self.assertTrue(all(len(c.args)==1 for c in runner.api.call_args_list)) + runner=self.runner();original=runner.api.side_effect + runner.api.side_effect=lambda path,*args: {"guid":OTHER} if path.endswith("/droplets/current") else original(path,*args) + with self.assertRaises(RuntimeError): held.arm(runner,None,self.task(),INSTANCE,SPACE,TEMPLATE,SOURCE,Mock()) + self.assertTrue(all(len(c.args)==1 for c in runner.api.call_args_list)) + + def test_rewarms_only_exact_unassigned_app_and_erases_ticket_from_receipts(self): + runner,task,record=self.runner(),self.task(),Mock() + held.arm(runner,None,task,INSTANCE,SPACE,TEMPLATE,SOURCE,record) + mutations=[c for c in runner.api.call_args_list if len(c.args)>1] + self.assertEqual([c.args[:2] for c in mutations],[ + (f"/v3/apps/{APP}/actions/stop","POST"), + (f"/v3/apps/{APP}/environment_variables","PATCH"), + (f"/v3/apps/{APP}/actions/start","POST"), + ]) + values=mutations[1].args[2]["var"] + ticket=values["LZC_POOL_TICKET"] + self.assertEqual(len(ticket),43) + self.assertNotIn(ticket,str(record.call_args_list)) + self.assertEqual(held.hashlib.sha256(ticket.encode()).hexdigest(),task.query.call_args_list[0].args[1][3]) + config=json.loads(values["LZC_RECOVERY_PLAN_HOLD"]) + self.assertEqual(config["instanceId"],INSTANCE) + self.assertEqual(config["appId"],APP) + self.assertEqual(record.call_args.args[0]["phase"],"ready") + self.assertTrue(task.query.call_args_list[1].kwargs["wait_ready"]) + + def test_failed_readiness_does_not_claim_ready(self): + task=self.task();task.query.side_effect=[[{"id":SLOT,"app_id":APP,"source_droplet_id":SOURCE}],[]] + record=Mock() + with self.assertRaises(RuntimeError): held.arm(self.runner(),None,task,INSTANCE,SPACE,TEMPLATE,SOURCE,record) + self.assertNotIn("ready",[c.args[0]["phase"] for c in record.call_args_list]) + + def test_delayed_receipt_polls_logs_without_replaying_sql(self): + client = Mock() + def call(args): + if args[0] == "run-task": + self.task_name = args[args.index("--name") + 1] + return "" + if not hasattr(self, "logs_seen"): + self.logs_seen = True + return "" + return f'[APP/TASK/{self.task_name}/0] OUT DRILL_QUERY=[{{"status":"ready"}}]' + client.call.side_effect = call + operator = held.OperatorTask(client, APP, {}) + with patch.object(held.time, "sleep"): + self.assertEqual(operator.query("SELECT $1", ["safe"]), [{"status":"ready"}]) + self.assertEqual(sum(c.args[0][0] == "run-task" for c in client.call.call_args_list), 1) + self.assertEqual(sum(c.args[0][0] == "logs" for c in client.call.call_args_list), 2) + + def test_public_or_symlinked_credentials_are_rejected(self): + with tempfile.TemporaryDirectory() as folder: + path=Path(folder)/"private.json";path.write_text("private-only");path.chmod(0o644) + with self.assertRaises(ValueError):held.private_file(path) + path.chmod(0o600);link=Path(folder)/"link";link.symlink_to(path) + with self.assertRaises(ValueError):held.private_file(link) + self.assertEqual(held.private_file(path),path) + + +if __name__ == "__main__": unittest.main() diff --git a/landing-zone-configurator/infra/recovery-lab/test_hosted_plan.py b/landing-zone-configurator/infra/recovery-lab/test_hosted_plan.py index 23d1715..09eef37 100644 --- a/landing-zone-configurator/infra/recovery-lab/test_hosted_plan.py +++ b/landing-zone-configurator/infra/recovery-lab/test_hosted_plan.py @@ -30,6 +30,7 @@ def controller(self, confirm=INSTANCE, warm=True): controller = object.__new__(hosted.Controller) controller.instance, controller.space, controller.template = INSTANCE, SPACE, TEMPLATE controller.confirm, controller.stopped = confirm, False + controller.require_held = False controller.boundary_verified = True private = tempfile.TemporaryDirectory() self.addCleanup(private.cleanup) @@ -50,6 +51,19 @@ def test_observe_mode_finished_and_foreign_jobs_cannot_mutate(self): with self.assertRaises(RuntimeError): controller.stop(APP, value, finished) api.assert_not_called() + def test_held_mode_rejects_unheld_and_released_plans(self): + controller = self.controller() + controller.require_held = True + with patch.object(controller, "api") as api: + for value in [event(), {**event(), "observation": "native-plan-released"}]: + with self.assertRaises(RuntimeError): controller.stop(APP, value, set()) + api.assert_not_called() + held = {**event(age=8), "observation": "native-plan-held"} + controller.armed_at = time.time()-10 + controller.stop(APP, held, set()) + self.assertTrue(controller.stopped) + self.assertIsNone(hosted.observation(json.dumps({**event(age=16), "observation":"native-plan-held"}, separators=(",", ":")), INSTANCE, time.time()-20, time.time())) + def test_only_one_exact_warm_runner_can_be_stopped(self): controller = self.controller() with patch.object(controller, "api", side_effect=controller.api) as api: