test(recovery): reliably interrupt a native Dev Application Plan

.github/workflows/configurator-release.yml

@@ -120,6 +120,8 @@ jobs:
         with:
           tofu_version: '1.12.6'
           tofu_wrapper: false
+      - name: Test isolated native Plan hold and interruption
+        run: LZC_TEST_TOFU_BIN="$(command -v tofu)" npx vitest run tests/worker-recovery-hold.test.ts
       - name: Test real OpenTofu contract and package isolated runner
         run: |
           npm run test:common-contract

.github/workflows/validate-configurator.yml

@@ -111,6 +111,8 @@ jobs:
         run: npm run test:common-contract
       - name: Test native Plan, Application and governance contracts
         run: LZC_TEST_TOFU_BIN="$(command -v tofu)" npm run test:plan
+      - name: Test isolated native Plan hold and interruption
+        run: LZC_TEST_TOFU_BIN="$(command -v tofu)" npx vitest run tests/worker-recovery-hold.test.ts
       - name: Test database migrations and tenant isolation
         run: npm run test:database -- --maxWorkers=2 --testTimeout=30000
         env:

landing-zone-configurator/app/apps/worker/src/observation.ts

@@ -71,7 +71,11 @@ export function watchPlanVersion(
 export function applicationPlanObservation(
   jobId: unknown,
   instanceId: unknown,
-  phase: "native-plan-observed" | "plan-command-finished",
+  phase:
+    | "native-plan-observed"
+    | "plan-command-finished"
+    | "native-plan-held"
+    | "native-plan-released",
 ) {
   const job = applicationPlanJobSchema.shape.id.safeParse(jobId);
   const instance =

landing-zone-configurator/app/apps/worker/src/recovery-hold.ts

@@ -0,0 +1,156 @@
+import { readFile } from "node:fs/promises";
+import { basename } from "node:path";
+import { z } from "zod";
+
+export const devPlanHoldMilliseconds = 30_000;
+const api = "https://api.system.01.cf.eu01.stackit.cloud";
+const origin = "https://lzc-dev-configurator.apps.01.cf.eu01.stackit.cloud";
+const configuration = z.strictObject({
+  version: z.literal(1),
+  instanceId: z.uuid(),
+  appId: z.uuid(),
+  spaceId: z.uuid(),
+  expiresAt: z.iso.datetime(),
+});
+const context = z.object({
+  cf_api: z.literal(api),
+  organization_name: z.literal("lzc-dev-runners"),
+  space_name: z.literal("plans"),
+  application_id: z.uuid(),
+  application_name: z.string().regex(/^lzc-warm-[a-f0-9-]{36}$/),
+  space_id: z.uuid(),
+  process_type: z.literal("web"),
+  application_uris: z.array(z.string()).length(0),
+});
+
+// Only an operator-prepared, one-shot Dev warm app can opt into this drill.
+export function devPlanHoldConfiguration(
+  job: {
+    mode: unknown;
+    instanceId?: string | undefined;
+    purpose?: string | undefined;
+    origin?: string | undefined;
+    local?: boolean | undefined;
+  },
+  env: NodeJS.ProcessEnv = process.env,
+  now = Date.now(),
+) {
+  if (
+    job.mode !== "application-plan" ||
+    job.local ||
+    job.origin !== origin ||
+    (job.purpose !== undefined && job.purpose !== "standard") ||
+    !env.LZC_RECOVERY_PLAN_HOLD ||
+    !z.uuid().safeParse(env.CF_INSTANCE_GUID).success
+  )
+    return undefined;
+  try {
+    const config = configuration.parse(JSON.parse(env.LZC_RECOVERY_PLAN_HOLD));
+    const cf = context.parse(JSON.parse(env.VCAP_APPLICATION ?? ""));
+    const expiresAt = Date.parse(config.expiresAt);
+    if (
+      config.instanceId !== job.instanceId ||
+      cf.application_id !== config.appId ||
+      cf.space_id !== config.spaceId ||
+      expiresAt <= now + devPlanHoldMilliseconds ||
+      expiresAt > now + 15 * 60_000
+    )
+      return undefined;
+    return config;
+  } catch {
+    return undefined;
+  }
+}
+
+async function stoppedNativePlan(group: number) {
+  const children = await readFile(
+    `/proc/${group}/task/${group}/children`,
+    "utf8",
+  ).catch(() => "");
+  for (const pid of [
+    group,
+    ...children.trim().split(/\s+/).filter(Boolean).map(Number),
+  ]) {
+    const [stat, command] = await Promise.all([
+      readFile(`/proc/${pid}/stat`, "utf8").catch(() => ""),
+      readFile(`/proc/${pid}/cmdline`, "utf8").catch(() => ""),
+    ]);
+    const fields = stat.slice(stat.lastIndexOf(") ") + 2).split(" ");
+    const args = command.split("\0");
+    if (
+      fields[0] === "T" &&
+      Number(fields[2]) === group &&
+      basename(args[0] ?? "") === "tofu" &&
+      args[1] === "plan"
+    )
+      return true;
+  }
+  return false;
+}
+
+export function createDevPlanHold(
+  enabled: ReturnType<typeof devPlanHoldConfiguration>,
+  observation: (phase: "native-plan-held" | "native-plan-released") => void,
+) {
+  let attempted = false;
+  let pid: number | undefined;
+  let confirmed = false;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const release = () => {
+    if (timer) clearTimeout(timer);
+    timer = undefined;
+    if (pid === undefined) return;
+    const group = pid;
+    pid = undefined;
+    try {
+      process.kill(-group, "SIGCONT");
+    } catch {
+      /* The child may already be gone. */
+    }
+    if (confirmed) observation("native-plan-released");
+  };
+  return {
+    hold(childPid: number | undefined, deadline: number) {
+      if (
+        !enabled ||
+        attempted ||
+        !childPid ||
+        !Number.isInteger(childPid) ||
+        childPid <= 1 ||
+        Date.parse(enabled.expiresAt) <= Date.now() + devPlanHoldMilliseconds ||
+        deadline <= Date.now() + devPlanHoldMilliseconds + 10_000
+      )
+        return;
+      attempted = true;
+      try {
+        process.kill(-childPid, "SIGSTOP");
+        pid = childPid;
+        timer = setTimeout(release, devPlanHoldMilliseconds);
+      } catch {
+        return;
+      }
+      // Confirm both the shell leader and its native Plan are stopped, not merely a finished shell.
+      void (async () => {
+        for (let attempt = 0; attempt < 50 && pid === childPid; attempt++) {
+          const status = await readFile(
+            `/proc/${childPid}/status`,
+            "utf8",
+          ).catch(() => "");
+          if (pid !== childPid) return;
+          if (
+            /^State:\s+T\b/m.test(status) &&
+            (await stoppedNativePlan(childPid)) &&
+            pid === childPid
+          ) {
+            confirmed = true;
+            observation("native-plan-held");
+            return;
+          }
+          await new Promise((done) => setTimeout(done, 10));
+        }
+        release();
+      })().catch(release);
+    },
+    close: release,
+  };
+}

landing-zone-configurator/app/apps/worker/src/runner.ts

@@ -30,6 +30,10 @@ import {
   summarizePlan,
 } from "@lzc/contracts";
 import { applicationPlanObservation, watchPlanVersion } from "./observation.js";
+import {
+  createDevPlanHold,
+  devPlanHoldConfiguration,
+} from "./recovery-hold.js";
 
 export const brokerOrigin =
   "https://lzc-dev-configurator.apps.01.cf.eu01.stackit.cloud";
@@ -444,8 +448,13 @@ export async function runWorker(
       );
     let activeCommand: string | undefined;
     let planObserved = false;
+    let holdNativePlan: (() => void) | undefined;
     const observePlan = (
-      phase: "native-plan-observed" | "plan-command-finished",
+      phase:
+        | "native-plan-observed"
+        | "plan-command-finished"
+        | "native-plan-held"
+        | "native-plan-released",
     ) => {
       if (
         mode !== "application-plan" ||
@@ -545,6 +554,7 @@ export async function runWorker(
               ) {
                 planObserved = true;
                 observePlan("native-plan-observed");
+                holdNativePlan?.();
               }
               content = progress.text;
               if (progress.invalid && !invalidProgressReported) {
@@ -610,6 +620,18 @@ export async function runWorker(
       outputTimer = setTimeout(pollOutput, 500);
       try {
         activeCommand = phase;
+        const recoveryHold = createDevPlanHold(
+          phase === "planning"
+            ? devPlanHoldConfiguration({
+                mode,
+                instanceId: applicationBinding?.instanceId,
+                purpose: applicationBinding?.purpose,
+                origin: options.brokerOrigin,
+                local: options.local,
+              })
+            : undefined,
+          observePlan,
+        );
         const stopObservation =
           phase === "planning" && mode === "application-plan"
             ? watchPlanVersion(
@@ -618,6 +640,7 @@ export async function runWorker(
                 () => {
                   planObserved = true;
                   observePlan("native-plan-observed");
+                  holdNativePlan?.();
                 },
               )
             : () => {};
@@ -645,6 +668,7 @@ export async function runWorker(
                 detached: true,
               },
             );
+            holdNativePlan = () => recoveryHold.hold(child.pid, deadline);
             let killTimer: ReturnType<typeof setTimeout> | undefined;
             const stop = (signal: NodeJS.Signals) => {
               try {
@@ -680,6 +704,8 @@ export async function runWorker(
           });
         } finally {
           stopObservation();
+          holdNativePlan = undefined;
+          recoveryHold.close();
         }
       } finally {
         activeCommand = undefined;

landing-zone-configurator/app/integration/runner-pool.test.ts

@@ -1,4 +1,5 @@
 import { randomBytes, randomUUID } from "node:crypto";
+import { readFile } from "node:fs/promises";
 import { fileURLToPath } from "node:url";
 import pg from "pg";
 import { afterAll, beforeAll, beforeEach, expect, it, vi } from "vitest";
@@ -518,3 +519,55 @@ it("hands a real database lease through the HTTP broker to exactly one waiting w
     await app.close();
   }
 });
+
+it("atomically rewarms only a fresh unassigned exact-source lease and rotates its capability", async () => {
+  const pool = create();
+  const slot = await ready(pool);
+  const sql = await readFile(
+    new URL("../../infra/recovery-lab/hold_lease.sql", import.meta.url),
+    "utf8",
+  );
+  const ticket = randomBytes(32).toString("base64url");
+  const { createHash } = await import("node:crypto");
+  const hash = createHash("sha256").update(ticket).digest("hex");
+  for (const [space, source] of [
+    [randomUUID(), config.dropletId],
+    [config.spaceId, randomUUID()],
+  ])
+    expect(
+      (await database.query(sql, [space, config.origin, source, hash]))
+        .rowCount,
+    ).toBe(0);
+  const leased = await database.query(sql, [
+    config.spaceId,
+    config.origin,
+    config.dropletId,
+    hash,
+  ]);
+  expect(leased.rows).toEqual([
+    { id: slot.id, app_id: slot.appId, source_droplet_id: config.dropletId },
+  ]);
+  await expect(pool.claim(slot.id, slot.ticket)).rejects.toThrow();
+  await pool.maintain();
+  expect(driver.prepareWarm).toHaveBeenCalledTimes(1);
+  expect(await pool.claim(slot.id, ticket)).toBeNull();
+  const assignment = job();
+  await pool.start(
+    assignment.id,
+    assignment.ticket,
+    config.origin,
+    "application",
+    async () => {},
+  );
+  expect(
+    (
+      await database.query(sql, [
+        config.spaceId,
+        config.origin,
+        config.dropletId,
+        hash,
+      ])
+    ).rowCount,
+  ).toBe(0);
+  expect((await pool.claim(slot.id, ticket))?.id).toBe(assignment.id);
+});