Faster e2e and refactors

.devcontainer/Dockerfile

@@ -39,16 +39,30 @@ RUN apt-get update \
 # Tool versions - centralized for easy updates.
 # This devcontainer currently supports amd64 tool downloads only.
 
+# Finding the latest versions:
+# KUBECTL_VERSION -> https://github.com/kubernetes/kubernetes/releases
+# KUSTOMIZE_VERSION -> https://github.com/kubernetes-sigs/kustomize/releases
+# KUBEBUILDER_VERSION -> https://github.com/kubernetes-sigs/kubebuilder
+# GOLANGCI_LINT_VERSION -> https://github.com/golangci/golangci-lint/releases
+# K3D_VERSION -> https://github.com/k3d-io/k3d
+# FLUX_VERSION -> https://github.com/fluxcd/flux2
+# FLUX_OPERATOR_VERSION -> https://github.com/controlplaneio-fluxcd/flux-operator
+# TASK_VERSION -> https://github.com/go-task/task/releases
+# TILT_VERSION -> https://github.com/tilt-dev/tilt/releases
+# ACTIONLINT_VERSION -> https://github.com/rhysd/actionlint/releases
+
 ENV PATH="/go/bin:/usr/local/go/bin:${PATH}" \
     KUBECTL_VERSION=v1.36.1 \
     KUSTOMIZE_VERSION=5.8.1 \
-    KUBEBUILDER_VERSION=4.14.0 \
+    KUBEBUILDER_VERSION=4.14.1 \
     GOLANGCI_LINT_VERSION=v2.12.2 \
     HELM_VERSION=v4.2.0 \
     K3D_VERSION=v5.8.3 \
-    FLUX_VERSION=2.8.6 \
-    FLUX_OPERATOR_VERSION=0.49.0 \
-    TASK_VERSION=v3.50.0
+    FLUX_VERSION=2.8.8 \
+    FLUX_OPERATOR_VERSION=0.50.0 \
+    TASK_VERSION=v3.51.1 \
+    TILT_VERSION=v0.37.3 \
+    ACTIONLINT_VERSION=1.7.12
 
 # https://github.com/fluxcd/flux2/releases
 # https://fluxoperator.dev/
@@ -107,6 +121,18 @@ RUN curl -fsSL "https://github.com/go-task/task/releases/download/${TASK_VERSION
     | tar -xzO task > /usr/local/bin/task \
     && chmod +x /usr/local/bin/task
 
+# Install actionlint (static linter for GitHub Actions workflows)
+RUN asset="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" \
+    && base="https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}" \
+    && tmpdir="$(mktemp -d)" \
+    && curl -fsSL "${base}/${asset}" -o "${tmpdir}/${asset}" \
+    && curl -fsSL "${base}/actionlint_${ACTIONLINT_VERSION}_checksums.txt" -o "${tmpdir}/checksums.txt" \
+    && cd "${tmpdir}" \
+    && grep "  ${asset}$" checksums.txt | sha256sum -c - \
+    && tar -xzf "${asset}" actionlint \
+    && install -m 0755 actionlint /usr/local/bin/actionlint \
+    && rm -rf "${tmpdir}"
+
 # Set working directory
 WORKDIR /workspaces
 
@@ -127,7 +153,8 @@ RUN chgrp -R godev /go && \
 RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.19.0 \
     && go install github.com/onsi/ginkgo/v2/ginkgo@v2.28.1 \
     && go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20260318145839-6c9615a2a166 \
-    && go install golang.org/x/tools/cmd/goimports@v0.43.0
+    && go install golang.org/x/tools/cmd/goimports@v0.43.0 \
+    && go install github.com/boyter/scc/v3@v3.7.0
 
 # Initialize golangci-lint cache by running it once on an empty directory
 # This downloads linter dependencies without needing source code
@@ -192,7 +219,6 @@ RUN printf '%s\n' \
     >> /etc/bash.bashrc
 
 # Install Tilt CLI (local dev loop orchestration; dev stage only)
-ENV TILT_VERSION=v0.37.1
 RUN arch="$(dpkg --print-architecture)" && \
     case "${arch}" in \
         amd64) tilt_arch="x86_64" ;; \

.devcontainer/devcontainer-lock.json

@@ -1,14 +1,14 @@
 {
   "features": {
-    "ghcr.io/devcontainers/features/common-utils:2.5.8": {
-      "version": "2.5.8",
-      "resolved": "ghcr.io/devcontainers/features/common-utils@sha256:c42fdefe6d737a3a6f61cc52b23c7c9a565d08cc4d9c303669a7cf2ee5fd81fc",
-      "integrity": "sha256:c42fdefe6d737a3a6f61cc52b23c7c9a565d08cc4d9c303669a7cf2ee5fd81fc"
+    "ghcr.io/devcontainers/features/common-utils:2.5.9": {
+      "version": "2.5.9",
+      "resolved": "ghcr.io/devcontainers/features/common-utils@sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a",
+      "integrity": "sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a"
     },
-    "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {
-      "version": "1.9.1",
-      "resolved": "ghcr.io/devcontainers/features/docker-outside-of-docker@sha256:dc89605f01ff2f24252c61f7c8ba2a58ccdbc14f2ebf87a7952d9e2b89834850",
-      "integrity": "sha256:dc89605f01ff2f24252c61f7c8ba2a58ccdbc14f2ebf87a7952d9e2b89834850"
+    "ghcr.io/devcontainers/features/docker-outside-of-docker:1.10.0": {
+      "version": "1.10.0",
+      "resolved": "ghcr.io/devcontainers/features/docker-outside-of-docker@sha256:c2c2cf829505ead8e4892c88c31b6594ae94a2bbb209e16e1fac456c1a3a624e",
+      "integrity": "sha256:c2c2cf829505ead8e4892c88c31b6594ae94a2bbb209e16e1fac456c1a3a624e"
     },
     "ghcr.io/devcontainers/features/git:1": {
       "version": "1.3.5",

.devcontainer/devcontainer.json

@@ -8,8 +8,8 @@
   "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/${localWorkspaceFolderBasename},type=bind",
   "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
   "features": {
-    "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {},
-    "ghcr.io/devcontainers/features/common-utils:2.5.8": {
+    "ghcr.io/devcontainers/features/docker-outside-of-docker:1.10.0": {},
+    "ghcr.io/devcontainers/features/common-utils:2.5.9": {
       "userUid": "automatic",
       "userGid": "automatic",
       "username": "vscode"
@@ -25,7 +25,8 @@
     19090,
     10350,
     16379,
-    19080
+    19080,
+    19081
   ],
   "portsAttributes": {
     "13000": {
@@ -47,6 +48,10 @@
     "19080": {
       "label": "Flux",
       "onAutoForward": "notify"
+    },
+    "19081": {
+      "label": "Allure",
+      "onAutoForward": "notify"
     }
   },
   "customizations": {

.github/workflows/ci.yml

@@ -273,9 +273,17 @@ jobs:
               && export INSTALL_MODE=plain-manifests-file
               && task test-e2e-quickstart-manifest
             needs_artifact: true
+            e2e_ginkgo_procs: "2"
+            k3d_agent_count: "0"
           - name: full
             script: "task test-e2e-full"
             needs_artifact: false
+            # Bumped 1 -> 4: the full suite was serialized to avoid the
+            # cross-target rule-change snapshot coupling, now fixed (per-target
+            # effective watch plan — docs/design/gittarget-isolation-on-rule-change.md).
+            # Validated locally at --procs=4; trialing the same in CI.
+            e2e_ginkgo_procs: "4"
+            k3d_agent_count: "0"
     env:
       PROJECT_IMAGE: ${{ needs.docker-build.outputs.image }}
       CI_CONTAINER: ${{ needs.build-ci-container.outputs.image }}
@@ -296,6 +304,51 @@ jobs:
         run: |
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
 
+      - name: Start disk usage sampler
+        run: |
+          # Sample free space on / every 10s for the whole e2e run so we can see
+          # the peak (not just the post-run value). /var/lib/docker — k3d node
+          # volumes, pulled images, build cache — lives on this filesystem.
+          ( while true; do
+              printf '%s avail=%s\n' "$(date -u +%H:%M:%S)" \
+                "$(df --output=avail -BM / | tail -1 | tr -d ' ')"
+              sleep 10
+            done ) > "df-samples-${{ matrix.name }}.log" 2>/dev/null &
+          echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
+
+      - name: Bring up cluster + Flux services (explicit, streaming logs)
+        run: |
+          # Install-mode-/image-independent slice of prepare-e2e. Runs the
+          # slow, hang-prone cluster + Flux + services bring-up here so its
+          # logs stream live, instead of being buffered inside Ginkgo's
+          # SynchronizedBeforeSuite (where a stall just looks like a silent
+          # 15m hang ending in a goroutine dump). The in-suite prepare-e2e
+          # no-ops this part because the stamps already exist.
+          docker run --rm \
+            --network host \
+            -v "${GITHUB_WORKSPACE}:${{ env.CI_WORKDIR }}" \
+            -v /var/run/docker.sock:/var/run/docker.sock \
+            -w "${{ env.CI_WORKDIR }}" \
+            -e IMAGE_DELIVERY_MODE=${{ env.IMAGE_DELIVERY_MODE }} \
+            -e K3D_AGENT_COUNT=${{ matrix.k3d_agent_count }} \
+            -e HOST_PROJECT_PATH=${{ github.workspace }} \
+            -e KUBECONFIG=${{ env.CI_WORKDIR }}/.stamps/cluster/k3d-gitops-reverser-test-e2e/.kube/config \
+            ${{ env.CI_CONTAINER }} \
+            bash -c "
+              git config --global --add safe.directory ${{ env.CI_WORKDIR }}
+              # The cluster is created here but reused by the test container
+              # below. start-cluster.sh writes the k3d context into KUBECONFIG;
+              # point it at a file on the mounted workspace so the next
+              # container inherits the context (otherwise it is lost with this
+              # ephemeral container and kubectl reports 'context does not exist').
+              # Living under .stamps/cluster/<ctx>/.kube keeps it tied to the
+              # cluster lifecycle: cleanup-installs.sh (per-namespace) leaves it
+              # alone, while clean-cluster (rm -rf .stamps/cluster/<ctx>) clears
+              # it together with the cluster it points at.
+              mkdir -p .stamps/cluster/k3d-gitops-reverser-test-e2e/.kube
+              task _services-ready
+            "
+
       - name: Run E2E tests in CI container
         run: |
           docker run --rm \
@@ -305,13 +358,65 @@ jobs:
             -w "${{ env.CI_WORKDIR }}" \
             -e PROJECT_IMAGE=${{ env.PROJECT_IMAGE }} \
             -e IMAGE_DELIVERY_MODE=${{ env.IMAGE_DELIVERY_MODE }} \
+            -e E2E_GINKGO_PROCS=${{ matrix.e2e_ginkgo_procs }} \
+            -e K3D_AGENT_COUNT=${{ matrix.k3d_agent_count }} \
             -e HOST_PROJECT_PATH=${{ github.workspace }} \
+            -e KUBECONFIG=${{ env.CI_WORKDIR }}/.stamps/cluster/k3d-gitops-reverser-test-e2e/.kube/config \
             ${{ env.CI_CONTAINER }} \
             bash -c "
               git config --global --add safe.directory ${{ env.CI_WORKDIR }}
               ${{ matrix.script }}
             "
 
+      - name: Report disk usage (peak + final)
+        if: always()
+        run: |
+          kill "${SAMPLER_PID}" 2>/dev/null || true
+          samples="df-samples-${{ matrix.name }}.log"
+          {
+            echo "### Disk usage (${{ matrix.name }})"
+            echo
+            echo "Lowest free space on \`/\` during the run:"
+            echo '```'
+            sort -t= -k2 -n "${samples}" 2>/dev/null | head -1 || echo "(no samples captured)"
+            echo '```'
+            echo "Final state:"
+            echo '```'
+            df -h /
+            echo
+            docker system df
+            echo '```'
+            echo "<details><summary>Full df sample timeline</summary>"
+            echo
+            echo '```'
+            cat "${samples}" 2>/dev/null || echo "(no samples captured)"
+            echo '```'
+            echo "</details>"
+          } | tee -a "$GITHUB_STEP_SUMMARY"
+
+      - name: Make E2E artifacts readable by the runner
+        if: always()
+        # The E2E suite runs in a root container that mounts the workspace, so
+        # everything it writes under .stamps/ (e.g. e2e-repo-artifacts/) is
+        # root-owned with restrictive perms. The upload step below runs on the
+        # host as the non-root runner user and fails with EACCES when its glob
+        # tries to scandir those dirs. Relax perms so the runner can traverse
+        # and read them. a+rX adds read for files and traverse for dirs only.
+        run: sudo chmod -R a+rX .stamps || true
+
+      - name: Upload E2E raw Ginkgo JSON reports
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: e2e-ginkgo-reports-${{ matrix.name }}
+          path: .stamps/**/ginkgo-report-*.json
+          # Reports live under the dot-dir .stamps/, which upload-artifact treats
+          # as hidden and excludes by default — without this the upload silently
+          # finds no files. See the "No files were found" gotcha in upload-artifact v4+.
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 60
+
   # Release job only runs on push to main after tests pass
   release-please:
     name: Release Please

AGENTS.md

@@ -24,6 +24,10 @@ task test      # Must pass all unit tests with >90% coverage
 task test-e2e  # Must pass end-to-end tests
 ```
 
+If you change a GitHub Actions workflow, also run `task lint-actions`, which lints
+`.github/workflows/ci.yml` with `actionlint`. Both `actionlint` and `golangci-lint`
+ship in the devcontainer image.
+
 ## PRE-IMPLEMENTATION BEHAVIOR
 
 1. **Check Docker availability for e2e tests**: Before running `task test-e2e`, verify Docker is running with `docker info` or ask user to start Docker daemon if needed
@@ -64,6 +68,12 @@ task test-e2e  # Must pass end-to-end tests
 - Add race condition protection
 - Use temporary directories for testing
 
+### CI Workflows (`.github/workflows/`)
+- After editing any workflow, run `task lint-actions` to catch errors with
+  `actionlint` before pushing
+- A workflow-only change still counts as a CI/config change, so it is **not**
+  covered by the markdown/docs-only validation exception
+
 ## TESTING REQUIREMENTS
 
 - Write unit tests with >90% coverage

Taskfile-build.yml

@@ -8,6 +8,7 @@ vars:
   CONTROLLER_GEN: '{{.CONTROLLER_GEN | default "controller-gen"}}'
   ENVTEST: '{{.ENVTEST | default "setup-envtest"}}'
   GOLANGCI_LINT: '{{.GOLANGCI_LINT | default "golangci-lint"}}'
+  SCC: '{{.SCC | default "scc"}}'
   INSTALL_NAME: '{{.INSTALL_NAME | default "gitops-reverser"}}'
   NAMESPACE: '{{.NAMESPACE | default "gitops-reverser"}}'
   DEFAULT_AUDIT_REDIS_ADDR: '{{.DEFAULT_AUDIT_REDIS_ADDR | default "valkey:6379"}}'
@@ -121,13 +122,18 @@ tasks:
     cmds:
       - |
         export KUBEBUILDER_ASSETS="$({{.ENVTEST}} use {{.ENVTEST_K8S_VERSION}} --bin-dir "$(pwd)/bin" -p path)"
-        go test $(go list ./... | grep -v /e2e) -coverprofile cover.out
+        go test $(go list ./... | grep -Ev '/test/e2e$') -coverprofile cover.out
 
   lint:
     desc: Run golangci-lint
     cmds:
       - '{{.GOLANGCI_LINT}} run'
 
+  lint-actions:
+    desc: Lint GitHub Actions workflows with actionlint
+    cmds:
+      - actionlint .github/workflows/ci.yml
+
   lint-fix:
     desc: Run golangci-lint and perform fixes
     cmds:
@@ -138,6 +144,11 @@ tasks:
     cmds:
       - '{{.GOLANGCI_LINT}} config verify'
 
+  loc:
+    desc: Count lines of project code with scc (excludes vendored external-sources)
+    cmds:
+      - '{{.SCC}} --exclude-dir external-sources,vendor,bin,dist {{.CLI_ARGS}}'
+
   build:
     desc: Build manager binary
     deps:

Taskfile.yml

@@ -5,6 +5,11 @@ version: '3'
 # See .env.example for the available keys.
 dotenv: ['.env']
 
+# Default tasks to run once per invocation so the e2e DAG can fan tasks
+# out via `deps:` without racing — e.g. `_cluster-ready` reachable through
+# both `_image-loaded` and `_flux-installed` must not start twice.
+run: once
+
 set:
   - errexit
   - nounset

Tiltfile

@@ -91,7 +91,6 @@ test_targets = [
     'test-e2e-full',
     'test-e2e-signing',
     'test-e2e-manager',
-    'test-e2e-audit-redis',
     'test-image-refresh',
 ]
 for name in test_targets:

docs/README.md

@@ -27,8 +27,11 @@ These directories are useful when changing internals, but they are not the prima
 docs surface:
 
 - [`design/`](design/): architecture notes, investigations, and implementation plans
+- [`finished/`](finished/): design docs and reports for work that has shipped or been resolved,
+  kept for historical context
 - [`ci/`](ci/): CI/devcontainer rationale and troubleshooting
 - [`future/`](future/): ideas that are intentionally deferred
+- [`serious-bug/`](serious-bug/): worked bug reports against upstream projects
 - [`audit-setup/`](audit-setup/): cluster-specific audit delivery notes and examples
 
 These root-level files are also working notes rather than polished user docs:

docs/architecture.md

@@ -884,7 +884,7 @@ For deeper context on specific decisions:
 - [GitTarget lifecycle and repo architecture](design/gittarget-lifecycle-and-repo-architecture.md) — GitTarget state machine details
 - [Watch and audit rule matching improvement](design/watch-audit-rule-matching-improvement.md) — known issues with namespace matching and startup bootstrap
 - [Kubernetes API resource catalog](design/kubernetes-api-resource-catalog.md) — the served-resource discovery model behind `APIResourceCatalog`
-- [WatchRule GVR resolution](design/watchrule-gvr-resolution-plan.md) — how bare rule resources resolve to concrete GVRs, and the snapshot trust model
+- [WatchRule GVR resolution](finished/watchrule-gvr-resolution-plan.md) — how bare rule resources resolve to concrete GVRs, and the snapshot trust model
 - [Multi-cluster audit ingestion implications](design/multi-cluster-audit-ingestion-implications.md) — what multi-cluster means beyond ingestion
 - [SOPS/age key management](design/sops-repo-bootstrap-and-key-management-architecture.md) — encryption architecture
 - [Audit webhook TLS design](design/audit-webhook-tls-design.md) — webhook transport security