diff --git a/api/v1alpha3/applytrigger_types.go b/api/v1alpha3/applytrigger_types.go index 146ea9a5..0766cac6 100644 --- a/api/v1alpha3/applytrigger_types.go +++ b/api/v1alpha3/applytrigger_types.go @@ -3,8 +3,6 @@ package v1alpha3 import ( - "time" - meta "github.com/fluxcd/pkg/apis/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -63,7 +61,9 @@ const ( // ApplyReasonRepair puts back an API edit that an earlier apply reverted on the cluster after // the edit was made. Git kept the edit; the cluster runs the older value until the repair. ApplyReasonRepair ApplyReason = "Repair" - // ApplyReasonCatchUp keeps the applier current with GitOps Reverser's own pushes. + // ApplyReasonCatchUp adopts what the API created in the applier's folders, so the applier owns + // and can later prune it, and converges the applier once at startup and when a GitTarget first + // names its trigger. ApplyReasonCatchUp ApplyReason = "CatchUp" ) @@ -142,9 +142,6 @@ type ApplyOperation struct { Message string `json:"message,omitempty"` } -// DefaultApplierQuietPeriod is FluxTrigger.spec.quietPeriod when it is omitted. -const DefaultApplierQuietPeriod = 30 * time.Second - // ApplyTrigger is what every trigger kind answers, so the controllers, the branch workers and the // dispatcher handle a FluxTrigger and an ArgoCDTrigger alike. // +kubebuilder:object:generate=false @@ -160,8 +157,6 @@ type ApplyTrigger interface { ApplierKind() string // Applier is the object the trigger holds, its namespace defaulted to the trigger's own. Applier() types.NamespacedName - // EffectiveQuietPeriod is how long the branch must be quiet before a CatchUp. - EffectiveQuietPeriod() time.Duration // SourceBinding is the repository and branch the applier applies, as the trigger last read // them, and false when it has not read them. SourceBinding() (repositoryURL, branch string, known bool) @@ -169,9 +164,6 @@ type ApplyTrigger interface { // false when the applier has none it shares this way or the trigger has not read it. Appliers // sharing one have one operation at a time. SharedSource() (source types.NamespacedName, known bool) - // Prefetches reports whether a push may be fetched into the shared source ahead of the apply: - // every other reader of it is held, so a new artifact applies nothing on its own. - Prefetches() bool // Operation is status.operation, and SetOperation replaces it and status.repairPending. Operation() *ApplyOperation RepairPending() bool diff --git a/api/v1alpha3/argocdtrigger_types.go b/api/v1alpha3/argocdtrigger_types.go index 15f202f6..71d831c6 100644 --- a/api/v1alpha3/argocdtrigger_types.go +++ b/api/v1alpha3/argocdtrigger_types.go @@ -4,16 +4,12 @@ package v1alpha3 import ( "strings" - "time" meta "github.com/fluxcd/pkg/apis/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) -// Why no quietPeriod: it times a CatchUp, and an Application is never asked to catch up. With -// automated sync off it applies nothing on its own, so it holds no stale state to correct. - // ArgoCDTriggerSpec names the Argo CD Application the trigger holds, and the cluster it is in. type ArgoCDTriggerSpec struct { // Immutable for the reason FluxTriggerSpec's references are: the trigger holds what it names. @@ -77,7 +73,7 @@ type ArgoCDTriggerStatus struct { Operation *ApplyOperation `json:"operation,omitempty"` // RepairPending is true while a Repair sync is pending for the Application: a sync reverted an - // API edit made after it was asked for, so the cluster runs an older value than Git until the + // API edit made after its flush, so the cluster runs an older value than Git until the // next sync. // +optional RepairPending bool `json:"repairPending,omitempty"` @@ -108,9 +104,9 @@ type ArgoCDTriggerSource struct { // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` // ArgoCDTrigger holds an Argo CD Application, with automated sync off, for GitOps Reverser, which -// starts every sync it runs: only on a branch with nothing live missing from the commit, after a -// commit GitOps Reverser did not push, after a refused edit, and after a reverted edit. GitTargets -// name it with spec.applyTriggerRef. +// starts every sync it runs, each after publishing the edits pending on the branch: after a commit +// GitOps Reverser did not push, after a refused edit, after a reverted edit, and to adopt what the +// API created or converge the Application (CatchUp). GitTargets name it with spec.applyTriggerRef. type ArgoCDTrigger struct { metav1.TypeMeta `json:",inline"` @@ -163,9 +159,6 @@ func (t *ArgoCDTrigger) Applier() types.NamespacedName { } } -// EffectiveQuietPeriod is the default: it only times a CatchUp, which an Application is never asked. -func (t *ArgoCDTrigger) EffectiveQuietPeriod() time.Duration { return DefaultApplierQuietPeriod } - // SourceBinding is status.source's repoURL, and the branch its targetRevision names ("HEAD" for an // empty one, which follows no branch GitOps Reverser writes). func (t *ArgoCDTrigger) SourceBinding() (string, string, bool) { @@ -185,9 +178,6 @@ func (t *ArgoCDTrigger) SharedSource() (types.NamespacedName, bool) { return types.NamespacedName{}, false } -// Prefetches is false: an Application has no source object to fetch ahead. -func (t *ArgoCDTrigger) Prefetches() bool { return false } - // Operation is status.operation. func (t *ArgoCDTrigger) Operation() *ApplyOperation { return t.Status.Operation } diff --git a/api/v1alpha3/fluxtrigger_types.go b/api/v1alpha3/fluxtrigger_types.go index d4bec7ad..74e1312b 100644 --- a/api/v1alpha3/fluxtrigger_types.go +++ b/api/v1alpha3/fluxtrigger_types.go @@ -3,8 +3,6 @@ package v1alpha3 import ( - "time" - meta "github.com/fluxcd/pkg/apis/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -43,16 +41,6 @@ type FluxTriggerSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec.kustomizationRef is immutable; delete and recreate the FluxTrigger to hold another Kustomization" // +kubebuilder:validation:XValidation:rule="self.name != ''",message="spec.kustomizationRef.name must not be empty" KustomizationRef meta.NamespacedObjectReference `json:"kustomizationRef"` - - // QuietPeriod is how long the branch must hold no open commit window and no pending writes, - // after GitOps Reverser's own pushes, before the Kustomization is asked to catch up with them. - // Every new write restarts the wait. A Go duration of at most "1h". - // +optional - // +kubebuilder:default="30s" - // +kubebuilder:validation:Type=string - // +kubebuilder:validation:Pattern="^([0-9]+(\\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+$" - // +kubebuilder:validation:XValidation:rule="duration(self) <= duration('1h')",message="spec.quietPeriod must be a Go duration of at most 1h" - QuietPeriod *metav1.Duration `json:"quietPeriod,omitempty"` } // FluxTriggerStatus is what the trigger knows about its Kustomization, and the apply it started. @@ -93,7 +81,7 @@ type FluxTriggerStatus struct { Operation *ApplyOperation `json:"operation,omitempty"` // RepairPending is true while a Repair apply is pending for the Kustomization: an apply reverted - // an API edit made after it was asked for, so the cluster runs an older value than Git until the + // an API edit made after its flush, so the cluster runs an older value than Git until the // next apply. // +optional RepairPending bool `json:"repairPending,omitempty"` @@ -123,13 +111,6 @@ type FluxTriggerSource struct { // +optional // +listType=atomic OtherConsumers []meta.NamespacedObjectKindReference `json:"otherConsumers,omitempty"` - - // Prefetch is true while GitOps Reverser fetches the GitRepository as soon as it learns of a push, - // before the apply: every consumer of it was listed and is held by a FluxTrigger, so a new - // artifact applies nothing on its own. False when otherConsumers is not empty, or the consumers - // could not be listed (the identity needs list on kustomizations, helmreleases and helmcharts). - // +optional - Prefetch bool `json:"prefetch,omitempty"` } // +kubebuilder:object:root=true @@ -145,10 +126,10 @@ type FluxTriggerSource struct { // +kubebuilder:printcolumn:name="Message",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].message`,priority=1 // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` -// FluxTrigger holds a Flux Kustomization for GitOps Reverser, which starts every apply it makes: -// only on a branch with nothing live missing from the commit, after a commit GitOps Reverser did -// not push, after a refused edit, after a reverted edit, and to catch up with its own pushes. -// GitTargets name it with spec.applyTriggerRef. +// FluxTrigger holds a Flux Kustomization for GitOps Reverser, which starts every apply it makes, +// each after publishing the edits pending on the branch: after a commit GitOps Reverser did not +// push, after a refused edit, after a reverted edit, and to adopt what the API created or converge +// the Kustomization (CatchUp). GitTargets name it with spec.applyTriggerRef. type FluxTrigger struct { metav1.TypeMeta `json:",inline"` @@ -199,14 +180,6 @@ func (t *FluxTrigger) Applier() types.NamespacedName { } } -// EffectiveQuietPeriod is spec.quietPeriod, resolving an omitted field to the default. -func (t *FluxTrigger) EffectiveQuietPeriod() time.Duration { - if t.Spec.QuietPeriod == nil { - return DefaultApplierQuietPeriod - } - return t.Spec.QuietPeriod.Duration -} - // SourceBinding is status.source's url and branch. func (t *FluxTrigger) SourceBinding() (string, string, bool) { if t.Status.Source == nil { @@ -224,9 +197,6 @@ func (t *FluxTrigger) SharedSource() (types.NamespacedName, bool) { return types.NamespacedName{Namespace: source.Namespace, Name: source.Name}, true } -// Prefetches is status.source.prefetch. -func (t *FluxTrigger) Prefetches() bool { return t.Status.Source != nil && t.Status.Source.Prefetch } - // Operation is status.operation. func (t *FluxTrigger) Operation() *ApplyOperation { return t.Status.Operation } diff --git a/api/v1alpha3/gittarget_types.go b/api/v1alpha3/gittarget_types.go index 18148272..3863b0e5 100644 --- a/api/v1alpha3/gittarget_types.go +++ b/api/v1alpha3/gittarget_types.go @@ -92,11 +92,12 @@ type GitTargetSpec struct { // ApplyTriggerRef names the apply trigger, in this GitTarget's namespace, that holds the applier // of this folder: a FluxTrigger (a Flux Kustomization) or an ArgoCDTrigger (an Argo CD - // Application). With it set, GitOps Reverser decides when that applier applies: only when the - // branch holds no open commit window and no pending writes, after a commit it did not push, - // after a refused edit (spec.onRefusal: Reapply), after an apply reverted a newer API edit, and, - // for a Kustomization, once the branch has been quiet for the trigger's quietPeriod, to keep it - // current with its own pushes. The trigger's applier must apply this target's repository and + // Application). With it set, GitOps Reverser decides when that applier applies, always after + // publishing the edits it holds: after a commit it did not push, after a refused edit + // (spec.onRefusal: Reapply), after an apply reverted a newer API edit, after the API created an + // object in the folder (so the applier owns it and prunes it later), and once when the operator + // starts or the trigger is first named, after the folder's initial snapshots are in Git. Objects + // another applier owns are not mirrored into the folder. The trigger's applier must apply this target's repository and // branch; the ApplyTriggerReady condition says whether it does. Several GitTargets may name one // trigger, and it is asked once for all of them. // +optional @@ -109,8 +110,8 @@ type GitTargetSpec struct { // "Ignore", the default, records the refusal in conditions and events and does nothing else. // // "Reapply" asks this target's applier (through spec.applyTriggerRef, which it requires) to - // apply the current tip once the branch holds no open window and no pending writes, which - // reverts the refused edit and nothing that is still waiting to be published. Only this + // apply the current tip after publishing the edits pending on the branch, which reverts the + // refused edit and nothing that was waiting to be published. Only this // target's applier is asked. If its GitRepository has to fetch to reach the tip, the // Kustomizations a FluxTrigger holds stay suspended and ignore the new artifact; only the // consumers no trigger holds, listed in the trigger's status.source.otherConsumers, apply it, as @@ -447,9 +448,9 @@ type GitTargetStatus struct { // GitTargetRemoteStatus is the answer to "where is my branch, and when did we last prove it". // -// It is written whenever the commit changes — including one we pushed ourselves — and otherwise -// only when the published timestamp is older than one refresh interval: the commit has to move -// with the fact it dates. +// It is sampled when the GitTarget reconciles, from what its branch worker last proved about the +// branch, and written at most once a minute. The first observation, and a correction after the +// repository changed under the target, are written at once. type GitTargetRemoteStatus struct { // Commit is the commit hash the branch is at on the remote. EMPTY means the branch is not on // the remote at all, which is not an error: a branch does not exist without a commit, and a diff --git a/api/v1alpha3/helpers_test.go b/api/v1alpha3/helpers_test.go index e8e6c2f6..6a6ebe7b 100644 --- a/api/v1alpha3/helpers_test.go +++ b/api/v1alpha3/helpers_test.go @@ -3,8 +3,6 @@ package v1alpha3 import ( - "time" - "testing" meta "github.com/fluxcd/pkg/apis/meta" @@ -336,7 +334,6 @@ func TestApplyTrigger_AppliersDefaultToTheTriggersNamespace(t *testing.T) { } assert.Equal(t, "team-a/apps", flux.Applier().String()) assert.Equal(t, "hub", flux.ClusterProvider()) - assert.Equal(t, DefaultApplierQuietPeriod, flux.EffectiveQuietPeriod()) argo := &ArgoCDTrigger{ ObjectMeta: metav1.ObjectMeta{Namespace: "team-a"}, @@ -364,27 +361,23 @@ func TestArgoCDTrigger_SourceBindingNamesTheBranch(t *testing.T) { } // TestNewApplyTrigger_KnowsBothKinds: a kind this release does not know is nil, never a guess. -// TestApplyTrigger_SharedSource: a FluxTrigger shares the GitRepository it last read, and prefetches -// only when its status says so; an ArgoCDTrigger shares nothing and never prefetches. +// TestApplyTrigger_SharedSource: a FluxTrigger shares the GitRepository it last read; an +// ArgoCDTrigger shares nothing. func TestApplyTrigger_SharedSource(t *testing.T) { flux := &FluxTrigger{} _, known := flux.SharedSource() assert.False(t, known, "a trigger that has not read its source shares none yet") - assert.False(t, flux.Prefetches()) flux.Status.Source = &FluxTriggerSource{ GitRepository: meta.NamespacedObjectReference{Namespace: "flux-system", Name: "platform"}, - Prefetch: true, } source, known := flux.SharedSource() assert.True(t, known) assert.Equal(t, "flux-system/platform", source.String()) - assert.True(t, flux.Prefetches()) argo := &ArgoCDTrigger{} _, known = argo.SharedSource() assert.False(t, known, "every sync pins its own revision") - assert.False(t, argo.Prefetches()) } func TestNewApplyTrigger_KnowsBothKinds(t *testing.T) { @@ -397,8 +390,7 @@ func TestNewApplyTrigger_KnowsBothKinds(t *testing.T) { // and SetOperation replaces the operation and repairPending together. func TestApplyTrigger_StatusAccessors(t *testing.T) { operation := &ApplyOperation{Phase: ApplyOperationRunning, Commit: "abc"} - quiet := &metav1.Duration{Duration: time.Minute} - flux := &FluxTrigger{Spec: FluxTriggerSpec{QuietPeriod: quiet}} + flux := &FluxTrigger{} argo := &ArgoCDTrigger{} for _, trigger := range []ApplyTrigger{flux, argo} { assert.Nil(t, trigger.Operation()) @@ -414,10 +406,8 @@ func TestApplyTrigger_StatusAccessors(t *testing.T) { assert.Equal(t, ApplyTriggerKindFlux, flux.TriggerKind()) assert.Equal(t, ApplierKindKustomization, flux.ApplierKind()) - assert.Equal(t, time.Minute, flux.EffectiveQuietPeriod()) assert.Equal(t, ApplyTriggerKindArgoCD, argo.TriggerKind()) assert.Equal(t, ApplierKindApplication, argo.ApplierKind()) - assert.Equal(t, DefaultApplierQuietPeriod, argo.EffectiveQuietPeriod()) assert.Equal(t, DefaultClusterProviderName, argo.ClusterProvider(), "an omitted clusterProviderRef is the default provider, as for a GitTarget") argo.Spec.ClusterProviderRef = &meta.LocalObjectReference{Name: "hub"} diff --git a/api/v1alpha3/zz_generated.deepcopy.go b/api/v1alpha3/zz_generated.deepcopy.go index 62b2724f..4939efae 100644 --- a/api/v1alpha3/zz_generated.deepcopy.go +++ b/api/v1alpha3/zz_generated.deepcopy.go @@ -883,11 +883,6 @@ func (in *FluxTriggerSpec) DeepCopyInto(out *FluxTriggerSpec) { **out = **in } out.KustomizationRef = in.KustomizationRef - if in.QuietPeriod != nil { - in, out := &in.QuietPeriod, &out.QuietPeriod - *out = new(v1.Duration) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FluxTriggerSpec. diff --git a/charts/gitops-reverser/templates/applier-rbac.yaml b/charts/gitops-reverser/templates/applier-rbac.yaml index dcd76bb2..3b6c2f9b 100644 --- a/charts/gitops-reverser/templates/applier-rbac.yaml +++ b/charts/gitops-reverser/templates/applier-rbac.yaml @@ -2,7 +2,7 @@ # What GitOps Reverser needs to drive a Kustomization a FluxTrigger holds: read and patch the # Kustomization and its GitRepository (it suspends and resumes the one and asks the other to fetch), # list the GitRepository's consumers (Kustomizations, and HelmReleases with the HelmCharts built -# from it; without it the source is never prefetched), and create SelfSubjectAccessReviews so the operator can check it holds that +# from it; without it the trigger lists none in status.source.otherConsumers), and create SelfSubjectAccessReviews so the operator can check it holds that # access before relying on it. Separate from the manager role because it is optional and # names another project's API: an install without Flux grants nothing on it. See # docs/design/apply-trigger.md. diff --git a/charts/gitops-reverser/values.yaml b/charts/gitops-reverser/values.yaml index 3f04cadf..a2f8edea 100644 --- a/charts/gitops-reverser/values.yaml +++ b/charts/gitops-reverser/values.yaml @@ -302,8 +302,9 @@ attribution: # bounds stream retention and the in-memory index together, and doubles as the replay horizon a # restart warms the index from. ttl: "10m" - # Bounded per-event wait for a matching audit fact before a watch event ships as the committer. - # Larger values raise attribution hit-rate at the cost of commit latency. + # Bounded per-event wait for a matching audit fact before a watch event ships unresolved, authored + # "unknown (attribution unresolved)". Larger values raise attribution hit-rate at the cost of + # commit latency. grace: "3s" # Caps on the facts held in memory. Per-type is the primary because it is the fair one: a burst on # one noisy type — a deletecollection over ten thousand objects, a large rollout — must not evict diff --git a/cmd/main.go b/cmd/main.go index bc8d9512..4ab6529b 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -181,6 +181,9 @@ func main() { // The read-only twin of the refusal reporter: the refresher re-reads an idle target's folder, // and a folder somebody else broke in Git produces no Kubernetes event to notice it by. workerManager.SetScanAcceptanceReporter(watchMgr.ReportGitPathScan) + // A worker discards a snapshot an apply made stale and asks for a fresh one, and asks no applier + // for anything before a GitTarget's initial snapshots are in Git: both read the streams. + workerManager.SetSnapshotHooks(watchMgr.InitialSnapshotsSettled, watchMgr.RequestRecheckForGitTarget) // The branch workers decide when a GitTarget's applier may apply; the dispatcher delivers it, // off their queues, to the applier its apply trigger holds, in the trigger's cluster. The status // writer records each operation on the trigger, also off their queues. @@ -614,9 +617,9 @@ func parseFlagsWithArgs(fs *flag.FlagSet, args []string) (appConfig, error) { "decides how often the precise path is taken; drops are counted on "+ "attribution_collection_without_uidset_total{reason}.") fs.DurationVar(&cfg.attributionGrace, "author-attribution-grace", watch.DefaultAttributionGraceWindow, - "Bounded per-event wait for a matching audit fact to arrive before a watch event ships as the "+ - "configured committer (duration string; default 3s). Larger values raise attribution hit-rate "+ - "at the cost of commit latency.") + "Bounded per-event wait for a matching audit fact to arrive before a watch event ships unresolved, "+ + "authored \"unknown (attribution unresolved)\" (duration string; default 3s). Larger values "+ + "raise attribution hit-rate at the cost of commit latency.") fs.DurationVar(&cfg.commitRequestTTL, "commit-request-ttl", controller.DefaultCommitRequestTTL, "How long a CommitRequest is kept once it finishes (duration string; default 48h, 0 keeps "+ "them). The deletion time is written onto the request as the "+ diff --git a/config/crd/bases/configbutler.ai_argocdtriggers.yaml b/config/crd/bases/configbutler.ai_argocdtriggers.yaml index ebd9260b..6f23190d 100644 --- a/config/crd/bases/configbutler.ai_argocdtriggers.yaml +++ b/config/crd/bases/configbutler.ai_argocdtriggers.yaml @@ -51,9 +51,9 @@ spec: openAPIV3Schema: description: |- ArgoCDTrigger holds an Argo CD Application, with automated sync off, for GitOps Reverser, which - starts every sync it runs: only on a branch with nothing live missing from the commit, after a - commit GitOps Reverser did not push, after a refused edit, and after a reverted edit. GitTargets - name it with spec.applyTriggerRef. + starts every sync it runs, each after publishing the edits pending on the branch: after a commit + GitOps Reverser did not push, after a refused edit, after a reverted edit, and to adopt what the + API created or converge the Application (CatchUp). GitTargets name it with spec.applyTriggerRef. properties: apiVersion: description: |- @@ -272,7 +272,7 @@ spec: repairPending: description: |- RepairPending is true while a Repair sync is pending for the Application: a sync reverted an - API edit made after it was asked for, so the cluster runs an older value than Git until the + API edit made after its flush, so the cluster runs an older value than Git until the next sync. type: boolean source: diff --git a/config/crd/bases/configbutler.ai_fluxtriggers.yaml b/config/crd/bases/configbutler.ai_fluxtriggers.yaml index e3a996a2..da10746b 100644 --- a/config/crd/bases/configbutler.ai_fluxtriggers.yaml +++ b/config/crd/bases/configbutler.ai_fluxtriggers.yaml @@ -53,10 +53,10 @@ spec: schema: openAPIV3Schema: description: |- - FluxTrigger holds a Flux Kustomization for GitOps Reverser, which starts every apply it makes: - only on a branch with nothing live missing from the commit, after a commit GitOps Reverser did - not push, after a refused edit, after a reverted edit, and to catch up with its own pushes. - GitTargets name it with spec.applyTriggerRef. + FluxTrigger holds a Flux Kustomization for GitOps Reverser, which starts every apply it makes, + each after publishing the edits pending on the branch: after a commit GitOps Reverser did not + push, after a refused edit, after a reverted edit, and to adopt what the API created or converge + the Kustomization (CatchUp). GitTargets name it with spec.applyTriggerRef. properties: apiVersion: description: |- @@ -123,17 +123,6 @@ spec: rule: self == oldSelf - message: spec.kustomizationRef.name must not be empty rule: self.name != '' - quietPeriod: - default: 30s - description: |- - QuietPeriod is how long the branch must hold no open commit window and no pending writes, - after GitOps Reverser's own pushes, before the Kustomization is asked to catch up with them. - Every new write restarts the wait. A Go duration of at most "1h". - pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+$ - type: string - x-kubernetes-validations: - - message: spec.quietPeriod must be a Go duration of at most 1h - rule: duration(self) <= duration('1h') required: - kustomizationRef type: object @@ -286,7 +275,7 @@ spec: repairPending: description: |- RepairPending is true while a Repair apply is pending for the Kustomization: an apply reverted - an API edit made after it was asked for, so the cluster runs an older value than Git until the + an API edit made after its flush, so the cluster runs an older value than Git until the next apply. type: boolean source: @@ -342,13 +331,6 @@ spec: type: object type: array x-kubernetes-list-type: atomic - prefetch: - description: |- - Prefetch is true while GitOps Reverser fetches the GitRepository as soon as it learns of a push, - before the apply: every consumer of it was listed and is held by a FluxTrigger, so a new - artifact applies nothing on its own. False when otherConsumers is not empty, or the consumers - could not be listed (the identity needs list on kustomizations, helmreleases and helmcharts). - type: boolean url: description: URL is the GitRepository's spec.url. type: string diff --git a/config/crd/bases/configbutler.ai_gittargets.yaml b/config/crd/bases/configbutler.ai_gittargets.yaml index 65be93b5..5bc2cf19 100644 --- a/config/crd/bases/configbutler.ai_gittargets.yaml +++ b/config/crd/bases/configbutler.ai_gittargets.yaml @@ -116,11 +116,12 @@ spec: description: |- ApplyTriggerRef names the apply trigger, in this GitTarget's namespace, that holds the applier of this folder: a FluxTrigger (a Flux Kustomization) or an ArgoCDTrigger (an Argo CD - Application). With it set, GitOps Reverser decides when that applier applies: only when the - branch holds no open commit window and no pending writes, after a commit it did not push, - after a refused edit (spec.onRefusal: Reapply), after an apply reverted a newer API edit, and, - for a Kustomization, once the branch has been quiet for the trigger's quietPeriod, to keep it - current with its own pushes. The trigger's applier must apply this target's repository and + Application). With it set, GitOps Reverser decides when that applier applies, always after + publishing the edits it holds: after a commit it did not push, after a refused edit + (spec.onRefusal: Reapply), after an apply reverted a newer API edit, after the API created an + object in the folder (so the applier owns it and prunes it later), and once when the operator + starts or the trigger is first named, after the folder's initial snapshots are in Git. Objects + another applier owns are not mirrored into the folder. The trigger's applier must apply this target's repository and branch; the ApplyTriggerReady condition says whether it does. Several GitTargets may name one trigger, and it is asked once for all of them. properties: @@ -345,8 +346,8 @@ spec: "Ignore", the default, records the refusal in conditions and events and does nothing else. "Reapply" asks this target's applier (through spec.applyTriggerRef, which it requires) to - apply the current tip once the branch holds no open window and no pending writes, which - reverts the refused edit and nothing that is still waiting to be published. Only this + apply the current tip after publishing the edits pending on the branch, which reverts the + refused edit and nothing that was waiting to be published. Only this target's applier is asked. If its GitRepository has to fetch to reach the tip, the Kustomizations a FluxTrigger holds stay suspended and ignore the new artifact; only the consumers no trigger holds, listed in the trigger's status.source.otherConsumers, apply it, as diff --git a/config/rbac/flux-applier-role.yaml b/config/rbac/flux-applier-role.yaml index 08502e1c..53224e35 100644 --- a/config/rbac/flux-applier-role.yaml +++ b/config/rbac/flux-applier-role.yaml @@ -1,8 +1,8 @@ # What GitOps Reverser needs to drive a Flux Kustomization a FluxTrigger holds: read the # Kustomization and the GitRepository it applies from, suspend and resume the Kustomization and # stamp reconcile.fluxcd.io/requestedAt on either, list the consumers of the GitRepository -# (Kustomizations, and HelmReleases with the HelmCharts built from it; without it the source is -# never prefetched), and check that it may. +# (Kustomizations, and HelmReleases with the HelmCharts built from it; without it the trigger lists +# none in status.source.otherConsumers), and check that it may. # # It is kept OUT of role.yaml because no GitTarget needs it until one names an applier, and an # install that does not use Flux should not hold patch rights on Flux objects. Drop this pair when diff --git a/config/samples/fluxtrigger.yaml b/config/samples/fluxtrigger.yaml index 1f7b5861..aba6be79 100644 --- a/config/samples/fluxtrigger.yaml +++ b/config/samples/fluxtrigger.yaml @@ -15,6 +15,3 @@ spec: kustomizationRef: namespace: flux-system name: example-apps - # How long the branch stays quiet after GitOps Reverser's own pushes before the Kustomization is - # asked to catch up with them. - quietPeriod: 30s diff --git a/docs/INDEX.md b/docs/INDEX.md index 7eb5ba77..9887b2e9 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -56,9 +56,10 @@ cover a provider CI cannot reach. **Before you name anything:** [`definitions.md`](definitions.md) — one name per concept (resource collection, watch, branch worker, author attribution, refusal, folder), the two clusters and the two namespace directions, and the nine rules that decide a CRD field, condition type, reason, enum value or printer column. It is -the third of three rulebooks, beside [`style-guide.md`](style-guide.md) for prose and -[`config-flag-conventions.md`](config-flag-conventions.md) for flags. A name that contradicts it is a -defect rather than a variation. +the fourth of four rulebooks, beside [`style-guide.md`](style-guide.md) for prose, +[`config-flag-conventions.md`](config-flag-conventions.md) for flags, and +[`spec/status-conditions-guide.md`](spec/status-conditions-guide.md) for conditions and reasons. A +name that contradicts it is a defect rather than a variation. **Before changing source watches:** [`facts/kubernetes-watch-options.md`](facts/kubernetes-watch-options.md) covers server-side @@ -124,7 +125,7 @@ decided), **design, decided** (decision made, not built), **partly built**, **bu **deferred** (parked, kept as a decision record). The label is the first thing in the page, so you never have to read a proposal to find out it already shipped. -### Open (31 pages) +### Open (32 pages) | Doc | Open question | |---|---| @@ -151,7 +152,8 @@ so you never have to read a proposal to find out it already shipped. | [`watch-and-catalog-architecture.md`](design/watch-and-catalog-architecture.md) | the target three-layer watch model — **needs a human call before building** | | [`metrics-observability-plan.md`](design/metrics-observability-plan.md) | **built.** the canonical metrics document, rewritten 2026-09 and shipped in the same wave. Reads the product as two ingestion paths — object state through watch, authorship through audit — joining at the resolver, and gives each stage one counter with a bounded `outcome` so the flow and its loss paths are drawable. Records what three review rounds corrected, including two reversals worth not re-litigating: the fact-loss counters must NOT be merged (an eviction counts a fact, a trim gap an occurrence, a decode error a whole batch), and a cumulative counter cannot replace a current count, which is why `status.streams` and `status.retainedDocuments` stayed. The instrument inventory lives in [`interpreting-metrics.md`](interpreting-metrics.md), not here. Open: the Grafana JSON and alert rules | | [`reconcile-triggering.md`](design/reconcile-triggering.md) | which controllers still fail to wake up | -| [`apply-trigger.md`](design/apply-trigger.md) | **built for Flux and Argo CD; a webhook applier is the aim.** Why GitOps Reverser decides when Git is applied, and why the applier never applies on its own: a Flux `Kustomization` is held suspended by its `FluxTrigger` and resumed for each apply, an Argo CD `Application` runs with automated sync off. Carries what building it taught (timers are triggers, a long interval is not exclusive control, a decision must hold at the write, a retry is a decision so only the worker makes one, an apply is an operation whose end is observed), the kustomize-controller v1.9.6 facts the suspended lifecycle rests on, its operation (resume token, outcome, the suspend after the apply), who owns `spec.suspend` (a trigger acts only while an eligible `GitTarget` references it, one trigger per applier), restart recovery, shared sources and prefetch, the measured ~50 ms cost, the apply window, and the decisions of 2026-10-17 and 2026-10-19 (the rule, the `allowCrossNamespace.appliers` grant, `Held`) | +| [`applier-echoes.md`](design/applier-echoes.md) | **built 2026-10-20, released as one change.** The applier's own write, back through the watch, is mirrored and re-planned onto a newer tip on replay, reverting a Git-side push. Two ideas close every way a stale picture can reach a commit: a write that changes nothing does not exist, and a pending write carries the actions decided on its tree, not a picture. The applier's write in the apply window is not mirrored (F1); a write is upserts and deletes decided once, so a replay never sweeps again and a save keeps its objects (F2); a resync never decides from a snapshot collected while Git was ahead or across an apply, by an apply epoch (F3); another applier's object is not mirrored into a triggered folder (F4); flush, then apply (S1); `CatchUp` only to adopt API creates and to converge, for Flux and Argo CD alike, and no apply before every initial snapshot is published (S2). Carries the collisions left open, the promises, the code plan, thirteen e2e scenarios, and the twelve decisions taken while building (among them: siblings do not count for F3, a 5-minute hold, initial snapshots exempt from F3, the prefetch removed, no heal resync, and the flush covering the sibling workers) | +| [`apply-trigger.md`](design/apply-trigger.md) | **built for Flux and Argo CD; a webhook applier is the aim.** Why GitOps Reverser decides when Git is applied, and why the applier never applies on its own: a Flux `Kustomization` is held suspended by its `FluxTrigger` and resumed for each apply, an Argo CD `Application` runs with automated sync off. Carries what building it taught (timers are triggers, a long interval is not exclusive control, a decision must hold at the write, a retry is a decision so only the worker makes one, an apply is an operation whose end is observed), the kustomize-controller v1.9.6 facts the suspended lifecycle rests on, its operation (resume token, outcome, the suspend after the apply), who owns `spec.suspend` (a trigger acts only while an eligible `GitTarget` references it, one trigger per applier), restart recovery, shared sources, the measured ~50 ms cost, flush then apply, `CatchUp` as adopt and converge for Flux and Argo CD alike, the apply window with the applier's own writes kept out of Git, resyncs held off while Git is ahead, another applier's objects kept out of a triggered folder, and the decisions of 2026-10-17, 2026-10-19 and 2026-10-20 (the rule, the `allowCrossNamespace.appliers` grant, `Held`, the reversals of 2026-10-20) | | [`push-cooldown.md`](design/push-cooldown.md) | **partly built.** Whether the fixed `5s` push cooldown still earns its place now that the commit window exists. Ordinary timer closures already space one author writing one target, but identity changes, saves with shorter timers and buffer-limit closures commit sooner, and serialization does not batch queued commits, so the cooldown is what batches them. Option C, the failure backoff, is built in #412 and shares the push timer; removing the success wait would now remove only `lastPushAt` and its branch, so that decision waits on the event-loop measurements in §9 | | [`multi-source-audit-ingress-hardening.md`](design/multi-source-audit-ingress-hardening.md) | how independent sources authenticate to a named audit route, when annotation routing is trustworthy, and how multi-provider ingestion remains fair | | [`release-image-reuse-plan.md`](design/release-image-reuse-plan.md) | PRs 2–5 unstarted | diff --git a/docs/TODO.md b/docs/TODO.md index 74c1e7ca..ad65eb2a 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -12,7 +12,7 @@ record made while the render-fidelity gate was closed) are fixed in #411: a request on a target that may not be written fails, whatever path ends it (`internal/git/write_gate.go`). -Retry progress on quiet branches is fixed: a failed publication schedules its own bounded retry +Retry progress on idle branches is fixed: a failed publication schedules its own bounded retry (`internal/git/retry.go`, [`push-cooldown.md`](design/push-cooldown.md) §7 option C). A window whose rebuild fails is no longer dropped: it stays pending for that retry ([`gittarget-branch-worker-pending-writes.md`](design/gittarget-branch-worker-pending-writes.md)). diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index 7ad611cf..c36df29a 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -33,10 +33,10 @@ default stored on every provider, is accepted. **Breaking.** `GitTarget.spec.onRefusal` accepts `Ignore` and `Reapply`. `PushEmptyCommit` is gone: GitOps Reverser no longer moves the branch with an empty commit when an edit is refused. With -`Reapply` it asks the target's applier to apply the current tip once the branch holds no open -commit window and no pending writes. The applier is the Flux `Kustomization` a `FluxTrigger` holds, -or the Argo CD `Application` an `ArgoCDTrigger` holds, and the `GitTarget` names the trigger in the -new `spec.applyTriggerRef`. Only that applier applies, rather than every applier watching the +`Reapply` it asks the target's applier to apply the current tip, after closing the open commit +window and publishing the writes pending on the branch. The applier is the Flux `Kustomization` a +`FluxTrigger` holds, or the Argo CD `Application` an `ArgoCDTrigger` holds, and the `GitTarget` +names the trigger in the new `spec.applyTriggerRef`. Only that applier applies, rather than every applier watching the branch; a `GitRepository` that has to fetch the tip wakes only the consumers no trigger holds. An allowed edit still in a commit window is published before the apply instead of being reverted with the refused one. The API rejects `Reapply` without `spec.applyTriggerRef`. @@ -61,25 +61,42 @@ See [who applies this folder](configuration.md#who-applies-this-folder-specapply A trigger also changes what else applies the folder. A `FluxTrigger` holds its `Kustomization` suspended, so Flux applies it only when GitOps Reverser resumes it: when somebody else pushes to the -branch, after a refusal or a reverted edit, and to catch up with its own pushes, or with the branch -once a `GitTarget` first names the trigger, after the branch has been quiet for the trigger's -`quietPeriod`. `flux get ks` shows it `Suspended`, and while the -operator is down it waits suspended: a graceful stop suspends one it had resumed, and after a crash -the next process does. Remove `spec.suspend` from whatever applies the -`Kustomization`'s manifest (the trigger reports `SuspendOwnedElsewhere` otherwise) and any Git-host -webhook into a Flux `Receiver` for the repository, as [the Flux guide](bi-directional-flux.md) -describes. A trigger acts only while an eligible `GitTarget` names it, so create the two in any -order: until one names it, it is `Ready` (`NotReferenced`) and holds nothing. Deleting the trigger, -or the last `GitTarget` naming it, hands the `Kustomization` back to Flux. The operator needs `get`, -`list` and `patch` on `kustomizations`, `list` on `helmreleases` and `helmcharts`, `get` and `patch` -on `gitrepositories`, and `create` on `selfsubjectaccessreviews`, in the trigger's cluster: Helm -`applierAccess.flux: true` grants them for the operator's own cluster. +branch, after a refusal or a reverted edit, to adopt an object created through the API in the folder +(Flux prunes only what it applied, so such an object would otherwise survive the deletion of its +file in Git), and to converge, once when the operator starts and once when a `GitTarget` first names +the trigger. An API edit asks for no apply. Each apply goes out once the writes pending on the +branch are published, and none before every initial snapshot of the `GitTarget`s naming the trigger +is published. A `GitTarget` that names a trigger has its folder created before it is applied: a +resync into a folder the branch does not hold yet commits the folder's bootstrap files (`README.md`, +`.gittargetignore`, and `.sops.yaml` with encryption) even when there is nothing to mirror. +`flux get ks` shows it `Suspended`, and while the operator is down it waits suspended: a graceful +stop suspends one it had resumed, and after a crash the next process does. Remove +`spec.suspend` from whatever applies the `Kustomization`'s manifest (the trigger reports +`SuspendOwnedElsewhere` otherwise) and any Git-host webhook into a Flux `Receiver` for the +repository, as [the Flux guide](bi-directional-flux.md) describes. A trigger acts only while an +eligible `GitTarget` names it, so create the two in any order: until one names it, it is `Ready` +(`NotReferenced`) and holds nothing. Deleting the trigger, or the last `GitTarget` naming it, hands +the `Kustomization` back to Flux. The operator needs `get`, `list` and `patch` on `kustomizations`, +`list` on `helmreleases` and `helmcharts`, `get` and `patch` on `gitrepositories`, and `create` on +`selfsubjectaccessreviews`, in the trigger's cluster: Helm `applierAccess.flux: true` grants them +for the operator's own cluster. For an Argo CD `Application`, GitOps Reverser starts every sync, so switch its automated sync off (`spec.syncPolicy.automated.enabled: false`, keeping `prune`) before its trigger holds it; one with automated sync on reports `Configured=False` (`AutomatedSyncEnabled`) on the trigger and is never -synced. The operator needs `get` and `update` on `applications`: Helm `applierAccess.argocd: true`. -See [bi-directional GitOps with Argo CD](bi-directional-argocd.md). +synced. An `Application` is asked to sync for the same reasons as a `Kustomization`, adopt and +converge included, so its hooks also run after an API create in the folder and when a `GitTarget` +first names the trigger. The operator needs `get` and `update` on `applications`: Helm +`applierAccess.argocd: true`. See [bi-directional GitOps with Argo CD](bi-directional-argocd.md). + +Naming a trigger also narrows what the folder mirrors. An object another applier owns, one whose +Flux owner labels (`kustomize.toolkit.fluxcd.io/name` and `kustomize.toolkit.fluxcd.io/namespace`) +or Argo CD tracking annotation (`argocd.argoproj.io/tracking-id`) name an applier other than the +trigger's, is no longer mirrored into that folder, and a resync leaves its document in Git alone. +Mirrored, it would be applied by two appliers, which fight each other. After naming a trigger, read +`gitopsreverser_apply_ownership_conflicts_total` and the log line `Not mirroring an object another +applier owns`: each object they report was mirrored before, and belongs to one applier only. See +[objects another applier owns](bi-directional.md#objects-another-applier-owns). ## `git_queue_drops_total` is now `git_queue_refusals_total` diff --git a/docs/api-first-publication.md b/docs/api-first-publication.md index 111698b2..4cef23f9 100644 --- a/docs/api-first-publication.md +++ b/docs/api-first-publication.md @@ -130,12 +130,13 @@ a publication succeeds. These clocks overlap. | Atomic write | Immediate local processing | A caller-defined batch arrives | Preserve a batch after finalizing earlier open work | | Retained-byte threshold | Default `8Mi` | Estimated open plus retained bytes reach the threshold | Close the open window under memory pressure | | Shutdown | Immediate attempt | The worker exits | Attempt to publish finalized work without waiting for cooldown | +| Apply due | Immediate | An applier on the branch has an apply due | Close the open window and publish every write decided so far without waiting for cooldown, so the apply includes them | The first publication can push as soon as a commit is ready. There is no initial five-second push wait. Setting `commit.window.idleTimeout: 0s` makes local commits immediately; subsequent publications still share the branch cooldown. Multiple targets do not each receive a separate push allowance. -For a healthy, quiet branch, a useful approximation is: +For a healthy branch with no apply due, a useful approximation is: ```text commit ready = last matching event at the worker + commit window + local planning time @@ -208,7 +209,7 @@ flowchart TD MATCH -->|yes| DONE["Publish and release retained writes"] MATCH -->|no| HASH["Use SHA already read by the rejected push"] HASH --> FETCH["Fetch current remote tip R and reset checkout"] - FETCH --> REPLAY["Re-plan every retained write, in order, on R"] + FETCH --> REPLAY["Apply every retained write's decided actions, in order, on R"] REPLAY --> DIFF{"Captured state still differs?"} DIFF -->|yes| COMMIT["Create replacement local commits"] DIFF -->|no| NOOP["Retain the no-op until remote confirmation"] @@ -218,9 +219,14 @@ flowchart TD REPLAY -->|error| HOLD["Keep writes; require successful recovery"] ``` -Replay re-runs planning against the fetched tree. It can change unpublished commit SHAs or turn -a write into a no-op. It retains the captured author, target, and resolved write configuration; -prune permissions can tighten before replay. It does not perform a three-way field merge. +Replay applies, against the fetched tree, the actions each write decided when it was first +committed. A write keeps only the objects whose plan changed its tree then, so an object that +changed nothing is never planned onto R, where its old content would revert what R changed; a +write with no action left settles at once and is never replayed. A resync keeps its upserts and +the documents its sweep removed, and never sweeps again on R. A save keeps every object it +captured. Replay can change unpublished commit SHAs or turn a write into a no-op. It retains the +captured author, target, and resolved write configuration; prune permissions can tighten before +replay. It does not perform a three-way field merge. For example, Alice changes `replicas` through the API while Bob changes `image` in Git on the same object. Alice's captured object contains both fields. Replaying it writes her `replicas` and @@ -347,7 +353,10 @@ fetched manifests, which is why a receiver delivery never triggers one. When Flux or Argo CD applies the same path on its own schedule, its refresh and apply cadence also affects whether a live edit survives long enough to be published. An [apply trigger](configuration.md#who-applies-this-folder-specapplytriggerref) takes that clock -away: the applier then applies only when GitOps Reverser asks, on a quiet branch. The +away: the applier then applies only when GitOps Reverser asks, after a flush that publishes every +write decided before the apply came due. An edit decided after the flush is recorded in the apply +window, so an apply that overwrites it is withheld and repaired. A push of GitOps Reverser's own +asks for no apply. The [bi-directional guide](bi-directional.md) describes the integration choices and limitations. ## The other clocks around publication diff --git a/docs/architecture.md b/docs/architecture.md index 92ae2813..a8cd062d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -79,7 +79,7 @@ lock the parent. The parent can still move after the advertisement, so what is g parent's tip as that advertisement showed it. **The corollary is the rule to keep in your head when adding code here:** anything that resolves, commits, or concludes without reaching a push advertisement must either fetch, or refuse to conclude. See -[inbound push notification](design/apply-trigger.md). +[how the branch worker reads the remote](design/push-notification-and-reconcile-trigger.md#part-1-how-the-branch-worker-reads-the-remote). That trust belongs to **one repository**, and so does the worker holding it. A worker is keyed by `(GitProvider namespace, GitProvider name, branch)`, which names no repository, while `spec.url` is @@ -95,7 +95,7 @@ for free: the push session reads the remote's advertisement and the server names accepted, so an active branch never schedules a look of its own. One that has gone quiet is asked on its 5-minute reconcile tick, and spends one ref advertisement per `--git-refresh-interval` (10m by default); only a branch that moved costs a fetch after that. What it learns is published -as `status.remote` (the revision, when it was last proved, and whether a push or a fetch proved +as `status.remote` (the commit, when it was last proved, and whether a push or a fetch proved it), and a folder somebody changed in Git is re-read and republished as `status.placement`. **A refresh never writes content.** It plans nothing and commits nothing of its own. What it does @@ -115,25 +115,31 @@ advertisement, and what the remote says decides whether the branch moved. It nev cluster snapshot: the cluster still holds the values from before the push, so a snapshot would publish them over the incoming change. -**GitOps Reverser decides when Git is applied to the cluster.** An apply is safe only when nothing -live is missing from the commit being applied, and only the workers writing the branch can see the -edits that are live and not yet pushed. So a `GitTarget` names an **apply trigger**, which holds -its **applier** so that it applies nothing on its own (a Flux `Kustomization` is held suspended, an -Argo CD `Application` has automated sync off), and the branch worker sends **apply requests** -instead. A request is sent only on a **quiet branch**, with no open commit window and no pending -writes on any worker writing that repository and branch: - -- for a commit somebody else pushed, after closing open windows and replaying the pending writes - onto it, so the applied commit carries the API side's answer; +**GitOps Reverser decides when Git is applied to the cluster.** An apply must not revert an edit +that is live and not yet in Git, and only the worker writing the branch can see those edits. So a +`GitTarget` names an **apply trigger**, which holds its **applier** so that it applies nothing on +its own (a Flux `Kustomization` is held suspended, an Argo CD `Application` has automated sync off), +and the branch worker sends **apply requests** instead. A request is sent only **after a flush**: +when an apply comes due, the worker closes the open commit window, publishes every write decided so +far at once, and sends the request for the tip that publication produced. An edit decided after +that is recorded in the apply window, so an apply that overwrites it is withheld and repaired. +Continuous editing therefore never holds an apply back. A request goes out: + +- for a commit somebody else pushed, after replaying the pending writes onto it, so the applied + commit carries the API side's answer; - for a refused edit the target opted to revert; - for an API edit that an apply reverted on the cluster after the edit was made, to put it back; -- for Flux only, to catch up with GitOps Reverser's own pushes once the branch has been quiet for - the trigger's `quietPeriod`, because Flux's inventory learns of an object only from an apply of a - commit that holds it. Argo CD is never asked to apply GitOps Reverser's own commits: with automated - sync off it applies nothing on its own, and a sync is pinned to the commit the worker checked. - -The worker keeps each reason pending until an apply operation answers it, and decides every retry -the same way. A request runs off the publication path and cannot fail a publication; the trigger's +- to catch up (`CatchUp`), for a Flux `Kustomization` and an Argo CD `Application` alike: to adopt + an object the API created in the folder, because an applier prunes only what it applied and a + later Git-side delete would otherwise leave the object running; and to converge, once when the + worker starts and once when a `GitTarget` first names the trigger, unless the applier already + applied the tip. A push of GitOps Reverser's own asks for nothing: the cluster already runs the + edits it holds. + +No applier is asked for anything before every initial snapshot of the `GitTarget`s it covers is +published, so the edits made while the operator was down reach Git before any apply. The worker +keeps each reason pending until an apply operation answers it, and decides every retry the same +way. A request runs off the publication path and cannot fail a publication; the trigger's `status.operation` records the operation it started. **Built:** the Flux `Kustomization` applier and the Argo CD `Application` applier, with automated sync off. **The aim:** a webhook for any other tool. See [the apply trigger design](design/apply-trigger.md). @@ -183,7 +189,8 @@ The solution, in the vocabulary used throughout this document: requests initial state for the new render-fidelity revision. Later attempts in the same watch loop can use a stored resume RV; expired history requires fresh initialization. A completed snapshot permits cleanup only within its observed collection and pruning policy. Cursors use - target UID, group/resource, and namespace, with a TTL refreshed by events and bookmarks. + target UID, group/resource, namespace, and label selector, with a TTL refreshed by events and + bookmarks. Unsupported initial-event requests use LIST with a buffered watch. Snapshot reconciliation is triggered by initialization, never by a timer. - **Audit, when enabled, only names the author.** It is an optional attribution lookup; a missing or @@ -277,7 +284,7 @@ graph LR end subgraph SOURCE["Source cluster selected by ClusterProvider"] - API["Kubernetes API\ndiscovery · Namespace policy · watches"] + API["Kubernetes API\ndiscovery · watches"] APPLIER["Applier\nFlux Kustomization\nor Argo CD Application"] end @@ -300,8 +307,8 @@ graph LR The diagram follows one `GitTarget`. A provider with no `spec.kubeConfig` selects the operator's own cluster as its source; any provider name, including `default`, can instead select a remote cluster through a kubeconfig Secret in the operator namespace. Each referenced `ClusterProvider` gets an independent -source context: client, discovery catalog, followability registry, source-namespace snapshot, and -reachability state. Its `GitTarget`s open their source watches through that context. The contexts are keyed +source context: client, discovery catalog, followability registry, and reachability state. Its +`GitTarget`s open their source watches through that context. The contexts are keyed by provider name and are released when no `GitTarget` references them. The applier is not one of the eight kinds. It belongs to Flux or Argo CD, lives in the cluster the @@ -339,6 +346,8 @@ namespace selection of its own. Both share the rule model: group; `*` is all. - `rules[].apiVersions`: omitted means the preferred served version. - `rules[].resources`: plural resource names or `*`. +- `rules[].objectSelector`: an optional label selector the API server applies to every list and + watch the rule opens. An object that stops matching leaves the collection as a deleted one does. - `WatchRule` adds `rules[].sourceNamespace`: omitted for the rule's own namespace, an exact name, or `*` for every namespace the source credential can read. Anything but the rule's own namespace passes the source-namespace gate (`SourceNamespaceAuthorized`). A `*` compiles to ONE cluster-wide @@ -347,8 +356,9 @@ namespace selection of its own. Both share the rule model: peer of any named-namespace collection on the same type, never a replacement. - `ClusterWatchRule` has no scope or namespace choice of its own: the kind is the choice. -Subresources are rejected in rule resources. Mirroring operates on top level resources; the selected -`/scale` subresource effect is translated separately into a parent `spec.replicas` field patch. +Subresources are rejected in rule resources. Mirroring operates on top level resources: a scale +made through the `/scale` subresource reaches Git as the parent's own watch event, and its audit +event serves only to name who scaled the parent. The two namespace controls intentionally live in different planes: @@ -409,9 +419,10 @@ Key fields: - `spec.commit.window`: when this target's commit windows close: `idleTimeout` (default `5s`) of silence, or `maxDuration` (default `1m`) after the window opened. - `spec.commit.message`: `liveTemplate` / `reconcileTemplate` Go templates. -- `spec.applyTriggerRef`: optional; the `FluxTrigger` (a Flux `Kustomization` GitOps Reverser asks - to apply, with its `quietPeriod`, default `30s`) or `ArgoCDTrigger` (an Argo CD `Application`, - automated sync off, whose syncs GitOps Reverser starts) in this namespace. +- `spec.applyTriggerRef`: optional; the `FluxTrigger` (a Flux `Kustomization` GitOps Reverser holds + suspended and asks to apply) or `ArgoCDTrigger` (an Argo CD `Application`, automated sync off, + whose syncs GitOps Reverser starts) in this namespace. In a folder a trigger applies, an object + another applier owns (by Flux's owner labels or Argo CD's tracking annotation) is not mirrored. - `spec.onRefusal`: `Ignore` (default) or `Reapply`, which asks the applier to reapply the tip when an edit to an existing document is refused. `Reapply` requires `spec.applyTriggerRef`. @@ -473,7 +484,7 @@ The controller verifies repository reachability and manages the signing key life ed25519 keypair when `signing.generateWhenMissing` is set. The portable artifact across GitOps ecosystems is the credentials Secret, not a foreign repository object. The credentials reader accepts the Kubernetes native, Flux, and Argo CD Secret key dialects (see -[design/git-credentials-interop.md](finished/git-credentials-interop.md)). +[finished/git-credentials-interop.md](finished/git-credentials-interop.md)). ### ClusterProvider @@ -536,7 +547,6 @@ flowchart LR CP -->|"source client"| DISC CP -->|"source client"| WATCH DISC --> OWN - NS -. "selector snapshot" .-> OWN WATCH --> OWN OWN --> RELEVANCE RELEVANCE --> RESOLVE @@ -642,7 +652,7 @@ flowchart TD MATCH -->|no| FETCH[Fetch latest remote tip] FETCH --> RESET[Hard reset local clone to remote] - RESET --> REPLAY[Replay retained pending writes
from sanitized API state] + RESET --> REPLAY[Apply each retained pending write's
decided actions onto the new tip] REPLAY --> REFRESH[Create fresh commit hashes] REFRESH --> CHECK @@ -655,24 +665,26 @@ local commit from before the retry. ### Somebody else pushed, and the cluster has to follow -A rejected push is one of three ways the branch worker learns the branch gained a commit it did -not push. The other two are a push-receiver delivery and the periodic refresh. All three set the -same piece of worker state, **the applier is behind**, and the worker clears it by sending an apply -request once the branch is quiet. +A rejected push is one of four ways the branch worker learns the branch gained a commit it did +not push. The others are a push-receiver delivery, the periodic refresh, and a Flux +`GitRepository` that fetched another commit than the one an apply asked for, which the worker +takes as a hint and checks the way it checks a delivery. All four set the same piece of worker +state, **the applier is behind**, and the worker clears it by sending an apply request after a +flush. ```mermaid flowchart TD - HINT["Receiver delivery"] --> ADV["Read the ref advertisement"] + HINT["Receiver delivery, or a source
that fetched another commit"] --> ADV["Read the ref advertisement"] REJECT["Rejected compare-and-swap push"] --> SEEN REFRESH["Periodic refresh fetch"] --> SEEN ADV --> OURS{"Tip is a commit
we pushed?"} OURS -->|yes| DROP["Nothing to do"] OURS -->|"no, or cannot tell"| SEEN["Applier is behind"] - SEEN --> PENDING{"Open windows or
pending writes?"} - PENDING -->|yes| FLUSH["Close the windows, fetch,
replay onto the foreign commit, push"] - FLUSH --> QUIET - PENDING -->|no| QUIET["Branch is quiet"] - QUIET --> REQ["Apply request to every applier on the branch,
naming the tip, reason ForeignCommit"] + SEEN --> PENDING{"Open window or
pending writes?"} + PENDING -->|yes| FLUSH["Close the window, publish the pending writes at once:
a rejected push fetches and replays them onto the foreign commit"] + FLUSH --> FLUSHED + PENDING -->|no| FLUSHED["Flushed: every write decided
before the apply came due is published"] + FLUSHED --> REQ["Apply request to every applier on the branch,
naming the tip, reason ForeignCommit"] style DROP fill:#eeeeee,stroke:#9e9e9e style REQ fill:#fff3e0,stroke:#ff9800 @@ -681,12 +693,38 @@ flowchart TD A delivery is handled on the worker's own queue, so it is never judged while a push is in flight: the push has recorded its commit before the delivery is read. A rollback onto one of the worker's older commits is foreign, because the advertisement names a tip other than the last one observed. +A delivery also closes the open commit window, unless a `CommitRequest` holds it, so its +publication finds the move. + +A Git-side push usually reaches the worker through the flush itself: the push of the pending writes +is rejected, the fetch finds a commit the worker did not push, the pending writes' decided actions +are applied onto it, and the applier is asked once, for the tip that holds both sides. Applying the +foreign commit on its own first would revert the pending API edits on the cluster and need a second +apply to put them back. -The same quiet-branch rule sends three other requests: `Refusal`, to the refusing target's applier +The same flush comes before three other requests: `Refusal`, to the refusing target's applier when `spec.onRefusal: Reapply` is set; `Repair`, to an applier whose late write over a newer API -edit the apply window withheld; and `CatchUp`, to every Flux `Kustomization` on the branch once it -has been quiet for the trigger's `quietPeriod` after the worker's own pushes, or after a `GitTarget` -first names the trigger. +edit the apply window withheld; and `CatchUp`, to a `Kustomization` or an `Application` alike, to +adopt an object the API created in a folder that applier applies (a live `ADDED` event whose newest +managers are not an applier's, or an object a resync added to Git) and to converge, once when the +worker starts and once when a `GitTarget` first names the trigger. A converge of the commit the +applier last applied asks nothing; an adopt is sent even then, because applying the same commit +again is what stamps the owner labels or tracking annotation on the new object. A push of the +worker's own leaves nothing pending. + +Three states still hold an apply back, because in them no flush can be made: a registered +`CommitRequest` (a save defines its own commit boundary, and the apply waits for it), a parent +recovery, and a checkout a failed write left dirty. An apply also waits until every initial +snapshot of the `GitTarget`s its applier covers is published (a `GitTarget` with no planned +collection yet, or with a blocked watch, holds its applier the same way), and an empty tip (a +remote that does not carry the branch) asks nothing until there is a commit to apply. + +The flush covers the **sibling** workers too: two `GitProvider`s naming one repository give its +branch two workers, and their `GitTarget`s can name one trigger. When an apply comes due, the +worker asks each sibling to close its window and publish up to a mark of its own at once, and +sends only once every sibling it asked is done. The workers share what they observe of the +branch, so each sends for the tip that holds the others' writes; a sibling's push is foreign to the +others, as a fetch of it would be, because its appliers may apply another cluster. Which apply is due lives on the worker, per applier: one attempt out at a time, and its own retry clock. The dispatcher in `internal/applier` carries out one attempt off the worker's queue and @@ -694,13 +732,13 @@ reports back. It reads each `GitTarget` again first (the same incarnation by uid applier through its trigger on this branch, still admitted, still consenting to a `Refusal`), checks that the `Kustomization`'s source fetches this repository and branch, has the `GitRepository` fetch the commit when its artifact lacks it, asks the worker whether its decision still holds (the commit is -still the tip and nothing is waiting to be published), writes the operation through to the trigger's +still the tip), writes the operation through to the trigger's `status.operation`, and only then resumes the `Kustomization` the trigger holds suspended, with a `requestedAt` token. It follows the apply until that token is handled, suspends the `Kustomization` again with a second token, and reports the operation complete once that one is handled ([`apply-trigger.md`](design/apply-trigger.md#the-flux-applier-suspended)). It never retries: an attempt that does not deliver gives the worker back what it carried, a refusal only while it still -stands, and the worker decides the retry on a quiet branch after that applier's backoff. A delivered +stands, and the worker decides the retry after a flush, once that applier's backoff has run out. A delivered request records an `ApplyRequested` Event, and the worker records the operation on the trigger's `status.operation`. A failed request never reaches back into the publication that caused it. @@ -716,6 +754,7 @@ lives. A populated target seeded from empty looks like: ```text team-a-config/ # GitTarget spec.path ├── README.md # operator-managed bootstrap file +├── .gittargetignore # operator-managed bootstrap file ├── .sops.yaml # present only when encryption is configured ├── team-a/ │ ├── configmaps/app-config.yaml @@ -753,9 +792,10 @@ and an error such as `410 Gone` requires rebuilding from current state. ### Recovery: resume, replay, or list plus mark-and-sweep The first attempt after process startup or watch replacement requests initial state, even when -Redis holds a cursor. `runTargetWatch` starts with `resumeFromCursor=false`: the new render-fidelity -revision needs fresh observation. Subsequent attempts in the same loop may open a normal watch -from the stored RV. If history has expired, or no usable cursor exists, initialization establishes +Redis holds a cursor. `runTargetWatch` passes `resumeFromCursor=false` until this watch's own +replay has completed: the new render-fidelity revision needs fresh observation. Attempts after that, +in the same loop, may open a normal watch from the stored RV. If history has expired, or no +usable cursor exists, initialization establishes current state again. Streaming initialization uses `sendInitialEvents=true` and `resourceVersionMatch=NotOlderThan`: @@ -784,7 +824,7 @@ The relevance filter reproduces that filter in product code, on the hot path: - **Sanitization** ([internal/sanitize](../internal/sanitize/)) strips status, managedFields, and volatile metadata before diffing, so runtime churn never masquerades as a desired-state change. -- **Followability** ([internal/typeset](../internal/typeset/)) encodes "controller-owned → don't mirror" +- **Followability** ([internal/typeset](../internal/typeset/)) encodes "controller-owned → don't mirror". A type that is not followable never gets a watch. - **No-op suppression.** A `*/status` write bumps `resourceVersion` but its sanitized projection equals the prior commit; the writer diffs it to a no-op and discards it. @@ -1131,8 +1171,8 @@ rather than being pruned. Both protect against accidental Git deletions on a dis its own catalog and registry, so a CRD served only by `prod-eu-1` is never resolved for a target mirroring `prod-us-1`. Catalogs refresh at startup and on the 30-second reconciliation cadence. CRD/APIService trigger informers run only in the **control plane**, where the operator's own CRDs live; a remote source's -API-surface change is discovered on that periodic refresh or when a rule/target reconciliation explicitly -refreshes that source. +API-surface change is discovered on that periodic refresh. A target's plan pass never calls discovery +itself; for a source nothing has observed yet, it asks for a refresh at once. ### TypeRegistry and followability @@ -1210,8 +1250,9 @@ A rule or target change and an API-surface change arrive on two different paths. A WatchRule / ClusterWatchRule / GitTarget change reaches a GitTarget through the **GitTarget controller**, which `Watches` those objects (generation change predicates) and queues the affected -GitTarget again. On reconcile the GitTarget refreshes its own source catalog and resolves its claimed -`(GVR, scope)` set again; a type a new rule starts watching gets a new watch opened (with a +GitTarget again. On reconcile the GitTarget declares its watch set again, and the watch plane's +next plan pass resolves its claimed `(GVR, scope)` set against the source's current catalog; a type +a new rule starts watching gets a new watch opened (with a `sendInitialEvents` replay) and a type no longer claimed has its watch closed. A **control-plane** CRD / APIService change reaches the same re-resolution through the watch manager @@ -1235,7 +1276,11 @@ write side is shared with live writes): the whole apply is **refused**: nothing is committed, `GitPathAccepted=False`, `Stalled=True`, and `Ready=False` with reason `UnsupportedContent` until a human cleans the path; - desired resources are upserted through the same content derived path as live writes; -- existing managed documents that are watched but absent from the desired set are deleted; +- existing managed documents that are watched but absent from the desired set are deleted, under + `spec.prune.mode: Always` only; +- in a folder an apply trigger applies, an object another applier owns (by Flux's owner labels or + Argo CD's tracking annotation, read before sanitize strips them) is neither upserted nor treated + as missing, so its document is left alone and counted in `apply_ownership_conflicts_total`; - the operator's own build directives (`kustomization.yaml`, `.sops.yaml`) and other allowlisted auxiliary YAML are retained, not materialised and not refused; - nothing is committed if the apply cannot complete safely. @@ -1253,10 +1298,20 @@ true from the path's structure alone block a GitTarget. The same gate runs on th refused path is never written into by a racing live event either. See [unsupported-folder-refusal-plan.md](spec/unsupported-folder-refusal-plan.md). -A reconcile is **type scoped** (`ScopeGVR`): the sweep is restricted to one `(group, resource)`, so -anchoring one type again never disturbs another's manifests. The desired set for the sweep is the -`sendInitialEvents` replay (everything marked `ADDED` up to `initial-events-end`), so a delete is -reconciled only after the replay completes. +A reconcile is **collection scoped** (`ResyncScope`): the sweep is restricted to the +`(group, resource, namespace)` the snapshot was gathered over, so anchoring one collection again +never disturbs another's manifests. The desired set for the sweep is the `sendInitialEvents` replay +(everything marked `ADDED` up to `initial-events-end`), so a delete is reconciled only after the +replay completes. + +The snapshot decides once. The upserts and deletes its first commit decided are what a replay +onto a newer tip applies, and the sweep never runs again there. On a branch an applier applies, a +snapshot collected while Git was ahead of the cluster (a commit pushed and not applied yet), +collected across an apply, or arriving while Git is ahead is discarded instead of decided, and a +fresh one is asked for once Git is no longer ahead; it would otherwise revert the unapplied commit. A +`GitTarget`'s initial snapshots are exempt, and after 5 minutes of discards the next snapshot is +used anyway. See +[stale snapshots](spec/reconcile-via-watchlist-mark-and-sweep.md#stale-snapshots-are-discarded). *** @@ -1285,7 +1340,7 @@ that names a different repository does not move a live worker; it replaces it, a starts with no clone, no base trust and no observation. `GitTarget.status.remote` is a **branch** fact published per target, and it is worth being exact -about that, because the reverse reading is tempting and wrong. The revision and its `lastVerifiedAt` +about that, because the reverse reading is tempting and wrong. The commit and its `lastVerifiedAt` are the branch worker's single observation, shared by every `GitTarget` on that branch: a push by one target proves it for all of them. So a fresh `lastVerifiedAt` says the branch tip was recently proved. It does **not** say that this target's folder was scanned, that its documents were @@ -1300,15 +1355,15 @@ the data plane wakes a target because a branch moved. So two targets on one branch can show different times. That is publication lag, not a difference in what was proved, and each tuple is internally consistent: one observation from one moment, never -the newest revision beside a stale clock. +the newest commit beside a stale clock. ### Everything about a branch happens on that branch's queue Reading or writing a branch's repository state is the branch worker's job, and the worker does it on one FIFO queue, on one goroutine. Live events, resyncs, `CommitRequest` attaches, periodic -refreshes and push-receiver deliveries all enter the same queue and are served in order. Whether -the branch is quiet is therefore a fact one goroutine holds, which is what lets the worker decide -when an apply request may go out. Nothing else opens a connection on a +refreshes and push-receiver deliveries all enter the same queue and are served in order. Which +writes are decided and which of them are published is therefore a fact one goroutine holds, which +is what lets the worker flush before an apply request goes out. Nothing else opens a connection on a branch's behalf, and nothing else touches its clone. That is what makes "the base is trusted" a statement one goroutine can hold. @@ -1331,10 +1386,12 @@ pair at a time: The window finalizes when its `idleTimeout` passes with no new matching event, its `maxDuration` runs out, the retained buffer reaches `--branch-buffer-max-size` (default `8Mi`), an `attach: Next` -`CommitRequest` closes it, or a resync request that is not a heal or shutdown arrives. The timers +`CommitRequest` closes it, a resync, an atomic batch, a push-receiver delivery or shutdown arrives, +or an apply comes due for an applier on the branch or on a sibling worker. The timers are the GitTarget's `spec.commit.window` until a `CommitRequest` attaches and replaces them. Successful local commits are retained until a fixed push cooldown (`5s`) allows a push, which prevents remote push storms during -bursts. Heal resyncs that arrive during a window are deferred and drained at the next idle boundary. +bursts. The one exception is an apply's flush: the writes decided before the apply came due are +pushed at once, cooldown or not, because the apply waits for them. ### Local clones and conflict retry @@ -1354,6 +1411,14 @@ its own tree. [PushAtomic](../internal/git/git_atomic_push.go) checks the remote remote diverged it smart fetches the latest tip, hard resets the local clone, replays the retained pending writes against the fresh tip (refreshing commit hashes), and retries up to the attempt limit. This is valid because every pending write is rebuilt from sanitized API state; nothing depends on locally edited files. +A write carries the actions its first commit decided, and a replay applies those and plans nothing +else. An object whose plan changed nothing is no action, so its old content can never revert what +the newer tip changed, and a write with no action left settles at once without a push. A resync +keeps the upserts that changed the tree and the documents its sweep removed: a replay plans each +upsert onto the new tree and removes each document if it is still there and the prune mode still +sweeps, and never sweeps again, so a document the newer tip added survives. A save is the exception: +a write carrying a `CommitRequest` keeps every object it captured, and the push decides its outcome. +See [the pending-writes design](design/gittarget-branch-worker-pending-writes.md#a-write-carries-the-actions-decided-on-its-tree). A branch whose remote has been **deleted** takes the same path. The advertisement does not carry the branch at all, which is reported as a moved remote with a zero hash, so the replay re-roots on the remote's @@ -1380,9 +1445,9 @@ with no refresher. is untrusted or the worktree is dirty (see the ground rule above); a healthy publishing target plans straight onto its own last push. Once writes are retained the guard flips off entirely, because a reset would destroy the local commits those writes already produced. Anything that needs a fresh -tree with work in hand therefore resets **and replays** rather than resetting alone, re-planning the -retained writes onto the new tip. One function on the event loop does that, -[`materialize`](../internal/git/branch_worker.go): every commit and push the loop makes runs it +tree with work in hand therefore resets **and replays** rather than resetting alone, applying the +retained writes' decided actions onto the new tip. One function on the event loop does that, +[`materialize`](../internal/git/pending_writes_loop.go): every commit and push the loop makes runs it first, and the loop's commit refuses a checkout that is not the projection of the retained writes. A worktree a failed write left dirty, retained writes whose commits a reset discarded, a parent change, and the snapshot a resync judges against all reach it. @@ -1409,7 +1474,7 @@ the branch. The kept work is bounded at intake ([`intake.go`](../internal/git/intake.go)). While a retry is pending, `--branch-buffer-max-size` counts everything accepted and not yet published: queued items, -the open window, the pending writes, deferred snapshots and waiting saves, each with a 1 KiB per-item charge. +the open window, the pending writes and waiting saves, each with a 1 KiB per-item charge. The first payload that does not fit pauses the branch's intake until a push lands; room under the budget again, or a remote that can be read but refuses the push, keeps it paused. A paused branch's watch streams wait to be woken instead of reconnecting, and their cursors stay where the last @@ -1466,9 +1531,6 @@ hydrates only touched files into buffers for the commit, and flushes only change [images-and-replicas edit-through design](design/support-boundary/finished/images-and-replicas-edit-through.md). - **Deletes:** use the manifest identity index, so a moved manifest can still be deleted even when it is not at the built-in path. -- **Field patches** (currently `/scale` → parent `spec.replicas`) are intentionally narrow: they only - patch an existing parent manifest and never fabricate a parent object from partial subresource data; a - `spec.replicas` assignment governed by a `replicas:` override is routed to the entry instead. ### File placement @@ -1506,8 +1568,12 @@ unsafely: logged per-resource, counted in the resync summary as `placementSkippe ### Bootstrap, encryption, and signing - **Bootstrap** ([bootstrapped_repo_template.go](../internal/git/bootstrapped_repo_template.go)): the - first write to a GitTarget path stages a `README.md` and, when encryption is configured, a `.sops.yaml` - with age recipient rules. Existing files are preserved. + first write to a GitTarget path stages a `README.md`, a `.gittargetignore` and, when encryption is + configured, a `.sops.yaml` with age recipient rules. Existing files are preserved. For a `GitTarget` + that names an apply trigger, a resync into a folder the branch does not hold yet commits these + files even with no object to write, because the folder has to exist before it is applied: Argo CD + retries a sync of a missing path for minutes, and Flux fails it. The bootstrap files are never + deleted, so the folder outlives its last resource. - **Encryption** ([encryption.go](../internal/git/encryption.go), [sops_encryptor.go](../internal/git/sops_encryptor.go), [sensitivity policy](../internal/types/sensitive_resource.go)): core Secrets are sensitive by default; @@ -1546,7 +1612,7 @@ immediately. That is not a failure: If a finalize closes an open window, the worker always schedules a push, so a window closed by an otherwise no-op resync is not stranded. 4. Outcomes resolve on push and are reported as conditions: a pushed commit sets `Ready=True` / - `Pushed=True` with `branch`/`sha`; a benign no-commit sets `Ready=True` with the reason on `Ready` and + `Pushed=True` with `branch`/`commit`; a benign no-commit sets `Ready=True` with the reason on `Ready` and `Pushed=False`; a failure sets `Ready=False` / `Stalled=True` with a message. **Between the window's finalize and that push the request is still in flight**, which matters because the controller keeps re-sending its attach until it reads an outcome: the worker marks the request committed rather than @@ -1578,8 +1644,9 @@ Controllers watch their dependencies so dependents reconcile quickly after spec them, `GitProvider`s, `ClusterProvider`s and `Namespace` labels. It does not watch appliers: it reads the Flux `Kustomization` or Argo CD `Application` on its tick to set `Configured`, so a cluster without the Flux or Argo CD CRDs needs no informer for them. -- `WatchRuleReconciler` / `ClusterWatchRuleReconciler` watch `GitTarget` and `GitProvider`, populate the - RuleStore, and trigger the rule-change reconcile. +- `WatchRuleReconciler` / `ClusterWatchRuleReconciler` watch `GitTarget`, `GitProvider` and + `ClusterProvider` (a `ClusterWatchRule` also `Namespace` labels), populate the RuleStore, and + trigger the rule-change reconcile. - `GitProviderReconciler` validates reachability and manages the signing key lifecycle. - `ClusterProviderReconciler` validates source-connection inputs and projects its readiness to dependent targets. It also removes the retired fact-purge finalizer from pre-release objects during an upgrade. @@ -1605,8 +1672,13 @@ flowchart TD B --> C[Create controller runtime manager] C --> D[Create RuleStore] D --> E[Create WorkerManager + register Runnable] - E --> F[Create Watch Manager + EventRouter; inject TypeRegistry] - F --> G[Register WatchRule + ClusterWatchRule controllers] + E --> R{receiver-bind-address set?} + R -->|yes| Ri[Add push receiver listener; route deliveries to branch workers] + R -->|no| Rj[No receiver; foreign pushes found by rejection and refresh] + Ri --> F + Rj --> F + F[Create Watch Manager + EventRouter; inject TypeRegistry] --> P[Create apply dispatcher + trigger status writer; wire them to the workers] + P --> G[Register WatchRule + ClusterWatchRule controllers] G --> H{redis-addr set?} H -->|yes| Hi[Create Redis cursor store + wire WatchCursorStore + readiness gate] H -->|no| Hj[Skip cursor store; reconnects initialize again] @@ -1616,12 +1688,7 @@ flowchart TD Hq -->|no| J[Configured-author: no fact streams or index; audit webhook skipped] I --> K[Setup + register Watch Manager] J --> K - K --> R{receiver-bind-address set?} - R -->|yes| Ri[Start push receiver listener; route deliveries to branch workers] - R -->|no| Rj[No receiver; foreign pushes found by rejection and refresh] - Ri --> L - Rj --> L - L[Register GitProvider + ClusterProvider + GitTarget + CommitRequest controllers] + K --> L[Register GitProvider + ClusterProvider + GitTarget + FluxTrigger + ArgoCDTrigger
+ CommitRequest controllers; admission webhooks] L --> M[Add cert watchers + health checks] M --> N[mgr.Start] ``` @@ -1666,7 +1733,7 @@ as one funnel and its loss paths as one selector: - **Queue.** `gitopsreverser_git_queue_refusals_total{kind}` counts work a worker refused at enqueue, which its producer offers again, and `_git_queue_depth` is read at scrape time rather than published by the worker loop. - **Write.** `gitopsreverser_git_documents_total{gittarget_*,group,version,resource,outcome}` is the - per-document census (`written`, `deleted_live`, `deleted_sweep`, `unchanged`, `retained`), with + per-document census (`written`, `deleted_live`, `deleted_sweep`, `unchanged`, `retained`, `refused`), with `_placements_total`, `_placement_refusals_total` and `_placement_kustomization_entries_total` answering where a new document landed and which resources were refused. - **Commit and push.** `gitopsreverser_git_commits_total{provider_*,branch,author_kind,message_source}` @@ -1684,13 +1751,17 @@ as one funnel and its loss paths as one selector: attempt the worker decided to send (`applier_kind` is `kustomization` or `application`; `reason` is `ForeignCommit`, `Refusal`, `Repair` or `CatchUp`), with `outcome` separating `sent` from `failed` (the worker decides the retry), `dropped` (no target may still ask for that -reason) and `stale` (the decision no longer held at the patch). It counts attempts, not -the requests still due. +reason) and `stale` (the tip moved before the patch). It counts attempts, not +the requests still due. `apply_echoes_total{outcome}` counts the applier's own writes the apply +window kept out of Git (`not_mirrored`) or mirrored because no window was open (`late`); +`apply_ownership_conflicts_total` counts objects another applier owns that were kept out of a +triggered folder; and `stale_snapshots_total{outcome}` counts resync snapshots discarded because an +apply made them stale, or used after the 5-minute bound. `receiver_deliveries_total{type,outcome}` counts receiver deliveries: `notified`, `unmatched`, `ignored` (a ping, a tag, another event type), `rejected` (bad signature) and `invalid`. The receiver cannot tell a foreign push from an own one; the worker decides that from the -advertisement, which is why a `notified` delivery on an otherwise quiet branch is followed by a -`CatchUp` and only a genuinely foreign one by a `ForeignCommit`. See +advertisement, which is why a `notified` delivery of the worker's own push is followed by no +request and only a foreign one by a `ForeignCommit`. See [Interpreting metrics](interpreting-metrics.md#apply-requests-and-the-push-receiver). **Authorship**, when `--author-attribution` is on: @@ -1720,10 +1791,11 @@ counter, `_secret_encryptions_total{outcome}`. **Configuration state** is the one family that is not about flow. `gitopsreverser_resource_condition{kind,resource_namespace,resource_name,type,status,reason}` publishes the kstatus trio of every object a human declared (`GitTarget`, `WatchRule`, -`ClusterWatchRule`, `GitProvider`, `ClusterProvider`) as one series per possible status, of which -exactly one is `1`. It is recorded at `reconcileStatus.commit()`, the single status choke point all -five controllers use, and the object's entry is dropped when the object goes: a condition series -that outlives its object reports `Ready=False` forever and never clears. It is the only family +`ClusterWatchRule`, `GitProvider`, `ClusterProvider`, `FluxTrigger`, `ArgoCDTrigger`) as one series +per possible status, of which exactly one is `1`. It is recorded at `reconcileStatus.commit()`, the +single status choke point every controller but the `CommitRequest` one uses, and the object's +entry is dropped when the object goes: a condition series that outlives its object reports +`Ready=False` forever and never clears. It is the only family carrying object names, which §7.1 of the plan permits for configuration objects and nothing else. Gauges here are **observable**: their value is read when Prometheus scrapes, not pushed from the @@ -1770,7 +1842,9 @@ Current limitations: - **An apply request cannot reserve time.** An edit made while the requested apply runs, to a field the applier sets, is reverted on the cluster. Its value stays in Git, because the applier's write over it is not mirrored, and the next apply restores it; until then the cluster runs the older - value. See [the apply window](design/apply-trigger.md#the-apply-window). + value. A sibling worker (another `GitProvider` on the same repository and branch) has no apply + window, so an edit made in its folders while the apply runs is reverted, and the sibling mirrors + the revert as an edit of its own. See [the apply window](design/apply-trigger.md#the-apply-window). - **Audit ingress uses a shared-CA trust boundary.** Named `/audit-webhook/` routes and the annotation-routed shared endpoint both require a client certificate signed by the audit CA, but that certificate is not bound to one provider. This is an accepted privileged-control-plane assumption, not @@ -1807,7 +1881,7 @@ Current limitations: | [internal/types/](../internal/types/) | shared resource identity/reference and sensitivity policy | | [internal/typeset/](../internal/typeset/) | type followability registry, lookup model, and the relevance filter (controller-owned → don't mirror) | | [internal/watch/](../internal/watch/) | discovery catalog, watch manager, per-`(GVR, scope)` raw watches with `sendInitialEvents` replay + mark-and-sweep, the author resolver, watched type tables, event router | -| [internal/webhook/](../internal/webhook/) | `/audit-webhook` ingress and attribution fact extraction (no body joiner), and the `/git-push` receiver | +| [internal/webhook/](../internal/webhook/) | `/audit-webhook` ingress and attribution fact extraction (no body joiner), the `/validate-operator-types` admission handler, and the `/git-push` receiver | *** @@ -1832,7 +1906,7 @@ Deeper dives live under [docs/design/](design/): - [Type followability](spec/type-followability.md) - [Typeset owns discovery grace](spec/typeset-owns-discovery-grace.md) - [Kubernetes API resource catalog](facts/kubernetes-api-resource-catalog.md) -- [GitTarget status design](spec/status-conditions-guide.md) +- [Status and conditions guide](spec/status-conditions-guide.md) - [The support contract: what we edit, what we refuse, and why](design/support-boundary/support-contract.md) - [The apply trigger](design/apply-trigger.md) - [Git credentials interop](finished/git-credentials-interop.md) diff --git a/docs/bi-directional-argocd.md b/docs/bi-directional-argocd.md index 5b31fca2..2d248301 100644 --- a/docs/bi-directional-argocd.md +++ b/docs/bi-directional-argocd.md @@ -37,18 +37,21 @@ for each of these, and for the settings that look like a fix and are not. An `Application` with automated sync on makes its trigger report `Configured=False` with `AutomatedSyncEnabled`, and GitOps Reverser never syncs it. -**There is no catch-up.** With automated sync off, Argo CD applies nothing on its own, so nothing it -holds goes stale. GitOps Reverser's own commits are already live, and after the next refresh the -`Application` reports them `Synced`. So an `ArgoCDTrigger` has no `quietPeriod`. A commit that -reached the branch before a `GitTarget` named the trigger is synced with the next push, refusal or -repair ([one sync](#one-sync)). +**A catch-up is about ownership.** With automated sync off, Argo CD applies nothing on its own. +GitOps Reverser's own commits hold values it read from the cluster, so they are already live, and +after the next refresh the `Application` reports them `Synced`: an API edit asks for no sync. An +object the API creates is different. Argo CD prunes only objects that carry its tracking annotation, +and only a sync stamps it, so an object created through the API in the folder would survive the +deletion of its file in Git. An `Application` therefore gets a `CatchUp` exactly as a Flux +`Kustomization` does: to adopt an object the API created, and to converge, once when the operator +starts and once when a `GitTarget` first names the trigger ([one sync](#one-sync)). ## Requirements | Requirement | Why | When it is not met | | --- | --- | --- | | Argo CD 3.1 or later | `spec.syncPolicy.automated.enabled` exists from 3.1.0, and is the only way to switch automated sync off and keep `prune` | Before 3.1 the API server drops `enabled`, the block reads as on, and the trigger reports `AutomatedSyncEnabled`. Remove the `automated` block instead, and accept that a file deleted in Git leaves its object running | -| Automated sync off | GitOps Reverser starts every sync, on a quiet branch | `AutomatedSyncEnabled`; the `Application` is never synced by GitOps Reverser | +| Automated sync off | GitOps Reverser starts every sync, after a flush | `AutomatedSyncEnabled`; the `Application` is never synced by GitOps Reverser | | A single Git source, `spec.source`, on the `GitProvider`'s repository | A sync is pinned to one commit of one repository | `UnsupportedSource` for `spec.sources` or `spec.sourceHydrator`; `SourceMismatch` for another repository | | `targetRevision` is the `GitTarget`'s branch (`main` or `refs/heads/main`) | The tip GitOps Reverser checks is the tip of that branch | `SourceMismatch` for `HEAD`, an empty `targetRevision`, a tag or a commit | | The source renders the folder GitOps Reverser writes: `directory.recurse: true` for a plain directory, or a kustomization at the path | GitOps Reverser writes `////.yaml` | A non-recursive directory sees no files, and the sync applies nothing | @@ -185,6 +188,12 @@ it. Without an `automated` block, its syncs do not prune: a file removed from Gi running. `selfHeal` means nothing with automated sync off. `syncOptions` and `retry` apply as they would to an automated sync. +**Objects another applier owns.** An object whose `argocd.argoproj.io/tracking-id` names another +`Application`, or whose `kustomize.toolkit.fluxcd.io/name` and `kustomize.toolkit.fluxcd.io/namespace` +labels name a Flux `Kustomization`, is not mirrored into the folder: a live change to it is not +mirrored, and a resync neither writes it nor removes its document. Two appliers applying one object +fight each other. See [objects another applier owns](bi-directional.md#objects-another-applier-owns). + ## The `ArgoCDTrigger` ```yaml @@ -287,11 +296,14 @@ writes the `Application` through the Kubernetes API, as `kubectl` would, not thr ## One sync -`ForeignCommit`, `Refusal` and `Repair` reach an `Application` -([the reasons](bi-directional.md#when-gitops-reverser-asks-for-an-apply)); `CatchUp` never does. -Every request starts a sync: there is no "already current" for Argo CD. So naming an `ArgoCDTrigger` -from a `GitTarget` syncs nothing by itself: a commit already on the branch reaches the -`Application` with the next push, refusal or repair, or a manual sync. +All four [reasons](bi-directional.md#when-gitops-reverser-asks-for-an-apply) reach an +`Application`, `CatchUp` included, as they reach a Flux `Kustomization`. Every request starts a sync; +the branch worker decides beforehand which ones are needed. A `CatchUp` that only converges asks +nothing when the `Application`'s last successful sync applied the tip. A `CatchUp` that adopts syncs +even then, because that sync is what stamps the tracking annotation on the objects the API created. +An `Application` a `GitTarget` names for the first time converges once that `GitTarget`'s initial +snapshots are published, so a commit already on the branch reaches it without waiting for the next +push. **A request sets the `Application`'s `operation` to the sync automated sync would start**, with these differences: @@ -336,16 +348,19 @@ starts in between makes it fail instead of being overwritten. A failed sync wait retry still counts as running: Argo CD keeps its `operation` set until it completed. **Hooks run on every sync**, as they would on an automated one: every `ForeignCommit` asks every -applier on the branch, whether or not the push touched its folder. - -**The apply window.** Until the sync's operation completes, however long that takes, an API edit to -a field the `Application` sets can be reverted on the cluster by the sync. GitOps Reverser recognizes -that write by Argo CD's field manager (`argocd-controller`, for a client-side and a server-side -apply alike, or `argocd-application-controller` for a `Replace=true` sync), withholds it from Git, -and asks for a `Repair` sync pinned to the tip that holds the edit, but only when Argo CD's entry is -newer than the edit: undoing an edit can leave an older entry of Argo CD's as the newest, and that -undo is mirrored. When Argo CD and another manager, or the edit, wrote in the same second, the write -is mirrored and counted as `ambiguous`. +applier on the branch, whether or not the push touched its folder, and a `CatchUp` syncs after an API +create in the folder and when a `GitTarget` first names the trigger. + +**The apply window.** Until 10 seconds after the sync's operation completes, however long that +takes, Argo CD's own write to an object nobody edited is the sync landing, and is not mirrored. An +API edit to a field the `Application` sets, one made after the flush included, can be reverted on +the cluster by the sync. GitOps Reverser recognizes that write by Argo CD's field manager +(`argocd-controller`, for a client-side and a server-side apply alike, or +`argocd-application-controller` for a `Replace=true` sync), withholds it from Git, and asks for a +`Repair` sync pinned to the tip that holds the edit, but only when Argo CD's entry is as new as the +edit or newer: undoing an edit can leave an older entry of Argo CD's as the newest, and that undo +is mirrored. When Argo CD and another manager wrote in the same second, the write is mirrored and +counted as `ambiguous`. A request that fails, and a sync that ends `Failed` or `Error`, is decided again from scratch, after that `Application`'s own backoff. A sync still running after 5 minutes is `Unresolved`: GitOps @@ -443,19 +458,23 @@ it. | Trigger `Configured=False`, `ApplierNamespaceNotAllowed` | The `Application` is in another namespace than the trigger, such as `argocd` | Set `allowCrossNamespace.appliers: true` on the trigger's `ClusterProvider` (Helm: `clusterProvider.default.allowCrossNamespace.appliers`), or move the trigger and `GitTarget` to the `Application`'s namespace | | Trigger `Configured=Unknown`, `NotBound` | A `GitTarget` names the trigger, but cannot be judged yet: its `GitProvider` or `ClusterProvider` does not exist | Create the missing object; the message names it | | Trigger `Configured=False`, `Forbidden` | The identity lacks one of the [permissions](#permissions) | Grant it, or set `applierAccess.argocd: true` | -| A Warning `ApplyRequestFailed` Event | A request could not be delivered, such as a sync somebody else started that did not end within 5 minutes | GitOps Reverser keeps asking on a quiet branch, at most every five minutes | -| A Warning `ApplyFailed` Event | The sync ended `Failed` or `Error`; the message carries Argo CD's | Fix what the message names; GitOps Reverser asks again on a quiet branch | +| A Warning `ApplyRequestFailed` Event | A request could not be delivered, such as a sync somebody else started that did not end within 5 minutes | GitOps Reverser keeps asking after its backoff, at most every five minutes | +| A Warning `ApplyFailed` Event | The sync ended `Failed` or `Error`; the message carries Argo CD's | Fix what the message names; GitOps Reverser asks again after its backoff | | A Warning `ApplyUnresolved` Event, trigger `OperationUnresolved` | The sync has not completed in 5 minutes: hooks, waves, or Argo CD's own retries | Nothing is asked of the `Application` until it completes; look at its operation state | | A file deleted in Git leaves its object running | The `Application` has no `automated.prune: true` | Add `automated: {enabled: false, prune: true}` | | The last sync stays `Running`: `Sync operation blocked by sync window` | A sync window blocks automated syncs | Remove the window from the `AppProject`; see [sync windows](#sync-windows-that-block-automated-syncs) | | A commit appears on every sync, adding or removing a `Job` | A `WatchRule` covers the objects a hook creates | Keep hook kinds out of the `WatchRule`s; see [hooks](#watching-the-objects-your-hooks-create) | +| An object in the folder is never mirrored, and the log says `Not mirroring an object another applier owns` | Another `Application` or a Flux `Kustomization` applies it; `apply_ownership_conflicts_total` counts it | Let one applier own it: take it out of the other applier's source, or stop watching it from this `GitTarget` | ## While GitOps Reverser is down GitOps Reverser becomes the only path to a deployment. While it is down, a Git-side push reaches the cluster through a manual sync (`argocd app sync`, or the UI), which is safe then because nothing is -publishing. When it starts again, its first look at the remote asks for no `ForeignCommit`: a sync -on startup could revert edits made while it was down, before its watches caught them. +publishing. When it starts again, its first look at the remote asks for no `ForeignCommit`. A +`CatchUp` that converges is pending instead, and is asked only once every initial snapshot of the +`GitTarget`s naming the trigger is published: a sync before that could revert edits made while the +operator was down, before its watches caught them. An `Application` whose last successful sync +applied the tip is asked nothing. ## What the tests prove @@ -463,13 +482,17 @@ on startup could revert edits made while it was down, before its watches caught | --- | --- | | The [complete example](#a-complete-example) decodes strictly, reports `Configured=True`, binds the `GitTarget`, and its request sets a sync pinned to the commit | [`argocd_guide_test.go`](../internal/applier/argocd_guide_test.go) | | The operation is the one automated sync builds, but for the pinned revision, the initiator and a pinned retry; a running sync is waited for and never replaced; a sync started between the read and the update makes it fail; an unanswered update is settled by a guarded write; every `Configured` reason; when a sync counts as finished | [`argocd_test.go`](../internal/applier/argocd_test.go) | -| An `Application` is never asked for a `CatchUp`, not even once a `GitTarget` first names its trigger, and is asked on a `ForeignCommit` and a `Refusal`; Argo CD's late write over a newer edit is withheld, by its own field managers | [`apply_trigger_test.go`](../internal/git/apply_trigger_test.go), [`apply_window_test.go`](../internal/git/apply_window_test.go) | +| An `Application` converges once a `GitTarget` first names its trigger, is asked on a `ForeignCommit` and a `Refusal`, and adopts an API create even at the tip; Argo CD's late write over a newer edit is withheld, by its own field managers | [`apply_trigger_test.go`](../internal/git/apply_trigger_test.go), [`apply_catchup_test.go`](../internal/git/apply_catchup_test.go), [`apply_window_test.go`](../internal/git/apply_window_test.go) | +| An object another applier owns, read from its tracking annotation, is neither mirrored nor swept | [`apply_ownership_test.go`](../internal/git/apply_ownership_test.go) | | A trigger nothing names reads nothing and is `Ready`; the binding over several `GitTarget`s; `ApplierNamespaceNotAllowed`; a sync whose `.operation` was removed while it ran ends | [`applytrigger_rule_test.go`](../internal/controller/applytrigger_rule_test.go), [`argocd_test.go`](../internal/applier/argocd_test.go) | | Two API edits seconds apart keep the later value, with no further commit | [Bi-directional (Argo CD)](../test/e2e/argocd_bi_directional_e2e_test.go) | | A Git-side push is synced by a `ForeignCommit` request, pinned to that commit | Bi-directional (Argo CD) | | A refused edit with `onRefusal: Reapply` is reverted by an Argo CD sync | Bi-directional (Argo CD) | | An `Application` with automated sync on is `AutomatedSyncEnabled` and is never synced | Bi-directional (Argo CD) | | Argo CD's bookkeeping never reaches Git, asserted as zero commits during a sync | Bi-directional (Argo CD) | +| An API create, and a delete and create with the same content, are adopted, so a Git-side delete prunes them; an edit made during an apply is in Git at once and back on the cluster after the `Repair`; a Git-side push is applied while edits keep arriving, and every edit is kept; a Git-side push survives a resync forced before its apply, and a Git-side addition survives one under `prune.mode: Always`; an API edit asks for no apply; another applier's object is neither mirrored nor adopted; a save made across a Git-side change resolves from the push | [Bi-directional applier echoes (Argo CD)](../test/e2e/bi_directional_echoes_e2e_test.go) | +| A Git-side push made while the previous one is being applied stays in Git and on the cluster | [Bi-directional push race (Argo CD)](../test/e2e/bi_directional_push_race_e2e_test.go) | +| After a restart, an API edit and an API create made while the operator was down end in Git and on the cluster, nothing reverts the edit, a Git-side push made meanwhile is applied, and the create is adopted | [Bi-directional restart (Argo CD)](../test/e2e/bi_directional_restart_e2e_test.go) | `task test-e2e-bi-directional` runs the Argo CD specs. Not covered end to end yet: a hub Argo CD in another cluster, `Forbidden` against cluster RBAC, and a `Replace=true` sync's field manager, which diff --git a/docs/bi-directional-flux.md b/docs/bi-directional-flux.md index b1b17710..64c3c6f2 100644 --- a/docs/bi-directional-flux.md +++ b/docs/bi-directional-flux.md @@ -15,7 +15,7 @@ prunes and reports health; it never applies on its own. | Before | With GitOps Reverser | | --- | --- | | The `Kustomization` applies on its interval, on every new artifact, and on a controller restart | It is suspended, and applies only when GitOps Reverser resumes it for one apply | -| A failed apply is retried after `retryInterval` | It is suspended again after a failure; GitOps Reverser retries on a quiet branch | +| A failed apply is retried after `retryInterval` | It is suspended again after a failure; GitOps Reverser retries after its own backoff | | The Git host's push webhook goes to a Flux `Receiver` | It goes to GitOps Reverser's receiver; the Flux `Receiver` for this repository is removed | | Nothing ties a folder to its `Kustomization` | A `FluxTrigger` holds it, and the `GitTarget` names the trigger in `spec.applyTriggerRef` | | `flux get ks` shows it ready | It shows it `Suspended`, which is how it is held | @@ -135,7 +135,6 @@ spec: kustomizationRef: namespace: flux-system name: checkout - quietPeriod: 30s ``` No Flux `Receiver` for `acme/platform` appears, on purpose: remove yours if you have one, and point @@ -170,12 +169,7 @@ source's other consumers in `status.source.otherConsumers`: the `Kustomization`s holds and the `HelmRelease`s whose chart comes from it, directly or through a `HelmChart` their `chartRef` names. Each of them applies a new artifact the moment it arrives. The list is information only and never affects `Ready`: one source for `apps` and `infra` is a normal setup. - -- While the list is empty and the trigger holds its own `Kustomization`, `status.source.prefetch` is - `true`: when GitOps Reverser learns of a push while the branch is busy, it has the source fetch at - once, so the artifact is ready when the apply is due. A consumer counts as held only while its - `FluxTrigger` reports `Held=True`, checked again on the cluster just before the fetch. -- Appliers on one branch that share a source apply one at a time. +Appliers on one branch that share a source apply one at a time. ## The `Kustomization` @@ -201,8 +195,15 @@ information only and never affects `Ready`: one source for `apps` and `infra` is [what a namespace may reach outside itself](configuration.md#what-a-namespace-may-reach-outside-itself-specallowcrossnamespace). - Several `GitTarget`s may name one `FluxTrigger`, for a `Kustomization` whose `path` covers several folders. It is applied once for all of them. -- Two `GitProvider`s may name one repository. The branch counts as quiet only when both of their - workers are. +- Two `GitProvider`s may name one repository. Before either branch worker asks, both flush: the + other worker publishes the writes it decided so far too, and a `GitRepository` they share serves + one operation at a time. +- **An object another applier owns is not mirrored into the folder.** An object whose + `kustomize.toolkit.fluxcd.io/name` and `kustomize.toolkit.fluxcd.io/namespace` labels name + another `Kustomization`, or whose `argocd.argoproj.io/tracking-id` names an Argo CD + `Application`, is left out: a live change to it is not mirrored, and a resync neither writes it + nor removes its document. Two appliers applying one object fight each other. See + [objects another applier owns](bi-directional.md#objects-another-applier-owns). ## The `FluxTrigger` @@ -218,17 +219,12 @@ spec: kustomizationRef: namespace: flux-system # default: the trigger's namespace name: checkout - quietPeriod: 30s # default 30s, at most 1h ``` `clusterProviderRef` names the cluster that holds the `Kustomization`: the operator's own cluster for `default`, or a management cluster where Flux runs. Both references are immutable: delete and recreate the trigger to hold another `Kustomization`. -`quietPeriod` is how long the branch must stay quiet after GitOps Reverser's own pushes before the -`Kustomization` applies them (`CatchUp`), and before the first apply once a `GitTarget` names the -trigger. Every new write restarts it. - ### How the trigger holds the `Kustomization` A trigger acts only while an eligible `GitTarget` references it: one in the trigger's namespace that @@ -345,7 +341,7 @@ with the identity of that `ClusterProvider`. That identity needs: | --- | --- | --- | | `kustomize.toolkit.fluxcd.io` | `kustomizations` | `get`, `list`, `patch` | | `source.toolkit.fluxcd.io` | `gitrepositories` | `get`, `patch` | -| `helm.toolkit.fluxcd.io` | `helmreleases` | `list`, to find the source's other consumers; without it the source is never prefetched | +| `helm.toolkit.fluxcd.io` | `helmreleases` | `list`, to find the source's other consumers; without it `status.source.otherConsumers` stays empty | | `source.toolkit.fluxcd.io` | `helmcharts` | `list`, for the same reason | | `authorization.k8s.io` | `selfsubjectaccessreviews` | `create`, which every authenticated identity has | @@ -358,11 +354,13 @@ grant is cluster-wide for convenience; GitOps Reverser only patches the `Kustomi ## One apply All four [reasons](bi-directional.md#when-gitops-reverser-asks-for-an-apply) reach a -`Kustomization`. A `ForeignCommit` or `CatchUp` asks nothing when the `Kustomization` has already -applied the commit; a `Refusal` and a `Repair` always apply, since they ask for an unchanged commit -to be applied again. A `Kustomization` a `GitTarget` names for the first time gets a `CatchUp`, so -a commit already on the branch reaches it once the branch is quiet. On a quiet branch, GitOps -Reverser: +`Kustomization`. A `CatchUp` that only converges asks nothing when the `Kustomization`'s last +successful operation applied the tip. Every other request applies, even of a commit already applied: +a `Refusal` and a `Repair` ask for an unchanged commit to be applied again, and a `CatchUp` that +adopts re-applies it so that Flux stamps its owner labels on the objects the API created, and takes +them into its inventory. A `Kustomization` a `GitTarget` names for the first time converges once that +`GitTarget`'s initial snapshots are published, so a commit already on the branch reaches it. After a +[flush](bi-directional.md#flush-then-apply), GitOps Reverser: 1. Makes the `GitRepository`'s artifact hold the commit: when it does not, it stamps `reconcile.fluxcd.io/requestedAt` on the `GitRepository` and waits for that fetch. The suspended @@ -411,11 +409,13 @@ as failed, and records an `OperationUnfollowed` Event. A resumed `Kustomization` nothing watching it. **The apply window.** From the moment an operation is asked for until 10 seconds after it completes, -a write by `kustomize-controller` over a newer API edit is withheld from Git, and GitOps Reverser -asks for a `Repair`; the trigger shows `repairPending` until that apply succeeds. Only a `kustomize-controller` entry newer -than the edit counts: undoing an edit can leave an older one as the newest entry, and that undo is -mirrored. When `kustomize-controller` and another manager, or the edit, wrote in the same second, -the write is mirrored and counted as `ambiguous`. +a write by `kustomize-controller` alone to an object nobody edited is the apply landing, and is not +mirrored. A write by `kustomize-controller` over a newer API edit, one made after the flush included, +is withheld from Git, and GitOps Reverser asks for a `Repair`; the trigger shows `repairPending` +until that apply succeeds. Only a `kustomize-controller` entry as new as the edit or newer counts: +undoing an edit can leave an older one as the newest entry, and that undo is mirrored. When +`kustomize-controller` and another manager wrote in the same second, the write is mirrored and +counted as `ambiguous`. **Reverting refused edits.** With `onRefusal: Reapply`, the target's `Kustomization` is resumed for one apply of the tip; no new commit is needed. If its `GitRepository` has to fetch to reach the @@ -466,7 +466,8 @@ flux get ks checkout -n flux-system | Trigger `Configured=Unknown`, `NotBound` | A `GitTarget` names the trigger, but cannot be judged yet: its `GitProvider` or `ClusterProvider` does not exist | Create the missing object; the message names it | | Trigger `Ready=True`, `NotReferenced`, and the `Kustomization` is not suspended | No `GitTarget` names the trigger, so it holds nothing | Name it in the `GitTarget`'s `spec.applyTriggerRef` | | Trigger `Configured=False`, `Forbidden` | The identity lacks one of the [permissions](#permissions) | Grant it, or set `applierAccess.flux: true` | -| A Warning `ApplyRequestFailed` Event | An apply could not be asked for; the message says why | GitOps Reverser keeps asking on a quiet branch, at most every five minutes | +| A Warning `ApplyRequestFailed` Event | An apply could not be asked for; the message says why | GitOps Reverser keeps asking after its backoff, at most every five minutes | +| An object in the folder is never mirrored, and the log says `Not mirroring an object another applier owns` | Another `Kustomization` or an Argo CD `Application` applies it; `apply_ownership_conflicts_total` counts it | Let one applier own it: take it out of the other applier's source, or stop watching it from this `GitTarget` | | A Warning `ApplyFailed` Event | Flux ran the apply and reported it failed | Read the `Kustomization`'s status; GitOps Reverser retries on its backoff | | A Warning `ApplyUnresolved` Event, trigger `OperationUnresolved` | The apply has not completed in 5 minutes | It is suspended and still followed; look at the `Kustomization`'s status and health checks | | A Warning `OperationUnfollowed` Event | An apply was left in flight with nothing following it | It was suspended and completed as failed. A recurring one is a bug worth reporting with the operator's logs | @@ -490,9 +491,11 @@ it is back, or until somebody runs `flux resume`, after which GitOps Reverser su An apply that was running when it stopped is not lost. The operation is recorded on the trigger before the `Kustomization` is resumed, so the next process follows it to its end by its tokens, with -the apply window open. After its first look at the remote it asks every `Kustomization` for a -`CatchUp`, and for no `ForeignCommit`: an apply on startup could revert edits made while it was -down, before its watches caught them. +the apply window open. After its first look at the remote, a `CatchUp` that converges is pending for +every `Kustomization`, and no `ForeignCommit`. It is asked only once every initial snapshot of the +`GitTarget`s naming the trigger is published: an apply before that could revert edits made while the +operator was down, before its watches caught them. A `Kustomization` whose last successful operation +applied the tip is asked nothing. ## What the tests prove @@ -505,17 +508,22 @@ down, before its watches caught them. | A trigger nothing names reads nothing and is `Ready`; the first eligible `GitTarget` adopts and the last releases, suspending an apply in flight first; a reference returning mid-release holds again; the binding over several `GitTarget`s; an older trigger not in line keeps none out, and a retiring holder stays first until its release is done; `ApplierNamespaceNotAllowed` | [`applytrigger_rule_test.go`](../internal/controller/applytrigger_rule_test.go) | | `allowAnySourceNamespace: true` is refused on create and on any spec change, a stored one included; `false` and its removal are accepted | [`removed_allow_any_source_namespace_test.go`](../internal/controller/removed_allow_any_source_namespace_test.go) | | Every operation checks the target again (incarnation, branch, admission, consent) and the source binding before it resumes, records itself first, and never asks for something weaker; a stop mid-apply suspends the `Kustomization` | [`dispatcher_test.go`](../internal/applier/dispatcher_test.go) | -| A failed attempt is decided again on a quiet branch; each `Kustomization` keeps its own backoff; a shared one is asked once; appliers sharing a source apply one at a time; a push is prefetched only while the branch is busy; a newly named `Kustomization` gets a `CatchUp`; an undo of an edit in the apply window is mirrored | [`apply_attempt_test.go`](../internal/git/apply_attempt_test.go), [`apply_groups_test.go`](../internal/git/apply_groups_test.go), [`apply_trigger_test.go`](../internal/git/apply_trigger_test.go), [`apply_window_test.go`](../internal/git/apply_window_test.go) | +| A failed attempt is decided again; each `Kustomization` keeps its own backoff; a shared one is asked once; appliers sharing a source apply one at a time; an apply waits for its sibling workers to flush; nothing is sent while no flush can be made; an undo of an edit in the apply window is mirrored, and the applier's own write is not | [`apply_attempt_test.go`](../internal/git/apply_attempt_test.go), [`apply_groups_test.go`](../internal/git/apply_groups_test.go), [`apply_trigger_test.go`](../internal/git/apply_trigger_test.go), [`apply_window_test.go`](../internal/git/apply_window_test.go) | +| A newly named `Kustomization` converges once; the startup converge asks nothing of one at the tip; an API create is adopted even at the tip; nothing is asked before the initial snapshots are published, or for an empty tip | [`apply_catchup_test.go`](../internal/git/apply_catchup_test.go), [`apply_trigger_test.go`](../internal/git/apply_trigger_test.go) | +| An object another applier owns is neither mirrored nor swept | [`apply_ownership_test.go`](../internal/git/apply_ownership_test.go) | | A trigger nothing names is `Ready` and holds nothing; named from a `GitTarget` on another branch than its source, it is `SourceMismatch` and adopts nothing | [Manager Flux Applier](../test/e2e/flux_applier_e2e_test.go) | | A `ForeignCommit` for a `SourceMismatch` applier never stamps the other branch's source and records `ApplyRequestFailed`; once the source is fixed the trigger adopts the `Kustomization` and Flux reaches the tip; dropping the reference hands it back, and naming the trigger again adopts it anew | Manager Flux Applier | | A Gitea push webhook to the operator's receiver gets a Git-side push applied; with no receiver and no API edit, the refresh finds one | Manager Flux Applier | | A failed apply is suspended again and retried; a `flux resume` is suspended again with a `ResumedElsewhere` Event; deleting the trigger hands the `Kustomization` back | Manager Flux Applier | | A manager restarted during an apply suspends the `Kustomization` as it stops; the new one completes that same operation and leaves it suspended | [Flux Restart Recovery](../test/e2e/flux_restart_recovery_e2e_test.go) | -| Two API edits seconds apart end at the later value with no extra commit, and Flux catches up by itself | [Bi-directional (Flux)](../test/e2e/flux_bi_directional_e2e_test.go) | +| Two API edits seconds apart end at the later value with no extra commit | [Bi-directional (Flux)](../test/e2e/flux_bi_directional_e2e_test.go) | | A Git-side push with an API edit on top lands both, and Flux applies the replayed commit | Bi-directional (Flux) | | A new `GitRepository` revision from a commit outside the applied folder is enough for the `Kustomization` to apply and revert drift | Bi-directional (Flux) | | A Git to cluster to Git round trip settles without a commit loop; a SOPS-encrypted Secret round-trips without a reverse commit; a Git revert removes the live object and the committed file | Bi-directional (Flux) | | A refused edit with `onRefusal: Reapply` is reverted by a resume, with no commit, and settles | [Manager Refusal Reapply](../test/e2e/refusal_reapply_e2e_test.go) | +| An API create, and a delete and create with the same content, are adopted, so a Git-side delete prunes them; an edit made during an apply is in Git at once and back on the cluster after the `Repair`; a Git-side push is applied while edits keep arriving, and every edit is kept; a Git-side push survives a resync forced before its apply, and a Git-side addition survives one under `prune.mode: Always`; an API edit asks for no apply; another applier's object is neither mirrored nor adopted; a save made across a Git-side change resolves from the push | [Bi-directional applier echoes (Flux)](../test/e2e/bi_directional_echoes_e2e_test.go) | +| A Git-side push made while the previous one is being applied stays in Git and on the cluster | [Bi-directional push race (Flux)](../test/e2e/bi_directional_push_race_e2e_test.go) | +| After a restart, an API edit and an API create made while the operator was down end in Git and on the cluster, nothing reverts the edit, a Git-side push made meanwhile is applied, and the create is adopted | [Bi-directional restart (Flux)](../test/e2e/bi_directional_restart_e2e_test.go) | `task test-e2e` runs the Manager specs and Flux Restart Recovery, and `task test-e2e-bi-directional` the bi-directional corner. Not covered end to end yet: a remote diff --git a/docs/bi-directional.md b/docs/bi-directional.md index 69982b69..8a119fac 100644 --- a/docs/bi-directional.md +++ b/docs/bi-directional.md @@ -89,7 +89,7 @@ The two appliers get the same contract by different means: | Held by | A `FluxTrigger` | An `ArgoCDTrigger` | | Kept from applying on its own by | `spec.suspend`, which the trigger owns | Automated sync off, which you set on the `Application` | | One apply is | A resume for one reconcile, then a suspend | A sync operation pinned to the commit | -| Catches up with GitOps Reverser's own pushes | Yes, once the branch is quiet for `quietPeriod` | No: nothing it holds goes stale | +| When it is asked to apply | [The same four reasons](#when-gitops-reverser-asks-for-an-apply), `CatchUp` included | The same four reasons | | The tool's own push webhook | Removed: GitOps Reverser's receiver replaces it | May stay: with automated sync off it only compares | | A human override | `flux resume` applies once; it is suspended again within a minute | Sync applies; GitOps Reverser waits for it to end | | Handing it back | Delete the trigger, or stop naming it from every `GitTarget`: the `Kustomization` is resumed | Delete the trigger, or stop naming it, then switch automated sync on | @@ -117,27 +117,42 @@ apply nobody checked: None of these can be filtered from outside. They are switched off, and GitOps Reverser makes the requests instead. -## The quiet branch +## Flush, then apply -Every apply GitOps Reverser asks for waits for a **quiet branch**: no open commit window and no -pending writes, on this branch worker or on any other worker writing the same repository and branch -through another `GitProvider`. That is the moment nothing live is missing from the tip. +Every apply GitOps Reverser asks for goes out after a **flush**. When an apply comes due, the branch +worker closes the open commit window, so its edits become a decided write, and takes the **flush +mark**: the last write decided so far. It publishes the writes up to the mark at once, without +waiting for the push cooldown, and asks for the tip that publication produced. Every edit decided +before the apply is therefore in the commit it applies. + +An edit decided after the mark does not hold the apply back. It is recorded in the applier's apply +window from the mark on, so an apply that writes over it is withheld from Git and repaired +([the life of an apply](#the-life-of-an-apply)). A user who keeps editing therefore never holds a +Git-side change back for as long as they type. ```mermaid stateDiagram-v2 direction LR - Quiet --> Busy: an API edit opens a commit window - Busy --> Busy: more edits, commits and pushes - Busy --> Quiet: every window closed, every write pushed - note right of Quiet - An apply starts only here, - after the decision is checked again + [*] --> Due: a reason is pending, past its backoff + Due --> Flushing: the open commit window is closed, the flush mark taken + Flushing --> Sent: the writes up to the mark are published + Sent --> [*]: the operation is followed to its end + note right of Flushing + An edit decided after the mark is recorded + in the apply window; it does not hold the send end note ``` -A foreign commit does not wait for open windows to close on their own. GitOps Reverser closes them, -replays the pending writes onto the foreign commit, pushes, and asks for the commit it just pushed. -The applied commit holds the API side's values, so applying it reverts nothing: +Three states still hold an apply back, because in them no flush can be made: a commit window a +[`CommitRequest`](configuration.md#commitrequest) holds (a save defines its own commit boundary, and +the apply waits for it), a missing parent branch being recovered, and a checkout a failed write left +dirty. In the last two the branch worker cannot say what is published. + +A foreign commit is applied after the same flush. A push webhook closes the open window at once; +without one, the flush's own push is rejected and its fetch finds the commit. Either way the pending +writes are applied onto the foreign commit and pushed, and the applier is asked once, for the commit +that holds both sides. It is never asked for the foreign commit on its own first. The applied commit +holds the API side's values, so applying it reverts nothing: ```mermaid sequenceDiagram @@ -163,18 +178,63 @@ sequenceDiagram | --- | --- | --- | | `ForeignCommit` | The branch gained a commit GitOps Reverser did not push | Every applier declared on the branch | | `Refusal` | An edit was refused and the target has `onRefusal: Reapply` | The refusing target's applier | -| `Repair` | An apply reverted an API edit made while it ran; Git kept the edit | That applier | -| `CatchUp` | The branch has been quiet for the `FluxTrigger`'s `quietPeriod` after GitOps Reverser's own pushes, since the operator started, or since a `GitTarget` first named the trigger | Every Flux `Kustomization` on the branch | - -GitOps Reverser's own commits hold values it read from the cluster, so applying them changes -nothing, and only Flux is asked to catch up with them: its inventory learns of an object only from -an apply of a commit that holds it, and an object it never applied is not pruned when Git later -deletes it. An `Application` with automated sync off applies nothing by itself, so nothing it holds -goes stale. +| `Repair` | An apply reverted an API edit made after the flush; Git kept the edit | That applier | +| `CatchUp` | To **adopt**: the API created an object in a folder the applier applies, or a resync added an object to Git. To **converge**: once when the operator starts, and once when a `GitTarget` first names the trigger | To adopt, that folder's applier; to converge, every applier on the branch at start, and the newly named one | + +The same rule holds for a Flux `Kustomization` and an Argo CD `Application`. The two differ only in +how an apply is carried out, never in when one is asked for. + +**An API edit asks for no apply.** The cluster already runs the edit, and the object keeps the owner +labels or tracking annotation an earlier apply stamped on it. GitOps Reverser's own pushes leave +nothing pending either. + +**A create is adopted.** An applier prunes only what it applied: Flux deletes only objects that +carry its owner labels and are in its inventory, and Argo CD prunes only objects that carry its +tracking annotation. Only an apply stamps them. An object the API created carries neither, even when +its content equals what Git holds, so when Git later deletes its file the applier leaves it running, +and a later resync writes it back into Git. A live create (an `ADDED` event whose newest field +managers are not an applier's) in a folder an applier applies therefore leaves a `CatchUp` pending, +and so does deleting an object and creating it again under the same name. An adopt is never skipped +as already applied, because applying the same commit again is what stamps the object. A burst of +creates is one apply: a pending `CatchUp` is one flag, and one attempt per applier is out at a time. + +**A converge asks nothing when it is already done.** The branch worker keeps, for each applier, the +commit its last successful operation applied, and recovers it on restart from the trigger's +`status.operation`. A converge of that commit asks nothing. + +**No apply before every initial snapshot is published.** An applier is asked for nothing, for any +reason, until every resource collection of every `GitTarget` it covers has a settled snapshot +(applied, refused or failed) and the branch worker holds no resync of that `GitTarget` still to +publish. At startup that is the moment the edits made while the operator was down are in Git; for a +`GitTarget` that newly names the trigger, the moment its folder holds the cluster. An earlier apply +would revert what is not mirrored yet. A `GitTarget` with no collection planned yet (its +`WatchRule` not resolved, or a CRD not discovered) and one whose watch is `Blocked` hold their +applier back the same way. A reason that comes due before stays pending, and so does one while the +remote does not carry the branch, because there is no commit to apply. + +**The folder exists before it is applied.** A resync of a `GitTarget` that names a trigger, into a +folder the branch does not hold yet, commits the folder's bootstrap files (`README.md`, +`.gittargetignore`, and `.sops.yaml` with encryption) even when there is no object to write: Argo CD +retries a sync of a missing path for minutes, and Flux fails it. The bootstrap files are never +deleted, so the folder outlives its last resource. Several `GitTarget`s may name one trigger, for a `Kustomization` or `Application` whose path covers several folders: the applier is asked once for all of them. +## Objects another applier owns + +A folder a trigger applies belongs to that trigger's applier. An object that another applier owns +is not mirrored into it: a live change to it is not mirrored, and a resync neither writes it nor +removes its document, even under `prune.mode: Always`. Mirrored, it would be applied by this applier +too, and two appliers applying one object fight each other. + +The owner is read from the object before its bookkeeping is stripped: Flux's +`kustomize.toolkit.fluxcd.io/name` and `kustomize.toolkit.fluxcd.io/namespace` labels, or else Argo +CD's `argocd.argoproj.io/tracking-id` annotation. Each excluded object is logged once and counted in +`apply_ownership_conflicts_total`. An object no applier owns, or one the folder's own applier owns, +is mirrored as usual. A folder with no trigger excludes nothing: mirroring another applier's objects +one way is what such a folder can exist for. + ## The life of an apply An apply is an **operation**, recorded on the trigger's `status.operation` before the applier is @@ -183,8 +243,8 @@ asked, so a restarted operator finds it and follows it to its end: ```mermaid stateDiagram-v2 [*] --> Due: ForeignCommit, Refusal, Repair or CatchUp - Due --> Due: tip moved or an edit is pending, decide again later - Due --> Requested: quiet branch, decision still holds, operation recorded + Due --> Due: held back, or the tip moved before the request, decide again + Due --> Requested: flushed, decision still holds, operation recorded Requested --> Running: the applier is seen working on it Running --> Settling: Flux only, applied and suspended again Running --> Succeeded @@ -202,23 +262,32 @@ stateDiagram-v2 ``` `Due` lives only in the branch worker's memory: an apply it means to ask for, and nothing shows it -until it is asked. Every other state is `status.operation.phase`. While an operation is in flight -(`Requested`, `Running`, `Settling` or `Unresolved`), the applier is asked nothing else. +until it is asked. It is held back by the applier's backoff, by the +[initial-snapshot rule](#when-gitops-reverser-asks-for-an-apply), and by the three states in which +[no flush can be made](#flush-then-apply). Every other state is `status.operation.phase`. While an +operation is in flight (`Requested`, `Running`, `Settling` or `Unresolved`), the applier is asked +nothing else. - **Just before it asks, GitOps Reverser checks the decision again**: the commit is still the branch - tip, no edit waits to be published, and each `GitTarget` still names this trigger on this branch, - is still admitted by its `ClusterProvider`s, and, for a `Refusal`, still consents; and the trigger - still holds the applier. When any of that - changed, it asks nothing and decides again at the next quiet moment. It never asks for something - weaker than what it was told. -- **An edit made while the apply runs keeps its value in Git.** The apply writes the older commit - over it on the cluster. GitOps Reverser recognizes that write by the applier's field manager, - withholds it from Git, and asks for a `Repair`, which applies the tip that holds the edit. Only a - write proven to be the applier's is withheld: its field manager alone, with an entry newer than the - edit. One that ties with the edit is mirrored, and a delete never is withheld. Each such write is + tip, each `GitTarget` still names this trigger on this branch, is still admitted by its + `ClusterProvider`s, and, for a `Refusal`, still consents; and the trigger still holds the applier. + When any of that changed, it asks nothing and decides again, after a new flush. It never asks for + something weaker than what it was told. +- **The apply's own writes are not mirrored.** Inside the apply window, a write whose newest field + managers are the applier's alone, made since the request was sent, to an object nobody edited in + the window, is the apply landing. It is kept out of Git and counted in + `apply_echoes_total{outcome="not_mirrored"}`: mirrored, it would carry the applied commit's values + onto a newer tip and revert a Git-side push. The applier's managers alone with no window open are + mirrored as an edit and counted as `late`. +- **An edit made after the flush keeps its value in Git.** The apply writes the older commit over it + on the cluster. GitOps Reverser recognizes that write by the applier's field manager, withholds + it from Git, and asks for a `Repair`, which applies the tip that holds the edit. Only a write + proven to be the applier's is withheld: its field manager alone, with an entry as new as the edit + or newer (in the same second, the edit arrived first). One that ties with another field manager + is mirrored, and a delete never is withheld. Each such write is counted in `apply_window_collisions_total`. - **A failed operation is decided again from scratch**, after that applier's own backoff (2 seconds - doubling to 5 minutes), on a quiet branch, at the tip as it is then. + doubling to 5 minutes), after a flush, at the tip as it is then. - **An operation still in flight after 5 minutes is `Unresolved`.** It is still followed, nothing ends it by timer, and nothing more is asked of that applier until it completes. @@ -228,12 +297,13 @@ it applied. ## Telling GitOps Reverser that somebody else pushed -GitOps Reverser learns that the branch moved in three ways, and each feeds the same decision: +GitOps Reverser learns that the branch moved in four ways, and each feeds the same decision: | Signal | Latency | Configure | | --- | --- | --- | | A push webhook from the Git host | Seconds | `GitProvider.spec.receiver` | | A rejected compare-and-swap push | At the next publication | Nothing: it is how every push works | +| A Flux `GitRepository` that fetched another commit than the one asked for | At the next apply attempt | Nothing: the branch worker then looks at the remote, as for a push webhook | | The periodic refresh | Within `--git-refresh-interval` (`10m`) plus the 5-minute reconcile tick | Nothing: on by default | To serve the receiver: @@ -269,11 +339,13 @@ push, so a snapshot would publish them over the incoming change. ## When both sides edit the same object The API side wins. When a push is rejected because somebody else pushed first, the branch worker -fetches the new tip, re-plans every pending write against it, and pushes again. Each pending write -carries the whole object as the watch captured it, so writing it onto the new tree either changes -nothing (the other writer made the same change) or writes the API side's value over the other -writer's. There is no merge: a watch event has no baseline that says which fields changed on each -side. [API-first publication](api-first-publication.md) has the replay in detail. +fetches the new tip, applies every pending write onto it again, and pushes. A pending write carries +the objects whose plan changed the tree when it was decided, each whole as the watch captured it, so +writing one onto the new tree either changes nothing (the other writer made the same change) or +writes the API side's value over the other writer's. An object that changed nothing was dropped when +the write was decided, so it never overwrites the push. There is no merge: a watch event has no +baseline that says which fields changed on each side. +[API-first publication](api-first-publication.md) has the replay in detail. ### What a write does to the file @@ -340,7 +412,7 @@ until something applies Git over it. | Policy | Response to a refused edit | Trade | | --- | --- | --- | | `Ignore` (default) | Report the refusal | Recovery is left to a human or a later apply | -| `Reapply` | Ask the target's applier to apply the tip, once the branch is quiet | The refused edit is reverted; nothing still waiting to be published is | +| `Reapply` | Ask the target's applier to apply the tip, after a flush | The refused edit is reverted; nothing still waiting to be published is | `Reapply` needs an apply trigger. It fires only for an edit to a document the folder already holds, never for a write that removes one, and a reverted edit does not clear the refusal: the target @@ -349,9 +421,12 @@ returns to `Ready=True` only after a resync covering the refused scope. The elig ## What is still exposed, and what it costs -- **An edit made while a requested apply runs** is reverted on the cluster until the `Repair`. It - keeps its value in Git. It can happen only during an apply GitOps Reverser asked for: after a - foreign push, a refusal with `Reapply`, or a Flux catch-up. +- **An edit made after the flush, while a requested apply runs,** is reverted on the cluster until + the `Repair`. It keeps its value in Git. It can happen only during an apply GitOps Reverser asked + for: after a foreign push, a refusal with `Reapply`, a `Repair`, or a `CatchUp`. +- **An edit through another `GitProvider` on the same repository and branch, made after the + flush,** is not protected. The flush publishes what that worker decided before the apply, but it + has no apply window, so it mirrors the apply's revert of a later edit as an edit of its own. - **A human** running `flux resume` or pressing Sync applies whenever they do. GitOps Reverser suspends a resumed `Kustomization` again within a minute. - **GitOps Reverser becomes the only path to a deployment.** While it is down, the shared folders diff --git a/docs/components.md b/docs/components.md index e9b3d62f..7f962f2e 100644 --- a/docs/components.md +++ b/docs/components.md @@ -119,7 +119,7 @@ write to that branch goes through its single event loop. | Component | Path | Role | |---|---|---| -| `WorkerManager` and `BranchWorker` | [`internal/git/`](../internal/git/) | clone, plan, commit window, atomic push, conflict replay | +| `WorkerManager` and `BranchWorker` | [`internal/git/`](../internal/git/) | clone, plan, commit window, atomic push, conflict replay, and when to ask an applier: the flush, the apply window, adopt and converge | | Manifest analyzer | [`internal/manifestanalyzer/`](../internal/manifestanalyzer/) | folder inventory, the structure-only acceptance gate, and resync planning | | Document editor | [`internal/git/manifestedit/`](../internal/git/manifestedit/) | edits one YAML document in place inside a multi-document file | | Manifest report | [`internal/manifestreport/`](../internal/manifestreport/) | projects a live object into a comparable manifest | @@ -183,10 +183,10 @@ flowchart LR | Observer | Observes | Cluster | Mechanism | |---|---|---|---| -| Controller cache | the operator's own custom resources, plus `Namespace` (label changes only) | config plane | controller-runtime informers ([`cmd/main.go:1130`](../cmd/main.go#L1130)) | -| API-surface triggers | **types**: `CustomResourceDefinition` and `APIService` objects | config plane only | one private `dynamicinformer` per resource ([`internal/watch/manager_catalog.go:458`](../internal/watch/manager_catalog.go#L458)) | -| Catalog refresh | **types**: the served API surface | every cluster, including remote | polled every 30s, plus rule changes and the trigger above ([`internal/watch/manager.go:288`](../internal/watch/manager.go#L288)) | -| Target watches | **objects** of claimed types | source cluster | raw `dynamic ... Watch()`, no informer and no cache ([`internal/watch/target_watch.go:969`](../internal/watch/target_watch.go#L969)) | +| Controller cache | the operator's own custom resources, plus `Namespace` (label changes only) | config plane | controller-runtime informers ([`cmd/main.go:1223`](../cmd/main.go#L1223)) | +| API-surface triggers | **types**: `CustomResourceDefinition` and `APIService` objects | config plane only | one private `dynamicinformer` per resource ([`internal/watch/manager_catalog.go:447`](../internal/watch/manager_catalog.go#L447)) | +| Catalog refresh | **types**: the served API surface | every cluster, including remote | polled every 30s, plus rule changes and the trigger above ([`internal/watch/owner.go:389`](../internal/watch/owner.go#L389)) | +| Target watches | **objects** of claimed types | source cluster | raw `dynamic ... Watch()`, no informer and no cache ([`internal/watch/target_watch.go:1420`](../internal/watch/target_watch.go#L1420)) | Two more paths reach the operator without being watches at all. The audit webhook is a **push** from kube-apiserver, and the admission webhooks are synchronous requests. Neither observes state. The @@ -212,8 +212,8 @@ stopped on `Forbidden` could then never be re-armed. Not in the observers. Three places currently recompute a GitTarget's watch set from the same watched-type table: -- the runner, at [`target_watch.go:105`](../internal/watch/target_watch.go#L105); -- the status roll-up, at [`stream_readiness.go:125`](../internal/watch/stream_readiness.go#L125); +- the runner, at [`target_watch.go:286`](../internal/watch/target_watch.go#L286); +- the status roll-up, at [`stream_readiness.go:145`](../internal/watch/stream_readiness.go#L145); - the resident table store itself, in [`watched_type_resolver.go`](../internal/watch/watched_type_resolver.go). They agree today only because each re-derives the same answer from the same inputs, and the runner diff --git a/docs/configuration.md b/docs/configuration.md index 9dcbf977..b2f67c66 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -137,7 +137,6 @@ The references are immutable: to hold another `Kustomization`, delete the trigge |---|---|---| | `clusterProviderRef` | `{"name":"default"}` | The `ClusterProvider` whose cluster holds the `Kustomization`. Its `accessFrom` must admit the trigger's namespace | | `kustomizationRef` | **required** | The `Kustomization` that applies the folders of the `GitTarget`s naming this trigger. See [who applies this folder](#who-applies-this-folder-specapplytriggerref) | -| `quietPeriod` | `30s` | How long the branch must stay quiet after GitOps Reverser's own pushes before a `CatchUp` | ### `ArgoCDTrigger` (namespaced): the Argo CD `Application` GitOps Reverser asks to sync @@ -180,7 +179,7 @@ there. ## `GitProvider` -`GitProvider` defines the Git remote, credentials, allowed branches, push strategy, and commit +`GitProvider` defines the Git remote, credentials, allowed branches, push receiver, and commit behavior. The important fields are: @@ -212,7 +211,7 @@ spec: ### What this repository is being used for (`status.branches`) -`status.branches` lists the branches the `GitTarget`s in this namespace are configured to write, +`status.branches` lists the branches the `GitTarget`s naming this provider are configured to write, with how many reference each: ```yaml @@ -819,9 +818,11 @@ editing them affects windows that open afterward, never one already running. A for that window. One branch worker holds one live window, bound to an author and target. Interleaved authors or -targets split a burst. Atomic writes, buffer limits, shutdown, and an `attach: Next` request can -close a window early. Push cooldown is independent: a zero timer does not promise an immediate -remote push. See the [complete trigger rules](spec/commit-window-refactor.md). +targets split a burst. Atomic writes, a resync, buffer limits, shutdown, an `attach: Next` request, +a push notice from the [receiver](#receiving-push-notifications-specreceiver), and the +[flush](bi-directional.md#flush-then-apply) before an apply can close a window early. Push cooldown +is independent: a zero timer does not promise an immediate remote push. See the +[complete trigger rules](spec/commit-window-refactor.md). Both values are **Go duration strings with a mandatory unit**, at most `24h`, checked by the API server: `"750ms"`, `"1.5m"` and `"1m30s"` are accepted, while `"30"`, `".5s"`, `"5 seconds"` and @@ -1134,8 +1135,8 @@ tick". [`gitopsreverser_git_fetches_total{reason="refresh"}`](interpreting-metri is what it costs. A refresh that finds a commit GitOps Reverser did not push is a foreign commit like any other: the -target's [applier](#who-applies-this-folder-specapplytriggerref) is asked to apply it once the branch is -quiet. That makes the refresh the backstop for deployments when no push webhook reaches the +target's [applier](#who-applies-this-folder-specapplytriggerref) is asked to apply it after a flush. +That makes the refresh the backstop for deployments when no push webhook reaches the operator. It is not a latency mechanism: it buys bounded staleness with no inbound surface and no Git-host configuration. Seconds-fresh is the job of the [push receiver](#receiving-push-notifications-specreceiver). @@ -1174,7 +1175,6 @@ spec: kustomizationRef: namespace: flux-system # default: the trigger's namespace name: checkout - quietPeriod: 30s --- apiVersion: configbutler.ai/v1alpha3 kind: GitTarget @@ -1192,7 +1192,6 @@ spec: | --- | --- | --- | | `FluxTrigger.spec.clusterProviderRef`, `ArgoCDTrigger.spec.clusterProviderRef` | `{name: default}` | The `ClusterProvider` whose cluster holds the applier, defaulted as a `GitTarget`'s is. Immutable | | `FluxTrigger.spec.kustomizationRef` | required | A Flux `Kustomization` whose `sourceRef` is a `GitRepository` that follows a branch. Immutable | -| `FluxTrigger.spec.quietPeriod` | `30s` | How long the branch must stay quiet after GitOps Reverser's own pushes, or after a `GitTarget` first names the trigger, before a `CatchUp` request. At most `1h` | | `ArgoCDTrigger.spec.applicationRef` | required | An Argo CD `Application` on a single source whose `targetRevision` names a branch, with automated sync off. Immutable | | `GitTarget.spec.applyTriggerRef` | none | The trigger, by `kind` and `name`, in the `GitTarget`'s namespace | @@ -1229,16 +1228,41 @@ spec: name: prod-eu-1-apps ``` -**When a request is made.** Only on a quiet branch: no open commit window and no pending writes on -the branch worker, or on any other worker writing the same repository and branch through another -`GitProvider`. +**When a request is made.** After a [flush](bi-directional.md#flush-then-apply): when an apply comes +due, the branch worker closes the open commit window, publishes the writes decided up to that +moment at once, and asks for the tip that publication produced. An edit decided after that is +recorded in the apply window, so an apply that writes over it is withheld from Git and repaired. The +branch workers of other `GitProvider`s writing the same repository and branch flush too, and the +request waits until they have. A registered `CommitRequest` (a save sets its own commit boundary), a +parent branch being recovered, or a checkout a failed write left dirty holds the request back. No +applier is asked for anything until every initial snapshot of every `GitTarget` naming its trigger +is published, and nothing is asked while the remote does not carry the branch. The rule is the same +for a `Kustomization` and an `Application`. | `reason` | When | Sent to | | --- | --- | --- | -| `ForeignCommit` | The branch gained a commit GitOps Reverser did not push. Open windows are closed and pending writes replayed and pushed first, so the request names that pushed commit | Every applier declared on the branch, once per applier | +| `ForeignCommit` | The branch gained a commit GitOps Reverser did not push. The pending writes are replayed onto it and pushed first, so the request names that pushed commit | Every applier declared on the branch, once per applier | | `Refusal` | An eligible refused edit, with `spec.onRefusal: Reapply` | This target's applier | -| `Repair` | An apply reverted an API edit made after it was asked for; Git kept the edit | That applier | -| `CatchUp` | The branch has been quiet for `quietPeriod` since GitOps Reverser's own pushes, since the operator started, or since a `GitTarget` first named the trigger | Every `Kustomization` declared on the branch; never an `Application` | +| `Repair` | An apply reverted an API edit made after the flush; Git kept the edit | That applier | +| `CatchUp` | To adopt: the API created an object in the folder, or a resync added an object to Git. To converge: once when the operator starts, and once when a `GitTarget` first names the trigger | To adopt, that folder's applier; to converge, every applier declared on the branch at start, and the newly named one | + +An API edit asks for no apply, and neither does a push of GitOps Reverser's own: the cluster already +runs what they hold. A create is different, because an applier prunes only what it applied: an +object the API created (an `ADDED` event whose newest field managers are not an applier's) carries +neither Flux's owner labels nor Argo CD's tracking annotation, so a later Git-side delete of its +file would leave it running. An adopt is therefore applied even at the commit the applier last +applied, since that apply is what stamps the object. A converge asks nothing when the applier's last +successful operation, as `status.operation` records it, applied the tip. + +**An object another applier owns is not mirrored into the folder.** When a `GitTarget` names a +trigger, an object whose Flux owner labels (`kustomize.toolkit.fluxcd.io/name` and +`kustomize.toolkit.fluxcd.io/namespace`) or Argo CD tracking annotation +(`argocd.argoproj.io/tracking-id`) name an applier other than the trigger's is excluded: a live +change to it is not mirrored, and a resync neither writes it nor removes its document, even under +`prune.mode: Always`. Two appliers applying one object fight each other. Each excluded object is +logged once and counted in `gitopsreverser_apply_ownership_conflicts_total`. A `GitTarget` without +`spec.applyTriggerRef` excludes nothing: mirroring another applier's objects one way is what such a +folder can exist for. **What it reports.** The trigger reports its applier and the operation it is running. The branch worker writes `status.operation` and `status.repairPending` at every move of the operation, so they are @@ -1263,7 +1287,6 @@ status: branch: main otherConsumers: # what else applies this GitRepository, held by no FluxTrigger - {kind: HelmRelease, namespace: platform, name: ingress} - prefetch: false operation: phase: Succeeded # Requested | Running | Settling | Unresolved | Succeeded | Failed reason: ForeignCommit # ForeignCommit | Refusal | Repair | CatchUp @@ -1282,9 +1305,8 @@ status: | `operation.token` | Identifies the request in the applier's own status: the `reconcile.fluxcd.io/requestedAt` value of the resume, or the `gitops-reverser request` entry in an Argo CD operation's `info` | | `operation.suspendToken` | Flux only: the token of the suspend after the apply. Once it is handled, nothing that started before the suspend is still running | | `operation.appliedRevision` | The revision the applier reports it applied. It can differ from `commit` when the branch moved between the check and the apply | -| `repairPending` | `true` while an apply reverted an API edit made after it was asked for: Git kept the edit, and the cluster runs the older value until the `Repair` | +| `repairPending` | `true` while an apply reverted an API edit made after its flush: Git kept the edit, and the cluster runs the older value until the `Repair` | | `source.otherConsumers` | Flux only: the `Kustomization`s and `HelmRelease`s that apply the same `GitRepository` and that no `FluxTrigger` holds. Information only; it never affects `Ready` | -| `source.prefetch` | Flux only: `true` while every consumer of the source is held, so GitOps Reverser may fetch the `GitRepository` ahead of the apply | A `Kustomization` resumed outside GitOps Reverser (`flux resume`) is suspended again and recorded as an `Unresolved` operation with no `reason` and no `commit`, which completes once the suspend is @@ -1338,10 +1360,10 @@ spec: | Value | What happens | | --- | --- | | `Ignore` (default) | The refusal is reported and nothing else happens | -| `Reapply` | Once the branch is quiet, this target's [applier](#who-applies-this-folder-specapplytriggerref) is asked to apply the current tip, with reason `Refusal`. The refused edit is reverted when the applier applies it; the request asks, and the applier's own status says whether it did | +| `Reapply` | After a flush, this target's [applier](#who-applies-this-folder-specapplytriggerref) is asked to apply the current tip, with reason `Refusal`. The refused edit is reverted when the applier applies it; the request asks, and the applier's own status says whether it did | `Reapply` requires `spec.applyTriggerRef`; a target without one is rejected at admission. The request goes -to this target's applier alone, and it waits for a quiet branch, so an allowed edit still in a +to this target's applier alone, and it goes out after a flush, so an allowed edit still in a commit window is published before the apply and is not reverted with the refused one. An `Application` is asked to sync the tip. When a `Kustomization`'s `GitRepository` has to fetch to reach the tip, the `Kustomization`s a `FluxTrigger` holds stay suspended and ignore the new artifact; @@ -1360,7 +1382,7 @@ from here, so both are excluded. Establishing that takes more than the refusal's kind, and more than the file. A render refusal fires for a brand-new resource an `images:` entry would override; removing one document from a file that holds two leaves the file in place, which looks exactly like an edit. Eligibility is therefore -decided per document, and a write that removes one anywhere in the same flush is excluded whole. +decided per document, and a write that removes one anywhere is excluded whole. Folder-level refusals, such as invalid YAML or unsupported content, never trigger it: they need a correction to the folder. @@ -2574,10 +2596,10 @@ When attribution is enabled, these flags tune the join: directly. **Watching a type your audit policy excludes costs more than attribution.** Every removal on such - a type spends the full grace before shipping as the committer, because the fact it is waiting for - was never recorded. If a watched type's commits are consistently committer-authored, check the - audit policy before anything else: a type in the policy's `level: None` list can never be - attributed, and no operator-side setting changes that. + a type spends the full grace before shipping unresolved, because the fact it is waiting for was + never recorded. If a watched type's commits are consistently authored + `unknown (attribution unresolved)`, check the audit policy before anything else: a type in the + policy's `level: None` list can never be attributed, and no operator-side setting changes that. - `--author-attribution-max-facts-per-type` (default `4096`) and `--author-attribution-max-facts` (default `65536`): how many facts the in-memory index holds, per type and in total, evicted oldest-first. Per-type is the fair cap: a burst on one noisy type must not evict every other type's diff --git a/docs/definitions.md b/docs/definitions.md index b1a69e46..f1969221 100644 --- a/docs/definitions.md +++ b/docs/definitions.md @@ -144,8 +144,9 @@ the parent branch is not followed. **Commit window.** The interval a branch worker batches one author's writes to one target over before it commits. Only a write opens one; nothing else does. It closes on its `idleTimeout` of silence or -its `maxDuration` after opening, whichever comes first. Windows do not merge across authors, which -is why attribution can be per-commit. +its `maxDuration` after opening, whichever comes first, or earlier for a flush, unless a +`CommitRequest` holds it. Windows do not merge across authors, which is why attribution can be +per-commit. **Attach.** What a `CommitRequest` does to a commit window: it gives the window its message and replaces the window's timers with its own. A request attaches to at most one window and never opens @@ -156,8 +157,11 @@ one. One word for the act, in the API (`window.attach`, `attachTimeout`), the co commit window, an atomic batch, a resync, or a save's empty record. The branch worker keeps its pending writes in order, commits them, and pushes them on one retry schedule. A pending write can be waiting to become a commit, committed locally and waiting for a push, or replayed onto a remote -that moved. `PendingWrite` and `pendingWrites` in Go. Not "log", -which also names diagnostic logs and suggests a durable journal, which this is not. +that moved. It carries the actions decided once, on its tree: an upsert per object that changed +the tree and a delete per document removed, while a save keeps every object it captured. A replay +applies those actions to the newer tip instead of planning the cluster's picture again. +`PendingWrite` and `pendingWrites` in Go. Not "log", which also names diagnostic logs and suggests +a durable journal, which this is not. **Publication.** One commit-and-push cycle by a branch worker. The thing a `GitTarget` reports in `status.remote`. @@ -221,15 +225,50 @@ happens to the `Kustomization` rather than what the trigger does. **Hand back** `spec.suspend`, so Flux applies on its own again. **Apply request.** What GitOps Reverser sends an applier: apply this commit, for this reason -(`ForeignCommit`, `Refusal`, `Repair`, `CatchUp`). It asks; the operation it starts is followed -until the applier's status shows the apply completed. A reason is **pending** for an applier from +(`ForeignCommit`, `Refusal`, `Repair`, `CatchUp`; a `CatchUp` adopts or converges, both defined +below). It asks; the operation it starts is followed until the applier's status shows the apply +completed. A reason is **pending** for an applier from the moment the branch worker decides it until an attempt **carries** it; an operation that succeeds **answers** it, and one that does not gives it back to pending. A pending `Repair` shows on the trigger as `status.repairPending`. -**Quiet branch.** A branch on which no branch worker writing that repository and branch holds an -open commit window or pending writes. Apply requests go out only on a quiet branch, because only -then is nothing live missing from the tip. +**Flush.** What a branch worker does before it sends an apply request: it closes the open commit +window, takes the **flush mark** (the last write decided so far), and publishes every write up to +the mark at once, without waiting for the push cooldown. The request is sent for the tip that +publication produced. An edit decided after the mark does not hold the request back: the applier's +[apply window](design/apply-trigger.md#the-apply-window) records it from the mark, so an apply that +overwrites it is withheld and a `Repair` puts it back. The flush covers the other branch workers +on the same repository and branch (two `GitProvider`s naming one repository): each closes its own +window and publishes up to a mark of its own, and the request waits until every one is done. + +**Echo.** The applier's own write landing: the watch event of an object the applier wrote while +applying a commit GitOps Reverser asked for. The branch worker knows one by its newest +`managedFields` entries, which are the applier's field managers alone, written at or after the +earliest send of the apply windows still open or carried, to an object nobody edited in the window. +An echo is not mirrored, because its content is the commit's, which Git already holds. An applier +write with no window open is mirrored as an edit, and counted as a late echo. + +**Adopt.** A `CatchUp` that has the applier take ownership of an object the API created in its +folder. An applier prunes only what it applied, and an apply is what stamps an object as applied: +Flux's owner labels and inventory, Argo CD's tracking annotation. So an object nobody applied stays +on the cluster when Git later deletes its file. An adopt is pending once the branch worker observes +an API create in a folder an applier applies, or a resync adds an object to Git, and it is never +skipped as already applied. A trigger's `Adopted` Event names something else, the trigger taking +hold of its `Kustomization`'s `spec.suspend`. + +**Converge.** A `CatchUp` that applies the tip because nothing says the applier runs it: once when +the branch worker starts, and once when a `GitTarget` first names the trigger (per incarnation, by +uid). The worker keeps, per applier, the commit its last succeeded operation applied, recovered on +restart from the trigger's `status.operation`, and a converge of that commit asks nothing. + +**Apply epoch.** A counter on the branch worker, raised at every send of an apply attempt and at +every end of one. When a snapshot's collection starts, the watch side reads it, with whether **Git +is ahead** of the cluster: an applier on the branch has a `ForeignCommit`, `Refusal` or `Repair` +pending, a converge it may be asked for, or an attempt out. A snapshot collected while Git was +ahead, or across an epoch change, holds a picture the apply has made stale, so the worker discards +it and asks for a fresh one once Git is no longer ahead. A `GitTarget`'s initial snapshots, the +first of each collection until one of it settles, are exempt, and its snapshots are held off for at +most 5 minutes. **Receiver.** `GitProvider.spec.receiver`, the endpoint a Git host notifies on a push. The word and its `type` values are Flux's. A delivery to it is a hint to look, never a fact about the branch. @@ -442,3 +481,4 @@ outage. | reconciler, orchestrator (for Flux or Argo CD) | applier | "reconciler" also names our own controllers | | owed, owes, debt, obligation | pending, decided, sent, waiting for | an accounting word names no state: say what is pending, for whom, and what ends it | | fence, fenced (for what a trigger does to its applier) | suspend, held | the `Kustomization` is suspended, and the trigger holds it; say which, and whether the suspend is handled | +| quiet branch | flush | no apply waits for a branch with nothing pending; it goes after a flush | diff --git a/docs/design/applier-echoes.md b/docs/design/applier-echoes.md new file mode 100644 index 00000000..fba749ce --- /dev/null +++ b/docs/design/applier-echoes.md @@ -0,0 +1,693 @@ +# Applier echoes: the applier's own writes, and applying while editing + +> **design**: proposed and built 2026-10-20, on the `design-applier-echoes` branch (PR #425), and +> released as one change. The plan below is kept as it was argued, in the present tense of the code +> it was read from; where the build departs from it, [Decided while +> building](#decided-while-building) says how. Index: [`../INDEX.md`](../INDEX.md). Builds on +> [`apply-trigger.md`](apply-trigger.md) (the apply window, `Repair`, `CatchUp`, and the +> quiet-branch rule this replaces), which now describes the built system, +> [`gittarget-branch-worker-pending-writes.md`](gittarget-branch-worker-pending-writes.md) (decide, +> `advance`, materialize, replay) and +> [`../spec/reconcile-via-watchlist-mark-and-sweep.md`](../spec/reconcile-via-watchlist-mark-and-sweep.md) +> (the resync). + +When GitOps Reverser asks Flux or Argo CD to apply a commit, the applier's write to each object +comes back through the watch. Today that write is mirrored like an API edit, and a mirrored write is +re-planned from the live object onto whatever tip the branch has when it is published. An echo of an +older commit, published after a Git-side push, **reverts that push**, in Git and then on the cluster. + +The fix rests on two ideas: + +- **A write that changes nothing in Git does not exist.** The writer already follows it for a + whole write; this design follows it for every object. +- **A pending write carries the actions decided on its tree, not a picture of the cluster.** Replay + applies those actions to a newer tip. It never plans a stale picture again. + +| | Change | +| --- | --- | +| **F1** | In the apply window, the applier's own write is not mirrored | +| **F2** | A write is a list of actions decided once, on its tree: an unchanged object is no action, a resync is explicit upserts and deletes, a save keeps its objects | +| **F3** | A resync never decides from a snapshot collected while Git was ahead of the cluster, or across an apply | +| **F4** | An object another applier owns is not mirrored into a folder a trigger applies | +| **S1** | Flush, then apply, instead of waiting for a quiet branch | +| **S2** | `CatchUp` only to adopt objects the API created and to converge, the same for Flux and Argo CD, decided in the branch worker; no apply before every initial snapshot is published; `quietPeriod` goes | + +The parts depend on each other: S1 removes a wait that the others make unnecessary, so they are +built and released together. + +## Decisions taken + +1. **The applier is recognized by its field-manager name, fixed per applier kind.** Kubernetes + records two different facts about a write, in two different places: + + | | Where it lives | What it says | Who sets it | + |---|---|---|---| + | Field manager | `metadata.managedFields`, on the object | a name the client sends with the write | the client | + | Service account | the API server's audit log only | who authenticated the request | the API server | + + A `managedFields` entry carries a manager name, an operation and a time, and no user: + + ```yaml + managedFields: + - manager: kustomize-controller # the apply + operation: Apply + time: "2026-10-20T07:32:42Z" + - manager: kubectl-edit # a person's edit + operation: Update + time: "2026-10-20T07:33:10Z" + ``` + + F1 reads only these. The names are constants in + [`apply_window.go`](../../internal/git/apply_window.go): `kustomize-controller`, fixed inside + Flux; `argocd-controller`, which Argo CD passes to every apply; and + `argocd-application-controller`, for a `Replace=true` sync that sends no manager, so the API server + takes the user agent. They do not change when the applier impersonates another account: the audit + user changes, the manager name does not. So the name is on the object, delivered with every event, + and needs no configuration and no audit webhook. If a later Flux or Argo CD release renamed its + manager, the constants would follow. + +2. **No service account on the triggers, and no check against the audit author.** The field manager + decides; nothing else is consulted. + +3. **Trust, not defense.** A write under the applier's manager name is the applier's. Whoever can + write as Flux or Argo CD already controls what is deployed, so the design guards only against + writers that are not malicious and still look alike: an undo, a same-second tie, a mutating + webhook inside the apply, a late echo. + +4. **Writes to one object from Git and from the API within seconds are rare, so they are defined + rather than prevented.** Each collision left open is listed with how close the two writes must be, + what it does, and whether it heals. + +5. **Git is applied as fast as it can be.** No wait for a quiet branch: the flush publishes what is + pending, and the apply goes. + +6. **Incoming Git changes are applied after the replay, in one apply with the worker's own writes.** + Never on their own first. + +7. **Flux and Argo CD are kept in line by one rule, decided in the branch worker.** The two appliers + differ only in how an apply is carried out (resume and suspend a `Kustomization`, sync an + `Application`), never in when one is asked for. This reverses two decisions in + [`apply-trigger.md`](apply-trigger.md): that an `Application` is never asked to catch up, and that + a newly bound `Application` waits for the next push. A sync, hooks included, on an API create or on + binding is rare, and one rule for both appliers is worth it. + +8. **A folder a trigger applies belongs to that trigger's applier** (F4). + +9. **A save is the strongest API intent, and is never trimmed.** A `CommitRequest` keeps every object + it captured, and its outcome is settled by the push, as the + [CommitRequest contract](../spec/commitrequest-design.md) says. Re-asserting a save over a + concurrent Git-side change of the same object is the "API wins" rule. + +## The race + +The regression spec pushes A, waits until the applier made A live, and pushes B at once. Argo CD +fails it, deterministically: + +```mermaid +sequenceDiagram + autonumber + participant G as Gitea (main) + participant W as Branch worker + participant A as Argo CD + participant K as ConfigMap (live) + + Note over W: 07:32:38 last push of its own, PushCooldown (5s) runs until 07:32:43 + G->>W: push A lands, refresh finds it (07:32:41) + W->>A: ForeignCommit: sync commit A + A->>K: apply A + K-->>W: watch event, managers argocd-controller (07:32:42) + Note over W: withholdLateApply mirrors it
decided on A: nothing to commit
it stays pending for the push, which waits for the cooldown + G->>G: test pushes B (07:32:41.9) + W->>G: 07:32:43 push, expecting tip A + G-->>W: rejected: "Remote branch not in expected state" + Note over W: replayOntoRemote: reset to B, re-plan the pending write
from the live object (still A) onto B + W->>G: commit "[UPDATE] bi-timing" says A: B is reverted + W->>A: ForeignCommit: sync the revert +``` + +Flux passes the same spec only when its echo lands after the push cooldown. + +**The root cause is ordering.** An API edit is a decision in the worker's queue: decided after older +writes, replayed after them, so it wins. A Git-side push is not a decision. It becomes the tree the +queue is replayed onto, so every pending decision lands *after* it, and each is re-planned from its +picture of the cluster: a decision taken from an older picture overwrites what the push changed. A +stale picture reaches a commit in six ways, and each change closes one: + +| A stale picture, held as | Closed by | +| --- | --- | +| the applier's own write, mirrored as an edit | F1 | +| an object that changed nothing, kept in a pending write and replayed | F2 | +| a resync's full membership, swept again on a newer tip that added an object | F2 | +| a resync snapshot collected before, or across, the apply that changed what it saw | F3 | +| an edit made around the flush, overwritten by the apply and never repaired | S1 | +| an initial snapshot that has not yet captured edits made while the operator was down | S2 | + +## F1: the applier's own write in the apply window is not mirrored + +`withholdLateApply` ([`apply_window.go`](../../internal/git/apply_window.go)) judges every live +event in an applier's open apply window. F1 keeps its judgment of edits as built and changes what +happens to the applier's own write when nobody edited the object. + +```mermaid +flowchart TD + E["live event, apply window open or carried
for an applier that covers its GitTarget"] --> D{"DELETE?"} + D -- yes --> M0["an edit; F2 drops it when nothing changes"] + D -- no --> A{"newest managedFields entries:
the applier's managers alone?"} + A -- "no, or tied with another manager" --> ED["an edit: mirrored, recorded in the window"] + A -- yes --> S{"entry written at or after the earliest send
of any window still open or carried?"} + S -- "no: an earlier apply's entry left newest by an undo" --> ED + S -- yes --> R{"an edit to it recorded in the window?"} + R -- no --> NM["the apply itself: not mirrored (F1)"] + R -- yes --> C{"the applier's entry against the edit"} + C -- "newer, or the same second" --> WH["late apply over the edit: withheld, Repair pending"] + C -- older --> UN["the API's undo of the edit: mirrored"] +``` + +- **The window** is the one built today: from sending an attempt until its operation completed, plus + 10 seconds, and an attempt sent within that grace carries the earlier window's edits and end. +- **The lower bound is the earliest send of every window still open or carried.** A delayed echo of + attempt A, arriving after attempt B was sent and inside A's grace, is still A's apply. +- **When an edit was recorded, today's comparison decides**, unchanged: an applier entry newer than + the edit is a late apply over it (withheld, `Repair`); an older one is the API's undo (mirrored). + The same second is withheld too ([decided while building](#decided-while-building), 7): the edit + arrived first, so it was written first. F1 changes only the case with no edit recorded, which today + is mirrored "as ever". +- **Edits are recorded from the flush mark** (S1), not only from the send. + +## F2: a write is a list of actions decided once, on its tree + +When a write is decided, the writer plans it against the tree it is decided on. F2 makes the result +of that plan, not the picture it was planned from, what the write carries until it is published: + +- **An upsert** for each object whose plan changes the tree, with the object's desired content. +- **A delete** for each document the plan removes. +- **Nothing** for an object whose plan changes nothing. It is dropped from the write then. + +A write with no action left settles at once. On replay, onto a newer tip after a rejected push, the +write's actions are applied again: an upsert is planned onto the new tree, a delete removes its +document if it is still there. Nothing else is planned again, so nothing that changed nothing, and +nothing a newer tip added, is touched. + +```mermaid +stateDiagram-v2 + [*] --> Decided + Decided --> Planned: planned once, on the write's tree + Planned --> Settled: no action + Planned --> Committed: upserts and deletes + Committed --> Published: push accepted + Committed --> Replayed: push rejected, remote moved + Replayed --> Committed: the same actions applied on the new tip + Settled --> [*] + Published --> [*] +``` + +How each kind of write follows it: + +- **A commit window's write** carries an upsert or a delete per live event that changed the tree. + Today an unchanged object inside a write that changed something is replayed with it; F2 drops it. +- **A resync** uses its snapshot's full membership once, to decide: which objects to upsert, and + which managed documents are orphans the sweep removes under the target's `prune.mode`. The write + then carries those upserts and those deletes, by identity. Replay applies exactly them and never + runs the sweep again, so a document a newer tip added (a Git-side addition the snapshot could not + know about) survives the replay. Today a replayed resync folds its whole membership over the new + tree, so under `prune.mode: Always` it deletes that addition: F2 closes that too. +- **A save** (a write carrying a `CommitRequest`) keeps an upsert for every object it captured, + unchanged ones included, and its outcome is settled by the push: `AlreadyPresent` when the push + confirms there was nothing to add, `Committed` when a replay produced a commit. That is today's + contract, kept on purpose (decision 9). A save cannot carry an echo: it attaches to a person's commit + window, and with F1 an applier opens none. + +Today the "changes nothing" rule exists for whole resync writes only (`PendingWrite.retained`, +[`commit_executor.go`](../../internal/git/commit_executor.go)). F2 makes it per object, for windows +and resyncs, and leaves saves as they are. + +## F3: a resync never decides from a snapshot collected while Git was ahead, or across an apply + +A resync mirrors a snapshot of live state, collected at one resource version +(`ResyncRequest.Desired`). When Git is ahead of the cluster, a commit pushed and not yet applied, a +snapshot holds the old state of every object that commit changes. A snapshot whose collection +overlaps an apply holds a mix. Either is a stale picture once the apply has run. + +- **Git is ahead of the cluster** while an applier on the branch has a `ForeignCommit`, `Repair` or + `Refusal` pending, a converge it may be asked for (past S2's initial-snapshot barrier), or an + attempt out. +- **The apply epoch** is a counter on the branch worker, raised at every send of an attempt and at + every completion of one. +- **When the watch side starts collecting a snapshot**, it reads both from the worker: whether Git is + ahead, and the epoch. The `ResyncRequest` carries them. +- **A snapshot is used only if** Git was not ahead when its collection started and the epoch is still + the one it started at when the worker decides it. Otherwise it is discarded, and a fresh one is + requested for the same `GitTarget` and scope once Git is no longer ahead (the re-anchor + `RequestRecheckForGitTarget` drives). The new snapshot is collected after the apply, so it never + overtakes a live event either. +- **The resync's own fetch is checked too.** A resync fetches the remote before it is planned. When + that fetch finds a commit the worker did not push, the snapshot is discarded and re-requested after + that commit is applied. +- **A heal resync that would steal a commit window** (today parked while a window is open) is + discarded and re-requested once the window is idle. No snapshot is ever parked. +- **A `CatchUp` to adopt does not count as Git being ahead**: it applies objects the cluster already + runs. +- **A branch with no trigger never waits**: nothing applies its commits. +- **A `GitTarget`'s initial snapshots are exempt**: the first snapshot of each collection it + watches, until one of it settles (applied, refused or failed). They are the startup baseline, + and S2 asks no applier for anything before they are in Git, so holding them for an apply would + wait on itself + ([decided while building](#decided-while-building), 3). +- **A `GitTarget`'s snapshots are held for at most 5 minutes**, from its first discard. Then a fresh + snapshot is asked for and used, logged and counted (`stale_snapshots_total{outcome="used"}`), and C2's + policy applies to what the unapplied commit changed + ([decided while building](#decided-while-building), 2). +- **Per worker.** Whether Git is ahead, and the epoch, are this worker's own: the applies due on a + sibling writing the same branch are not consulted + ([decided while building](#decided-while-building), 1). + +A resync is reconciliation, not latency-critical, and with S1 Git is ahead of the cluster for seconds +at a time. + +## F4: an object another applier owns is not mirrored into a triggered folder + +A folder a trigger applies belongs to that trigger's applier. An object another applier applies +(another `Kustomization`, another `Application`), mirrored into the folder, would then be applied by +this applier too, and two appliers applying one object fight each other. + +- **The watch side reads the owner from the raw object**, before sanitize strips it: Flux's owner + labels (`kustomize.toolkit.fluxcd.io/name` and `/namespace`), Argo CD's tracking annotation + (`argocd.argoproj.io/tracking-id`, which names its `Application`). It stamps the owner on the live + event and on each snapshot item. +- **The worker compares it with the applier of the event's `GitTarget`'s trigger.** An object owned by + another applier is excluded on both paths: a live event is not mirrored, and a snapshot item is + neither upserted nor treated as missing, so the sweep leaves its document alone. +- **Excluded objects are counted and logged**, so the misconfiguration is seen. +- **An object with no owner, or owned by this folder's applier, is mirrored as usual.** +- **A folder with no trigger excludes nothing**: mirroring another applier's objects one way is what + such a folder can exist for. + +## S1: flush, then apply + +Today an attempt goes out only on a quiet branch: no open commit window and no pending writes, on +this worker and every sibling writing the same branch. That kept an apply from reverting an edit not +yet published, and it starves: a user who keeps editing holds back a Git-side change for as long as +they type. S1 replaces the wait with a flush: + +1. **When an apply is due, the open commit window is closed**, so its edits become a decided write. +2. **Then the worker takes the flush mark**: the last write decided so far, which now includes the + window's. +3. **From the mark, the applier's next apply window records edits**, before the attempt is sent. An + edit decided after the mark is therefore one whose overwrite by the apply is withheld (F1), and `Repair` + puts it back. +4. **The writes up to the mark are published.** Writes decided after it do not hold the send, so + continuous editing cannot starve an apply. +5. **The apply is sent for the tip that publication produced.** At sending, it checks only that the + remote is still at that commit. + +Every edit is therefore either published before the send, so the apply includes it, or recorded in +the window, so an apply that overwrites it is withheld and repaired. + +Two states still hold an apply back, as they do today: a commit window held by a `CommitRequest` is +never closed early (a save defines its own commit boundary, and the apply waits for it), and the +worker does not apply while it recovers a missing parent or its checkout is not clean, the two states +in which it cannot say what is published. + +```mermaid +sequenceDiagram + autonumber + participant U as Person (API) + participant W as Branch worker + participant G as Gitea + participant F as Applier + participant K as Object (live) + + Note over W: an apply is due + W->>W: close the open commit window (edit E0 in it), then take the flush mark + U->>K: edit E1, just after the mark + K-->>W: E1: an edit, recorded + W->>G: publish the writes up to the mark (E0), tip C + W->>F: apply C, which holds E0 + U->>K: edit E2, during the apply + K-->>W: E2: an edit, recorded + F->>K: the apply writes C over E1 and E2 + K-->>W: the applier's writes over recorded edits: withheld, Repair pending + W->>G: E1 and E2 published + F-->>W: operation completed + W->>F: flush, then Repair: apply the tip, which holds E1 and E2 + F->>K: E1 and E2 are back +``` + +### When incoming Git changes are applied + +A Git-side push usually reaches the worker through the flush itself, when its push is rejected: + +```mermaid +sequenceDiagram + autonumber + participant W as Branch worker + participant G as Gitea + participant F as Applier + + Note over G: someone pushes B + W->>G: push the pending writes, expecting the old tip + G-->>W: rejected: the remote moved + W->>G: fetch: tip B, a commit the worker did not push, so ForeignCommit is pending + W->>W: apply the pending writes' actions onto B + W->>G: push: tip B+1 holds B and the API edits + W->>F: ForeignCommit: apply B+1 +``` + +The applier is asked once, for the tip that holds both sides (`applyInbox.observe` marks the fetch as +foreign; `sendAttempt` puts `ForeignCommit` before every other reason). Applying B on its own first +would revert the pending API edits on the cluster, need a second apply for the `Repair`, and show a +flicker in between: a `Deployment` whose template an edit touched rolls out to B's value and back. +Waiting costs one push. F1 and F2 make sure the actions applied in step 4 are only the API's, never a +stale picture that reverts B. + +## S2: `CatchUp` only to adopt and to converge, the same for both appliers + +With the flush guarding the cluster, an apply after every push of the worker's own protects nothing. +What is left is ownership: **an applier prunes only what it applied.** Flux deletes only objects that +carry its owner labels and are in its inventory; Argo CD prunes only objects that carry its tracking +annotation. Both are stamped by an apply. An object the API creates in an applied folder has neither, +so when Git later deletes its file, the applier leaves it on the cluster, and a later resync writes +it back into Git. + +| The API... | Does the applier need an apply? | +|---|---| +| edited an object the applier applied | no: the cluster runs the edit, and the object keeps its owner labels or tracking annotation | +| deleted an object | no: the object is gone, and the next apply prunes nothing | +| **created an object**, including deleting one and creating it again under the same name | **yes**: the new object carries no owner labels and no tracking annotation, even when its content equals what Git holds | + +- **Adopt.** A `CatchUp` is pending for an applier when the worker observes a live create (an + `ADDED` event) in a folder that applier applies, whose newest managers are not an applier's and + that no other applier owns (F4). It is pending whether or not the create changed Git: a delete and a + create of the same object in one commit window change nothing in Git and still leave an object no + applier owns. An object a resync adds to Git (an existing object a new `WatchRule` starts mirroring) + counts too. +- **Converge.** A `CatchUp` is pending once when the worker starts, and once when a `GitTarget` first + names the trigger (per `GitTarget` incarnation, by uid). +- **No apply before every initial snapshot is published.** An applier is asked for nothing, for any + reason (`ForeignCommit`, `Refusal`, `Repair` or `CatchUp`), until every resource collection of + every `GitTarget` it covers has completed its initial events and the resync they produced is + published or settled (a refused or failed initial snapshot counts as settled). At startup that is + the moment the edits made while the operator was down are in Git; for a `GitTarget` that newly + names the trigger it is the moment its folder holds the cluster. An earlier apply would destroy + what is not yet mirrored. A `GitTarget` whose watch is `Blocked` holds its applier's requests + until it is `Streaming`. A reason that comes due before stays pending. +- **Already applied, from the worker's own record.** The worker follows every operation and keeps, + per applier, the commit the last one that succeeded applied (`status.operation.appliedRevision`, + recovered on restart). A converge of that commit asks nothing. **An adopt is never skipped** that + way: the applier may be at the tip and still not own the new object, and re-applying the same commit + is what stamps it. +- **An empty tip asks nothing**: when the remote does not carry the branch there is no commit to + apply, and the source reports the missing branch itself. The reason stays pending until there is + one. +- **Batched by itself.** A pending `CatchUp` is one flag, and one attempt per applier is out at a + time, so a burst of creates is one apply. `quietPeriod` goes. + +```mermaid +flowchart TD + L["live ADDED event in a folder an applier applies"] --> W{"newest managers an applier's,
or owned by another applier?"} + W -- yes --> X["not an API create: nothing pending"] + W -- no --> AD["CatchUp pending: adopt"] + R["a resync adds an object to Git"] --> AD + N["the worker starts, or a GitTarget first names the trigger"] --> CV["CatchUp pending: converge"] + AD --> B{"every initial snapshot of every
covered GitTarget published?"} + CV --> B + O["ForeignCommit, Refusal or Repair pending"] --> B + B -- not yet --> P["stays pending"] + B -- yes --> K{"converge only, and the applier
already applied the tip?"} + K -- yes --> NO["asks nothing"] + K -- no --> F["flush, then apply the tip (S1)"] +``` + +## Collisions this leaves open + +Each needs the **same object** written from two sides within seconds. The **interval** is how close +the two writes must be; the **recovery** is how long the wrong state lasts once it happened. + +| | Collision | Interval | Effect | Recovery | Seen | +|---|---|---|---|---|---| +| C1 | An API edit to an object while an apply of it runs | the apply | the cluster runs the applied value; Git is right | heals: `Repair`, seconds | `apply_window_collisions_total{withheld}`, `Repair` on the trigger | +| C2 | An API edit or a save, and a Git-side push to the same object, before the edit is published | until the edit's push | the edit's upsert lands on the push: the API wins for every field of that object, including ones only Git changed | does not heal: the repository's "API wins" rule | the commit | +| C3 | An echo that arrives more than 10s after its operation completed, after the worker learned of a newer tip | after the grace | mirrored as an edit, reverting that tip's change to the object | does not heal | `apply_echoes_total{outcome="late"}` | +| C4 | An undo written in the same second as the apply's write to that object, or as the edit it undoes when an apply wrote that object in that second | one second | the undo looks like the apply and is not mirrored, or is withheld and a `Repair` puts the edit back | does not heal | not distinguishable | +| C5 | A prune by the applier, decided after the worker learned of a newer tip that adds the object back | the apply | the delete is mirrored and applied onto the re-add | does not heal | the commit | +| C6 | A mutating webhook or defaulting adds fields during an apply that Git does not hold | none | not mirrored (F1); a later resync mirrors them | not needed: the cluster runs them | the resync's commit | +| C7 | The applier's write and another manager's in the same second, after the worker learned of a newer tip | one second | mirrored as ambiguous; its content can revert that tip's change to the object | does not heal | `apply_window_collisions_total{ambiguous}` | +| C8 | A resync's sweep deletes a document whose object is gone from the cluster, and a Git-side push edits that document before the delete is published | until the delete's push | the delete is applied onto the edit: the cluster's state wins, as `prune.mode` asks | does not heal | the commit | + +C2 and C8 are policies, not races. C3, C4, C5 and C7 each need a coincidence on one object around an +apply; C3 and C7 are visible as metrics. + +## What it promises + +- **P1.** Inside the apply window, an echo of an applied commit never reverts a Git-side push: F1 + keeps the applier's write out, and F2 drops an object that changes nothing. The exceptions are a + late echo (C3), a prune racing a re-add (C5) and a same-second tie (C7). +- **P2.** A resync never decides from a snapshot collected while Git was ahead of the cluster, or + across an apply (F3). +- **P3.** A replay applies only the actions its write decided: it never sweeps again, so a document a + newer tip added survives it (F2). +- **P4.** An API edit is mirrored, except an undo in the same second as an apply of that object (C4), + and an edit to an object another applier owns (F4). +- **P5.** An API edit an apply overwrote is put back on the cluster by one `Repair`: every edit is + either published before the apply is sent, or recorded in its window (S1). +- **P6.** An object the API creates in a folder an applier applies, or deletes and creates again, is + adopted by that applier, Flux and Argo CD alike, so a later Git-side delete prunes it (S2). +- **P7.** No applier is asked for anything before every initial snapshot of the `GitTarget`s it + covers is published, so edits made while the operator was down are mirrored first (S2). +- **P8.** An object another applier owns is never mirrored into a folder a trigger applies (F4). +- **P9.** A save's outcome is settled by the push, as the CommitRequest contract says (decision 9). + +## Showing the applier's work + +F1 sees each apply land, object by object: every write it keeps out is the applier answering a +request. One new counter, +`apply_echoes_total{provider_namespace, provider_name, branch, outcome}`, has `not_mirrored` (F1) +and `late` (C3: the applier's managers alone, no window open, mirrored as an edit), and one line is +logged per window, when it closes or is replaced: "Observed the applier's apply", with the commit +and the number of objects kept out. Objects F4 excludes are counted in +`apply_ownership_conflicts_total{provider_namespace, provider_name, branch}`, and the snapshots F3 +holds off in `stale_snapshots_total{provider_namespace, provider_name, branch, outcome}`, as +`discarded` or `used` (let through by the 5-minute bound). + +## The code plan + +Read from the code at `47b171b9`. + +### Removed + +| Construct | Where | Why it can go | +|---|---|---| +| `branchQuiet` | [`apply_trigger.go`](../../internal/git/apply_trigger.go) | the quiet branch is the wait S1 replaces with the flush | +| `workWaiting` | `apply_trigger.go` | queued work items, here or on a sibling, no longer hold an apply | +| The "nothing pending" half of `decisionHolds` (`busyWriting`, `inflightItems`, `siblingBusy`) | `apply_trigger.go` | `ApplyRequest.Current` checks only that the remote is still at the commit asked for | +| `branchSiblings.othersBusy`, `BranchWorker.siblingBusy`, `publishWriteBusy`, the `busyWriting` atomic, and the `wakeOthers` calls for quietness | [`apply_siblings.go`](../../internal/git/apply_siblings.go), [`branch_worker.go`](../../internal/git/branch_worker.go) | quietness across siblings existed only for the quiet branch; the claims for source groups stay | +| `stashDeferredHeal`, `applyDeferredHeals`, `deferredHeals` | [`resync_flush.go`](../../internal/git/resync_flush.go), `branch_worker.go` | no snapshot is parked anymore (F3) | +| The re-run of the mark-and-sweep on replay | `resync_flush.go` (`applyResyncToWorktree`), [`commit_executor.go`](../../internal/git/commit_executor.go) | a replayed resync applies its decided actions (F2) | +| `ApplierRef.CatchesUp` | [`apply_request.go`](../../internal/git/apply_request.go) | both appliers catch up (S2) | +| `FluxTrigger.spec.quietPeriod`, `EffectiveQuietPeriod` on both trigger types and the `ApplyTrigger` interface, `DefaultApplierQuietPeriod` | [`api/v1alpha3`](../../api/v1alpha3/applytrigger_types.go) | nothing waits for time to pass before a `CatchUp`; the CRDs are regenerated | +| `branchTarget.quietPeriod`, `applierState.quietPeriod`, `quietSince`, the quiet term of `dueAt`, the reset of `quietSince` in `decide` | `apply_trigger.go`, `branch_worker.go`, [`pending_writes_loop.go`](../../internal/git/pending_writes_loop.go) | they only timed the quiet period | +| The `OutcomeCurrent` of a `ForeignCommit` or `CatchUp` for a suspended `Kustomization` already at the commit | [`internal/applier/flux.go`](../../internal/applier/flux.go) | the worker decides from what each applier last applied | +| The `OutcomeCurrent` for an empty commit, with the stamp of the source | `flux.go`, [`internal/applier/argocd.go`](../../internal/applier/argocd.go) | the worker sends nothing for an empty tip | +| `prefetchSources` and the inbox's prefetch, and `FluxTrigger` `status.source.prefetch` | [`apply_groups.go`](../../internal/git/apply_groups.go), `apply_trigger.go`, `api/v1alpha3` | it overlapped the quiet-branch wait, which is gone, and does not shorten the flush ([decided while building](#decided-while-building), 4) | + +### Changed + +| Construct | Where | After | +|---|---|---| +| `withholdLateApply`, no edit recorded | `apply_window.go` | the applier's write alone is not mirrored, counted `not_mirrored` (F1) | +| `withholdLateApply`, the lower bound | `apply_window.go` | an applier entry older than the earliest send of every window still open or carried is an edit; `applyWindow` keeps that earliest send as it carries edits (F1) | +| `withholdLateApply`, a closed window | `apply_window.go` | the applier's managers alone with no window open: mirrored, counted `late` (C3) | +| The apply window's edit records | `apply_window.go`, `apply_trigger.go` | start at the flush mark: edits decided after the mark are carried into the window the attempt opens (S1) | +| Planning a write (`applyPendingWriteEvents`, `executeEventWrite`) | `commit_executor.go` | produces the write's actions: an upsert or a delete per object that changes the tree; an unchanged object is dropped, except in a save (F2) | +| Planning a resync (`applyResyncToWorktree`) | `resync_flush.go` | decides once, from the snapshot's membership, the upserts and the sweep's deletes by identity; the write carries them (F2) | +| Replay (`replayOntoRemote`, `replayPendingWrites`) | `branch_worker.go` | applies each write's actions on the new tree: an upsert is planned onto it, a delete removes its document if present; no sweep (F2) | +| `PendingWrite.retained`, `settleCommitted` | `commit_executor.go`, `pending_writes_loop.go` | a write with no action left settles; a save is never trimmed and keeps today's push-decided outcome (F2, decision 9) | +| The apply epoch | `branch_worker.go`, `apply_trigger.go` | a counter raised at every send and every completion of an attempt, and a "Git is ahead" flag, both readable by the watch side (F3) | +| `ResyncRequest` | [`types.go`](../../internal/git/types.go), [`internal/watch`](../../internal/watch/target_watch.go) | carries the epoch and the "Git is ahead" flag read when its collection started (F3) | +| `handleResyncRequest` | `resync_flush.go` | a snapshot collected while Git was ahead, across an epoch change, or arriving while Git is ahead, or a heal while a commit window is open, is discarded and re-requested when that ends (F3) | +| The fetch before a resync is planned (`materialize`, `fetchReasonForcedRecheck`) | `pending_writes_loop.go` | a fetch that finds a foreign commit discards the resync and re-requests it after that commit is applied (F3) | +| The watch side's event stamp, and each snapshot item | [`internal/watch/target_watch.go`](../../internal/watch/target_watch.go) | also carries the owning applier, read from Flux's owner labels or Argo CD's tracking annotation on the raw object (F4) | +| `handleLiveEvents`, the resync plan | `branch_worker.go`, `resync_flush.go` | an object owned by an applier other than the folder's is excluded: not mirrored, not upserted, not swept; counted (F4) | +| `writeBusy` | `apply_trigger.go` | only the states in which no flush can be made: a registered `CommitRequest`, a parent recovery, a dirty worktree (S1) | +| `closeWindowForForeignCommit` | `apply_trigger.go` | when any applier has a reason due: closes the open window unless a `CommitRequest` holds it, then takes the flush mark (S1) | +| `serviceAppliers` | `apply_trigger.go` | sends once the writes up to the flush mark are published, and only past the initial-snapshot barrier (S1, S2) | +| `ApplyRequest.Current`, `ApplyStale` | `apply_request.go` | stale only when the tip moved | +| `applyInbox.observe` | `apply_trigger.go` | a push of the worker's own leaves nothing pending; the first observation leaves the startup converge (S2) | +| `handleLiveEvents`, adopt | `branch_worker.go` | an `ADDED` event whose newest managers are not an applier's, owned by no other applier, for a target an applier covers, leaves an adopt pending (S2) | +| `applierState` | `apply_trigger.go` | adds the adopt and converge flags, the commit the last succeeded operation applied, and which covered collections have published their initial snapshot (S2) | +| `syncAppliers` | `apply_trigger.go` | a newly named applier starts with a converge pending, for both kinds, behind its `GitTarget`'s initial snapshot (S2) | + +### Stays as built + +The apply window itself (opened per attempt, settled on completion, 10s grace, edits carried into +the next window), the withhold of a late apply over a newer edit with its `Repair`, the ambiguous +tie, one attempt in flight per applier and per source group (`claimGroup`, `groupOutElsewhere`, +`publishGroupsOut`), the dispatcher and the following of operations, restart recovery, the Flux +suspend lifecycle, the receiver and `pushHinted`, the `CatchUp` reason (narrowed by S2), the startup +baseline, the save contract and `resolvePushedCommitRequests`, and `inflightItems` for the +queue-depth gauge. + +### Documents that change with it + +- [`../definitions.md`](../definitions.md): **quiet branch** goes; the flush, **echo**, **adopt** and + **apply epoch** come in. +- [`apply-trigger.md`](apply-trigger.md): the apply window gains F1 and the flush mark, the + quiet-branch rule becomes S1, F3 and F4 join, and `CatchUp` is rewritten as S2, with the two + reversed decisions. +- [`gittarget-branch-worker-pending-writes.md`](gittarget-branch-worker-pending-writes.md): a write + carries actions; replay applies them (F2). +- [`../spec/reconcile-via-watchlist-mark-and-sweep.md`](../spec/reconcile-via-watchlist-mark-and-sweep.md): + a resync decides its upserts and deletes once, and a replay never sweeps again (F2). +- [`../interpreting-metrics.md`](../interpreting-metrics.md): `apply_requests_total`'s `stale` + outcome and its "never `CatchUp` for an `application`", and the three new counters. +- [`../bi-directional.md`](../bi-directional.md) (the quiet branch, the reasons table, the state + diagram), [`../bi-directional-flux.md`](../bi-directional-flux.md), + [`../bi-directional-argocd.md`](../bi-directional-argocd.md), + [`../architecture.md`](../architecture.md), [`../api-first-publication.md`](../api-first-publication.md), + [`../configuration.md`](../configuration.md) and [`../UPGRADING.md`](../UPGRADING.md): "on a quiet + branch" becomes "after a flush", `quietPeriod` goes, adopt and converge are described for both + appliers, and F4's ownership rule is stated where a triggered folder is set up. +- [`config/samples/fluxtrigger.yaml`](../../config/samples/fluxtrigger.yaml), the field comment in + [`gittarget_types.go`](../../api/v1alpha3/gittarget_types.go), and the comments in + [`internal/telemetry/apply_requests.go`](../../internal/telemetry/apply_requests.go) and + [`internal/telemetry/apply_window.go`](../../internal/telemetry/apply_window.go). + +## Tests + +### New end-to-end scenarios + +Each runs for Flux and for Argo CD, in the bi-directional corner, on a folder GitOps Reverser +mirrors and the applier applies. + +| # | Scenario | Proves | +| --- | --- | --- | +| E1 | **Push race**: push A, wait until the applier made A live, push B at once; three rounds. B must stay in Git and on the cluster. (`bi_directional_push_race_e2e_test.go`, committed as `Pending`; this change removes that.) | P1 | +| E2 | **Adopt, then prune**: create an object through the API in the folder; wait for the adopt; delete its file in Git. The object must be gone from the cluster and stay gone from Git. | P6 | +| E3 | **Recreate, then prune**: an object the applier applied is deleted and created again through the API with identical content, so Git does not change; wait for the adopt; delete its file in Git. The object must be gone. | P6 | +| E4 | **Negative control for E2**: with adopt disabled, the object survives the delete. Run once to confirm E2 and E3 exercise ownership; not kept. | that E2 and E3 test what they claim | +| E5 | **Edit during an apply**: push to Git so an apply starts, and edit an object the apply changes while it runs. The edit is in Git at once and back on the cluster after one `Repair`; count the commits. | P4, P5, C1 | +| E6 | **Edits do not starve an apply**: edit an object every second while a Git-side push waits. The push is applied within one apply's time, and every edit ends in Git and on the cluster. | S1, P5 | +| E7 | **Resync while Git is ahead**: push to Git and force a heal resync before the apply finishes. The push survives in Git and reaches the cluster. | P2 | +| E8 | **A Git-side addition survives a resync** (`prune.mode: Always`): add a file in Git and force a resync at once. The document stays in Git and its object reaches the cluster. | P2, P3 | +| E9 | **Restart with work made while down**: stop the operator; edit a ConfigMap and a Secret, create a third object through the API, and push an unrelated change to Git; start it. Both edits and the new object are in Git, nothing is reverted on the cluster, the Git-side change is applied, and the new object is adopted, all after the initial snapshots. | P6, P7 | +| E10 | **An API edit asks for no apply**: two quick API edits to an existing object; no apply is requested, and the later value survives. Replaces the Flux spec "asks Flux to apply only on a quiet branch". | S2 | +| E11 | **Another applier's object**: a second `Kustomization` or `Application` applies an object into a namespace the folder mirrors. It is not mirrored, not adopted, and counted. | P8 | +| E12 | **A save across a Git-side change**: open a commit window, save it with a `CommitRequest` while a Git-side push changes the same object. The save resolves from the push, as the CommitRequest contract says. | P9, C2 | +| E13 | **Timing, updated**: the timing spec's "API edit → Flux applied it" becomes "API create → adopted", for both appliers. | the latency of adopt | + +### Below end to end + +- **A worker-level test without timing luck**, on a real Git server (`startRealGitServer`, since + `file://` never compares the expected tip): an unchanged object decided inside the push cooldown, + the remote moves, the push fails and replays; no commit. The same inside a batch with a changed + object: only the changed object is committed. (F2) +- **A resync replay on a real Git server** with unchanged X, changed Y, and a document Z the remote + added before the replay, under `prune.mode: Always`: Y is committed, X is untouched, Z survives. + (F2, P3) +- **A snapshot collected across an apply**, driven deterministically: collection starts during an + apply, the apply completes, the snapshot arrives with nothing pending. It is discarded and + re-requested; so is one that started while Git was ahead. (F3) +- **An edit in the open window when an apply comes due, and one decided after the mark**, driven in + the worker: the first is published before the send, the second is recorded and the apply over it is withheld, and + `Repair` is pending. (S1) +- **Unit tests of F1**: alone after the earliest send (not mirrored); after a recorded edit (withheld, + ambiguous or undo, as today); an older entry (an edit); a delayed echo of an earlier attempt inside + its grace (not mirrored). +- **Unit tests of F4**: an event or snapshot item owned by another applier is excluded and not swept; + one owned by this folder's applier, or by none, is mirrored; a folder with no trigger excludes + nothing. +- **Unit tests of S2**: an API `ADDED` event leaves an adopt; an applier's does not; an adopt is sent + even when the applier is at the tip; no reason is sent before every initial snapshot is published. +- **Rewritten**: the tests of the quiet branch, the busy and sibling gates, `decisionHolds`, + `CatchesUp`, `quietPeriod`, the parked heals and Flux's `OutcomeCurrent` in + `apply_trigger_test.go`, `apply_attempt_test.go`, `apply_operation_test.go`, + `apply_recovery_test.go`, `refusal_reapply_test.go`, `applytrigger_schema_test.go`, + `flux_guide_test.go` and `helpers_test.go`, and the resync tests that replay a sweep. + +## Decided while building + +The build departs from the plan above in twelve places, decided on 2026-10-20: + +1. **Siblings do not count for F3.** Whether Git is ahead, and the apply epoch, are per worker: an + apply due on another worker writing the same branch does not hold this worker's resyncs. This + settles the first question the plan left open. +2. **F3 holds off a `GitTarget`'s resyncs for at most 5 minutes**, from the first snapshot it + discarded. Then the next snapshot is used, logged and counted + (`stale_snapshots_total{outcome="used"}`), and C2's policy, API wins, applies to what the + unapplied commit changed. A resync asked for again forever would leave drift unmirrored for as + long as an apply fails. This settles the second open question. +3. **A collection's initial snapshot is exempt from F3.** The + plan counted a pending converge as Git being ahead, while S2's barrier held every apply, + converge included, until the initial snapshots were published: each waited for the other, at + startup and when a `GitTarget` first names a trigger. The initial snapshots are the startup + baseline, so they are decided as they arrive. The exemption is tracked per collection on the + watch side: a snapshot is initial until a snapshot of its collection settles (applied, refused or + failed; a discarded or superseded one does not settle), and the barrier is every collection of + the `GitTarget` settled with no resync of it left to publish. A latch per `GitTarget`, read from + whether its streams were `Streaming`, counted a `GitTarget` declared before its `WatchRule`'s + collection was planned as done; its real initial snapshot was then discarded, and never asked + for again while the converge it waited on could not complete. +4. **The prefetch is removed**, with `FluxTrigger` `status.source.prefetch`. The plan kept it only + if it still shortened the flush; it does not. +5. **An edit carried from the window before guards only writes made before the next send.** The + plan carried a window's edits into the next attempt's window unchanged, so the next attempt's own + apply of an object edited in the earlier window was judged a late apply over the edit, withheld, + and drew a spurious `Repair`. One attempt is out at a time, so the earlier apply wrote before the + next one was sent: a carried edit now counts only against an applier entry written before the + current window's send. +6. **A held `GitTarget` is asked for a fresh snapshot when the 5 minutes run out**, on the applier + timer, rather than when the next stale snapshot happens to arrive. +7. **The applier's write alone in the same second as a recorded edit is withheld**, not mirrored as + ambiguous. Events arrive in `resourceVersion` order, so the edit, recorded first, was written + first, and the applier's write after it is its apply over the edit. Mirrored, it reverted the edit + in Git and on the cluster, which E6 caught when Flux applied within the second of an edit. The cost + is C4's: an undo in that second, left with an applier entry of that second as the newest, is + withheld and the `Repair` puts the edit back. A tie with another manager is still mirrored: it may + be the edit itself. +8. **A folder an applier applies is created before it is applied.** The plan's converge on first + naming asked Argo CD to sync a folder nothing had been mirrored into yet; Argo CD retried the + missing path for minutes, which held every later apply of that `Application`. A resync of a + `GitTarget` naming a trigger, into a folder the branch does not hold, commits the bootstrap + files even with no object. Whether the folder exists is read from HEAD's tree; nothing parses + its content. +9. **A `GitTarget` with no planned collection has not settled its initial snapshots.** It counted as + settled with nothing to deliver, so a converge went out before its `WatchRule`'s collection was + planned (a CRD not discovered yet). It now holds its applier, as a `Blocked` watch does. +10. **A Flux source that fetched another commit is a hint to look.** With no receiver, a converge + decided before a Git-side push kept asking for a commit the `GitRepository` no longer served, + backing off, until the worker's own push or the refresher found the move. The `RequestFailed` + now carries the revision the source holds, and the worker looks at the remote as at a push + notice. +11. **There is no heal resync.** The plan discarded a heal that would close an open window, and + asked for it again once the window was idle. Nothing produces one: every resync the watch plane + sends is a replay, which finalizes the window. The `Heal` flag, its discard and its window-idle + re-request are removed, with the tests that drove a heal by hand. +12. **The flush covers the sibling workers.** The plan removed sibling quietness, and S1 flushed only + the sending worker. A sibling sharing the trigger then kept its unpublished edit while the apply + reverted it on the cluster, and mirrored the revert, having sent nothing and holding no apply + window (found in review). When an apply comes due, the worker asks each sibling to flush up to a + mark of its own and sends once all have; workers share what they observe of the branch, so the + attempt is sent for the tip that holds the siblings' writes. A sibling's push is foreign to the + others, as it was when they found it by a fetch: its appliers may apply another cluster. A + sibling's edit made while the apply runs stays unprotected, as it was before this change. + +## References + +- [`apply-trigger.md`](apply-trigger.md): the apply window, `Repair`, `CatchUp`, the flush, and the + startup baseline. +- [`gittarget-branch-worker-pending-writes.md`](gittarget-branch-worker-pending-writes.md): decide, + `advance`, materialize, and replay. +- [`../spec/commitrequest-design.md`](../spec/commitrequest-design.md): the save contract F2 keeps. +- [`../spec/reconcile-via-watchlist-mark-and-sweep.md`](../spec/reconcile-via-watchlist-mark-and-sweep.md): + the resync and its sweep. +- [`internal/git/apply_window.go`](../../internal/git/apply_window.go): `withholdLateApply`. +- [`internal/git/commit_executor.go`](../../internal/git/commit_executor.go): planning a write. +- [`internal/git/resync_flush.go`](../../internal/git/resync_flush.go): the resync. +- [`internal/git/apply_trigger.go`](../../internal/git/apply_trigger.go): the flush and `CatchUp`. +- [`internal/git/apply_epoch.go`](../../internal/git/apply_epoch.go): stale snapshots (F3). +- [`internal/git/apply_ownership.go`](../../internal/git/apply_ownership.go): another applier's + objects (F4), and adopt. +- [`internal/git/branch_worker.go`](../../internal/git/branch_worker.go): `PushCooldown`, + `afterFailedPush`, `replayOntoRemote`. +- [`test/e2e/bi_directional_push_race_e2e_test.go`](../../test/e2e/bi_directional_push_race_e2e_test.go): + scenario E1. diff --git a/docs/design/apply-trigger.md b/docs/design/apply-trigger.md index 551888f0..69240e6c 100644 --- a/docs/design/apply-trigger.md +++ b/docs/design/apply-trigger.md @@ -1,10 +1,13 @@ # The apply trigger -> **design**: decided 2026-10-16, extended 2026-10-17 and 2026-10-19; built: the Flux applier (its -> `Kustomization` held suspended), the Argo CD applier (automated sync off), the trigger kinds that -> hold them and act only while an eligible `GitTarget` references them, the cross-namespace grant -> for appliers, the push receiver, `onRefusal: Reapply` and the apply window. The webhook applier is -> the aim. Index: [`../INDEX.md`](../INDEX.md). +> **design**: decided 2026-10-16, extended 2026-10-17, 2026-10-19 and 2026-10-20; built: the Flux +> applier (its `Kustomization` held suspended), the Argo CD applier (automated sync off), the +> trigger kinds that hold them and act only while an eligible `GitTarget` references them, the +> cross-namespace grant for appliers, the push receiver, `onRefusal: Reapply`, the apply window, +> and, since 2026-10-20, flush then apply, the applier's own writes kept out of Git, and `CatchUp` +> as adopt and converge for both appliers. The argument for the 2026-10-20 change is +> [`applier-echoes.md`](applier-echoes.md). The webhook applier is the aim. Index: +> [`../INDEX.md`](../INDEX.md). > How to use it: [`../bi-directional.md`](../bi-directional.md), > [`../bi-directional-flux.md`](../bi-directional-flux.md) and > [`../bi-directional-argocd.md`](../bi-directional-argocd.md). @@ -34,20 +37,27 @@ The cause is not a bug in either tool. Each apply was correct; it ran at the wro ## The principle -**An apply is safe only when nothing live is missing from the commit being applied.** Between an -edit and its push, the edit is live and absent from Git: in an open commit window, or in the -writes waiting to be pushed. Only the branch worker can see those. So only it can say when an apply -is safe, and every trigger the applier runs by itself fires without that knowledge. +**An apply is safe only when every live edit is in the commit being applied, or known to the +worker so it can be put back.** Between an edit and its push, the edit is live and absent from Git: +in an open commit window, or in the writes waiting to be pushed. Only the branch worker can see +those. So only it can say when an apply is safe, and every trigger the applier runs by itself fires +without that knowledge. -An apply is started only on a **quiet branch**: no open commit window and no pending writes. Four -reasons make one due: +An apply is started only after a **flush** ([below](#flush-then-apply)): the open commit window is +closed, the writes decided so far are published, and the apply is sent for the tip that +publication produced. An edit decided after that is recorded in the apply window, so an apply that +overwrites it is withheld and repaired. Four reasons make one due: | Reason | When | Applied by | | --- | --- | --- | -| `ForeignCommit` | The branch gained a commit GitOps Reverser did not push. Open windows are closed early and pending writes replayed onto it and pushed first, so the applied commit carries the API side's answer | Every applier on the branch | +| `ForeignCommit` | The branch gained a commit GitOps Reverser did not push. The flush replays the pending writes onto it and pushes them first, so the applied commit carries the API side's answer | Every applier on the branch | | `Refusal` | An edit to a document the folder holds was refused at the write boundary, and the target opted in with `onRefusal: Reapply` and is not suspended. At most one request a minute per target; a refusal inside that minute is carried by a trailing one | The refusing target's applier | | `Repair` | The apply window withheld a late apply over a newer API edit, so the cluster runs an older value than Git | The applier that wrote it | -| `CatchUp` | The branch has been quiet for `quietPeriod` after GitOps Reverser's own pushes, since it started, or since a `GitTarget` first named the trigger | Every Flux applier on the branch | +| `CatchUp` | Ownership, not safety ([below](#catchup-adopt-and-converge)). An **adopt**: the API created an object in the folder, or a resync added one to Git, and an apply stamps it as the applier's, so a later Git-side delete prunes it. A **converge**: once when the worker starts, and once when a `GitTarget` first names the trigger, unless the applier already applied the tip | The folder's applier for an adopt; for a converge, every applier on the branch at start, and the named one on first naming. Flux and Argo CD alike | + +No applier is asked for anything before every initial snapshot of the `GitTarget`s it covers is +published, and a push of GitOps Reverser's own leaves nothing pending: the cluster already runs the +edits it carries. ## What we learned @@ -63,7 +73,7 @@ unsuspended on a 24h interval and asked it to reconcile. Flux still applied when fired, when kustomize-controller restarted (from whatever artifact it held, which is why `CatchUp` was once a safety rule), when a failed apply retried, when the `Kustomization`'s own spec changed, and on every new artifact of its `GitRepository`, including one another consumer's request fetched. -None of those waits for a quiet branch. The third review called it best-effort coordination, and it +None of those waits for the worker. The third review called it best-effort coordination, and it was. A suspended `Kustomization` does none of them, so it is held suspended, and resumed for one apply at a time ([below](#the-flux-applier-suspended)). @@ -87,11 +97,11 @@ write that starts the apply. **A retry is a decision too, so only the worker may make one.** The second review found every remaining problem in the hand-off between the worker's decisions and the deliverer's retries: a -retry skipped the quiet check, outlived a refusal that had recovered, ran for a recreated target, or -waited on another applier's backoff. The fix was structural rather than four patches. The worker +retry skipped the worker's check, outlived a refusal that had recovered, ran for a recreated target, +or waited on another applier's backoff. The fix was structural rather than four patches. The worker keeps every reason pending until an operation that succeeded answers it. The dispatcher carries out one attempt and reports back; it never retries, and never asks for something weaker than it was told. A -retry is decided like a first attempt: on a quiet branch, at the tip as it is then. +retry is decided like a first attempt: after a flush, at the tip as it is then. **An apply is an operation, and its end is observed, never guessed.** The third review found the apply window closing early in three ways: kustomize-controller writes `lastAttemptedRevision` @@ -108,10 +118,10 @@ the right unit is what makes the 1:n cases fall out: | Unit | Owns | Why | | --- | --- | --- | -| The **branch** (repository and branch) | Quietness | A `Kustomization` applies the tip whichever folder an edit is in. Two `GitProvider`s naming one repository give one branch two workers, so a worker is quiet only when its siblings are too | -| The **applier**, held by its trigger | Pending reasons, retries, the operation | One operation at a time, its own backoff, one for every `GitTarget` naming the trigger. The trigger's status records it, and its `quietPeriod` is declared once | +| The **branch** (repository and branch), per worker | The flush, and whether Git is ahead (the apply epoch) | A `Kustomization` applies the tip whichever folder an edit is in, so a flush publishes every pending write of the worker. Two `GitProvider`s naming one repository give one branch two workers: a flush covers both, they share what they observe of the branch and the source claims, and each keeps its epoch for itself | +| The **applier**, held by its trigger | Pending reasons, retries, the operation, the commit it last applied | One operation at a time, its own backoff, one for every `GitTarget` naming the trigger. The trigger's status records it | | The **source** (cluster and `GitRepository`) | One operation at a time across its appliers | The artifact is shared: a sibling's fetch landing between an operation's recheck and its reconcile would apply another commit | -| The **`GitTarget`**, by uid | Configuration, consent and eligibility | Consent to a `Refusal` is per target, and a target deleted and recreated under the same name inherits nothing due to the old one. A trigger acts only while an eligible `GitTarget` references it | +| The **`GitTarget`**, by uid | Configuration, consent, eligibility and its initial snapshots | Consent to a `Refusal` is per target, and a target deleted and recreated under the same name inherits nothing due to the old one. A trigger acts only while an eligible `GitTarget` references it, and its applier is asked for nothing until each `GitTarget` it covers has published its initial snapshots | ## The design, as built @@ -120,33 +130,36 @@ flowchart LR subgraph WORKER["Branch worker (one per GitProvider and branch)"] OBS["Observations
push · fetch · notification"] DUE["Due, per applier
ForeignCommit · Refusal · Repair · CatchUp"] - QUIET{"Quiet branch?
this worker and its siblings"} + FLUSH{"Flush
close the window, publish up to the mark"} end EXEC["Dispatcher
one attempt, no retries"] TRIG["Trigger status
operation · repairPending"] FLUX["Flux
GitRepository · Kustomization"] ARGO["Argo CD
Application"] - OBS --> DUE --> QUIET - QUIET -- "yes, and due" --> EXEC + OBS --> DUE --> FLUSH + FLUSH -- "published, past the barrier" --> EXEC EXEC -- "record before the write" --> TRIG EXEC -- "recheck, fetch, resume with a token,
follow, suspend again" --> FLUX EXEC -- "recheck, set a pinned sync,
follow its token" --> ARGO - EXEC -. "report: Requested · Running · Settling ·
Unresolved · Succeeded · Failed · Current ·
RequestFailed · Rejected · Dropped · Stale" .-> DUE + EXEC -. "report: Requested · Running · Settling ·
Unresolved · Succeeded · Failed ·
RequestFailed · Rejected · Dropped · Stale" .-> DUE ``` - **Observing.** Every observation of the remote passes one point. The first after a start is a - baseline (a `CatchUp` is due, never a `ForeignCommit`, because an apply on startup would revert - edits made while the operator was down). A tip the worker pushed makes a `CatchUp` due; any other - a `ForeignCommit`. When in doubt it is foreign, which costs one apply on a quiet branch. An - applier a `GitTarget` names for the first time gets what a first observation gives, a `CatchUp`: - a commit observed before it was named was due to nobody, and was taken, so it would otherwise not - be applied until some later push. For a `Kustomization` already at the tip that costs nothing - (`Current`). The worker lists its `GitTarget`s on every pass, cached, so a new name is seen at - once. -- **Deciding.** At the end of every wake, if the branch is quiet, each applier with an apply due, - no operation out, shares no source with one that has, and is past its backoff - (and, for a `CatchUp`, its quiet period) gets one attempt. The attempt carries away what it + baseline: a converge is pending, never a `ForeignCommit`, because an apply on startup would revert + edits made while the operator was down. A tip the worker pushed leaves nothing pending; any other + is a `ForeignCommit`. When in doubt it is foreign, which costs one apply of the tip. An applier a + `GitTarget` names for the first time (per `GitTarget` incarnation, by uid) gets a converge too: a + commit observed before it was named was pending for nobody, and was taken, so it would otherwise + not be applied until some later push. The worker lists its `GitTarget`s on every pass, cached, so + a new name is seen at once. An API create in a folder an applier applies, and an object a resync + adds to Git, leave an adopt pending ([below](#catchup-adopt-and-converge)). +- **Deciding.** At the end of every wake, an applier is due when it has a reason pending, no + operation out, shares no source with one that has, and is past its backoff and the + initial-snapshot barrier. An empty tip asks nothing, and a converge alone of the commit the + applier last applied asks nothing and is answered. When one is due the worker flushes + ([below](#flush-then-apply)), and once the writes up to the flush mark are published, each due + applier gets one attempt, for the tip that publication produced. The attempt carries away what it covers; each target in it carries its uid and its own reason. The worker takes the operation's token; nothing is on the trigger yet. - **Attempting.** The dispatcher re-reads each target. It asks only for those that are still the @@ -163,26 +176,116 @@ flowchart LR `Held=True` (still adopting, releasing, or being deleted) is not resumed (`RequestFailed`). A source on another repository or branch, a `Kustomization` whose `spec.suspend` another manager applies, or an `Application` with automated sync on, is `Rejected`. A decision that no longer holds when - the write is due (the commit is no longer the tip, or something is waiting to be published) asks - nothing (`Stale`, no failure). + the write is due (the commit is no longer the tip) asks nothing (`Stale`, no failure). - **Following.** The dispatcher follows the operation and reports each move, so the attempt is out for exactly the apply window. Flux: `Running`, `Settling` once suspended again, then `Succeeded` or `Failed` ([below](#the-flux-applier-suspended)). Argo CD: the operation state carrying the request's token ([below](#the-argo-cd-applier)). One not completed after 5 minutes is `Unresolved`: a Flux `Kustomization` is suspended at once, and the operation is followed on, at an interval doubling to a minute. The applier object being deleted, or its trigger being deleted, - completes it as `Failed`; no `GitTarget` naming the trigger any more abandons it, and a Flux + completes it as `Failed`; no `GitTarget` naming the trigger anymore abandons it, and a Flux `Kustomization` is suspended, and that suspend handled, before it is let go. - **Settling.** An operation that succeeded is done. A stale attempt gives back what it carried and - is decided again at the next quiet moment. Any other outcome, a failed operation included, gives - back what the attempt carried, a refusal only if it still stands, and starts that applier's - backoff (2s doubling to 5m, at least 15s after a rejection). `ApplyRequestFailed` or - `ApplyFailed` is recorded for the first failure of a run, and `ApplyUnresolved` when an operation - becomes `Unresolved`; `apply_operations_unresolved` counts those still out, and the trigger is - `Ready=False` (`OperationUnresolved`) until its operation completes. + is decided again at once, at the tip as it is then, with no backoff. Any other outcome, a failed + operation included, gives back what the attempt carried, a refusal only if it still stands, and + starts that applier's backoff (2s doubling to 5m, at least 15s after a rejection). + `ApplyRequestFailed` or `ApplyFailed` is recorded for the first failure of a run, and + `ApplyUnresolved` when an operation becomes `Unresolved`; `apply_operations_unresolved` counts + those still out, and the trigger is `Ready=False` (`OperationUnresolved`) until its operation + completes. - **Learning about pushes.** A push webhook into GitOps Reverser's receiver (seconds), a rejected - compare-and-swap push (at the next publication), or the periodic refresh (minutes). All three - feed the same observation point. + compare-and-swap push (at the next publication), the periodic refresh (minutes), or a Flux + attempt whose `GitRepository` fetched another commit than the one asked for (`RequestFailed`), + which is looked at as a push notice is: a hint, never a fact. All of them feed the same + observation point. + +### Flush, then apply + +When an apply comes due for any applier, the worker flushes before it sends: + +1. **The open commit window is closed**, so its edits become a decided write + (`git_commit_windows_total{close_reason="apply_due"}`). +2. **The worker takes the flush mark**: the last write decided so far. +3. **From the mark, the due applier records the API edits decided after it**, and its next apply + window holds them from the start ([below](#the-apply-window)). +4. **The writes up to the mark are published at once**, without waiting for the 5-second push + cooldown. Writes decided after the mark do not hold the send, so continuous editing cannot + starve an apply. +5. **The attempt is sent for the tip that publication produced.** It is `Stale` only when that tip + moved: `ApplyRequest.Current` checks nothing else. + +Every edit is therefore published before the send, so the apply includes it, or recorded in the +window, so an apply that overwrites it is withheld and a `Repair` puts it back. + +Three states still hold an apply back: a registered `CommitRequest` (a save defines its own commit +boundary, and the apply waits for it), a parent-branch recovery, and a checkout a failed write left +dirty. In the last two the worker cannot say what is published. + +A Git-side push usually reaches the worker through the flush itself. Say somebody pushed `B`: the +worker's push is rejected, the fetch finds `B`, a commit the worker did not push (a `ForeignCommit` +is pending), the pending writes' actions are applied onto it, and the push lands `B+1`. The +applier is then asked once, with `ForeignCommit`, for `B+1`, which holds both sides. Applying `B` on +its own first would revert the pending API edits on the cluster and need a second apply to put them +back. A push notice from the Git host (the receiver) still closes an open window, unless a +`CommitRequest` holds it, so its publication finds the move. + +The flush covers the sibling workers (other `GitProvider`s on the same repository and branch), whose +`GitTarget`s can name the same trigger: an apply sent while a sibling holds an unpublished edit would +revert it on the cluster, and the sibling, which sent nothing and has no apply window, would mirror +the revert. When an apply comes due, the worker asks each sibling to flush: it closes its open +window, takes a mark of its own, and publishes up to it at once. The worker sends once every sibling +it asked is done. Every worker shares what it observes of the branch with its siblings, so the tip +the attempt is sent for holds their publications. A sibling's push is foreign to the other workers, +as a fetch of it would be: its appliers may apply another cluster than theirs, so nothing says its +content is live where their appliers apply, and an echo of content already live is not mirrored. A +sibling's edit decided +after its mark does not hold the send, and is not protected while the apply runs, as before. + +### `CatchUp`: adopt and converge + +With the flush guarding the cluster, an apply after every push of the worker's own protects +nothing, so such a push leaves nothing pending, and an API edit asks for no apply at all. What is +left is ownership: **an applier prunes only what it applied.** Flux deletes only objects that carry +its owner labels and are in its inventory; Argo CD prunes only objects that carry its tracking +annotation. An apply stamps both. An object the API creates carries neither, so when Git later +deletes its file the applier leaves it on the cluster, and a later resync writes it back. + +- **Adopt.** A `CatchUp` is pending for an applier when the worker observes a live create (an + `ADDED` event) in a folder that applier applies, whose newest managers are not an applier's and + that no other applier owns ([below](#another-appliers-objects)), and when a resync adds an object + to Git. It is never skipped as already applied: applying the same commit again is what stamps the + owner labels or the tracking annotation. +- **Converge.** A `CatchUp` is pending once when the worker starts (its first observation of the + branch), and once when a `GitTarget` first names the trigger, per `GitTarget` incarnation, by + uid. The worker keeps, per applier, the commit the last succeeded operation applied, from the + operation's report and, after a restart, from the trigger's `status.operation`. A converge of that + commit asks nothing. +- **No apply before every initial snapshot is published.** An applier is asked for nothing, for any + reason, until every collection of every `GitTarget` it covers has a settled snapshot (applied, + refused or failed; tracked per collection on the watch side) and the worker has published every + resync it holds for that `GitTarget`. At startup that is when the edits made while the operator + was down are in Git; for a `GitTarget` that newly names the trigger, when its folder holds the + cluster. A `GitTarget` with no planned collection (its `WatchRule` not resolved yet, a CRD not + discovered) and one whose watch is `Blocked` hold their applier's requests the same way. A reason + that comes due before stays pending. +- **The folder exists before it is applied.** A resync of a `GitTarget` that names a trigger, into + a folder the branch does not hold yet, commits the folder's bootstrap files (`README.md`, + `.gittargetignore`, and `.sops.yaml` with encryption) even when it has no object to write. Argo CD + retries a sync of a missing path for minutes, and Flux fails it; the bootstrap files are never + deleted, so the folder outlives its last resource. +- **An empty tip asks nothing.** When the remote does not carry the branch, nothing is sent, and the + reason stays pending until there is a commit to apply. +- **Batched by itself.** A pending `CatchUp` is one flag, and one attempt per applier is out at a + time, so a burst of creates is one apply. +- **One reason per target in an attempt**, by precedence: `ForeignCommit`, then `Repair`, then + `Refusal`, then `CatchUp`. Any succeeded operation applies the tip and answers every pending + reason. + +This reverses two earlier decisions ([2026-10-20, decision 3](#decisions)). An Argo CD +`Application` now gets a `CatchUp`, adopt and converge alike, where it was never asked for one. And +a newly bound `Application` converges once its `GitTarget`'s initial snapshots are published, where +it waited for the next push. A sync on an API create or on binding runs the `Application`'s hooks; +such a sync is rare, and one rule for both appliers is worth it. ## The Flux applier, suspended @@ -249,19 +352,20 @@ drifted object with no new commit): resumed again before T2 is handled, the `Kustomization` is suspended again. 7. **Complete** once T2 is handled, and keep the window open 10 s more. -A `ForeignCommit` or `CatchUp` asks nothing (`Current`) when the `Kustomization` is suspended and -both its artifact and `lastAppliedRevision` hold the commit: nothing applied since, and nothing will -until it is resumed. A `Refusal` and a `Repair` always run the operation. A failed apply is suspended -like any other, which is what stops Flux's own `retryInterval` retry; the worker owns the retry. +Every reason that reaches the client runs the operation. The worker decides what needs no apply +before it sends (a converge of the commit the applier last applied, an empty tip), so the client +never answers for it, and an adopt runs even at the tip, because the apply is what stamps the +object. A failed apply is suspended like any other, which is what stops Flux's own `retryInterval` +retry; the worker owns the retry. An `Unresolved` operation is suspended at once (the suspend does not stop the running apply, but it stops whatever would follow), and that suspend's token then decides when it completes. There is no automatic cancellation. -`CatchUp` stays, as convergence rather than safety: a suspended `Kustomization` never applies a -stale artifact, but Flux's inventory learns of an object only from an apply of a commit that holds -it, and an object created through the API and never applied by Flux would not be pruned when Git -later deletes it. +`CatchUp` is ownership rather than safety ([above](#catchup-adopt-and-converge)): a suspended +`Kustomization` never applies a stale artifact, but Flux's inventory learns of an object only from +an apply of a commit that holds it, and an object created through the API and never applied by Flux +would not be pruned when Git later deletes it. ### Who owns `spec.suspend` @@ -324,7 +428,7 @@ later deletes it. first adoption write until it has handed the `Kustomization` back or over, also while its configuration is invalid (then `Held=Unknown`, `NotConfigured`, and the message says the `Kustomization` is left suspended under `gitops-reverser`). It hands it back when it is deleted, - and when no eligible `GitTarget` references it any more, conclusively: a reference that cannot be + and when no eligible `GitTarget` references it anymore, conclusively: a reference that cannot be judged yet keeps the hold, since a release is not undone. Until the hand-back is done the trigger is `Held=False`, `Releasing`, and stays the holder in line. It suspends the `Kustomization` with a token if an apply has it resumed, waits until that is handled, then releases `spec.suspend` with @@ -343,7 +447,7 @@ later deletes it. `Kustomization`; the dispatcher reads the trigger uncached before every step that may suspend. A `Kustomization` that is gone lets the trigger go. A `ClusterProvider` that is gone does not end the hand-back: it goes on through whatever client still reaches the cluster, which for a provider - without `kubeConfig` is the operator's own. A cluster no client reaches any more, a suspend or + without `kubeConfig` is the operator's own. A cluster no client reaches anymore, a suspend or release the identity may not make, or a hand-back still unfinished 10 minutes after `Held` turned `Releasing` (each a `Warning` `ReleaseFailed` Event, which says where the `Kustomization` was left) lets the trigger go. @@ -367,31 +471,28 @@ live event is mirrored before the windows of what it finds are open: suspended) is followed whenever it is listed, once per suspend token, so its window opens while the process runs too. -The first observation is still a baseline that makes a `CatchUp` due, so a recovered operation -decides nothing about which apply is due. +The first observation is still a baseline that leaves a converge pending, so a recovered operation +decides nothing about which apply is due. The commit each applier last applied is read back from its +trigger's `status.operation`, so that converge asks nothing of an applier already at the tip. -### Shared sources and prefetch +### Shared sources - The trigger lists the consumers of its `GitRepository` in the applier's cluster: `Kustomization`s with that `sourceRef`, and `HelmRelease`s whose chart comes from it, directly or through a `HelmChart` their `chartRef` names (at whichever `HelmRelease` version the cluster - serves; consumers that cannot be determined are an error, so no prefetch). Those no `FluxTrigger` - on the same `ClusterProvider` holds, that is reports `Held=True`, are - `status.source.otherConsumers`, information only: it never affects `Ready`, and no `Warning` is - sent, because one `GitRepository` for `apps` and `infra` is a normal setup. -- `status.source.prefetch` is true only when that list was read and is empty, and the trigger holds - its own `Kustomization`. The dispatcher checks once more, on the cluster, that every consumer is - held just before it stamps the source, and stamps only the `GitRepository` whose consumers it - checked: the `Kustomization`'s live `sourceRef` must still name it. Then a push somebody else - made, observed while the branch is busy, has the source fetch at once, so the artifact is - ready when the apply is due. On a quiet branch the attempt sent at once fetches for itself. No - prefetch while any `FluxTrigger` on that source has an operation in flight, because a resumed - consumer applies whatever the artifact becomes. A listing that is forbidden counts as not holding. + serves; a listing that fails is logged and publishes nothing). Those no `FluxTrigger` on the same + `ClusterProvider` holds, that is reports `Held=True`, are `status.source.otherConsumers`, + information only: it never affects `Ready`, and no `Warning` is sent, because one `GitRepository` + for `apps` and `infra` is a normal setup. Each of them applies a new artifact the moment it + arrives. - Appliers that share a source have one operation at a time, and take turns: the one whose last turn is oldest goes first, so a busy sibling cannot starve another. This holds across the workers of GitProviders that name the same repository and branch: a worker claims the source's - group before it sends, under the siblings' lock, and a sibling's operation of that source holds - a prefetch back too. + group before it sends, under the siblings' lock. +- The source is stamped only inside an operation, to make its artifact hold the commit (step 1 + [above](#one-operation-every-reason)). Nothing fetches a push into a shared `GitRepository` ahead + of the apply: that prefetch overlapped the wait for a quiet branch, which is gone, and a flush + leaves it nothing to shorten. `status.source.prefetch` went with it. ### What the measurement found @@ -420,11 +521,12 @@ Taken 2026-10-17: 1. **Suspension is the only Flux mode,** as automated sync off is the only Argo CD mode. A second, request-based mode would keep every hole of the 24h setup alive, and two completion protocols. -2. **A shared source is information only:** `status.source.otherConsumers`, prefetch off while it is - not empty, no `SharedSource` condition. +2. **A shared source is information only:** `status.source.otherConsumers`, no `SharedSource` + condition. It also held the prefetch off while it was not empty; the prefetch is removed + (2026-10-20, decision 7). 3. **An external resume is always suspended again,** with a `Normal` Event each time. 4. **`CatchUp` stays for Flux,** as convergence. The same inventory question exists for Argo CD, - which was decided without `CatchUp`. + which was decided without `CatchUp`. Reversed 2026-10-20 (decision 3): Argo CD catches up too. 5. **The resources are triggers,** with long kind names: `FluxTrigger` (`spec.kustomizationRef`) and `ArgoCDTrigger` (`spec.applicationRef`), named by `GitTarget.spec.applyTriggerRef` in the same namespace. Each is its own kind because suspension needs one owner object with a finalizer, @@ -454,6 +556,30 @@ Taken 2026-10-19, after the seventh review: mid-release. 6. **A newly bound `Application` waits for the next push.** A first sync on binding would run its hooks for a commit it may already have, which is why an `Application` gets no `CatchUp`. + Reversed 2026-10-20 (decision 3): it converges on first naming. + +Taken 2026-10-20, with [`applier-echoes.md`](applier-echoes.md), which carries the argument: + +1. **Flush, then apply** ([above](#flush-then-apply)). No apply waits for a quiet branch, on this + worker or a sibling: the flush publishes what is pending, on this worker and on every sibling + writing the branch, and the apply goes. +2. **The applier's own write in its apply window is not mirrored** ([below](#the-apply-window)). +3. **`CatchUp` only adopts and converges, decided in the branch worker, the same for Flux and Argo + CD** ([above](#catchup-adopt-and-converge)). This reverses decision 4 of 2026-10-17, so an + Argo CD `Application` now catches up, and decision 6 of 2026-10-19, so a newly bound + `Application` converges on first naming instead of waiting for the next push. A push of the + worker's own leaves nothing pending, and `FluxTrigger` `spec.quietPeriod` is removed from the + API. +4. **No apply before every initial snapshot is published**, for any reason. +5. **A resync never decides from a snapshot collected while Git was ahead, or across an apply** + ([below](#resyncs-while-git-is-ahead)). Per worker, so siblings are not consulted, and a + `GitTarget`'s snapshots are held off for at most 5 minutes. +6. **An object another applier owns is not mirrored into a triggered folder** + ([below](#another-appliers-objects)). +7. **The prefetch is removed**, with `status.source.prefetch`: it overlapped the quiet-branch wait, + and does not shorten the flush. +8. **The worker decides what needs no apply, and never sends an empty commit**, so the clients' + `Current` outcome is gone. ## The apply window @@ -463,10 +589,9 @@ the edit yet. Making the request synchronous would not help: waiting for the app user from editing during it, and refusing edits would halt the API for everyone to protect a few seconds. -The quiet-branch rule confines the window to three moments: right after somebody else's push, right -after a refusal the target asked to have reverted, and after `quietPeriod` of silence on a branch -GitOps Reverser pushed to, or that a `GitTarget` newly names a trigger for. Inside it, **the API -wins**, as it does for any object both sides changed: +The flush confines the window to what it cannot avoid ([above](#flush-then-apply)): every edit is +either published before the attempt is sent, so the apply includes it, or recorded in the window +from the flush mark. Inside it, **the API wins**, as it does for any object both sides changed: - **The window is known.** It opens when the attempt is sent and stays open until its operation completed: for Flux, the resume's token and the suspend after it handled; for Argo CD, the operation @@ -477,28 +602,107 @@ wins**, as it does for any object both sides changed: attempt that asked for nothing does not end the earlier grace. A write of the new apply to an object edited during the earlier one is then withheld too, since the managers cannot tell the two applies apart: that costs one redundant `Repair`, never an edit. -- **The late apply is withheld.** A write whose latest `managedFields` entries are the applier's - own managers alone (`kustomize-controller`; `argocd-controller` or - `argocd-application-controller`) and newer than the API edit, inside the window, to an object in - the applier's folders that the API side edited after the attempt was sent, is not mirrored. Git - keeps the edit, and the withheld write makes a `Repair` due to its applier: an apply of the tip at - the next quiet moment, with no quiet period, for Flux and Argo CD alike. The cluster runs the - reverted value until then, and the trigger shows `repairPending`. +- **Edits are recorded from the flush mark.** An edit decided after the mark is not in the commit + the attempt applies, so the due applier records it before the attempt is sent, and the window the + attempt opens holds it from the start. +- **The apply itself is not mirrored.** A write whose newest `managedFields` entries are the + applier's own managers alone (`kustomize-controller`; `argocd-controller` or + `argocd-application-controller`), written at or after the earliest send of every window still + open or carried, to an object nobody edited in the window, is the apply landing: its content is + the commit's, which Git already holds. It is not mirrored, makes nothing due, and is counted in + `apply_echoes_total{outcome="not_mirrored"}`. Mirrored, it would become a pending write planned + from the live object, and a replay of that write onto a newer tip would revert that tip's change + to the object. The bound is the earliest send of every window still open or carried, so a + delayed echo of one attempt, arriving after the next was sent and inside the first one's grace, + is still that apply. When a window closes or is replaced, one line is logged, + `Observed the applier's apply`, with the commit and the number of objects kept out. +- **The late apply is withheld.** A write by the applier's managers alone, newer than the API edit, + inside the window, to an object in the applier's folders that the API side edited in the window, + is not mirrored. Git keeps the edit, and the withheld write makes a `Repair` due to its applier: + an apply of the tip after the next flush, for Flux and Argo CD alike. The cluster runs the + reverted value until then, and the trigger shows `repairPending`. An edit carried from the window + before guards only against an applier write made before this attempt was sent: one attempt is out + at a time, so a write since is this attempt's own apply, of a commit holding every edit published + before it. - **Only proof withholds.** `managedFields` times resolve to the second. When the applier ties with another manager, the write could be the edit itself, and withholding an edit loses it outright; it is mirrored as before. It is also remembered as an edit, even as the first write in - the window, so the applier's write alone a second later is still withheld. An applier entry in the - same second as the edit is mirrored the same way. One older than the edit is no write of the + the window, so the applier's write alone a second later is still withheld. An applier entry alone + in the same second as the edit is withheld: the edit arrived first, in `resourceVersion` order, so + it was written first, and only an undo in that second is lost. One older than the edit is no write of the applier's at all: an update that removes every field a manager owns deletes that manager's entry, so undoing an edit (`kubectl label cm/x tmp-` after `tmp=1`) can leave the applier's entry from an earlier apply as the newest. A late apply moves the applier's entry forward, so an older one is - the API's undo, mirrored as an edit. A DELETE carries nothing about its deleter and is never - withheld. All are logged. + the API's undo, mirrored as an edit. For the same reason an applier entry older than the earliest + send, on an object nobody edited in the window, is an edit and is mirrored. A DELETE carries + nothing about its deleter: it is always an edit, which a pending write drops only when it changes + nothing in Git. +- **With no window open, the applier's write is late.** The applier's managers alone on an object + with no window open are mirrored as an edit and counted in `apply_echoes_total{outcome="late"}`: + an echo that arrived after the grace, or an apply somebody else started. Every collision is logged with the object, the applier and the requested commit, and counted in `apply_window_collisions_total` (`withheld` or `ambiguous`). What remains: an edit reverted on the cluster until the repair, an ambiguous tie mirrored as the applier's commit, and an applier's write -that reaches the watch more than 10 seconds after its operation completed. +that reaches the watch more than 10 seconds after its operation completed, mirrored and counted as +`late`. + +## Resyncs while Git is ahead + +**A resync never decides from a snapshot collected while Git was ahead of the cluster, or across an +apply.** A resync mirrors a snapshot of live state. While a commit is pushed and not yet applied, a +snapshot holds the old state of every object that commit changes, and one whose collection overlaps +an apply holds a mix. Decided, either reverts the push. + +- **Git is ahead** while an applier on the branch has a `ForeignCommit`, `Repair` or `Refusal` + pending, a converge it may be asked for (past the initial-snapshot barrier), or an attempt out. + An adopt does not count: it applies objects the cluster already runs. +- **The apply epoch** is a counter on the worker, raised at every send of an attempt and at every + end of one. +- **The watch side reads both when a snapshot's collection starts**, just before the watch that + delivers its initial events opens, or before the LIST fallback. The resync request carries them. +- **The worker discards** a snapshot collected while Git was ahead, across an epoch change, or + arriving while Git is ahead, and one whose own pre-resync fetch finds a commit the worker did not + push. A discarded snapshot is answered as discarded, which is no failure and is not counted as a + failed background resync. Once Git is no longer ahead, the worker asks the watch plane to + re-anchor that `GitTarget`'s watches (the forced recheck a refused Git path uses), which collects + a fresh snapshot after the apply. Until it lands, the `GitTarget`'s render-fidelity scope stays + pending: the render-fidelity gate holds its live writes back, and the fresh snapshot recovers + them. +- **What never waits.** A branch with no trigger, because nothing applies its commits. A + `GitTarget`'s initial snapshots, the first of each collection it watches until one of it settles + (applied, refused or failed): they are the startup baseline, and no applier is asked for anything + before they are in Git, so holding them for an apply would wait on itself. And a `GitTarget` + whose snapshots have been held for 5 minutes from the first discard: a fresh snapshot is asked + for then and used, logged and counted, because an apply that keeps failing must not leave drift + unmirrored for as long as it fails. The "API wins" rule then applies to what the + unapplied commit changed. +- **Per worker.** Whether Git is ahead, and the epoch, are this worker's own; a sibling worker's + applies are not consulted. + +`stale_snapshots_total` counts each stale snapshot as `discarded`, or `used` when the 5-minute +bound let it through. + +## Another applier's objects + +**A folder a trigger applies belongs to that trigger's applier.** An object another applier applies +(another `Kustomization`, another `Application`), mirrored into the folder, would be applied by this +applier too, and two appliers applying one object fight each other. + +- The watch side reads the owner from the raw object, before sanitize strips it: Flux's owner labels + `kustomize.toolkit.fluxcd.io/name` and `/namespace` (a `Kustomization`), else Argo CD's tracking + annotation `argocd.argoproj.io/tracking-id` (`:/:/`, where + `` is `_` for an `Application` outside Argo CD's own namespace). It stamps + the owner on live events and on snapshot entries. +- The worker compares it with the applier of the event's `GitTarget`'s trigger. An object owned by + another applier is excluded: a live event is not mirrored, and a snapshot entry is neither + upserted nor treated as missing, so the sweep leaves its document alone, under + `prune.mode: Always` too. +- An object owned by nobody, or by the folder's own applier, is mirrored as before. A folder with no + trigger excludes nothing: mirroring another applier's objects one way is what such a folder can + exist for. +- Each excluded object is logged once and counted in `apply_ownership_conflicts_total`, so the + misconfiguration is seen. ## The Argo CD applier @@ -512,10 +716,12 @@ and is never synced. The user guide is [`../bi-directional-argocd.md`](../bi-dir Automated sync off is Argo CD's equivalent of a suspended `Kustomization`, and it needs no suspend after an apply: an Argo CD operation that has completed is not followed by another one nobody -started. **It gets no `CatchUp`**: an `ArgoCDTrigger` has no `quietPeriod`, and the worker never -asks an `Application` for one. So an `Application` a `GitTarget` names for the first time is synced at the next push, a -refusal or a repair, not at once: a sync on binding would run its hooks for a commit it may already -have. +started. **It gets every reason, `CatchUp` included,** by the same rules as Flux +([above](#catchup-adopt-and-converge)): Argo CD prunes only objects that carry its tracking +annotation, and a sync is what stamps it on an object the API created. An `Application` a +`GitTarget` names for the first time converges once that `GitTarget`'s initial snapshots are +published, unless it already synced the tip, so a sync on binding runs its hooks. Both reverse +earlier decisions ([2026-10-20, decision 3](#decisions)). Every rule the client relies on is Argo CD's own, read from its source (3.5.0): @@ -540,7 +746,8 @@ Three choices were decided (2026-10-17): 2. **The initiator** is `InitiatedBy{Username: "gitops-reverser", Automated: true}`. GitOps Reverser replaces automated sync, so a sync window that blocks automated syncs blocks its syncs too. 3. **Hooks run on every sync**, so on every `ForeignCommit`, which asks every applier on the - branch. Narrowing it by changed paths waits for a measured complaint. + branch, and, since 2026-10-20, on every adopt and converge. Narrowing it by changed paths waits + for a measured complaint. A hub Argo CD keeps `Application`s in a management cluster: the trigger's `spec.clusterProviderRef` reaches it as it reaches a Flux in another cluster. Out of scope: multi-source `Application`s (and @@ -551,8 +758,8 @@ reaches it as it reaches a Flux in another cluster. Out of scope: multi-source ` **A webhook applier** for any other tool: a third trigger kind that POSTs `{"repository", "branch", "commit", "reason"}`, signed `X-Signature: sha256=` the way Flux's `generic-hmac` -`Receiver` checks. It gets every reason, `CatchUp` included, since nothing says the tool does not -apply on its own. No in-process plugin: it would run with every credential the operator holds. +`Receiver` checks. It gets every reason, `CatchUp` included, as Flux and Argo CD do. No in-process +plugin: it would run with every credential the operator holds. ## What it costs @@ -571,9 +778,9 @@ apply on its own. No in-process plugin: it would run with every credential the o its generation unapplied, and its dependents wait until GitOps Reverser's retry succeeds. - **The `Kustomization`'s manifest must omit `suspend`.** This is checked, not just documented. - **The residual race.** Between the artifact recheck and the reconcile reading the artifact, - somebody can stamp the `GitRepository`. With prefetch held off during operations, that is a person - or the source's own interval. The outcome table catches it, and the apply window still withholds - a late write over a newer edit. + somebody can stamp the `GitRepository`. GitOps Reverser stamps a source only inside an operation, + one operation per source at a time, so that is a person or the source's own interval. The outcome + table catches it, and the apply window still withholds a late write over a newer edit. - **One replica.** The receiver, the branch workers and the dispatcher run in one pod, and the chart refuses more than one replica. Leader election is not switched on; more than one replica needs ownership coordination first, and deliveries routed to the pod that owns the branch. @@ -583,6 +790,9 @@ apply on its own. No in-process plugin: it would run with every credential the o - **Every applier on the branch applies on a `ForeignCommit`**, whether or not the push touched its folder. Harmless for Flux; for Argo CD it runs hooks. Narrowing it needs the applier's own source paths, so it waits for a measured complaint. +- **An Argo CD `Application` also syncs, hooks included, to adopt and to converge**: on an API + create in its folder, once when the operator starts unless it already synced the tip, and when a + `GitTarget` first names its trigger. - **A sync somebody else started holds an Argo CD request**, for up to 5 minutes, and then fails it; the worker asks again after its backoff. - **A trigger may hold any applier its `ClusterProvider` reaches**, with @@ -605,21 +815,24 @@ suspended. Next, in order, each one pull request: Test gaps to close along the way: a remote source cluster and an applier in another cluster (a hub Argo CD among them) in e2e, `Forbidden` against real RBAC, the receiver and applier roles installed by the Helm chart, two `GitTarget`s sharing one `Kustomization` in e2e, an apply-window collision in -e2e, a prefetch observed end to end, a `Replace=true` Argo CD sync's field manager measured rather +e2e, a `Replace=true` Argo CD sync's field manager measured rather than read from the source, and `-race` in CI. **What adopting it changes for gitops-api.** Its gate shrinks to "one save, one window", which protects commit messages and has nothing to do with Flux. Its push handler relays each push to the -`GitProvider`'s receiver instead of stamping Flux. And publications trail an editing session by at -least `quietPeriod`, since Flux applies GitOps Reverser's own commits only after the branch has -been quiet; `quietPeriod` is the knob for that trade. +`GitProvider`'s receiver instead of stamping Flux. And GitOps Reverser's own commits ask for no +apply, since the cluster already runs the edits they carry; an object created through the API is +adopted by one apply. ## What proves it | Claim | Test | | --- | --- | -| The decision: quiet branch, reasons, quiet period, baseline | [`apply_trigger_test.go`](../../internal/git/apply_trigger_test.go) | -| Retries are decided by the worker; recovered refusals, withdrawn consent and recreated targets are not retried; each applier keeps its own clock; one attempt for a shared applier; a busy sibling worker holds the branch | [`apply_attempt_test.go`](../../internal/git/apply_attempt_test.go) | +| The decision: the baseline, the reasons, a `ForeignCommit` asking every applier and answering everything else pending, nothing sent while no flush can be made, a push notice closing the window | [`apply_trigger_test.go`](../../internal/git/apply_trigger_test.go) | +| `CatchUp`: the startup converge asks nothing of an applier at the tip, an API create is adopted even at the tip, a first naming converges once, nothing is asked before the initial snapshots are published, an empty tip asks nothing | [`apply_catchup_test.go`](../../internal/git/apply_catchup_test.go) | +| A snapshot collected across an apply, or arriving while Git is ahead, is discarded; initial snapshots are exempt; a branch with no trigger never waits; snapshots are held no longer than the limit | [`apply_epoch_test.go`](../../internal/git/apply_epoch_test.go) | +| Another applier's object is not mirrored from a live write, and a snapshot of it is neither upserted nor swept | [`apply_ownership_test.go`](../../internal/git/apply_ownership_test.go) | +| Retries are decided by the worker; recovered refusals, withdrawn consent and recreated targets are not retried; each applier keeps its own clock; one attempt for a shared applier; an apply waits for its sibling workers to flush; a stale attempt is decided again without backoff | [`apply_attempt_test.go`](../../internal/git/apply_attempt_test.go) | | The operation is recorded before the write that starts it, moves through its phases on the trigger's status, an attempt that asked nothing restores the record before it, and a withheld rollback makes a `Repair` due | [`apply_operation_test.go`](../../internal/git/apply_operation_test.go), [`status_writer_test.go`](../../internal/applier/status_writer_test.go) | | Every attempt rechecks targets, asks for nothing weaker than it was told, and resumes no `Kustomization` on another branch; a stop mid-apply suspends the `Kustomization` | [`dispatcher_test.go`](../../internal/applier/dispatcher_test.go), [`flux_test.go`](../../internal/applier/flux_test.go) | | The suspended lifecycle against the source facts: an apply is not finished when attempted; a token handled while suspended is no apply; a resume alone reverts a refusal; a suspend does not stop a running apply; the artifact is rechecked before the resume; a failed apply is suspended and Flux's requeue applies nothing; a shared source applies only the consumer asked | [`flux_lifecycle_test.go`](../../internal/applier/flux_lifecycle_test.go), [`flux_consumers_test.go`](../../internal/applier/flux_consumers_test.go), [`fluxsim`](../../internal/applier/fluxsim/fluxsim_test.go) | @@ -627,10 +840,10 @@ been quiet; `quietPeriod` is the knob for that trade. | The rule: a trigger nothing names reads nothing and is `Ready`; the first eligible reference adopts and the last releases, suspending an apply in flight first; a reference returning mid-release holds again; a re-created `GitTarget` releases and re-adopts; the binding over several `GitTarget`s; an older trigger not in line keeps none out, and a retiring holder stays first until its release is done; a holder left unconfigured says the `Kustomization` is suspended; `ApplierNamespaceNotAllowed` for both kinds | [`applytrigger_rule_test.go`](../../internal/controller/applytrigger_rule_test.go) | | `allowAnySourceNamespace: true` is refused on create and on any spec change, a stored one included; `false` and its removal are accepted; a `WatchRule` that relied on it is refused naming the new field | [`removed_allow_any_source_namespace_test.go`](../../internal/controller/removed_allow_any_source_namespace_test.go), [`watchrule_source_namespace_test.go`](../../internal/controller/watchrule_source_namespace_test.go) | | A restarted worker follows an operation left in flight with its window open, restores `repairPending`, and follows an apply started elsewhere; one recorded but never resumed ends `Failed` | [`apply_recovery_test.go`](../../internal/git/apply_recovery_test.go), [`dispatcher_test.go`](../../internal/applier/dispatcher_test.go) | -| Appliers sharing a source have one operation at a time; a push is prefetched only while the branch is busy and no operation of the source is in flight | [`apply_groups_test.go`](../../internal/git/apply_groups_test.go), [`flux_consumers_test.go`](../../internal/applier/flux_consumers_test.go) | +| Appliers sharing a source have one operation at a time and take turns, across workers too; a shared source applies only the consumer asked | [`apply_groups_test.go`](../../internal/git/apply_groups_test.go), [`flux_consumers_test.go`](../../internal/applier/flux_consumers_test.go) | | An applier in another cluster is reached through its own `ClusterProvider`, behind that provider's `accessFrom` | [`dispatcher_test.go`](../../internal/applier/dispatcher_test.go), [`clusterprovider_admission_test.go`](../../internal/authz/clusterprovider_admission_test.go), [`applytrigger_controller_test.go`](../../internal/controller/applytrigger_controller_test.go) | -| The applier's late write over a newer edit is withheld, and nothing that could be the edit is: a tie, and an undo that leaves an older applier entry, are mirrored | [`apply_window_test.go`](../../internal/git/apply_window_test.go), [`last_field_managers_test.go`](../../internal/watch/last_field_managers_test.go) | -| A newly named `Kustomization` gets a `CatchUp`, and a newly named `Application` none | [`apply_trigger_test.go`](../../internal/git/apply_trigger_test.go) | +| The applier's late write over a newer edit is withheld, and nothing that could be the edit is: a tie, and an undo that leaves an older applier entry, are mirrored; the apply itself is not mirrored, a delayed echo of the attempt before included; an applier entry older than the send is an edit; an applier write with no window open is late | [`apply_window_test.go`](../../internal/git/apply_window_test.go), [`last_field_managers_test.go`](../../internal/watch/last_field_managers_test.go) | +| A newly named applier is caught up | [`apply_trigger_test.go`](../../internal/git/apply_trigger_test.go) | | The receiver's path names no provider, and every miss answers the same | [`receiver_test.go`](../../internal/webhook/receiver_test.go) | | The original race: two API edits end at the later value with no extra commit | [bi-directional Flux corner](../../test/e2e/flux_bi_directional_e2e_test.go) | | A trigger nothing names is `Ready` and holds nothing; `SourceMismatch` adopts nothing and never stamps the other branch's source; fixed, the `Kustomization` is adopted and Flux reaches the tip; dropping the reference hands it back and naming the trigger again adopts it; a Gitea webhook and the refresh alone each get a push applied; a failed apply is suspended and retried by the worker; a `flux resume` is suspended again; deleting the trigger hands the `Kustomization` back | [Manager Flux Applier](../../test/e2e/flux_applier_e2e_test.go) | @@ -638,8 +851,8 @@ been quiet; `quietPeriod` is the knob for that trade. | A refused edit is reverted by a resume of its `Kustomization` | [Refusal reapply](../../test/e2e/refusal_reapply_e2e_test.go) | | The Flux guide's example is accepted by the code | [`flux_guide_test.go`](../../internal/applier/flux_guide_test.go) | | An Argo CD sync is the one automated sync builds, pinned to the commit, with a pinned retry; a running operation is waited out and never replaced; an operation set between the read and the update makes it fail; one whose `.operation` was removed while it ran ends; every `Configured` reason, `AutomatedSyncEnabled` among them; when a sync counts as finished | [`argocd_test.go`](../../internal/applier/argocd_test.go), [`dispatcher_test.go`](../../internal/applier/dispatcher_test.go) | -| An `Application` is never asked for a `CatchUp`; Argo CD's late write over a newer edit is withheld by its own managers | [`apply_trigger_test.go`](../../internal/git/apply_trigger_test.go), [`apply_window_test.go`](../../internal/git/apply_window_test.go) | -| Each trigger kind holds one applier in one cluster, immutably; `quietPeriod` is bounded and defaulted on a `FluxTrigger` alone; a trigger nothing names is `Ready` with no finalizer, and a `GitTarget` naming it re-runs its check | [`applytrigger_schema_test.go`](../../internal/controller/applytrigger_schema_test.go) | +| An `Application` is asked on a `ForeignCommit` and on a `Refusal`; Argo CD's late write over a newer edit is withheld by its own managers | [`apply_trigger_test.go`](../../internal/git/apply_trigger_test.go), [`apply_window_test.go`](../../internal/git/apply_window_test.go) | +| Each trigger kind holds one applier in one cluster, immutably; a trigger nothing names is `Ready` with no finalizer, and a `GitTarget` naming it re-runs its check | [`applytrigger_schema_test.go`](../../internal/controller/applytrigger_schema_test.go) | | With Argo CD: two API edits keep the later value with no extra commit; a Git-side push is synced by a pinned `ForeignCommit`; a refused edit is reverted by a sync; an `Application` with automated sync on is `AutomatedSyncEnabled` and never synced | [bi-directional Argo CD corner](../../test/e2e/argocd_bi_directional_e2e_test.go) | | The Argo CD guide's example is accepted by the code | [`argocd_guide_test.go`](../../internal/applier/argocd_guide_test.go) | diff --git a/docs/design/branch-worker-event-model.md b/docs/design/branch-worker-event-model.md index 9c81c07c..93e8036c 100644 --- a/docs/design/branch-worker-event-model.md +++ b/docs/design/branch-worker-event-model.md @@ -1,6 +1,6 @@ # Branch worker events, durable execution, and recovery -A single failed push on a quiet branch used to leave a `CommitRequest` in `WaitingForPush` +A single failed push on an idle branch used to leave a `CommitRequest` in `WaitingForPush` indefinitely, even after Git recovered. #412 repaired the retry half of that liveness gap, and #413 finished the in-process half: bounded Git calls, a bounded outage backlog with explicit pause and resume, and a status that says so. The branch worker remains the owner of its checkout. @@ -105,7 +105,7 @@ not enforce that exactly one alternative is set. | Attach a save | `CommitRequest` controller through the event router | `handleAttachCommitRequest` | | Withdraw a save | `CommitRequest` controller through the event router | `handleWithdrawCommitRequest` | | Complete scoped snapshot | Watch replay through `EnqueueResync` | `handleResyncRequest` | -| Observe remote and folder | `GitTarget` reconcile through `EnqueueRefresh` | `handleRefreshRequest` | +| Observe remote and folder | `GitTarget` reconcile through `EnqueueRefresh`; a push notice through `EnqueuePushNotification` | `handleRefreshRequest` | This inventory is the starting point for extracting explicit transitions. The atomic handler is supported, but it has no non-test producer; step 5a of the @@ -164,7 +164,7 @@ These gaps are visible in the current code. Making the queue durable addresses o ### A failed push stranded a save in `WaitingForPush` (fixed in #412) Before #412, `pushPending` retained failed writes but stopped the push timer without arming a -retry. If no more writes arrived, a quiet branch stayed unpublished indefinitely, and ordinary +retry. If no more writes arrived, an idle branch stayed unpublished indefinitely, and ordinary refresh skips a branch with retained work, so periodic `GitTarget` reconciliation did not repair it. The failure now schedules its own retry. The rest of this section records why the fix had to be on the worker. @@ -355,8 +355,9 @@ window membership, policy decisions, and operation results. The HA plan owns the Live interfaces, process pointers, and reply channels remain adapters outside that format. Snapshot replacement has execution semantics of its own. Live writes, overlapping snapshots, -and save attachments can fence replacement of an earlier queue position. Deferred heals wait -behind an open window. Preserve those decisions when persisting the +and save attachments can fence replacement of an earlier queue position. A snapshot collected +while Git was ahead of the cluster, or across an apply, is discarded and asked for again. Preserve +those decisions when persisting the [snapshot protocol](../future/ha-gittarget-distribution-plan.md#snapshot-and-replay-delivery). [Resync handling](../../internal/git/resync_flush.go) still reports local application separately from publication. @@ -420,7 +421,7 @@ The current worker has six timer sources: | Retry | Retry a failed publication on its backoff, or probe for a missing parent | | Attach timeout | Settle a save still waiting for an eligible window | | Refusal action | Recheck consent before a coalesced Refusal request is due | -| Applier | Send an apply attempt that came due on a quiet branch | +| Applier | Send an apply attempt once its backoff or the initial-snapshot barrier allows, close an apply window after its grace, and ask again for a snapshot held off for its limit | The retry schedule has its own timer; the push timer is only the success cooldown. Parent recovery has no timer of its own since #413: while recovery is open, the retry deadline is @@ -518,7 +519,7 @@ Journal crashes, retention, encryption, and HA failure tests belong to the | Scenario | Required observation | |---|---| -| Save's first push fails, then the branch goes quiet | Built (#412): retry publishes and resolves the same request without another edit | +| Save's first push fails, then the branch goes idle | Built (#412): retry publishes and resolves the same request without another edit | | Held save outlives the controller's safety window | Built (#412): withdrawal cannot falsely fail it; recovery remains scheduled | | Push fails under continuous arrivals | Built (#412): arrivals cannot bypass the retry budget | | Parent returns, but publication still fails | Built (#413): new commits respect the one retry deadline while recovery remains open | @@ -550,8 +551,8 @@ explicit transfer or terminal decision before that responsibility can be release Snapshots establish render fidelity even while ordinary live writes are refused. Suspension suppresses writes while allowing observation. A single dispatcher-level write gate would block -the recovery that can reopen a target. `Heal=true` snapshots defer behind any open window; the -inspected production replay producer uses `Heal=false`. +the recovery that can reopen a target. Every resync closes the open commit window before it is +decided. [Refresh handling](../../internal/git/refresh.go) normally skips a busy checkout. During parent recovery it can service a due probe and publish retained work. Disabling periodic refresh does diff --git a/docs/design/event-pipeline-overview.md b/docs/design/event-pipeline-overview.md index ce73d8ad..501a9719 100644 --- a/docs/design/event-pipeline-overview.md +++ b/docs/design/event-pipeline-overview.md @@ -1,8 +1,10 @@ # Event pipeline overview: from a watch event to a Git commit > **Snapshot, 2026-10-04, at the end of #413.** A picture-first tour of how a change -> in the cluster becomes a commit today, with the replay paths drawn out and an honest list of -> what is still missing. It describes behavior that exists. The plan that changes it is +> in the cluster becomes a commit, with the replay paths drawn out and an honest list of what is +> still missing. What was removed since is taken out; the apply trigger, which came later, is not +> drawn here (its timer, and the flush that closes a window when an apply comes due): see +> [`apply-trigger.md`](apply-trigger.md). The plan that changes it is > [`gittarget-branch-worker-pending-writes.md`](gittarget-branch-worker-pending-writes.md); the longer-term event model > is [`branch-worker-event-model.md`](branch-worker-event-model.md). @@ -231,9 +233,7 @@ the fence (`tailPassed`) forces a new position instead. The payload the old mark request swapped into it earlier; since step 5b that request runs, where before it was dropped and its caller never answered. -A resync marked `Heal` (a re-anchor that must not close another target's window) is parked until -the window is idle. No production producer sends one today; the watch replay sends `Heal=false`, -which closes the open window first to keep arrival order. +Every resync closes the open window first, to keep arrival order. No snapshot is parked. ## 4. Inside the branch worker: inputs and the loop @@ -261,7 +261,7 @@ flowchart TB TIMERS --> SEL SEL --> H["handler"] - H --> AFTER["after every wake:
service waiting saves,
apply parked heals,
publish held bytes to the intake gate"] + H --> AFTER["after every wake:
service waiting saves,
publish held bytes to the intake gate"] AFTER --> SEL ``` @@ -285,7 +285,7 @@ flowchart LR ``` Other things that close a window: the idle or maximum-duration timer, an atomic batch or a -non-heal resync (to keep arrival order), a save with `attach: Next`, and shutdown. A closed window +resync (to keep arrival order), a save with `attach: Next`, and shutdown. A closed window becomes one decided write. The other write gate, `spec.suspend`, applies later, when the write is committed: a suspended @@ -393,7 +393,7 @@ The intake gate ([`intake.go`](../../internal/git/intake.go)) applies only while pending; a healthy branch never pauses, because its pending writes drain at the next push. - **What it counts.** Payload on the FIFO, charged at enqueue so producers cannot fill the FIFO - between two loop iterations, plus what the loop holds: the open window, the pending writes, parked heals, + between two loop iterations, plus what the loop holds: the open window, the pending writes, and saves waiting for a window. Every item is charged 1 KiB beyond its payload and message, so empty saves and refusal commits count and the budget bounds the item count too. Sizes are serialized YAML, estimated once by the producer. @@ -511,7 +511,7 @@ This is the honest list. Each item names who plans to close it, or says nobody d ### Bounded memory and pause/resume -Step 5b closed the gaps this section used to list: queued payload, empty records, parked heals, +Step 5b closed the gaps this section used to list: queued payload, empty records, and waiting saves are counted; the pause holds until a push lands; producers wait instead of polling; resyncs respect the retry deadline; and an oversized snapshot has an explicit path. What remains: @@ -570,6 +570,6 @@ remains: | Pending writes: decide, materialize, settle | [`internal/git/pending_writes_loop.go`](../../internal/git/pending_writes_loop.go) | | Retry deadline | [`internal/git/retry.go`](../../internal/git/retry.go) | | Publication report and backlog gauges | [`internal/git/publication.go`](../../internal/git/publication.go) | -| Resyncs and heals | [`internal/git/resync_flush.go`](../../internal/git/resync_flush.go) | +| Resyncs | [`internal/git/resync_flush.go`](../../internal/git/resync_flush.go) | | Saves | [`internal/git/commit_request_attach_loop.go`](../../internal/git/commit_request_attach_loop.go) | | Missing parent probe | [`internal/git/parent_recovery.go`](../../internal/git/parent_recovery.go) | diff --git a/docs/design/gittarget-branch-worker-pending-writes.md b/docs/design/gittarget-branch-worker-pending-writes.md index 67045061..fe7ff683 100644 --- a/docs/design/gittarget-branch-worker-pending-writes.md +++ b/docs/design/gittarget-branch-worker-pending-writes.md @@ -56,7 +56,7 @@ message, and save were still intact. Step 2 fixes it. Three pieces of loop-owned state: - **`pendingWrites`**: decided entries, in order, that are not yet published. Entry kinds are the existing - `PendingWrite` kinds (window, atomic, resync, request record, refusal touch). Accepted entries + `PendingWrite` kinds (window, atomic, resync, request record). Accepted entries survive saturation. There is no `owedSnapshot` entry (a placeholder that would request a snapshot again) replacing discarded accepted work. - **`checkout`**: `{root, rootGen, applied}`. The checkout equals `root` plus the commits of @@ -111,12 +111,14 @@ termination keep their explicit shutdown behavior; they cannot be described as p 1. Replay **accepted decisions** from the pending writes onto the remote when the checkout requires rebuilding. Preserve order, messages, attribution, and save boundaries. A terminal refusal - settles only its entry, as step 4 establishes. Each decision is judged against the tree the one - before it left, so commit count and SHAs can change, and a decision that changes nothing there - makes no commit. Two decisions to one object onto a remote that already holds the second one's - content replay as a revert and a reapply: the final tree is right, and each decision keeps its - commit and its save. This is accepted as is; collapsing it would cost a save on the first - decision its commit (`TestDecidedWrite_SameObjectEditsReplayAsDecidedOntoARemoteHoldingTheLast`). + settles only its entry, as step 4 establishes. A replay applies the actions each write decided + on its first commit and plans nothing else (see + [a write carries the actions decided on its tree](#a-write-carries-the-actions-decided-on-its-tree)). + Each decision is judged against the tree the one before it left, so commit count and SHAs can + change, and a decision whose actions change nothing there makes no commit. Two decisions to one + object onto a remote that already holds the second one's content replay as a revert and a + reapply: the final tree is right, and each decision keeps its commit and its save. This is + accepted as is; collapsing it would cost a save on the first decision its commit (`TestDecidedWrite_SameObjectEditsReplayAsDecidedOntoARemoteHoldingTheLast`). 2. Resume **unaccepted observations** from the last safely admitted watch cursor after intake reopens. While paused, close the affected sessions and wait for branch capacity, with a cancellation-safe wakeup and a bounded fallback recheck. Do not reconnect and collect rejected @@ -142,7 +144,7 @@ Source review at `373bf8d7` confirms the pending-writes path and step 4 refusal `skipUnchangedLiveUpdate` stored the content hash before enqueue, so a refused UPDATE was skipped on cursor resume, which then advanced the cursor. - **The byte budget was incomplete** (fixed in step 5b). `syncAdmission` used `pendingWritesBytes` - and a pending retry. `buildRequestRecordWrite` and `buildRefusalTouchWrite` leave `ByteSize` at + and a pending retry. `buildRequestRecordWrite` and `buildRefusalTouchWrite` left `ByteSize` at zero, so repeated empty saves grew the pending writes without reaching the byte threshold, and payload on the FIFO was not counted at all. - **Retry pacing had an exception** (fixed after the 2026-10-04 review). `decide` let resyncs @@ -377,8 +379,8 @@ producer wait. The intake gate is [`intake.go`](../../internal/git/intake.go). - **One budget, bytes and count.** While a failed attempt waits for its retry, `--branch-buffer-max-size` bounds everything accepted and not yet published: payload on the FIFO, charged at enqueue so - producers cannot fill the FIFO between two loop iterations; the open window; the pending writes; deferred - heals; and saves waiting for a window. Every item is charged `pendingWriteOverheadBytes` (1 KiB) + producers cannot fill the FIFO between two loop iterations; the open window; the pending writes; + and saves waiting for a window. Every item is charged `pendingWriteOverheadBytes` (1 KiB) beyond its payload and message, so the byte budget also bounds the item count, empty records included. The producer sizes each event and snapshot once, off the loop, and the loop reuses the size. The charge is serialized YAML, not heap: a queued object is held as an unstructured map of @@ -514,9 +516,8 @@ page is marked built. ## What stays -The open window and its timers, FIFO admission and resync coalescing at enqueue, deferred heals -(still needed so a heal never closes a sibling target's window), the attach loop, the -refusal-touch schedule, `baseTrusted`, the standby state (`newBranchParent`), and the push's +The open window and its timers, FIFO admission and resync coalescing at enqueue, the attach loop, +`baseTrusted`, the standby state (`newBranchParent`), and the push's compare-and-swap with the parent-change admission check. ## Invariants to keep @@ -546,7 +547,48 @@ separate jobs, and only one of them can wait: tree, and the caller waits at most five minutes. So a resync that cannot be materialized now still replies with the error, as today, and the scope stays unproven, which is the conservative direction. If the deferred write is refused when it finally materializes, step 4 reports that - refusal on the target's status. + refusal on the target's status. A snapshot the worker finds stale (collected while Git was ahead + of the cluster, or across an apply) is answered as discarded, which is no failure, and a fresh one + is asked for; see + [stale snapshots](../spec/reconcile-via-watchlist-mark-and-sweep.md#stale-snapshots-are-discarded). + +## A write carries the actions decided on its tree + +Built with [applier echoes](applier-echoes.md#f2-a-write-is-a-list-of-actions-decided-once-on-its-tree) +(F2). A pending write keeps what its first commit decided, and the picture of the cluster it was +decided from is used once: + +- **Decided once.** The first time a write is committed (`executePendingWrite`), its plan decides + its actions on that tree and records them on the write (`actionsDecided`). A window or an atomic + write keeps only the events whose plan changed the tree, compared per object + (`eventsThatChanged`). A resync keeps the upserts that changed the tree and the documents its + sweep removed, by identity (`resyncActions`). +- **An unchanged object is no action.** It is dropped from the write then. A write with no action + left settles at once (`settleCommitted`, `PendingWrite.retained`): it is neither retained nor + pushed, so no replay can plan it onto a newer tip, where its old content would revert what that + tip changed. A suspended target's resync decides nothing and settles the same way. +- **Replay applies those actions and nothing else.** Onto a newer tip, each kept event or upsert is + planned onto the new tree. A resync's delete removes its document if the new tree still holds it, + and only while the target's prune mode, read again before the replay, still sweeps, so a policy + tightened since is obeyed. The sweep never runs again, so a document the newer tip added survives + the replay, under `prune.mode: Always` too. +- **A save keeps every object.** A write carrying a `CommitRequest` is never trimmed: it keeps each + object it captured, unchanged ones included, and the push decides its outcome (`AlreadyPresent` + or `Committed`), as the [CommitRequest contract](../spec/commitrequest-design.md) says. + +Pinned by [`applier_echoes_test.go`](../../internal/git/applier_echoes_test.go): an unchanged object +is no action, a resync replay applies its decided actions, a replayed resync obeys a tightened prune +mode, and a suspended target's resync is not retained. + +### An apply's flush skips the push cooldown + +The push cooldown (`PushCooldown`, `5s`) still paces ordinary publication, with one exception. When +an apply comes due for an applier on the branch, the worker closes the open commit window +(`git_commit_windows_total{close_reason="apply_due"}`), takes the flush mark (the last write decided +so far), and publishes the writes up to that mark at once, cooldown or not (`maybeSchedulePush`). +The apply waits for that publication and is sent for the tip it produced. Writes decided after the +mark do not hold the send and wait for the cooldown as before. See +[S1 in applier echoes](applier-echoes.md#s1-flush-then-apply). ## Decisions diff --git a/docs/design/push-cooldown.md b/docs/design/push-cooldown.md index f254e3ab..d390a3c6 100644 --- a/docs/design/push-cooldown.md +++ b/docs/design/push-cooldown.md @@ -43,7 +43,9 @@ a manager restart, or Redis. All targets sharing that worker share the pause; ot ```go if len(l.pendingWrites) == 0 { return } if l.awaitingRetry() { return } // a failed attempt has its own deadline -if l.lastPushAt.IsZero() { l.pushPending(); return } // never pushed: go now +if l.lastPushAt.IsZero() || l.publishesAtOnce() { // never pushed, or an apply waits: go now + l.pushPending(); return +} if time.Since(l.lastPushAt) >= PushCooldown { l.pushPending(); return } if l.pushTimer == nil { l.pushTimer = time.NewTimer(...) } // otherwise wait out the remainder ``` @@ -51,7 +53,9 @@ if l.pushTimer == nil { l.pushTimer = time.NewTimer(...) } // otherwise wait out `lastPushAt` advances **only on a successful push**, so the cooldown paces successful publication and does not pace retries. The publication backoff does that separately. Parent recovery owns its attempts while active, and new commits wait for its deadline; see the -[worker event model](branch-worker-event-model.md). +[worker event model](branch-worker-event-model.md). A flush for an apply skips the cooldown: the +writes up to the flush mark are pushed at once (`publishesAtOnce`), because the apply waits for +them ([the apply trigger](apply-trigger.md#flush-then-apply)). ## 2. Ordinary timer closures already space a single identity @@ -155,9 +159,9 @@ is half right. | `pushTimer` and its select arm | Success-cooldown scheduling only since step 3b | | `PushCooldown` constant and its documentation | 1 constant, several doc paragraphs | -The failure backoff uses its own timer in `retrySchedule`. The worker currently has five timer -sources: window, success push, attach, refusal action, and retry. Parent recovery uses the retry -deadline. Removing the success cooldown would leave four sources. +The failure backoff uses its own timer in `retrySchedule`. The worker currently has six timer +sources: window, success push, attach, refusal action, applier, and retry. Parent recovery uses the +retry deadline. Removing the success cooldown would leave five sources. **Stays exactly as it is:** @@ -258,7 +262,7 @@ stateDiagram-v2 note right of Idle No timer armed. - A healthy quiet branch is silent. + A healthy idle branch is silent. end note ``` diff --git a/docs/design/push-notification-and-reconcile-trigger.md b/docs/design/push-notification-and-reconcile-trigger.md index 28e76cbb..54b1e14e 100644 --- a/docs/design/push-notification-and-reconcile-trigger.md +++ b/docs/design/push-notification-and-reconcile-trigger.md @@ -4,7 +4,10 @@ > current design and what is still open. This page is the record of how the apply trigger was > decided. It stays in `design/` because its Part 1 (how the branch worker reads the remote) is > still cited from the Git code by its section numbers, which predate the last restructuring of -> this page. +> this page. Parts 2 to 4 are the design as it was decided then, and much of it has changed since: +> no apply waits for a quiet branch (an apply comes after a flush), `quietPeriod` is gone, an +> Argo CD `Application` gets `CatchUp` too, and a push of the worker's own asks for no apply. Read +> [`apply-trigger.md`](apply-trigger.md) for current behavior. GitOps Reverser and the applier (Flux or Argo CD) share a branch. This page is how they talk: how GitOps Reverser learns that somebody else pushed, and how it decides when the applier applies. diff --git a/docs/design/support-boundary/argocd-bi-directional.md b/docs/design/support-boundary/argocd-bi-directional.md index 3687b16f..f26c5941 100644 --- a/docs/design/support-boundary/argocd-bi-directional.md +++ b/docs/design/support-boundary/argocd-bi-directional.md @@ -146,8 +146,9 @@ setting. Verdict: **cannot be used to be slow.** ## The configuration that works Turn automated sync **off** and let GitOps Reverser start every sync. Nothing in Argo CD then -applies on its own, so the only syncs are ones started on a quiet branch, pinned to a commit with -nothing live missing from it. Argo CD's own rules make it safe to drive this way: a sync is started +applies on its own, so the only syncs are ones started after a flush, pinned to a commit that +holds every edit published before it; an edit made after the flush is repaired should the sync +overwrite it. Argo CD's own rules make it safe to drive this way: a sync is started by setting the `Application`'s `operation` as automated sync would (`controller/appcontroller.go` `autoSync`), and never while another operation is set (`util/argo/argo.go` `SetAppOperation`). With automated sync off, a refresh only compares: `autoSync` returns at once when @@ -185,7 +186,7 @@ forgotten. | PreSync hook (veto) | **No** | also aborts legitimate deploys; can't tell them apart | | self-heal timeout / backoff | **No** | throttles repeats; first drift reverted immediately | | `selfHeal: false` + push webhook | **Mostly** | nothing reverts drift, but every new commit, GitOps Reverser's own included, is synced over pending edits | -| automated sync off + an `ArgoCDTrigger` | **Yes** | GitOps Reverser starts every sync, on a quiet branch, pinned to a checked commit | +| automated sync off + an `ArgoCDTrigger` | **Yes** | GitOps Reverser starts every sync, after a flush, pinned to a checked commit | | admission gate on Argo's writes | (recovers "revert unauthorized") | per-write authority decision; external to Argo | ## References diff --git a/docs/future/ha-gittarget-distribution-plan.md b/docs/future/ha-gittarget-distribution-plan.md index ec7207c6..2fc14f30 100644 --- a/docs/future/ha-gittarget-distribution-plan.md +++ b/docs/future/ha-gittarget-distribution-plan.md @@ -244,9 +244,9 @@ fresh replay, never loss of the old journal tail. Tie every member and completion marker to a snapshot ID and its exact scope. An incomplete snapshot cannot authorize deletion; an explicitly complete empty one can authorize a scoped sweep. Preserve ordering fences where live writes, overlapping snapshots, or saves prevent -replacement of an earlier queue position. Persist supersession and deferred-heal state when -needed to reproduce the worker's decisions. A successful local resync reply does not acknowledge -remote publication. +replacement of an earlier queue position. Persist supersession and stale-snapshot state (a +discarded snapshot asked for again, and since when) when needed to reproduce the worker's +decisions. A successful local resync reply does not acknowledge remote publication. ## Publication recovery diff --git a/docs/future/idea-argocd-ui-user-attribution.md b/docs/future/idea-argocd-ui-user-attribution.md new file mode 100644 index 00000000..1b2b50b5 --- /dev/null +++ b/docs/future/idea-argocd-ui-user-attribution.md @@ -0,0 +1,190 @@ +# Idea: name the Argo CD user behind a UI edit + +Status: investigation only, nothing built. Read against Argo CD `master` at +[`4e682980`](https://github.com/argoproj/argo-cd/tree/4e682980460d2ebbaac20cf78f4795ce44c3cf34) +(2026-10-18). + +## The question + +For an ArgoCon demo: someone clicks around in the Argo CD UI, and the change shows up in a GitOps +folder. Can the commit name the person who was logged in, rather than Argo CD? + +## Short answer + +Not from the Kubernetes audit log. Partly from Argo CD's own audit Events, today. Exactly, after a +small upstream change. + +| Source | Names the person? | Joins to the write exactly? | +|---|---|---| +| Kubernetes audit (what we use now) | No: `system:serviceaccount:argocd:argocd-server` | Yes | +| Argo CD audit Event | Yes, as `annotations.user` and in `message` | No, see below | +| Argo CD audit Event, with the upstream change | Yes | Yes, by uid + resourceVersion | + +## Why the audit log cannot name the person + +Every UI action that writes to the cluster is executed by `argocd-server` with its own credentials. +The audit event's `user` is the server's service account, and we take the author from `user`, or +from `impersonatedUser` when it is set +([audit_event_parsing.go](../../internal/queue/audit_event_parsing.go)). + +Argo CD's impersonation feature does not change this. It impersonates the service account an +AppProject sets per destination in `destinationServiceAccounts` +([`DeriveServiceAccountToImpersonate`](https://github.com/argoproj/argo-cd/blob/4e682980460d2ebbaac20cf78f4795ce44c3cf34/util/settings/impersonation.go#L19)), +never the logged-in user. At best the author becomes a better-named service account. + +## What Argo CD writes to its audit Event + +`argocd-server` records each user action as a `core/v1` Event through +[`AuditLogger.logEvent`](https://github.com/argoproj/argo-cd/blob/4e682980460d2ebbaac20cf78f4795ce44c3cf34/util/argo/audit_logger.go#L49). +These Events are on by default: `--enable-k8s-event` / `ARGOCD_ENABLE_K8S_EVENT` defaults to `all`. +The user name sits on the **Event**, never on the changed resource. + +These are all the fields it sets. Every other field of the Event is left empty. + +| Field | Value | +|---|---| +| `metadata.name` | `.`. The only place the time is kept to the nanosecond. | +| `metadata.namespace` | The Application's namespace for app Events; Argo CD's own namespace for resource Events. | +| `metadata.labels` | App labels plus AppProject labels, filtered by `resource.includeEventLabelKeys` / `resource.excludeEventLabelKeys` in `argocd-cm`. Empty unless configured. Resource Events never have labels. | +| `metadata.annotations.user` | The logged-in user, see [Who "user" is](#who-user-is). Present only when there is one. | +| `metadata.annotations.dest-server` | `app.spec.destination.server` (app Events only). | +| `metadata.annotations.dest-namespace` | `app.spec.destination.namespace` (app Events only). | +| `metadata.annotations.resource-namespace` | The resource's namespace (resource Events only). | +| `involvedObject.{apiVersion,kind,name,namespace,uid,resourceVersion}` | The object the Event is about: the **Application** for app Events, the **resource** for resource Events. | +| `reason` | `ResourceCreated`, `ResourceUpdated`, `ResourceDeleted`, `ResourceActionRan` or `OperationStarted`. | +| `type` | `Normal` | +| `message` | `" "`, for example `alice@example.com patched resource apps/Deployment 'guestbook-ui'`. `"Unknown user"` when there is none. | +| `source.component` | `argocd-server` | +| `firstTimestamp`, `lastTimestamp` | `time.Now()` when the Event is written. These serialize to whole seconds. | +| `count` | `1` | + +### Which UI action writes which Event + +Line numbers refer to +[`server/application/application.go`](https://github.com/argoproj/argo-cd/blob/4e682980460d2ebbaac20cf78f4795ce44c3cf34/server/application/application.go). + +| UI action | `reason` | `message` after the user name | `involvedObject` | Its `resourceVersion` | +|---|---|---|---|---| +| Edit live manifest ([L1574](https://github.com/argoproj/argo-cd/blob/4e682980460d2ebbaac20cf78f4795ce44c3cf34/server/application/application.go#L1574)) | `ResourceUpdated` | `patched resource / ''` | Application | The app's, unrelated to the patch | +| Delete resource ([L1617](https://github.com/argoproj/argo-cd/blob/4e682980460d2ebbaac20cf78f4795ce44c3cf34/server/application/application.go#L1617)) | `ResourceDeleted` | `deleted resource / ''` | Application | The app's | +| Resource action, such as restart ([L2686](https://github.com/argoproj/argo-cd/blob/4e682980460d2ebbaac20cf78f4795ce44c3cf34/server/application/application.go#L2686)) | `ResourceActionRan` | `ran action on resource //` | Application | The app's | +| … and a second Event for the same action | `ResourceActionRan` | `ran action ` | **the resource** (has its uid) | **Wrong: the API version, such as `v1`** | +| Create app ([L395](https://github.com/argoproj/argo-cd/blob/4e682980460d2ebbaac20cf78f4795ce44c3cf34/server/application/application.go#L395)) | `ResourceCreated` | `created application` | Application | The new object's, so it matches | +| Edit app: `Update`, `UpdateSpec`, `Patch` ([L1045](https://github.com/argoproj/argo-cd/blob/4e682980460d2ebbaac20cf78f4795ce44c3cf34/server/application/application.go#L1045)) | `ResourceUpdated` | `updated application spec` | Application | The **old** one, from before the update | +| Delete app ([L1233](https://github.com/argoproj/argo-cd/blob/4e682980460d2ebbaac20cf78f4795ce44c3cf34/server/application/application.go#L1233)) | `ResourceDeleted` | `deleted application` | Application | The old one, which is fine for a delete | +| Sync ([L2212](https://github.com/argoproj/argo-cd/blob/4e682980460d2ebbaac20cf78f4795ce44c3cf34/server/application/application.go#L2212)) | `OperationStarted` | `initiated sync to ` | Application | The new one, after the operation was set | +| Terminate operation ([L2548](https://github.com/argoproj/argo-cd/blob/4e682980460d2ebbaac20cf78f4795ce44c3cf34/server/application/application.go#L2548)) | `ResourceUpdated` | `terminated running operation` | Application | The old one | + +All of these are written **after** argocd-server's own write succeeded. A write that fails produces +no Event. For a sync that write only sets the `Application`'s `operation`: `OperationStarted` says a +sync was started, not that any resource was written, and the sync itself can still fail. + +A Lua action can also **create** resources. Those creates are named in no Event at all. + +For contrast, the application-controller writes Events through the same logger, but always without a +user, so they never carry `annotations.user`. + +An Edit-live-manifest Event looks like this: + +```yaml +apiVersion: v1 +kind: Event +metadata: + name: guestbook.18a2f3c4d5e6f7a8 + namespace: argocd + annotations: + user: alice@example.com + dest-server: https://kubernetes.default.svc + dest-namespace: guestbook +involvedObject: + apiVersion: argoproj.io/v1alpha1 + kind: Application + name: guestbook + namespace: argocd + uid: 0b6c… # the Application's uid, not the Deployment's + resourceVersion: "48211" +reason: ResourceUpdated +type: Normal +message: alice@example.com patched resource apps/Deployment 'guestbook-ui' +source: + component: argocd-server +firstTimestamp: "2026-10-18T09:14:03Z" +lastTimestamp: "2026-10-18T09:14:03Z" +count: 1 +``` + +### Who "user" is + +[`session.Username`](https://github.com/argoproj/argo-cd/blob/4e682980460d2ebbaac20cf78f4795ce44c3cf34/util/session/sessionmanager.go#L693) +returns: + +- for a local Argo CD account, its name, such as `admin` +- for SSO through Dex or another OIDC provider, the `email` claim when present, otherwise Dex's + `federated_claims.user_id`, otherwise `sub` + +For a demo, an SSO login gives a real email address. + +## What we could build today: an approximate match + +To match, we would read `argocd-server`'s Events and pair each one with an audit fact for the same +write: + +- The audit fact says `argocd-server` wrote `apps/Deployment guestbook-ui` in `guestbook` at time T. +- The Event says `alice patched resource apps/Deployment 'guestbook-ui'` shortly after T, on an + Application whose `dest-namespace` is `guestbook`. + +The match is weaker than an audit match, for three reasons: + +- **No resource key** on the patch and delete Events. The resource exists only as group, kind and + name inside free text, and its namespace has to be taken from the Application's destination. +- **Time is the only order.** Two people editing the same object within the window cannot be told + apart. The nanosecond time hidden in `metadata.name` helps, but it records when the Event was + written, not when the write happened. +- **An Event proves nothing about who wrote it.** Anyone allowed to create Events in the Argo CD + namespace can write `user: alice`. To trust one, we would check through audit that + `argocd-server` created it, or present the result as Argo CD's claim and not as a fact. + +That makes it a separate attribution tier, "Argo CD reports alice", alongside the audit match, never +mixed into it silently. It is good enough for a demo, but not for an audit trail. + +## The upstream change that would make the match exact + +`argocd-server` already holds the identity of what it wrote at the moment it logs the Event. It just +doesn't record it: + +1. **Edit live manifest.** `kubectl.PatchResource` returns the patched object (`manifest`), with its + uid and **new** resourceVersion, and the Event is logged on the very next lines. Writing a + resource Event from `manifest` gives us uid + resourceVersion. +2. **Delete resource.** The tree node `res` already has the resource's uid. A resource Event with + that uid is enough, since a delete is matched by uid. +3. **Resource actions.** `patchResource` and `createResource` throw away the API server's response. + Returning it would give the new resourceVersion for patches and a uid for creates, which currently + have no Event at all. +4. **Bug fix, worth sending on its own.** `LogResourceEvent` sets + `involvedObject.resourceVersion` to `res.Version`, which is the **API version** + ([audit_logger.go#L164](https://github.com/argoproj/argo-cd/blob/4e682980460d2ebbaac20cf78f4795ce44c3cf34/util/argo/audit_logger.go#L164)). + `ResourceNode` does have a `ResourceVersion` field, but it comes from the cache, from before the + action, so the real fix is to pass in the object the write returned. +5. **App edits and terminate** log the object from before the update (`app`, `a`) instead of the + result (`res`, `updated`). Swapping them gives the new resourceVersion. + +With uid + resourceVersion on the Event, it carries the same key as one of our audit facts +([author_fact.go](../../internal/queue/author_fact.go)). We could join it exactly and drop the +time-window guesswork. For writes made through the Argo CD UI, the user name would then be as exact +as the audit match. + +### The alternative upstream change: forward the identity itself + +If `argocd-server` impersonated the logged-in user (and their groups) on resource writes, the +Kubernetes audit event would carry `impersonatedUser: alice`. We already read that field, so this +would need no change on our side. But Argo CD would need `impersonate` rights on users, which is a +security-model change. Its current impersonation design deliberately stops at service accounts. +It's a strong message for a talk, and a long road to a merge. + +## For the ArgoCon proposal + +- **Works today:** click in the UI and the change lands in Git, authored as `argocd-server`. +- **Demo-able with modest work:** the real user name, from the approximate match on Argo CD's audit + Events, with the caveats above. +- **The ask to the Argo CD community:** put the uid and resourceVersion of what you wrote on your + audit Event. Items 1 to 3 are small, item 4 is a bug, and together they make UI attribution exact. diff --git a/docs/installing-apps-as-krm.md b/docs/installing-apps-as-krm.md index d7d4edb3..6bf612c6 100644 --- a/docs/installing-apps-as-krm.md +++ b/docs/installing-apps-as-krm.md @@ -104,7 +104,10 @@ GitOps Reverser ignores a few purely-runtime kinds by default — Pods, Events, Endpoints/EndpointSlices, Leases, ControllerRevisions, Jobs, CronJobs — but it does **not** ignore Deployments, ReplicaSets, Services, or ConfigMaps, because those are often exactly the intent you *do* want to capture. Drawing the line is -your job, and it is worth doing deliberately. +your job, and it is worth doing deliberately. The one line the operator draws +itself is for a folder an apply trigger applies: it leaves out objects another +Flux `Kustomization` or Argo CD `Application` owns +([why](bi-directional.md#objects-another-applier-owns)). **How to draw it:** diff --git a/docs/interpreting-metrics.md b/docs/interpreting-metrics.md index 34948b73..e8c800b0 100644 --- a/docs/interpreting-metrics.md +++ b/docs/interpreting-metrics.md @@ -205,10 +205,10 @@ boundary, the commit, the push. Background: | `git_queue_refusals_total` | counter | `provider_namespace`, `provider_name`, `branch`, `kind` | Work refused at enqueue: the queue was full, intake was paused through an outage, or the worker was stopping. `kind` is `write` / `attach` / `resync` / `refresh`. The producer keeps a refused item and offers it again: the watch delivers a write again from its cursor, the controller re-sends a save, a resync is gathered again, and the next reconcile asks for a refresh. What it can cost, when its watch cursor expires before the write comes back, is the write's own commit and, under `prune.mode: OnEvent`, a deleted object's removal from Git. | | `git_commit_failures_total` | counter | `provider_namespace`, `provider_name`, `branch`, `kind`, `reason` | A window or request that died between routing and pushing. `kind` is `window` / `atomic`; `reason` is `refused` (a Git path a human must fix) / `error`. Every increment is a window's events lost until the next resync. An unreachable remote does not count: a decided window waits for the publication retry instead. | | `git_writes_dropped_total` | counter | `provider_namespace`, `provider_name`, `branch`, `kind`, `reason` | A live event (`kind="event"`) or atomic write (`kind="atomic"`) the branch worker dropped because its `GitTarget`'s render fidelity was not established. `reason` is `render_fidelity_pending`: a collection of the `GitTarget` has not finished its replay, which closes every collection of that target until it does. Or it is `render_fidelity_diverged`: a rendered value does not match the live object. Nothing replays a dropped write, so each one is an edit missing from Git until the object changes again or its collection replays. The worker logs the first drop under each gate status at Info, naming the collections the gate waits for. | -| `git_commit_windows_total` | counter | `gittarget_namespace`, `gittarget_name`, `close_reason`, `timer_source` | One per closed commit window, whether or not it then produced a commit. `close_reason` is `idle_timeout` / `max_duration` / `attach_next` / `identity_change` / `buffer_limit` / `resync_before_apply` / `atomic_before_apply` / `foreign_commit` (the branch moved, or a Git host said it may have, so the window is published now) / `shutdown`; `timer_source` is `target` or `commit_request`. See [tuning commit windows](#tuning-commit-windows). | +| `git_commit_windows_total` | counter | `gittarget_namespace`, `gittarget_name`, `close_reason`, `timer_source` | One per closed commit window, whether or not it then produced a commit. `close_reason` is `idle_timeout` / `max_duration` / `attach_next` / `identity_change` / `buffer_limit` / `resync_before_apply` / `atomic_before_apply` / `foreign_commit` (a Git host said the branch may have moved, so the window is published now) / `apply_due` (an apply came due for an applier on the branch, so the window's edits are published before the apply is sent) / `shutdown`; `timer_source` is `target` or `commit_request`. See [tuning commit windows](#tuning-commit-windows). | | `git_commit_window_duration_seconds` | histogram | `gittarget_namespace`, `gittarget_name` | How long each window collected: from the write that opened it to the start of its finalize. Its count equals `git_commit_windows_total` summed over `close_reason` and `timer_source`. | | `git_queue_depth` | gauge | `provider_namespace`, `provider_name`, `branch` | Pending + in-flight + committed-but-unpushed. Read at scrape time. | -| `git_retained_bytes` | gauge | `provider_namespace`, `provider_name`, `branch` | What the worker holds until a push lands, in the units of the retained-byte budget: the open window, the retained writes, deferred snapshots and waiting saves, each with a 1 KiB per-item charge. Serialized bytes, not heap. | +| `git_retained_bytes` | gauge | `provider_namespace`, `provider_name`, `branch` | What the worker holds until a push lands, in the units of the retained-byte budget: the open window, the retained writes and waiting saves, each with a 1 KiB per-item charge. Serialized bytes, not heap. | | `git_retained_writes` | gauge | `provider_namespace`, `provider_name`, `branch` | Decided writes waiting for a push. | | `git_intake_paused` | gauge | `provider_namespace`, `provider_name`, `branch` | 1 while the worker refuses new writes, saves and resyncs because an outage filled its budget; 0 otherwise. | | `git_oldest_retained_write_timestamp_seconds` | gauge | `provider_namespace`, `provider_name`, `branch` | When the oldest write still waiting for a push was decided, as a Unix timestamp. No sample while nothing waits. | @@ -359,7 +359,7 @@ the fresh snapshot then restores the object's content without that change's own **Backpressure during an outage looks like loss and is not.** A branch worker keeps every decided write while its remote cannot be reached, and bounds that at intake: while a failed publication waits for its retry, work that would take what it holds past the retained-byte budget (queued work, -the open window, retained writes, deferred snapshots and waiting saves) pauses its intake. It then +the open window, retained writes and waiting saves) pauses its intake. It then refuses new writes, saves and resyncs the way a full queue does, and both series rise. Intake stays paused until a push lands. The watch does not advance its cursor past a refused event; its stream waits for the branch to reopen intake and then delivers the event again. Only a cursor that expires @@ -621,6 +621,11 @@ picture towards `max_duration`. [sizing the branch worker queue](#sizing-the-branch-worker-queue-against-git_queue_refusals_total). - `resync_before_apply` and `atomic_before_apply` close the window so a resync or an atomic write can apply after it; `shutdown` is the worker stopping. +- `apply_due` closes the window because an apply came due for an applier on the branch: its edits + are published at once, ahead of the push cooldown, so the apply includes them. A window a + `CommitRequest` holds is never closed this way; the apply waits for the save. Its share follows the + apply traffic (Git-side pushes, API creates, repairs), and no timer changes it. `foreign_commit` + is the same early close for a push a Git host reported. The count includes windows that then committed nothing, because their writes already matched Git or their finalize failed. A push that replays commits onto a moved remote closes no window, and neither @@ -824,17 +829,20 @@ else pushed. See [who applies this folder](configuration.md#who-applies-this-fol | Metric | Type | Labels | Meaning | | --- | --- | --- | --- | -| `apply_requests_total` | counter | `applier_kind`, `reason`, `outcome` | One per attempt to one applier. `applier_kind` is `kustomization` (a Flux `Kustomization`) or `application` (an Argo CD `Application`); `reason` is `ForeignCommit` / `Refusal` / `Repair` / `CatchUp` (never `CatchUp` for an `application`); `outcome` is `sent` (the applier was asked, or it needed no request: a `ForeignCommit` or `CatchUp` found the `Kustomization` suspended with the commit applied already, or the remote no longer carries the branch, so there is no commit to apply; it says nothing about how the apply went), `failed` (could not ask, including an applier that applies another repository or branch, an `Application` with automated sync on, or one whose running sync did not end within 5 minutes; the branch worker keeps the reason pending and decides the retry on a quiet branch, so a failure that persists keeps counting at its backoff) or `dropped` (no `GitTarget` may still ask for that reason: each was deleted or recreated, moved to another branch or applier, is no longer admitted by its `ClusterProvider`, or withdrew its consent to a `Refusal`) or `stale` (asked nothing because, by the time of the write, the commit was no longer the branch tip or an edit was waiting to be published; the worker decides again at the next quiet moment) | +| `apply_requests_total` | counter | `applier_kind`, `reason`, `outcome` | One per attempt to one applier. `applier_kind` is `kustomization` (a Flux `Kustomization`) or `application` (an Argo CD `Application`); `reason` is `ForeignCommit` / `Refusal` / `Repair` / `CatchUp`, for either kind (`CatchUp` adopts an object the API created in the folder, or converges once when the worker starts and once when a `GitTarget` first names the trigger); `outcome` is `sent` (the applier was asked and the ask landed; it says nothing about how the apply went. What needs no apply, a converge of the commit the applier last applied or a branch the remote does not carry, the branch worker never sends, so it is not counted), `failed` (could not ask, including an applier that applies another repository or branch, an `Application` with automated sync on, or one whose running sync did not end within 5 minutes; the branch worker keeps the reason pending and decides the retry after a flush, so a failure that persists keeps counting at its backoff) or `dropped` (no `GitTarget` may still ask for that reason: each was deleted or recreated, moved to another branch or applier, is no longer admitted by its `ClusterProvider`, or withdrew its consent to a `Refusal`) or `stale` (asked nothing because the branch tip moved before the ask landed; the worker decides again at once, at the new tip, and frequent stale attempts mean the branch keeps moving within an attempt's lifetime) | | `apply_revision_mismatches_total` | counter | `applier_kind` | One per apply operation that succeeded on another revision than the commit GitOps Reverser checked. A `Kustomization` applies whatever its `GitRepository` artifact holds when it reconciles; the artifact is read again just before the resume, so this is somebody fetching the source in between (a person, or the source's own interval). The apply window still withholds a late write over a newer edit; a steady rate means something else keeps fetching the source | -| `apply_window_collisions_total` | counter | `provider_namespace`, `provider_name`, `branch`, `outcome` | One per write by the applier, while an apply it was asked for runs, to an object the API side edited after the request was sent: the apply arriving late over a newer edit. `outcome` is `withheld` (proven to be the applier's write, by its field manager alone, with an entry newer than the edit: `kustomize-controller`, or `argocd-controller` / `argocd-application-controller`; so not mirrored: Git keeps the API's value, and a `Repair` of the applier follows, which restores it on the cluster) or `ambiguous` (the applier and another field manager, or the applier and the edit, wrote in the same second, so it was mirrored as ever; if it was the apply, the revert is in Git as the applier's commit). An applier entry older than the edit is not counted: it is the API's undo of an edit, mirrored | +| `apply_window_collisions_total` | counter | `provider_namespace`, `provider_name`, `branch`, `outcome` | One per write by the applier, while an apply it was asked for runs, to an object the API side edited after the flush before that request: the apply arriving late over a newer edit. `outcome` is `withheld` (proven to be the applier's write, by its field manager alone, with an entry as new as the edit or newer: `kustomize-controller`, or `argocd-controller` / `argocd-application-controller`; so not mirrored: Git keeps the API's value, and a `Repair` of the applier follows, which restores it on the cluster) or `ambiguous` (the applier and another field manager wrote in the same second, so it was mirrored as ever; if it was the apply, the revert is in Git as the applier's commit). An applier entry older than the edit is not counted: it is the API's undo of an edit, mirrored | +| `apply_echoes_total` | counter | `provider_namespace`, `provider_name`, `branch`, `outcome` | One per write by the applier's own field managers alone, judged by the apply window: an apply GitOps Reverser asked for landing, object by object. `outcome` is `not_mirrored` (the applier wrote the object last, at or after the earliest send of the apply windows still open, and nobody edited it in the window: the apply itself, whose content is the commit Git already holds, so it is not mirrored) or `late` (the applier's managers alone wrote the object last with no apply window open: an echo arriving more than 10 seconds after its operation completed, or an apply somebody else started; it is mirrored as an edit). `not_mirrored` rises with every apply and needs no action. A steady `late` rate means applies keep landing outside the windows, and such a write can revert a newer Git-side change to the object: look for a person or a tool applying the folder by hand (`flux reconcile`, a manual sync), or for watch events reaching the worker long after the apply | +| `apply_ownership_conflicts_total` | counter | `provider_namespace`, `provider_name`, `branch` | One per object another applier owns that the branch worker kept out of a folder an apply trigger applies: a live write not mirrored, or a snapshot object neither upserted nor swept. The owner is read from Flux's owner labels (`kustomize.toolkit.fluxcd.io/name` and `/namespace`) or Argo CD's tracking annotation (`argocd.argoproj.io/tracking-id`). Any non-zero rate is a misconfiguration: another `Kustomization` or `Application` applies objects into a namespace the folder mirrors, and two appliers applying one object fight each other. The worker logs each object once, naming both appliers; narrow the `WatchRule`, or move the object to one applier | +| `stale_snapshots_total` | counter | `provider_namespace`, `provider_name`, `branch`, `outcome` | One per resync snapshot the branch worker found stale: collected while Git was ahead of the cluster (a commit pushed and not yet applied), collected across an apply, or arriving while Git is ahead. `outcome` is `discarded` (not decided; a fresh snapshot of its `GitTarget` is asked for once Git is no longer ahead, and the `GitTarget`'s live writes are held back by the render-fidelity gate until it lands) or `used` (the `GitTarget`'s snapshots had been held for 5 minutes while an apply kept failing, so this one was decided anyway, and whatever the unapplied commit changed in its objects is reverted, as the API-wins rule says). `discarded` comes in short bursts around applies and needs no action; a discard is not a resync failure and is not counted in `git_resync_failures_total`. Any `used` means an applier is failing for minutes at a time: read the trigger's `status.operation` and the `ApplyFailed` Events. See [stale snapshots](spec/reconcile-via-watchlist-mark-and-sweep.md#stale-snapshots-are-discarded) | | `apply_operations_unresolved` | gauge | `provider_namespace`, `provider_name`, `branch` | Apply operations on the branch that have not completed 5 minutes after they were requested. Each is still followed, holds its applier's apply window open, and holds back the applier's next request until it completes, so a value that stays above zero is an applier stuck mid-apply: read its status, and the `ApplyUnresolved` Event on the `GitTarget`s. Read at scrape time | | `receiver_deliveries_total` | counter | `type`, `outcome` | One per verified-or-refused delivery to `/git-push/`. `type` is the `GitProvider.spec.receiver.type`; `outcome` is `notified` (handed to the branch worker), `unmatched` (no worker for that branch), `ignored` (a ping, an event other than a push, a tag or other non-branch ref, or a `generic-hmac` body naming another repository), `rejected` (a missing or wrong signature or token, or an oversized body) or `invalid` (verified but unparseable) | A delivery counted `notified` is only a hint: the worker reads the advertisement, and fetches under `git_fetches_total{reason="refresh"}` only when the branch actually moved. A `ForeignCommit` request follows when it did. So on a branch where only GitOps Reverser pushes, `notified` deliveries are -followed by `CatchUp` requests (to a `Kustomization`; an `Application` gets none) and never by -`ForeignCommit`. +followed by no request at all: a push of the worker's own leaves nothing pending, because the cluster +already runs the edits it holds. ```promql # Requests that could not be delivered, by reason. A failure that persists keeps counting, at the @@ -848,6 +856,16 @@ sum by (provider_namespace, provider_name, branch, outcome) ( increase(gitopsreverser_apply_window_collisions_total[1h])) > 0 # An applier stuck mid-apply: nothing more is asked of it until its operation completes. max by (provider_namespace, provider_name, branch) (gitopsreverser_apply_operations_unresolved) > 0 +# Applier writes mirrored as edits because no apply window was open. A steady rate can revert +# Git-side changes; something applies the folder outside the windows. +sum by (provider_namespace, provider_name, branch) ( + rate(gitopsreverser_apply_echoes_total{outcome="late"}[1h])) > 0 +# Another applier's objects inside a triggered folder: a misconfiguration, logged per object. +sum by (provider_namespace, provider_name, branch) ( + increase(gitopsreverser_apply_ownership_conflicts_total[1h])) > 0 +# Snapshots used although Git was ahead: an applier has been failing for at least 5 minutes. +sum by (provider_namespace, provider_name, branch) ( + increase(gitopsreverser_stale_snapshots_total{outcome="used"}[1h])) > 0 ``` ## Audit attribution (optional) @@ -1318,6 +1336,8 @@ sum by (outcome) (rate(gitopsreverser_secret_encryptions_total[5m])) | `rate(gitopsreverser_attribution_fact_stream_decode_errors_total[10m]) > 0` | A schema or version mismatch on the fact stream; facts are being skipped and lost. | | `(time() - …_fact_follower_last_success_timestamp_seconds > 600) or (…_transport_info == 1 unless on() …_fact_follower_last_success_timestamp_seconds)`, `for: 10m` | The fact follower is wedged; attribution is degrading to committer-authored cluster-wide. Both arms are needed — see below. | | `rate(gitopsreverser_git_resync_failures_total[15m]) > 0` sustained | Background resyncs are not committing; the folder relies on steady-state events to catch up. | +| `increase(gitopsreverser_apply_ownership_conflicts_total[1h]) > 0` | Another `Kustomization` or `Application` applies objects into a folder an apply trigger applies; they are kept out of Git, and the log names both appliers. | +| `increase(gitopsreverser_stale_snapshots_total{outcome="used"}[1h]) > 0` | An applier has been failing for at least 5 minutes, so a resync decided a snapshot taken while Git was ahead and reverted part of the unapplied commit. | | `gitopsreverser_api_catalog_group_versions{state="degraded"} > 0` | Part of the API surface is hidden behind a broken APIService. | | `time() - gitopsreverser_watch_plan_oldest_dirty_since_timestamp_seconds > 120`, `for: 5m` | A GitTarget cannot be planned — its source cluster or catalog is unreachable — and its mirror is not converging. The `time() -` is not optional: the gauge is a Unix timestamp, so a bare `> 120` is true of every instant since 1970 and would fire on ordinary planning. | | `rate(gitopsreverser_watch_plan_passes_total{outcome="timed_out"}[15m]) > 0` sustained | Plan passes are hitting their per-target deadline; the target stays dirty and installs nothing. | diff --git a/docs/spec/README.md b/docs/spec/README.md index 69c41c43..b1fc6b72 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -11,7 +11,8 @@ Three folders, three meanings. This is the one that binds. Everything in this folder states **current behaviour that the code relies on**, and most of it is cited by path from Go source — that citation is how the next person finds out you changed a contract. The handful that are not Go-cited -(`gvk-gvr-mapping-layer`, `sops-single-file-no-multidoc`, `e2e-test-design`) are +(`gvk-gvr-mapping-layer`, `sops-single-file-no-multidoc`, `e2e-test-design`, +`watch-event-ordering-and-attribution-grace`) are here because they are the only written record of a rule the code still obeys. If you change one of these behaviours, change the document in the same commit. @@ -43,8 +44,17 @@ three lifecycle classes and was unreadable spread across them. The rows below st | [`gvk-gvr-mapping-layer.md`](gvk-gvr-mapping-layer.md) | the GVK↔GVR bijection contract | | [`where-validation-lives.md`](where-validation-lives.md) | schema → CEL → **the reconciler**; a webhook only for what exists solely at admission | | [`sops-single-file-no-multidoc.md`](sops-single-file-no-multidoc.md) | one encrypted file is one document | -| [`scale-subresource-audit-rehydration.md`](scale-subresource-audit-rehydration.md) | `/scale` → bounded field patch; every other subresource ignored | +| [`scale-subresource-audit-rehydration.md`](scale-subresource-audit-rehydration.md) | history: the `/scale` field-patch design; a scale is now mirrored from the watch, and the field-patch writer is what remains | | [`commit-window-refactor.md`](commit-window-refactor.md) | one grouped commit = one (author, GitTarget) | | [`commitrequest-design.md`](commitrequest-design.md) | how a request binds to a same-actor commit window and reports its outcome | +| [`commitrequest-admission-authorship.md`](commitrequest-admission-authorship.md) | a request's submitter comes from admission, present-or-never, apart from audit facts | +| [`status-conditions-guide.md`](status-conditions-guide.md) | the conditions and reasons every kind reports | +| [`gitpath-foreign-content-stringency.md`](gitpath-foreign-content-stringency.md) | a `GitTarget` owns its subtree and refuses foreign content; `.gittargetignore` | +| [`unsupported-folder-refusal-plan.md`](unsupported-folder-refusal-plan.md) | history of the `GitPathAccepted` refusal of unsupported folder content | +| [`typeset-owns-discovery-grace.md`](typeset-owns-discovery-grace.md) | the typeset owns discovery grace, and the API-resource catalog is thin | | [`gittarget-isolation-on-rule-change.md`](gittarget-isolation-on-rule-change.md) | a rule change on target A never touches target B | | [`audit-readiness-probe-plan.md`](audit-readiness-probe-plan.md) | why liveness must never depend on Redis | +| [`mutation-capture-lab-design.md`](mutation-capture-lab-design.md) | the lab and corpus of what watch, audit and admission carry, as recorded | +| [`e2e-test-design.md`](e2e-test-design.md) | how the e2e suite is structured: one Git repository per test file | +| [`e2e-serial-registry.md`](e2e-serial-registry.md) | which e2e specs must run `Serial`, and why | +| [`e2e-bi-directional-corner.md`](e2e-bi-directional-corner.md) | the opt-in e2e category where Argo CD and the bi-directional specs run | diff --git a/docs/spec/attribution.md b/docs/spec/attribution.md index 931ee59c..a886697f 100644 --- a/docs/spec/attribution.md +++ b/docs/spec/attribution.md @@ -14,9 +14,9 @@ > What is still **open** is not here. It is ranked in > [`../design/open-asks-priority.md`](../design/open-asks-priority.md). Removal-wait alternatives are in > [`../design/attribution-removal-wait-options.md`](../design/attribution-removal-wait-options.md). -> The [planned filtered-removal policy](../design/watches-labels-simplification.md#attribution-for-filtered-removals) -> adds stricter evidence and immediate release of eligible exact matches for label-filtered -> `DELETED` events. It is not implemented; the behavior below remains the current contract. +> The [filtered-removal policy](../design/watches-labels-simplification.md#attribution-for-filtered-removals) +> for label-filtered `DELETED` events is built, and is the last of the +> [four rules that are easy to miss](#four-rules-that-are-easy-to-miss) below. > > Related: [`watch-event-ordering-and-attribution-grace.md`](watch-event-ordering-and-attribution-grace.md), > [`commitrequest-admission-authorship.md`](commitrequest-admission-authorship.md), @@ -108,9 +108,8 @@ repository means. **In code.** [`routeLiveTargetWatchEvent`](../../internal/watch/target_watch.go) routes an event whose object has `GetDeletionTimestamp() != nil` as a **Delete** regardless of whether the watch type -is `MODIFIED` or `DELETED`; the Delete path emits no body, so the file is removed. This is computed -*before* the operation is matched against the rule's verbs, so a WatchRule that excludes deletes -consistently does not act on the logical delete either. Later finalizer-clearing events re-issue +is `MODIFIED` or `DELETED`; the Delete path emits no body, so the file is removed. A rule has no +operation filter, so every rule acts on the logical delete. Later finalizer-clearing events re-issue "delete X" against an already-absent path, and the writer diffs them to a no-op. **One operational caveat.** Logical absence is a Git statement, and it must not make us blind to a diff --git a/docs/spec/audit-readiness-probe-plan.md b/docs/spec/audit-readiness-probe-plan.md index e0b43768..351fa84e 100644 --- a/docs/spec/audit-readiness-probe-plan.md +++ b/docs/spec/audit-readiness-probe-plan.md @@ -1,10 +1,16 @@ # Audit-pipeline readiness probe: improvement plan -> **Status (2026-06-12): implemented** — `/readyz` now gates on the audit listener serving, the -> TLS cert being loaded, and a startup-only Redis connection; `/healthz` stays a bare `Ping`. The -> e2e harness's live-pipeline gate (`a133137`) is intentionally **kept** for now: it proves the -> apiserver→manager path, which `/readyz` cannot assert (readiness only checks local -> preconditions). §7's e2e simplification remains a future follow-up. +> **Status: implemented; this page is the plan it was built from.** The code is +> [`cmd/readiness.go`](../../cmd/readiness.go). `/readyz` gates on the audit listener serving, the +> audit TLS cert being loaded, and, when Redis is configured (`--redis-addr`), a startup-only Redis +> connection: the first successful `PING`, which latches. `/healthz` stays a bare `Ping`. The body +> below describes the code as it was before the change, so its "today" and its line references are +> from then. It differs from what was built in three ways. The audit pipeline no longer enqueues +> audit events in Redis (Redis holds watch cursors and, with the Redis transport, attribution +> facts), so the gate pings the shared Redis connection (`RedisStore.Ping`) rather than a by-type +> queue. The `redis_up`-style gauge (§5, step 5) was not built. §7's e2e simplification was not made: +> the harness's live-pipeline gate (`warmup_audit_path` in `hack/e2e/inject-webhook-tls.sh`) is +> kept, because it proves the kube-apiserver to manager path, which `/readyz` cannot assert. ## Summary diff --git a/docs/spec/commit-window-refactor.md b/docs/spec/commit-window-refactor.md index c68b8d75..b7220109 100644 --- a/docs/spec/commit-window-refactor.md +++ b/docs/spec/commit-window-refactor.md @@ -12,7 +12,7 @@ refactor was in flight; the implementation has now landed. ## Goals -The git writer turns Kubernetes audit events and reconcile snapshots into Git +The branch worker turns live watch events and snapshot resyncs into Git commits. The design has three main goals: - Keep Git history readable during bursty operations such as `kubectl apply -k`, @@ -34,14 +34,14 @@ second cooldown in the branch worker. A short burst from one logical operation should usually become one commit per author and target, not one commit per Kubernetes object. For example, one `kubectl apply -k` that creates four ConfigMaps for the same `GitTarget` and -same user becomes one grouped commit after the commit window goes quiet. +same user becomes one grouped commit once the window's `idleTimeout` passes with no write. ### Honest authorship -Grouped audit commits are authored by the event user. The operator remains the -committer because it physically writes the Git commit. Atomic reconcile writes -are authored by the operator because they represent a controller snapshot, not a -single human action. +A window's commit is authored by its events' author, from author attribution. The +operator remains the committer because it physically writes the Git commit. Resync +writes are authored by the configured committer because they represent a snapshot +of the cluster, not a single human action. ### Target isolation @@ -55,83 +55,106 @@ same burst. If the remote branch moves before our push lands, unpublished local commits are discarded and rebuilt on top of the new remote tip. Replay uses the metadata resolved when the work entered the pending lifecycle, so a deleted or changed -`GitTarget` does not change how already-pending work is written. +`GitTarget` does not change how already-pending work is written. The one exception +is the prune mode, which is re-read before a replay and only ever tightened. Replay +also applies the actions each write decided when it was first committed, rather +than planning the cluster's picture again onto the new tip. ## Current Flow ```mermaid flowchart TD - A[WriteRequest or audit event] --> B{Commit mode} - B -->|PerEvent| C[Open window: same author and target] + A[Live watch event or resync] --> B{Kind} + B -->|live event| C[Open window: same author and target] C --> E{Finalize trigger} E -->|author or target change| F[PendingWriteCommit] E -->|idleTimeout silence| F E -->|maxDuration| F E -->|byte cap| F E -->|attach: Next| F + E -->|apply due: flush| F + E -->|Git host push notice| F E -->|shutdown| F - B -->|Atomic| Q[Finalize open window if present] + B -->|resync| Q[Finalize open window if present] Q -.-> F - Q -->|then| D[PendingWriteAtomic] + Q -->|then| D[PendingWriteResync] F --> G[pendingWrites] D --> G - G --> H[CommitPlanner projects 1:1] - H --> I[CommitExecutor writes local commits] + G --> I[First commit decides each write's actions on its tree] + I -->|no action left| S[settled: not retained, not pushed] I --> J{Push} J -->|success| K[clear pendingWrites and advance lastPushAt] - J -->|remote moved| L[sync to remote tip] - L --> M[rebuild from pendingWrites] - M --> I + J -->|remote moved| L[reset to remote tip] + L --> M[apply the decided actions onto the new tip] + M --> J J -->|transient failure| N[keep local commits and pendingWrites for retry] ``` `BranchWorker` owns the live open window, timers, repository preparation, push cooldowns, and retry scheduling. It retains `[]PendingWrite` until a push -succeeds. Local commit creation does not clear pending work. +succeeds. Local commit creation does not clear pending work. A write whose first +commit changes nothing in Git is the exception: it settles at once and is neither +retained nor pushed. A save (a write carrying a `CommitRequest`) is always +retained, and the push decides its outcome. Per-event writes are processed as a stream. The branch worker keeps one open window at a time, and that window contains only one author and one target. The window finalizes on its idle timeout or its maximum duration, and immediately on -author change, target change, the byte cap, an `attach: Next` request, or shutdown. +author change, target change, the byte cap, an `attach: Next` request, a resync, +shutdown, an apply coming due, or a Git host's push notice. An apply coming due +closes the window so its edits are published before the apply request is sent +([the flush](../design/apply-trigger.md#flush-then-apply), `close_reason="apply_due"`); +the same holds when a sibling worker on the same repository and branch flushes. A push notice closes it so its writes are +published, and replayed onto the move, at once. Neither applies while a +`CommitRequest` holds the window: no flush is made while a request is registered, +and a push notice leaves an attached window to its save. A timer of `0s` closes the window in the same step that collected the write. Repeated writes to the same Git path inside the open window are last-write-wins while preserving first-seen path order. -Atomic writes are different: they bypass the live-event window, but they do not -jump ahead of already-windowed live events. If an atomic write arrives while the +Resyncs and atomic writes are different: they bypass the live-event window, but +they do not jump ahead of already-windowed live events. If one arrives while the branch worker has an open event window, the worker finalizes that window into a -commit pending write first, then appends the atomic pending write. This -preserves arrival order while keeping atomic writes as one caller-defined batch. +commit pending write first, then appends its own pending write. This preserves +arrival order while keeping a resync or an atomic write as one batch. ## Core Types `WriteRequest` is the external input contract for Git writes. Callers choose between: -- `CommitModePerEvent`, used for live audit events that may be windowed. With +- `CommitModePerEvent`, used for live watch events that may be windowed. With `idleTimeout: 0s`, each event finalizes immediately. -- `CommitModeAtomic`, used for reconcile snapshots that must land as one commit. +- `CommitModeAtomic`, one commit for all events in the request. No production + caller sends one today: a snapshot arrives as a resync (`ResyncRequest`). `PendingWrite` is the durability unit retained until push succeeds: - `PendingWriteCommit` represents one finalized commit-shaped event window. - `PendingWriteAtomic` represents one caller-defined atomic write. - -Pending writes carry resolved target metadata: target identity, destination -path, bootstrap options, and encryption configuration. That metadata is the -replay contract. - -`CommitPlan` is the planner output. It contains ordered `CommitUnit`s. Each -`PendingWrite` maps to exactly one `CommitUnit`, and each unit maps to at most -one local Git commit. - -`CommitExecutor` executes commit units. It applies files, stages bootstrap -templates, configures encryption from the unit, skips no-op units, renders the -message, chooses commit metadata, and creates the local commit. +- `PendingWriteResync` represents one GitTarget's snapshot, applied by + mark-and-sweep. +- `PendingWriteRequestRecord` records a `CommitRequest`'s message in an empty + commit when no window reached the request and it asked for + `whenNothingToCommit: CommitEmpty`. + +Pending writes carry resolved target metadata (`ResolvedTargetMetadata`): target +identity, destination path, bootstrap options, encryption configuration, and +prune mode. That metadata is the replay contract. + +[`commit_executor.go`](../../internal/git/commit_executor.go) commits the pending +writes in order, at most one local Git commit per write. A write's first commit +decides its actions on that tree: a window or atomic write keeps only the events +whose plan changed the tree, a resync keeps the upserts that changed it and the +documents its sweep removed, and a save keeps every object it captured. A replay +(`replayPendingWrites`) applies those actions onto the newer tip. The executor +stages bootstrap templates, configures encryption from the write's metadata, +skips a write that changes nothing, renders the message, chooses commit metadata, +and creates the local commit. ## Window Rules -The branch worker is the only owner of commit grouping for live audit events. +The branch worker is the only owner of commit grouping for live events. It appends to or finalizes the open window on event arrival. Rules: @@ -151,7 +174,9 @@ planner-time regrouping pass. Message precedence is a non-empty literal override, then `commit.message.liveTemplate` for live windows or `commit.message.reconcileTemplate` for atomic snapshots and resyncs. Every live window -uses the same context, including singleton and `0s` windows. Literal text never changes authorship. +uses the same context, including singleton and `0s` windows. A `CommitRequest`'s message is the +literal override, and `commit.message.requestTemplate` can frame it (see +[`commitrequest-design.md`](commitrequest-design.md)). Literal text never changes authorship. Resolved templates and literal overrides travel with pending writes through retries and conflict replay, so editing a target does not change an already finalized write's message configuration. @@ -162,6 +187,7 @@ The live template receives `LiveCommitMessageData`: - `Count` - `Operations` - `Resources` +- `RequestMessage` `Resources` is deduplicated by Git path and rendered in first-seen order. @@ -180,13 +206,14 @@ Push outcomes: push retries the same commits. - Push error plus failed remote-state fetch: treat it like transient. Preserve state and surface the original push error. -- Conflict: if the remote tip differs, sync to the latest remote tip, rebuild - commits from retained pending writes, and retry in the same call. +- Conflict: if the remote tip differs, reset onto the latest remote tip, apply + the retained writes' decided actions onto it, and retry in the same call. Replay correctness is defined by final tree correctness and a successful push, not by preserving unpublished local commit objects. If replay finds that the new -remote tree already contains the desired content, the no-op unit is skipped and -no replacement commit is created. +remote tree already contains the desired content, the write is skipped and no +replacement commit is created. A write the new tree refuses is skipped and +settled as a refusal, and the writes after it are still replayed. ## Operational Controls @@ -205,21 +232,24 @@ finalizes the moment the target changes. An unreadable target, or one declaring no window, takes the `5s` and `1m` defaults. A `CommitRequest` that attaches replaces both timers for that window. -The byte cap is an operator startup setting, `--branch-buffer-max-bytes`, -because it protects pod memory rather than describing user-facing Git history. +The byte cap is an operator startup setting, `--branch-buffer-max-size` +(default `8Mi`), because it protects pod memory rather than describing +user-facing Git history. The push cooldown is fixed at 5 seconds. It keeps fast local commits from spamming the remote while still keeping ordinary single-change latency close to -`idleTimeout + push RTT`. +`idleTimeout + push RTT`. A flush skips it: when an apply comes due, the writes up +to the flush mark are published at once. ## Tests The implementation is covered at three useful levels: -- Planner tests assert 1:1 projection, atomic plan shape, arrival order, and - one-resolution-per-target encryption lookup. +- Pending-write tests assert metadata derivation, arrival order, + one-resolution-per-target encryption lookup, prune-mode tightening, and that a + decided write replays as decided. - Executor tests assert message-kind routing, no-op skipping, and encryption - coming from `CommitUnit` metadata. + coming from the pending write's metadata. - Branch-worker tests assert open-window finalize triggers, retained pending writes, transient vs conflict push handling, replay with deleted target metadata, atomic/commit interleave order, and no-op replay behavior. diff --git a/docs/spec/commitrequest-admission-authorship.md b/docs/spec/commitrequest-admission-authorship.md index ee0d8c95..9818f09f 100644 --- a/docs/spec/commitrequest-admission-authorship.md +++ b/docs/spec/commitrequest-admission-authorship.md @@ -52,8 +52,8 @@ its message in a pushed empty commit (`Pushed=True`). A `WindowMismatch` never f | `Ready` | a pushed commit or benign no-commit | progress or a finalize failure | `Reconciling=True` is the normal in-progress state, with the phase the branch worker reports as its -reason: `Progressing` until the worker registers the request, then `WaitingForWindow`, -`CollectingWindow` and `WaitingForPush`. +reason: `Progressing` until the worker registers the request (`WaitingForWorker` while the GitTarget +has no branch worker yet), then `WaitingForWindow`, `CollectingWindow` and `WaitingForPush`. ## Wiring @@ -64,5 +64,5 @@ An installation may therefore have either, both, or neither source of author inf Related live contracts: - [architecture: CommitRequest Finalize](../architecture.md#commitrequest-finalize) -- [configuration: CommitRequest conditions](../configuration.md#commitrequest-finalize) +- [configuration: CommitRequest conditions](../configuration.md#when-there-is-nothing-to-commit) - [status conditions guide](status-conditions-guide.md) diff --git a/docs/spec/commitrequest-design.md b/docs/spec/commitrequest-design.md index 79d4694f..46896136 100644 --- a/docs/spec/commitrequest-design.md +++ b/docs/spec/commitrequest-design.md @@ -1,6 +1,6 @@ # CommitRequest window finalization -> **spec**: current behavior, reviewed at `373bf8d7` on 2026-10-03. +> **spec**: current behavior, reviewed at `c12f7970` on 2026-10-20. > The code depends on this document; change one, change the other. > Index: [`../INDEX.md`](../INDEX.md) @@ -47,7 +47,13 @@ registration, `maxDuration` from the attach. Each is a pointer, so an omitted va `"0s"` stay distinguishable. A zero `attachTimeout` attaches to a window already open, or gives up at once, and never shortens the collection that follows; a zero `maxDuration` finalizes right after the attach. -Other flush triggers can close an attached window early, carrying its message. Each request attaches +Other window closers can close an attached window early, carrying its message: an author or target +change, the byte cap, a resync, or shutdown. An apply coming due does not. While a request is +registered, from its registration until it resolves, the branch worker makes no +[flush](../design/apply-trigger.md#flush-then-apply) and sends no apply request, because a save +defines its own commit boundary: an apply landing between the request and its writes would revert +its first edits before they reach Git. A Git host's push notice leaves an attached window open too. +Each request attaches to at most one window and cannot rename a finalized commit, including one waiting for push. A bundle provides no ordering guarantee. The wait does not reserve a transaction. Competing requests are served in registration order, first come, first served. @@ -130,10 +136,10 @@ for a push to say. So is `NoWindow` under `Resolve`. Under `CommitEmpty` a `NoWi its message in an empty commit, and it resolves when that commit reaches the remote.) A window that produced no diff used to resolve `AlreadyPresent` at finalize, on the strength of the local plan. That was only sound while every cycle fetched before it planned. It no longer does (see -[inbound push notification](../design/apply-trigger.md)), so the plan may have run -against a tree the remote has moved past, and the replay that follows a rejected push can turn the -same captured object into a commit. "Already present" is a claim about the remote, so the -remote settles it: a no-diff request resolves `AlreadyPresent` when the push confirms there was +[how the branch worker reads the remote](../design/push-notification-and-reconcile-trigger.md#12-the-push-is-the-read)), +so the plan may have run against a tree the remote has moved past, and the replay that follows a +rejected push can turn the same captured object into a commit. "Already present" is a claim about +the remote, so the remote settles it: a no-diff request resolves `AlreadyPresent` when the push confirms there was nothing to add, and `Committed` when the replay produced a commit after all. **A request whose write is decided stays identifiable until publication.** It remains attached to diff --git a/docs/spec/current-manifest-support-review.md b/docs/spec/current-manifest-support-review.md index 41b04c8b..f5d8b4d7 100644 --- a/docs/spec/current-manifest-support-review.md +++ b/docs/spec/current-manifest-support-review.md @@ -2,7 +2,18 @@ > **spec** — current behaviour. The code depends on this document; change one, change the other. Index: [`../INDEX.md`](../INDEX.md) > -> Status: architecture review, captured 2026-06-04 +> Status: design history. An architecture review captured 2026-06-04 that proposed the +> materialized manifest model. That model has shipped: the `ManifestStore` in +> [`internal/manifestanalyzer`](../../internal/manifestanalyzer), planned and flushed once per +> write by [`internal/git/plan_flush.go`](../../internal/git/plan_flush.go). "Current +> Architecture", "How It Works Today", "Key Code Paths", "What Works Well", and "Cons And Gaps" +> describe the code of that date; `FolderReconciler`, the path-derived scan, and the per-batch +> `manifestLocator` are gone. Three decisions below have changed since. Non-YAML files are no +> longer exempt (decision 4): a file in no recognized role refuses the folder +> ([gitpath-foreign-content-stringency.md](gitpath-foreign-content-stringency.md)). The operator +> refuses no folder for unwatched or out-of-scope KRM (decision 3); only the dry-run scan reports +> them. The managed drop is a policy knob: a resync sweeps only under `spec.prune.mode: Always`. +> For current behavior, read [manifest-system.md](manifest-system.md). > Related: > `implementation-plan.md`, > [reconcile-via-watchlist-mark-and-sweep.md](reconcile-via-watchlist-mark-and-sweep.md), diff --git a/docs/spec/e2e-bi-directional-corner.md b/docs/spec/e2e-bi-directional-corner.md index 3843f6db..db54bac0 100644 --- a/docs/spec/e2e-bi-directional-corner.md +++ b/docs/spec/e2e-bi-directional-corner.md @@ -23,7 +23,7 @@ already carries Flux, because the shared e2e dependencies (Gitea, Valkey, Prometheus, cert-manager) are delivered as Flux `HelmRelease`s from [`test/e2e/setup/flux/`](../../test/e2e/setup/flux/) — Flux is load-bearing and cannot be removed. Argo CD is *not* load-bearing: only bi-directional tests need -it. Installing it in all four CI legs would tax every leg to serve two specs. +it. Installing it in every CI leg would tax every leg to serve the corner's specs. **Cohesion.** [`docs/bi-directional.md`](../bi-directional.md) said, under "What is not complete yet", when this corner was designed: @@ -173,7 +173,7 @@ dedicated CI leg: | Piece | Value | | --- | --- | | Category label | `bi-directional` (already exists on the Flux spec) | -| Sub-labels | `flux`, `argocd`, `provenance` | +| Sub-labels | `flux`, `argocd`, `provenance`, `timing`, `applier-echoes`, `push-race` | | Env gate | `E2E_ENABLE_BI_DIRECTIONAL` | | Task | `task test-e2e-bi-directional` | | CI leg | `bi-directional` | @@ -186,17 +186,37 @@ The other specs prove what the loop does and only bound how long it may take. Kustomization or an Argo CD Application applies, then times each sample from the stimulus (an API edit, or a Git-side push landing on Gitea) to the effect an operator would look for: -- the commit on the branch; -- Flux applying it after the default 30s quiet period; -- the trigger holding its applier again; -- the live object changing after a Git-side push through the receiver. +- the commit on the branch, for an API edit and for an API create; +- the create adopted by the applier (a `CatchUp`); +- the live object changing after a Git-side push through the receiver; +- the applier done with that push: an Argo CD sync finished, or Flux applied it and is held again. -Each duration goes into a `gmeasure` experiment, which Ginkgo prints as an N/min/median/max -table and writes into the JSON report. `task allure-e2e-report` attaches it to the spec as +Each duration goes into a `gmeasure` experiment, which Ginkgo prints as a table (N, min, +median, mean, stddev, max) and writes into the JSON report. `task allure-e2e-report` attaches it to the spec as "Measurements". Every sample also has a budget, and a sample over it fails the spec. The budgets catch a path that fell back to the 5-minute steady tick, not a slow runner. `E2E_TIMING_SAMPLES` sets the sample count (default 3). +### The applier-echo specs + +Three more files run the scenarios of +[applier echoes](../design/applier-echoes.md#new-end-to-end-scenarios) on the same kind of +shared folder, each as one describe for Flux and one for Argo CD: + +- [`bi_directional_echoes_e2e_test.go`](../../test/e2e/bi_directional_echoes_e2e_test.go) + (label `applier-echoes`): adopting an API create (also one deleted and created again), an + edit made during an apply, a Git-side push under continuous edits, a resync forced before an + apply finishes, a Git-side addition across a resync under `prune.mode: Always`, no apply after + an API edit, an object another applier owns, and a save made across a Git-side change. +- [`bi_directional_push_race_e2e_test.go`](../../test/e2e/bi_directional_push_race_e2e_test.go) + (label `push-race`): two Git-side pushes in quick succession both reach the cluster, and the + later one stays in Git. +- [`bi_directional_restart_e2e_test.go`](../../test/e2e/bi_directional_restart_e2e_test.go) + (label `applier-echoes`): an API edit and an API create made while the operator is scaled to + zero are mirrored before anything is applied; then a Git-side push made meanwhile is applied, + and the new object is adopted. It scales the controller, so both describes are `Serial` (see + [the Serial registry](e2e-serial-registry.md)). + ### The third spec: `provenance_markers_e2e_test.go` The corner hosts a third spec, and it is the reason the corner is worth more than the @@ -226,14 +246,14 @@ results are written up in It shares the `argocd` namespace with the Argo CD spec but never patches `argocd-secret` and owns its own repo, namespace, and resource names, so it runs in -parallel with both. `BI_DIRECTIONAL_GINKGO_PROCS` is therefore `3`. +parallel with both. -Filter changes so the default suite stops paying for it: +The filters that keep the default suite from paying for it: -- `test/e2e/Taskfile-e2e.yml`: `E2E_LABEL_FILTER` default `!image-refresh` - → `!image-refresh && !bi-directional` -- `.github/workflows/ci.yml`, `full-core` leg: `!manager && !image-refresh` - → `!manager && !image-refresh && !bi-directional` +- `test/e2e/Taskfile-e2e.yml`: `E2E_LABEL_FILTER` defaults to + `!image-refresh && !bi-directional && !source-cluster && !ado` +- `.github/workflows/ci.yml`, `full-core` leg: + `!manager && !image-refresh && !bi-directional && !source-cluster` `full-manager` (`manager`) is unaffected — the bi-directional specs carry no `manager` label. @@ -278,7 +298,7 @@ chart**. `_argocd-installed` `helm pull`s the pinned chart into the ServiceMonitors need. - **Only** `test-e2e-bi-directional` depends on `_argocd-installed`. `prepare-e2e` does not. The suite's own in-process `task prepare-e2e` call - (`e2e_suite_test.go:197`) stays untouched and no-ops on the warm stamp. + (`prepareE2EClusterOnce` in `e2e_suite_test.go`) stays untouched and no-ops on the warm stamp. Deliberately **not** a Flux `HelmRelease` alongside gitea/valkey, even though that is the local idiom: `hack/e2e/wait-flux-services.sh` waits on every `HelmRelease` @@ -335,7 +355,8 @@ devcontainer**: Per the existing per-file CRD-group convention, add `crdGroupArgoBiDirectional = "argo-bi-directional.e2e.example.com"` to [`test/e2e/icecream.go`](../../test/e2e/icecream.go) and register it in the -suite's CRD pre-clean loop (`e2e_suite_test.go:225`). +suite's CRD pre-clean loop (`iceCreamCRDGroups`, which `prepareE2EClusterOnce` in +`e2e_suite_test.go` walks). The Flux spec keeps its exact-commit-count discipline, which works because it drives Flux as a manually triggered applier until its last phase hands the @@ -410,8 +431,9 @@ report `ApplyTriggerReady=True`. ### Spec A — a Git-side push is synced through a pinned `ForeignCommit` -1. Assert Argo CD has run **no operation**: GitOps Reverser's own bootstrap push, - and its start, ask for nothing. +1. Assert that naming the `Application` converges it **once**: a `CatchUp` that + succeeds after the folder's initial snapshot is in Git. After it, GitOps + Reverser's own bootstrap push asks for nothing more. 2. Push a clean `order-1` from the Git side, and nothing else. 3. Assert the `Application`'s operation state is GitOps Reverser's: initiated by `gitops-reverser` with `automated: true`, reason `ForeignCommit` in its info, @@ -502,7 +524,8 @@ The Argo spec therefore covers `IceCreamOrder` CRs only, and ## Consequent code change: `internal/sanitize` -*Implemented.* +*Implemented* as an exact-key `switch` in `isOperationalAnnotation`, which also +matches `kcp.io/cluster`. The sketch below is the design. Argo's tracking annotation is controller bookkeeping and must not reach Git. @@ -511,8 +534,8 @@ that prefix are legitimate user intent that belongs in Git: `argocd.argoproj.io/sync-wave`, `sync-options`, `compare-options`, `hook`. Stripping the prefix would silently delete a user's sync ordering. -`isOperationalAnnotation` is prefix-only today -([`types.go:75-82`](../../internal/sanitize/types.go#L75-L82)); it gains an +`isOperationalAnnotation` was prefix-only +([`types.go`](../../internal/sanitize/types.go)); it gains an exact-key set: ```go @@ -544,11 +567,10 @@ instance label will be committed to Git, and that is not fixable in the sanitize ### Adjacent gap found while auditing -No test covers the `kro.run/` label prefix or the `applyset.kubernetes.io/` -prefixes, though both are in the strip list. Also `kro.run/` is stripped from -labels but **not** from annotations, while `applyset.kubernetes.io/` is stripped -from both — an inconsistency that may or may not be intentional. Worth a -table-driven case in `internal/sanitize/types_test.go` while the file is open. +`internal/sanitize/types_test.go` now has table-driven cases for the `kro.run/` +label prefix and the `applyset.kubernetes.io/` prefixes. `kro.run/` is still +stripped from labels but **not** from annotations, while `applyset.kubernetes.io/` +is stripped from both, an inconsistency that may or may not be intentional. ## CI wiring @@ -563,8 +585,8 @@ New matrix entry in `.github/workflows/ci.yml`: ``` No `e2e_label_filter` and no `e2e_ginkgo_procs`: the task pins its own label filter -and Ginkgo procs (`BI_DIRECTIONAL_GINKGO_PROCS`, default `3`), so values here would -be ignored. +and Ginkgo procs (`BI_DIRECTIONAL_GINKGO_PROCS`, default `4`), so values here would +be ignored. `BI_DIRECTIONAL_TIMEOUT` (default `35m`) bounds the run. `coverage: "1"` — the leg uses the default `config-dir` install, which carries the `GOCOVERDIR` overlay, so Codecov unions its `e2e` upload with the other legs and @@ -575,8 +597,9 @@ check on `main` (only *Project image*, *Unit tests*, *Lint* are), so adding a le carries no merge-gating risk. The two specs are pulled out of `full-core`, so that leg gets faster. -Three procs: the Flux, Argo CD and provenance spec files own separate Gitea repos, -namespaces and resource names, and none is `Serial`. +Four procs: the spec files own separate Gitea repos, namespaces and resource names, +so they run side by side. The two restart describes are `Serial` and run alone after +every parallel spec, so their runtime adds to the leg's. ## Risks @@ -623,7 +646,11 @@ namespaces and resource names, and none is `Serial`. - [`docs/finished/e2e-ci-runner-sharding-plan.md`](../finished/e2e-ci-runner-sharding-plan.md): leg membership and rebalancing - [`test/e2e/flux_bi_directional_e2e_test.go`](../../test/e2e/flux_bi_directional_e2e_test.go) and [`test/e2e/argocd_bi_directional_e2e_test.go`](../../test/e2e/argocd_bi_directional_e2e_test.go): - the two specs in this corner + the two bi-directional specs - [`test/e2e/bi_directional_timing_e2e_test.go`](../../test/e2e/bi_directional_timing_e2e_test.go): the loop's latency, as numbers +- [`test/e2e/bi_directional_echoes_e2e_test.go`](../../test/e2e/bi_directional_echoes_e2e_test.go), + [`test/e2e/bi_directional_push_race_e2e_test.go`](../../test/e2e/bi_directional_push_race_e2e_test.go) + and [`test/e2e/bi_directional_restart_e2e_test.go`](../../test/e2e/bi_directional_restart_e2e_test.go): + the applier-echo scenarios - [`internal/sanitize/types.go`](../../internal/sanitize/types.go): the strip lists diff --git a/docs/spec/e2e-serial-registry.md b/docs/spec/e2e-serial-registry.md index 211524bd..c845e343 100644 --- a/docs/spec/e2e-serial-registry.md +++ b/docs/spec/e2e-serial-registry.md @@ -11,7 +11,7 @@ under bounded Ginkgo parallelism (Phase 2.5 of > The two audit-consumer specs, `bi_directional`, and `aggregated_apiserver` have > all been de-serialized (see the "De-serialized" section). `crd_lifecycle` was > de-serialized and then **re-serialized** (commit `3d249e3`) when a residual -> cluster-wide CRD-discovery churn re-flaked it. The current Serial set is the three +> cluster-wide CRD-discovery churn re-flaked it. The current Serial set is the four > singleton-controller specs, `crd_lifecycle`, and `playground`. ## Rule @@ -38,6 +38,7 @@ the `full-core` leg beside the restart specs instead of at the end of the longes |---|---|---|---| | [test/e2e/restart_reconcile_e2e_test.go](../../test/e2e/restart_reconcile_e2e_test.go) | `Restart Reconcile Safety` | Rollout-restarts the controller deployment. | The controller is a singleton; restarting it disrupts in-flight reconciles/commits for every other spec. | | [test/e2e/flux_restart_recovery_e2e_test.go](../../test/e2e/flux_restart_recovery_e2e_test.go) | `Flux Restart Recovery` | Rollout-restarts the controller deployment while a Flux apply runs, and labels a node so that apply's Deployment can schedule. | The controller is a singleton, as for `Restart Reconcile Safety`; a node label is cluster-wide. | +| [test/e2e/bi_directional_restart_e2e_test.go](../../test/e2e/bi_directional_restart_e2e_test.go) | `Bi Directional restart with work made while down (Flux)`, `(Argo CD)` | Scales the controller deployment to zero and back, to make edits and a Git-side push while the operator is down. | The controller is a singleton, as for `Restart Reconcile Safety`. | | [test/e2e/image_refresh_test.go](../../test/e2e/image_refresh_test.go) | `image refresh dependency chain` | Changes the controller image / redeploys the controller. | Same singleton controller; an image swap perturbs all concurrent specs. | | [test/e2e/crd_lifecycle_e2e_test.go](../../test/e2e/crd_lifecycle_e2e_test.go) | `CRD Lifecycle` | Installs/deletes a cluster-scoped CRD and watches all `customresourcedefinitions` cluster-wide, asserting exact Git file presence/absence. | A concurrent CRD install/delete bumps the global discovery-catalog generation and re-resolves every GitTarget's watched-type tables, delaying this spec's reconcile (CRD file appears late / post-delete sweep lags — the 649↔673 flake). A cluster-wide catalog bump cannot be scoped by name. | | [test/e2e/tilt_playground_e2e_test.go](../../test/e2e/tilt_playground_e2e_test.go) | `playground` | Reusable manual-playground fixture: fixed (non-randomized) `tilt-playground` namespace and `playground` repo/provider names, preserved across runs. | A singleton by construction — its resource names are intentionally stable so `task playground-*` can re-attach, so per-run name isolation does not apply. | @@ -46,10 +47,10 @@ the `full-core` leg beside the restart specs instead of at the end of the longes `CRD Lifecycle` ([crd_lifecycle_e2e_test.go](../../test/e2e/crd_lifecycle_e2e_test.go)) **was de-serialized and then re-serialized** — it is in the Serial table above, not -here. Finding #2's fix ([manager.go:1191](../../internal/watch/manager.go#L1191)) — -resnapshot a target only when its *resolved* plan hash changes — removed the worst +here. Finding #2's fix ([`markInvalidatedTargets` in owner.go](../../internal/watch/owner.go)), +which resnapshots a target only when its *resolved* plan hash changes, removed the worst symptom: a catalog refresh no longer drags unrelated targets into `reconcile: sync …` -resnapshots, and per-file CRD groups ([icecream.go:30](../../test/e2e/icecream.go#L30)) +resnapshots, and per-file CRD groups ([icecream.go](../../test/e2e/icecream.go)) keep the `IceCreamOrder` CRD name-isolated. But a residual effect remained — a concurrent CRD install/delete still bumps the cluster-wide discovery-catalog generation and re-resolves every GitTarget's watched-type tables, delaying this @@ -89,22 +90,22 @@ was de-serialized too. It already owned a dedicated repo and per-file CRD group; its historical "+2 commits only under parallelism" came from cluster-wide GVR catalog churn (another spec installing/deleting a CRD) dragging unrelated targets into a resnapshot whose `reconcile: sync …` commits inflated its exact count. -Finding #2's fix ([manager.go:1191](../../internal/watch/manager.go#L1191)) — -a target only resnapshots when its *resolved* plan hash changes — removed that +Finding #2's fix ([`markInvalidatedTargets` in owner.go](../../internal/watch/owner.go)), +under which a target only resnapshots when its *resolved* plan hash changes, removed that mechanism, the same fix that unblocked `crd_lifecycle`. `Aggregated API server` ([aggregated_apiserver_e2e_test.go](../../test/e2e/aggregated_apiserver_e2e_test.go)) was de-serialized after its `Serial` justification proved **stale**. The note claimed it "installs/removes a cluster-scoped `APIService` while it is in flight", but the wardle APIService (`v1alpha1.wardle.example.com`) is installed once at -cluster setup by the apiservice-audit-proxy HelmRelease -(setup/flux/releases/aggregated-api.yaml) -and stays Available for the whole suite. The spec only *reads* it — it never -registers or removes it — so there is no in-flight discovery perturbation to +cluster setup from `test/e2e/setup/manifests/sample-apiserver/` +and stays Available for the whole suite. The spec only *reads* it (it never +registers or removes it), so there is no in-flight discovery perturbation to serialize against. Everything the spec mutates is name-isolated (own namespace, -own repo, namespace-scoped flunder WatchRule, namespaced Flunders), and its one -cluster-wide assertion (`sum(gitopsreverser_audit_events_total{category="error"}) == 0`) -is a global zero-invariant that is parallel-safe by design. +own repo, namespace-scoped flunder WatchRule, namespaced Flunders). The global +zero-invariant on `gitopsreverser_audit_events_total{category="error"}` runs once +for the whole suite (`assertNoAnomalousAuditOutcomes` in +[e2e_suite_test.go](../../test/e2e/e2e_suite_test.go)), not in this spec. ## Triage list (watch during stability runs) diff --git a/docs/spec/e2e-test-design.md b/docs/spec/e2e-test-design.md index 8724b845..0c9b111f 100644 --- a/docs/spec/e2e-test-design.md +++ b/docs/spec/e2e-test-design.md @@ -40,7 +40,8 @@ This shape won because it gave a good trade-off between isolation and practicali What it does not promise: -- full `ginkgo -p` safety for the whole package yet +- `ginkgo -p` safety for every spec: the specs that touch cluster-wide state run `Serial` + (see [e2e-serial-registry.md](e2e-serial-registry.md)) - removal of all remaining shell from the harness - total independence from shared cluster-side services @@ -114,8 +115,11 @@ These are the canonical Task entry points on the current worktree: | Command | Purpose | |---|---| -| `task test-e2e` | Standard e2e package, excluding CI-only workflow checks and the bi-directional corner | +| `task test-e2e` | Standard e2e package, excluding the `image-refresh` workflow check and the opt-in corners | | `task test-e2e-bi-directional` | Bi-directional corner (Flux + Argo CD); the only lane that installs Argo CD | +| `task test-e2e-source-cluster` | Source-cluster corner: kcp workspaces as remote source clusters; the only lane that installs kcp | +| `task test-e2e-ado` | Mirror into a hosted Azure DevOps repository (needs `E2E_ADO_REPO_URL` and `E2E_ADO_PAT`) | +| `task test-e2e-aggregated-api` | Aggregated API server specs only | | `task test-image-refresh` | Image rebuild / reload invalidation chain | | `task test-e2e-quickstart-helm` | Quickstart framework with Helm install | | `task test-e2e-quickstart-manifest` | Quickstart framework with manifest install | @@ -132,9 +136,10 @@ The intended interpretation of these entry points is: than as part of the normal `task test-e2e` path - `task test-e2e-quickstart-helm` is the important validation for the Helm install path - `task test-e2e-quickstart-manifest` is the important validation for the single-file manifest install path -- `task test-e2e` is a broad spec run but **excludes** the bi-directional corner (its label filter - defaults to `!image-refresh && !bi-directional`), because that corner needs an Argo CD install - `task prepare-e2e` does not perform — run it with `task test-e2e-bi-directional` +- `task test-e2e` is a broad spec run but **excludes** the opt-in corners (its label filter + defaults to `!image-refresh && !bi-directional && !source-cluster && !ado`), because they need + what `task prepare-e2e` does not provide (Argo CD, kcp, an Azure DevOps repository): run them + with their own tasks ## Lifecycle Layers @@ -147,9 +152,9 @@ These are shared across runs until you explicitly tear down the cluster: - Flux-managed support services - Gitea - Valkey - - Prometheus stack + - Prometheus operator - Cert-manager - - Traefik + - Reflector - Gitea organization bootstrap - port-forwards - age key material under the cluster stamp directory @@ -187,20 +192,17 @@ the cluster as having one active install variant for the current run. These belong to the current e2e run, not to the controller install: -- active repo name -- checkout path -- HTTP token -- SSH keypair -- generated `secrets.yaml` -- webhook metadata -- checkout readiness marker +- repo name, clone URLs, Secret names and receiver webhook metadata, in the `RepoArtifacts` that + `SetupRepo` returns +- HTTP token and SSH keypair, written into the generated `secrets.yaml` +- the local checkout -On the current worktree, these live under: +These live under: -- `.stamps/cluster///git-/...` +- `.stamps/e2e-repo-artifacts///secrets.yaml` +- `.stamps/repos//` (the checkout) -This is intentionally conservative and keeps the Task migration closer to the earlier layout, even though these -artifacts are conceptually more run-scoped than install-scoped. +Both are outside the cluster stamp tree, so an install refresh does not touch them. ### Layer 4: Per-suite Kubernetes resources @@ -209,9 +211,9 @@ Secrets, CRDs, or Flux resources. These namespaces are typically named: -- `-test-manager` +- `-test-manager-watchrule` - `-test-signing` -- `-test-audit-consumer` +- `-test-commit-request` - and so on Those namespaces are created in `BeforeAll` and removed in `AfterAll` for the suite. @@ -229,11 +231,15 @@ has already copied into the target namespace. The central setup happens in [test/e2e/e2e_suite_test.go](../../test/e2e/e2e_suite_test.go). -`ensureE2EPrepared()` is intentionally narrow: +It is a `SynchronizedBeforeSuite`, and intentionally narrow: -1. Runs `task prepare-e2e` (cluster, controller install, webhook TLS, age key, port-forwards) -2. Sets the kubectl context -3. Ensures `E2E_AGE_KEY_FILE` points at the prepared age key +1. On parallel process #1 only: takes the per-cluster run lock, runs `task prepare-e2e` (cluster, + controller install, webhook TLS, age key, port-forwards), sets the kubectl context, removes + every `IceCreamOrder` CRD group, ensures the shared Gitea organization + (`prepareE2EClusterOnce`), waits out the namespaces of an earlier run with the same seed, and + resets the observability counters +2. In every process: points `E2E_AGE_KEY_FILE` at the prepared age key when it is unset + (`configureE2EProcess`) `BeforeSuite` does **not** bootstrap any Gitea repo. Per-file repo setup is the responsibility of each repo-using test file via `SetupRepo(...)`. @@ -245,19 +251,19 @@ The only cluster-level variable still consumed across files is: Repo state (names, checkout paths, Secret names, optional receiver webhook info) flows through a typed `RepoArtifacts` struct returned by `SetupRepo`, not through package-global env vars. -One important current detail is that the remaining audit-consumer specs share a lazy repo -helper in `audit_helpers_test.go`, so `Commit Window Batching` and `Commit Request` can -run first without hidden ordering assumptions. +The audit-consumer specs (`Commit Window Batching`, `Commit Request`) each provision their own repo +via `SetupRepo`, like every other file. ## Repo Model: One Repo Per E2E Test File -Each repo-using e2e file owns its own repo: +Each repo-using e2e file owns its own repo, for example: | File | Test namespace | Repo | |---|---|---| -| `e2e_test.go` | `testNamespaceFor("manager")` | `e2e-manager-` | +| `watchrule_configmap_secret_e2e_test.go` (one of the `manager` files) | `testNamespaceFor("manager-watchrule")` | `e2e-manager-watchrule-` | | `signing_e2e_test.go` | `testNamespaceFor("signing")` | `e2e-signing-` | -| `commit_window_batching_e2e_test.go` / `commit_request_e2e_test.go` | `testNamespaceFor("audit-consumer")` via shared helper; per-spec producer namespaces | `e2e-audit-consumer-` | +| `commit_window_batching_e2e_test.go` | `testNamespaceFor("commit-window-batching")` | `e2e-commit-window-` | +| `commit_request_e2e_test.go` | `testNamespaceFor("commit-request")`, and `commit-request-bundle` for the bundle describe | `e2e-commit-request-`, `e2e-commit-request-bundle-` | | `commit_author_attribution_e2e_test.go` | `testNamespaceFor("commit-author")` | `e2e-commit-author-` | | `quickstart_framework_e2e_test.go` | `testNamespaceFor("quickstart-framework")` | `e2e-quickstart-framework-` | | `flux_bi_directional_e2e_test.go` | `testNamespaceFor("bi-directional")` | `e2e-bi-directional-` | @@ -268,18 +274,18 @@ Each repo-using e2e file owns its own repo: Each file calls `SetupRepo(ctx, testNamespace, repoName)` once in `BeforeAll`, stores the returned `*RepoArtifacts` in a file-local variable, and applies `artifacts.SecretsYAML` to the test namespace. -Repo stamps live under: - -- `.stamps/cluster///git-/` +Repo artifacts live under: -not under the install namespace. +- `.stamps/e2e-repo-artifacts///` +- `.stamps/repos//` (the checkout) -The `audit_helpers_test.go` file additionally uses a `sync.Once`-backed -`ensureAuditConsumerRepo()` helper so either audit-consumer container can initialize the shared -repo first without cross-container ordering coupling. +not under the cluster stamp tree. ## Why CI Could Pass Even If This Was Fragile +This section is history: it explains a failure from when the repo fixtures lived under the install +namespace's stamp tree. They now live under `.stamps/e2e-repo-artifacts/`, outside it. + Two reasons can both be true at once: ### 1. The problematic cleanup was conditional on reinstall paths @@ -298,15 +304,16 @@ That is exactly why the issue surfaced around `image_refresh_test.go`: it repeat Current CI e2e matrix in .github/workflows/ci.yml: -- `quickstart-helm-and-makefile-image-refresh` -- `quickstart-manifest` -- `full` - -The `full` job currently runs: +- `full-manager` +- `full-core` +- `bi-directional` +- `source-cluster` +- `quickstart-install` +- `image-refresh` -- `task test-e2e` +The `full-manager` and `full-core` legs both run `task test-e2e`, each with its own label filter. -`task test-e2e` runs the entire e2e package (there is no longer a smoke-only subset). +`task test-e2e` runs the e2e package minus the opt-in corners (there is no smoke-only subset). So "CI ran before" does not prove the fixture boundaries were healthy. It only proves the exercised combination of: @@ -323,15 +330,19 @@ Yes. That happens in hack/cleanup-installs.sh. -Specifically, for each namespace with an install stamp it does: +Specifically, for each namespace with an install stamp it deletes the namespace, then runs: + +```bash +find "${base}/${ns}" -mindepth 1 -maxdepth 1 ! -name 'git-*' -exec rm -rf {} + +``` where: - `base=.stamps/cluster/${CTX}` - `ns` is the install namespace, usually `gitops-reverser` -The script still cleans install-scoped data under that namespace tree, but it now preserves `git-*` directories so the -shared repo fixtures survive an install refresh during the same e2e run. +The script cleans install-scoped data under that namespace tree and still preserves `git-*` directories, from when +the repo fixtures lived there. `SetupRepo` now writes them outside that tree. ## Current Stamp Layout @@ -343,7 +354,7 @@ On the current worktree, the important paths are: - `.stamps/cluster//flux.installed` - `.stamps/cluster//flux-setup.ready` - `.stamps/cluster//services.ready` -- `.stamps/cluster//gitea/bootstrap/...` +- `.stamps/cluster//age-key.txt` ### Install namespace @@ -358,11 +369,7 @@ On the current worktree, the important paths are: ### Run-scoped Gitea artifacts -- `.stamps/cluster///git-/active-repo.txt` -- `.stamps/cluster///git-/checkout-path.txt` -- `.stamps/cluster///git-/token.txt` -- `.stamps/cluster///git-/secrets.yaml` -- `.stamps/cluster///git-/checkout.ready` +- `.stamps/e2e-repo-artifacts///secrets.yaml` ### Local Git checkout @@ -370,26 +377,27 @@ On the current worktree, the important paths are: ## Suite Inventory -The e2e package currently contains: +The e2e package includes, among others: | Suite | Label(s) | Purpose | |---|---|---| -| `e2e_test.go` | `manager` | Main reconciliation and GitOps behavior | +| `controller_basics_e2e_test.go` and the other `manager` files | `manager` | Main reconciliation and GitOps behavior | | `signing_e2e_test.go` | `signing` | Commit signing behavior | -| `commit_window_batching_e2e_test.go` | `audit-consumer` | Commit-window batching through the audit pipeline | -| `commit_request_e2e_test.go` | `audit-consumer` | CommitRequest finalization through the audit pipeline | +| `commit_window_batching_e2e_test.go` | `commit-window-batching`, `audit-consumer` | Commit-window batching through the audit pipeline | +| `commit_request_e2e_test.go` | `commit-request`, `audit-consumer` | CommitRequest finalization through the audit pipeline | | `commit_author_attribution_e2e_test.go` | `manager` | OIDC claim to Git author attribution | | `image_refresh_test.go` | `image-refresh` | Build/load/restart invalidation chain | | `quickstart_framework_e2e_test.go` | `quickstart-framework` | Installer-level quickstart flow | | `flux_bi_directional_e2e_test.go` | `bi-directional`, `flux` | Flux plus gitops-reverser shared ownership | | `argocd_bi_directional_e2e_test.go` | `bi-directional`, `argocd` | Argo CD plus gitops-reverser shared ownership | +| the other bi-directional corner files | `bi-directional` and a sub-label | See [e2e-bi-directional-corner.md](e2e-bi-directional-corner.md) | | `demo_e2e_test.go` | `demo` | Demo repo preparation, intentionally leaves resources | ## Suite Definition `task test-e2e` runs the standard e2e package, excluding the rebuild-heavy `image-refresh` -workflow check. There is no smoke-only subset and no `Label("smoke")`. The `smoke` label was -removed from every spec. +workflow check and the opt-in corners (`bi-directional`, `source-cluster`, `ado`). There is no +smoke-only subset and no `Label("smoke")`. The `smoke` label was removed from every spec. It stays anchored to the standard `config-dir` install. The default run is not meant to prove all install methods on every invocation; the dedicated `quickstart-*` tasks cover @@ -470,26 +478,11 @@ The real question is where they should live: - under the install namespace stamp tree - or under a dedicated run-scoped Gitea tree -This worktree currently keeps them under the install namespace stamp tree and relies on cleanup preserving `git-*` -directories during reinstall flows. +Resolved: `SetupRepo` writes them under a dedicated tree, `.stamps/e2e-repo-artifacts///`, +with the checkout under `.stamps/repos//`, so an install refresh cannot remove them. ## Current Preference -The current preference is to keep this migration conservative and avoid changing the storage model more than needed. - -That means the following perspective is valid and intentional: - -- the generated secrets, SSH keys, and similar helper artifacts are coupled to the specific e2e run -- they are also coupled to the specific install state that exists in that cluster at that moment -- the long-lived local Git checkout is already tracked separately under `.stamps/repos/` -- because of that, it is reasonable to keep them under `.stamps/cluster///git-/` for now - -In other words: - -- the checkout itself is clearly reusable across runs -- some of the generated credentials and helper manifests are much more ephemeral -- preserving the namespace-scoped layout may be preferable in the short term if it keeps the Task - migration closer to the old behavior - -So the document should not treat a separated `gitea/runs/` layout as the only correct answer. It is one clean option, -but not the only defensible one. +Superseded by the resolution above. The earlier preference kept the generated secrets and SSH keys +under `.stamps/cluster///git-/`, coupled to the install state. They now live +under `.stamps/e2e-repo-artifacts/`, and the long-lived checkout stays under `.stamps/repos/`. diff --git a/docs/spec/gitpath-foreign-content-stringency.md b/docs/spec/gitpath-foreign-content-stringency.md index 607d1be0..00663481 100644 --- a/docs/spec/gitpath-foreign-content-stringency.md +++ b/docs/spec/gitpath-foreign-content-stringency.md @@ -22,15 +22,29 @@ > [plan_flush.go](../../internal/git/plan_flush.go), enforced at the top of `flush`. > - **Bootstrap** — a fully-commented > [.gittargetignore](../../internal/git/bootstrapped-repo-template/.gittargetignore) staged -> alongside README.md / .sops.yaml. -> - **Status reason** — `IgnoreShadowsManagedPath` ([gittarget_controller.go](../../internal/controller/gittarget_controller.go), -> wired in [event_router.go](../../internal/watch/event_router.go), kept in the stalled set in -> [stream_status.go](../../internal/controller/stream_status.go)). +> alongside README.md / .sops.yaml. A resync into a folder an apply trigger applies, which the +> branch does not hold yet, commits these files even with no object, so the applier is never +> asked to apply a missing path (`stageResyncFolder` in +> [resync_flush.go](../../internal/git/resync_flush.go)). +> - **Status reason:** `IgnoreShadowsManagedPath` ([gittarget_controller.go](../../internal/controller/gittarget_controller.go)), +> picked by `GitPathRefusalReason` in +> [acceptance_refusal.go](../../internal/manifestanalyzer/acceptance_refusal.go). Every +> `GitPathAccepted=False` is `Stalled=True`, whatever its reason (`gitTargetReadinessGates` in +> gittarget_controller.go). > - **Tests** — unit ([gittargetignore_test.go](../../internal/manifestanalyzer/gittargetignore_test.go)), > writer-integration ([gittargetignore_writer_test.go](../../internal/git/gittargetignore_writer_test.go)), > and e2e ([foreign_content_e2e_test.go](../../test/e2e/foreign_content_e2e_test.go)). Gitlink/submodule > detection and the manually-moved-file residual edge remain the noted later hardening. > +> The body below is the design as argued on 2026-06-27. §2 and §5 describe the scanner before +> this landed (the discarded non-YAML list, the skipped symlinks), so their `analyzer.go` line +> links, other than the one to `collectFiles`, no longer point at that code. As built, the code +> differs from the body in three details: the `.gitignore` parser is go-git v6's +> (`github.com/go-git/go-git/v6/plumbing/format/gitignore`); the parse-time denylist also holds +> `*.yml`, `**/*`, `**/*.yaml`, and `**/*.yml` (`isCatastrophicIgnorePattern`); and the +> benign-passenger set also accepts `LICENSE.txt`, `LICENCE.txt`, `COPYING.txt`, and +> `NOTICE.txt`. +> > Companion to > [unsupported-folder-refusal-plan.md](unsupported-folder-refusal-plan.md). That doc settled *how* a > refusal is mechanically wired and surfaced (the `GitPathAccepted` condition + kstatus trio). This doc @@ -75,7 +89,7 @@ permits. The gate ([acceptance.go](../../internal/manifestanalyzer/acceptance.go)) is strict and well-tested about **YAML it parses**. Structure-only refusals (the live + resync path via `AcceptStructureOnly`, -[acceptance.go:191](../../internal/manifestanalyzer/acceptance.go#L191)): +[acceptance.go:229](../../internal/manifestanalyzer/acceptance.go#L229)): | Refusal | IssueKind | |---|---| @@ -341,7 +355,7 @@ This is a **structural** refusal — it depends only on the bytes/entries in the discovery or followability. That places it in exactly the same safe class as the existing Tier-1 refusals, so it inherits their guarantees and their seam: -- It runs in `AcceptStructureOnly` ([acceptance.go:191](../../internal/manifestanalyzer/acceptance.go#L191)), +- It runs in `AcceptStructureOnly` ([acceptance.go:229](../../internal/manifestanalyzer/acceptance.go#L229)), so it gates **both** the resync apply and the live write path, and it **cannot false-refuse on a discovery wobble** (the property the unsupported-folder plan guards so carefully). A foreign file is a fact about the disk, not about the API. @@ -352,7 +366,7 @@ so it inherits their guarantees and their seam: status plumbing.** It reuses everything `unsupported-folder-refusal-plan.md` already built. There are two real changes, both upstream of `Accept`, both in `collectFiles` -([analyzer.go:281](../../internal/manifestanalyzer/analyzer.go#L281)) — the single walk that already +([analyzer.go:288](../../internal/manifestanalyzer/analyzer.go#L288)), the single walk that already classifies every entry: 1. **Apply the root `.gittargetignore` first (the filter).** Read **exactly** `/.gittargetignore` @@ -406,7 +420,7 @@ semantics rather than reinventing glob handling. - **Empty directories:** ignore. Git does not track them; refusing them buys nothing and annoys users. - **The operator's own bootstrap output must self-accept.** `README.md`, `.sops.yaml`, and `.gittargetignore` must be in the effective allowlist or we refuse our own writes. `.sops.yaml` already is - ([acceptance.go:149](../../internal/manifestanalyzer/acceptance.go#L149) `WriterAllowlist`); `README.md` + ([acceptance.go:191](../../internal/manifestanalyzer/acceptance.go#L191) `WriterAllowlist`); `README.md` and `.gittargetignore` must be added. Audit the full bootstrap template ([bootstrapped_repo_template.go](../../internal/git/bootstrapped_repo_template.go)) for any other non-KRM file it stages and allowlist each one explicitly. diff --git a/docs/spec/gittarget-isolation-on-rule-change.md b/docs/spec/gittarget-isolation-on-rule-change.md index 2d46814d..24bb3bd2 100644 --- a/docs/spec/gittarget-isolation-on-rule-change.md +++ b/docs/spec/gittarget-isolation-on-rule-change.md @@ -1,13 +1,19 @@ # GitTarget Isolation on Rule Changes -> **Status: ✅ Implemented and shipped.** Snapshot selection is now driven purely -> by a per-target *effective watch plan* hash (resolved GVR + scope + -> destination); the global `force` flag is gone. Validated by unit -> tests, a non-Serial e2e regression, and a `--procs=4` smoke run (21/21). See -> the Rollout Plan below for the per-step landing notes. The sections before it -> are kept as the original design rationale. Rule `operations` were later removed -> from both rule kinds ([collection rename](../design/collection-terminology-rename.md)), -> so the plan no longer carries an operation component. +> **Status: implemented, then superseded; this page is design history.** The fix +> shipped as written below: the global `force` flag went away and snapshot +> selection became a per-target hash of the effective watch plan. The mechanism +> has since been replaced. There are no shared informers and no rule-set hash: +> each GitTarget runs one watch stream per resource collection, and a rule change +> diffs that target's watch plan and starts, restarts, or stops only the +> collections that changed ([target watch plan](../design/target-watch-plan.md), +> `internal/watch/target_watch_plan.go`). The functions and tests this page names +> (`currentRuleSetSnapshots`, `snapshotTargetsNeedingDelivery`, +> `computeRequestedGVRs`, `rule_change_snapshot_test.go`, and the rest) no longer +> exist. The isolation guarantee is still pinned end to end by +> `test/e2e/gittarget_isolation_e2e_test.go` ("Manager GitTarget Isolation", +> label `manager`). Rule `operations` were later removed from both rule kinds +> ([collection rename](../design/collection-terminology-rename.md)). ## Problem diff --git a/docs/spec/gvk-gvr-mapping-layer.md b/docs/spec/gvk-gvr-mapping-layer.md index da73081e..45307b4d 100644 --- a/docs/spec/gvk-gvr-mapping-layer.md +++ b/docs/spec/gvk-gvr-mapping-layer.md @@ -1,14 +1,26 @@ # Manifest GVK/GVR Mapping Layer -> **spec** — current behaviour. The code depends on this document; change one, change the other. Index: [`../INDEX.md`](../INDEX.md) +> **spec**, kept as design history. Index: [`../INDEX.md`](../INDEX.md) > -> Status: design, captured 2026-06-04 +> Status: built 2026-06-04, then superseded by [type-followability.md](type-followability.md). The +> layer this page designs (`internal/mapping` with `ResourceMapper`, `StructureOnlyMapper`, +> `StaticSnapshotMapper` and `mapping.Status`; `watch.CatalogMapper`; the catalog's `byGVK`, +> `LookupGVK`, `LookupGVR` and `APIResourceEntry`; `RuleGVRResolver`) no longer exists. The mapper +> is now `typeset.Lookup` ([`internal/typeset/lookup.go`](../../internal/typeset/lookup.go)): +> `Ready()` plus an exact-match `ByGVK(gvk) (TypeRecord, bool)`, and a caller gates on +> `TypeRecord.Followable()` instead of a status vocabulary. The controller hands the git writer its +> `typeset.Registry`; tests use `typeset.NewSnapshotRegistry`; a Lookup that is not ready is the +> structure-only mode, which is still the `manifest-analyzer` default. The kubeconfig-backed mapper +> and the `APIMappingReady` condition were never built. What carries over is the reasoning: +> discovery is the only source of GVK/GVR truth, a mapping must match exactly, and degraded +> discovery is never evidence of absence. +> +> Captured 2026-06-04. > Related: > [current-manifest-support-review.md](current-manifest-support-review.md), > [reconcile-via-watchlist-mark-and-sweep.md](reconcile-via-watchlist-mark-and-sweep.md), -> [../kubernetes-api-resource-catalog.md](../facts/kubernetes-api-resource-catalog.md), +> [kubernetes-api-resource-catalog.md](../facts/kubernetes-api-resource-catalog.md), > [`internal/watch/api_resource_catalog.go`](../../internal/watch/api_resource_catalog.go), -> `internal/watch/rule_gvr_resolver.go`, > [`internal/manifestanalyzer/analyzer.go`](../../internal/manifestanalyzer/analyzer.go) ## Summary @@ -174,7 +186,9 @@ degraded. > repository's no-stutter lint, `MappingResult` is `mapping.Result` and > `MappingStatus` is `mapping.Status`. The `Mapping*` status constants below kept > their names. `internal/mapping` has no dependency on `internal/watch`, so the -> analyzer can resolve mappings without importing the watch manager. +> analyzer can resolve mappings without importing the watch manager. All of it was +> later replaced by `internal/typeset`, which keeps that leaf property (see the +> status at the top). The concrete Go names can move, but the interface should make the dependency explicit: diff --git a/docs/spec/manifest-system.md b/docs/spec/manifest-system.md index f5ee2318..53203e40 100644 --- a/docs/spec/manifest-system.md +++ b/docs/spec/manifest-system.md @@ -52,21 +52,34 @@ that is [`../layout/new-file-placement-rules.md`](../layout/new-file-placement-r **A GitTarget makes an all-or-nothing claim on its folder.** It either manages everything in the subtree or it refuses the folder. There is no partial ownership, because a partially-managed folder cannot be reasoned about — see -[`current-manifest-support-review.md`](current-manifest-support-review.md). +[`current-manifest-support-review.md`](current-manifest-support-review.md). The claim +covers files that are not manifests too: a file in no recognized role refuses the +folder ([`gitpath-foreign-content-stringency.md`](gitpath-foreign-content-stringency.md)). **Never partially materialize a multi-document file.** If one document in a file is out of scope, unparseable, or non-KRM, the file is not half-written. The refusal is the whole file. -**Refuse the folder rather than prune unwatched KRM.** Encountering API-backed KRM -in the folder that no WatchRule covers means the folder is refused -(`GitPathAccepted=False`), *not* that the document is deleted as drift. Deleting a +**Never prune unwatched KRM.** API-backed KRM in the folder that no WatchRule +covers is *not* deleted as drift: a resync sweeps one resource collection at a +time, so a document of a type nobody watches stays as Git holds it. Deleting a user's manifest because we were not told to watch its type is the worst thing this -system could do. +system could do. The operator does not refuse the folder for it either, because +whether a type is watched is a discovery fact that can blink; its gate +(`AcceptStructureOnly`) refuses on structure alone. The dry-run scan, which has an +API source (`Accept`), does report a type it cannot follow, or a namespace outside +the GitTarget's scope, as a refusal. + +**An object another applier owns stays out of a folder an applier applies.** When a +GitTarget names an apply trigger, an object another applier owns (Flux's owner labels, or +Argo CD's tracking annotation) is not mirrored into its folder, and the sweep +leaves its document alone: two appliers applying one object fight each other. A +folder with no trigger mirrors such objects as before. **At most one editable document per resource identity.** Two documents claiming the -same identity is not a prune opportunity, it is an invalid repository state: it -blocks startup and takes a live target down (`RepositoryValid=False`). The older +same identity is not a prune opportunity, it is an invalid repository state: the +GitTarget refuses the folder (`GitPathAccepted=False`) and writes nothing to it +until a human removes one copy. The older "first-occurrence-wins, delete the losers" idea was explicitly abandoned. **The API wins — full-object ownership, never field-subset ownership.** When the @@ -87,7 +100,9 @@ mix ciphertext with plaintext — [`sops-single-file-no-multidoc.md`](sops-singl **No bookmark, no sweep.** Initial reconcile is a streaming list-watch with mark-and-sweep. The sweep is a set-difference taken at the joined `initial-events-end` bookmark, and a partial mark must never drive a sweep — a -missing bookmark fails closed. See +missing bookmark fails closed. The sweep removes a document only under +`spec.prune.mode: Always`; the default, `OnEvent`, mirrors an observed delete and +infers none. See [`reconcile-via-watchlist-mark-and-sweep.md`](reconcile-via-watchlist-mark-and-sweep.md). **One grouped commit = exactly one (author, GitTarget) tuple** — @@ -139,6 +154,10 @@ So when something changes Git *out of band* — an external push, a human deleti files on the branch — **nothing re-runs the path's reconcile to bring Git back in line.** The drift is detected but not acted on. +A folder an apply trigger applies is the exception, in the other direction: when +the branch worker learns of a commit it did not push, it asks the applier to apply +it (a `ForeignCommit`), so the cluster follows Git. + The recommended fix, not yet built, is a per-GitTarget **subtree OID** compare that arms exactly one replay-mark session when the subtree moves underneath us. This is the one substantive open item inherited from the old `manifest/` folder and is diff --git a/docs/spec/manifestedit-field-ownership-spike.md b/docs/spec/manifestedit-field-ownership-spike.md index e627aeec..cce8d5fc 100644 --- a/docs/spec/manifestedit-field-ownership-spike.md +++ b/docs/spec/manifestedit-field-ownership-spike.md @@ -3,9 +3,8 @@ > **spec** — current behaviour. The code depends on this document; change one, change the other. Index: [`../INDEX.md`](../INDEX.md) > > Status: decided -> Related: [manifestedit-abstraction-plan.md](manifest-system.md) -> (step 5), [manifest-inventory-file-agnostic-placement.md](manifest-system.md), -> [bi-directional.md](../bi-directional.md), +> Related: [manifest-system.md](manifest-system.md), which replaced the abstraction plan and the +> file-agnostic-placement plan this page answered, [bi-directional.md](../bi-directional.md), > POC decision record: `internal/git/manifestedit/DECISION.md` The abstraction plan flagged field ownership — "does the reverser own the whole @@ -38,7 +37,7 @@ now it genuinely *is* part of the API object. So the loop closes: - Edit a field in the cluster → the reverser writes it to Git. -- Edit a field in Git → the bi-directional reconciler applies it to the cluster → +- Edit a field in Git → the applier (Flux or Argo CD) applies it to the cluster → the reverser keeps it in Git. - A field that is in Git but in neither the cluster nor a pending Git-to-cluster apply is, by definition, orphaned state. Deleting it is correct. @@ -78,7 +77,9 @@ they fight the API-first model: - **Server-Side-Apply `managedFields`-driven ownership.** Principled in theory, but `fieldsV1` set-encoding, shared ownership across managers, and apply-vs-update semantics make a faithful predicate expensive to build and brittle to maintain — - for a guarantee bi-directional GitOps already provides more simply. No. + for a guarantee bi-directional GitOps already provides without any predicate. No. (The + branch worker does read an object's newest `managedFields` entries, but only to + recognize the applier's own write, an echo; never to decide which fields Git keeps.) If a real, concrete need for partial ownership ever appears, it must come back through its own proposal with a worked use case that bi-directional GitOps cannot diff --git a/docs/spec/mutation-capture-lab-design.md b/docs/spec/mutation-capture-lab-design.md index 72d6a6f2..05d92bcf 100644 --- a/docs/spec/mutation-capture-lab-design.md +++ b/docs/spec/mutation-capture-lab-design.md @@ -8,9 +8,10 @@ webhooks** — at every interesting moment — and commits those structures as a **corpus**. It is not a second implementation of GitOps Reverser; it is reference material and a regression harness. -The corpus is captured against **k8s v1.36.1+k3s1** (recorded in -`test/mutationlab/corpus/CLUSTER.md`). **All seventeen** catalogued scenarios are captured today — see -the [Difficult Cases Catalog](#difficult-cases-catalog). +The corpus is captured against **k8s v1.36.4+k3s1** (recorded in +`test/mutationlab/corpus/CLUSTER.md`). **All seventeen** catalogued scenarios are captured today (see +the [Difficult Cases Catalog](#difficult-cases-catalog)), plus the rows that extend it, which +`test/mutationlab/README.md` indexes. The lab reuses the product's webhook URLs (`/validate-all`, `/audit-webhook`) on the same ports and TLS mounts, so capturing is just an **image swap** on an already-prepared e2e cluster — no new audit @@ -102,7 +103,7 @@ visible rather than silent. ## Findings -These are conclusions drawn from the captured corpus on v1.36.1. They are versioned facts: the regen- +These are conclusions drawn from the captured corpus on v1.36.4. They are versioned facts: the regen- and-diff workflow re-checks them on a Kubernetes bump. ### 1. The watch carries the full object exactly where the audit body goes shallow @@ -111,8 +112,8 @@ This is the headline product result. Two captured data points sit side by side i - **Deletecollection (Row 9).** One user request: the audit *request* body is only `DeleteOptions` and name-less, yet each of the N per-object watch `DELETED` events carries the full removed object. (On - v1.36.1 the audit *response* body additionally carries a `List` of the removed objects — so the - shallow-body concern is real for the request and narrower for the response — but a name-less request + v1.36.4 the audit *response* body additionally carries a `List` of the removed objects, so the + shallow-body concern is real for the request and narrower for the response, but a name-less request still cannot be relied on for per-object identity across versions or audit levels.) - **Aggregated API (Row 15).** The official kube-apiserver audit event carries **no `requestObject` and no `responseObject`** (the apiserver proxies the aggregated request and has no schema to render @@ -180,7 +181,7 @@ are not equal — there is no shared native join key. The admission recorder's This is the dominant GitOps write path (Flux/Argo write by apply), so its shape across the mechanisms is load-bearing for watch mode. -- **Server-side apply (Row 3).** An apply that *changes* the object is, on v1.36.1, audited as +- **Server-side apply (Row 3).** An apply that *changes* the object is, on v1.36.4, audited as `verb: patch` (the apply rides a `?fieldManager=…&force=true` PATCH, not an `apply` verb), reaches admission as `UPDATE` carrying the apply field manager in `options`, and surfaces on the watch as a single `MODIFIED` whose `managedFields` shows one `manager: …, operation: Apply` entry. So for the @@ -228,7 +229,7 @@ with `automountServiceAccountToken: false` so no random volume name churns the c | 12 | Rejected during validation | ✅ | **no** watch event | failed response (`code` 4xx) | recorder is **always called** (parallel validation) and record-and-rejects | admission saw a write that never persisted | | 13 | Optimistic-concurrency conflict | ✅ | no watch event, no final change | failed response, `code: 409`, `Status` body | **none** — rejected at storage before admission | failure carries a Status, not the object | | 14 | Multi-version CRD conversion | ✅ | `ADDED`/`MODIFIED` in the v2 storage version (`size` string) | bodies in the submitted v1 (`sizeBytes`); +`conversion` source both ways | submitted v1 (the `Equivalent`+`*` webhook delivers it) | three different shapes for one write | -| 15 | Aggregated API write | ✅ | **full object** (`ADDED`, spec included) | official: **empty** request/response body; proxy-enriched: full body | validating webhook **observed** firing on this version | the body-quality cliff — and watch fills it | +| 15 | Aggregated API write | ✅ | **full object** (`ADDED`, spec included) | **empty** request/response body | validating webhook **observed** firing on this version | the body-quality cliff, and watch fills it | | 16 | Watch resync (`410 Gone`) | ✅ | `ERROR` (`Status` 410); driver verifies relist recovery | n/a | n/a | proves watch needs a list backstop | | 17 | Bookmark | ✅ | `BOOKMARK` with resourceVersion | n/a | n/a | the safe resume anchor | @@ -343,6 +344,10 @@ test/mutationlab/corpus/ aggregated-api-write/ # watch.added (full) · audit.create (empty) ``` +The rows that extend the catalog (8a, 15a, 15b, 18, 19 in `test/mutationlab/README.md`) add their +own directories beside these, such as `configmap/selector-membership/` and +`flunder/aggregated-api-delete/`. + `CLUSTER.md` is load-bearing: a captured shape is only meaningful against a known apiserver version. ### Normalization diff --git a/docs/spec/reconcile-via-watchlist-mark-and-sweep.md b/docs/spec/reconcile-via-watchlist-mark-and-sweep.md index bb52a75c..3ada850f 100644 --- a/docs/spec/reconcile-via-watchlist-mark-and-sweep.md +++ b/docs/spec/reconcile-via-watchlist-mark-and-sweep.md @@ -2,7 +2,12 @@ > **spec** — current behaviour. The code depends on this document; change one, change the other. Index: [`../INDEX.md`](../INDEX.md) > -> Status: design direction, captured 2026-06-04. +> Status: implemented. Captured 2026-06-04 as the design direction, and the sections up to +> "Source Code Concepts That Could Be Thrown Away" keep that voice: the code they name for removal +> (`FolderReconciler`, the path-derived identity, the two-snapshot handshake, the per-event locator, +> `Manager.GetClusterStateForGitDest`) is gone, and the resync is `planResyncToWorktree` in +> [`internal/git/resync_flush.go`](../../internal/git/resync_flush.go). The sections from "Decided +> once, replayed as decided" to "Stale snapshots are discarded" describe current behavior. > Related: > [current-manifest-support-review.md](current-manifest-support-review.md), > [gvk-gvr-mapping-layer.md](gvk-gvr-mapping-layer.md), @@ -267,6 +272,103 @@ Do **not** delete these — their decision logic moves into Plan computation / A - Acceptance failures (duplicate / non-KRM / unwatched API-backed KRM / mixed managed-allowlisted file) short-circuit before any stream is opened: refuse, reconcile nothing. +- The plan is evaluated on that pair only. When the push is rejected because the + remote moved, a newer tip receives the plan's decided actions, never a new sweep + (see [Decided once, replayed as decided](#decided-once-replayed-as-decided)). + +## Decided once, replayed as decided + +A resync uses its snapshot's full membership once, when its pending write is +first committed. The plan above decides two things on that tree: the upserts that +changed it, and the documents the sweep removed (by identity, under the target's +`prune.mode`). The write then carries only those actions. An object whose upsert +changed nothing is no action, and a resync that decided nothing settles at once +and is never pushed. A resync into a folder an apply trigger applies that the +branch does not hold yet also decides the folder's bootstrap files, with or +without objects, so the applier is never asked to apply a missing path. + +A replay onto a newer tip, after a rejected push, applies exactly those actions: + +- each upsert is planned onto the new tree; +- each delete removes its document if the new tree still holds it, and only while + the target's prune mode, read again before the replay, still sweeps, so a policy + tightened since is obeyed; +- nothing else: the sweep never runs again. + +So a document the newer tip added, which the snapshot could not know about, +survives the replay, under `prune.mode: Always` too. Before this rule a replayed +resync folded its whole membership over the new tree and swept that addition as +an orphan. The write-side mechanics are in +[the pending-writes design](../design/gittarget-branch-worker-pending-writes.md#a-write-carries-the-actions-decided-on-its-tree). + +## Objects another applier owns + +In a folder an apply trigger applies (a `GitTarget` with `spec.applyTriggerRef`), +an object another applier owns is outside the resync. Mirrored into the folder, +it would be applied by this folder's applier too, and two appliers applying one +object fight each other. + +- The watch side reads the owner from the raw object, before sanitize strips it: + Flux's owner labels (`kustomize.toolkit.fluxcd.io/name` and `/namespace`, naming + a `Kustomization`), else Argo CD's tracking annotation + (`argocd.argoproj.io/tracking-id`, naming an `Application`). It stamps the owner + on every snapshot entry and live event. +- A snapshot entry owned by an applier other than the one the folder's trigger + holds is neither upserted nor treated as missing, so the sweep leaves its + document alone, under `prune.mode: Always` too. A live event for it is not + mirrored. +- An object nobody owns, or the folder's own applier owns, is a member as before. + A folder with no trigger excludes nothing. +- Each exclusion is counted in `apply_ownership_conflicts_total` and logged once + per object; see + [Interpreting metrics](../interpreting-metrics.md#apply-requests-and-the-push-receiver). + +Design: [applier echoes, F4](../design/applier-echoes.md#f4-an-object-another-applier-owns-is-not-mirrored-into-a-triggered-folder). + +## Stale snapshots are discarded + +On a branch an applier applies, a snapshot can hold a stale picture. While Git is +ahead of the cluster (a commit pushed and not yet applied), a snapshot holds the +old state of every object that commit changes, and one collected across an apply +holds a mix. Deciding either would revert the push in Git and then on the cluster. + +The branch worker keeps an **apply epoch**, raised at every send and every end of +an apply attempt, and knows whether **Git is ahead**: an applier on the branch has +a `ForeignCommit`, `Repair` or `Refusal` pending, a converge it may be asked for, +or an attempt out. An adopt does not count, because it applies objects the +cluster already runs. The watch side reads both when a snapshot's collection +starts (before the initial-events watch opens, or before the LIST fallback), +and the resync request carries them. + +The worker discards a snapshot collected while Git was ahead, collected across an +epoch change, or arriving while Git is ahead, and one whose pre-resync fetch finds +a commit the worker did not push. A discarded snapshot is answered as discarded, +which is not a failure and is not counted in `git_resync_failures_total`. Once Git +is no longer ahead, the worker asks the watch plane to re-anchor that +`GitTarget`'s streams, which collect a fresh snapshot after the apply. Until it +lands, the `GitTarget`'s render-fidelity scope stays pending: its live writes are +held back by the render-fidelity gate, and the fresh snapshot recovers them. No +snapshot is ever parked. + +Three rules keep this from holding a resync forever: + +- A branch with no trigger never waits: nothing applies its commits. +- A collection's initial snapshot is exempt: the first snapshot of each collection + a `GitTarget` watches, until one of them settles (applied, refused or failed; a + discarded or superseded one does not settle). The watch side tracks this per + collection. Initial snapshots are the startup baseline, and no applier is asked + for anything before they are in Git, so holding them for an apply would wait on + itself. +- After a `GitTarget`'s snapshots have been held for 5 minutes, counted from its + first discard, a fresh snapshot is asked for at that moment and used anyway, + logged and counted. An apply that keeps + failing must not leave drift unmirrored for as long as it fails; whatever the + unapplied commit changed in those objects is then reverted, by the API-wins rule. + +Only the worker's own appliers count: other `GitProvider`s writing the same +repository and branch are not consulted. Both outcomes are counted in +`stale_snapshots_total` (`discarded`, and `used` for the 5-minute bound). Design: +[applier echoes, F3](../design/applier-echoes.md#f3-a-resync-never-decides-from-a-snapshot-collected-while-git-was-ahead-or-across-an-apply). ## Lazy Materialization Synergy diff --git a/docs/spec/scale-subresource-audit-rehydration.md b/docs/spec/scale-subresource-audit-rehydration.md index 2a6d58f3..45e862fc 100644 --- a/docs/spec/scale-subresource-audit-rehydration.md +++ b/docs/spec/scale-subresource-audit-rehydration.md @@ -1,17 +1,22 @@ # Subresource Audit Resolution -> **spec** — current behaviour. The code depends on this document; change one, change the other. Index: [`../INDEX.md`](../INDEX.md) +> **History**, kept in `spec/` because [`internal/git/types.go`](../../internal/git/types.go) cites +> it for `FieldPatch`. Index: [`../INDEX.md`](../INDEX.md) > -> Status: design proposal, captured 2026-06-08, revised 2026-06-08. -> -> Trigger: `kubectl scale deployment` updates the live Deployment through the -> `deployments/scale` subresource, but the committed Deployment manifest is not -> updated today. -> -> Scope update: see -> [subresource-scope-reduction.md](scale-subresource-audit-rehydration.md). Future work -> should narrow this from generic subresource translation to built-in and CRD -> `/scale` only. +> Status: superseded. This page was the design (captured 2026-06-08) for mirroring +> `kubectl scale` from the `/scale` audit event as a field patch, and records the scale-only +> reduction that followed. Neither is how it works now. Since watch-first ingestion, the audit +> pipeline produces attribution facts only, and object state comes from the watch: a +> `kubectl scale` reaches Git as the parent's `MODIFIED` watch event, carrying the new +> `spec.replicas`, and is mirrored like any other edit. The audit handler keeps a mutating `/scale` +> event only to name who scaled the parent, and drops every other subresource before recording +> (`shouldForwardSubresource` in +> [`audit_handler.go`](../../internal/webhook/audit_handler.go)). The translator +> (`translateScaleToAssignments`), `IsScaleSubresource`, `auditutil.BuiltinScaleReplicasPath`, the +> Redis stream, the consumer and its outcomes below are gone. The writer half remains: +> `git.FieldPatch` and `manifestedit.PatchFields` still apply a bounded patch to an existing parent +> document, but nothing in production produces a field patch today. The scope-reduction page the +> body links is no longer kept; those links point at this page. ## Problem diff --git a/docs/spec/sops-single-file-no-multidoc.md b/docs/spec/sops-single-file-no-multidoc.md index ab9299bc..c725fd61 100644 --- a/docs/spec/sops-single-file-no-multidoc.md +++ b/docs/spec/sops-single-file-no-multidoc.md @@ -5,10 +5,10 @@ > Status: decided > Captured: 2026-06-08 > Related: -> [file-agnostic-placement.md](../layout/new-file-placement-rules.md), +> [../layout/new-file-placement-rules.md](../layout/new-file-placement-rules.md), > [../layout/contextual-namespace.md](../layout/contextual-namespace.md), -> [../sops-repo-bootstrap-and-key-management-architecture.md](../finished/sops-repo-bootstrap-and-key-management-architecture.md), -> [../sops-repo-bootstrap-out-of-scope.md](../finished/sops-repo-bootstrap-out-of-scope.md) +> [../finished/sops-repo-bootstrap-and-key-management-architecture.md](../finished/sops-repo-bootstrap-and-key-management-architecture.md), +> [../finished/sops-repo-bootstrap-out-of-scope.md](../finished/sops-repo-bootstrap-out-of-scope.md) ## Decision @@ -17,7 +17,7 @@ SOPS file**. We do **not** write SOPS-encrypted multi-document YAML (no `\n---\n`-separated documents inside an encrypted file). Plaintext manifests may still be multi-document where that is convenient (see -[file-agnostic-placement.md](../layout/new-file-placement-rules.md)); this decision is +[new-file-placement-rules.md](../layout/new-file-placement-rules.md)); this decision is scoped to files we encrypt with SOPS. ## Why this came up @@ -146,7 +146,11 @@ and full compatibility with stock SOPS and Flux. it never appends an encrypted resource as an extra `---` document. - Placement logic for encrypted resources must therefore not co-locate multiple resources in a single encrypted file, even where the plaintext placement rules - would allow a multi-doc file. + would allow a multi-doc file. Placement refuses to append a sensitive resource + to a file that already holds a document (`sensitive_append`), and refuses to + append a plaintext resource to a file that holds an encrypted one + (`plaintext_onto_encrypted`), in + [`placement.go`](../../internal/manifestanalyzer/placement.go). - Re-encrypting a single-resource file is acceptable and expected when that resource's content changes; it is not the thing we are avoiding. What we avoid is one resource's change forcing a re-encrypt that touches *other* resources. diff --git a/docs/spec/status-conditions-guide.md b/docs/spec/status-conditions-guide.md index c85da1e1..e7afab37 100644 --- a/docs/spec/status-conditions-guide.md +++ b/docs/spec/status-conditions-guide.md @@ -10,7 +10,7 @@ Controllers write it, users don't. It's a separate subresource (`/status`) — ` persist changes to it, and RBAC for it is granted separately from the main resource: ```yaml -- resources: ["gitdestinations/status"] +- resources: ["gittargets/status"] verbs: ["get", "patch", "update"] ``` @@ -30,9 +30,10 @@ A field that fails this is not merely expensive. It is wrong three ways, and onl 1. Every status write is an etcd write and a `resourceVersion` bump, which invalidates the cached copy held by every watcher of that type, not just ours. -2. It is a **feedback loop**. A status write fires an Update event that the controller's own `For()` - turns back into a queued request, which is the self-triggering edge the section below exists to - suppress. A field that moves on every pass defeats that suppression by construction. +2. It is a **feedback loop**. A status write fires an Update event that every controller watching + the object turns back into a queued request (a WatchRule watches its GitTarget's status, for + one), which is the edge the section below exists to suppress. A field that moves on every pass + defeats that suppression by construction. 3. It destroys the field's own readability. A value that changes constantly cannot be read as a statement about convergence, because there is no steady state to compare against. @@ -74,15 +75,15 @@ in the same shape: Distributing a fact is not re-proving it, and it costs nothing. - **Publication** is the status write, and it is sampled: each target writes what it holds WHEN IT RECONCILES, on its own cadence, with a floor of one write per minute measured from the last - write. Nothing in the data plane wakes a target because a branch moved. The one exception is a - reconcile that has just ASKED the remote something, which comes back for the answer rather than - leaving it undelivered for a tick. + write. Nothing in the data plane wakes a target because a branch moved. The exceptions are a + reconcile that has ASKED the remote something, and one holding an answer the floor has not + let out yet: each comes back for the answer rather than leaving it undelivered for a tick. - **The tuple stays internally consistent**: one observation from one moment, never the newest - revision beside a stale clock. That is what makes a sampled value readable, and it is why - `revision` and `lastVerifiedAt` are published together or not at all. + commit beside a stale clock. That is what makes a sampled value readable, and it is why + `commit` and `lastVerifiedAt` are published together or not at all. Two corrections skip the floor, because they are not throughput: the first observation, and taking -back a revision that names a repository the target no longer points at. +back a commit proved against a repository the target no longer points at. The promise that follows is worth stating wherever such a field is documented: it can lag the latest observation, and it says nothing at all about the subject when nothing has observed it. @@ -121,26 +122,28 @@ its timeout on an object that was never going to converge. `reconcileStatus.commit` computes the difference between the status as read and the status as written, and sends **nothing** when they are equal. A status write bumps `resourceVersion`, which -fires an Update watch event, which the controller's own `For()` turns straight back into a queued -request — so an unconditional write makes every reconcile cost roughly two. Status fields that move +fires an Update watch event, which every controller watching the object turns back into a queued +request, so an unconditional write makes every reconcile cost more reconciles. Status fields that move on every pass (a "last reconcile attempt" timestamp) defeat this by construction and were removed; `lastTransitionTime` plus `controller_runtime_reconcile_total` answer the same question without making every object mutable on read. -The write is a status **patch with optimistic concurrency**, and a conflict is dropped rather than -retried: a conflict means the object moved under this reconcile, so the status just computed -describes a generation that is no longer current, and publishing it would publish a stale -observation. Convergence is deferred, not guaranteed to be immediate. A conflict caused by a *spec* -edit re-enqueues at once; one caused by a metadata- or status-only write does **not**, because every -`For()` in this project carries `GenerationChangedPredicate`. That case converges on the next -dependency event or the requeue cadence `commitRule`/`requeueFor` already picked from the same -verdict — which is why that cadence, and not a retry loop here, is the backstop. +The write is a status **patch with optimistic concurrency**, and a conflict drops the write rather +than retrying it: a conflict means the object moved under this reconcile, so the status it +computed describes a `resourceVersion` that is no longer current, and publishing it would publish a +stale observation. The conflict is cache lag (one status writer per kind, and controller-runtime +never reconciles one key twice at once), and nothing else re-enqueues the object, because the +`For()` of every controller using `reconcileStatus` filters status-only updates +(`GenerationChangedPredicate`, or a variant of it that also admits a reconcile request). So the loss is +recorded, and the reconcile comes back after `RequeueWriteLostInterval` (100ms) instead of on its +normal cadence, and recomputes from fresh data. `CommitRequest` does not use this helper: its +`For()` has no such filter, and it retries its terminal status write in place. ### Reason vocabulary Generic reasons are aliases of [`github.com/fluxcd/pkg/apis/meta`](https://pkg.go.dev/github.com/fluxcd/pkg/apis/meta) -— `Succeeded`, `Failed`, `Progressing`, `DependencyNotReady` — a module this project already depends -on. Sharing the vocabulary means one alerting rule works across every kind here *and* across every +(`Succeeded`, `Failed`, `Progressing`, `Suspended`), a module this project already depends on. +Sharing the vocabulary means one alerting rule works across every kind here *and* across every Flux kind in the same cluster. A reason that restates the condition type (`Ready=True, reason=Ready`) answers nothing and is not used. @@ -209,12 +212,26 @@ GitTarget adds domain conditions that explain the summary: ```go const ( - TypeValidated = "Validated" - TypeEncryptionConfigured = "EncryptionConfigured" - TypeStreamsRunning = "StreamsRunning" - TypeGitPathAccepted = "GitPathAccepted" - TypeGitTargetReady = "GitTargetReady" - TypeLayoutResolved = "LayoutResolved" + TypeValidated = "Validated" + TypeEncryptionConfigured = "EncryptionConfigured" + TypeSourceClusterReachable = "SourceClusterReachable" + TypeGitProviderReady = "GitProviderReady" + TypeClusterProviderReady = "ClusterProviderReady" + TypeStreamsRunning = "StreamsRunning" + TypeGitPathAccepted = "GitPathAccepted" + TypeRenderMatchesLive = "RenderMatchesLive" + TypeLayoutResolved = "LayoutResolved" + TypeApplyTriggerReady = "ApplyTriggerReady" // only when spec names an apply trigger +) +``` + +WatchRule and ClusterWatchRule add their own: + +```go +const ( + TypeResourcesResolved = "ResourcesResolved" + TypeStreamsRunning = "StreamsRunning" + TypeGitTargetReady = "GitTargetReady" // WatchRule only: whether every rule item's RESOLVED source-namespace scope is authorized. TypeSourceNamespaceAuthorized = "SourceNamespaceAuthorized" @@ -319,7 +336,7 @@ trio; the bounded "did the work actually happen" signal lives on the `Pushed` do ```go const ( TypeReady = "Ready" // True at a non-error terminal outcome - TypeReconciling = "Reconciling" // True while the close delay/finalize is in progress + TypeReconciling = "Reconciling" // True until the request reaches a terminal outcome TypeStalled = "Stalled" // True when the finalize failed (kstatus Failed) TypeAuthorAttributed = "AuthorAttributed" // Whether admission captured the command submitter TypePushed = "Pushed" // True once the commit is in the remote repository @@ -337,7 +354,10 @@ Canonical reads: `Pushed=Unknown` → kstatus InProgress - committed: `Ready=True`, `Pushed=True`, `Stalled=False`, reason `Committed` → kstatus Current - benign no-commit (nothing to save / already present / foreign open window): `Ready=True`, `Pushed=False`, - `Stalled=False`, with the specific reason on `Ready` → kstatus Current (a correct, non-error outcome) + `Stalled=False`, with the specific reason on `Ready` (`NoWindow`, `AlreadyPresent`, `WindowMismatch`) + → kstatus Current (a correct, non-error outcome). With `whenNothingToCommit: CommitEmpty`, nothing to + save or already present records an empty commit instead: `Pushed=True`, the reason still naming the + cause - failed finalize: `Ready=False`, `Stalled=True`, reason `FinalizeFailed` → kstatus Failed `AuthorAttributed=True` (`AttributedFromAdmission`) means the command submitter was captured. `False` diff --git a/docs/spec/type-followability.md b/docs/spec/type-followability.md index 8a7c3a64..9e24a9fa 100644 --- a/docs/spec/type-followability.md +++ b/docs/spec/type-followability.md @@ -2,11 +2,12 @@ > **spec** — current behaviour. The code depends on this document; change one, change the other. Index: [`../INDEX.md`](../INDEX.md) > -> Status: greenfield proposal, captured 2026-06-08 +> Status: implemented in [`internal/typeset`](../../internal/typeset/model.go); captured +> 2026-06-08 as a greenfield proposal. Where the code differs from the proposal, this page +> says so in place, and the code wins. > -> Supersedes the layered model in -> [api-catalog-watched-type-architecture.md](type-followability.md). -> That doc grounds in today's packages and ends up with seven layers and two +> Supersedes the layered model of `api-catalog-watched-type-architecture.md` (since +> deleted), which grounded in the packages of the time and ended up with seven layers and two > parallel reports (followability requirements + a health report). This one starts > from the question we actually want answered and works backward. @@ -26,9 +27,9 @@ This matters because the layered model invented condition names that do not map a real decision. A `Watch` condition that is `False` does not tell an operator anything actionable; the type is simply **not followable because it is missing a required verb**. Naming the check after the failure ("Watch: False") is less clear -than naming it after the requirement and letting the reason speak ("not followable -— missing required verb: patch"). So the model has one list of named requirement -checks, and the failing check *is* the explanation. +than naming it after the requirement and letting the reason speak +(`not followable — missing required verb: watch`). So the model has one list of named +requirement checks, and the failing check *is* the explanation. ## Fewer layers @@ -52,16 +53,19 @@ cluster ─ scan ─▶ Observation raw per-type facts: discovery + CR ``` - **Scan** gathers raw facts and produces an `Observation` per served type. It - joins discovery with `CustomResourceDefinition` and `APIService` evidence and the - built-in scale registry. It makes no product decisions. + joins discovery with the built-in scale registry and the operator's resource + policies. It makes no product decisions. The proposal also joined + `CustomResourceDefinition` and `APIService` evidence; that is not built, so origin + is inferred from the API group (see `origin` below). - **`TypeRegistry`** is the only place that turns observations + policy into a followability verdict. It owns identity rules, the deny/sensitive policy, the live set, and the 60-second removal grace. Every consumer reads from it. - **`TargetView`** is a GitTarget's projection of *already-followable* records into - namespace scopes. It never recomputes followability. + namespace scopes. It never recomputes followability. As built it is the + `WatchedTypeTable` in `internal/watch`. -The mapper is not a layer — it is `TypeRegistry.ByGVK`. WatchRule expansion is not a -layer — it is a function that builds a `TargetView` by asking the registry. +The mapper is not a layer: it is `TypeRegistry.ByGVK` (as built, `typeset.Lookup`). +WatchRule expansion is not a layer: it is a function that builds a `TargetView` by asking the registry. ## Identity is 1:1, in both directions @@ -111,12 +115,22 @@ always has the same machine-readable answer. | `stable` | the type is not mid-disappearance, or is inside the removal grace | `absence-expired` | | `identity` | GVK ↔ GVR is 1:1 in both directions | `gvk-not-unique`, `gvr-not-unique` | | `scope` | the type is known namespaced or cluster-scoped | `scope-unknown` | -| `verbs` | discovery advertises `get`, `list`, `watch`, `patch` (detail names the missing verb) | `missing-verb` | -| `origin` | classified `builtin`, `crd`, or `aggregated` with evidence | `origin-unknown` | +| `verbs` | discovery advertises `get`, `list`, `watch` (detail names the missing verb) | `missing-verb` | +| `origin` | classified `builtin`, `crd`, or `aggregated` | `origin-unknown` | | `policy` | product policy permits mirroring this type | `denied-by-policy` | | `sensitivity` | not sensitive, or sensitivity has supported encryption/write handling | `sensitive-unsupported` | | `scale` | scale is unused, or its parent replica path is known (not guessed) | `scale-path-unresolved` | +As built, three checks are narrower than the proposal: + +- `verbs` does not require `patch`. Mirroring is a read path, so a type without + `patch` is still mirrorable. +- `origin` is a shape heuristic on the API group: the core group, `*.k8s.io`, and the + legacy built-in groups are `builtin`, everything else is `crd`, both with confidence + `inferred`. It never fails for a served type, and nothing is classified `aggregated`. +- `scale` binds a parent replica path only for built-ins. A CRD that serves `/scale` + has no path, so it fails with `scale-path-unresolved` and is refused. + Verdict derivation is mechanical: - all required checks `pass` → `followable`; @@ -155,14 +169,14 @@ Refused for a missing verb — the case the old `Watch` condition described badl ```yaml gvk: metrics.k8s.io/v1beta1 PodMetrics gvr: metrics.k8s.io/v1beta1 pods -origin: { kind: aggregated, confidence: observed, evidence: v1beta1.metrics.k8s.io } +origin: { kind: builtin, confidence: inferred } verdict: refused -summary: not followable — missing required verb: watch, patch +summary: "not followable — missing required verb: watch" checks: - { requirement: served, result: pass } - { requirement: trusted, result: pass } - { requirement: identity, result: pass } - - { requirement: verbs, result: fail, reason: missing-verb, detail: "watch, patch" } + - { requirement: verbs, result: fail, reason: missing-verb, detail: "watch" } ``` Refused for ambiguous identity: @@ -170,7 +184,7 @@ Refused for ambiguous identity: ```yaml gvk: example.com/v1 Widget verdict: refused -summary: not followable — GVK served by two GVRs +summary: "not followable — GVK served by more than one resource: widgets, widgetz" checks: - { requirement: identity, result: fail, reason: gvk-not-unique, detail: "widgets, widgetz" } ``` @@ -210,7 +224,7 @@ type TypeRecord struct { type Followability struct { Verdict Verdict // followable | retained | refused | unknown - Summary string // one line, e.g. "not followable — missing required verb: patch" + Summary string // one line, e.g. "not followable — missing required verb: watch" Checks []Check // every requirement, funnel order } @@ -233,24 +247,34 @@ record — including the verdict and every check — so a caller never needs a s "inspect" call to render the reason. ```go -type TypeRegistry interface { - Ready() bool - Generation() uint64 - - // Always return the full record. The bool reports whether the type is known to - // the registry at all; callers gate behavior on record.Followable(). - ByGVK(ctx context.Context, gvk schema.GroupVersionKind) (TypeRecord, bool, error) - ByGVR(ctx context.Context, gvr schema.GroupVersionResource) (TypeRecord, bool, error) - - // Followable returns only verdict in {followable, retained}. All returns every - // known type for inventory and "why not" views. - Followable(ctx context.Context) ([]TypeRecord, error) - All(ctx context.Context) ([]TypeRecord, error) -} +// As built: the concrete *typeset.Registry, one per source cluster. +func (r *Registry) Ready() bool +func (r *Registry) Generation() uint64 +// Revision bumps when the followable set changes (a grace drop included) or the +// scan generation moves: the cache key for per-GitTarget projections. +func (r *Registry) Revision() uint64 + +// Always return the full record. The bool reports whether the type is known to +// the registry at all; callers gate behavior on record.Followable(). +func (r *Registry) ByGVK(gvk schema.GroupVersionKind) (TypeRecord, bool) +func (r *Registry) ByGVR(gvr schema.GroupVersionResource) (TypeRecord, bool) + +// Followable returns only verdict in {followable, retained}. All returns every +// known type for inventory and "why not" views. +func (r *Registry) Followable() []TypeRecord +func (r *Registry) All() []TypeRecord ``` -Per-GitTarget projection consumes records; it does not recompute them. A rule that -matches nothing followable returns the refused record so the operator sees why. +Consumers outside `internal/watch` read the smaller `Lookup` interface: `Ready()` and +`ByGVK`. + +Per-GitTarget projection consumes records; it does not recompute them. The proposal +below is not built in this shape: the projection is `WatchedTypeTable` +(`internal/watch/watched_type_table.go`), whose `WatchedType` copies GVK, GVR, scope, and +served version from the record and maps each namespace scope to its object selector. +There is no `Rejection`: a rule whose selector matches nothing followable resolves +(`ResourcesResolved=True`) and watches zero types, and why a type is not followable is in +its registry record and in the `type is not followable` log line. ```go type TargetView struct { @@ -301,28 +325,34 @@ type ScaleBinding struct { } ``` -`SpecReplicasPath` comes from `CustomResourceDefinition` `spec.versions[*].subresources.scale` -for CRDs and from a small built-in registry for built-ins, so both look identical to -the writer. The CRD JSONPath rules live in -[../../../facts/subresources.md](../facts/subresources.md) and are not repeated -here. The scale write path depends only on `SpecReplicasPath`; selector facts are for -reporting. The `scale` requirement is `skip` when scale is unused and only `fail`s -(`scale-path-unresolved`) when a needed parent path is missing. +The design takes `SpecReplicasPath` from `CustomResourceDefinition` +`spec.versions[*].subresources.scale` for CRDs and from a small built-in registry for +built-ins, so both look identical to the writer. The CRD JSONPath rules live in +[subresources.md](../facts/subresources.md) and are not repeated here. As built, only +the built-in registry ([`internal/typeset/scale.go`](../../internal/typeset/scale.go)) +fills the binding; a CRD parent with `/scale` gets `Usable: false`. The `scale` +requirement is `skip` when the type serves no `/scale` and `fail`s +(`scale-path-unresolved`) when it serves one without a known parent path. Nothing reads +the binding yet: a `/scale` audit event only names who scaled the parent, and the +replica count reaches Git through the parent's own watch event. ## Live set and the 60-second grace Additions are fast; removals are slow. A newly served type that passes every requirement enters the live set immediately. A previously live type that fails its -next `served`/`trusted` observation is held as `retained` for a fixed **60 seconds** -before it leaves the live set as `refused`/`unknown`. The grace is product safety, -not tuning, so it is not configurable: it stops a short discovery blink from turning -into a large Git sweep. +next observation is held as `retained`. A type that discovery stops listing is held +for a fixed **60 seconds** (`RemovalGrace`), after which the registry drops its record. +A type whose group/version discovery reports as failed keeps its last-known facts, +marked untrusted, and stays `retained` for as long as the failure persists. The grace +is product safety, not tuning, so it is not configurable: it stops a short discovery +blink from turning into a large Git sweep. While retained, `ByGVK`/`ByGVR` still return the record and `Followable()` is still -true, so planning, snapshots, informers, and writer identity keep working. The -verdict reads `retained` and the failing check explains why. Once the grace expires, -the type drops from `Followable()` and a deliberate sweep can act on a stable -absence. +true, so the watched-type tables and writer identity keep the type. A GitTarget's watch +setup does not run while one of its watched types is retained: it fails closed rather +than act on a reduced view. The verdict reads `retained` and the failing check explains +why. Once the grace expires, the record is gone: the type leaves `Followable()` and the +lookups no longer know it. ## What each consumer asks @@ -331,36 +361,30 @@ whether they need the non-followable ones. | Consumer | Reads | Needs refused records? | | --- | --- | --- | -| Manifest mapper | `Identity`, `Followable()` | no | -| WatchRule → `TargetView` | `Identity`, verbs, scope, `Followable()` | yes, as `Rejection` | -| Snapshot / informer | `Identity`, scope, `retained` state | no | -| Audit consumer | parent `Identity`, `Subresources.Scale.SpecReplicasPath` | no | -| Git writer | `Sensitive`, `Origin`, `Subresources.Scale` | no | -| CLI / status / GUI | the whole record, including `refused` | yes, via `All()` | - -The `/scale` case is the pressure test: the audit consumer calls `ByGVR` for the -parent, checks `Followable()`, and reads `Subresources.Scale.SpecReplicasPath`. If -that path is empty the `scale` check is `fail` and the translator refuses rather than -guessing `.spec.replicas`. - -## Relationship to today's code - -Greenfield names, but they land on existing packages: - -- **Scan / `Observation`** ← today's `APIResourceCatalog` refresh plus the new CRD / - `APIService` enrichment. -- **`TypeRegistry`** ← the resolved surface that does not exist yet; absorbs - `CatalogMapper` (becomes `ByGVK`), the GVK/GVR ambiguity policy, the sensitive / - deny policy, and the live-set hysteresis currently scattered across - `RuleGVRResolver` and `WatchedTypeTable`. -- **`TargetView`** ← today's `WatchedTypeTable`, stripped of identity, ambiguity, and - removal-grace logic now owned by the registry. +| Manifest mapper (git writer, `manifest-analyzer`) | `Identity`, `Followable()` | no | +| WatchRule → `WatchedTypeTable` | `Identity`, `Preferred`, `Followable()` | no | +| Target watch setup | `retained` state | no | +| Status, metrics, followability log | the whole record, including `refused` | yes, via `All()` | + +The design also had the audit consumer and the git writer read `Subresources.Scale`, +`Sensitive`, and `Origin`. Neither does: the audit path reads no type record, and the +writer takes sensitivity from the operator's sensitive-resource policy. + +## Where it lives in the code + +- **Scan / `Observation`**: `APIResourceCatalog` in `internal/watch` normalizes one + discovery result into a `typeset.Scan`; `typeset.ObservationsFromEntries` turns its + entries into observations. There is no CRD or `APIService` enrichment. +- **`TypeRegistry`**: `typeset.Registry`, one per source cluster, fed by + `UpdateFromScan` (see [typeset-owns-discovery-grace.md](typeset-owns-discovery-grace.md)). + It replaced `CatalogMapper` and `RuleGVRResolver`, and owns the GVK/GVR ambiguity + policy, the deny policy, and the removal grace. +- **`TargetView`**: `WatchedTypeTable` in `internal/watch`, which carries no identity, + ambiguity, or removal-grace logic of its own. ## References -- [api-catalog-watched-type-architecture.md](type-followability.md) -- [catalog-mapper-vs-watched-type-table.md](type-followability.md) -- [subresource-scope-reduction.md](scale-subresource-audit-rehydration.md) +- [scale-subresource-audit-rehydration.md](scale-subresource-audit-rehydration.md) - [gvk-gvr-mapping-layer.md](gvk-gvr-mapping-layer.md) - [resource-types.yaml](../facts/resource-types.yaml) - [subresources.md](../facts/subresources.md) diff --git a/docs/spec/type-lifecycle-events-and-wobble-settling.md b/docs/spec/type-lifecycle-events-and-wobble-settling.md index 4468513e..735d13e4 100644 --- a/docs/spec/type-lifecycle-events-and-wobble-settling.md +++ b/docs/spec/type-lifecycle-events-and-wobble-settling.md @@ -1,23 +1,22 @@ # Type lifecycle events, the wobble-settle phase, and consolidation -> **spec** — current behaviour. The code depends on this document; change one, change the other. Index: [`../INDEX.md`](../INDEX.md) -> Status: design direction, captured 2026-06-09. **Implemented 2026-06-09 (M12 first -> slices):** Proposals 1–2 in [internal/typeset/lifecycle.go](../../internal/typeset/lifecycle.go) -> (`LifecycleEvent`/`Observer`/`Subscribe`, `SettleWindow`, settle + flap coalescing in -> `Registry.Update`); M12 per-type reconcile/sweep via -> `manifestanalyzer.BuildScopedPlan`, the `ScopeGVR` resync request, the registry -> subscription + drain goroutine in internal/watch/type_lifecycle.go, -> and `EventRouter.EmitTypeReconcile/SweepForGitDest`. Proposal 3's consolidation is the -> shared `Manager.typeWobbling` predicate. The per-type path is gated on `SnapshotSynced`, -> so bootstrap is unchanged; `Unknown` granularity and bootstrap-decoupling remain open. -> Origin: the per-type reconcile track ([dream.md](type-followability.md), -> `per-type-reconcile-and-streaming-tail.md`) -> and the [e2e flakiness findings](e2e-serial-registry.md). -> Related: -> [discovery-catalog-typeset-boundary.md](type-followability.md), -> [type-followability.md](type-followability.md), -> [catalog-mapper-vs-watched-type-table.md](type-followability.md), -> [api-catalog-watched-type-architecture.md](type-followability.md). +> **spec**, kept as design history for its consumer half. Index: [`../INDEX.md`](../INDEX.md) +> +> Status: captured 2026-06-09 as a design direction. **Built and current:** Proposals 1 and 2 +> in [internal/typeset/lifecycle.go](../../internal/typeset/lifecycle.go) +> (`LifecycleEvent`/`Observer`/`Subscribe`, the 5-second `SettleWindow`, settle and flap +> coalescing in the registry's update). The events are computed on every update and have no +> production subscriber: the M12 consumers (a per-type reconcile and sweep driven by the events, +> `internal/watch/type_lifecycle.go`, `EventRouter.EmitTypeReconcile/SweepForGitDest`, a +> `ScopeGVR` resync request gated on `SnapshotSynced`) were deleted with the watch-first rewrite. +> The planned consumer is the `stop` classification in +> [target-watch-plan.md](../design/target-watch-plan.md#what-a-collection-leaving-means). Of +> Proposal 3 what exists is the shared `typeWobbling` predicate in +> `internal/watch/scope_resolve.go`: a GitTarget's watch setup fails closed while any of its +> watched types is `retained`. `Unknown` granularity remains open. +> Origin: the per-type reconcile track and the [e2e flakiness findings](e2e-serial-registry.md). +> Related: [type-followability.md](type-followability.md), +> [typeset-owns-discovery-grace.md](typeset-owns-discovery-grace.md). ## Why now @@ -67,7 +66,7 @@ M10 projects this per-GitTarget into the `WatchedTypeTable`, re-resolved on a ru fingerprint or a catalog `Generation()` change, and already holds a type the catalog momentarily stops serving rather than dropping it (the pending-removal fail-closed in `resolveSnapshotGVRs`). GVK↔GVR is treated as **1:1** (a -multi-resource GVK is a refused `TypeConflict`), giving the per-type work an +multi-resource GVK is refused with `gvk-not-unique`), giving the per-type work an unambiguous key. None of that changes here. This design adds a transition layer on top of it and @@ -154,7 +153,7 @@ changed?" is currently answered in three layers that each re-derive a slice of i pending-removal. Layer 1 and 2 are the right boundary and stay (see -[discovery-catalog-typeset-boundary.md](type-followability.md)). +[typeset-owns-discovery-grace.md](typeset-owns-discovery-grace.md)). **Layer 3's re-classification is the duplication to remove**: once the registry emits transitions, the watched-type store stops diffing and re-classifying — it *subscribes* to its types' transitions and updates membership directly. Concretely, @@ -195,7 +194,7 @@ because adding events without removing the diffs would be strictly worse. `TypeRemoved` (`absence-expired`), never on `Retained`/`Unknown`/wobble. - **`RemovalGrace` still gates deletion.** The settle window only gates activation; it never shortens removal. -- **GVK↔GVR is 1:1.** A multi-resource GVK stays a refused `TypeConflict`. +- **GVK↔GVR is 1:1.** A multi-resource GVK stays refused with `gvk-not-unique`. - **`Unknown` (catalog unavailable) is global, not per-type.** When the registry cannot assess the surface at all, no per-type reconcile should proceed — this stays a fail-closed whole-surface gate (§Open questions refines it). diff --git a/docs/spec/typeset-owns-discovery-grace.md b/docs/spec/typeset-owns-discovery-grace.md index e2107731..49892273 100644 --- a/docs/spec/typeset-owns-discovery-grace.md +++ b/docs/spec/typeset-owns-discovery-grace.md @@ -1,8 +1,10 @@ # Typeset owns discovery grace: thinning the API-resource catalog -> **spec** — current behaviour. The code depends on this document; change one, change the other. Index: [`../INDEX.md`](../INDEX.md) +> **spec**, a staged plan: S1–S3 are built and describe current behavior, S4 is open. The code +> depends on this document; change one, change the other. Index: [`../INDEX.md`](../INDEX.md) -**Status: S1–S3 IMPLEMENTED (2026-06-11); S4 open.** As-built deltas from the plan below +**Status: S1–S3 IMPLEMENTED (2026-06-11); S4 open.** §2 describes the code before S2 and S3 +landed, so read it as the starting point, not as today. As-built deltas from the plan below (all minor, same shape): - The catalog kept its `Refresh(disco) (bool, error)` signature; the scan is read back @@ -15,7 +17,15 @@ - The catalog keeps the last normalized scan as its only cross-scan state — mechanical, per §3.1's allowance: it is the change fingerprint (generation bumps only on changed facts, so registry revisions don't churn on steady rescans) and the re-derive source - for `refreshTypeRegistry`'s lazy callers (rule status, watched-type tables). + for `refreshClusterTypeRegistry`'s lazy callers (rule status, watched-type tables). +- There is one registry per source cluster (`refreshClusterTypeRegistry` publishes each + cluster's scan to its own), and the feed is `UpdateFromScan(scan)`: §2.1's "Registry + feed" row names the pre-S2 `Update`, which survives only for `NewSnapshotRegistry`. + §2.1's fixtures row is also out of date: `SplitFieldPath` and + `internal/auditutil/subresource_policy.go` are gone, and `BuiltinScale` has no caller + outside `internal/typeset`. +- After the grace a record is dropped from the registry rather than kept as `Refused` + (§3.2): lookups stop knowing the type, and the lifecycle emits `TypeRemoved`. - **The demand axis is gone (2026-08-26).** `typeset.Materializer` and its materialization lifecycle were deleted: the watch-first rewrite left them with no production caller, and the demand intersection ("followable AND claimed") now lives in diff --git a/docs/spec/unsupported-folder-refusal-plan.md b/docs/spec/unsupported-folder-refusal-plan.md index 8af54056..f8904cbc 100644 --- a/docs/spec/unsupported-folder-refusal-plan.md +++ b/docs/spec/unsupported-folder-refusal-plan.md @@ -1,23 +1,24 @@ # Refuse unsupported folder content — design + implementation plan -> **spec** — current behaviour. The code depends on this document; change one, change the other. Index: [`../INDEX.md`](../INDEX.md) +> **spec**, kept as design history. Index: [`../INDEX.md`](../INDEX.md) > -> Status: IMPLEMENTED — shipped as `GitPathAccepted` (internal/git/git_path_refusal.go, -> test/e2e/unsupported_folder_e2e_test.go). Originally written 2026-06-26, revised 2026-06-27; -> the "PROPOSAL" label was stale and was corrected 2026-07-11. The mechanics (refuse a GitTarget path the -> operator cannot safely manage) are settled. The **status surface** is reopened: this revision records the -> back-and-forth that turned a single overloaded `StreamsReady=Blocked` refusal into a deliberate -> two-sided status model, and then — after the `k8s-crd-design-review` skill grew a `kstatus-readiness` -> reference — into a **two-layer kstatus model** (positive `Ready` + abnormal-true `Reconciling`/`Stalled` -> over domain conditions). §3 lays the discarded options next to the chosen one so the reasoning survives; -> the goal is a status contract legible to Flux / Argo CD / `helm --wait`, not just to humans reading YAML. +> Status: IMPLEMENTED. The refusal shipped as `GitPathAccepted` +> ([`internal/git/git_path_refusal.go`](../../internal/git/git_path_refusal.go), +> [`test/e2e/unsupported_folder_e2e_test.go`](../../test/e2e/unsupported_folder_e2e_test.go)), with +> D1 to D4 in §7 as decided. Originally written 2026-06-26, revised 2026-06-27. The status surface is +> settled, and [status-conditions-guide.md](status-conditions-guide.md) is its current contract: the +> kstatus trio over domain conditions, as §3.5 recommends. Read the rest of this page as the record of +> how that was decided. Where it differs from the code, the code wins: GitTarget has no `status.phase` +> (so no `Degraded`), the printer columns carry no `Reconciling`/`Stalled`, the live writer runs +> `AcceptStructureOnly` (the mapping-aware refusals are still deferred), and the line-number links +> point at the code as it was in June 2026. > > **Decision recorded** in > [e2e-coverage-gaps-and-improvements-plan.md §4.1](../design/e2e-coverage-gaps-and-improvements-plan.md): the > operator must **refuse** a GitTarget path it cannot safely manage, for the cases where we already -> know the content is a problem — not silently keep writing. Written against the **actual** current code, -> not the superseded `Synced`/`materialization` model in -> [status-design-git-target.md](status-conditions-guide.md). +> know the content is a problem. It must not silently keep writing. This page was written against +> the code of June 2026, after the `Synced`/`materialization` model of the since-deleted +> `status-design-git-target.md` was superseded. --- diff --git a/docs/spec/watch-event-ordering-and-attribution-grace.md b/docs/spec/watch-event-ordering-and-attribution-grace.md index a3f1fc60..dfc630bd 100644 --- a/docs/spec/watch-event-ordering-and-attribution-grace.md +++ b/docs/spec/watch-event-ordering-and-attribution-grace.md @@ -3,7 +3,7 @@ > **contract:** current watch processing and ordering boundaries. Index: [`../INDEX.md`](../INDEX.md) > > Status: **implementation explainer** -> Checked: 2026-09-28 +> Checked: 2026-10-20 > Related: [Kubernetes watch options](../facts/kubernetes-watch-options.md), > [Watch-first ingestion architecture](../finished/watch-first-ingestion-architecture.md), > [Attribution contract](attribution.md), @@ -76,11 +76,12 @@ also authors the change. A fact arriving after a commit is written does not rewr The index determines what counts as a resolution. For a removal query, even an exact UID/RV PATCH match is held while [`FactIndex.Await`](../../internal/queue/fact_index.go) waits for deletion evidence, up to the grace deadline. `ExactCapable=false` does not disable the exact lookup. -When the exact fact is missing, an earlier writer's fallback can be returned at expiry; ordered -delivery does not establish that this author caused a label-exit mutation. The proposed +When the exact fact is missing, an earlier writer's fallback can be returned at expiry. A removal +from a label-selected collection is the exception: under the [filtered-removal attribution policy](../design/watches-labels-simplification.md#attribution-for-filtered-removals) -requires stricter evidence and releases eligible exact matches immediately, preserving the same -inline ordering. That policy is not built. +it reads only the fact at the object's exact UID/RV, returns as soon as that fact arrives, and is +unresolved without it, so no earlier writer can author a label exit. Both paths keep the same +inline ordering. ### Preserve ordered release if attribution becomes concurrent @@ -161,12 +162,14 @@ test today. queue checks live in [`resync_push_test.go`](../../internal/git/resync_push_test.go). - Refuse an event because the worker queue is full. The routing error must leave its watch cursor unadvanced; queue admission is the boundary asserted by this check. -- For the proposed selector replacement, pause the retired producer immediately before enqueue and +- For a selector replacement, pause the retired producer immediately before enqueue and initialize the replacement. The retired work must enter ahead of the replacement snapshot or be canceled. It cannot arrive behind that snapshot and restore an excluded object. The [label-selection design](../design/watches-labels-simplification.md#scope-integration-and-queue-ordering) - requires a producer handoff for this case; the current cancellation checks alone do not - establish it. + requires a producer handoff for this case, and `producerGate` in + [`target_watch.go`](../../internal/watch/target_watch.go) provides it: retiring a stream waits + out an enqueue in flight and admits none after. The check lives in + [`producer_gate_test.go`](../../internal/watch/producer_gate_test.go). - Exercise overlapping watches with delayed attribution on one. Assert each watch's own order; do not infer a global mutation order from their combined arrivals. The label-selection design refuses conflicting selectors but retains the existing same-selector namespace overlap. diff --git a/docs/spec/where-validation-lives.md b/docs/spec/where-validation-lives.md index 5eee18e8..c1ac1fb3 100644 --- a/docs/spec/where-validation-lives.md +++ b/docs/spec/where-validation-lives.md @@ -11,7 +11,7 @@ Validate at the **first** rung that can express the rule: 1. **OpenAPI schema** — types, enums, required, min/max, immutability where the shape allows. 2. **CEL `XValidation`** — anything expressible from the object *itself*, including transitions against `oldSelf`. Example: `spec.kubeConfig` immutability on `ClusterProvider` - ([`clusterprovider_types.go:68`](../../api/v1alpha3/clusterprovider_types.go#L68)). + ([`clusterprovider_types.go:25`](../../api/v1alpha3/clusterprovider_types.go#L25)). 3. **The reconciler** — everything **cross-object**, because CEL cannot read another object. Refusal surfaces as `Validated=False` with a reason, and **returns before** the object reaches the data plane. @@ -40,21 +40,27 @@ Three things, in order of how often they get argued backwards: The worked example is `ClusterProvider.spec.accessFrom`. An earlier design proposed enforcing it "in two places", one of them a webhook. What shipped enforces it in **one**, on every reconcile, before `DeclareForGitTarget` -([`gittarget_controller.go:311`](../../internal/controller/gittarget_controller.go#L311), -[`gittarget_source_cluster.go:68`](../../internal/controller/gittarget_source_cluster.go#L68)) — +([`gittarget_controller.go:473`](../../internal/controller/gittarget_controller.go#L473), +[`gittarget_source_cluster.go:37`](../../internal/controller/gittarget_source_cluster.go#L37)), and that is why the quickstart is a single command again. ## The one webhook we ship, and why it is not an exception to the rule -[`/validate-operator-types`](../../internal/webhook/validate_operator_types_handler.go#L26) +[`/validate-operator-types`](../../internal/webhook/validate_operator_types_handler.go#L27) is **not a validation gate**. It captures the *submitter's identity* on a command kind -(`CommitRequest`) so a commit can be authored by a real Kubernetes user, and it **always -allows**. Identity is the textbook case for admission: it exists in the `AdmissionRequest` and -on no persisted field, so no reconciler could recover it later. See +(`CommitRequest`) so a commit can be authored by a real Kubernetes user, and for a command it +**always allows**. Identity is the textbook case for admission: it exists in the `AdmissionRequest` +and on no persisted field, so no reconciler could recover it later. See [`commitrequest-admission-authorship.md`](commitrequest-admission-authorship.md). -(`/validate-all` — [`admission_allow_handler.go`](../../internal/webhook/admission_allow_handler.go#L12) -— is an always-allow observation surface wired only by the e2e SUT. The chart does not ship it.) +The same endpoint carries one rejection, and it is feedback, not a gate: a `WatchRule` that would +bring a second source namespace to a `GitTarget` declaring `spec.serializeNamespace: false` is +rejected at the moment the mistake is made. The writer refuses the same configuration at the write +whether or not the webhook ran, which is why its `failurePolicy` is `Ignore`. This is the feedback +latency trade above, taken where a webhook is installed anyway, and never as the only gate. + +(`/validate-all`, in [`admission_allow_handler.go`](../../internal/webhook/admission_allow_handler.go#L17), +is an always-allow observation surface wired only by the e2e SUT. The chart does not ship it.) ## Applying this diff --git a/hack/crdfields/fields.yaml b/hack/crdfields/fields.yaml index f0d8a53f..bf278cad 100644 --- a/hack/crdfields/fields.yaml +++ b/hack/crdfields/fields.yaml @@ -228,8 +228,6 @@ kinds: doc: >- The `Kustomization` that applies the folders of the `GitTarget`s naming this trigger. See [who applies this folder](#who-applies-this-folder-specapplytriggerref) - - path: quietPeriod - doc: How long the branch must stay quiet after GitOps Reverser's own pushes before a `CatchUp` - kind: ArgoCDTrigger title: "`ArgoCDTrigger` (namespaced): the Argo CD `Application` GitOps Reverser asks to sync" diff --git a/internal/applier/applier.go b/internal/applier/applier.go index ea63a976..bd0e0f1d 100644 --- a/internal/applier/applier.go +++ b/internal/applier/applier.go @@ -140,9 +140,6 @@ func selfSubjectAccessReviewGVR() schema.GroupVersionResource { // Outcome is what one request did. type Outcome string -// OutcomeCurrent asked for nothing: the applier needed no request (see each client's Request). -const OutcomeCurrent Outcome = "Current" - // Configured and ApplyTriggerReady reasons (docs/configuration.md, "Who applies this folder"). // Succeeded and Failed are the generic reasons, aliased from Flux's meta package as the conditions // guide asks. @@ -198,7 +195,7 @@ type Source struct { } // Started is the operation one Request started, may have started (Unconfirmed), or read back from -// the trigger's record (Recovered), for Progress to follow; with Outcome Current, nothing was asked. +// the trigger's record (Recovered), for Progress to follow. type Started struct { // Outcome is what was asked, if anything. Outcome Outcome diff --git a/internal/applier/argocd.go b/internal/applier/argocd.go index 2ddf0fb3..40f33df4 100644 --- a/internal/applier/argocd.go +++ b/internal/applier/argocd.go @@ -5,6 +5,7 @@ package applier import ( "context" "encoding/json" + "errors" "fmt" "strings" "time" @@ -139,9 +140,8 @@ func NewArgoCD(client dynamic.Interface) *ArgoCD { // it returns an Unconfirmed operation rather than an error, which Progress follows by its token, // so the apply window stays open until it is known to have run or never to run (settleUnseen). // -// An empty commit (the remote does not carry the branch) has nothing to pin, and asks nothing -// (OutcomeCurrent): Argo CD reports the missing branch itself on its next refresh, and the branch -// coming back is an observation that leaves a new request pending. +// The worker never asks for an empty commit: when the remote does not carry the branch there is +// nothing to pin, and Argo CD reports the missing branch itself on its next refresh. func (a *ArgoCD) Request(ctx context.Context, ref types.NamespacedName, ask Ask) (Started, error) { var waitUntil time.Time var abandonedState string @@ -152,7 +152,7 @@ func (a *ArgoCD) Request(ctx context.Context, ref types.NamespacedName, ask Ask) return Started{}, err } if ask.Commit == "" { - return Started{Outcome: OutcomeCurrent}, nil + return Started{}, permanentError{errors.New("no commit to apply: the remote does not carry the branch")} } if mustWait(app, abandonedState) { if abandonedState, err = a.awaitOperation(ctx, ref, app, &waitUntil); err != nil { diff --git a/internal/applier/argocd_test.go b/internal/applier/argocd_test.go index fb79e2b6..e350ba64 100644 --- a/internal/applier/argocd_test.go +++ b/internal/applier/argocd_test.go @@ -576,7 +576,7 @@ func TestArgoCDRequest_RefusesWhatIsNotUnderControl(t *testing.T) { wantErr: "has automated sync on", wantPerm: true}, {name: "the decision went stale", app: manualApp(), commit: testCommit, current: func() bool { return false }, wantStale: true}, - {name: "no commit to pin", app: manualApp(), wantOutcome: OutcomeCurrent}, + {name: "no commit to pin", app: manualApp(), wantErr: "no commit to apply", wantPerm: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -781,10 +781,6 @@ func TestArgoCD_Progress(t *testing.T) { assert.Equal(t, Progress{Outcome: git.ApplySucceeded}, progress(t)) setState(t, theirOperation(), "Succeeded", false) assert.Equal(t, git.ApplyFailed, progress(t).Outcome, "ours was cleared before it recorded an outcome") - - current, err := argo.Progress(context.Background(), platformApp, &Started{Outcome: OutcomeCurrent}) - require.NoError(t, err) - assert.Equal(t, git.ApplySucceeded, current.Outcome, "nothing asked is nothing to wait for") } // TestArgoCD_ASyncWhoseOperationWasRemovedEnds is review 7's A1. Removing .operation is the diff --git a/internal/applier/dispatcher.go b/internal/applier/dispatcher.go index 23466e64..53855551 100644 --- a/internal/applier/dispatcher.go +++ b/internal/applier/dispatcher.go @@ -33,10 +33,10 @@ const ( // EventReasonApplyRequested is a request that reached the applier. EventReasonApplyRequested = "ApplyRequested" // EventReasonApplyRequestFailed is the first failed attempt of a run of them. The branch worker - // keeps the reason pending and asks again on a quiet branch; the Event is not repeated for its retries. + // keeps the reason pending and asks again after a flush; the Event is not repeated for its retries. EventReasonApplyRequestFailed = "ApplyRequestFailed" // EventReasonApplyFailed is a requested apply the applier ran and did not complete, the first of - // a run of failed attempts. The branch worker asks again on a quiet branch. + // a run of failed attempts. The branch worker asks again after a flush. EventReasonApplyFailed = "ApplyFailed" // EventReasonApplyUnresolved is a requested apply that did not complete within // UnresolvedAfter. It is still followed, and its apply window stays open until it completes. @@ -96,7 +96,7 @@ const pollTimeout = 10 * time.Second // carried out on its own goroutine and reported back to the worker that decided it: the request, // then the operation it started as it moves, until it completes or the worker abandons it. The // worker keeps the reason pending until an operation answers it, and decides every retry on a quiet -// branch. So nothing goes out that a worker did not check on a quiet branch, and nothing here can +// branch. So nothing goes out that a worker did not check after a flush, and nothing here can // turn a delayed decision into an apply nobody checked. // // What it does check is what can change between the worker's decision and the write. It reads each @@ -384,14 +384,15 @@ func (d *Dispatcher) attempt(ctx context.Context, req git.ApplyRequest) (git.App case isPermanent(err): return d.failure(ctx, req, eligible, report(git.ApplyRejected, err), log), nil case err != nil: - return d.failure(ctx, req, eligible, report(git.ApplyRequestFailed, err), log), nil + res := report(git.ApplyRequestFailed, err) + var elsewhere *sourceElsewhereError + if errors.As(err, &elsewhere) { + res.SourceRevision = elsewhere.revision + } + return d.failure(ctx, req, eligible, res, log), nil } d.count(ctx, req, outcomeSent) - if started.Outcome == OutcomeCurrent { - log.V(1).Info("the applier needs no request for this commit; asking nothing") - return report(git.ApplyCurrent, nil), nil - } log.Info("apply requested", "outcome", started.Outcome) if started.Unconfirmed != nil { log.Info("the request may not have reached the applier; following it by its token", @@ -673,7 +674,7 @@ func (d *Dispatcher) completed( "after", took.String(), "message", progress.Message, "priorFailures", req.PriorFailures) if req.PriorFailures == 0 { d.event(ctx, req, corev1.EventTypeWarning, EventReasonApplyFailed, - "%s did not complete the apply of %s (%s), and will be asked again once the branch is quiet: %v", + "%s did not complete the apply of %s (%s), and will be asked again after its backoff: %v", describeApplier(req.Applier), DescribeCommit(req.Commit), req.Reason, err) } result.Err = err @@ -734,7 +735,7 @@ func (d *Dispatcher) failure( if req.PriorFailures == 0 && d.recorder != nil { for _, e := range eligible { d.recorder.Eventf(e.target, corev1.EventTypeWarning, EventReasonApplyRequestFailed, - "could not ask %s to apply %s (%s), and will ask again once the branch is quiet: %v", + "could not ask %s to apply %s (%s), and will ask again after its backoff: %v", describeApplier(req.Applier), DescribeCommit(req.Commit), e.reason, res.Err) } } @@ -886,114 +887,3 @@ func applierKindOf(req git.ApplyRequest) string { } return applierKindKustomization } - -// PrefetchSource asks the GitRepository a FluxTrigger's Kustomization applies to fetch now, ahead of -// the apply, and returns at once. It asks nothing when the trigger's status.source.prefetch is not -// true, when a FluxTrigger on the same source has an operation in flight (a resumed Kustomization -// applies whatever the artifact becomes), or when the artifact already holds the commit. It applies -// nothing, so it reports nothing. -func (d *Dispatcher) PrefetchSource(req git.PrefetchRequest) { - if req.Applier.Kind != v1alpha3.ApplierKindKustomization { - return - } - d.runMu.RLock() - ctx := d.run - if ctx != nil { - d.attempts.Add(1) - } - d.runMu.RUnlock() - if ctx == nil { - return - } - go func() { - defer d.attempts.Done() - ctx, cancel := context.WithTimeout(ctx, d.AttemptTimeout) - defer cancel() - log := d.log.WithValues("applier", req.Applier.String(), "commit", req.Commit) - if err := d.prefetch(ctx, req); err != nil && !errors.Is(err, errNotPrefetched) { - log.V(1).Info("did not prefetch the source", "reason", err.Error()) - return - } - log.V(1).Info("prefetched the source") - }() -} - -// errPrefetchHeld is a prefetch held back by the trigger's source. -var errPrefetchHeld = errors.New("the source may not be prefetched now") - -// prefetch checks the trigger and its source's other triggers, then the source's consumers as they -// are now (heldNow), and stamps the source, which every check was of: status.source's GitRepository, -// only while the Kustomization still applies it. -func (d *Dispatcher) prefetch(ctx context.Context, req git.PrefetchRequest) error { - trigger := &v1alpha3.FluxTrigger{} - key := client.ObjectKey{Namespace: req.Applier.Trigger.Namespace, Name: req.Applier.Trigger.Name} - if err := d.reader.Get(ctx, key, trigger); err != nil { - return err - } - if trigger.Status.Source == nil || !trigger.Status.Source.Prefetch || git.ApplierOf(trigger) != req.Applier { - return errPrefetchHeld - } - var triggers v1alpha3.FluxTriggerList - if err := d.reader.List(ctx, &triggers); err != nil { - return err - } - for i := range triggers.Items { - other := &triggers.Items[i] - if other.ClusterProvider() != trigger.ClusterProvider() || other.Status.Source == nil || - other.Status.Source.GitRepository != trigger.Status.Source.GitRepository { - continue - } - if operation := other.Status.Operation; operation != nil && !operation.Phase.Completed() { - return errPrefetchHeld - } - } - flux, err := d.heldNow(ctx, req, trigger, triggers.Items) - if err != nil { - return err - } - checked, _ := trigger.SharedSource() - return flux.Prefetch(ctx, applierName(req.Applier), checked, req.Commit) -} - -// heldNow checks again, in the applier's cluster, what the trigger's status.source.prefetch said when -// its reconciler last ran, up to a minute ago: every consumer of its GitRepository is a -// Kustomization a FluxTrigger on the same cluster names, and is suspended, held by GitOps Reverser -// with its suspend handled now. A consumer that appeared since, or one Flux applies on its own again, would apply -// the prefetched artifact at once, over live edits not pushed yet (errPrefetchHeld). It returns the -// Flux client of the applier's cluster. -func (d *Dispatcher) heldNow( - ctx context.Context, req git.PrefetchRequest, trigger *v1alpha3.FluxTrigger, triggers []v1alpha3.FluxTrigger, -) (*Flux, error) { - applierClient, err := d.applierClients(ctx, req.Applier.Cluster) - if err != nil { - return nil, err - } - flux := NewFlux(applierClient) - flux.Now = d.Now - named := map[types.NamespacedName]bool{} - for i := range triggers { - other := &triggers[i] - if other.DeletionTimestamp.IsZero() && other.ClusterProvider() == trigger.ClusterProvider() { - named[other.Applier()] = true - } - } - source := trigger.Status.Source.GitRepository - consumers, err := flux.Consumers(ctx, types.NamespacedName{Namespace: source.Namespace, Name: source.Name}) - if err != nil { - return nil, err - } - for _, consumer := range consumers { - key := types.NamespacedName{Namespace: consumer.Namespace, Name: consumer.Name} - if consumer.Kind != v1alpha3.ApplierKindKustomization || !named[key] { - return nil, fmt.Errorf("%w: %s %s is not held by a FluxTrigger", errPrefetchHeld, consumer.Kind, key) - } - suspension, err := flux.Suspension(ctx, key) - if err != nil { - return nil, fmt.Errorf("read %s %s: %w", kindKustomization, key, err) - } - if !suspension.Suspended || !suspension.Held || !suspension.Handled { - return nil, fmt.Errorf("%w: %s %s is not held suspended now", errPrefetchHeld, kindKustomization, key) - } - } - return flux, nil -} diff --git a/internal/applier/dispatcher_test.go b/internal/applier/dispatcher_test.go index 59f8d0f9..b920af56 100644 --- a/internal/applier/dispatcher_test.go +++ b/internal/applier/dispatcher_test.go @@ -448,8 +448,24 @@ func TestDispatcher_AnApplierClusterThatDoesNotAdmitAsksNothing(t *testing.T) { assert.Empty(t, f.source.clusters, "the hub's client is never even asked for") } -// TestDispatcher_AFailureIsReportedAndNeverRetried: retrying is the branch worker's decision, on a -// quiet branch. The dispatcher reports the failure, and records ApplyRequestFailed only for the +// TestDispatcher_ASourceElsewhereIsReportedWithItsRevision: a GitRepository that fetched and holds +// another commit than the one asked for fails the attempt, and the report carries the revision it +// holds, which the branch worker takes as a hint that the branch moved. +func TestDispatcher_ASourceElsewhereIsReportedWithItsRevision(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, behindSource()...) + f.sim.Fetches(platformKey, "main@sha1:"+olderCommit) + + res := f.attempt(request(checkout, v1alpha3.ApplyReasonForeignCommit, "a")) + assert.Equal(t, git.ApplyRequestFailed, res.Outcome) + assert.Equal(t, "main@sha1:"+olderCommit, res.SourceRevision) + + f.sim.Fetches(platformKey, "main@sha1:"+testCommit) + assert.Empty(t, f.attempt(request(checkout, v1alpha3.ApplyReasonForeignCommit, "a")).SourceRevision, + "a source that brings the commit says nothing") +} + +// TestDispatcher_AFailureIsReportedAndNeverRetried: retrying is the branch worker's decision, after a +// flush. The dispatcher reports the failure, and records ApplyRequestFailed only for the // first failure of a run. func TestDispatcher_AFailureIsReportedAndNeverRetried(t *testing.T) { f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}) @@ -576,10 +592,11 @@ func TestDispatcher_AStaleDecisionAsksNothing(t *testing.T) { assert.Equal(t, int64(1), f.count(v1alpha3.ApplyReasonRefusal, outcomeStale)) } -// TestDispatcher_CatchUpAlreadyCurrent: a CatchUp the applier already satisfies (suspended, with -// the commit in its artifact and applied) counts as sent and records nothing, since nothing was -// asked of it. -func TestDispatcher_CatchUpAlreadyCurrent(t *testing.T) { +// TestDispatcher_ACatchUpOfTheAppliedCommitIsAsked: the branch worker decides what needs no apply +// (it sends no converge of the commit an applier last applied), so a CatchUp the dispatcher carries +// is asked even of a Kustomization suspended with the commit applied: an adopt re-applies it on +// purpose, which is what stamps Flux's owner labels on an object the API created. +func TestDispatcher_ACatchUpOfTheAppliedCommitIsAsked(t *testing.T) { ks := kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")) _ = unstructured.SetNestedField(ks.Object, true, "spec", "suspend") _ = unstructured.SetNestedField(ks.Object, "main@sha1:"+testCommit, "status", "lastAppliedRevision") @@ -590,9 +607,8 @@ func TestDispatcher_CatchUpAlreadyCurrent(t *testing.T) { res := f.attempt(request(checkout, v1alpha3.ApplyReasonCatchUp, "a")) - assert.Equal(t, git.ApplyCurrent, res.Outcome) - assert.Empty(t, f.patches()) - assert.Empty(t, f.events()) + assert.Equal(t, git.ApplyRequested, res.Outcome) + assert.NotEmpty(t, f.patches(), "the Kustomization was resumed") } // TestDispatcher_RequestApplyReportsFromItsOwnGoroutine: RequestApply returns at once and the @@ -717,7 +733,7 @@ func TestDispatcher_AnApplyPastItsBudgetIsUnresolvedAndStillFollowed(t *testing. } // TestDispatcher_AFailedApplyIsSuspendedAgain: a failed apply is suspended like any other, which is -// what stops Flux's own retryInterval retry; the worker decides the retry, on a quiet branch. +// what stops Flux's own retryInterval retry; the worker decides the retry, after a flush. func TestDispatcher_AFailedApplyIsSuspendedAgain(t *testing.T) { f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, behindSource()...) f.sim.FailApplies("ConfigMap/apps/x dry-run failed") diff --git a/internal/applier/flux.go b/internal/applier/flux.go index 3589cc8a..6e2662b4 100644 --- a/internal/applier/flux.go +++ b/internal/applier/flux.go @@ -18,7 +18,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" - "github.com/ConfigButler/gitops-reverser/api/v1alpha3" "github.com/ConfigButler/gitops-reverser/internal/git" "github.com/ConfigButler/gitops-reverser/internal/repourl" ) @@ -43,8 +42,7 @@ const ( kindGitRepository = "GitRepository" ) -// OutcomeResumed resumed the Kustomization for one apply. OutcomeCurrent asks nothing: a CatchUp or -// ForeignCommit found it suspended, with the commit both in its artifact and applied. +// OutcomeResumed resumed the Kustomization for one apply. const OutcomeResumed Outcome = "Resumed" // SuspendFieldManager is the field manager GitOps Reverser holds a Kustomization's spec.suspend and @@ -92,9 +90,10 @@ const fetchPollInterval = 100 * time.Millisecond // for that fetch. An artifact that ends on another commit is errStale when the decision no longer // holds, and a failure otherwise: the source cannot fetch the commit the worker checked. // -// A ForeignCommit or a CatchUp asks nothing (OutcomeCurrent) when the Kustomization is suspended -// and has applied commit: nothing applied since, and nothing will until it is resumed. A Refusal -// and a Repair always run the operation, because they ask for an apply of an unchanged commit. +// Every reason runs the operation. The branch worker decides what needs no apply: it follows every +// operation, keeps the revision each applier last applied, and sends no converge of that commit +// (docs/design/applier-echoes.md, S2). An adopt re-applies the same commit on purpose: that is what +// stamps Flux's owner labels on an object the API created. // // Just before the resume it reads the Kustomization again, since the fetch can take a while: one // that was replaced, applies another source now, or has its spec.suspend applied by another @@ -106,8 +105,8 @@ const fetchPollInterval = 100 * time.Millisecond // requestedAt, so it enqueues a reconcile that reads both, which applies. It is made over the // Kustomization as last read, and fails rather than land on one that changed since. // -// An empty commit (the remote does not carry the branch) has nothing to apply: the GitRepository -// is stamped, so Flux reports the missing branch itself, and nothing is resumed. +// The worker never asks for an empty commit: when the remote does not carry the branch there is +// nothing to apply, and the GitRepository reports the missing branch itself. func (f *Flux) Request(ctx context.Context, ref types.NamespacedName, ask Ask) (Started, error) { ks, err := f.client.Resource(kustomizationGVR()).Namespace(ref.Namespace).Get(ctx, ref.Name, metav1.GetOptions{}) if err != nil { @@ -125,18 +124,11 @@ func (f *Flux) Request(ctx context.Context, ref types.NamespacedName, ask Ask) ( return Started{}, err } if ask.Commit == "" { - if _, err := f.stampSource(ctx, sourceRef); err != nil { - return Started{}, err - } - return Started{Outcome: OutcomeCurrent}, nil + return Started{}, permanentError{errors.New("no commit to apply: the remote does not carry the branch")} } if err := f.fetchCommit(ctx, sourceRef, source, ask); err != nil { return Started{}, err } - if (ask.Reason == v1alpha3.ApplyReasonForeignCommit || ask.Reason == v1alpha3.ApplyReasonCatchUp) && - suspended(ks) && revisionHolds(statusString(ks, "lastAppliedRevision"), ask.Commit) { - return Started{Outcome: OutcomeCurrent}, nil - } return f.resume(ctx, ref, ks, sourceRef, ask) } @@ -158,8 +150,22 @@ func (f *Flux) fetchCommit( if !ask.holds() { return errStale } - return fmt.Errorf("%s %s fetched %s, not commit %s", kindGitRepository, sourceRef, - statusString(source, "artifact", "revision"), ask.Commit) + return &sourceElsewhereError{ + source: sourceRef, revision: statusString(source, "artifact", "revision"), commit: ask.Commit, + } +} + +// sourceElsewhereError is a GitRepository that fetched, and holds another commit than the one asked +// for: usually the branch moved on, past a decision the branch worker made before it heard of the +// move. The dispatcher hands the revision to the worker (git.ApplyResult.SourceRevision), which looks +// at the remote instead of asking again for a commit the source may no longer serve. +type sourceElsewhereError struct { + source types.NamespacedName + revision, commit string +} + +func (e *sourceElsewhereError) Error() string { + return fmt.Sprintf("%s %s fetched %s, not commit %s", kindGitRepository, e.source, e.revision, e.commit) } // maxResumeConflicts bounds how often one request reads the Kustomization again after its resume @@ -337,8 +343,7 @@ func (f *Flux) fetch(ctx context.Context, sourceRef types.NamespacedName) (*unst // `flux resume`) is suspended again. A Recovered or Unconfirmed operation whose token is nowhere on the // Kustomization was never resumed, or its token was stamped over (a suspend of its own the process // stopped before reporting, `flux reconcile`): it is suspended, and judged once that suspend is -// handled on what ran since it was asked for. Nothing else is asked: an OutcomeCurrent request is -// finished. +// handled on what ran since it was asked for. func (f *Flux) Progress(ctx context.Context, ref types.NamespacedName, s *Started) (Progress, error) { if s.Outcome != OutcomeResumed { return Progress{Outcome: git.ApplySucceeded}, nil diff --git a/internal/applier/flux_consumers.go b/internal/applier/flux_consumers.go index fb395b3a..57aa8a13 100644 --- a/internal/applier/flux_consumers.go +++ b/internal/applier/flux_consumers.go @@ -4,7 +4,6 @@ package applier import ( "context" - "errors" "fmt" "sort" @@ -17,10 +16,9 @@ import ( "k8s.io/apimachinery/pkg/types" ) -// A GitRepository's consumers, and prefetching it (docs/design/apply-trigger.md, "Shared sources and prefetch"). -// A suspended Kustomization ignores a new artifact, so a GitRepository whose every consumer is held -// suspended by a FluxTrigger can fetch on every push without applying anything. One consumer that is -// not held applies each new artifact the moment it arrives, so then the source is never prefetched. +// A GitRepository's consumers (docs/design/apply-trigger.md, "Shared sources"). A suspended +// Kustomization ignores a new artifact; one consumer that is not held applies each new artifact the +// moment it arrives, so a FluxTrigger lists the ones no trigger holds in its status. // A HelmRelease applies a GitRepository through a HelmChart that source-controller builds from it: // one helm-controller creates from the release's spec.chart, or one the release names in @@ -205,36 +203,3 @@ func consumes(obj *unstructured.Unstructured, path []string, source types.Namesp } return namespace == source.Namespace } - -// errNotPrefetched is a prefetch that was not made, because the artifact already holds the commit. -var errNotPrefetched = errors.New("the artifact already holds the commit") - -// Prefetch stamps checked, the GitRepository of the Kustomization ref whose consumers were checked, -// so it fetches now, unless its artifact already holds commit (errNotPrefetched). A Kustomization -// that applies another GitRepository now was repointed since its trigger published checked: that -// source's consumers were never checked, so nothing is stamped (errPrefetchHeld). It waits for -// nothing: the operation that applies the commit reads the artifact again before it resumes. -func (f *Flux) Prefetch(ctx context.Context, ref, checked types.NamespacedName, commit string) error { - ks, err := f.client.Resource(kustomizationGVR()).Namespace(ref.Namespace).Get(ctx, ref.Name, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("get %s %s: %w", kindKustomization, ref, err) - } - sourceRef, err := gitRepositoryRef(ks) - if err != nil { - return err - } - if sourceRef != checked { - return fmt.Errorf("%w: %s %s applies %s %s now, not %s", errPrefetchHeld, kindKustomization, ref, - kindGitRepository, sourceRef, checked) - } - source, err := f.client.Resource(gitRepositoryGVR()).Namespace(sourceRef.Namespace). - Get(ctx, sourceRef.Name, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("get %s %s: %w", kindGitRepository, sourceRef, err) - } - if artifactHolds(source, commit) { - return errNotPrefetched - } - _, err = f.stampSource(ctx, sourceRef) - return err -} diff --git a/internal/applier/flux_consumers_test.go b/internal/applier/flux_consumers_test.go index 0495f8f3..5bbe0961 100644 --- a/internal/applier/flux_consumers_test.go +++ b/internal/applier/flux_consumers_test.go @@ -17,11 +17,9 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" dynamicfake "k8s.io/client-go/dynamic/fake" k8stesting "k8s.io/client-go/testing" - "sigs.k8s.io/controller-runtime/pkg/client" "github.com/ConfigButler/gitops-reverser/api/v1alpha3" "github.com/ConfigButler/gitops-reverser/internal/applier/fluxsim" - "github.com/ConfigButler/gitops-reverser/internal/git" ) // helmReleaseObj is a HelmRelease whose chart comes from the GitRepository "platform", in its namespace @@ -193,147 +191,4 @@ func TestFluxProbe_ASharedSourceAppliesOnlyTheConsumerAsked(t *testing.T) { assert.True(t, suspended(sim.Kustomization(checkoutKey))) } -// prefetchFixture is a dispatcher whose FluxTrigger "checkout" publishes source flux-system/platform -// with prefetch, over a cluster whose GitRepository is behind and whose Kustomization it holds. -func prefetchFixture(t *testing.T, prefetch bool, extra ...client.Object) *dispatcherFixture { - t.Helper() - return prefetchFixtureOver(t, prefetch, heldSource(t), extra...) -} - -// prefetchFixtureOver is prefetchFixture over the source cluster objects source. -func prefetchFixtureOver(t *testing.T, prefetch bool, source []runtime.Object, - extra ...client.Object) *dispatcherFixture { - t.Helper() - f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, source...) - setTriggerSource(t, f, "checkout", prefetch, nil) - for _, obj := range extra { - require.NoError(t, f.control.Create(context.Background(), obj)) - } - return f -} - -// setTriggerSource writes a FluxTrigger's published source, and its operation. -func setTriggerSource(t *testing.T, f *dispatcherFixture, name string, prefetch bool, - operation *v1alpha3.ApplyOperation) { - t.Helper() - trigger := &v1alpha3.FluxTrigger{} - require.NoError(t, f.control.Get(context.Background(), client.ObjectKey{Namespace: targetNS, Name: name}, trigger)) - trigger.Status.Source = &v1alpha3.FluxTriggerSource{ - GitRepository: fluxmeta.NamespacedObjectReference{Namespace: fluxSystemNS, Name: "platform"}, - URL: testRepoURL, Branch: testBranch, Prefetch: prefetch, - } - trigger.Status.Operation = operation - require.NoError(t, f.control.Update(context.Background(), trigger)) -} - -func prefetchRequest(commit string) git.PrefetchRequest { - return git.PrefetchRequest{Applier: checkout, Commit: commit} -} - -// TestDispatcher_PrefetchFetchesTheSourceOnlyWhenItMay: the source is stamped when every consumer is -// held and nothing on it is in flight, and left alone otherwise. -func TestDispatcher_PrefetchFetchesTheSourceOnlyWhenItMay(t *testing.T) { - f := prefetchFixture(t, true) - require.NoError(t, f.dispatcher.prefetch(context.Background(), prefetchRequest(testCommit))) - assert.Equal(t, []string{"gitrepositories " + fluxSystemNS}, f.patches(), "fetched; nothing applied") - - held := prefetchFixture(t, false) - require.ErrorIs(t, held.dispatcher.prefetch(context.Background(), prefetchRequest(testCommit)), errPrefetchHeld, - "a source another consumer applies on its own is never prefetched") - assert.Empty(t, held.patches()) - - busy := prefetchFixture(t, true, &v1alpha3.FluxTrigger{ - ObjectMeta: metav1.ObjectMeta{Namespace: "team-b", Name: "infra"}, - Spec: v1alpha3.FluxTriggerSpec{ - ClusterProviderRef: &fluxmeta.LocalObjectReference{Name: v1alpha3.DefaultClusterProviderName}, - KustomizationRef: fluxmeta.NamespacedObjectReference{Namespace: fluxSystemNS, Name: "infra"}, - }, - Status: v1alpha3.FluxTriggerStatus{ - Source: &v1alpha3.FluxTriggerSource{ - GitRepository: fluxmeta.NamespacedObjectReference{Namespace: fluxSystemNS, Name: "platform"}, - }, - Operation: &v1alpha3.ApplyOperation{Phase: v1alpha3.ApplyOperationRunning, Token: "T1"}, - }, - }) - require.ErrorIs(t, busy.dispatcher.prefetch(context.Background(), prefetchRequest(testCommit)), errPrefetchHeld, - "not while a consumer of the source is resumed") - assert.Empty(t, busy.patches()) - - current := prefetchFixture(t, true) - require.ErrorIs(t, current.dispatcher.prefetch(context.Background(), prefetchRequest(olderCommit)), - errNotPrefetched, "the artifact already holds it") - assert.Empty(t, current.patches()) -} - -// heldSource is behindSource with flux-system/checkout held by GitOps Reverser, and extra. -func heldSource(t *testing.T, extra ...runtime.Object) []runtime.Object { - t.Helper() - source := behindSource() - source[0] = fluxsim.OwnSuspend(t, source[0].(*unstructured.Unstructured), true, SuspendFieldManager, - metav1.ManagedFieldsOperationApply) - return append(source, extra...) -} - -// TestDispatcher_PrefetchChecksTheConsumersAsTheyAreNow is the fourth review's finding C: the -// trigger's status.source.prefetch is up to a minute old, so the consumers are read again before -// the source is stamped. A consumer that appeared since, or one Flux applies on its own, would apply -// the prefetched artifact over live edits not pushed yet. -func TestDispatcher_PrefetchChecksTheConsumersAsTheyAreNow(t *testing.T) { - infraTrigger := func() *v1alpha3.FluxTrigger { - return &v1alpha3.FluxTrigger{ - ObjectMeta: metav1.ObjectMeta{Namespace: targetNS, Name: "infra"}, - Spec: v1alpha3.FluxTriggerSpec{ - ClusterProviderRef: &fluxmeta.LocalObjectReference{Name: v1alpha3.DefaultClusterProviderName}, - KustomizationRef: fluxmeta.NamespacedObjectReference{Namespace: fluxSystemNS, Name: "infra"}, - }, - } - } - infra := func(held bool) *unstructured.Unstructured { - ks := kustomizationObj(fluxSystemNS, "infra", gitRepoSourceRef("platform")) - if held { - return fluxsim.OwnSuspend(t, ks, true, SuspendFieldManager, metav1.ManagedFieldsOperationApply) - } - return ks - } - resumed := heldSource(t) - require.NoError(t, unstructured.SetNestedField(resumed[0].(*unstructured.Unstructured).Object, false, - "spec", "suspend")) - - for name, f := range map[string]*dispatcherFixture{ - "a Kustomization no trigger names": prefetchFixtureOver(t, true, heldSource(t, infra(true))), - "a Kustomization Flux applies on its own": prefetchFixtureOver(t, true, heldSource(t, infra(false)), - infraTrigger()), - "a HelmRelease": prefetchFixtureOver(t, true, - heldSource(t, helmReleaseObj("apps", "podinfo", fluxSystemNS))), - "its own Kustomization, resumed": prefetchFixtureOver(t, true, resumed), - } { - t.Run(name, func(t *testing.T) { - require.ErrorIs(t, f.dispatcher.prefetch(context.Background(), prefetchRequest(testCommit)), - errPrefetchHeld) - assert.Empty(t, f.patches(), "nothing is fetched") - }) - } - - f := prefetchFixtureOver(t, true, heldSource(t, infra(true)), infraTrigger()) - require.NoError(t, f.dispatcher.prefetch(context.Background(), prefetchRequest(testCommit))) - assert.Equal(t, []string{"gitrepositories " + fluxSystemNS}, f.patches(), "every consumer is held") -} - -// TestDispatcher_PrefetchStampsOnlyTheSourceItChecked: the trigger's status.source is up to a minute -// old, and the Kustomization was repointed since, to a second GitRepository over the same repository -// and branch, which a Kustomization Flux applies on its own consumes too. The checks pass for the old -// source, which nothing consumes now, so stamping the Kustomization's new one would have that consumer -// apply the commit at once, over live edits not pushed yet. -func TestDispatcher_PrefetchStampsOnlyTheSourceItChecked(t *testing.T) { - source := heldSource(t, - gitRepositoryObj(fluxSystemNS, "platform-next", testRepoURL, branchRef(testBranch), "main@sha1:"+olderCommit), - kustomizationObj(fluxSystemNS, "billing", gitRepoSourceRef("platform-next"))) - require.NoError(t, unstructured.SetNestedStringMap(source[0].(*unstructured.Unstructured).Object, - map[string]string{"kind": kindGitRepository, "name": "platform-next"}, "spec", "sourceRef")) - f := prefetchFixtureOver(t, true, source) - - require.ErrorIs(t, f.dispatcher.prefetch(context.Background(), prefetchRequest(testCommit)), errPrefetchHeld) - assert.Empty(t, f.patches(), "neither source is fetched") -} - var _ runtime.Object = (*unstructured.Unstructured)(nil) diff --git a/internal/applier/flux_guide_test.go b/internal/applier/flux_guide_test.go index 420cd64e..919d8f9c 100644 --- a/internal/applier/flux_guide_test.go +++ b/internal/applier/flux_guide_test.go @@ -67,7 +67,6 @@ func TestFluxGuide_TheCompleteExampleIsAccepted(t *testing.T) { *target.Spec.ApplyTriggerRef) assert.Equal(t, target.Namespace, trigger.Namespace, "a GitTarget names a trigger in its own namespace") assert.Equal(t, v1alpha3.RefusalActionReapply, target.Spec.OnRefusal) - assert.LessOrEqual(t, trigger.EffectiveQuietPeriod(), time.Hour) // The Flux objects follow the guide's advice. interval, _, _ := unstructured.NestedString(repository.Object, "spec", "interval") diff --git a/internal/applier/flux_lifecycle_test.go b/internal/applier/flux_lifecycle_test.go index 671f3144..dc17f87a 100644 --- a/internal/applier/flux_lifecycle_test.go +++ b/internal/applier/flux_lifecycle_test.go @@ -181,11 +181,6 @@ func TestFluxRequest_Outcomes(t *testing.T) { commit: testCommit, reason: v1alpha3.ApplyReasonForeignCommit, want: OutcomeResumed, wantFetch: true, }, - { - name: "a branch absent on the remote fetches, and resumes nothing", artifact: "main@sha1:" + testCommit, - suspended: true, commit: "", reason: v1alpha3.ApplyReasonForeignCommit, - want: OutcomeCurrent, wantFetch: true, - }, { name: "a refusal at the applied commit resumes", artifact: "main@sha1:" + testCommit, applied: "main@sha1:" + testCommit, suspended: true, @@ -197,14 +192,16 @@ func TestFluxRequest_Outcomes(t *testing.T) { commit: testCommit, reason: v1alpha3.ApplyReasonRepair, want: OutcomeResumed, }, { - name: "a foreign commit applied while suspended asks nothing", artifact: "main/" + testCommit, + // The worker decides what needs no apply; the client applies what it is asked. An adopt + // re-applies the same commit on purpose: that is what stamps the owner labels. + name: "a foreign commit applied while suspended resumes", artifact: "main/" + testCommit, applied: "main@sha1:" + testCommit, suspended: true, - commit: testCommit, reason: v1alpha3.ApplyReasonForeignCommit, want: OutcomeCurrent, + commit: testCommit, reason: v1alpha3.ApplyReasonForeignCommit, want: OutcomeResumed, }, { - name: "a catch-up applied while suspended asks nothing", artifact: "sha1:" + testCommit, + name: "a catch-up applied while suspended resumes", artifact: "sha1:" + testCommit, applied: "main@sha1:" + testCommit, suspended: true, - commit: testCommit, reason: v1alpha3.ApplyReasonCatchUp, want: OutcomeCurrent, + commit: testCommit, reason: v1alpha3.ApplyReasonCatchUp, want: OutcomeResumed, }, { // Not suspended yet (the trigger has not adopted it): it may apply on its own, so the @@ -464,6 +461,9 @@ func TestFluxRequest_AFetchThatEndsElsewhere(t *testing.T) { require.ErrorContains(t, err, "fetched main@sha1:"+olderCommit+", not commit "+testCommit) assert.False(t, isPermanent(err)) require.NotErrorIs(t, err, errStale) + var elsewhere *sourceElsewhereError + require.ErrorAs(t, err, &elsewhere) + assert.Equal(t, "main@sha1:"+olderCommit, elsewhere.revision, "the revision the source holds") a := ask(v1alpha3.ApplyReasonForeignCommit) a.Current = func() bool { return false } diff --git a/internal/controller/applytrigger_controller.go b/internal/controller/applytrigger_controller.go index 4fddbac5..51cde223 100644 --- a/internal/controller/applytrigger_controller.go +++ b/internal/controller/applytrigger_controller.go @@ -223,7 +223,7 @@ func (r *ApplyTriggerReconciler) commitTrigger( // holdFlux returns a FluxTrigger's Held condition. A Configured trigger holds its Kustomization, // with its finalizer put in place before the first adoption write, and then publishes its source -// consumers: prefetch also needs the Kustomization held. One that is not Configured holds nothing, +// consumers. One that is not Configured holds nothing, // and one in line behind another holder lets go of a finalizer it carries, handing nothing back. func (r *ApplyTriggerReconciler) holdFlux( ctx context.Context, st *reconcileStatus, trigger *configbutleraiv1alpha3.FluxTrigger, check triggerCheck, @@ -237,7 +237,7 @@ func (r *ApplyTriggerReconciler) holdFlux( } held := r.holdKustomization(ctx, trigger) if check.checked { - r.publishConsumers(ctx, trigger, held) + r.publishConsumers(ctx, trigger) } return held, nil case configured.Reason == applier.ReasonNotReferenced: diff --git a/internal/controller/applytrigger_controller_test.go b/internal/controller/applytrigger_controller_test.go index 47713fe2..504f6d4b 100644 --- a/internal/controller/applytrigger_controller_test.go +++ b/internal/controller/applytrigger_controller_test.go @@ -288,7 +288,7 @@ func TestApplyTriggerReconcile_FluxVerdicts(t *testing.T) { allowed: true, wantStatus: metav1.ConditionTrue, wantReason: ReasonSucceeded, wantSource: &configbutleraiv1alpha3.FluxTriggerSource{ GitRepository: meta.NamespacedObjectReference{Namespace: "flux-system", Name: "platform"}, - URL: "https://github.com/acme/platform", Branch: "main", Prefetch: true, + URL: "https://github.com/acme/platform", Branch: "main", }}, {name: "no such Kustomization, which may be on its way", allowed: true, wantStatus: metav1.ConditionFalse, wantReason: applier.ReasonApplierNotFound}, diff --git a/internal/controller/applytrigger_schema_test.go b/internal/controller/applytrigger_schema_test.go index 976b9214..70b083c7 100644 --- a/internal/controller/applytrigger_schema_test.go +++ b/internal/controller/applytrigger_schema_test.go @@ -18,7 +18,7 @@ import ( ) // The API server holds the trigger kinds to their contract: each names one applier and its -// cluster, neither can be pointed elsewhere once created, quietPeriod is bounded and defaulted, and +// cluster, neither can be pointed elsewhere once created, and // a GitTarget names a trigger by a kind this release serves. var _ = Describe("Apply trigger schema", func() { ctx := context.Background() @@ -48,17 +48,6 @@ var _ = Describe("Apply trigger schema", func() { } } - It("defaults a FluxTrigger's quiet period, and bounds it", func() { - trigger := flux("flux-default") - Expect(create(trigger)).To(Succeed()) - Expect(trigger.Spec.QuietPeriod).NotTo(BeNil()) - Expect(trigger.Spec.QuietPeriod.Duration).To(Equal(configbutleraiv1alpha3.DefaultApplierQuietPeriod)) - - long := flux("flux-long") - long.Spec.QuietPeriod = &metav1.Duration{Duration: 2 * time.Hour} - Expect(create(long)).To(MatchError(ContainSubstring("spec.quietPeriod must be a Go duration of at most 1h"))) - }) - It("defaults the cluster to the default ClusterProvider, as a GitTarget does, and requires names", func() { omitted := argo("argo-default-cluster") omitted.Spec.ClusterProviderRef = nil @@ -116,9 +105,6 @@ var _ = Describe("Apply trigger schema", func() { To(MatchError(ContainSubstring("spec.kustomizationRef is immutable"))) Expect(update(fluxTrigger, func() { fluxTrigger.Spec.ClusterProviderRef.Name = "other" })). To(MatchError(ContainSubstring("spec.clusterProviderRef is immutable"))) - Expect(update(fluxTrigger, func() { - fluxTrigger.Spec.QuietPeriod = &metav1.Duration{Duration: time.Minute} - })).To(Succeed(), "the quiet period is not what it holds") argoTrigger := argo("argo-immutable") Expect(create(argoTrigger)).To(Succeed()) diff --git a/internal/controller/constants.go b/internal/controller/constants.go index ae80699a..1b3168f1 100644 --- a/internal/controller/constants.go +++ b/internal/controller/constants.go @@ -78,10 +78,9 @@ const ( // // It is deliberately distinct from GitTargetReady, which stays the health of the referenced // GitTarget and must never be reused for source authorization. Its three values are not - // interchangeable: False is a refusal (terminal, Stalled=True), while Unknown covers both "the - // answer is still being established" and "a rule with an already-resolved scope has lost the - // ability to re-evaluate its policy and is retaining that scope" — neither of which may be - // rendered as a permanent failure. + // interchangeable: False is a refusal (terminal, Stalled=True), while Unknown means the answer is + // still being established (the rule is blocked by validation), which must not be rendered as a + // permanent failure. ConditionTypeSourceNamespaceAuthorized = "SourceNamespaceAuthorized" // ConditionTypeAuthorAttributed indicates whether a CommitRequest's commit author // was named from the submitter captured at admission. It is binary and immediately @@ -213,7 +212,7 @@ const ( // this project already depends on. Sharing the vocabulary is the point: one alerting rule // written against reason=Succeeded works across every kind here AND across every Flux kind in // the same cluster, which is not true of a per-project spelling. Domain - // reasons (UnsupportedContent, WriteBoundaryRefused, NoAdmittedSourceNamespaces, ...) stay ours: + // reasons (UnsupportedContent, WriteBoundaryRefused, SourceMismatch, ...) stay ours: // those carry information a generic reason cannot, and declaring them is exactly what the // upstream vocabulary asks projects to do. diff --git a/internal/controller/fluxtrigger_hold.go b/internal/controller/fluxtrigger_hold.go index 4e4b6514..147e5ae0 100644 --- a/internal/controller/fluxtrigger_hold.go +++ b/internal/controller/fluxtrigger_hold.go @@ -667,19 +667,16 @@ func (r *ApplyTriggerReconciler) event(object client.Object, eventType, reason, } } -// publishConsumers sets the trigger's status.source.otherConsumers and status.source.prefetch: the -// consumers of its GitRepository in the applier's cluster, other than its own Kustomization, that no -// FluxTrigger on the same ClusterProvider holds. Prefetch also needs its own Kustomization held, -// which held, this pass's Held, says. A listing that fails publishes none and no prefetch, -// because the precondition cannot be checked. Information only (decision 2): it never affects Ready. -func (r *ApplyTriggerReconciler) publishConsumers( - ctx context.Context, trigger *configbutleraiv1alpha3.FluxTrigger, held conditionValue, -) { +// publishConsumers sets the trigger's status.source.otherConsumers: the consumers of its GitRepository +// in the applier's cluster, other than its own Kustomization, that no FluxTrigger on the same +// ClusterProvider holds. A listing that fails publishes none. Information only (decision 2): it never +// affects Ready. +func (r *ApplyTriggerReconciler) publishConsumers(ctx context.Context, trigger *configbutleraiv1alpha3.FluxTrigger) { source := trigger.Status.Source if source == nil || r.ApplierClients == nil { return } - source.OtherConsumers, source.Prefetch = nil, false + source.OtherConsumers = nil ctx, cancel := context.WithTimeout(ctx, applierCheckTimeout) defer cancel() flux, err := r.fluxClient(ctx, trigger) @@ -690,7 +687,7 @@ func (r *ApplyTriggerReconciler) publishConsumers( Namespace: source.GitRepository.Namespace, Name: source.GitRepository.Name, }) if err != nil { - logf.FromContext(ctx).V(1).Info("cannot list the GitRepository's consumers; not prefetching it", + logf.FromContext(ctx).V(1).Info("cannot list the GitRepository's consumers", "gitRepository", source.GitRepository.Namespace+"/"+source.GitRepository.Name, "error", err.Error()) return } @@ -706,7 +703,6 @@ func (r *ApplyTriggerReconciler) publishConsumers( } source.OtherConsumers = append(source.OtherConsumers, consumer) } - source.Prefetch = len(source.OtherConsumers) == 0 && held.Status == metav1.ConditionTrue } // heldKustomizations is every Kustomization a FluxTrigger on the ClusterProvider holds: one whose diff --git a/internal/controller/fluxtrigger_hold_test.go b/internal/controller/fluxtrigger_hold_test.go index 196f32eb..02d9d31f 100644 --- a/internal/controller/fluxtrigger_hold_test.go +++ b/internal/controller/fluxtrigger_hold_test.go @@ -457,8 +457,7 @@ func kustomizationManagers(ks *unstructured.Unstructured) []string { } // TestFluxTrigger_PublishesTheSourcesOtherConsumers is decision 2: the consumers of the -// GitRepository that no FluxTrigger holds are listed, information only, and the source is prefetched -// only when there are none. +// GitRepository that no FluxTrigger holds are listed, information only. func TestFluxTrigger_PublishesTheSourcesOtherConsumers(t *testing.T) { objs := applierFluxObjects() infra := objs[0].(*unstructured.Unstructured).DeepCopy() @@ -480,7 +479,6 @@ func TestFluxTrigger_PublishesTheSourcesOtherConsumers(t *testing.T) { {Kind: "HelmRelease", Namespace: "apps", Name: "podinfo"}, {Kind: "Kustomization", Namespace: "flux-system", Name: "infra"}, }, source.OtherConsumers, "itself is not another consumer") - assert.False(t, source.Prefetch) assert.True(t, conditionIsTrue(f.trigger(t).Status.Conditions, ConditionTypeReady), "information only") // The fourth review's finding C: a trigger holds its Kustomization only once it says so @@ -517,15 +515,7 @@ func TestFluxTrigger_PublishesTheSourcesOtherConsumers(t *testing.T) { }).Namespace("apps").Delete(context.Background(), "podinfo", metav1.DeleteOptions{})) _, err = f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) require.NoError(t, err) - got := f.trigger(t).Status.Source - assert.Empty(t, got.OtherConsumers) - assert.True(t, got.Prefetch, "every consumer is held: the source is prefetched") - - trigger := f.trigger(t) - f.r.publishConsumers(context.Background(), trigger, - conditionValue{Status: metav1.ConditionFalse, Reason: ReasonSuspending}) - assert.Empty(t, trigger.Status.Source.OtherConsumers) - assert.False(t, trigger.Status.Source.Prefetch, "nor while its own Kustomization is not held yet") + assert.Empty(t, f.trigger(t).Status.Source.OtherConsumers, "every consumer is held") } // setTriggerCondition publishes a condition on the FluxTrigger team-a/name. diff --git a/internal/git/acceptance_gate_test.go b/internal/git/acceptance_gate_test.go index 32693cd0..fa2631ea 100644 --- a/internal/git/acceptance_gate_test.go +++ b/internal/git/acceptance_gate_test.go @@ -138,7 +138,7 @@ func TestResyncRefusal_DoesNotStageSOPSBootstrap(t *testing.T) { }, } - _, err = w.executeResyncPendingWrite(context.Background(), repo, worktree, pending) + _, err = w.executeResyncPendingWrite(context.Background(), repo, worktree, &pending) var refused *manifestanalyzer.AcceptanceRefusedError require.ErrorAs(t, err, &refused) diff --git a/internal/git/ado_multiack_test.go b/internal/git/ado_multiack_test.go index b2ef60f0..ee65f729 100644 --- a/internal/git/ado_multiack_test.go +++ b/internal/git/ado_multiack_test.go @@ -57,7 +57,7 @@ type adoSimulator struct { // ledger tallies every request the backend sees, with bytes, so an operation's cost in // round trips to the Git host can be measured rather than guessed. See - // git_request_ledger_test.go and docs/design/push-notification-and-reconcile-trigger.md §1.6. + // git_request_ledger_test.go and docs/design/push-notification-and-reconcile-trigger.md, Part 1. ledger *gitRequestLedger // around, when set, serves each request in place of the backend and decides whether and how diff --git a/internal/git/applier_echoes_test.go b/internal/git/applier_echoes_test.go new file mode 100644 index 00000000..dc8eb787 --- /dev/null +++ b/internal/git/applier_echoes_test.go @@ -0,0 +1,488 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/client" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// The tests here pin docs/design/applier-echoes.md below end to end. The Git ones run on a real Git +// server (startRealGitServer): file:// never compares the tip a push expects, so a test of what a +// rejected push replays would pass without one ever being rejected. + +const echoTarget = "team-a" + +// echoFixture is a real Git server whose main already holds team-a's folder, a warm worker for main, +// and its event loop. pushFromOutside moves main behind the worker's back, as a person would. +type echoFixture struct { + *ledgerFixture + + loop *branchWorkerEventLoop + // clone is the outside clone pushFromOutside commits from. + clone string +} + +func newEchoFixture(t *testing.T, slug string, prune configv1alpha3.PruneMode, docs map[string]string) *echoFixture { + t.Helper() + f := newLedgerFixture(t, slug, true) + f.worker.mapper = configMapMapper() + createGitTargetWithPruneMode(t, f.worker, echoTarget, echoTarget, prune) + clone := filepath.Join(t.TempDir(), "outside") + gitIn(t, filepath.Dir(clone), "clone", "--quiet", f.repoDir, clone) + gitIn(t, clone, "config", "user.name", "Outside") + gitIn(t, clone, "config", "user.email", "outside@example.com") + e := &echoFixture{ledgerFixture: f, clone: clone} + for name, value := range docs { + e.pushFromOutside(name, value) + } + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + e.loop = newBranchWorkerEventLoop(f.worker, 0) + t.Cleanup(e.loop.stopTimers) + return e +} + +// echoFile is where the worker keeps the ConfigMap name of team-a. +func echoFile(name string) string { return echoTarget + "/default/configmaps/" + name + ".yaml" } + +// echoDocument is the document the worker writes for echoEvent(name, value). +func echoDocument(name, value string) string { + return "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: " + name + + "\n namespace: default\ndata:\n key: " + value + "\n" +} + +// echoEvent is a live update of the ConfigMap name to value, by alice, for team-a. +func echoEvent(name, value string) Event { + event := configMapTargetEvent(name, "alice", echoTarget) + event.Operation = string(itypes.OperationUpdate) + event.Object.Object["data"] = map[string]interface{}{"key": value} + return event +} + +// echoDesired is a snapshot entry of the ConfigMap name holding value. +func echoDesired(name, value string) manifestanalyzer.DesiredResource { + return manifestanalyzer.DesiredResource{ + Resource: echoEvent(name, value).Identifier, + Object: echoEvent(name, value).Object, + } +} + +// pushFromOutside commits one document to main from another clone, and returns the commit. +func (f *echoFixture) pushFromOutside(name, value string) string { + f.t.Helper() + gitIn(f.t, f.clone, "pull", "--quiet", "--ff-only", "origin", "main") + full := filepath.Join(f.clone, echoFile(name)) + require.NoError(f.t, os.MkdirAll(filepath.Dir(full), 0o750)) + require.NoError(f.t, os.WriteFile(full, []byte(echoDocument(name, value)), 0o600)) + gitIn(f.t, f.clone, "add", echoFile(name)) + gitIn(f.t, f.clone, "commit", "--quiet", "-m", "outside: "+name+"="+value) + gitIn(f.t, f.clone, "push", "--quiet", "origin", "HEAD:main") + return gitIn(f.t, f.clone, "rev-parse", "HEAD") +} + +// onMain is the document main holds at file, or "" when it holds none. +func (f *echoFixture) onMain(file string) string { + f.t.Helper() + listed := gitIn(f.t, f.repoDir, "ls-tree", "--name-only", "-r", "main") + for _, path := range splitLines(listed) { + if path == file { + return gitIn(f.t, f.repoDir, "show", "main:"+file) + "\n" + } + } + return "" +} + +// changedSince is the documents main changed after commit: the folder's bootstrap files, which a +// first write stages, are left out. +func (f *echoFixture) changedSince(commit string) []string { + f.t.Helper() + var documents []string + for _, file := range splitLines(gitIn(f.t, f.repoDir, "diff", "--name-only", commit, "main")) { + if strings.HasPrefix(file, echoTarget+"/default/") { + documents = append(documents, file) + } + } + return documents +} + +func splitLines(s string) []string { + var lines []string + start := 0 + for i := range len(s) { + if s[i] == '\n' { + lines = append(lines, s[start:i]) + start = i + 1 + } + } + if start < len(s) { + lines = append(lines, s[start:]) + } + return lines +} + +// writeWindow sends one window of live writes, closed at once (the loop's window is zero). +func (f *echoFixture) writeWindow(events ...Event) { + f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: events, CommitMode: CommitModePerEvent}}) +} + +// TestApplierEchoes_AnUnchangedObjectIsNoAction is F2 for a window, without timing luck: the write +// is decided inside the push cooldown, the remote moves, and the push is rejected and replayed. An +// object whose plan changed nothing is no action, so the replay never plans it onto the newer tip, +// where it would revert what that tip changed. +func TestApplierEchoes_AnUnchangedObjectIsNoAction(t *testing.T) { + t.Run("alone, it is never retained", func(t *testing.T) { + f := newEchoFixture(t, "echo-unchanged-alone", configv1alpha3.PruneOnEvent, map[string]string{"x": "x1"}) + f.loop.lastPushAt = time.Now() + + f.writeWindow(echoEvent("x", "x1")) // the applier's echo of what Git holds + require.Empty(t, f.loop.pendingWrites, "a write that changes nothing settles at once") + + outside := f.pushFromOutside("x", "x2") + f.loop.pushPending() + + assert.Equal(t, echoDocument("x", "x2"), f.onMain(echoFile("x")), "the Git-side push survives") + assert.Empty(t, f.changedSince(outside), "and nothing was committed over it") + }) + + t.Run("beside a changed object, only the changed one is committed", func(t *testing.T) { + f := newEchoFixture(t, "echo-unchanged-batch", configv1alpha3.PruneOnEvent, + map[string]string{"x": "x1", "y": "y1"}) + f.loop.lastPushAt = time.Now() + + f.writeWindow(echoEvent("x", "x1"), echoEvent("y", "y2")) + require.Len(t, f.loop.pendingWrites, 1) + require.Len(t, f.loop.pendingWrites[0].Events, 1, "the write keeps the object that changed") + assert.Equal(t, "y", f.loop.pendingWrites[0].Events[0].Identifier.Name) + + outside := f.pushFromOutside("x", "x2") + f.loop.pushPending() + require.Empty(t, f.loop.pendingWrites, "published after the replay") + + assert.Equal(t, echoDocument("x", "x2"), f.onMain(echoFile("x")), "the Git-side push survives the replay") + assert.Equal(t, echoDocument("y", "y2"), f.onMain(echoFile("y"))) + assert.Equal(t, []string{echoFile("y")}, f.changedSince(outside)) + }) + + t.Run("a save keeps every object it captured", func(t *testing.T) { + f := newEchoFixture(t, "echo-unchanged-save", configv1alpha3.PruneOnEvent, map[string]string{"x": "x1"}) + f.loop.lastPushAt = time.Now() + save := attachReq("alice", time.Hour) + save.GitTargetName = echoTarget + save.MaxDuration = 0 + serviceAttach(f.loop, save) + + f.writeWindow(echoEvent("x", "x1")) + require.Len(t, f.loop.pendingWrites, 1, "a save is never trimmed") + require.Len(t, f.loop.pendingWrites[0].Events, 1) + + f.pushFromOutside("x", "x2") + f.loop.pushPending() + + res, resolved := outcome(t, f.worker) + require.True(t, resolved) + assert.Equal(t, FinalizeCommitted, res.Outcome, "the replay re-asserted the save over the push") + assert.Equal(t, echoDocument("x", "x1"), f.onMain(echoFile("x")), "API wins for a save (C2)") + }) +} + +// TestApplierEchoes_AResyncReplayAppliesItsDecidedActions is F2 and P3 for a resync, under +// prune.mode: Always: the snapshot decides once, on its tree, which objects to upsert and which +// documents to sweep, and a replay applies exactly those. A document a newer tip added survives, +// and an object the snapshot found unchanged is left as the newer tip has it. +func TestApplierEchoes_AResyncReplayAppliesItsDecidedActions(t *testing.T) { + f := newEchoFixture(t, "echo-resync-replay", configv1alpha3.PruneAlways, + map[string]string{"x": "x1", "y": "y1", "gone": "g1"}) + f.loop.lastPushAt = time.Now() + + req := &ResyncRequest{ + Desired: []manifestanalyzer.DesiredResource{echoDesired("x", "x1"), echoDesired("y", "y2")}, + ResourceVersion: "7", + GitTargetName: echoTarget, + GitTargetNamespace: "default", + Result: make(chan ResyncResult, 1), + } + f.loop.handleQueueItem(resyncItem(req)) + result := <-req.Result + require.NoError(t, result.Err) + assert.Equal(t, 1, result.Stats.Updated) + assert.Equal(t, 1, result.Stats.Deleted, "the snapshot's sweep removed the document the cluster lacks") + require.Len(t, f.loop.pendingWrites, 1) + actions := f.loop.pendingWrites[0].resyncActions + require.NotNil(t, actions) + require.Len(t, actions.upserts, 1, "the unchanged object is no action") + assert.Equal(t, "y", actions.upserts[0].Resource.Name) + require.Len(t, actions.deletes, 1) + assert.Equal(t, "gone", actions.deletes[0].resource.Name) + + f.pushFromOutside("z", "z1") + outside := f.pushFromOutside("x", "x2") + f.loop.pushPending() + require.Empty(t, f.loop.pendingWrites, "published after the replay") + + assert.Equal(t, echoDocument("z", "z1"), f.onMain(echoFile("z")), "a Git-side addition survives the replay") + assert.Equal(t, echoDocument("x", "x2"), f.onMain(echoFile("x")), "an object the snapshot left alone stays") + assert.Equal(t, echoDocument("y", "y2"), f.onMain(echoFile("y"))) + assert.Empty(t, f.onMain(echoFile("gone")), "the decided delete still lands") + assert.ElementsMatch(t, []string{echoFile("y"), echoFile("gone")}, f.changedSince(outside)) +} + +// TestApplierEchoes_AReplayedResyncObeysATightenedPruneMode pins that a replayed sweep delete is +// gated by the policy in force at the replay: an operator who stopped pruning since the resync was +// decided is obeyed. +func TestApplierEchoes_AReplayedResyncObeysATightenedPruneMode(t *testing.T) { + f := newEchoFixture(t, "echo-resync-tightened", configv1alpha3.PruneAlways, + map[string]string{"y": "y1", "gone": "g1"}) + f.loop.lastPushAt = time.Now() + + req := &ResyncRequest{ + Desired: []manifestanalyzer.DesiredResource{echoDesired("y", "y2")}, + GitTargetName: echoTarget, + GitTargetNamespace: "default", + Result: make(chan ResyncResult, 1), + } + f.loop.handleQueueItem(resyncItem(req)) + require.NoError(t, (<-req.Result).Err) + require.Len(t, f.loop.pendingWrites, 1) + + var target configv1alpha3.GitTarget + require.NoError(t, f.worker.Client.Get(f.worker.ctx, keyOf(echoTarget), &target)) + target.Spec.Prune = &configv1alpha3.PrunePolicy{Mode: configv1alpha3.PruneNever} + require.NoError(t, f.worker.Client.Update(f.worker.ctx, &target)) + + f.pushFromOutside("z", "z1") + f.loop.pushPending() + require.Empty(t, f.loop.pendingWrites) + + assert.Equal(t, echoDocument("gone", "g1"), f.onMain(echoFile("gone")), + "pruning stopped, so the delete is not replayed") + assert.Equal(t, echoDocument("y", "y2"), f.onMain(echoFile("y"))) +} + +// keyOf is the key of a GitTarget in the default namespace. +func keyOf(name string) client.ObjectKey { return client.ObjectKey{Namespace: "default", Name: name} } + +// withTrigger names a FluxTrigger on team-a and gives the worker a requester that reports nothing, +// so the test reports for it. The loop's commit window is an hour, so a window stays open until +// something closes it. +func (f *echoFixture) withTrigger() *recordingRequester { + f.t.Helper() + var target configv1alpha3.GitTarget + require.NoError(f.t, f.worker.Client.Get(f.worker.ctx, keyOf(echoTarget), &target)) + target.Spec.ApplyTriggerRef = testApplier(echoTarget) + require.NoError(f.t, f.worker.Client.Update(f.worker.ctx, &target)) + createTestTrigger(f.t, f.worker.Client, &target) + requester := &recordingRequester{manual: true} + f.worker.applyRequester = requester + f.loop.stopTimers() + f.loop = newBranchWorkerEventLoop(f.worker, time.Hour) + f.t.Cleanup(f.loop.stopTimers) + f.loop.followRecordedOperations(time.Now()) + // The converges the start and the naming left pending, answered already: these tests drive + // other reasons. + f.worker.inbox.take() + for _, state := range f.loop.appliers { + state.converge = false + } + return requester +} + +// echoApplier is the applier team-a's trigger holds. +func (f *echoFixture) echoApplier() *applierState { + f.t.Helper() + require.Len(f.t, f.loop.appliers, 1) + for _, state := range f.loop.appliers { + return state + } + return nil +} + +// TestApplierEchoes_FlushThenApply is S1, driven in the worker: when an apply comes due, the open +// window is closed and the writes up to the flush mark are published before the send, cooldown or +// not, so the apply holds them. An edit decided after the mark does not hold the send: it is recorded +// in the apply window, and the apply landing over it is withheld, with a Repair pending. +func TestApplierEchoes_FlushThenApply(t *testing.T) { + f := newEchoFixture(t, "echo-flush", configv1alpha3.PruneOnEvent, map[string]string{"x": "x1", "y": "y1"}) + requester := f.withTrigger() + f.loop.lastPushAt = time.Now() // the push cooldown runs: a flush does not wait for it + + f.writeWindow(echoEvent("x", "x2")) // E0, in the open window when the apply comes due + require.NotNil(t, f.loop.openWindow) + restore := failPushes(t, f.worker) + f.echoApplier().repair = true + f.loop.endWake(0) + + assert.Nil(t, f.loop.openWindow, "the apply closed the open window") + assert.Empty(t, requester.taken(), "the writes up to the mark are not published yet") + require.NotNil(t, f.echoApplier().flushEdits, "edits are recorded from the mark") + + f.writeWindow(echoEvent("y", "y2")) // E1, decided after the mark + require.NotNil(t, f.loop.openWindow, "it opens a window of its own") + + restore() + f.loop.retry.due = time.Now().Add(-time.Second) + f.loop.runRetry() + f.loop.endWake(0) + + sent := requester.taken() + require.Len(t, sent, 1, "the writes up to the mark are published, and E1's open window does not hold the send") + assert.Equal(t, configv1alpha3.ApplyReasonRepair, sent[0].Reason) + assert.Equal(t, echoDocument("x", "x2"), f.onMain(echoFile("x")), "the apply holds E0") + assert.Equal(t, gitIn(t, f.repoDir, "rev-parse", "main"), sent[0].Commit, "for the tip the flush produced") + assert.Equal(t, echoDocument("y", "y1"), f.onMain(echoFile("y")), "E1 is not published yet") + + // The apply writes the commit over E1, the applier's manager alone and newer than the edit. + late := echoEvent("y", "y1") + late.LastFieldManagers = []string{FluxFieldManager} + late.LastFieldManagedAt = time.Now().Add(time.Hour) + assert.True(t, f.loop.withholdLateApply(late), "E1 was recorded in the window: the apply over it is withheld") + assert.True(t, f.echoApplier().repair, "and a Repair puts E1 back once it is published") +} + +// TestApplierEchoes_AResyncThatAddsAnObjectAdoptsIt is S2: an object a resync adds to Git runs on +// the cluster already, and its applier never applied it, so a later Git-side delete would leave it +// there. It is adopted; a resync that only updates adopts nothing. +func TestApplierEchoes_AResyncThatAddsAnObjectAdoptsIt(t *testing.T) { + f := newEchoFixture(t, "echo-resync-adopt", configv1alpha3.PruneOnEvent, map[string]string{"x": "x1"}) + f.withTrigger() + + resync := func(desired ...manifestanalyzer.DesiredResource) { + req := &ResyncRequest{Desired: desired, GitTargetName: echoTarget, GitTargetNamespace: "default", + Result: make(chan ResyncResult, 1)} + f.loop.applyResync(req) + f.loop.advance() + require.NoError(t, (<-req.Result).Err) + } + resync(echoDesired("x", "x2")) + assert.False(t, f.echoApplier().adopt, "an update adopts nothing") + + resync(echoDesired("x", "x2"), echoDesired("new", "n1")) + assert.True(t, f.echoApplier().adopt, "the object the resync added is adopted") +} + +// TestApplierEchoes_ASuspendedResyncIsNotRetained: a suspended target's resync scans and writes +// nothing, so it decides nothing: it settles at once, and is neither retained nor pushed. +func TestApplierEchoes_ASuspendedResyncIsNotRetained(t *testing.T) { + f := newEchoFixture(t, "echo-resync-suspended", configv1alpha3.PruneOnEvent, map[string]string{"x": "x1"}) + var target configv1alpha3.GitTarget + require.NoError(t, f.worker.Client.Get(f.worker.ctx, keyOf(echoTarget), &target)) + target.Spec.Suspend = true + require.NoError(t, f.worker.Client.Update(f.worker.ctx, &target)) + + req := &ResyncRequest{Desired: []manifestanalyzer.DesiredResource{echoDesired("x", "x2")}, + GitTargetName: echoTarget, GitTargetNamespace: "default", Result: make(chan ResyncResult, 1)} + f.loop.applyResync(req) + f.loop.advance() + require.NoError(t, (<-req.Result).Err) + assert.Empty(t, f.loop.pendingWrites, "nothing to publish") + assert.True(t, f.loop.lastPushAt.IsZero(), "so nothing was pushed") +} + +// TestApplierEchoes_AnAppliedFolderIsCreatedByItsResync: a folder an applier applies must exist +// before the applier is asked to apply it, or Argo CD retries a sync of a missing path for minutes +// and Flux fails it. A resync of a GitTarget naming a trigger creates the folder with its bootstrap +// files, with no object to write; once it exists, an empty resync commits nothing. A GitTarget no +// applier applies keeps the old rule: an empty resync commits nothing. +func TestApplierEchoes_AnAppliedFolderIsCreatedByItsResync(t *testing.T) { + // Initial: the seed below is a foreign commit the resync's own fetch finds, which F3 discards a + // later snapshot for; the first one of a collection is the baseline, and is used. + emptyResync := func(f *echoFixture) { + t.Helper() + req := &ResyncRequest{GitTargetName: echoTarget, GitTargetNamespace: "default", Initial: true, + Result: make(chan ResyncResult, 1)} + f.loop.applyResync(req) + f.loop.advance() + require.NoError(t, (<-req.Result).Err) + } + seed := func(f *echoFixture) { + t.Helper() + require.NoError(t, os.WriteFile(filepath.Join(f.clone, "NOTES.md"), []byte("elsewhere\n"), 0o600)) + gitIn(t, f.clone, "add", "NOTES.md") + gitIn(t, f.clone, "commit", "--quiet", "-m", "outside: a file outside the folder") + gitIn(t, f.clone, "push", "--quiet", "origin", "HEAD:main") + } + main := func(f *echoFixture) string { return gitIn(t, f.repoDir, "rev-parse", "main") } + + applied := newEchoFixture(t, "echo-bootstrap-applied", configv1alpha3.PruneOnEvent, nil) + seed(applied) + applied.withTrigger() + emptyResync(applied) + assert.NotEmpty(t, applied.onMain(echoTarget+"/README.md"), "the folder exists, with its bootstrap files") + created := main(applied) + emptyResync(applied) + assert.Equal(t, created, main(applied), "a folder that exists needs no commit") + + plain := newEchoFixture(t, "echo-bootstrap-plain", configv1alpha3.PruneOnEvent, nil) + seed(plain) + before := main(plain) + emptyResync(plain) + assert.Equal(t, before, main(plain), "no applier applies it: an empty resync commits nothing") + assert.Empty(t, plain.onMain(echoTarget+"/README.md")) +} + +// siblingOf is a second worker on the fixture's repository and branch, under a GitProvider of its +// own, whose GitTarget team-b names team-a's trigger; both are registered as siblings. Its loop's +// commit window is an hour, so a window stays open until something closes it. It sends no apply +// itself (no requester), so what publishes its writes is the flush the fixture's worker asks of it. +func (f *echoFixture) siblingOf() *branchWorkerEventLoop { + f.t.Helper() + provider := &configv1alpha3.GitProvider{Spec: configv1alpha3.GitProviderSpec{URL: f.sim.RepoURL}} + provider.Name, provider.Namespace = f.worker.GitProviderRef+"-b", "default" + require.NoError(f.t, f.worker.Client.Create(f.worker.ctx, provider)) + worker := NewBranchWorker(f.worker.Client, logr.Discard(), provider.Name, "default", f.worker.Branch, + RepoIdentity{URL: f.sim.RepoURL}, nil, BranchWorkerLimits{}) + worker.ctx, worker.mapper = f.worker.ctx, configMapMapper() + f.t.Cleanup(func() { _ = os.RemoveAll(worker.repoRootPath()) }) + createPlainGitTarget(f.t, worker, "team-b", "team-b") + var target configv1alpha3.GitTarget + require.NoError(f.t, worker.Client.Get(worker.ctx, keyOf("team-b"), &target)) + target.Spec.ApplyTriggerRef = testApplier(echoTarget) + require.NoError(f.t, worker.Client.Update(worker.ctx, &target)) + require.NoError(f.t, worker.ensureRepositoryInitialized(worker.ctx)) + + siblings := newBranchSiblings() + siblings.add(f.worker) + siblings.add(worker) + loop := newBranchWorkerEventLoop(worker, time.Hour) + f.t.Cleanup(loop.stopTimers) + return loop +} + +// TestApplierEchoes_AnApplyWaitsForASiblingsEdit is S1 across the workers writing one branch: two +// GitProviders on one repository, whose GitTargets name one trigger. The sibling holds an edit in an +// open window when the apply comes due. Sent then, the apply would revert it on the cluster, and the +// sibling, with no apply window of its own, would mirror the revert. The worker asks the sibling to +// flush, and sends only once it has, for the tip that holds the edit. +func TestApplierEchoes_AnApplyWaitsForASiblingsEdit(t *testing.T) { + f := newEchoFixture(t, "echo-sibling", configv1alpha3.PruneOnEvent, map[string]string{"x": "x1"}) + requester := f.withTrigger() + sibling := f.siblingOf() + edit := configMapTargetEvent("z", "bob", "team-b") + sibling.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{edit}, CommitMode: CommitModePerEvent}}) + require.NotNil(t, sibling.openWindow, "the sibling's edit is in its open window") + + f.echoApplier().repair = true + f.loop.endWake(0) + assert.Empty(t, requester.taken(), "the sibling has not published its edit") + + sibling.endWake(0) + assert.Nil(t, sibling.openWindow, "the sibling closed its window for the flush") + assert.NotEmpty(t, f.onMain("team-b/default/configmaps/z.yaml"), "and published the edit at once") + + f.loop.endWake(0) + sent := requester.taken() + require.Len(t, sent, 1, "the sibling flushed: the apply goes") + assert.Equal(t, gitIn(t, f.repoDir, "rev-parse", "main"), sent[0].Commit, + "for the tip that holds the sibling's edit, which the sibling shared") +} diff --git a/internal/git/apply_attempt_test.go b/internal/git/apply_attempt_test.go index 8e07fc65..d14422f9 100644 --- a/internal/git/apply_attempt_test.go +++ b/internal/git/apply_attempt_test.go @@ -9,6 +9,7 @@ package git import ( "context" + "errors" "testing" "time" @@ -65,10 +66,9 @@ func addAlphaUnsentRefusal(loop *branchWorkerEventLoop) refusalKey { return key } -// TestApplier_ARetryIsDecidedAgainOnAQuietBranch: an attempt that failed is not replayed blind. It -// waits out its backoff, then waits for a quiet branch like a first attempt, and names the tip as -// it is then. -func TestApplier_ARetryIsDecidedAgainOnAQuietBranch(t *testing.T) { +// TestApplier_ARetryIsDecidedAgain: an attempt that failed is not replayed blind. It waits out its +// backoff, then is decided like a first attempt, after a flush, and names the tip as it is then. +func TestApplier_ARetryIsDecidedAgain(t *testing.T) { loop, requester := manualLoop(t, applierTestTarget("alpha", true)) observeForeignCommit(loop.w) loop.serviceAppliers() @@ -81,11 +81,11 @@ func TestApplier_ARetryIsDecidedAgainOnAQuietBranch(t *testing.T) { assert.NotNil(t, loop.applierTimer, "the backoff is armed") dueNow(loop) - loop.openWindow = &openWindow{} + holdApplies(loop) loop.serviceAppliers() - assert.Empty(t, requester.taken(), "not while the branch is busy") + assert.Empty(t, requester.taken(), "not while a save holds the flush") - loop.openWindow = nil + releaseApplies(loop) loop.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByPush}) loop.serviceAppliers() retry := requester.taken() @@ -99,11 +99,10 @@ func TestApplier_ARetryIsDecidedAgainOnAQuietBranch(t *testing.T) { // The requester used to turn a Refusal whose target withdrew consent into a CatchUp and send it at // once, skipping the quiet period, so a behind artifact was fetched over an unpublished edit. Now // the requester only reports that the reason no longer holds, and the worker decides: no Refusal, -// and a CatchUp only after the quiet period. +// only the converge the first naming left pending, as a CatchUp. func TestApplier_WithdrawnConsentNeverBecomesAnImmediateApply(t *testing.T) { - loop, requester := manualLoop(t, applierTestTarget("alpha", true, reapplying), - fluxTrigger("alpha", quietFor(time.Hour))) - loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) // leaves a CatchUp pending + loop, requester := manualLoop(t, applierTestTarget("alpha", true, reapplying)) + loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) // leaves a converge pending addAlphaUnsentRefusal(loop) loop.serviceAppliers() out := requester.taken() @@ -121,15 +120,18 @@ func TestApplier_WithdrawnConsentNeverBecomesAnImmediateApply(t *testing.T) { dueNow(loop) loop.serviceAppliers() - assert.Empty(t, requester.taken(), "the catch-up waits out the hour-long quiet period") - assert.Empty(t, loop.unsentRefusals, "and the Refusal is gone with the consent") - assert.True(t, loop.appliers[applierRef("alpha")].catchUp) + next := requester.taken() + require.Len(t, next, 1) + assert.Equal(t, configv1alpha3.ApplyReasonCatchUp, next[0].Reason, "the converge, never the Refusal") + assert.Empty(t, loop.unsentRefusals, "the Refusal is gone with the consent") } // TestApplier_ARecoveredRefusalIsNotRetried is the second review's second finding: a refusal that // recovered while its attempt was out came back with the failed attempt and was requested again. func TestApplier_ARecoveredRefusalIsNotRetried(t *testing.T) { - loop, requester := manualLoop(t, applierTestTarget("alpha", true, reapplying)) + loop, requester := manualLoop(t, applierTestTarget("alpha", true, reapplying), + fluxTrigger("alpha", recording(recordedOperation(configv1alpha3.ApplyOperationSucceeded), false))) + loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByFetch}) // the tip it applied key := addAlphaUnsentRefusal(loop) loop.serviceAppliers() out := requester.taken() @@ -202,23 +204,15 @@ func TestApplier_EachApplierRetriesOnItsOwnClock(t *testing.T) { } // TestApplier_OneAttemptForEveryTargetNamingIt: GitTargets that share a trigger are asked for -// together, each with its own reason, and wait for the trigger's quiet period, because its apply -// reaches every one of their folders. +// together, each with its own reason, because its apply reaches every one of their folders. func TestApplier_OneAttemptForEveryTargetNamingIt(t *testing.T) { shared := func(g *configv1alpha3.GitTarget) { g.Spec.ApplyTriggerRef = testApplier("shared") } loop, requester := applyTriggerLoop(t, applierTestTarget("alpha", true, shared, reapplying), applierTestTarget("bravo", true, shared), - fluxTrigger("shared", quietFor(time.Hour)), + fluxTrigger("shared"), ) - loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) - loop.serviceAppliers() - assert.Equal(t, time.Hour, loop.appliers[applierRef("shared")].quietPeriod()) - - loop.quietSince = time.Now().Add(-2 * time.Minute) - loop.serviceAppliers() - assert.Empty(t, requester.taken(), "past alpha's quiet period, not bravo's") - + loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) // the startup converge addAlphaUnsentRefusal(loop) loop.serviceAppliers() out := requester.taken() @@ -235,8 +229,8 @@ func TestApplier_OneAttemptForEveryTargetNamingIt(t *testing.T) { UID: "uid-bravo", Reason: configv1alpha3.ApplyReasonCatchUp, }, - }, out[0].Targets, "a Refusal applies the tip, so it answers bravo's pending CatchUp as well") - assert.False(t, loop.appliers[applierRef("shared")].catchUp) + }, out[0].Targets, "a Refusal applies the tip, so it answers the pending converge as well") + assert.False(t, loop.appliers[applierRef("shared")].converge) } // TestApplier_AnAttemptOutHoldsTheNext: one attempt per applier at a time. A reason that becomes pending @@ -280,74 +274,70 @@ func TestApplier_ATargetThatNoLongerNamesAnApplierHasNothingPending(t *testing.T assert.Nil(t, loop.applierTimer) } -// TestSiblings_ABusyWorkerOnTheSameBranchHoldsTheOther: two GitProviders naming one repository give -// one branch two workers, and a Kustomization applies the tip whichever of them wrote it. So one -// worker sends nothing while the other holds an open window, and is woken when it closes. -func TestSiblings_ABusyWorkerOnTheSameBranchHoldsTheOther(t *testing.T) { - loop, requester := applyTriggerLoop(t, applierTestTarget("alpha", true)) - loop.w.repo = RepoIdentity{URL: "https://github.com/acme/platform"} - loop.w.applyWake = make(chan struct{}, 1) - sibling := newMetricsTestWorker() - sibling.repo = RepoIdentity{URL: "git@github.com:acme/platform.git"} - sibling.applyWake = make(chan struct{}, 1) - other := newMetricsTestWorker() - other.repo = RepoIdentity{URL: "https://github.com/acme/elsewhere"} - siblings := newBranchSiblings() - for _, w := range []*BranchWorker{loop.w, sibling, other} { - siblings.add(w) - } - - other.publishWriteBusy(true) - sibling.publishWriteBusy(true) - observeForeignCommit(loop.w) - loop.serviceAppliers() - assert.Empty(t, requester.taken(), "the sibling's open window is on this branch too") - assert.True(t, loop.quietSince.IsZero()) +// answerFlush is a sibling's wake answering the flush another worker asked of it, with no writes of +// its own to publish beyond what the test put in its pending writes. +func answerFlush(loop *branchWorkerEventLoop) { + loop.takeSiblingFlush() + loop.reportSiblingFlush() +} - sibling.publishWriteBusy(false) +// TestSiblings_AnApplyWaitsForItsSiblingsFlush: two GitProviders naming one repository give one +// branch two workers, whose GitTargets can name one trigger. An apply sent while a sibling holds an +// edit not yet published would revert it on the cluster, and the sibling, with no apply window, would +// mirror the revert. So the flush covers the siblings: the worker waits until each has published the +// writes it decided before it was asked, and a write decided after does not hold the send. +func TestSiblings_AnApplyWaitsForItsSiblingsFlush(t *testing.T) { + first, firstRequests, second, _ := siblingLoops(t) + second.decisions = 1 + second.pendingWrites = []PendingWrite{{decision: 1}} + + first.serviceAppliers() + assert.Empty(t, firstRequests.taken(), "the sibling has not flushed yet") + require.Positive(t, second.w.flushAsked.Load(), "it was asked to") + + answerFlush(second) + assert.Equal(t, uint64(1), second.siblingFlushMark, "the mark is the last write it decided") + assert.True(t, second.publishesAtOnce(), "what the mark covers is pushed at once, cooldown or not") + first.serviceAppliers() + assert.Empty(t, firstRequests.taken(), "its write up to the mark is not published yet") + + second.decisions = 2 + second.pendingWrites = []PendingWrite{{decision: 2}} // published the first, decided another + second.reportSiblingFlush() select { - case <-loop.w.applyWake: + case <-first.w.applyWake: default: - t.Fatal("a sibling going quiet must wake the worker it held back") + t.Fatal("a sibling done flushing must wake the worker waiting for it") } - loop.serviceAppliers() - assert.Len(t, requester.taken(), 1, "a worker on another repository never held it") - - siblings.remove(sibling) - writing, working := loop.w.siblingBusy() - assert.False(t, writing) - assert.False(t, working) + first.serviceAppliers() + assert.Len(t, firstRequests.taken(), 1, "a write decided after the mark does not hold the send") } -// TestSiblings_AnItemOnASiblingHoldsTheAttemptButNotTheQuietPeriod: a sibling's queued or -// in-hand item holds the attempt back the way this worker's own queued work does. It keeps -// quietSince, because the branch is busy only once that work opens a window, and the sibling's -// loop wakes this one as it releases its last item. -func TestSiblings_AnItemOnASiblingHoldsTheAttemptButNotTheQuietPeriod(t *testing.T) { - loop, requester := applyTriggerLoop(t, applierTestTarget("alpha", true)) - sibling := newMetricsTestWorker() - siblings := newBranchSiblings() - for _, w := range []*BranchWorker{loop.w, sibling} { - w.repo = RepoIdentity{URL: "https://github.com/acme/platform"} - w.applyWake = make(chan struct{}, 1) - siblings.add(w) +// TestSiblings_ShareWhatTheyObserve: what a sibling observes of the branch moves the tip every +// worker on it sends for, and wakes them. A sibling's push is somebody else's commit here, as a +// fetch of it would say: the sibling's appliers may apply another cluster, so nothing says its +// content is live where this worker's appliers apply. +func TestSiblings_ShareWhatTheyObserve(t *testing.T) { + first, _, second, _ := siblingLoops(t) + first.w.inbox.take() + select { + case <-first.w.applyWake: + default: } - require.True(t, sibling.Enqueue(windowEvent("alpha", "3", "kubectl-edit"))) - observeForeignCommit(loop.w) - loop.serviceAppliers() - assert.Empty(t, requester.taken(), "the sibling's item is a live edit not yet decided") - assert.False(t, loop.quietSince.IsZero(), "the quiet period runs on") - - <-sibling.eventQueue // its loop takes the item, and then releases it, as run does - sibling.releaseInflightItem() + second.w.recordRemoteObservation("ccc", ObservedByPush) + assert.Equal(t, "ccc", first.w.inbox.lastTip()) + assert.True(t, first.w.inbox.foreignPending(), "a sibling's push is foreign here") select { - case <-loop.w.applyWake: + case <-first.w.applyWake: default: - t.Fatal("a sibling releasing its last item must wake the worker it held back") + t.Fatal("a shared observation must wake the sibling, so a ForeignCommit is decided at once") } - loop.serviceAppliers() - assert.Len(t, requester.taken(), 1) + + first.w.inbox.take() + second.w.recordRemoteObservation("ddd", ObservedByFetch) + assert.Equal(t, "ddd", first.w.inbox.lastTip()) + assert.True(t, first.w.inbox.foreignPending(), "and so is what a sibling fetched") } func TestApplierRetryDelay_DoublesToACap(t *testing.T) { @@ -362,15 +352,15 @@ func TestApplierRetryDelay_DoublesToACap(t *testing.T) { } // TestApplier_AStaleAttemptIsDecidedAgainWithoutBackoff: an attempt whose decision no longer held -// at the patch (the branch moved on, or became busy) asked nothing, and that is no failure. What it -// carried comes back, and the next quiet moment decides again at the new tip. +// at the patch (the branch moved on) asked nothing, and that is no failure. What it carried comes +// back, and the next wake decides again at the new tip. func TestApplier_AStaleAttemptIsDecidedAgainWithoutBackoff(t *testing.T) { loop, requester := manualLoop(t, applierTestTarget("alpha", true)) observeForeignCommit(loop.w) loop.serviceAppliers() out := requester.taken() require.Len(t, out, 1) - require.True(t, out[0].Current(), "decided at the tip on a quiet branch") + require.True(t, out[0].Current(), "decided at the tip") loop.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByPush}) assert.False(t, out[0].Current(), "the branch moved on") @@ -384,62 +374,16 @@ func TestApplier_AStaleAttemptIsDecidedAgainWithoutBackoff(t *testing.T) { assert.Zero(t, retry[0].PriorFailures) } -// TestDecisionHolds_OnlyAtTheTipOnAQuietBranch: the check the requester makes before it patches. -func TestDecisionHolds_OnlyAtTheTipOnAQuietBranch(t *testing.T) { +// TestDecisionHolds_OnlyAtTheTip: the check the requester makes before it patches. Only a moved tip +// makes a decision stale: an edit is published before the send, or recorded in the apply window. +func TestDecisionHolds_OnlyAtTheTip(t *testing.T) { w := newMetricsTestWorker() w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) assert.True(t, w.decisionHolds("aaa")) assert.False(t, w.decisionHolds("older"), "another commit is not the tip") - w.publishWriteBusy(true) - assert.False(t, w.decisionHolds("aaa"), "a window open since") - w.publishWriteBusy(false) - - require.True(t, w.Enqueue(windowEvent("alpha", "3", "kubectl-edit"))) - assert.False(t, w.decisionHolds("aaa"), "a queued event is a live edit not yet decided") -} - -// TestDecisionHolds_NotWhileTheLoopHandlesAnItem is the fifth review's second finding. The check -// read the queue's length, and busyWriting, which a wake publishes only as it ends: once the loop -// had taken its last item both said quiet, for as long as that item took to commit and push. The -// item counts until its wake has published what it left behind, for this worker's attempts and for -// a sibling's. The loop runs for real here, held inside the item by the render fidelity gate. -func TestDecisionHolds_NotWhileTheLoopHandlesAnItem(t *testing.T) { - loop, _ := manualLoop(t, applierTestTarget("alpha", true)) - w := loop.w - sibling := newMetricsTestWorker() - siblings := newBranchSiblings() - for _, worker := range []*BranchWorker{w, sibling} { - worker.repo = RepoIdentity{URL: "https://github.com/acme/platform"} - worker.applyWake = make(chan struct{}, 1) - worker.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) - siblings.add(worker) - } - gate := NewRenderFidelityGate() - w.renderFidelityGate = gate - ctx, cancel := context.WithCancel(context.Background()) - w.ctx = ctx - done := make(chan struct{}) - - gate.mu.Lock() // the loop's write check waits here - go func() { - defer close(done) - loop.run() - }() require.True(t, w.Enqueue(windowEvent("alpha", "3", "kubectl-edit"))) - require.Eventually(t, func() bool { return len(w.eventQueue) == 0 }, 5*time.Second, time.Millisecond) - assert.False(t, w.busyWriting.Load(), "the wake has published nothing yet") - assert.False(t, w.decisionHolds("aaa"), "the item the loop is handling is a live edit") - assert.False(t, sibling.decisionHolds("aaa"), "for a sibling too") - gate.mu.Unlock() - - require.Eventually(t, func() bool { return w.inflightItems.Load() == 0 }, 5*time.Second, time.Millisecond) - assert.True(t, w.busyWriting.Load(), "the window it opened is published before the item is released") - assert.False(t, sibling.decisionHolds("aaa")) - - restartAll(gate, itypes.NewResourceReference("alpha", "team-a"), fidelityScope("apps", "deployments")) - cancel() - <-done + assert.True(t, w.decisionHolds("aaa"), "a queued edit holds nothing") } // TestPushNotice_TheTipAlreadySeenDoesNotCutAWindow is the customer review's third blocker: the @@ -479,3 +423,66 @@ func TestWriteBusy_ASaveAnnouncedBeforeItsWrites(t *testing.T) { loop.pendingCRs = map[commitRequestID]*pendingCommitRequest{{}: {}} assert.True(t, loop.writeBusy()) } + +// queuedRefreshes takes the refreshes the worker queued for its loop. +func queuedRefreshes(w *BranchWorker) []*RefreshRequest { + var refreshes []*RefreshRequest + for { + select { + case item := <-w.eventQueue: + if item.Refresh != nil { + refreshes = append(refreshes, item.Refresh) + } + default: + return refreshes + } + } +} + +// TestApplyAttempt_ASourceElsewhereIsAHintToLook: an applier's source that fetched another commit +// than the one asked for says the branch may have moved, past a decision made before the worker heard +// of it. With no receiver, the worker would otherwise ask again for a commit the source no longer +// serves, until its own push or the refresher found the move; it looks, as at a push notice. A source +// at the tip known here says nothing new. +func TestApplyAttempt_ASourceElsewhereIsAHintToLook(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + observeForeignCommit(loop.w) + failWith := func(revision string) { + t.Helper() + loop.serviceAppliers() + sent := requester.taken() + require.Len(t, sent, 1) + sent[0].Report(ApplyResult{ID: sent[0].ID, Outcome: ApplyRequestFailed, + Err: errors.New("the source fetched another commit"), SourceRevision: revision}) + loop.serviceAppliers() + dueNow(loop) + } + + failWith("main@sha1:bbb") + assert.Empty(t, queuedRefreshes(loop.w), "the source holds the tip known here") + + failWith("main@sha1:ccc") + refreshes := queuedRefreshes(loop.w) + require.Len(t, refreshes, 1, "the worker looks at the remote") + assert.True(t, refreshes[0].PushNotice) + assert.Equal(t, "ccc", refreshes[0].After) +} + +// TestApplyAttempt_ANewFlushMarkStartsTheEditsOver: an applier can stay due without being sent (a +// sibling holds its group), and the next flush takes a newer mark, which publishes the edits recorded +// before it. Carried into the window, those would withhold the apply's own write of a commit that +// holds them, with a false collision and an extra Repair. A flush in progress keeps its record. +func TestApplyAttempt_ANewFlushMarkStartsTheEditsOver(t *testing.T) { + loop, _ := manualLoop(t, applierTestTarget("alpha", true)) + loop.followRecordedOperations(time.Now()) + state := loop.appliers[applierRef("alpha")] + state.flushEdits = map[string]time.Time{"x": time.Now()} + + loop.flushing = true + require.True(t, loop.flushed([]*applierState{state})) + assert.Len(t, state.flushEdits, 1, "the flush in progress keeps what it recorded") + + loop.flushing = false + require.True(t, loop.flushed([]*applierState{state})) + assert.Empty(t, state.flushEdits, "a new mark published them") +} diff --git a/internal/git/apply_catchup_test.go b/internal/git/apply_catchup_test.go new file mode 100644 index 00000000..00821e77 --- /dev/null +++ b/internal/git/apply_catchup_test.go @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// S2 (docs/design/applier-echoes.md): a CatchUp only adopts objects the API created and converges, +// for Flux and Argo CD alike, decided in the branch worker; no apply goes out before every initial +// snapshot is published. + +// appliedAt is a trigger whose last recorded operation succeeded, applying revision. +func appliedAt(revision string) func(*configv1alpha3.FluxTrigger) { + return recording(&configv1alpha3.ApplyOperation{ + Phase: configv1alpha3.ApplyOperationSucceeded, Reason: configv1alpha3.ApplyReasonCatchUp, + Commit: "older", AppliedRevision: revision, + }, false) +} + +// createdEvent is a live ADDED of a Deployment in target's folder, written last by managers. +func createdEvent(target string, managers ...string) Event { + event := windowEvent(target, "1", managers...) + event.Operation = string(itypes.OperationCreate) + return event +} + +// TestCatchUp_TheStartupConvergeAsksNothingOfAnApplierAtTheTip: the first observation leaves a +// converge pending for every applier. One whose trigger recorded the tip as applied is asked +// nothing; the other is asked a CatchUp of the tip. +func TestCatchUp_TheStartupConvergeAsksNothingOfAnApplierAtTheTip(t *testing.T) { + loop, requester := manualLoop(t, + applierTestTarget("fresh", true), + applierTestTarget("current", true), fluxTrigger("current", appliedAt("main@sha1:AAA")), + ) + loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) + loop.serviceAppliers() + + requests := requester.taken() + require.Len(t, requests, 1) + assert.Equal(t, applierRef("fresh"), requests[0].Applier) + assert.Equal(t, configv1alpha3.ApplyReasonCatchUp, requests[0].Reason) + assert.Equal(t, "aaa", requests[0].Commit) + assert.False(t, loop.appliers[applierRef("current")].converge, "already applied: answered") + + report(requests[0], ApplySucceeded) + loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByPush}) + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "the worker's own push asks nothing") +} + +// TestCatchUp_AnAPICreateIsAdoptedEvenAtTheTip: an object the API created carries no owner labels and +// no tracking annotation, so its applier is asked to apply again, even the commit it applied last: +// that is what stamps them. A create of the applier's own, or of an object another applier owns, +// adopts nothing. The same holds for an Argo CD Application. +func TestCatchUp_AnAPICreateIsAdoptedEvenAtTheTip(t *testing.T) { + for name, tc := range map[string]struct { + target *configv1alpha3.GitTarget + manager string + }{ + "flux": {applierTestTarget("alpha", true), FluxFieldManager}, + "argo cd": {applierTestTarget("alpha", true, argoApplier), ArgoCDFieldManager}, + } { + t.Run(name, func(t *testing.T) { + loop, requester := manualLoop(t, tc.target) + loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) + loop.serviceAppliers() + converge := requester.taken() + require.Len(t, converge, 1) + converge[0].Report(ApplyResult{ID: converge[0].ID, Outcome: ApplySucceeded, AppliedRevision: "aaa"}) + loop.serviceAppliers() + require.Empty(t, requester.taken()) + + ownApply := createdEvent("alpha", tc.manager) + loop.handleLiveEvents(&WriteRequest{Events: []Event{ownApply}, CommitMode: CommitModePerEvent}) + owned := createdEvent("alpha", "kubectl-create") + owned.AppliedBy = AppliedBy{Kind: configv1alpha3.ApplierKindKustomization, Namespace: "x", Name: "other"} + loop.handleLiveEvents(&WriteRequest{Events: []Event{owned}, CommitMode: CommitModePerEvent}) + loop.openWindow = nil + loop.serviceAppliers() + require.Empty(t, requester.taken(), "neither is an API create of the folder's own objects") + + loop.handleLiveEvents(&WriteRequest{ + Events: []Event{createdEvent("alpha", "kubectl-create")}, CommitMode: CommitModePerEvent, + }) + loop.openWindow = nil + loop.serviceAppliers() + adopt := requester.taken() + require.Len(t, adopt, 1, "adopted, although the applier is at the tip") + assert.Equal(t, configv1alpha3.ApplyReasonCatchUp, adopt[0].Reason) + assert.Equal(t, "aaa", adopt[0].Commit) + }) + } +} + +// TestCatchUp_NothingIsAskedBeforeTheInitialSnapshotsArePublished: every reason waits, a +// ForeignCommit included, until every collection of every GitTarget the applier covers has a settled +// snapshot and the worker published what it made of it; the barrier is looked at again on a +// short timer. +func TestCatchUp_NothingIsAskedBeforeTheInitialSnapshotsArePublished(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + applied := false + loop.w.initialSnapshots = func(itypes.ResourceReference) bool { return applied } + observeForeignCommit(loop.w) + + loop.serviceAppliers() + assert.Empty(t, requester.taken()) + assert.NotNil(t, loop.applierTimer, "the barrier is looked at again") + assert.True(t, loop.appliers[applierRef("alpha")].foreign, "the reason stays pending") + + applied = true + loop.serviceAppliers() + requests := requester.taken() + require.Len(t, requests, 1) + assert.Equal(t, configv1alpha3.ApplyReasonForeignCommit, requests[0].Reason) +} + +// TestCatchUp_AnEmptyTipAsksNothing: the remote does not carry the branch, so there is no commit to +// apply; the source reports the missing branch itself. The converge stays pending for when it does. +func TestCatchUp_AnEmptyTipAsksNothing(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + loop.w.inbox.observe(RemoteObservation{By: ObservedByFetch}) + loop.serviceAppliers() + assert.Empty(t, requester.taken()) + assert.True(t, loop.appliers[applierRef("alpha")].converge) + + loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) + loop.serviceAppliers() + requests := requester.taken() + require.Len(t, requests, 1, "the push that creates the branch gives it a commit") + assert.Equal(t, "aaa", requests[0].Commit) +} + +// TestCatchUp_AFirstNamingConvergesOnce: a GitTarget that names an applier already listed leaves one +// converge pending; listing it again leaves none. +func TestCatchUp_AFirstNamingConvergesOnce(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) + loop.serviceAppliers() + first := requester.taken() + require.Len(t, first, 1) + first[0].Report(ApplyResult{ID: first[0].ID, Outcome: ApplySucceeded, AppliedRevision: "main@sha1:aaa"}) + loop.serviceAppliers() + + loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByPush}) + shared := applierTestTarget("bravo", true, + func(g *configv1alpha3.GitTarget) { g.Spec.ApplyTriggerRef = testApplier("alpha") }) + require.NoError(t, loop.w.Client.Create(t.Context(), shared)) + loop.serviceAppliers() + second := requester.taken() + require.Len(t, second, 1, "bravo names alpha's trigger for the first time") + assert.Equal(t, "bbb", second[0].Commit) + report(second[0], ApplySucceeded) + + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "listing it again converges nothing") +} + +func TestRevisionNamesCommit_ReadsEveryForm(t *testing.T) { + for _, revision := range []string{"main@sha1:ABC", "sha1:abc", "main/abc", "feature/x/abc", "abc"} { + assert.True(t, revisionNamesCommit(revision, "abc"), revision) + } + assert.False(t, revisionNamesCommit("main@sha1:abd", "abc")) + assert.False(t, revisionNamesCommit("", "abc")) + assert.False(t, revisionNamesCommit("abc", "")) +} diff --git a/internal/git/apply_epoch.go b/internal/git/apply_epoch.go new file mode 100644 index 00000000..9c24f3b9 --- /dev/null +++ b/internal/git/apply_epoch.go @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "context" + "fmt" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// A resync never decides from a stale picture (docs/design/applier-echoes.md, F3). +// +// A resync mirrors a snapshot of live state. While Git is ahead of the cluster, a commit pushed and +// not yet applied, a snapshot holds the old state of every object that commit changes, and one whose +// collection overlaps an apply holds a mix. Either, decided, reverts the push. So the watch side reads +// the worker's ApplyState when a snapshot's collection starts, and the snapshot is used only if Git +// was not ahead then, the apply epoch has not moved since, and Git is not ahead when it arrives. +// Otherwise it is discarded, and a fresh snapshot of the GitTarget is asked for once Git is no longer +// ahead (resyncRecheck re-anchors its streams). No snapshot is ever parked. +// +// Three exceptions keep that from holding anything forever: +// +// - A branch with no trigger never waits: nothing applies its commits. +// - A collection's initial snapshot (the first one applied for it, ResyncRequest.Initial) is +// exempt. It is the startup baseline, and S2 asks no applier for anything before it is in Git, +// so holding it for an apply would wait on itself. +// - A GitTarget whose snapshots were held for snapshotHoldLimit uses the next one anyway: an +// apply that keeps failing must not leave drift unmirrored for as long as it fails. The C2 +// policy, API wins, then applies to whatever the push changed. + +// snapshotHoldLimit is how long F3 may hold off a GitTarget's resyncs, from the first snapshot it +// discarded: applierRetryMax, so an applier at full backoff gets at least one more attempt first. +const snapshotHoldLimit = applierRetryMax + +// Stale snapshot outcomes, the outcome label of stale_snapshots_total. +const ( + staleSnapshotDiscarded = "discarded" + staleSnapshotUsed = "used" +) + +// Why a snapshot was discarded, for the log. +const ( + snapshotAheadAtStart = "collected while Git was ahead of the cluster" + snapshotAcrossApply = "collected across an apply" + snapshotAheadOnArrival = "arrived while Git is ahead of the cluster" + snapshotForeignFetch = "its fetch found a commit the cluster does not run yet" +) + +// ApplyState reports where this worker's appliers stand, for the watch side to read when a +// snapshot's collection starts. It is safe from any goroutine. +func (w *BranchWorker) ApplyState() ApplyState { + ahead := w.appliersAhead.Load() || (w.hasAppliers.Load() && w.inbox.foreignPending()) + return ApplyState{Epoch: w.applyEpoch.Load(), GitAhead: ahead} +} + +// raiseApplyEpoch marks a send or an end of an attempt: a snapshot collected across it is stale. +func (w *BranchWorker) raiseApplyEpoch() { w.applyEpoch.Add(1) } + +// gitAhead reports whether Git may hold a commit the cluster does not run yet: an applier on the +// branch has a reason pending that applies the tip, or an attempt out. A converge counts once the +// applier may be asked for it; an adopt does not count, because it applies objects the cluster +// already runs. A branch with no applier is never ahead. +func (l *branchWorkerEventLoop) gitAhead() bool { + if len(l.appliers) == 0 { + return false + } + return l.appliersAhead() || l.w.inbox.foreignPending() +} + +// appliersAhead is gitAhead without what the inbox holds. +func (l *branchWorkerEventLoop) appliersAhead() bool { + if len(l.unsentRefusals) > 0 { + return true + } + for _, state := range l.appliers { + if state.inflight != nil || state.foreign || state.repair || (state.converge && l.pastBarrier(state)) { + return true + } + } + return false +} + +// publishApplyState publishes the loop's half of ApplyState. It runs at the end of every wake. +func (l *branchWorkerEventLoop) publishApplyState() { + l.w.hasAppliers.Store(len(l.appliers) > 0) + l.w.appliersAhead.Store(len(l.appliers) > 0 && l.appliersAhead()) +} + +// handleResyncRequest decides one revision-pinned resync in order on the worker goroutine, unless its +// snapshot is stale (snapshotStale), in which case it is discarded and asked for again. +func (l *branchWorkerEventLoop) handleResyncRequest(req *ResyncRequest) { + if reason := l.snapshotStale(req); reason != "" { + l.discardSnapshot(req, reason) + return + } + l.applyResync(req) +} + +// snapshotStale says why a resync's snapshot must not be decided, or "" when it may be. +func (l *branchWorkerEventLoop) snapshotStale(req *ResyncRequest) string { + if !l.snapshotsWait(req) { + return "" + } + switch start := req.CollectionStart; { + case start != nil && start.GitAhead: + return snapshotAheadAtStart + case start != nil && start.Epoch != l.w.applyEpoch.Load(): + return snapshotAcrossApply + case l.gitAhead(): + return snapshotAheadOnArrival + } + return "" +} + +// snapshotsWait reports whether F3 applies to a resync: the branch has an applier, the snapshot is +// not its collection's initial one, and the GitTarget's snapshots have not been held for +// snapshotHoldLimit already. A snapshot let through by the limit is logged and counted as used. +func (l *branchWorkerEventLoop) snapshotsWait(req *ResyncRequest) bool { + if len(l.appliers) == 0 || req.Initial { + return false + } + since, held := l.snapshotHeldSince[resyncTargetKey(req)] + if !held || l.w.now().Sub(since) < snapshotHoldLimit { + return true + } + if l.gitAhead() { + l.w.Log.Info("Using a snapshot although Git is ahead of the cluster: its GitTarget's snapshots have "+ + "been held for the limit, and drift must not stay unmirrored for as long as an apply fails", + "gitTarget", resyncTargetKey(req), "heldFor", l.w.now().Sub(since).Round(time.Second).String()) + l.recordStaleSnapshot(staleSnapshotUsed) + } + return false +} + +// discardSnapshot answers a resync whose snapshot is stale, and asks for a fresh one of its GitTarget +// once Git is no longer ahead. +func (l *branchWorkerEventLoop) discardSnapshot(req *ResyncRequest, reason string) { + key := resyncTargetKey(req) + l.w.Log.Info("Discarding a resync snapshot; a fresh one is asked for once the apply is done", + "gitTarget", key, "scope", req.Scope.String(), "reason", reason) + req.reply(ResyncResult{Err: fmt.Errorf("%w: %s", ErrResyncDiscarded, reason)}) + l.recordStaleSnapshot(staleSnapshotDiscarded) + if l.snapshotRerequests == nil { + l.snapshotRerequests = map[string]itypes.ResourceReference{} + } + l.snapshotRerequests[key] = itypes.NewResourceReference(req.GitTargetName, req.GitTargetNamespace) + if l.snapshotHeldSince == nil { + l.snapshotHeldSince = map[string]time.Time{} + } + if _, held := l.snapshotHeldSince[key]; !held { + l.snapshotHeldSince[key] = l.w.now() + } +} + +// heldForTheLimit reports whether a GitTarget's snapshots have been held for snapshotHoldLimit: its +// next one is asked for, and used, whatever is ahead. +func (l *branchWorkerEventLoop) heldForTheLimit(key string) bool { + since, held := l.snapshotHeldSince[key] + return held && l.w.now().Sub(since) >= snapshotHoldLimit +} + +// snapshotHoldDeadline is when the earliest held GitTarget reaches snapshotHoldLimit, zero when none +// waits for it: the applier timer wakes the loop then, to ask for its snapshot. +func (l *branchWorkerEventLoop) snapshotHoldDeadline() time.Time { + var deadline time.Time + for key := range l.snapshotRerequests { + if since, held := l.snapshotHeldSince[key]; held { + deadline = earliest(deadline, since.Add(snapshotHoldLimit)) + } + } + return deadline +} + +// snapshotUsed records that a GitTarget's snapshot was decided: its hold is over. +func (l *branchWorkerEventLoop) snapshotUsed(req *ResyncRequest) { + delete(l.snapshotHeldSince, resyncTargetKey(req)) +} + +// rerequestSnapshots asks for the snapshots discarded, once what made them stale is over. It runs +// at the end of every wake. +func (l *branchWorkerEventLoop) rerequestSnapshots() { + if len(l.snapshotRerequests) == 0 { + return + } + ahead := l.gitAhead() + for key, target := range l.snapshotRerequests { + if ahead && !l.heldForTheLimit(key) { + continue + } + delete(l.snapshotRerequests, key) + if l.w.resyncRecheck == nil { + continue + } + l.w.Log.Info("Asking for a fresh snapshot of a GitTarget whose snapshot was discarded", + "gitTarget", key) + l.w.resyncRecheck(target) + } +} + +// initialSnapshotsPublished reports whether a GitTarget's initial snapshots are published: every +// collection it watches has a settled snapshot, applied, refused or failed (the watch side says so, +// initialSnapshots), and the worker holds no resync of it, queued or decided and not yet published. +func (l *branchWorkerEventLoop) initialSnapshotsPublished(ref itypes.ResourceReference) bool { + if l.w.initialSnapshots != nil && !l.w.initialSnapshots(ref) { + return false + } + key := ref.String() + if l.w.queuedResyncsFor(key) { + return false + } + for i := range l.pendingWrites { + if write := &l.pendingWrites[i]; write.Kind == PendingWriteResync && + write.GitTargetNamespace+"/"+write.GitTargetName == key { + return false + } + } + return true +} + +// recordStaleSnapshot counts one stale snapshot, by what was done with it. +func (l *branchWorkerEventLoop) recordStaleSnapshot(outcome string) { + if telemetry.StaleSnapshotsTotal == nil { + return + } + telemetry.StaleSnapshotsTotal.Add(context.WithoutCancel(l.w.ctx), 1, + metric.WithAttributes(l.w.providerAttrs(attribute.String("outcome", outcome))...)) +} diff --git a/internal/git/apply_epoch_test.go b/internal/git/apply_epoch_test.go new file mode 100644 index 00000000..8d84c185 --- /dev/null +++ b/internal/git/apply_epoch_test.go @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// F3 (docs/design/applier-echoes.md): a resync never decides from a snapshot collected while Git was +// ahead of the cluster, or across an apply. These drive the worker's side deterministically. + +// epochLoop is a loop whose applier covers alpha, with alpha's initial snapshots published, and the +// GitTargets it was asked to snapshot again. +func epochLoop(t *testing.T) (*branchWorkerEventLoop, *recordingRequester, *[]itypes.ResourceReference) { + t.Helper() + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + asked := &[]itypes.ResourceReference{} + loop.w.resyncRecheck = func(target itypes.ResourceReference) { *asked = append(*asked, target) } + loop.followRecordedOperations(time.Now()) + require.True(t, loop.initialSnapshotsPublished(alphaRef)) + return loop, requester, asked +} + +// alphaRef is the GitTarget alpha. +var alphaRef = itypes.NewResourceReference("alpha", "team-a") + +// alphaSnapshot is a snapshot of alpha whose collection started at start. +func alphaSnapshot(start *ApplyState) *ResyncRequest { + return &ResyncRequest{GitTargetName: "alpha", GitTargetNamespace: "team-a", CollectionStart: start, + Result: make(chan ResyncResult, 1)} +} + +func discarded(t *testing.T, req *ResyncRequest) bool { + t.Helper() + select { + case res := <-req.Result: + return errors.Is(res.Err, ErrResyncDiscarded) + default: + return false + } +} + +// TestApplyEpoch_ASnapshotCollectedAcrossAnApplyIsDiscarded: the collection starts while an apply is +// out, the apply completes, and the snapshot arrives with nothing pending. It holds a mix of before +// and after the apply, so it is discarded, and asked for again: by then nothing is ahead. +func TestApplyEpoch_ASnapshotCollectedAcrossAnApplyIsDiscarded(t *testing.T) { + loop, requester, asked := epochLoop(t) + before := loop.w.ApplyState() + require.False(t, before.GitAhead) + + observeForeignCommit(loop.w) + loop.serviceAppliers() + sent := requester.taken() + require.Len(t, sent, 1) + loop.publishApplyState() + during := loop.w.ApplyState() + assert.True(t, during.GitAhead, "an attempt is out") + assert.Greater(t, during.Epoch, before.Epoch, "the send raised the epoch") + + report(sent[0], ApplySucceeded) + loop.serviceAppliers() + loop.publishApplyState() + after := loop.w.ApplyState() + require.False(t, after.GitAhead, "nothing is pending once the apply completed") + assert.Greater(t, after.Epoch, during.Epoch, "the completion raised it again") + + started := before + req := alphaSnapshot(&started) + loop.handleResyncRequest(req) + assert.True(t, discarded(t, req), "collected across the apply") + loop.rerequestSnapshots() + assert.Equal(t, []itypes.ResourceReference{itypes.NewResourceReference("alpha", "team-a")}, *asked) + + whileAhead := alphaSnapshot(&during) + loop.handleResyncRequest(whileAhead) + assert.True(t, discarded(t, whileAhead), "collected while Git was ahead") +} + +// TestApplyEpoch_ASnapshotArrivingWhileGitIsAheadWaitsForTheApply: a snapshot that arrives while a +// ForeignCommit is pending is discarded, and asked for again only once the apply answered it. +func TestApplyEpoch_ASnapshotArrivingWhileGitIsAheadWaitsForTheApply(t *testing.T) { + loop, requester, asked := epochLoop(t) + observeForeignCommit(loop.w) + + req := alphaSnapshot(nil) + loop.handleResyncRequest(req) + assert.True(t, discarded(t, req)) + loop.rerequestSnapshots() + assert.Empty(t, *asked, "Git is still ahead") + + loop.serviceAppliers() + sent := requester.taken() + require.Len(t, sent, 1) + loop.rerequestSnapshots() + assert.Empty(t, *asked, "the apply is out") + + report(sent[0], ApplySucceeded) + loop.serviceAppliers() + loop.rerequestSnapshots() + assert.Len(t, *asked, 1, "asked for again once the apply completed") +} + +// TestApplyEpoch_InitialSnapshotsAreExempt: the first snapshot of a collection is its baseline, and +// is never held for an apply: S2 asks for no apply before it is in Git, so holding it would wait on +// itself. A later one of the same collection is judged by F3. +func TestApplyEpoch_InitialSnapshotsAreExempt(t *testing.T) { + loop, _, _ := epochLoop(t) + observeForeignCommit(loop.w) + + initial := alphaSnapshot(&ApplyState{GitAhead: true}) + initial.Initial = true + assert.Empty(t, loop.snapshotStale(initial), "the collection's first snapshot: exempt") + assert.Equal(t, snapshotAheadAtStart, loop.snapshotStale(alphaSnapshot(&ApplyState{GitAhead: true})), + "a later one: F3 applies") +} + +// TestApplyEpoch_ABranchWithNoTriggerNeverWaits: nothing applies its commits. +func TestApplyEpoch_ABranchWithNoTriggerNeverWaits(t *testing.T) { + loop, _ := manualLoop(t, applierTestTarget("alpha", false)) + loop.followRecordedOperations(time.Now()) + observeForeignCommit(loop.w) + assert.Empty(t, loop.snapshotStale(alphaSnapshot(&ApplyState{GitAhead: true}))) + assert.False(t, loop.w.ApplyState().GitAhead) +} + +// TestApplyEpoch_SnapshotsAreHeldNoLongerThanTheLimit: an apply that keeps failing holds a +// GitTarget's snapshots for snapshotHoldLimit at most, from the first one discarded; the next is used. +func TestApplyEpoch_SnapshotsAreHeldNoLongerThanTheLimit(t *testing.T) { + loop, _, _ := epochLoop(t) + now := time.Now() + loop.w.clock = func() time.Time { return now } + observeForeignCommit(loop.w) + + loop.handleResyncRequest(alphaSnapshot(nil)) + require.Contains(t, loop.snapshotHeldSince, "team-a/alpha") + assert.Equal(t, snapshotAheadOnArrival, loop.snapshotStale(alphaSnapshot(nil))) + + now = now.Add(snapshotHoldLimit) + assert.Empty(t, loop.snapshotStale(alphaSnapshot(nil)), "held for the limit: used anyway") +} + +// TestApplyEpoch_InitialSnapshotsWaitForTheWorkersResyncs: settled for every collection, as the watch +// side says, is not enough; the worker must not hold a resync of the GitTarget, queued or pending. +func TestApplyEpoch_InitialSnapshotsWaitForTheWorkersResyncs(t *testing.T) { + loop, _ := manualLoop(t, applierTestTarget("alpha", true)) + applied := false + loop.w.initialSnapshots = func(itypes.ResourceReference) bool { return applied } + loop.followRecordedOperations(time.Now()) + assert.False(t, loop.initialSnapshotsPublished(alphaRef), "the watch side has not settled them") + + applied = true + loop.pendingWrites = []PendingWrite{ + {Kind: PendingWriteResync, GitTargetName: "alpha", GitTargetNamespace: "team-a"}, + } + assert.False(t, loop.initialSnapshotsPublished(alphaRef), "a resync of it is not published yet") + + loop.pendingWrites = nil + assert.True(t, loop.initialSnapshotsPublished(alphaRef)) +} + +// TestApplyEpoch_AHeldGitTargetIsAskedAgainAtTheLimit: while an apply keeps failing, a discarded +// snapshot is not asked for again until the hold reaches snapshotHoldLimit; the applier timer is set +// for that moment, and then the fresh snapshot is asked for although Git is still ahead. +func TestApplyEpoch_AHeldGitTargetIsAskedAgainAtTheLimit(t *testing.T) { + loop, _, asked := epochLoop(t) + now := time.Now() + loop.w.clock = func() time.Time { return now } + observeForeignCommit(loop.w) + + loop.handleResyncRequest(alphaSnapshot(nil)) + assert.Equal(t, now.Add(snapshotHoldLimit), loop.snapshotHoldDeadline()) + loop.rerequestSnapshots() + assert.Empty(t, *asked, "Git is ahead, and the limit is not reached") + + now = now.Add(snapshotHoldLimit) + assert.True(t, loop.heldForTheLimit("team-a/alpha")) + loop.rerequestSnapshots() + assert.Equal(t, []itypes.ResourceReference{itypes.NewResourceReference("alpha", "team-a")}, *asked) + assert.True(t, loop.snapshotHoldDeadline().IsZero(), "nothing is waiting any more") +} diff --git a/internal/git/apply_groups.go b/internal/git/apply_groups.go index 9792fd57..cbb1cd09 100644 --- a/internal/git/apply_groups.go +++ b/internal/git/apply_groups.go @@ -8,7 +8,7 @@ import ( "github.com/ConfigButler/gitops-reverser/api/v1alpha3" ) -// Shared sources (docs/design/apply-trigger.md, "Shared sources and prefetch"). +// Shared sources (docs/design/apply-trigger.md, "Shared sources"). // // Flux appliers on one branch that apply one GitRepository form a group, and a group has one // operation at a time: the artifact is shared, and an operation reads it again just before it @@ -18,11 +18,6 @@ import ( // workers, and their triggers can apply one GitRepository, so a worker claims the group before it // sends, and publishes the groups it has attempts out for. // -// When every consumer of the GitRepository is held suspended by a FluxTrigger, a new artifact -// applies nothing, so a push somebody else made is fetched as soon as it is observed (prefetch) -// rather than when the branch is next quiet: the artifact is then ready when the apply is due. It -// is asked only while the branch is busy, because on a quiet branch the attempt sent at once fetches -// for itself, and never while an operation of the group is in flight. // sourceGroup is the group a trigger's applier belongs to: its cluster and GitRepository, or "" for // an applier that shares nothing this way. @@ -89,43 +84,3 @@ func (l *branchWorkerEventLoop) turnOrder() []ApplierRef { }) return refs } - -// prefetchSources asks each group whose trigger may prefetch, and that has no attempt out, to fetch -// a push somebody else made. One request per group: its appliers share the GitRepository. -func (l *branchWorkerEventLoop) prefetchSources() { - w := l.w - tip, due := w.inbox.takePrefetch() - if !due || w.applyRequester == nil { - return - } - busy := map[string]bool{} - first := map[string]*applierState{} - for _, ref := range l.sortedAppliers() { - state := l.appliers[ref] - group := state.group() - if group == "" { - continue - } - busy[group] = busy[group] || state.inflight != nil - if _, seen := first[group]; !seen && state.targets[0].prefetch { - first[group] = state - } - } - groups := make([]string, 0, len(first)) - for group := range first { - groups = append(groups, group) - } - sort.Strings(groups) - for _, group := range groups { - if busy[group] || w.groupOutOnSibling(group) { - continue - } - state := first[group] - w.Log.V(1).Info("Fetching a push ahead of its apply", "branch", w.Branch, - "applier", state.ref.String(), "commit", tip) - w.applyRequester.PrefetchSource(PrefetchRequest{ - Applier: state.ref, - Commit: tip, - }) - } -} diff --git a/internal/git/apply_groups_test.go b/internal/git/apply_groups_test.go index 0e404e64..26048841 100644 --- a/internal/git/apply_groups_test.go +++ b/internal/git/apply_groups_test.go @@ -2,9 +2,8 @@ package git -// Shared sources (docs/design/apply-trigger.md, "Shared sources and prefetch"): Flux appliers that apply one -// GitRepository have one operation at a time, and a push somebody else made is fetched ahead of the -// apply while the branch is busy, when every consumer of the source is held. +// Shared sources (docs/design/apply-trigger.md, "Shared sources"): Flux appliers that apply one +// GitRepository have one operation at a time. import ( "fmt" @@ -18,11 +17,11 @@ import ( ) // applying gives a trigger the source its reconciler published. -func applying(gitRepository string, prefetch bool) func(*configv1alpha3.FluxTrigger) { +func applying(gitRepository string) func(*configv1alpha3.FluxTrigger) { return func(trigger *configv1alpha3.FluxTrigger) { trigger.Status.Source = &configv1alpha3.FluxTriggerSource{ GitRepository: meta.NamespacedObjectReference{Namespace: "flux-system", Name: gitRepository}, - URL: "https://example.com/repo", Branch: "main", Prefetch: prefetch, + URL: "https://example.com/repo", Branch: "main", } } } @@ -31,9 +30,9 @@ func applying(gitRepository string, prefetch bool) func(*configv1alpha3.FluxTrig // after the other, and two on different GitRepositories at once. func TestGroups_OneOperationAtATimePerSource(t *testing.T) { loop, requester := manualLoop(t, - applierTestTarget("alpha", true), fluxTrigger("alpha", applying("platform", false)), - applierTestTarget("bravo", true), fluxTrigger("bravo", applying("platform", false)), - applierTestTarget("charlie", true), fluxTrigger("charlie", applying("other", false)), + applierTestTarget("alpha", true), fluxTrigger("alpha", applying("platform")), + applierTestTarget("bravo", true), fluxTrigger("bravo", applying("platform")), + applierTestTarget("charlie", true), fluxTrigger("charlie", applying("other")), ) observeForeignCommit(loop.w) loop.serviceAppliers() @@ -58,8 +57,8 @@ func TestGroups_OneOperationAtATimePerSource(t *testing.T) { // turn waits for its siblings instead of going again. func TestGroups_TheSourceIsSharedInTurn(t *testing.T) { loop, requester := manualLoop(t, - applierTestTarget("alpha", true), fluxTrigger("alpha", applying("platform", false)), - applierTestTarget("bravo", true), fluxTrigger("bravo", applying("platform", false)), + applierTestTarget("alpha", true), fluxTrigger("alpha", applying("platform")), + applierTestTarget("bravo", true), fluxTrigger("bravo", applying("platform")), ) observeForeignCommit(loop.w) loop.serviceAppliers() @@ -76,50 +75,6 @@ func TestGroups_TheSourceIsSharedInTurn(t *testing.T) { } } -// TestPrefetch_APushIsFetchedAheadWhileTheBranchIsBusy: a push somebody else made, observed while the -// branch is busy, has the source fetch now, once per source, so its artifact is ready when the -// branch is quiet; a quiet branch asks the applier at once, which fetches for itself. -func TestPrefetch_APushIsFetchedAheadWhileTheBranchIsBusy(t *testing.T) { - loop, requester := manualLoop(t, - applierTestTarget("alpha", true), fluxTrigger("alpha", applying("platform", true)), - applierTestTarget("bravo", true), fluxTrigger("bravo", applying("platform", true)), - applierTestTarget("held", true), fluxTrigger("held", applying("shared", false)), - ) - loop.serviceAppliers() // lists the appliers - loop.openWindow = &openWindow{} - observeForeignCommit(loop.w) - - loop.serviceAppliers() - assert.Empty(t, requester.taken(), "nothing is applied on a busy branch") - require.Len(t, requester.prefetches, 1, "one fetch for the shared source; none for a shared one") - assert.Equal(t, applierRef("alpha"), requester.prefetches[0].Applier) - assert.Equal(t, "bbb", requester.prefetches[0].Commit) - - loop.serviceAppliers() - assert.Len(t, requester.prefetches, 1, "once per push") - - loop.openWindow = nil - loop.serviceAppliers() - assert.Len(t, requester.taken(), 2, "the apply is asked once the branch is quiet") - assert.Len(t, requester.prefetches, 1) -} - -// TestPrefetch_NotWhileAnOperationOfTheSourceIsInFlight: a resumed Kustomization applies whatever the -// artifact becomes, so no fetch is asked for while one of its group is out. -func TestPrefetch_NotWhileAnOperationOfTheSourceIsInFlight(t *testing.T) { - loop, requester := manualLoop(t, - applierTestTarget("alpha", true), fluxTrigger("alpha", applying("platform", true)), - ) - observeForeignCommit(loop.w) - loop.serviceAppliers() - require.Len(t, requester.taken(), 1) - - loop.openWindow = &openWindow{} - loop.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByFetch}) - loop.serviceAppliers() - assert.Empty(t, requester.prefetches) -} - // siblingLoops is two workers on one repository and branch, under two GitProviders, whose // FluxTriggers alpha and bravo apply the GitRepository platform, each with a ForeignCommit pending. func siblingLoops(t *testing.T) ( @@ -127,9 +82,9 @@ func siblingLoops(t *testing.T) ( ) { t.Helper() first, firstRequests := manualLoop(t, - applierTestTarget("alpha", true), fluxTrigger("alpha", applying("platform", true))) + applierTestTarget("alpha", true), fluxTrigger("alpha", applying("platform"))) second, secondRequests := manualLoop(t, - applierTestTarget("bravo", true), fluxTrigger("bravo", applying("platform", true))) + applierTestTarget("bravo", true), fluxTrigger("bravo", applying("platform"))) siblings := newBranchSiblings() for _, loop := range []*branchWorkerEventLoop{first, second} { loop.w.repo = RepoIdentity{URL: "https://github.com/acme/platform"} @@ -146,10 +101,14 @@ func siblingLoops(t *testing.T) ( func TestGroups_OneOperationAtATimePerSourceAcrossWorkers(t *testing.T) { first, firstRequests, second, secondRequests := siblingLoops(t) + first.serviceAppliers() + answerFlush(second) first.serviceAppliers() sent := firstRequests.taken() require.Len(t, sent, 1) second.serviceAppliers() + answerFlush(first) + second.serviceAppliers() assert.Empty(t, secondRequests.taken(), "the other worker's operation on the shared source is out") report(sent[0], ApplySucceeded) @@ -159,6 +118,8 @@ func TestGroups_OneOperationAtATimePerSourceAcrossWorkers(t *testing.T) { default: t.Fatal("the group going idle must wake the worker it held back") } + second.serviceAppliers() // its flush asks the first again + answerFlush(first) second.serviceAppliers() next := secondRequests.taken() require.Len(t, next, 1) @@ -190,16 +151,3 @@ func TestGroups_AClaimIsCheckedAndRecordedAtOnce(t *testing.T) { } assert.True(t, second.w.claimGroup(group)) } - -// TestPrefetch_NotWhileASiblingsOperationOfTheSourceIsInFlight: a resumed Kustomization applies -// whatever the artifact becomes, whichever worker resumed it. -func TestPrefetch_NotWhileASiblingsOperationOfTheSourceIsInFlight(t *testing.T) { - first, firstRequests, second, secondRequests := siblingLoops(t) - first.serviceAppliers() - require.Len(t, firstRequests.taken(), 1) - - second.openWindow = &openWindow{} - second.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByFetch}) - second.serviceAppliers() - assert.Empty(t, secondRequests.prefetches) -} diff --git a/internal/git/apply_operation_test.go b/internal/git/apply_operation_test.go index a359db2b..5a0d2210 100644 --- a/internal/git/apply_operation_test.go +++ b/internal/git/apply_operation_test.go @@ -25,7 +25,7 @@ import ( // Argo CD sync of replicas 2 runs, the API edits replicas to 3, and the sync's write of 2 arrives // after it: the window withholds it, so Git keeps 3, and the cluster runs 2. An Application is never // sent a CatchUp, so nothing used to sync again. The withheld write leaves a Repair pending: a -// second sync, pinned to the tip that holds the edit, sent on the quiet branch with no quiet +// second sync, pinned to the tip that holds the edit, sent after a flush with no quiet // period. func TestApplier_AWithheldRollbackIsRepairedWithASecondSync(t *testing.T) { loop, requester := manualLoop(t, applierTestTarget("alpha", true, argoApplier)) @@ -54,10 +54,9 @@ func TestApplier_AWithheldRollbackIsRepairedWithASecondSync(t *testing.T) { assert.Empty(t, requester.taken(), "a repair is sent once") } -// TestApplier_ARepairWaitsForTheBranchNotForAQuietPeriod: a Repair is decided like a Refusal. It -// waits for a quiet branch, and for the attempt still out, but not for the quiet period; and a -// failed one comes back. -func TestApplier_ARepairWaitsForTheBranchNotForAQuietPeriod(t *testing.T) { +// TestApplier_ARepairWaitsForTheAttemptOut: a Repair is decided like a Refusal. It waits for the +// attempt still out and for a flush, but not for a quiet period; and a failed one comes back. +func TestApplier_ARepairWaitsForTheAttemptOut(t *testing.T) { loop, requester := manualLoop(t, applierTestTarget("alpha", true)) observeForeignCommit(loop.w) loop.serviceAppliers() @@ -70,15 +69,14 @@ func TestApplier_ARepairWaitsForTheBranchNotForAQuietPeriod(t *testing.T) { assert.Empty(t, requester.taken(), "the attempt is still out") report(first[0], ApplySucceeded) - loop.openWindow = &openWindow{} + holdApplies(loop) loop.serviceAppliers() - assert.Empty(t, requester.taken(), "not while the branch is busy") + assert.Empty(t, requester.taken(), "not while a save holds the flush") - loop.openWindow = nil - loop.quietSince = time.Time{} + releaseApplies(loop) loop.serviceAppliers() repair := requester.taken() - require.Len(t, repair, 1, "the moment the branch is quiet") + require.Len(t, repair, 1, "the moment a flush can be made") assert.Equal(t, configv1alpha3.ApplyReasonRepair, repair[0].Reason) report(repair[0], ApplyFailed) @@ -305,7 +303,7 @@ func TestApplier_TheTriggerStatusFollowsEveryTransition(t *testing.T) { // Record, the trigger is told nothing about it. func TestApplier_AnAttemptEndedBeforeItsRecordLeavesTheTriggerAlone(t *testing.T) { for _, outcome := range []ApplyOutcome{ - ApplyRejected, ApplyDropped, ApplyStale, ApplyRequestFailed, ApplyCurrent, + ApplyRejected, ApplyDropped, ApplyStale, ApplyRequestFailed, } { t.Run(string(outcome), func(t *testing.T) { loop, requester := manualLoop(t, applierTestTarget("alpha", true, argoApplier)) diff --git a/internal/git/apply_ownership.go b/internal/git/apply_ownership.go new file mode 100644 index 00000000..4bafe82e --- /dev/null +++ b/internal/git/apply_ownership.go @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "context" + "strings" + + "go.opentelemetry.io/otel/metric" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" +) + +// A folder a trigger applies belongs to that trigger's applier (docs/design/applier-echoes.md, F4). +// An object another applier applies, mirrored into it, would be applied by this applier too, and two +// appliers applying one object fight each other. So an object owned by another applier is excluded +// from a triggered folder: a live event of it is not mirrored, and a snapshot of it is neither +// upserted nor swept. A folder with no trigger excludes nothing: mirroring another applier's objects +// one way is what such a folder can exist for. +// +// The owner is read from the raw object, where the watch holds it before sanitize strips it: Flux's +// owner labels, and Argo CD's tracking annotation, which names its Application. + +const ( + // fluxOwnerNameLabel and fluxOwnerNamespaceLabel name the Kustomization that applied an object. + fluxOwnerNameLabel = "kustomize.toolkit.fluxcd.io/name" + fluxOwnerNamespaceLabel = "kustomize.toolkit.fluxcd.io/namespace" + // argoCDTrackingAnnotation is :/:/, where + // is the Application's name, prefixed with _ when it lives outside Argo CD's own + // namespace (Application.InstanceName in external-sources/argo-cd). + argoCDTrackingAnnotation = "argocd.argoproj.io/tracking-id" +) + +// AppliedBy is the applier that owns an object, as the object itself says. The zero value is an +// object no applier owns. +type AppliedBy struct { + // Kind is v1alpha3.ApplierKindKustomization or v1alpha3.ApplierKindApplication. + Kind string + // Namespace is the applier's namespace; empty for an Argo CD Application in Argo CD's own + // namespace, which its tracking annotation does not name. + Namespace string + Name string +} + +// IsZero reports whether no applier owns the object. +func (a AppliedBy) IsZero() bool { return a.Kind == "" } + +// String is the owner for logs. +func (a AppliedBy) String() string { return a.Kind + "/" + a.Namespace + "/" + a.Name } + +// is reports whether the owner is the applier ref. +func (a AppliedBy) is(ref ApplierRef) bool { + if a.Kind != ref.Kind || a.Name != ref.Name { + return false + } + return a.Namespace == ref.Namespace || (a.Kind == v1alpha3.ApplierKindApplication && a.Namespace == "") +} + +// AppliedByOf reads the applier that owns an object from its raw labels and annotations: Flux's owner +// labels, or else Argo CD's tracking annotation. It must be called before sanitize. +func AppliedByOf(u *unstructured.Unstructured) AppliedBy { + if u == nil { + return AppliedBy{} + } + labels := u.GetLabels() + if name, namespace := labels[fluxOwnerNameLabel], labels[fluxOwnerNamespaceLabel]; name != "" && namespace != "" { + return AppliedBy{Kind: v1alpha3.ApplierKindKustomization, Namespace: namespace, Name: name} + } + tracking := u.GetAnnotations()[argoCDTrackingAnnotation] + app, _, found := strings.Cut(tracking, ":") + if !found || app == "" { + return AppliedBy{} + } + if namespace, name, qualified := strings.Cut(app, "_"); qualified { + return AppliedBy{Kind: v1alpha3.ApplierKindApplication, Namespace: namespace, Name: name} + } + return AppliedBy{Kind: v1alpha3.ApplierKindApplication, Name: app} +} + +// applierOf is the applier whose trigger a GitTarget names on this branch, nil when it names none. +func (l *branchWorkerEventLoop) applierOf(name, namespace string) *applierState { + for _, ref := range l.sortedAppliers() { + state := l.appliers[ref] + for _, target := range state.targets { + if target.ref.Name == name && target.ref.Namespace == namespace { + return state + } + } + } + return nil +} + +// ownedElsewhere reports whether an object of the GitTarget's folder is owned by an applier other +// than the one the folder's trigger names, and counts and logs it when it is. +func (l *branchWorkerEventLoop) ownedElsewhere(name, namespace, object string, owner AppliedBy) bool { + if owner.IsZero() { + return false + } + applier := l.applierOf(name, namespace) + if applier == nil || owner.is(applier.ref) { + return false + } + l.recordOwnershipConflict(namespace+"/"+name, object, applier.ref, owner) + return true +} + +// noteCreated leaves an adopt pending for the applier of the folder an API create landed in (S2 in +// docs/design/applier-echoes.md). An applier prunes only what it applied, and an object the API +// created carries neither Flux's owner labels nor Argo CD's tracking annotation, even when its +// content equals what Git holds, so a later Git-side delete would leave it on the cluster. A create +// whose newest managers are an applier's is that applier's own apply, and adopts nothing; one another +// applier owns never reaches here (ownedElsewhere). +func (l *branchWorkerEventLoop) noteCreated(event Event) { + if writtenByAnApplier(event.LastFieldManagers) { + return + } + l.adopt(event.GitTargetName, event.GitTargetNamespace, event.Identifier.String()) +} + +// adopt leaves an adopt pending for the applier of a GitTarget's folder, when its trigger names one. +func (l *branchWorkerEventLoop) adopt(name, namespace, object string) { + applier := l.applierOf(name, namespace) + if applier == nil { + return + } + if !applier.adopt { + l.w.Log.Info("An object the API created is in a folder an applier applies; it is adopted at the "+ + "next apply, so a later Git-side delete prunes it", "gitTarget", namespace+"/"+name, + "object", object, "applier", applier.ref.String()) + } + applier.adopt = true +} + +// writtenByAnApplier reports whether every one of managers is a field manager of an applier kind. +func writtenByAnApplier(managers []string) bool { + if len(managers) == 0 { + return false + } + for _, kind := range []string{v1alpha3.ApplierKindKustomization, v1alpha3.ApplierKindApplication} { + if _, alone := (ApplierRef{Kind: kind}).writtenBy(managers); alone { + return true + } + } + return false +} + +// withoutOwnedElsewhere is a resync's snapshot without the objects another applier owns, and those +// objects by objectKey, for the sweep to leave alone. +func (l *branchWorkerEventLoop) withoutOwnedElsewhere( + req *ResyncRequest, +) ([]manifestanalyzer.DesiredResource, map[string]bool) { + if len(req.Owners) == 0 { + return req.Desired, nil + } + desired := make([]manifestanalyzer.DesiredResource, 0, len(req.Desired)) + var kept map[string]bool + for _, dr := range req.Desired { + owner := req.Owners[dr.Resource] + if !l.ownedElsewhere(req.GitTargetName, req.GitTargetNamespace, dr.Resource.String(), owner) { + desired = append(desired, dr) + continue + } + if kept == nil { + kept = map[string]bool{} + } + kept[objectKey(dr.Resource)] = true + } + return desired, kept +} + +// ownershipLogLimit bounds the objects recordOwnershipConflict remembers having logged. +const ownershipLogLimit = 1024 + +// recordOwnershipConflict counts an object another applier owns, kept out of a triggered folder, and +// logs it once per object. +func (l *branchWorkerEventLoop) recordOwnershipConflict(target, object string, applier ApplierRef, owner AppliedBy) { + if telemetry.ApplyOwnershipConflictsTotal != nil { + telemetry.ApplyOwnershipConflictsTotal.Add(context.WithoutCancel(l.w.ctx), 1, + metric.WithAttributes(l.w.providerAttrs()...)) + } + key := target + "|" + object + if _, logged := l.ownershipLogged[key]; logged { + return + } + if l.ownershipLogged == nil || len(l.ownershipLogged) >= ownershipLogLimit { + // The log only spares recent objects a repeat; the counter records every exclusion. + // Objects come and go (an Argo CD hook Job per sync), so it starts over rather than grow. + l.ownershipLogged = map[string]struct{}{} + } + l.ownershipLogged[key] = struct{}{} + l.w.Log.Info("Not mirroring an object another applier owns into a folder its trigger's applier applies: "+ + "two appliers applying one object fight each other", + "gitTarget", target, "object", object, "applier", applier.String(), "ownedBy", owner.String()) +} diff --git a/internal/git/apply_ownership_test.go b/internal/git/apply_ownership_test.go new file mode 100644 index 00000000..f2986823 --- /dev/null +++ b/internal/git/apply_ownership_test.go @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "context" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// F4 (docs/design/applier-echoes.md): an object another applier owns is not mirrored into a folder a +// trigger applies. + +func ownedObject(labels map[string]string, annotations map[string]string) *unstructured.Unstructured { + u := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "v1", "kind": "ConfigMap"}} + u.SetLabels(labels) + u.SetAnnotations(annotations) + return u +} + +func TestAppliedByOf_ReadsFluxLabelsAndArgoCDTracking(t *testing.T) { + cases := []struct { + name string + obj *unstructured.Unstructured + want AppliedBy + }{ + {"no owner", ownedObject(nil, nil), AppliedBy{}}, + {"flux", ownedObject( + map[string]string{fluxOwnerNameLabel: "apps", fluxOwnerNamespaceLabel: "flux-system"}, nil), + AppliedBy{Kind: configv1alpha3.ApplierKindKustomization, Namespace: "flux-system", Name: "apps"}}, + {"flux, name alone", ownedObject(map[string]string{fluxOwnerNameLabel: "apps"}, nil), AppliedBy{}}, + {"argo cd", ownedObject(nil, map[string]string{argoCDTrackingAnnotation: "shop:/ConfigMap:team-a/cfg"}), + AppliedBy{Kind: configv1alpha3.ApplierKindApplication, Name: "shop"}}, + {"argo cd, an application in another namespace", + ownedObject(nil, map[string]string{argoCDTrackingAnnotation: "tenants_shop:apps/Deployment:team-a/api"}), + AppliedBy{Kind: configv1alpha3.ApplierKindApplication, Namespace: "tenants", Name: "shop"}}, + {"argo cd, malformed", ownedObject(nil, map[string]string{argoCDTrackingAnnotation: "shop"}), AppliedBy{}}, + {"nil", nil, AppliedBy{}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, AppliedByOf(tc.obj)) + }) + } +} + +func TestAppliedBy_IsTheApplierItNames(t *testing.T) { + flux := applierRef("alpha") + assert.True(t, AppliedBy{Kind: flux.Kind, Namespace: flux.Namespace, Name: flux.Name}.is(flux)) + assert.False(t, AppliedBy{Kind: flux.Kind, Namespace: "elsewhere", Name: flux.Name}.is(flux)) + assert.False(t, AppliedBy{Kind: configv1alpha3.ApplierKindApplication, Name: flux.Name}.is(flux)) + + argo := argoApplierRef("shop") + assert.True(t, AppliedBy{Kind: argo.Kind, Name: "shop"}.is(argo), "the control plane's own namespace is unnamed") + assert.True(t, AppliedBy{Kind: argo.Kind, Namespace: argo.Namespace, Name: "shop"}.is(argo)) + assert.False(t, AppliedBy{Kind: argo.Kind, Namespace: "tenants", Name: "shop"}.is(argo)) +} + +// TestOwnership_ALiveWriteOfAnotherAppliersObjectIsNotMirrored: in alpha's triggered folder, an +// object another Kustomization owns is kept out, counted; one alpha's applier owns, or nobody does, +// is mirrored. bravo's folder names no trigger and excludes nothing. +func TestOwnership_ALiveWriteOfAnotherAppliersObjectIsNotMirrored(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + loop, _ := manualLoop(t, applierTestTarget("alpha", true), applierTestTarget("bravo", false)) + loop.followRecordedOperations(time.Now()) + other := AppliedBy{Kind: configv1alpha3.ApplierKindKustomization, Namespace: "flux-system", Name: "infra"} + own := AppliedBy{Kind: configv1alpha3.ApplierKindKustomization, Namespace: "team-a", Name: "alpha"} + + assert.True(t, loop.ownedElsewhere("alpha", "team-a", "cm/x", other)) + assert.False(t, loop.ownedElsewhere("alpha", "team-a", "cm/y", own)) + assert.False(t, loop.ownedElsewhere("alpha", "team-a", "cm/z", AppliedBy{})) + assert.False(t, loop.ownedElsewhere("bravo", "team-a", "cm/x", other), "a folder with no trigger excludes nothing") + got, _ := telemetry.CollectInt64Sum(reader, "gitopsreverser_apply_ownership_conflicts_total", nil) + assert.Equal(t, int64(1), got) + + event := windowEvent("alpha", "2", "kubectl-edit") + event.AppliedBy = other + loop.handleLiveEvents(&WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}) + assert.Nil(t, loop.openWindow, "the object another applier owns opened no window") +} + +// TestOwnership_ASnapshotObjectOfAnotherApplierIsNeitherUpsertedNorSwept: the resync drops the object +// from what it upserts, and the sweep leaves its document alone even under prune.mode: Always. +func TestOwnership_ASnapshotObjectOfAnotherApplierIsNeitherUpsertedNorSwept(t *testing.T) { + loop, _ := manualLoop(t, applierTestTarget("alpha", true)) + loop.followRecordedOperations(time.Now()) + theirs := desiredCM("theirs", "red") + ours := desiredCM("ours", "blue") + req := &ResyncRequest{ + GitTargetName: "alpha", GitTargetNamespace: "team-a", + Desired: []manifestanalyzer.DesiredResource{theirs, ours}, + Owners: map[types.ResourceIdentifier]AppliedBy{ + theirs.Resource: {Kind: configv1alpha3.ApplierKindApplication, Name: "infra"}, + }, + } + desired, kept := loop.withoutOwnedElsewhere(req) + require.Len(t, desired, 1) + assert.Equal(t, "ours", desired[0].Resource.Name) + assert.Equal(t, map[string]bool{objectKey(theirs.Resource): true}, kept) + + writer := newContentWriter(types.SensitiveResourcePolicy{}) + worktree := newWorktreeForTest(t) + theirFile := seedPlacedManifest(t, worktree, "apps/theirs.yaml", cmManifest("theirs", "green")) + w := &BranchWorker{contentWriter: writer, mapper: configMapMapper()} + stats, _, _, err := w.planResyncToWorktree(context.Background(), worktree, "", + ResolvedTargetMetadata{PruneMode: configv1alpha3.PruneAlways}, desired, + resyncScopePtr(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, ""), kept) + require.NoError(t, err) + assert.Zero(t, stats.Deleted, "the other applier's document is not swept") + content, err := os.ReadFile(theirFile) + require.NoError(t, err) + assert.Contains(t, string(content), "green", "and not upserted") +} diff --git a/internal/git/apply_recovery.go b/internal/git/apply_recovery.go index 9d5e82dc..f161830b 100644 --- a/internal/git/apply_recovery.go +++ b/internal/git/apply_recovery.go @@ -30,7 +30,7 @@ import ( // followRecordedOperations reads the triggers' recorded status on every pass, and follows what it // finds. Every pass, and not only while the branch has appliers: a GitTarget naming a trigger for // the first time is seen here, and gets its first apply (syncAppliers). The listing is cached. A -// listing that fails is tried again on the next pass. +// listing that fails is tried again on the applier timer (listRetryAt), as takeReasons' is. func (l *branchWorkerEventLoop) followRecordedOperations(now time.Time) { w := l.w if w.applyRequester == nil { @@ -39,9 +39,11 @@ func (l *branchWorkerEventLoop) followRecordedOperations(now time.Time) { targets, err := w.appliersOnBranch(w.ctx) if err != nil { w.Log.Error(err, "Cannot read the apply triggers' recorded operations; retrying", "branch", w.Branch) + l.listRetryAt = time.Now().Add(applierRetryInterval) l.resetApplierTimer(applierRetryInterval) return } + l.listRetryAt = time.Time{} l.syncAppliers(targets) for _, ref := range l.sortedAppliers() { l.followRecordedFor(l.appliers[ref], now) @@ -82,6 +84,7 @@ func recordedStatus(state *applierState) (*v1alpha3.ApplyOperation, bool) { func (l *branchWorkerEventLoop) followRecorded(state *applierState, recorded *v1alpha3.ApplyOperation, now time.Time) { w := l.w l.nextApplyID++ + w.raiseApplyEpoch() abandoned, abandon := context.WithCancel(w.ctx) attempt := &inflightAttempt{ id: l.nextApplyID, abandon: abandon, recovered: true, @@ -89,6 +92,7 @@ func (l *branchWorkerEventLoop) followRecorded(state *applierState, recorded *v1 } state.inflight, state.lastTurn = attempt, attempt.id state.followedSuspend = recorded.SuspendToken + l.reportApplyWindow(state) state.openApplyWindow(attempt.id, recorded.Commit, now) state.operation, state.operationID = recorded.DeepCopy(), attempt.id diff --git a/internal/git/apply_recovery_test.go b/internal/git/apply_recovery_test.go index ec8cc9d4..8ea33c7f 100644 --- a/internal/git/apply_recovery_test.go +++ b/internal/git/apply_recovery_test.go @@ -44,6 +44,7 @@ func TestRecovery_AnOperationLeftInFlightIsFollowedWithItsWindowOpen(t *testing. fluxTrigger("alpha", recording(recorded, false))) writer := &recordingStatusWriter{} loop.w.triggerStatus = writer + loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByFetch}) // the tip it applies loop.followRecordedOperations(time.Now()) @@ -64,7 +65,7 @@ func TestRecovery_AnOperationLeftInFlightIsFollowedWithItsWindowOpen(t *testing. followed[0].Report(ApplyResult{ID: followed[0].ID, Outcome: ApplySettling, SuspendToken: "T2"}) followed[0].Report(ApplyResult{ID: followed[0].ID, Outcome: ApplySucceeded, SuspendToken: "T2", AppliedRevision: "main@sha1:bbb"}) - loop.openWindow = &openWindow{} // busy: the repair the withheld write left pending waits + holdApplies(loop) // the repair the withheld write left pending waits loop.serviceAppliers() status, _ := writer.last(applierRef("alpha").Trigger) completed := status.Operation @@ -75,7 +76,7 @@ func TestRecovery_AnOperationLeftInFlightIsFollowedWithItsWindowOpen(t *testing. assert.Equal(t, "T2", completed.SuspendToken) assert.Empty(t, requester.taken()) - loop.openWindow, loop.quietSince = nil, time.Time{} + releaseApplies(loop) loop.serviceAppliers() repair := requester.taken() require.Len(t, repair, 1, "the withheld write leaves a repair pending, as in a process that never stopped") @@ -87,11 +88,12 @@ func TestRecovery_AnOperationLeftInFlightIsFollowedWithItsWindowOpen(t *testing. func TestRecovery_APendingRepairIsRestored(t *testing.T) { loop, requester := manualLoop(t, applierTestTarget("alpha", true), fluxTrigger("alpha", recording(recordedOperation(configv1alpha3.ApplyOperationSucceeded), true))) + loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByFetch}) // the tip it applied loop.followRecordedOperations(time.Now()) assert.True(t, onlyApplier(t, loop).repair) loop.serviceAppliers() repair := requester.taken() - require.Len(t, repair, 1, "the repair is sent on the quiet branch") + require.Len(t, repair, 1, "the repair is sent after a flush") assert.Equal(t, configv1alpha3.ApplyReasonRepair, repair[0].Reason) assert.Nil(t, repair[0].Recovered) } diff --git a/internal/git/apply_request.go b/internal/git/apply_request.go index a9cc8363..37cf0ea9 100644 --- a/internal/git/apply_request.go +++ b/internal/git/apply_request.go @@ -36,11 +36,6 @@ func (a ApplierRef) String() string { return a.Cluster + "/" + a.Kind + "/" + a.Namespace + "/" + a.Name } -// CatchesUp reports whether the applier is sent a CatchUp after GitOps Reverser's own pushes. An -// Argo CD Application is not: with automated sync off it applies nothing on its own, so it holds no -// stale state for a catch-up to correct. -func (a ApplierRef) CatchesUp() bool { return a.Kind != v1alpha3.ApplierKindApplication } - // TriggerRef is one apply trigger: its kind (v1alpha3.ApplyTriggerKind*), namespace and name. type TriggerRef struct { Kind string @@ -83,10 +78,10 @@ func ResolveApplyTrigger( return trigger, nil } -// ApplyRequest is one attempt, decided by a branch worker on a quiet branch, to have one applier +// ApplyRequest is one attempt, decided by a branch worker after a flush, to have one applier // apply a commit. The worker decides every attempt, retries included; the requester only carries it // out, once, off the worker's goroutine, and reports how it went. So nothing is ever sent that the -// worker did not check on a quiet branch, against the GitTargets as they were at that moment. +// worker did not check after a flush, against the GitTargets as they were at that moment. type ApplyRequest struct { // ID identifies the attempt, so its report can be matched to it. ID uint64 @@ -109,11 +104,11 @@ type ApplyRequest struct { // requester reports a failure only for an attempt at zero, so a standing failure is reported // once rather than on every retry. PriorFailures int - // Current reports whether the decision still holds: Commit is still the branch tip, and nothing - // live is waiting to be published on this worker or a sibling. The requester asks it just before - // the write that starts the apply, and asks nothing when it is false (ApplyStale): an apply of an - // older commit, or over an edit not yet pushed, is exactly what the quiet-branch rule exists to - // prevent. Nil is always current. It does not block. + // Current reports whether the decision still holds: Commit is still the branch tip. The requester + // asks it just before the write that starts the apply, and asks nothing when it is false + // (ApplyStale): an apply of an older commit would revert what the newer one holds. An edit not + // yet pushed does not make it stale: the flush published every edit decided before the attempt, + // and the apply window holds the rest. Nil is always current. It does not block. Current func() bool // Record writes the operation this attempt starts on its trigger's status, and returns once it // is written: the only way the operation reaches the trigger before the requester reports it. @@ -168,9 +163,6 @@ const ( // ApplyFailed is an operation that completed without applying the commit: the applier reported // a failure, or the operation is gone. It may have applied part of it. ApplyFailed ApplyOutcome = "Failed" - // ApplyCurrent asked nothing: a CatchUp or ForeignCommit found the applier already at the - // commit (a suspended Kustomization that applied it), or the remote does not carry the branch. - ApplyCurrent ApplyOutcome = "Current" // ApplyRequestFailed could not ask; a later attempt may. ApplyRequestFailed ApplyOutcome = "RequestFailed" // ApplyRejected could not ask, and asking again will not help until the applier's @@ -181,8 +173,8 @@ const ( // consent to a Refusal. The worker decides again from what the targets are now. ApplyDropped ApplyOutcome = "Dropped" // ApplyStale asked nothing, because the decision no longer held when the write was due: the - // branch moved on, or became busy (ApplyRequest.Current). It is no failure; the worker decides - // again at the next quiet moment, at the tip as it is then. + // branch moved on (ApplyRequest.Current). It is no failure; the worker decides again at once, at + // the tip as it is then. ApplyStale ApplyOutcome = "Stale" ) @@ -191,7 +183,7 @@ func (o ApplyOutcome) Ends() bool { switch o { case ApplyRequested, ApplyRunning, ApplySettling, ApplyUnresolved: return false - case ApplySucceeded, ApplyFailed, ApplyCurrent, ApplyRequestFailed, ApplyRejected, ApplyDropped, ApplyStale: + case ApplySucceeded, ApplyFailed, ApplyRequestFailed, ApplyRejected, ApplyDropped, ApplyStale: } return true } @@ -211,6 +203,10 @@ type ApplyResult struct { AppliedRevision string // Message says what a Succeeded operation applied instead of the commit. Message string + // SourceRevision is the revision the applier's source fetched when it holds another commit than + // the one asked for (a RequestFailed carries it). It is a hint that the branch moved, never a + // fact: the worker looks at the remote itself (settleApplyResults). + SourceRevision string // Err is the cause of a RequestFailed, Rejected or Failed attempt. Err error } @@ -243,25 +239,15 @@ type ApplyTriggerStatusWriter interface { RecordApplyOperation(ctx context.Context, trigger TriggerRef, operation *v1alpha3.ApplyOperation) error } -// ApplyRequester carries out apply requests. RequestApply and PrefetchSource must not block: they -// are called on the branch worker's loop goroutine. +// ApplyRequester carries out apply requests. RequestApply must not block: it is called on the branch +// worker's loop goroutine. type ApplyRequester interface { RequestApply(req ApplyRequest) - // PrefetchSource asks the source req.Applier applies to fetch req.Commit now, ahead of an apply, - // when its trigger says it may and no operation on that source is in flight. It applies nothing - // and reports nothing. - PrefetchSource(req PrefetchRequest) -} - -// PrefetchRequest asks the source an applier applies to fetch a commit ahead of its apply. -type PrefetchRequest struct { - Applier ApplierRef - Commit string } // StrongerApplyReason is the reason a target is asked for when two reasons are pending for it. A -// CatchUp asks for the least: it asks nothing of a Kustomization that is suspended with the commit -// applied, where the others always run the operation, so it never replaces any of them. A Repair and a ForeignCommit +// CatchUp asks for the least: any apply of the tip adopts and converges, so it never replaces any +// of the others. A Repair and a ForeignCommit // outrank a Refusal because they need no consent, and a ForeignCommit outranks a Repair because the // commit it applies is news to the applier. The empty reason ranks below all four. func StrongerApplyReason(a, b v1alpha3.ApplyReason) v1alpha3.ApplyReason { diff --git a/internal/git/apply_sibling_flush.go b/internal/git/apply_sibling_flush.go new file mode 100644 index 00000000..05f490fa --- /dev/null +++ b/internal/git/apply_sibling_flush.go @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// Flush, then apply (docs/design/applier-echoes.md, S1), across the workers writing one branch. Two +// GitProviders naming one repository give its branch two workers, and their GitTargets can name one +// trigger. An apply one of them sends writes the commit it was sent for over every object the applier +// owns, so a sibling's edit not published by then is reverted on the cluster, and the sibling, which +// sent nothing and has no apply window, mirrors the revert as an edit of its own. +// +// So the flush covers the siblings: when an apply comes due, the worker asks each sibling to flush, +// and each closes its open window, takes a mark of its own, and publishes up to it at once. The +// worker sends once every sibling it asked is done, for the tip their publications produced, which +// they shared with it (shareObservation), and which is foreign to it. As in S1, a sibling's writes decided after its mark do not +// hold the send; an edit a sibling makes while the apply runs is not protected, as before. + +// askFlush asks every other worker on w's repository and branch to flush, and returns the request's +// generation, zero when w has no sibling. A sibling asked a later generation has flushed for this +// one too, because its mark is taken after this one was asked. +func (s *branchSiblings) askFlush(w *BranchWorker) uint64 { + key, ok := siblingKey(w) + if s == nil || !ok { + return 0 + } + s.mu.Lock() + var others []*BranchWorker + for sibling := range s.byKey[key] { + if sibling != w { + others = append(others, sibling) + } + } + if len(others) == 0 { + s.mu.Unlock() + return 0 + } + s.flushGen++ + gen := s.flushGen + for _, sibling := range others { + sibling.flushAsked.Store(max(sibling.flushAsked.Load(), gen)) + } + s.mu.Unlock() + for _, sibling := range others { + sibling.wakeApplier() + } + return gen +} + +// othersFlushed reports whether every other worker on w's repository and branch that was asked for +// gen has flushed for it. A worker that joined after the request was never asked, and a worker that +// stopped is not waited for: what it held is published or gone with it. +func (s *branchSiblings) othersFlushed(w *BranchWorker, gen uint64) bool { + key, ok := siblingKey(w) + if !ok { + return true + } + s.mu.Lock() + defer s.mu.Unlock() + for sibling := range s.byKey[key] { + if sibling != w && sibling.flushAsked.Load() >= gen && sibling.flushDone.Load() < gen { + return false + } + } + return true +} + +// takeSiblingFlush starts a flush a sibling asked for: the open window is closed, and the mark is the +// last write decided so far. It runs at the end of every wake, before the wake's writes are advanced, +// so they are published at once (publishesAtOnce). A save, a parent recovery or a dirty checkout +// hold it, as they hold the worker's own flush; the asking sibling waits for them. +func (l *branchWorkerEventLoop) takeSiblingFlush() { + asked := l.w.flushAsked.Load() + if asked <= l.siblingFlushGen || l.writeBusy() { + return + } + l.finalizeOpenWindowWithReason(windowFinalizeReasonApplyDue) + l.siblingFlushGen, l.siblingFlushMark = asked, l.decisions +} + +// reportSiblingFlush says the sibling flush taken is done once every write up to its mark is +// published, and wakes the siblings, so the one waiting sends. +func (l *branchWorkerEventLoop) reportSiblingFlush() { + if l.siblingFlushGen <= l.w.flushDone.Load() || l.pendingUpTo(l.siblingFlushMark) { + return + } + l.w.flushDone.Store(l.siblingFlushGen) + if l.w.siblings != nil { + l.w.siblings.wakeOthers(l.w) + } +} + +// siblingsFlushed reports whether the siblings asked when this worker's flush began have flushed. +// Their publications moved the branch, and each one's observation of its push reached this worker's +// inbox (shareObservation) before it said it was done, so the tip the attempt is sent for holds them. +func (l *branchWorkerEventLoop) siblingsFlushed() bool { + return l.siblingAsk == 0 || l.w.siblings.othersFlushed(l.w, l.siblingAsk) +} + +// shareObservation tells every other worker on w's repository and branch what w observed of the +// branch, so the tip each one sends an attempt for holds w's publications, and wakes them. To them +// the commit is somebody else's, as a fetch of it would have told them: w's appliers may apply +// another cluster than theirs, or their folders, so nothing says its content is live where their +// appliers apply. Their applier is asked for it, and an echo of content already live there is not +// mirrored (F1). +func (s *branchSiblings) shareObservation(w *BranchWorker, observed RemoteObservation) { + key, ok := siblingKey(w) + if s == nil || !ok { + return + } + s.mu.Lock() + others := make([]*BranchWorker, 0, len(s.byKey[key])) + for sibling := range s.byKey[key] { + if sibling != w { + others = append(others, sibling) + } + } + s.mu.Unlock() + shared := observed + if shared.By == ObservedByPush { + shared.By = ObservedByFetch + } + for _, sibling := range others { + sibling.inbox.observe(shared) + sibling.wakeApplier() + } +} + +// publishesAtOnce reports whether a write the pending writes hold is waited for by a flush, this +// worker's or a sibling's: it is pushed at once, cooldown or not. +func (l *branchWorkerEventLoop) publishesAtOnce() bool { + return (l.flushing && l.pendingUpTo(l.flushMark)) || + (l.siblingFlushGen > l.w.flushDone.Load() && l.pendingUpTo(l.siblingFlushMark)) +} diff --git a/internal/git/apply_siblings.go b/internal/git/apply_siblings.go index 33bb8ab2..269bbb4f 100644 --- a/internal/git/apply_siblings.go +++ b/internal/git/apply_siblings.go @@ -9,15 +9,16 @@ import ( ) // A branch worker is keyed by its GitProvider, so two GitProviders naming one repository give one -// branch two workers. A Kustomization applies the tip of that branch whichever of them wrote it, so -// the branch is quiet only when BOTH are: an apply one worker asks for while the other holds an open -// window would revert that window's edits before they reach Git. branchSiblings is what lets each -// see the other. +// branch two workers. Their appliers can apply one GitRepository, and a source group has one +// operation at a time (apply_groups.go), so the workers claim the group before they send. +// branchSiblings is what lets each see the other's claims. // branchSiblings groups the live workers by the repository and branch they write. type branchSiblings struct { mu sync.Mutex byKey map[string]map[*BranchWorker]struct{} + // flushGen numbers the flushes workers ask of their siblings (apply_sibling_flush.go). + flushGen uint64 } func newBranchSiblings() *branchSiblings { @@ -49,7 +50,7 @@ func (s *branchSiblings) add(w *BranchWorker) { w.siblings = s } -// remove forgets a worker that is stopping, and wakes the others: whatever it held busy is gone. +// remove forgets a worker that is stopping, and wakes the others: the groups it claimed are released. func (s *branchSiblings) remove(w *BranchWorker) { key, ok := siblingKey(w) if s == nil || !ok { @@ -64,29 +65,8 @@ func (s *branchSiblings) remove(w *BranchWorker) { s.wakeOthers(w) } -// othersBusy reports whether another worker on w's repository and branch is busy writing, and -// whether one has work items queued or in hand: a queued watch event is an edit that is live and -// not yet decided, and the item a loop is handling may be committing and pushing before the wake -// that publishes it ends. -func (s *branchSiblings) othersBusy(w *BranchWorker) (bool, bool) { - key, ok := siblingKey(w) - if !ok { - return false, false - } - s.mu.Lock() - defer s.mu.Unlock() - writing, working := false, false - for sibling := range s.byKey[key] { - if sibling != w { - writing = writing || sibling.busyWriting.Load() - working = working || sibling.inflightItems.Load() > 0 - } - } - return writing, working -} - // wakeOthers wakes every other worker on w's repository and branch, so one that held back an -// attempt because w was busy looks again. +// attempt for a group w had out looks again. func (s *branchSiblings) wakeOthers(w *BranchWorker) { key, ok := siblingKey(w) if !ok { @@ -105,14 +85,6 @@ func (s *branchSiblings) wakeOthers(w *BranchWorker) { } } -// publishWriteBusy records whether this worker holds live writes not yet on the remote, and wakes -// its siblings when it stops: they may have held an attempt back for it. -func (w *BranchWorker) publishWriteBusy(busy bool) { - if was := w.busyWriting.Swap(busy); was && !busy && w.siblings != nil { - w.siblings.wakeOthers(w) - } -} - // claimGroup records that w sends an attempt for an applier of group, and reports false, recording // nothing, when another worker on its repository and branch has one out for that group. Checking // and recording under one lock is what keeps two workers from sending for one group at once. @@ -185,19 +157,7 @@ func (w *BranchWorker) groupOutOnSibling(group string) bool { return group != "" && w.siblings != nil && w.siblings.groupOutElsewhere(w, group) } -// releaseInflightItem takes one item off the in-flight count, and wakes the siblings when it was -// the last: they may have held an attempt back for it. +// releaseInflightItem takes one item off the in-flight count, which the queue-depth gauge reads. func (w *BranchWorker) releaseInflightItem() { - if w.inflightItems.Add(-1) == 0 && w.siblings != nil { - w.siblings.wakeOthers(w) - } -} - -// siblingBusy reports whether another worker on this repository and branch is busy writing, and -// whether one has work items queued or in hand (othersBusy). -func (w *BranchWorker) siblingBusy() (bool, bool) { - if w.siblings == nil { - return false, false - } - return w.siblings.othersBusy(w) + w.inflightItems.Add(-1) } diff --git a/internal/git/apply_trigger.go b/internal/git/apply_trigger.go index 233ba21b..8ab72943 100644 --- a/internal/git/apply_trigger.go +++ b/internal/git/apply_trigger.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "sort" + "strings" "sync" "sync/atomic" "time" @@ -20,39 +21,45 @@ import ( ) // This file is the branch worker's half of the apply trigger: deciding WHEN the appliers of the -// GitTargets on this branch apply. See docs/design/apply-trigger.md. +// GitTargets on this branch apply. See docs/design/apply-trigger.md and +// docs/design/applier-echoes.md. // -// An apply is safe only when nothing live is missing from the commit being applied, and only the -// workers on this branch can see the edits that are live and not yet pushed: their open commit -// windows and pending writes. So an attempt goes out only on a QUIET branch, with neither on this -// worker or on any other worker writing the same repository and branch. Four reasons ask for one: +// An apply must not revert an edit that is live and not yet in Git, and only the worker can see +// those: its open commit window and pending writes. So an attempt goes out after a FLUSH: the open +// window is closed, the writes decided up to that moment (the flush mark) are published, and the +// attempt is sent for the tip that publication produced. An edit decided after the mark is recorded +// in the apply window, so an apply that overwrites it is withheld and a Repair puts it back +// (apply_window.go). Four reasons ask for an attempt: // // - ForeignCommit: an observation of the remote found the branch somewhere this worker did not -// push it. Every applier on the branch is asked as soon as the branch is quiet, and an open -// window is closed rather than waited out, because a user who keeps editing would otherwise -// hold a Git-side change back for as long as they type. +// push it. Every applier on the branch is asked. // - Refusal: a refused edit on a target with spec.onRefusal: Reapply (refusal_reapply.go). -// - CatchUp: this worker's own pushes. A suspended Kustomization applies nothing on its own, so -// this is convergence, not safety: Flux's inventory learns of an object only from an apply of a -// commit that holds it, and an object created through the API and never applied is not pruned -// when Git deletes it. Asked only once the branch has stayed quiet for the trigger's quiet -// period, and never of an Argo CD Application (decided without it; ApplierRef.CatchesUp). +// - CatchUp: ownership, not safety, the same for Flux and Argo CD. An applier prunes only what it +// applied: an object the API created in its folder carries no owner labels and no tracking +// annotation, so when Git later deletes its file the applier leaves it on the cluster. A CatchUp +// ADOPTS it: it is pending once the worker observes an API create in a folder an applier applies, +// or a resync adds an object to Git, and is never skipped as already applied, because applying +// the same commit again is what stamps the object. A CatchUp also CONVERGES: once when the worker +// starts, and once when a GitTarget first names the trigger; that one asks nothing of an applier +// that already applied the tip (S2 in docs/design/applier-echoes.md). // - Repair: the apply window withheld a late apply over a newer API edit (apply_window.go), so -// the cluster runs an older value than Git. Asked like a Refusal: on a quiet branch, with no -// quiet period, and of either kind of applier, because nothing else would put the edit back on -// an Argo CD Application. +// the cluster runs an older value than Git. Asked like a Refusal, and of +// either kind of applier, because nothing else would put the edit back on an Argo CD +// Application. // -// The three relations are kept apart on purpose. The BRANCH is the unit of quietness, because a -// Kustomization applies the tip whichever GitTarget's folder an edit is in. The APPLIER is the unit -// of pending reasons: one attempt in flight at a time, its own retry clock, and one attempt for all -// the GitTargets that name it. The GITTARGET, by uid, is the unit of configuration and consent. +// Two relations are kept apart on purpose. The APPLIER is the unit of pending reasons: one attempt in +// flight at a time, its own retry clock, and one attempt for all the GitTargets that name it. The +// GITTARGET, by uid, is the unit of configuration and consent. +// +// No applier is asked for anything, for any reason, before every initial snapshot of the GitTargets +// it covers is published: at startup the edits made while the operator was down are then in Git, and +// a newly named GitTarget's folder holds the cluster. An empty tip asks nothing. // // The worker keeps every pending reason until an operation answers it. The requester carries out one // attempt and reports its operation as it moves; it never retries, so a retry is decided here like -// any first attempt, on a quiet branch, against what the targets are now. An operation is followed -// for as long as it takes: one past the requester's budget is Unresolved, and keeps its window open -// and its applier's next attempt back, because a timer that closes the window does not stop the -// apply. +// any first attempt, after a flush, against what the targets are now. An operation is followed for as +// long as it takes: one past the requester's budget is Unresolved, and keeps its window open and its +// applier's next attempt back, because a timer that closes the window does not stop the apply. // applyInbox holds what other goroutines tell the loop about applies: the reasons pending for the // branch's appliers because of what was OBSERVED about the remote, and the reports of attempts that @@ -65,66 +72,53 @@ type applyInbox struct { // means the remote does not carry the branch. known bool tip string - // foreign and catchUp are pending until the loop takes them on a quiet branch. prefetch is a push - // somebody else made, until the loop takes it (apply_groups.go). + // foreign and converge are pending until the loop takes them onto the appliers. foreign bool - catchUp bool - prefetch bool + converge bool // results are the reports of finished attempts, until the loop reads them. results []ApplyResult } // observe classifies one observation. The first one is a baseline, not a move: nothing says what -// the appliers last applied, so a CatchUp is pending for them, never a ForeignCommit. A request on a -// restart would otherwise apply the tip before the watches have captured the edits made while the -// operator was down. +// the appliers last applied, so the startup converge is pending for them, never a ForeignCommit. A +// request on a restart would otherwise apply the tip before the watches have captured the edits made +// while the operator was down. // // After that, a commit that differs from the last one observed is either this worker's push, proved -// by the push itself, or somebody else's, because every other observation is a fetch. When in doubt -// the classification errs towards foreign, which costs one apply of the tip on a quiet branch. +// by the push itself, which leaves nothing pending, or somebody else's, because every other +// observation is a fetch. When in doubt the classification errs towards foreign, which costs one +// apply of the tip. func (d *applyInbox) observe(observed RemoteObservation) { d.mu.Lock() defer d.mu.Unlock() if !d.known { - d.known, d.tip, d.catchUp = true, observed.Commit, true + d.known, d.tip, d.converge = true, observed.Commit, true return } if observed.Commit == d.tip { return } d.tip = observed.Commit - if observed.By == ObservedByPush { - d.catchUp = true - return + if observed.By != ObservedByPush { + d.foreign = true } - d.foreign, d.prefetch = true, true -} - -// takePrefetch returns whether a push somebody else made is to be fetched ahead, with the tip, and -// clears it. -func (d *applyInbox) takePrefetch() (string, bool) { - d.mu.Lock() - defer d.mu.Unlock() - due := d.prefetch - d.prefetch = false - return d.tip, due } // take returns the pending reasons and clears them. The caller restores it if it cannot act on it. func (d *applyInbox) take() (string, bool, bool) { d.mu.Lock() defer d.mu.Unlock() - tip, foreign, catchUp := d.tip, d.foreign, d.catchUp - d.foreign, d.catchUp = false, false - return tip, foreign, catchUp + tip, foreign, converge := d.tip, d.foreign, d.converge + d.foreign, d.converge = false, false + return tip, foreign, converge } // restore puts back what take returned, for a caller that could not act on it. -func (d *applyInbox) restore(foreign, catchUp bool) { +func (d *applyInbox) restore(foreign, converge bool) { d.mu.Lock() defer d.mu.Unlock() d.foreign = d.foreign || foreign - d.catchUp = d.catchUp || catchUp + d.converge = d.converge || converge } // foreignPending reports whether a ForeignCommit is pending, without taking it. @@ -134,6 +128,14 @@ func (d *applyInbox) foreignPending() bool { return d.foreign } +// lastTip is the last commit observed, empty before the first observation or when the remote does +// not carry the branch. +func (d *applyInbox) lastTip() string { + d.mu.Lock() + defer d.mu.Unlock() + return d.tip +} + // tipIs reports whether the last commit observed is commit. func (d *applyInbox) tipIs(commit string) bool { d.mu.Lock() @@ -160,10 +162,9 @@ const applierRetryInterval = 10 * time.Second // branchTarget is one GitTarget on this branch that names an applier, as listed on this pass. type branchTarget struct { - ref itypes.ResourceReference - uid k8stypes.UID - applier ApplierRef - quietPeriod time.Duration + ref itypes.ResourceReference + uid k8stypes.UID + applier ApplierRef // consents is whether it asks for a Refusal request now (ConsentsToReapply). consents bool // recorded and repairPending are the trigger's status.operation and status.repairPending, as listed: @@ -171,10 +172,8 @@ type branchTarget struct { recorded *v1alpha3.ApplyOperation repairPending bool // group names the GitRepository a Flux trigger's Kustomization applies, in its cluster; empty - // for an Argo CD Application or a source not read yet. prefetch is the trigger's - // status.source.prefetch (apply_groups.go). - group string - prefetch bool + // for an Argo CD Application or a source not read yet (apply_groups.go). + group string } // appliersOnBranch lists the GitTargets on this worker's (provider, branch) that name an apply @@ -205,12 +204,10 @@ func (w *BranchWorker) appliersOnBranch(ctx context.Context) ([]branchTarget, er ref: itypes.NewResourceReference(target.Name, target.Namespace), uid: target.UID, applier: ApplierOf(trigger), - quietPeriod: trigger.EffectiveQuietPeriod(), consents: ConsentsToReapply(target), recorded: trigger.Operation(), repairPending: trigger.RepairPending(), group: sourceGroup(trigger), - prefetch: trigger.Prefetches(), }) } sort.Slice(listed, func(i, j int) bool { return listed[i].ref.String() < listed[j].ref.String() }) @@ -222,10 +219,19 @@ type applierState struct { ref ApplierRef // targets are the GitTargets that name it, as listed on the last pass. targets []branchTarget - // foreign, catchUp and repair are pending until an attempt carries them. - foreign bool - catchUp bool - repair bool + // foreign, adopt, converge and repair are pending until an attempt carries them. adopt and + // converge are both asked as a CatchUp. + foreign bool + adopt bool + converge bool + repair bool + // named is the GitTarget incarnations that named the applier so far: one naming it for the first + // time leaves a converge pending. + named map[k8stypes.UID]bool + // appliedRevision is the revision the last operation that succeeded applied, as the applier + // reported it, recovered on restart from the trigger's status.operation. A converge of the commit + // it names asks nothing. + appliedRevision string // inflight is the attempt whose report has not ended it yet. While it is out, nothing else is // sent to this applier: a reason that becomes pending meanwhile waits for it. inflight *inflightAttempt @@ -240,6 +246,10 @@ type applierState struct { // finished it (apply_window.go). Nil when no attempt was sent, or the last asked for no apply // after the window before it had closed. window *applyWindow + // flushEdits records the API-side edits made since the flush mark of an attempt that is due and + // not sent yet, by objectKey: they are not in the commit the attempt applies, so its window holds + // them from the start (S1). Nil while no flush is waiting for this applier. + flushEdits map[string]time.Time // operation is the last operation an attempt started, as its trigger's status.operation shows // it, and operationID the attempt it belongs to. Nil until this worker started one. operation *v1alpha3.ApplyOperation @@ -266,7 +276,8 @@ func (s *applierState) repairPending() bool { type inflightAttempt struct { id uint64 foreign bool - catchUp bool + adopt bool + converge bool repair bool refusals map[refusalKey]unsentRefusal // phase is the last report of its operation, empty before the first. @@ -286,45 +297,49 @@ type inflightAttempt struct { abandon context.CancelFunc } -// quietPeriod is how long the branch must be quiet before a CatchUp: the trigger's, as listed with -// each GitTarget naming it. The longest wins should a listing straddle an edit of the trigger. -func (s *applierState) quietPeriod() time.Duration { - var longest time.Duration - for _, target := range s.targets { - longest = max(longest, target.quietPeriod) - } - return longest +// hasReason reports whether a reason is pending for the applier, given its unsent refusals. +func (s *applierState) hasReason(refusals map[refusalKey]unsentRefusal) bool { + return s.foreign || s.repair || s.adopt || s.converge || len(refusals) > 0 } -// writeBusy reports whether the loop holds anything live that is not yet on the remote, or is in a -// state where it cannot say. A request is never sent while it does. A registered CommitRequest -// counts: a save is announced before its writes arrive, and an apply landing between the two would -// revert the save's first edits before they reach Git. +// barrierRecheckInterval is how soon a reason held back by the initial-snapshot barrier is looked at +// again: the barrier lifts when the watch side and a publication say so, neither of which wakes the +// loop on its own. +const barrierRecheckInterval = time.Second + +// writeBusy reports whether the loop is in a state in which no flush can be made, and so no request +// sent: a registered CommitRequest, whose save defines its own commit boundary (a save is announced +// before its writes arrive, and an apply landing between the two would revert its first edits before +// they reach Git), a parent recovery, or a checkout a failed write left dirty. In the last two the +// worker cannot say what is published. func (l *branchWorkerEventLoop) writeBusy() bool { - return l.openWindow != nil || len(l.pendingWrites) > 0 || len(l.deferredHeals) > 0 || - len(l.pendingCRs) > 0 || l.recovery.active || l.w.worktreeDirty() + return len(l.pendingCRs) > 0 || l.recovery.active || l.w.worktreeDirty() } -// closeWindowForForeignCommit closes the open commit window early when the branch has moved under -// it, or a Git host has said it may have, so its writes are published, and replayed onto the move, -// now rather than when the window's timers allow. See the file comment for why not wait. -func (l *branchWorkerEventLoop) closeWindowForForeignCommit() { +// closeWindowForPushHint closes the open commit window early when a Git host has said the branch may +// have moved, so its writes are published, and replayed onto the move, now rather than when the +// window's timers allow. A window a CommitRequest holds is left to its save. +func (l *branchWorkerEventLoop) closeWindowForPushHint() { hinted := l.pushHinted l.pushHinted = false - if l.openWindow == nil || l.w.applyRequester == nil { + if !hinted || l.openWindow == nil || l.openWindow.pendingCR != nil || l.w.applyRequester == nil { return } - if !hinted && !l.w.inbox.foreignPending() { - return - } - l.w.Log.Info("The branch moved, or may have; closing the commit window to publish it now", + l.w.Log.Info("A Git host reported a push; closing the commit window to publish it now", "branch", l.w.Branch) l.finalizeOpenWindowWithReason(windowFinalizeReasonForeignCommit) } -// serviceAppliers settles the reports that came back, and sends whatever attempts are due if the -// branch is quiet. It runs at the end of every wake, after the wake's writes have been decided and -// committed, so "quiet" is what the wake left behind. +// serviceAppliers settles the reports that came back, and sends whatever attempts are due, after a +// flush (docs/design/applier-echoes.md, S1). It runs at the end of every wake, after the wake's +// writes have been decided and committed. +// +// When an apply comes due, the open commit window is closed, so its edits become a decided write, +// and the flush mark is taken: the last write decided so far. From the mark, the edits the due +// appliers' next apply windows will hold are recorded (withholdLateApply). The writes up to the mark +// are published, at once, and the attempts are sent for the tip that publication produced. Writes +// decided after the mark do not hold the send, so continuous editing cannot starve an apply: an edit +// decided after it is recorded, and an apply that overwrites it is withheld and repaired. func (l *branchWorkerEventLoop) serviceAppliers() { w := l.w if w.applyRequester == nil { @@ -335,59 +350,207 @@ func (l *branchWorkerEventLoop) serviceAppliers() { now := time.Now() l.settleApplyResults(now) l.followRecordedOperations(now) - if !l.branchQuiet(now) || l.workWaiting() { - // Queued work, here or on a sibling: the next wake looks again, and a sibling's loop wakes - // this one once it has handled its last item. The branch is still quiet until that work - // opens a window, so quietSince is kept. A push observed meanwhile is fetched ahead of the - // apply. - l.prefetchSources() + work := l.takeReasons() + // A listing that failed is retried on the applier timer (listRetryAt), which every arming below + // keeps. + closing := earliest(earliest(l.reportApplyWindows(now), l.snapshotHoldDeadline()), l.listRetryAt) + + if !work { + l.endFlush() + l.armApplierTimer(now, closing) return } - // Quiet: the attempts sent now fetch what they apply themselves. - w.inbox.takePrefetch() - - tip, foreign, catchUp := w.inbox.take() - if !foreign && !catchUp && len(l.unsentRefusals) == 0 && !l.appliersHaveWork() { - l.stopApplierTimer() + due, wait := l.dueAppliers(now) + wait = earliest(wait, closing) + if len(due) == 0 { + l.endFlush() + l.armApplierTimer(now, wait) return } + if !l.flushed(due) { + // The publication wakes the loop when it lands (a push, or its retry); a save, a parent + // recovery or a dirty checkout end with a wake of their own, and so does a sibling's flush. + l.armApplierTimer(now, wait) + return + } + // Publishing may have observed news: a rejected push fetches, and a foreign commit it found is + // pending for every applier now. The attempts carry it, for the tip that holds both sides. + l.takeReasons() + tip := w.inbox.lastTip() + for _, state := range due { + if l.attemptOut(state) || !l.w.claimGroup(state.group()) { + // A sibling sent one for the group since; its end wakes this worker. + continue + } + l.sendAttempt(state, tip, l.unsentRefusalsFor(state)) + } + l.endFlush() + l.armApplierTimer(now, wait) +} + +// takeReasons moves the reasons the inbox holds onto the appliers listed now, and reports whether +// any applier has work. A listing that fails leaves them in the inbox, and sets listRetryAt, when +// the applier timer looks again; only a listing that succeeds clears it (followRecordedOperations +// lists the same appliers first, every pass). +func (l *branchWorkerEventLoop) takeReasons() bool { + w := l.w + _, foreign, converge := w.inbox.take() + if !foreign && !converge && len(l.unsentRefusals) == 0 && !l.appliersHaveWork() { + return false + } targets, err := w.appliersOnBranch(w.ctx) if err != nil { - w.inbox.restore(foreign, catchUp) + w.inbox.restore(foreign, converge) w.Log.Error(err, "Cannot list the appliers on this branch; retrying", "branch", w.Branch) - l.resetApplierTimer(applierRetryInterval) - return + l.listRetryAt = time.Now().Add(applierRetryInterval) + return false } + l.listRetryAt = time.Time{} l.syncAppliers(targets) for _, state := range l.appliers { state.foreign = state.foreign || foreign - state.catchUp = state.catchUp || (catchUp && state.ref.CatchesUp()) + state.converge = state.converge || converge } - l.armApplierTimer(now, l.sendDue(tip, now)) + return true } -// branchQuiet reports whether the branch holds nothing live that is not yet on the remote, on this -// worker or on any other writing the same repository and branch, and keeps quietSince: when it last -// became quiet, zero while it is busy. -func (l *branchWorkerEventLoop) branchQuiet(now time.Time) bool { - busy := l.writeBusy() - l.w.publishWriteBusy(busy) - siblingWriting, _ := l.w.siblingBusy() - if busy || siblingWriting { - l.quietSince = time.Time{} - l.stopApplierTimer() - return false +// dueAppliers is the appliers an attempt is due for now, in turn order, and the earliest time one +// still waiting comes due. The appliers are taken in turn order, so in a group, which has one attempt +// out at a time, the attempt goes to the applier whose turn is longest ago. +// +// A reason waits for the applier's backoff, and for the initial-snapshot barrier (pastBarrier). An +// empty tip asks nothing: the remote does not carry the branch, and the source reports that itself; +// the reason stays pending for when it does. A converge alone of the commit the applier last applied +// asks nothing, and is answered. +func (l *branchWorkerEventLoop) dueAppliers(now time.Time) ([]*applierState, time.Time) { + var due []*applierState + var wait time.Time + tip := l.w.inbox.lastTip() + for _, ref := range l.turnOrder() { + state := l.appliers[ref] + if l.attemptOut(state) { + continue + } + refusals := l.unsentRefusalsFor(state) + if l.alreadyConverged(state, refusals, tip) { + state.converge = false + l.w.Log.V(1).Info("The applier already applied the tip; the converge asks nothing", + "branch", l.w.Branch, "applier", state.ref.String(), "commit", tip) + } + switch { + case !state.hasReason(refusals) || tip == "": + case !l.pastBarrier(state): + wait = earliest(wait, now.Add(barrierRecheckInterval)) + case now.Before(state.retryAt): + wait = earliest(wait, state.retryAt) + default: + due = append(due, state) + } } - if l.quietSince.IsZero() { - l.quietSince = now + return due, wait +} + +// alreadyConverged reports whether only a converge is pending for the applier, and it already applied +// the tip. An adopt is never skipped this way: the applier may be at the tip and still not own an +// object the API created, and applying the same commit again is what stamps it. +func (l *branchWorkerEventLoop) alreadyConverged( + state *applierState, refusals map[refusalKey]unsentRefusal, tip string, +) bool { + return state.converge && !state.adopt && !state.foreign && !state.repair && len(refusals) == 0 && + revisionNamesCommit(state.appliedRevision, tip) +} + +// pastBarrier reports whether every initial snapshot of every GitTarget the applier covers is +// published (initialSnapshotsPublished), so it may be asked for something. Before, an apply would +// destroy what is not mirrored yet: at startup the edits made while the operator was down, and for a +// GitTarget that newly names the trigger, what its folder does not hold yet. +func (l *branchWorkerEventLoop) pastBarrier(state *applierState) bool { + for _, target := range state.targets { + if !l.initialSnapshotsPublished(target.ref) { + return false + } } return true } +// revisionNamesCommit reports whether an applier's revision names commit: Flux writes main@sha1:, +// sha1:, main/ or a bare hash, Argo CD a bare hash. +func revisionNamesCommit(revision, commit string) bool { + if revision == "" || commit == "" { + return false + } + return strings.EqualFold(revisionCommit(revision), commit) +} + +// revisionCommit is the commit an applier's revision names: the hash after its last ':' or '/'. +func revisionCommit(revision string) string { + if i := strings.LastIndexAny(revision, ":/"); i >= 0 { + return revision[i+1:] + } + return revision +} + +// flushed makes the flush an apply needs (see serviceAppliers), and reports whether the writes up to +// its mark are published, so the attempts may go. The first call of a flush closes the open window +// and takes the mark; later calls only look whether the publication landed. +func (l *branchWorkerEventLoop) flushed(due []*applierState) bool { + if l.writeBusy() { + return false + } + for _, state := range due { + // A new mark publishes every edit recorded before it, so those are in the commit the + // attempt applies: carried into its window, they would withhold the apply's own write. + if state.flushEdits == nil || !l.flushing { + state.flushEdits = map[string]time.Time{} + } + } + if !l.flushing { + l.finalizeOpenWindowWithReason(windowFinalizeReasonApplyDue) + l.flushing, l.flushMark = true, l.decisions + // The siblings writing this branch flush too (apply_sibling_flush.go). + l.siblingAsk = l.w.siblings.askFlush(l.w) + if l.pendingUpTo(l.flushMark) { + l.w.Log.Info("An apply is due; publishing the pending writes first", "branch", l.w.Branch, + "pendingWrites", len(l.pendingWrites)) + l.advance() + } + } + return !l.pendingUpTo(l.flushMark) && l.siblingsFlushed() +} + +// pendingUpTo reports whether a write decided up to mark is still pending. +func (l *branchWorkerEventLoop) pendingUpTo(mark uint64) bool { + for i := range l.pendingWrites { + if l.pendingWrites[i].decision <= mark { + return true + } + } + return false +} + +// endFlush ends the flush once nothing is due: the edits recorded for an applier that has nothing +// pending any more are dropped with it. +func (l *branchWorkerEventLoop) endFlush() { + l.flushing, l.siblingAsk = false, 0 + for _, state := range l.appliers { + if state.inflight == nil && !state.hasReason(l.unsentRefusalsFor(state)) { + state.flushEdits = nil + } + } +} + +// earliest is the earlier of two deadlines, a zero one being none. +func earliest(a, b time.Time) time.Time { + if a.IsZero() || (!b.IsZero() && b.Before(a)) { + return b + } + return a +} + // appliersHaveWork reports whether any applier has a reason pending or an attempt out. func (l *branchWorkerEventLoop) appliersHaveWork() bool { for _, state := range l.appliers { - if state.foreign || state.catchUp || state.repair || state.inflight != nil { + if state.foreign || state.adopt || state.converge || state.repair || state.inflight != nil { return true } } @@ -396,13 +559,13 @@ func (l *branchWorkerEventLoop) appliersHaveWork() bool { // syncAppliers makes the applier states match the GitTargets listed now. An applier no GitTarget // names any more is forgotten, with whatever was pending for it: nobody asks for it, and its -// operation, if one is out, is abandoned. An applier named for the first time starts where a first -// observation does, with a CatchUp pending where its kind catches up: a commit observed before it -// was named was pending for nobody, and was taken, so without one it would not be applied until -// some later push. For a -// Kustomization that already applied the tip it costs nothing (Current); an Application gets no -// CatchUp, and waits for the next push. An unsent refusal of a GitTarget that is gone, or is now -// another incarnation, is dropped the same way. +// operation, if one is out, is abandoned. A GitTarget that names an applier for the first time leaves +// a converge pending, for Flux and Argo CD alike: a commit observed before was pending for nobody, +// and was taken, so without one it would not be applied until some later push. It waits for that +// GitTarget's initial snapshots (pastBarrier), and asks nothing of an applier that already applied +// the tip. An applier listed for the first time starts from what its trigger recorded as last +// applied. An unsent refusal of a GitTarget that is gone, or is now another incarnation, is dropped +// the same way. func (l *branchWorkerEventLoop) syncAppliers(targets []branchTarget) { if l.appliers == nil { l.appliers = map[ApplierRef]*applierState{} @@ -426,12 +589,7 @@ func (l *branchWorkerEventLoop) syncAppliers(targets []branchTarget) { } l.publishUnresolved() for ref, named := range named { - state := l.appliers[ref] - if state == nil { - state = &applierState{ref: ref, catchUp: ref.CatchesUp()} - l.appliers[ref] = state - } - state.targets = named + l.listApplier(ref, named) } for key, refusal := range l.unsentRefusals { if uids[key.target] != refusal.uid { @@ -440,47 +598,34 @@ func (l *branchWorkerEventLoop) syncAppliers(targets []branchTarget) { } } -// sendDue sends an attempt to every applier that has a reason pending, is not waiting on one, and whose -// retry and quiet periods have run out. The appliers are taken in turn order, so in a group, which -// has one attempt out at a time, the attempt goes to the applier whose turn is longest ago. It -// returns the earliest time one still waiting comes due. -func (l *branchWorkerEventLoop) sendDue(tip string, now time.Time) time.Time { - var earliest time.Time - wait := func(at time.Time) { - if earliest.IsZero() || at.Before(earliest) { - earliest = at - } +// listApplier records the GitTargets that name an applier now, and leaves a converge pending for each +// one that names it for the first time. +func (l *branchWorkerEventLoop) listApplier(ref ApplierRef, named []branchTarget) { + state := l.appliers[ref] + if state == nil { + state = &applierState{ref: ref, named: map[k8stypes.UID]bool{}, appliedRevision: lastApplied(named)} + l.appliers[ref] = state } - for _, ref := range l.turnOrder() { - state := l.appliers[ref] - if l.attemptOut(state) { - continue - } - refusals := l.unsentRefusalsFor(state) - if !state.foreign && !state.repair && len(refusals) == 0 && !state.catchUp { - continue - } - due := l.dueAt(state, refusals) - if now.Before(due) { - wait(due) - continue + state.targets = named + for _, target := range named { + if !state.named[target.uid] { + state.named[target.uid], state.converge = true, true } - if !l.w.claimGroup(state.group()) { - // A sibling sent one for the group since attemptOut looked; its end wakes this worker. - continue - } - l.sendAttempt(state, tip, refusals) } - return earliest } -// dueAt is when an applier's next attempt may go: once its backoff has run out, and, when only a -// CatchUp is pending, once the branch has been quiet for the trigger's quiet period. -func (l *branchWorkerEventLoop) dueAt(state *applierState, refusals map[refusalKey]unsentRefusal) time.Time { - if state.foreign || state.repair || len(refusals) > 0 { - return state.retryAt +// lastApplied is the revision the last operation that succeeded applied, as an applier's trigger +// recorded it in status.operation: its appliedRevision, or the commit it was asked for when the +// applier reported none. Empty when the last recorded operation did not succeed. +func lastApplied(targets []branchTarget) string { + recorded, _ := recordedStatus(&applierState{targets: targets}) + if recorded == nil || recorded.Phase != v1alpha3.ApplyOperationSucceeded { + return "" + } + if recorded.AppliedRevision != "" { + return recorded.AppliedRevision } - return latest(state.retryAt, l.quietSince.Add(state.quietPeriod())) + return recorded.Commit } // unsentRefusalsFor is the unsent refusals of the GitTargets naming this applier that still consent. @@ -508,8 +653,8 @@ func (l *branchWorkerEventLoop) unsentRefusalsFor(state *applierState) map[refus // sendAttempt hands one attempt for an applier to the requester and moves what it carries out of // the applier's pending reasons, into the attempt. Each target is asked for its own reason: every // one for a ForeignCommit, then for a Repair; otherwise a Refusal for a target with an unsent -// refusal, and a CatchUp for the rest when one is pending. Any operation that succeeds applies the -// tip, so it answers a pending CatchUp or Repair too. +// refusal, and a CatchUp for the rest when an adopt or a converge is pending. Any operation that +// succeeds applies the tip, so it answers a pending CatchUp or Repair too. func (l *branchWorkerEventLoop) sendAttempt(state *applierState, tip string, refusals map[refusalKey]unsentRefusal) { refused := map[itypes.ResourceReference]bool{} for key := range refusals { @@ -526,7 +671,7 @@ func (l *branchWorkerEventLoop) sendAttempt(state *applierState, tip string, ref own = v1alpha3.ApplyReasonRepair case refused[target.ref]: own = v1alpha3.ApplyReasonRefusal - case state.catchUp: + case state.adopt || state.converge: own = v1alpha3.ApplyReasonCatchUp default: continue @@ -536,16 +681,24 @@ func (l *branchWorkerEventLoop) sendAttempt(state *applierState, tip string, ref } l.nextApplyID++ + l.w.raiseApplyEpoch() abandoned, abandon := context.WithCancel(l.w.ctx) sentAt := time.Now() attempt := &inflightAttempt{ - id: l.nextApplyID, foreign: state.foreign, catchUp: state.catchUp, repair: state.repair, - refusals: refusals, abandon: abandon, + id: l.nextApplyID, foreign: state.foreign, adopt: state.adopt, converge: state.converge, + repair: state.repair, refusals: refusals, abandon: abandon, previous: state.operation, previousID: state.operationID, } - state.foreign, state.catchUp, state.repair, state.inflight = false, false, false, attempt + state.foreign, state.adopt, state.converge, state.repair = false, false, false, false + state.inflight = attempt state.lastTurn = attempt.id + l.reportApplyWindow(state) state.openApplyWindow(attempt.id, tip, sentAt) + // The edits decided after the flush mark are not in the commit, and the apply may overwrite them. + for key, at := range state.flushEdits { + state.window.recordEdit(key, at) + } + state.flushEdits = nil for key := range refusals { delete(l.unsentRefusals, key) } @@ -583,7 +736,7 @@ func (l *branchWorkerEventLoop) sendAttempt(state *applierState, tip string, ref // settleApplyResults reads the reports that came back. A report of an operation still out records // where it stands, and holds the attempt. An operation that succeeded is done. Any other ending // gives back what the attempt carried and, unless it was only stale, waits out its applier's -// backoff, so its retry is decided again on a quiet branch at whatever the tip is then, never +// backoff, so its retry is decided again after a flush at whatever the tip is then, never // replayed blind. A report for an applier no GitTarget names any more is dropped with it. func (l *branchWorkerEventLoop) settleApplyResults(now time.Time) { defer l.publishUnresolved() @@ -600,17 +753,22 @@ func (l *branchWorkerEventLoop) settleApplyResults(now time.Time) { } state.inflight = nil attempt.abandon() + l.w.raiseApplyEpoch() state.settleApplyWindow(attempt.id, res.Outcome.Operated(), now) if attempt.phase == ApplyUnresolved { l.w.Log.Info("An unresolved apply completed; its apply window closes after the grace", "branch", l.w.Branch, "applier", state.ref.String(), "outcome", string(res.Outcome)) } switch res.Outcome { - case ApplySucceeded, ApplyCurrent: + case ApplySucceeded: state.failures, state.retryAt = 0, time.Time{} + state.appliedRevision = res.AppliedRevision + if state.appliedRevision == "" && state.operation != nil { + state.appliedRevision = state.operation.Commit + } continue case ApplyStale: - // The decision went stale before the write: nothing failed, and the next quiet moment + // The decision went stale before the write: nothing failed, and the next wake // decides again, at the tip as it is then. l.restoreAttempt(state, attempt) continue @@ -620,9 +778,10 @@ func (l *branchWorkerEventLoop) settleApplyResults(now time.Time) { state.failures++ state.retryAt = now.Add(applierRetryDelay(state.failures, res.Outcome == ApplyRejected)) l.restoreAttempt(state, attempt) - message := "An apply attempt did not deliver; deciding it again on a quiet branch" + l.lookIfSourceMoved(state, res) + message := "An apply attempt did not deliver; deciding it again after its backoff" if res.Outcome == ApplyFailed { - message = "The applier did not complete the apply; deciding it again on a quiet branch" + message = "The applier did not complete the apply; deciding it again after its backoff" } l.w.Log.Info(message, "branch", l.w.Branch, "applier", state.ref.String(), "outcome", string(res.Outcome), @@ -630,6 +789,19 @@ func (l *branchWorkerEventLoop) settleApplyResults(now time.Time) { } } +// lookIfSourceMoved takes an applier's source holding another commit than the tip the worker knows +// as a hint that the branch moved, and looks at the remote, as a Git host's push notice is looked at. +// A worker with no receiver otherwise learns of a Git-side push only through its own rejected push, +// or the refresher, and asks again for a commit the source no longer serves until then. +func (l *branchWorkerEventLoop) lookIfSourceMoved(state *applierState, res ApplyResult) { + if res.SourceRevision == "" || revisionNamesCommit(res.SourceRevision, l.w.inbox.lastTip()) { + return + } + l.w.Log.Info("The applier's source holds another commit than the tip known here; looking at the remote", + "branch", l.w.Branch, "applier", state.ref.String(), "sourceRevision", res.SourceRevision) + l.w.EnqueuePushNotification(revisionCommit(res.SourceRevision)) +} + // advanceOperation records where an attempt's operation stands. Unresolved is said once, and stays // until the operation ends: it is the requester's verdict that the operation is overdue, which a // later sight of it running does not take back. @@ -662,7 +834,7 @@ func operationPhase(outcome ApplyOutcome) (v1alpha3.ApplyOperationPhase, bool) { return v1alpha3.ApplyOperationSucceeded, true case ApplyFailed: return v1alpha3.ApplyOperationFailed, true - case ApplyCurrent, ApplyRequestFailed, ApplyRejected, ApplyDropped, ApplyStale: + case ApplyRequestFailed, ApplyRejected, ApplyDropped, ApplyStale: } return "", false } @@ -792,7 +964,8 @@ func (l *branchWorkerEventLoop) publishUnresolved() { // revert. func (l *branchWorkerEventLoop) restoreAttempt(state *applierState, attempt *inflightAttempt) { state.foreign = state.foreign || attempt.foreign - state.catchUp = state.catchUp || attempt.catchUp + state.adopt = state.adopt || attempt.adopt + state.converge = state.converge || attempt.converge state.repair = state.repair || attempt.repair for key, refusal := range attempt.refusals { if !l.w.refusalStands(key) { @@ -817,33 +990,17 @@ func (l *branchWorkerEventLoop) applierAwaiting(id uint64) *applierState { return nil } -// workWaiting reports whether work items are queued on this worker, or queued or in hand on another -// writing this repository and branch. -func (l *branchWorkerEventLoop) workWaiting() bool { - _, siblingWorking := l.w.siblingBusy() - return len(l.w.eventQueue) > 0 || siblingWorking -} - // branchKey is the repository and branch this worker writes. func (w *BranchWorker) branchKey() BranchKey { return BranchKey{RepoNamespace: w.GitProviderNamespace, RepoName: w.GitProviderRef, Branch: w.Branch} } // decisionHolds is ApplyRequest.Current for an attempt decided at tip: the branch is still at that -// commit, and nothing live is waiting to be published on this worker or a sibling. A work item -// counts from its enqueue until its wake has published what it left behind (inflightItems), not -// only while it waits in the queue: a queued watch event is an edit that is live and not yet -// decided, and the item the loop is handling may be committing and pushing while busyWriting still -// says what the last wake left. It is called from the requester's goroutine, so it reads only what -// is safe to read there. -// -// An attempt sent from the wake of a queue item can find that item still counted, should the -// requester ask within moments. It is then Stale, which costs no backoff, and the report wakes the -// loop without an item, so the next attempt is not held by one of its own. +// commit. Nothing else holds it: every edit is either published before the send, or recorded in the +// apply window and repaired should the apply overwrite it (S1). It is called from the requester's +// goroutine, so it reads only what is safe to read there. func (w *BranchWorker) decisionHolds(tip string) bool { - siblingWriting, siblingWorking := w.siblingBusy() - return w.inbox.tipIs(tip) && !w.busyWriting.Load() && w.inflightItems.Load() == 0 && - !siblingWriting && !siblingWorking + return w.inbox.tipIs(tip) } // reportApplyResult is ApplyRequest.Report: it records the report and wakes the loop to settle it. diff --git a/internal/git/apply_trigger_test.go b/internal/git/apply_trigger_test.go index a680157c..3e36ad71 100644 --- a/internal/git/apply_trigger_test.go +++ b/internal/git/apply_trigger_test.go @@ -38,7 +38,7 @@ func targetRefs(req ApplyRequest) []itypes.ResourceReference { return refs } -func TestApplyDebt_FirstObservationIsABaseline(t *testing.T) { +func TestApplyInbox_FirstObservationIsABaseline(t *testing.T) { var inbox applyInbox inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) @@ -48,16 +48,14 @@ func TestApplyDebt_FirstObservationIsABaseline(t *testing.T) { assert.True(t, catchUp, "but nothing says it has, so a catch-up is pending") } -func TestApplyDebt_ClassifiesEachMove(t *testing.T) { +func TestApplyInbox_ClassifiesEachMove(t *testing.T) { cases := []struct { name string next RemoteObservation wantForeign bool - wantCatchUp bool }{ {name: "an unchanged tip leaves nothing pending", next: RemoteObservation{Commit: "aaa", By: ObservedByFetch}}, - {name: "our own push leaves a catch-up pending", next: RemoteObservation{Commit: "bbb", By: ObservedByPush}, - wantCatchUp: true}, + {name: "our own push leaves nothing pending", next: RemoteObservation{Commit: "bbb", By: ObservedByPush}}, {name: "a fetch that found the branch moved leaves a ForeignCommit pending", next: RemoteObservation{Commit: "bbb", By: ObservedByFetch}, wantForeign: true}, {name: "a deleted branch is a move like any other", @@ -70,22 +68,22 @@ func TestApplyDebt_ClassifiesEachMove(t *testing.T) { inbox.take() inbox.observe(tc.next) - tip, foreign, catchUp := inbox.take() + tip, foreign, converge := inbox.take() assert.Equal(t, tc.next.Commit, tip) assert.Equal(t, tc.wantForeign, foreign) - assert.Equal(t, tc.wantCatchUp, catchUp) + assert.False(t, converge, "only the first observation converges") }) } } -func TestApplyDebt_RestorePutsBackWhatWasTaken(t *testing.T) { +func TestApplyInbox_RestorePutsBackWhatWasTaken(t *testing.T) { var inbox applyInbox inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByFetch}) - _, foreign, catchUp := inbox.take() + _, foreign, converge := inbox.take() require.True(t, foreign) - inbox.restore(foreign, catchUp) + inbox.restore(foreign, converge) assert.True(t, inbox.foreignPending()) } @@ -149,6 +147,14 @@ func applyTriggerLoop(t *testing.T, targets ...client.Object) (*branchWorkerEven } // observeForeignCommit makes the worker's inbox hold a ForeignCommit at commit "bbb". +// holdApplies puts the loop in a state in which no flush can be made, so no apply is sent: a save is +// registered, and it defines its own commit boundary (writeBusy). releaseApplies ends it. +func holdApplies(l *branchWorkerEventLoop) { + l.pendingCRs = map[commitRequestID]*pendingCommitRequest{{Namespace: "team-a", Name: "save"}: {}} +} + +func releaseApplies(l *branchWorkerEventLoop) { l.pendingCRs = nil } + func observeForeignCommit(w *BranchWorker) { w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) w.inbox.take() @@ -182,14 +188,16 @@ func TestServiceAppliers_ForeignCommitAsksEveryApplierOnTheBranch(t *testing.T) } } -func TestServiceAppliers_NothingIsSentWhileAnythingLiveIsUnpublished(t *testing.T) { +// TestServiceAppliers_NothingIsSentWhileNoFlushCanBeMade: an apply waits only for the states in +// which no flush can be made: a registered save, a parent recovery, a checkout a failed write left +// dirty. Its reason stays pending. +func TestServiceAppliers_NothingIsSentWhileNoFlushCanBeMade(t *testing.T) { cases := []struct { name string busy func(l *branchWorkerEventLoop) }{ - {name: "an open commit window", busy: func(l *branchWorkerEventLoop) { l.openWindow = &openWindow{} }}, - {name: "a pending write", busy: func(l *branchWorkerEventLoop) { l.pendingWrites = []PendingWrite{{}} }}, - {name: "a parked heal", busy: func(l *branchWorkerEventLoop) { l.deferredHeals = []*ResyncRequest{{}} }}, + {name: "a registered save", busy: holdApplies}, + {name: "a parent recovery", busy: func(l *branchWorkerEventLoop) { l.recovery.active = true }}, {name: "a worktree a failed write left dirty", busy: func(l *branchWorkerEventLoop) { l.w.markWorktreeDirty("test") }}, @@ -203,37 +211,11 @@ func TestServiceAppliers_NothingIsSentWhileAnythingLiveIsUnpublished(t *testing. loop.serviceAppliers() assert.Empty(t, requester.taken()) - assert.True(t, loop.quietSince.IsZero(), "a busy branch is not quiet") - assert.True(t, loop.w.inbox.foreignPending(), "and the pending ForeignCommit waits for it") + assert.True(t, loop.appliers[applierRef("alpha")].foreign, "the pending ForeignCommit waits") }) } } -func TestServiceAppliers_CatchUpWaitsForTheQuietPeriod(t *testing.T) { - loop, requester := applyTriggerLoop(t, - applierTestTarget("soon", true), fluxTrigger("soon", quietFor(time.Minute)), - applierTestTarget("late", true), fluxTrigger("late", quietFor(time.Hour)), - ) - loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) - - loop.serviceAppliers() - assert.Empty(t, requester.taken(), "the branch has only just become quiet") - require.NotNil(t, loop.applierTimer, "the earliest catch-up is armed") - require.Len(t, loop.appliers, 2) - - // Two minutes of quiet: the one-minute applier is due, the one-hour one is not. - loop.quietSince = time.Now().Add(-2 * time.Minute) - loop.serviceAppliers() - requests := requester.taken() - require.Len(t, requests, 1) - assert.Equal(t, configv1alpha3.ApplyReasonCatchUp, requests[0].Reason) - assert.Equal(t, "aaa", requests[0].Commit) - assert.Equal(t, []itypes.ResourceReference{itypes.NewResourceReference("soon", "team-a")}, - targetRefs(requests[0])) - assert.True(t, loop.appliers[applierRef("late")].catchUp, "the one-hour applier still waits") - assert.NotNil(t, loop.applierTimer) -} - // argoApplier makes a GitTarget name the ArgoCDTrigger of its own name, which holds the Application // argocd/. func argoApplier(g *configv1alpha3.GitTarget) { @@ -242,31 +224,6 @@ func argoApplier(g *configv1alpha3.GitTarget) { } } -// TestServiceAppliers_AnApplicationIsNeverSentACatchUp: with automated sync off Argo CD applies -// nothing on its own, so GitOps Reverser's own pushes (and the baseline after a start) ask an -// Application for nothing, however long the branch stays quiet; a Kustomization beside it on the same -// branch is still caught up. -func TestServiceAppliers_AnApplicationIsNeverSentACatchUp(t *testing.T) { - loop, requester := applyTriggerLoop(t, - applierTestTarget("argo", true, argoApplier), - applierTestTarget("flux", true), - ) - loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) // the baseline - loop.serviceAppliers() - loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByPush}) - loop.serviceAppliers() - loop.quietSince = time.Now().Add(-time.Hour) - loop.serviceAppliers() - - requests := requester.taken() - require.Len(t, requests, 1, "only the Kustomization catches up") - assert.Equal(t, applierRef("flux"), requests[0].Applier) - assert.Equal(t, configv1alpha3.ApplyReasonCatchUp, requests[0].Reason) - argo := argoApplierRef("argo") - require.Contains(t, loop.appliers, argo) - assert.False(t, loop.appliers[argo].catchUp) -} - // TestServiceAppliers_AnApplicationIsAskedOnAForeignCommitAndARefusal: the two reasons that are // not catching up reach an Application as they reach a Kustomization. func TestServiceAppliers_AnApplicationIsAskedOnAForeignCommitAndARefusal(t *testing.T) { @@ -297,81 +254,6 @@ func TestServiceAppliers_AnApplicationIsAskedOnAForeignCommitAndARefusal(t *test }) } -func TestServiceAppliers_AWriteRestartsTheQuietPeriod(t *testing.T) { - loop, requester := applyTriggerLoop(t, applierTestTarget("alpha", true)) - loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) - loop.serviceAppliers() - loop.quietSince = time.Now().Add(-time.Hour) - - loop.openWindow = &openWindow{} - loop.serviceAppliers() - assert.True(t, loop.quietSince.IsZero()) - assert.Nil(t, loop.applierTimer) - - loop.openWindow = nil - loop.serviceAppliers() - assert.Empty(t, requester.taken(), "the quiet period restarted when the write arrived") - assert.WithinDuration(t, time.Now(), loop.quietSince, time.Second) -} - -// TestServiceAppliers_AWriteDecidedAndPushedInOneWakeRestartsTheQuietPeriod is the regression the -// bi-directional e2e found. A window with a zero idle timeout opens, closes, commits and pushes in a -// single wake that ends quiet, so a quiet period read only from how wakes END was inherited from -// the quiet wakes before it: the catch-up went out at once, Flux applied that commit over the next -// API edit, and the revert was mirrored as Flux's commit. -func TestServiceAppliers_AWriteDecidedAndPushedInOneWakeRestartsTheQuietPeriod(t *testing.T) { - loop, requester := applyTriggerLoop(t, applierTestTarget("alpha", true)) - loop.serviceAppliers() - loop.quietSince = time.Now().Add(-time.Hour) - - loop.decide(PendingWrite{}) - loop.pendingWrites, loop.pendingWritesBytes = nil, 0 // the same wake pushed it - loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByPush}) - loop.serviceAppliers() - - assert.Empty(t, requester.taken(), "the catch-up waits a quiet period from the push, not from before it") - assert.WithinDuration(t, time.Now(), loop.quietSince, time.Second) - assert.NotNil(t, loop.applierTimer) -} - -// TestServiceAppliers_ASecondWriteInsideTheQuietPeriodRestartsIt is the other half of the race the -// bi-directional e2e spec covers with real timers: two API edits a few seconds apart, each decided -// and pushed in one wake. The catch-up pending for the first push must not go out between them, and -// when it goes out it names the second push. -func TestServiceAppliers_ASecondWriteInsideTheQuietPeriodRestartsIt(t *testing.T) { - loop, requester := applyTriggerLoop( - t, - applierTestTarget("alpha", true), - fluxTrigger("alpha", quietFor(time.Minute)), - ) - loop.w.inbox.observe(RemoteObservation{Commit: "base", By: ObservedByFetch}) - loop.w.inbox.take() - - pushInOneWake := func(commit string) { - loop.decide(PendingWrite{}) - loop.pendingWrites, loop.pendingWritesBytes = nil, 0 - loop.w.inbox.observe(RemoteObservation{Commit: commit, By: ObservedByPush}) - loop.serviceAppliers() - } - - pushInOneWake("first") - loop.quietSince = time.Now().Add(-50 * time.Second) // ten seconds short of the quiet period - loop.serviceAppliers() - assert.Empty(t, requester.taken(), "the first push's catch-up is not due yet") - - pushInOneWake("second") - loop.quietSince = time.Now().Add(-50 * time.Second) // past the first push's period, not the second's - loop.serviceAppliers() - assert.Empty(t, requester.taken(), "the second write restarted the quiet period") - - loop.quietSince = time.Now().Add(-time.Minute) - loop.serviceAppliers() - requests := requester.taken() - require.Len(t, requests, 1) - assert.Equal(t, configv1alpha3.ApplyReasonCatchUp, requests[0].Reason) - assert.Equal(t, "second", requests[0].Commit, "the catch-up names the later push, never the earlier") -} - func TestServiceAppliers_AForeignCommitAnswersEverythingElsePending(t *testing.T) { loop, requester := applyTriggerLoop(t, applierTestTarget("alpha", true)) loop.addUnsentRefusal(refusalKeyFor(itypes.NewResourceReference("alpha", "team-a"), "deployments"), @@ -381,10 +263,10 @@ func TestServiceAppliers_AForeignCommitAnswersEverythingElsePending(t *testing.T loop.serviceAppliers() requests := requester.taken() - require.Len(t, requests, 1, "one apply of the tip covers the refusal and the catch-up") + require.Len(t, requests, 1, "one apply of the tip covers the refusal and the converge") assert.Equal(t, configv1alpha3.ApplyReasonForeignCommit, requests[0].Reason) assert.Empty(t, loop.unsentRefusals) - assert.False(t, loop.appliers[applierRef("alpha")].catchUp) + assert.False(t, loop.appliers[applierRef("alpha")].converge) } func TestServiceAppliers_NoApplierOnTheBranchClearsThePendingReasons(t *testing.T) { @@ -404,8 +286,8 @@ func TestServiceAppliers_NoApplierOnTheBranchClearsThePendingReasons(t *testing. // observed while no GitTarget names an applier, so the ForeignCommit is taken and dropped; then a // GitTarget names a trigger, whose Kustomization was suspended before its GitRepository fetched the // merge. The new applier had nothing pending, so the merge was never deployed until some later push. -// A newly named applier now starts where a first observation does, with a CatchUp pending (for -// Flux free when the commit is already applied); an Application, which gets no CatchUp, waits for the next push. +// A GitTarget that names an applier for the first time leaves a converge pending, for a Kustomization +// and an Application alike (S2). func TestServiceAppliers_ANewlyNamedApplierIsCaughtUp(t *testing.T) { loop, requester := applyTriggerLoop(t, applierTestTarget("flux", false), fluxTrigger("flux"), @@ -433,16 +315,15 @@ func TestServiceAppliers_ANewlyNamedApplierIsCaughtUp(t *testing.T) { }, })) loop.serviceAppliers() - loop.quietSince = time.Now().Add(-time.Hour) - loop.serviceAppliers() requests := requester.taken() - require.Len(t, requests, 1, "one apply, for the newly named Kustomization") - assert.Equal(t, applierRef("flux"), requests[0].Applier) - assert.Equal(t, configv1alpha3.ApplyReasonCatchUp, requests[0].Reason) - assert.Equal(t, "bbb", requests[0].Commit, "the merge it never fetched") - require.Contains(t, loop.appliers, argoApplierRef("argo"), "the Application is listed") - assert.False(t, loop.appliers[argoApplierRef("argo")].catchUp, "and waits for the next push") + require.Len(t, requests, 2, "one apply for each newly named applier") + for _, req := range requests { + assert.Equal(t, configv1alpha3.ApplyReasonCatchUp, req.Reason) + assert.Equal(t, "bbb", req.Commit, "the merge it never fetched") + } + assert.ElementsMatch(t, []ApplierRef{applierRef("flux"), argoApplierRef("argo")}, + []ApplierRef{requests[0].Applier, requests[1].Applier}) } func TestServiceAppliers_NoRequesterSwitchesTheTriggerOff(t *testing.T) { @@ -455,40 +336,27 @@ func TestServiceAppliers_NoRequesterSwitchesTheTriggerOff(t *testing.T) { assert.True(t, loop.w.inbox.foreignPending(), "nothing took the pending ForeignCommit") } -// TestCloseWindowForForeignCommit_PublishesTheOpenWindowNow: a window closes on silence, so a user -// who keeps editing would hold a foreign commit back for as long as they type. It is closed instead, -// and its write published and replayed onto the move. -func TestCloseWindowForForeignCommit_PublishesTheOpenWindowNow(t *testing.T) { - cases := []struct { - name string - cause func(l *branchWorkerEventLoop) - }{ - {name: "a foreign commit observed", cause: func(l *branchWorkerEventLoop) { observeForeignCommit(l.w) }}, - {name: "a Git host's push notice on a busy branch", cause: func(l *branchWorkerEventLoop) { - l.handleRefreshRequest(&RefreshRequest{PushNotice: true}) - }}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - worker, _, _ := setupCommitPushSplitWorker(t) - createPlainGitTarget(t, worker, "team-a", "team-a") - worker.applyRequester = &recordingRequester{} - loop := newBranchWorkerEventLoop(worker, time.Hour) - t.Cleanup(loop.stopTimers) - - loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("edited", "alice", "team-a")}, - CommitMode: CommitModePerEvent, - }}) - require.NotNil(t, loop.openWindow, "an hour-long window is still open") - - tc.cause(loop) - loop.closeWindowForForeignCommit() - - assert.Nil(t, loop.openWindow) - assert.Len(t, loop.pendingWrites, 1, "the window's write is decided, to be published") - }) - } +// TestCloseWindowForPushHint_PublishesTheOpenWindowNow: a Git host said the branch may have moved, +// so the open window is closed and its write published, and replayed onto the move, now. An apply +// that comes due closes it the same way, as the flush (TestFlush_*). +func TestCloseWindowForPushHint_PublishesTheOpenWindowNow(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + createPlainGitTarget(t, worker, "team-a", "team-a") + worker.applyRequester = &recordingRequester{} + loop := newBranchWorkerEventLoop(worker, time.Hour) + t.Cleanup(loop.stopTimers) + + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("edited", "alice", "team-a")}, + CommitMode: CommitModePerEvent, + }}) + require.NotNil(t, loop.openWindow, "an hour-long window is still open") + + loop.handleRefreshRequest(&RefreshRequest{PushNotice: true}) + loop.closeWindowForPushHint() + + assert.Nil(t, loop.openWindow) + assert.Len(t, loop.pendingWrites, 1, "the window's write is decided, to be published") } // applierLedgerFixture is a real Git server, a GitTarget naming an applier, and an event loop whose @@ -586,9 +454,7 @@ func TestApplierOf_KeysTheApplierByItsTriggerClusterAndKind(t *testing.T) { argo := testTrigger("team-a", &configv1alpha3.ApplyTriggerReference{ Kind: configv1alpha3.ApplyTriggerKindArgoCD, Name: "apps", }).(configv1alpha3.ApplyTrigger) - app := ApplierOf(argo) - assert.Equal(t, argoApplierRef("apps"), app) - assert.False(t, app.CatchesUp(), "an Application is never asked to catch up") + assert.Equal(t, argoApplierRef("apps"), ApplierOf(argo)) } // TestResolveApplyTrigger_NamesNothingItCannotFind: a GitTarget that names no trigger, a kind this @@ -634,3 +500,54 @@ func TestResolveApplyTrigger_NamesNothingItCannotFind(t *testing.T) { ) require.ErrorContains(t, err, "read FluxTrigger team-a/apps") } + +// TestServiceAppliers_AFailedListingKeepsItsRetry: a listing of the branch's appliers that fails +// leaves the reasons pending and is tried again on the applier timer. The wake that found the failure +// armed that timer and then disarmed it, with nothing else to wait for, so the apply waited on some +// unrelated activity to wake the loop. +func TestServiceAppliers_AFailedListingKeepsItsRetry(t *testing.T) { + loop, requester := applyTriggerLoop(t, applierTestTarget("alpha", true)) + reader := loop.w.Client + loop.w.Client = interceptor.NewClient(reader.(client.WithWatch), interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return errors.New("the API server is away") + }, + }) + observeForeignCommit(loop.w) + + loop.serviceAppliers() + assert.Empty(t, requester.taken()) + require.NotNil(t, loop.applierTimer, "the listing is tried again") + assert.WithinDuration(t, time.Now().Add(applierRetryInterval), loop.listRetryAt, time.Second) + + loop.w.Client = reader + loop.serviceAppliers() + assert.Len(t, requester.taken(), 1, "the retry sends what was pending") + assert.True(t, loop.listRetryAt.IsZero()) +} + +// TestServiceAppliers_AFailedRecordedListingKeepsItsRetry: the pass's first listing, of the recorded +// operations, failing while nothing else is pending must leave its retry armed: the same pass found +// no reason to send, and disarmed the timer. +func TestServiceAppliers_AFailedRecordedListingKeepsItsRetry(t *testing.T) { + loop, _ := applyTriggerLoop(t, applierTestTarget("alpha", true)) + loop.serviceAppliers() + loop.w.inbox.take() + for _, state := range loop.appliers { + state.converge = false + } + reader := loop.w.Client + loop.w.Client = interceptor.NewClient(reader.(client.WithWatch), interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return errors.New("the API server is away") + }, + }) + + loop.serviceAppliers() + require.NotNil(t, loop.applierTimer, "the listing is tried again") + assert.False(t, loop.listRetryAt.IsZero()) + + loop.w.Client = reader + loop.serviceAppliers() + assert.True(t, loop.listRetryAt.IsZero(), "a listing that succeeds clears it") +} diff --git a/internal/git/apply_window.go b/internal/git/apply_window.go index 27e1db88..1ed3d0aa 100644 --- a/internal/git/apply_window.go +++ b/internal/git/apply_window.go @@ -27,6 +27,13 @@ import ( // is withheld. Git keeps the edit, and the cluster runs the reverted value until an apply puts it // back, so the withheld write leaves a Repair pending for its applier (apply_trigger.go). // +// The applier's write to an object nobody edited is the apply itself, landing: its content is the +// commit's, which Git already holds. It is not mirrored either (docs/design/applier-echoes.md, F1). +// Mirrored, it became a pending write planned from the live object, and a replay of that write onto +// a newer tip reverted the newer tip's change to the object. The window recognizes it by the +// applier's managers alone on an entry written at or after the earliest send of the windows still +// open or carried; an older one is an earlier apply's entry left newest by an undo, and an edit. +// // What the rule must never do is withhold an API edit, which would lose it outright. So it acts // only on proof: the object's latest managedFields entries are the applier's managers alone, and // they are newer than the edit. Alone is not enough: an update that removes every field a manager @@ -86,6 +93,15 @@ const ( collisionAmbiguous = "ambiguous" ) +// Applier echo outcomes, the outcome label of apply_echoes_total. +const ( + // echoNotMirrored is the applier's own write in its apply window, to an object nobody edited. + echoNotMirrored = "not_mirrored" + // echoLate is a write by the applier's managers alone with no window open: an echo that arrived + // after the grace, or an apply nobody asked for. It is mirrored as an edit (C3). + echoLate = "late" +) + // applyWindow is one applier's open apply window: from sending an attempt until its operation // completed, plus applyWindowGrace, and never before the window it replaced would have closed. An // Unresolved operation holds it open. @@ -100,9 +116,25 @@ type applyWindow struct { edited map[string]time.Time // carried holds the objects the window before this one had seen edited, when that window was // still open as this attempt was sent, and carriedUntil is when it would have closed: the - // earlier apply's writes can arrive until then (openApplyWindow). - carried map[string]time.Time - carriedUntil time.Time + // earlier apply's writes can arrive until then (openApplyWindow). carriedSentAt is the earliest + // send of the windows carried, zero when none is. + carried map[string]time.Time + carriedUntil time.Time + carriedSentAt time.Time + // echoes counts the applier's writes the window kept out of Git: the apply landing, object by + // object. reported is set once that count was logged (reportApplyWindows). + echoes int + reported bool +} + +// earliestSend is the lower bound of the applier's own writes at now: the earliest send of this +// window and of the windows it still carries. A delayed echo of the attempt before, arriving after +// this one was sent and inside that one's grace, is still that apply. +func (a *applyWindow) earliestSend(now time.Time) time.Time { + if !a.carriedSentAt.IsZero() && now.Before(a.carriedUntil) { + return a.carriedSentAt + } + return a.sentAt } // open reports whether the window is open at now. @@ -117,10 +149,14 @@ func objectKey(id itypes.ResourceIdentifier) string { } // editedSince reports whether the API side wrote an object since the attempt was sent, or, until -// the window before it would have closed, since that one was, and when it last did. -func (a *applyWindow) editedSince(key string, now time.Time) (time.Time, bool) { +// the window before it would have closed, since that one was, and when it last did. An edit carried +// from the window before guards only against an applier write made before this attempt was sent: +// one attempt is out at a time, so the earlier apply wrote before then, and a write since is this +// attempt's apply of a commit that holds every edit published before it (S1). +func (a *applyWindow) editedSince(key string, now, written time.Time) (time.Time, bool) { at, edited := a.edited[key] - if carriedAt, carried := a.carried[key]; carried && now.Before(a.carriedUntil) { + if carriedAt, carried := a.carried[key]; carried && now.Before(a.carriedUntil) && + written.Before(a.sentAt.Truncate(time.Second)) { return latest(at, carriedAt), true } return at, edited @@ -144,6 +180,7 @@ func (state *applierState) openApplyWindow(attempt uint64, commit string, now ti maps.Copy(next.carried, before.carried) } next.carriedUntil = before.closesAt + next.carriedSentAt = before.earliestSend(now) } state.window = next } @@ -173,58 +210,131 @@ func (state *applierState) covers(event Event) bool { }) } -// withholdLateApply reports whether a live event is an applier's apply arriving late over a newer -// API edit, and must not be mirrored. Every other event is recorded as an API-side edit in the -// windows of the appliers that cover it. It is called for each live event before it reaches the -// commit window. +// withholdLateApply reports whether a live event is the applier's own write in its apply window, +// and must not be mirrored: the apply itself landing on an object nobody edited (F1), or the apply +// arriving late over a newer API edit (withheld, with a Repair pending). Every other event is +// recorded as an API-side edit in the windows of the appliers that cover it, and in the edits a due +// attempt's window will hold once it is sent (applierState.flushEdits). It is called for each live +// event before it reaches the commit window. func (l *branchWorkerEventLoop) withholdLateApply(event Event) bool { if len(l.appliers) == 0 { return false } now := time.Now() - key := objectKey(event.Identifier) for _, ref := range l.sortedAppliers() { applier := l.appliers[ref] - window := applier.window - if !window.open(now) || !applier.covers(event) { + if !applier.covers(event) { continue } - byApplier, byApplierAlone := ref.writtenBy(event.LastFieldManagers) - editedAt, editedSince := window.editedSince(key, now) - written := event.LastFieldManagedAt switch { - case !byApplier || event.Operation == string(itypes.OperationDelete): - window.recordEdit(key, written) - case !byApplierAlone: - // It may be the API edit itself, written in the same second: mirroring it is the - // choice that cannot lose an edit. It counts as one, too, even as the first write in the - // window, so the applier's write alone after it is still seen as late. It collides only - // with an edit made before it. - if editedSince { - l.recordApplyWindowCollision(applier, window, event, collisionAmbiguous) + case applier.window.open(now): + if l.judgeInWindow(applier, event, now) { + return true + } + case event.Operation != string(itypes.OperationDelete): + if _, alone := ref.writtenBy(event.LastFieldManagers); alone { + // No apply this worker asked for is landing: an echo after the grace, or an apply + // somebody else started. It is mirrored as an edit, and counted (C3). + l.recordApplyEcho(echoLate) } - window.recordEdit(key, written) - case !editedSince: - // The applier applying an object nobody edited since the attempt: its content is the - // commit's, which Git already holds. Mirrored as ever. - case written.Equal(editedAt): - // The applier's entry and the edit in the same second: it could be either. + } + if applier.flushEdits != nil { + // Decided after the flush mark, so not in the commit the due attempt applies: its window + // holds it from the start (S1). + key := objectKey(event.Identifier) + applier.flushEdits[key] = latest(applier.flushEdits[key], event.LastFieldManagedAt) + } + } + return false +} + +// judgeInWindow judges one live event against the open apply window of an applier that covers it, +// recording it as an edit unless it is the applier's own write, and reports whether it is not to +// be mirrored. +func (l *branchWorkerEventLoop) judgeInWindow(applier *applierState, event Event, now time.Time) bool { + window := applier.window + key := objectKey(event.Identifier) + byApplier, byApplierAlone := applier.ref.writtenBy(event.LastFieldManagers) + written := event.LastFieldManagedAt + editedAt, editedSince := window.editedSince(key, now, written) + switch { + case !byApplier || event.Operation == string(itypes.OperationDelete): + window.recordEdit(key, written) + case !byApplierAlone: + // It may be the API edit itself, written in the same second: mirroring it is the + // choice that cannot lose an edit. It counts as one, too, even as the first write in the + // window, so the applier's write alone after it is still seen as late. It collides only + // with an edit made before it. + if editedSince { l.recordApplyWindowCollision(applier, window, event, collisionAmbiguous) - window.recordEdit(key, written) - case written.Before(editedAt): - // The applier's entry is older than the edit, so the applier did not write this: an API - // update that removed every field its manager owned took that entry with it, and left - // the applier's from an earlier apply as the newest. It is an edit, the undo of one. - window.recordEdit(key, written) - default: - l.recordApplyWindowCollision(applier, window, event, collisionWithheld) - applier.repair = true - return true } + window.recordEdit(key, written) + case written.Before(window.earliestSend(now).Truncate(time.Second)): + // Older than every send still open: an earlier apply's entry, left newest by an update + // that removed every field the editor owned. It is an edit, the undo of one. + window.recordEdit(key, written) + case !editedSince: + // The apply itself, on an object nobody edited since the attempt: its content is the + // commit's, which Git already holds. Not mirrored. + window.echoes++ + l.recordApplyEcho(echoNotMirrored) + return true + case written.Before(editedAt): + // The applier's entry is older than the edit, so the applier did not write this: an API + // update that removed every field its manager owned took that entry with it, and left + // the applier's from an earlier apply as the newest. It is an edit, the undo of one. + window.recordEdit(key, written) + default: + // The applier's write alone, at or after the edit. In the same second it is still the + // apply over the edit: events arrive in resourceVersion order, so the edit, recorded + // already, was written first, and mirroring this would revert it in Git and on the + // cluster. Only an undo in that same second is lost (C4 in docs/design/applier-echoes.md). + l.recordApplyWindowCollision(applier, window, event, collisionWithheld) + applier.repair = true + return true } return false } +// recordApplyEcho counts one write of an applier's own that the apply window judged, by outcome. +func (l *branchWorkerEventLoop) recordApplyEcho(outcome string) { + if telemetry.ApplyEchoesTotal == nil { + return + } + telemetry.ApplyEchoesTotal.Add(context.WithoutCancel(l.w.ctx), 1, + metric.WithAttributes(l.w.providerAttrs(attribute.String("outcome", outcome))...)) +} + +// reportApplyWindow logs, once, how many of the applier's writes its window kept out of Git: what it +// saw of the apply landing. It runs when the window is about to be replaced, or is found closed. +func (l *branchWorkerEventLoop) reportApplyWindow(state *applierState) { + window := state.window + if window == nil || window.reported { + return + } + window.reported = true + l.w.Log.Info("Observed the applier's apply", "branch", l.w.Branch, "applier", state.ref.String(), + "commit", window.commit, "objects", window.echoes) +} + +// reportApplyWindows reports every window that has closed, and returns when the earliest one still +// open closes, zero when none will. +func (l *branchWorkerEventLoop) reportApplyWindows(now time.Time) time.Time { + var next time.Time + for _, ref := range l.sortedAppliers() { + state := l.appliers[ref] + window := state.window + switch { + case window == nil || window.reported: + case !window.open(now): + l.reportApplyWindow(state) + case !window.closesAt.IsZero() && (next.IsZero() || window.closesAt.Before(next)): + next = window.closesAt + } + } + return next +} + // recordApplyWindowCollision logs and counts one write by an applier, inside its apply window, to // an object the API side edited since the attempt was sent. func (l *branchWorkerEventLoop) recordApplyWindowCollision( diff --git a/internal/git/apply_window_test.go b/internal/git/apply_window_test.go index d7bf33ca..ac98a896 100644 --- a/internal/git/apply_window_test.go +++ b/internal/git/apply_window_test.go @@ -21,14 +21,14 @@ import ( itypes "github.com/ConfigButler/gitops-reverser/internal/types" ) -// windowClock is when the managedFields entries of the next windowEvent were written: a second -// after the previous one, as the API server stamps them. -var windowClock = time.Unix(1_700_000_000, 0) +// windowClock is when the managedFields entries of the last windowEvent were written. +var windowClock time.Time // windowEvent is a live UPDATE to the Deployment "api" in target's folder, written last by managers, -// a second after the previous one. +// a second after the previous one and never before now, as the API server stamps them: an applier's +// entry older than the attempt's send is an earlier apply's (apply_window.go). func windowEvent(target, replicas string, managers ...string) Event { - windowClock = windowClock.Add(time.Second) + windowClock = latest(windowClock, time.Now().Truncate(time.Second)).Add(time.Second) return windowEventAt(target, replicas, windowClock, managers...) } @@ -62,6 +62,13 @@ func sentAttempt(t *testing.T) (*branchWorkerEventLoop, ApplyRequest) { return loop, out[0] } +func echoes(t *testing.T, reader *sdkmetric.ManualReader, outcome string) int64 { + t.Helper() + got, _ := telemetry.CollectInt64Sum(reader, "gitopsreverser_apply_echoes_total", + map[string]string{"outcome": outcome}) + return got +} + func collisions(t *testing.T, reader *sdkmetric.ManualReader, outcome string) int64 { t.Helper() got, _ := telemetry.CollectInt64Sum(reader, "gitopsreverser_apply_window_collisions_total", @@ -88,7 +95,8 @@ func TestApplyWindow_TheAppliersLateWriteOverANewerEditIsWithheld(t *testing.T) // deletes its managedFields entry, so the newest entry left is kustomize-controller's, from an // earlier apply. That undo looked like the applier's write alone over a newer edit: it was withheld // and a Repair sent, which put tmp=1 back. A real late apply moves the applier's entry forward; an -// undo does not. So only an applier entry strictly newer than the edit is withheld. +// undo does not. So only an applier entry at or after the edit is withheld: in the same second the +// edit, which arrived first, was written first, and the applier's write alone is its apply over it. func TestApplyWindow_AnUndoOfAnEditIsMirrored(t *testing.T) { reader, err := telemetry.InitTestExporter() require.NoError(t, err) @@ -99,12 +107,12 @@ func TestApplyWindow_AnUndoOfAnEditIsMirrored(t *testing.T) { assert.False(t, loop.withholdLateApply(windowEventAt("alpha", "2", earlierApply, FluxFieldManager)), "its undo leaves only an older entry of the applier's, and is mirrored") assert.False(t, loop.appliers[applierRef("alpha")].repair, "so no Repair puts the edit back") - assert.False(t, loop.withholdLateApply(windowEventAt("alpha", "2", edit, FluxFieldManager)), - "a write of the applier's in the same second as the edit could be either, and is mirrored") - assert.Equal(t, int64(1), collisions(t, reader, collisionAmbiguous)) + assert.True(t, loop.withholdLateApply(windowEventAt("alpha", "2", edit, FluxFieldManager)), + "the applier's write alone in the same second as the edit, after it, is its apply over it") + assert.Equal(t, int64(0), collisions(t, reader, collisionAmbiguous)) assert.True(t, loop.withholdLateApply(windowEventAt("alpha", "2", edit.Add(time.Second), FluxFieldManager)), "an apply arriving after the edit moves the applier's entry forward, and is withheld") - assert.Equal(t, int64(1), collisions(t, reader, collisionWithheld)) + assert.Equal(t, int64(2), collisions(t, reader, collisionWithheld)) } // TestApplyWindow_OnlyAWithheldWriteIsKeptFromTheCommitWindow follows both events through the live @@ -125,16 +133,19 @@ func TestApplyWindow_OnlyAWithheldWriteIsKeptFromTheCommitWindow(t *testing.T) { } } -// TestApplyWindow_WhatIsNeverWithheld: the applier's write is mirrored as ever when it could be -// the edit itself, when nobody edited the object since the attempt, when it is a delete, and for a -// folder another applier (or none) applies. +// TestApplyWindow_WhatIsNeverWithheld: the applier's write is mirrored when it could be the edit +// itself, when it is a delete, and for a folder another applier (or none) applies. Its write to an +// object nobody edited since the attempt is not mirrored, and leaves no Repair: it is the apply +// itself, writing what Git holds (F1). func TestApplyWindow_WhatIsNeverWithheld(t *testing.T) { reader, err := telemetry.InitTestExporter() require.NoError(t, err) loop, _ := sentAttempt(t) - assert.False(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), - "an object nobody edited since the attempt: the apply writes what Git holds") + assert.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), + "an object nobody edited since the attempt: the apply writes what Git holds, and is not mirrored") + assert.False(t, loop.appliers[applierRef("alpha")].repair, "nothing to put back") + assert.Equal(t, int64(1), echoes(t, reader, echoNotMirrored)) assert.False(t, loop.withholdLateApply(windowEvent("alpha", "3", "kubectl-edit"))) assert.False(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager, "kubectl-edit")), @@ -179,12 +190,18 @@ func TestApplyWindow_ClosesAGraceAfterFluxFinished(t *testing.T) { // TestApplyWindow_AnAttemptThatAppliedNothingHasNoWindow: a stale, failed or dropped attempt // stamped nothing, so nothing the applier writes afterwards is the apply it asked for. func TestApplyWindow_AnAttemptThatAppliedNothingHasNoWindow(t *testing.T) { - for _, outcome := range []ApplyOutcome{ApplyStale, ApplyRequestFailed, ApplyRejected, ApplyDropped, ApplyCurrent} { + for _, outcome := range []ApplyOutcome{ApplyStale, ApplyRequestFailed, ApplyRejected, ApplyDropped} { t.Run(string(outcome), func(t *testing.T) { loop, req := sentAttempt(t) assert.False(t, loop.withholdLateApply(windowEvent("alpha", "3", "kubectl-edit"))) report(req, outcome) loop.serviceAppliers() + state := loop.appliers[applierRef("alpha")] + if state.window != nil { + // A stale attempt is decided again at once: the window is the next attempt's. + require.NotEqual(t, req.ID, state.window.attempt, "the attempt that applied nothing has none") + return + } assert.False(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager))) }) } @@ -255,6 +272,14 @@ func completedThenNext(t *testing.T) (*branchWorkerEventLoop, *recordingRequeste return loop, requester, next[0] } +// sentAfterTheEarlierApply dates the applier's current window's send an hour on, so the writes the +// next windowEvents date come before it: one attempt is out at a time, so the earlier apply's late +// writes were made before the next attempt was sent. A carried edit guards only those. +func sentAfterTheEarlierApply(t *testing.T, loop *branchWorkerEventLoop) { + t.Helper() + onlyApplier(t, loop).window.sentAt = time.Now().Add(time.Hour) +} + // TestApplyWindow_ANewAttemptKeepsTheGraceOfTheOneBefore is the fifth review's third finding. A new // attempt replaced its applier's window, and the edits the one before had seen went with it, while // that earlier apply's writes could still arrive within its grace. Its late write over the edit was @@ -263,6 +288,7 @@ func TestApplyWindow_ANewAttemptKeepsTheGraceOfTheOneBefore(t *testing.T) { reader, err := telemetry.InitTestExporter() require.NoError(t, err) loop, _, _ := completedThenNext(t) + sentAfterTheEarlierApply(t, loop) assert.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), "A's late write over the edit is withheld while B is out") @@ -274,14 +300,15 @@ func TestApplyWindow_ANewAttemptKeepsTheGraceOfTheOneBefore(t *testing.T) { // window of its own, but A's grace still runs, so A's late write is withheld until it ends, and // mirrored after. func TestApplyWindow_AnAttemptThatAppliedNothingKeepsTheGraceBeforeIt(t *testing.T) { - for _, outcome := range []ApplyOutcome{ApplyCurrent, ApplyStale} { + for _, outcome := range []ApplyOutcome{ApplyStale, ApplyDropped} { t.Run(string(outcome), func(t *testing.T) { loop, _, next := completedThenNext(t) report(next, outcome) - loop.openWindow = &openWindow{} // a busy branch: a stale B is not sent again yet + holdApplies(loop) // a stale B is not sent again yet loop.serviceAppliers() state := onlyApplier(t, loop) require.Nil(t, state.inflight) + sentAfterTheEarlierApply(t, loop) assert.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), "A's late write is withheld within A's grace") @@ -303,8 +330,9 @@ func TestApplyWindow_TheCarriedGraceEndsWhenTheEarlierWindowWouldHave(t *testing require.False(t, window.carriedUntil.IsZero()) window.carriedUntil = time.Now().Add(-time.Millisecond) - assert.False(t, loop.withholdLateApply(windowEvent("alpha", "3", FluxFieldManager)), - "A's grace is over: the applier's write is B's apply") + assert.True(t, loop.withholdLateApply(windowEvent("alpha", "3", FluxFieldManager)), + "A's grace is over: the applier's write is B's apply landing, which is not mirrored") + assert.False(t, onlyApplier(t, loop).repair, "and leaves nothing to put back") assert.False(t, loop.withholdLateApply(windowEvent("alpha", "4", "kubectl-edit"))) assert.True(t, loop.withholdLateApply(windowEvent("alpha", "3", FluxFieldManager)), "an edit made while B runs is B's own") @@ -312,10 +340,11 @@ func TestApplyWindow_TheCarriedGraceEndsWhenTheEarlierWindowWouldHave(t *testing // TestApplyWindow_ConsecutiveOperationsOnOneApplier composes the steps: A completes, B is sent within // A's grace, A's late write over the edit is withheld, B completes, and the Repair that write left -// pending goes out once, on the quiet branch, at the tip. Each window closes a grace after its own attempt. +// pending goes out once, after a flush, at the tip. Each window closes a grace after its own attempt. func TestApplyWindow_ConsecutiveOperationsOnOneApplier(t *testing.T) { loop, requester, second := completedThenNext(t) assert.Equal(t, configv1alpha3.ApplyReasonForeignCommit, second.Reason) + sentAfterTheEarlierApply(t, loop) require.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager))) loop.serviceAppliers() @@ -337,3 +366,69 @@ func TestApplyWindow_ConsecutiveOperationsOnOneApplier(t *testing.T) { assert.False(t, state.repairPending()) assert.WithinDuration(t, time.Now().Add(applyWindowGrace), state.window.closesAt, time.Second) } + +// TestApplyWindow_AnApplierEntryOlderThanTheSendIsAnEdit is F1's lower bound: an entry of the +// applier's older than every send still open was written by an earlier apply, and is newest only +// because an update removed every field the editor owned. It is the undo of an edit, mirrored, and +// it counts as one, so the apply landing over it afterwards is withheld. +func TestApplyWindow_AnApplierEntryOlderThanTheSendIsAnEdit(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + loop, _ := sentAttempt(t) + beforeSend := time.Now().Add(-time.Hour).Truncate(time.Second) + + assert.False(t, loop.withholdLateApply(windowEventAt("alpha", "3", beforeSend, FluxFieldManager)), + "an earlier apply's entry is an edit, and is mirrored") + assert.Equal(t, int64(0), echoes(t, reader, echoNotMirrored)) + assert.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), + "the apply landing over it is withheld") + assert.True(t, loop.appliers[applierRef("alpha")].repair) +} + +// TestApplyWindow_ACarriedEditDoesNotWithholdTheNextApply: B is sent within A's grace, carrying the +// edit made while A ran. B applies a commit that holds every edit published before it was sent, so +// B's own write over that object, made after B's send, is B's apply landing: not withheld, and no +// Repair. Only A's late write, made before B was sent, is withheld because of the carried edit. +func TestApplyWindow_ACarriedEditDoesNotWithholdTheNextApply(t *testing.T) { + loop, _, _ := completedThenNext(t) + window := onlyApplier(t, loop).window + window.sentAt = time.Now().Add(-time.Hour) // B's send, before every write windowEvent dates + + assert.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), + "B's write is its apply landing, not mirrored") + assert.False(t, onlyApplier(t, loop).repair, "and B is not repaired over an edit its commit holds") +} + +// TestApplyWindow_ADelayedEchoOfTheAttemptBeforeIsNotMirrored: B is sent within A's grace, and A's +// apply of an object nobody edited arrives after B was sent. Its entry is older than B's send, but +// not than A's, which is still carried: it is A's apply landing, and is not mirrored. +func TestApplyWindow_ADelayedEchoOfTheAttemptBeforeIsNotMirrored(t *testing.T) { + loop, _, _ := completedThenNext(t) + window := onlyApplier(t, loop).window + require.False(t, window.carriedSentAt.IsZero()) + window.sentAt = time.Now().Add(time.Hour) // B was sent after A's write + betweenSends := window.carriedSentAt.Add(time.Second) + + echo := windowEventAt("alpha", "2", betweenSends, FluxFieldManager) + echo.Identifier.Name = "nobody-edited" + assert.True(t, loop.withholdLateApply(echo), "A's echo, inside A's grace") + assert.False(t, onlyApplier(t, loop).repair) + + window.carriedUntil = time.Now().Add(-time.Millisecond) + assert.False(t, loop.withholdLateApply(echo), "once A's grace is over the entry is older than every send") +} + +// TestApplyWindow_AnApplierWriteWithNoWindowIsLate: with no apply window open, the applier's +// managers alone are mirrored as an edit, and counted late (C3). +func TestApplyWindow_AnApplierWriteWithNoWindowIsLate(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + loop, _ := manualLoop(t, applierTestTarget("alpha", true), applierTestTarget("bravo", false)) + loop.followRecordedOperations(time.Now()) + require.Len(t, loop.appliers, 1) + + assert.False(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager))) + assert.False(t, loop.withholdLateApply(windowEvent("alpha", "2", "kubectl-edit"))) + assert.False(t, loop.withholdLateApply(windowEvent("bravo", "2", FluxFieldManager)), "no applier covers bravo") + assert.Equal(t, int64(1), echoes(t, reader, echoLate)) +} diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index 814f27e5..23c83bc8 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -145,21 +145,32 @@ type BranchWorker struct { triggerStatus ApplyTriggerStatusWriter inbox applyInbox // applyWake wakes the loop to service the appliers without queueing a work item: an attempt - // reported back, or another worker writing this repository and branch went quiet. + // reported back, or another worker writing this repository and branch released a source group. applyWake chan struct{} // siblings are the workers writing the same repository and branch as this one, under another - // GitProvider; their open windows, pending writes and work items make the branch busy too. - // busyWriting is this worker's own, published for them at the end of every wake; inflightItems - // below covers the wake in between. Nil siblings (the CLI, tests) means this worker is alone. - // See apply_siblings.go. - siblings *branchSiblings - busyWriting atomic.Bool + // GitProvider; they take turns on the source groups their appliers share. Nil siblings (the CLI, + // tests) means this worker is alone. See apply_siblings.go. + siblings *branchSiblings // groupsOut are the source groups this worker has an attempt out for, so a sibling sends none // for the same GitRepository meanwhile; guarded by the siblings' lock (apply_groups.go). groupsOut map[string]struct{} // unresolvedOperations is how many of this branch's apply operations are Unresolved, as the // loop last published it, for the apply_operations_unresolved gauge. unresolvedOperations atomic.Int64 + // applyEpoch, appliersAhead and hasAppliers are the loop's ApplyState for the watch side, which + // reads it when a snapshot's collection starts (apply_epoch.go). + applyEpoch atomic.Uint64 + appliersAhead atomic.Bool + hasAppliers atomic.Bool + // flushAsked is the latest flush a sibling asked of this worker, and flushDone the latest it + // published up to its mark for (apply_sibling_flush.go). + flushAsked atomic.Uint64 + flushDone atomic.Uint64 + // initialSnapshots reports whether every collection a GitTarget watches has a settled + // snapshot, and resyncRecheck asks the watch side for fresh snapshots of a GitTarget. Set by the + // WorkerManager before Start; nil in the CLI and in tests (apply_epoch.go). + initialSnapshots func(target itypes.ResourceReference) bool + resyncRecheck func(target itypes.ResourceReference) // Event processing eventQueue chan WorkItem @@ -211,7 +222,7 @@ type BranchWorker struct { // across every error return on the write path, and a lock there would put a second ordering // constraint on code whose only job is to record that something went wrong. // - // See the package's design note in docs/design/push-notification-and-reconcile-trigger.md §1.5. + // See the package's design note in docs/design/push-notification-and-reconcile-trigger.md, Part 1. baseTrustedState atomic.Bool // checkoutApplied is how many of the loop's retained writes the checkout holds: the checkout is @@ -328,6 +339,9 @@ type BranchWorker struct { // records that boundary. pendingResyncsMu sync.Mutex pendingResyncs map[resyncKey]*resyncMarker + // queuedResyncs counts, per GitTarget, the resync markers on the FIFO the loop has not taken. + // Guarded by pendingResyncsMu. See queuedResyncsFor. + queuedResyncs map[string]int // crOutcomes holds resolved CommitRequest outcomes for the controller to poll // via LookupCommitRequestOutcome. The event loop is the only writer (on its @@ -374,14 +388,6 @@ type branchWorkerLogFirsts struct { push sync.Once } -// healKey identifies a deferred heal by the (GitTarget, scope) it corrects, so re-stashing a heal -// for the same target+type replaces the parked one rather than queuing a duplicate. -type healKey struct { - name string - namespace string - scope string -} - type windowFinalizeReason string const ( @@ -395,6 +401,7 @@ const ( windowFinalizeReasonBufferLimit windowFinalizeReason = "buffer-limit" windowFinalizeReasonShutdown windowFinalizeReason = "shutdown" windowFinalizeReasonForeignCommit windowFinalizeReason = "foreign-commit" + windowFinalizeReasonApplyDue windowFinalizeReason = "apply-due" ) // NewBranchWorker creates a worker for a (provider, branch) combination. A zero @@ -768,6 +775,10 @@ func (w *BranchWorker) EnqueueResync(request *ResyncRequest) bool { w.inflightItems.Add(1) select { case w.eventQueue <- WorkItem{Resync: marker}: + if w.queuedResyncs == nil { + w.queuedResyncs = map[string]int{} + } + w.queuedResyncs[resyncTargetKey(request)]++ w.pendingResyncsMu.Unlock() w.Log.V(1).Info("Resync request enqueued", "resources", len(request.Desired), @@ -880,9 +891,27 @@ func (w *BranchWorker) takePendingResync(marker *resyncMarker) (*ResyncRequest, if w.pendingResyncs[key] == marker { delete(w.pendingResyncs, key) } + target := resyncTargetKey(marker.request) + if queued, counted := w.queuedResyncs[target]; counted && queued > 1 { + w.queuedResyncs[target] = queued - 1 + } else { + delete(w.queuedResyncs, target) + } return marker.request, marker.charge } +// queuedResyncsFor reports whether a resync of the GitTarget is on the FIFO, not taken yet. +func (w *BranchWorker) queuedResyncsFor(target string) bool { + w.pendingResyncsMu.Lock() + defer w.pendingResyncsMu.Unlock() + return w.queuedResyncs[target] > 0 +} + +// resyncTargetKey is the GitTarget a resync is for, as namespace/name. +func resyncTargetKey(request *ResyncRequest) string { + return request.GitTargetNamespace + "/" + request.GitTargetName +} + // enqueueRequest places a write request on the FIFO and reports whether it was // accepted. A false return means the item was refused (a full queue, paused intake, or a // stopping worker), so a caller that gates durable state on the write (a watch cursor) must not @@ -1150,11 +1179,14 @@ type branchWorkerEventLoop struct { // retry.go. retry retrySchedule - // deferredHeals holds heal resyncs (periodic re-anchors, removed-type sweeps) parked while a - // commit window is open, so a heal never force-finalizes (steals) that window — including a - // sibling GitTarget's held CommitRequest window on this shared worker (Rec 1 / 8f2ad84). They - // are drained by applyDeferredHeals at every idle boundary. Loop-goroutine only. - deferredHeals []*ResyncRequest + // snapshotRerequests are the GitTargets whose snapshot was discarded as stale, until a fresh one + // is asked for; snapshotHeldSince is when each GitTarget's first one was, until one is used. See + // apply_epoch.go. + snapshotRerequests map[string]itypes.ResourceReference + snapshotHeldSince map[string]time.Time + // ownershipLogged is the objects another applier owns already logged as kept out of a folder. + // See apply_ownership.go. + ownershipLogged map[string]struct{} // pendingCRs holds CommitRequests registered (via AttachCommitRequest) but not // yet resolved, keyed by identity. A request is parked here until a same-author @@ -1174,22 +1206,35 @@ type branchWorkerEventLoop struct { // authorized request is gone with nothing to report it. refusalPending map[refusalKey]pendingRefusalReapply // unsentRefusals holds the refusals the worker decided to send their target's applier a Refusal - // request for, until an attempt carries them on a quiet branch. It is keyed like refusalPending, + // request for, until an attempt carries them after a flush. It is keyed like refusalPending, // by target AND collection, so a collection that recovers cancels its own request and no other // (refusalRecovered). See apply_trigger.go. unsentRefusals map[refusalKey]unsentRefusal // appliers holds the reasons pending for each applier named on this branch and how its attempts - // are going, and nextApplyID numbers the attempts. quietSince is when the branch last became quiet (zero - // while it is busy), and applierTimer fires when the earliest waiting attempt comes due. See - // apply_trigger.go. + // are going, and nextApplyID numbers the attempts. applierTimer fires when the earliest waiting + // attempt comes due. See apply_trigger.go. appliers map[ApplierRef]*applierState nextApplyID uint64 - quietSince time.Time applierTimer *time.Timer + // listRetryAt is when a failed listing of the branch's appliers is tried again, zero when the + // last one succeeded (followRecordedOperations, takeReasons). + listRetryAt time.Time // pushHinted records that a Git host reported a push while the branch was busy. The wake closes - // the open window so its publication finds the move (closeWindowForForeignCommit). + // the open window so its publication finds the move (closeWindowForPushHint). pushHinted bool + // decisions numbers the decided writes (PendingWrite.decision). flushing is set while an apply is + // due and waits for the writes up to flushMark, the last decision when it came due, to be + // published (serviceAppliers). + decisions uint64 + flushing bool + flushMark uint64 + // siblingAsk is the generation this worker's flush asked its siblings for, zero with none. + // siblingFlushGen and siblingFlushMark are the last flush a sibling asked of this worker, and its + // mark. See apply_sibling_flush.go. + siblingAsk uint64 + siblingFlushGen uint64 + siblingFlushMark uint64 // attachTimer fires at the earliest attach deadline of a waiting CommitRequest, so a request // that no window reaches still resolves on time. An attached request needs no timer of its own: @@ -1263,7 +1308,7 @@ func (l *branchWorkerEventLoop) run() { l.applierTimer = nil // serviceAppliers, at the end of the wake, sends what has come due. case <-l.w.applyWake: - // An attempt reported back, or a sibling worker on this branch went quiet; + // An attempt reported back, or a sibling released a source group; // serviceAppliers settles and sends. } l.endWake(0) @@ -1280,19 +1325,22 @@ func (l *branchWorkerEventLoop) handleQueueItem(item WorkItem) { // // - Bind any waiting CommitRequest to an open window, resolve any whose attach deadline has // passed, and re-arm the deadline timer. -// - Drain any heal resync parked while a window was open, now that this wake may have finalized -// it (a silence timeout, a CommitRequest finalize). A no-op while a window is still open or -// nothing is parked. -// - Close the open window early if the branch moved under it (closeWindowForForeignCommit). +// - Close the open window early if a Git host reported a push (closeWindowForPushHint). // - Drive the pending writes: commit what the wake decided, schedule its push (advance). -// - Send the pending apply requests, if the wake left the branch quiet (serviceAppliers). +// - Send the apply requests that are due, after a flush, once the GitTargets they cover have +// their initial snapshots published (serviceAppliers). +// - Ask again for the snapshots discarded as stale once Git is no longer ahead, and publish the +// ApplyState (apply_epoch.go). // - Publish the result, moving the handled item's charge to what the loop holds. func (l *branchWorkerEventLoop) endWake(released int64) { l.serviceCommitRequests() - l.applyDeferredHeals() - l.closeWindowForForeignCommit() + l.closeWindowForPushHint() + l.takeSiblingFlush() l.advance() + l.reportSiblingFlush() l.serviceAppliers() + l.rerequestSnapshots() + l.publishApplyState() l.publishLoopState(released) } @@ -1371,12 +1419,7 @@ func (l *branchWorkerEventLoop) dispatch(item WorkItem) int64 { // finalizing it as identity, the byte cap or a zero window require. func (l *branchWorkerEventLoop) handleLiveEvents(request *WriteRequest) { for _, event := range request.Events { - if status, closed := l.w.renderFidelityClosed(event.GitTargetName, event.GitTargetNamespace); closed { - l.dropAtRenderFidelity(itypes.NewResourceReference(event.GitTargetName, event.GitTargetNamespace), - status, writeDroppedEvent, event.Identifier.String()) - continue - } - if l.withholdLateApply(event) { + if !l.mirrors(event) { continue } if l.openWindow != nil && !l.openWindow.canAppend(event) { @@ -1393,10 +1436,6 @@ func (l *branchWorkerEventLoop) handleLiveEvents(request *WriteRequest) { "resource", event.Identifier.String(), "eventTarget", event.GitTargetNamespace+"/"+event.GitTargetName) l.finalizeOpenWindowWithReason(windowFinalizeReasonIdentityChange) - // The window just closed and a new one for this event has not opened yet: an idle - // boundary. Drain any parked heal here so it gets a turn even under sustained, - // author-alternating load that never lets the silence timer fire (Rec 1 non-starvation). - l.applyDeferredHeals() } opened := false @@ -1449,6 +1488,28 @@ func (l *branchWorkerEventLoop) handleLiveEvents(request *WriteRequest) { } } +// mirrors reports whether a live event reaches the commit window. It does not while the target's +// render fidelity is not established, for an object another applier owns in a folder a trigger +// applies (F4), and for the applier's own write in its apply window (apply_window.go). +func (l *branchWorkerEventLoop) mirrors(event Event) bool { + if status, closed := l.w.renderFidelityClosed(event.GitTargetName, event.GitTargetNamespace); closed { + l.dropAtRenderFidelity(itypes.NewResourceReference(event.GitTargetName, event.GitTargetNamespace), + status, writeDroppedEvent, event.Identifier.String()) + return false + } + if l.ownedElsewhere(event.GitTargetName, event.GitTargetNamespace, event.Identifier.String(), + event.AppliedBy) { + return false + } + if l.withholdLateApply(event) { + return false + } + if event.Operation == string(itypes.OperationCreate) { + l.noteCreated(event) + } + return true +} + // closeOrArmWindow finalizes the open window when its first deadline has passed, and otherwise // arms the commit timer for that deadline. A zero timer therefore closes the window in the same // step that collected the write. @@ -1471,10 +1532,6 @@ func (l *branchWorkerEventLoop) closeOrArmWindow() { // the atomic write joins the pending writes behind it. func (l *branchWorkerEventLoop) handleAtomicRequest(request *WriteRequest) { l.finalizeOpenWindowWithReason(windowFinalizeReasonAtomicBeforeApply) - // Finalizing the window opened an idle boundary: a heal parked behind that window - // arrived BEFORE this atomic, so drain it here to keep arrival order (window, then heal, - // then atomic) rather than letting the atomic overtake it. - l.applyDeferredHeals() pendingWrite, err := l.w.buildAtomicPendingWrite(l.w.ctx, request) if err != nil { @@ -1489,9 +1546,6 @@ func (l *branchWorkerEventLoop) handleAtomicRequest(request *WriteRequest) { func (l *branchWorkerEventLoop) handleShutdown() { l.w.Log.Info("Handling shutdown, finalizing open window and pushing pending commits") l.finalizeOpenWindowWithReason(windowFinalizeReasonShutdown) - // The window is closed: drain any parked heal so its drift commits (and pushes below) on a - // clean shutdown rather than leaking its caller's reply channel. - l.applyDeferredHeals() if len(l.pendingWrites) > 0 { // Shutdown bypasses the cooldown and the retry: pending work needs to land before the // worker exits, even if a push was just sent. The push commits what was just decided. @@ -1689,7 +1743,8 @@ func (l *branchWorkerEventLoop) maybeSchedulePush() { if len(l.pendingWrites) == 0 || l.awaitingRetry() { return } - if l.lastPushAt.IsZero() { + if l.lastPushAt.IsZero() || l.publishesAtOnce() { + // An apply waits for this publication: it is made at once, cooldown or not. l.pushPending() return } @@ -1936,7 +1991,7 @@ func (w *BranchWorker) stoppingLocked() bool { return w.stoppingState } // // The push session reads the remote's ref advertisement on a connection the cycle was making // anyway, and a cycle that commits nothing still reaches it, so a trusted base needs no fetch to -// plan against. That is the whole saving. See docs/design/push-notification-and-reconcile-trigger.md §1.3 and §1.5. +// plan against. That is the whole saving. See docs/design/push-notification-and-reconcile-trigger.md, Part 1. // // Only a checkout holding no retained writes may fetch here at all: a reset would destroy the // local commits those writes produced. With writes retained, the event loop's materialize has @@ -2942,6 +2997,8 @@ func (r windowFinalizeReason) metricLabel() string { return "shutdown" case windowFinalizeReasonForeignCommit: return "foreign_commit" + case windowFinalizeReasonApplyDue: + return "apply_due" case windowFinalizeReasonUnspecified: return "unspecified" default: @@ -3123,6 +3180,7 @@ func (w *BranchWorker) publishObservation(observed RemoteObservation) { // still describes the repository the GitTarget points at. See RemoteObservation.Repo. w.lastObservation.Store(&observed) w.inbox.observe(observed) + w.siblings.shareObservation(w, observed) // Delivered here, at the one point where an observation is made, rather than by each caller // against whichever GitTargets it happened to be holding. The fact is about the BRANCH, so it // goes to the branch, and every target on it reads the same answer with the time it was diff --git a/internal/git/branch_worker_base_trust_test.go b/internal/git/branch_worker_base_trust_test.go index 37800fd9..43a5744f 100644 --- a/internal/git/branch_worker_base_trust_test.go +++ b/internal/git/branch_worker_base_trust_test.go @@ -2,7 +2,7 @@ package git -// The base-trust state machine from docs/design/push-notification-and-reconcile-trigger.md §1.5. +// The base-trust state machine from docs/design/push-notification-and-reconcile-trigger.md, Part 1. // // Nothing reads these flags to make a decision yet and the setter is hard-wired off, so none of // this can change behavior — which is exactly why the tests are written now. The hard part of the diff --git a/internal/git/branch_worker_empty_repo_test.go b/internal/git/branch_worker_empty_repo_test.go index 383e0d4a..70612387 100644 --- a/internal/git/branch_worker_empty_repo_test.go +++ b/internal/git/branch_worker_empty_repo_test.go @@ -143,7 +143,8 @@ func TestBranchWorker_AnEmptyRepositoryThatGainsABranchIsNotOrphaned(t *testing. assert.Equal(t, FinalizeCommitted, res.Outcome) } -// §4.2 test 2. The same with a write that changes nothing: the branch stays absent. +// §4.2 test 2. The same with a write that changes nothing: the branch stays absent. A write that +// changes nothing does not exist, so it settles before the remote is asked anything. func TestBranchWorker_AnEmptyRepositoryThatGainsABranchKeepsANoOpAbsent(t *testing.T) { f := newRealServerNewBranch(t, "empty-gains-main-noop", nil) @@ -152,7 +153,7 @@ func TestBranchWorker_AnEmptyRepositoryThatGainsABranchKeepsANoOpAbsent(t *testi deleted.Operation = "DELETE" deleted.Object = nil f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{deleted}, CommitMode: CommitModePerEvent}}) - require.Len(t, f.loop.pendingWrites, 1) + require.Empty(t, f.loop.pendingWrites, "settled at once, even in the cooldown") simulateClientCommitOnDisk(t, f.repoDir, "main", "README.md", "main appeared\n") f.loop.pushPending() @@ -267,8 +268,11 @@ func TestBranchWorker_ADetachedRemoteHeadFollowsGoGitsResolution(t *testing.T) { } } -// noOpWrite sends a live write that changes nothing: the delete of an object Git never had. +// noOpWrite saves a live write that changes nothing: the delete of an object Git never had. A +// save keeps it (a plain write that changes nothing settles without a push), so it is published, +// and its push observes the remote. func noOpWrite(loop *branchWorkerEventLoop) { + serviceAttach(loop, standbyRequest()) deleted := configMapTargetEvent("never-written", "alice", newBranchTarget) deleted.Operation = "DELETE" deleted.Object = nil diff --git a/internal/git/branch_worker_fetch_metric_test.go b/internal/git/branch_worker_fetch_metric_test.go index 25fa2505..d2981f59 100644 --- a/internal/git/branch_worker_fetch_metric_test.go +++ b/internal/git/branch_worker_fetch_metric_test.go @@ -2,7 +2,7 @@ package git -// Red-first, per docs/design/push-notification-and-reconcile-trigger.md §1.7: today's behavior is pinned as a +// Red-first, per docs/design/push-notification-and-reconcile-trigger.md, Part 1: today's behavior is pinned as a // passing assertion, so the change that removes the head-of-cycle fetch is proved by this // assertion flipping rather than by argument. // diff --git a/internal/git/branch_worker_metrics_test.go b/internal/git/branch_worker_metrics_test.go index 651eabd8..b044e7da 100644 --- a/internal/git/branch_worker_metrics_test.go +++ b/internal/git/branch_worker_metrics_test.go @@ -406,14 +406,14 @@ func TestExecuteResyncPendingWrite_ResetsCommittedWhenNothingChanges(t *testing. } // First execution writes and commits. - commits, err := w.executeResyncPendingWrite(t.Context(), repo, worktree, pendingWrite) + commits, err := w.executeResyncPendingWrite(t.Context(), repo, worktree, &pendingWrite) require.NoError(t, err) require.Equal(t, 1, commits) require.True(t, committed, "the first execution committed") // Second execution is the replay shape: the worktree already holds the desired state, so // nothing changes and no commit is made. The flag must come back down. - commits, err = w.executeResyncPendingWrite(t.Context(), repo, worktree, pendingWrite) + commits, err = w.executeResyncPendingWrite(t.Context(), repo, worktree, &pendingWrite) require.NoError(t, err) require.Equal(t, 0, commits) assert.False(t, committed, diff --git a/internal/git/branch_worker_new_branch_test.go b/internal/git/branch_worker_new_branch_test.go index 47f24770..37993656 100644 --- a/internal/git/branch_worker_new_branch_test.go +++ b/internal/git/branch_worker_new_branch_test.go @@ -175,6 +175,14 @@ func liveWrite(loop *branchWorkerEventLoop, name string) { }}) } +// noOpSave saves a live write of cm1, which the tests' parent already holds. A write that changes +// nothing does not exist and settles without a push (docs/design/applier-echoes.md, F2); a save is +// never trimmed, so its push still reaches the remote, which settles it. +func noOpSave(loop *branchWorkerEventLoop) { + serviceAttach(loop, standbyRequest()) + liveWrite(loop, "cm1") +} + // treeHas reports whether the commit's tree carries the file. func treeHas(t *testing.T, repo *git.Repository, commit plumbing.Hash, file string) bool { t.Helper() @@ -311,9 +319,10 @@ func TestBranchWorker_ResyncOnAbsentBranchDiffsAgainstTheMovedParent(t *testing. } // TestBranchWorker_NoDiffLiveWriteOnAbsentBranchStaysOnStandby is "keep normal deployments -// quiet". A live write the parent already holds commits nothing, and its push must not create the -// write branch merely to record that nothing changed. It still reaches the remote: "already -// present" is a claim only the advertisement can settle. +// quiet". A live write the parent already holds commits nothing, and must not create the write +// branch merely to record that nothing changed. A write that changes nothing does not exist, so it +// settles without a push; a save of it still reaches the remote, because "already present" is a +// claim only the advertisement can settle. func TestBranchWorker_NoDiffLiveWriteOnAbsentBranchStaysOnStandby(t *testing.T) { t.Run("the parent already holds the document", func(t *testing.T) { f := newNewBranchFixture(t, map[string]string{cmFile("cm1"): cmDocument("cm1")}) @@ -323,7 +332,21 @@ func TestBranchWorker_NoDiffLiveWriteOnAbsentBranchStaysOnStandby(t *testing.T) liveWrite(loop, "cm1") - require.Empty(t, loop.pendingWrites, "the no-diff write was published") + require.Empty(t, loop.pendingWrites, "the no-diff write settled at once") + assert.True(t, loop.lastPushAt.IsZero(), "with nothing to publish, nothing was pushed") + _, onRemote := f.featureOnRemote() + assert.False(t, onRemote, "and the write branch stays absent") + }) + + t.Run("a save the parent already holds", func(t *testing.T) { + f := newNewBranchFixture(t, map[string]string{cmFile("cm1"): cmDocument("cm1")}) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + f.warmOnParent() + + noOpSave(loop) + + require.Empty(t, loop.pendingWrites, "the no-diff save was published") assert.False(t, loop.lastPushAt.IsZero(), "it reached the remote to confirm") _, onRemote := f.featureOnRemote() assert.False(t, onRemote, "and the write branch stays absent") diff --git a/internal/git/branch_worker_parent_branch_test.go b/internal/git/branch_worker_parent_branch_test.go index 562e3f1d..e7178bf8 100644 --- a/internal/git/branch_worker_parent_branch_test.go +++ b/internal/git/branch_worker_parent_branch_test.go @@ -249,9 +249,9 @@ func TestBranchWorker_AParentChangeRebuildsRetainedWrites(t *testing.T) { defer loop.stopTimers() loop.lastPushAt = time.Now() - liveWrite(loop, "cm1") + noOpSave(loop) require.Len(t, loop.pendingWrites, 1) - require.True(t, loop.pendingWrites[0].CommitSHA.IsZero(), "a no-op is retained during the cooldown") + require.True(t, loop.pendingWrites[0].CommitSHA.IsZero(), "a no-op save is retained during the cooldown") require.True(t, f.worker.SetParentBranch(parent)) liveWrite(loop, "cm2") @@ -354,7 +354,7 @@ func TestBranchWorker_ObservesTheParentOfAnAbsentBranch(t *testing.T) { loop := newBranchWorkerEventLoop(f.worker, 0) defer loop.stopTimers() - liveWrite(loop, "cm1") + noOpSave(loop) o := observed(f) assert.Equal(t, ObservedByPush, o.By) diff --git a/internal/git/branch_worker_parent_change_test.go b/internal/git/branch_worker_parent_change_test.go index c697f91e..3518ff7d 100644 --- a/internal/git/branch_worker_parent_change_test.go +++ b/internal/git/branch_worker_parent_change_test.go @@ -137,7 +137,7 @@ func TestBranchWorker_AParentChangeDuringANoOpPushIsTakenByTheNextWrite(t *testi defer loop.stopTimers() roots := pushRoots(t, func() { f.worker.SetParentBranch("release") }) - liveWrite(loop, "cm1") // already in Git: settles without creating the branch + noOpSave(loop) // already in Git: settles without creating the branch _, onRemote := f.featureOnRemote() require.False(t, onRemote) require.Len(t, *roots, 1) diff --git a/internal/git/branch_worker_queue_depth_test.go b/internal/git/branch_worker_queue_depth_test.go index 0cb90781..e775bdcb 100644 --- a/internal/git/branch_worker_queue_depth_test.go +++ b/internal/git/branch_worker_queue_depth_test.go @@ -24,7 +24,7 @@ import ( // queueDepth() sums the in-flight count and the retained-work flag, so an item that opened a commit // window is covered by the first until the second is published. Releasing the count first leaves a // window where neither covers it, and the loop does real work inside that window -// (serviceCommitRequests, applyDeferredHeals, advance) before the flag is published. A scrape +// (serviceCommitRequests, advance, serviceAppliers) before the flag is published. A scrape // landing there reports a drained queue for a worker holding an uncommitted event, which can satisfy // a drain gate early. The loop releases the count only after the wake that handled the item. func TestHandledItem_PublishesRetainedWorkBeforeReleasingInflight(t *testing.T) { diff --git a/internal/git/branch_worker_split_test.go b/internal/git/branch_worker_split_test.go index ca5d341c..0afa7c7f 100644 --- a/internal/git/branch_worker_split_test.go +++ b/internal/git/branch_worker_split_test.go @@ -5,6 +5,7 @@ package git import ( "context" "errors" + "fmt" "os" "path/filepath" "strings" @@ -1006,12 +1007,15 @@ func TestEventLoop_CrossAuthorSamePathPingPongUsesOnlyAuthorBoundaries(t *testin loop := newBranchWorkerEventLoop(worker, time.Hour) loop.lastPushAt = time.Now() - for _, event := range []Event{ + for i, event := range []Event{ configMapEvent("F", "alice", "team-a"), configMapEvent("F", "bob", "team-a"), configMapEvent("X", "alice", "team-a"), configMapEvent("F", "alice", "team-a"), } { + // Each write changes the object, so every window has something to commit: a write that + // changes nothing settles at once, and would leave no window to count. + event.Object.Object["data"] = map[string]interface{}{"key": fmt.Sprintf("v%d", i)} loop.handleQueueItem(WorkItem{Request: &WriteRequest{ Events: []Event{event}, CommitMode: CommitModePerEvent, @@ -1124,11 +1128,11 @@ func TestEventLoop_AtomicPushFailure_DoesNotAdvanceCooldownOrLosePendingWrite(t pendingWrite, err := worker.buildAtomicPendingWrite(worker.ctx, request) require.NoError(t, err) - require.NoError(t, worker.commitPendingWrites([]PendingWrite{*pendingWrite})) loop := newBranchWorkerEventLoop(worker, time.Second) - loop.pendingWrites = []PendingWrite{*pendingWrite} - loop.pendingWritesBytes = pendingWrite.ByteSize + loop.decide(*pendingWrite) + loop.commitDecided() + require.Len(t, loop.pendingWrites, 1, "the atomic write is committed and pending its push") remotePath := strings.TrimPrefix(remoteURL, "file://") require.NoError(t, os.RemoveAll(remotePath)) diff --git a/internal/git/commit_executor.go b/internal/git/commit_executor.go index 70a2295f..8cd74465 100644 --- a/internal/git/commit_executor.go +++ b/internal/git/commit_executor.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "path/filepath" + "slices" "time" gogit "github.com/go-git/go-git/v6" @@ -86,7 +87,7 @@ func (w *BranchWorker) runPendingWrites( // Where this write starts: a refused write is undone back to here, keeping the writes // before it. before := headOr(repo, start) - created, hash, source, err := w.executePendingWrite(ctx, repo, worktree, pendingWrites[i]) + created, hash, source, err := w.executePendingWrite(ctx, repo, worktree, &pendingWrites[i]) if err != nil && skipRefused && undoRefusedWrite(repo, before, err) { refusals[i] = err continue @@ -159,13 +160,26 @@ func restoreWorktree(repo *gogit.Repository, start plumbing.Hash) error { return discardWorktreeLeftovers(repo) } -// retained reports whether the loop keeps this write once it is committed: every write does except -// a resync that changed nothing, which is neither retained nor pushed (see Committed). The checkout +// retained reports whether the loop keeps this write once it is committed: a write that changes +// nothing in Git does not exist (docs/design/applier-echoes.md, F2), so a write whose plan left no +// action, a window or a resync alike, is neither retained nor pushed. A save is the exception: it +// keeps its objects, and the push decides its outcome (resolvePushedCommitRequests). The checkout // counts the same writes (checkoutApplied), so the two can never disagree. func (p PendingWrite) retained() bool { - return p.Kind != PendingWriteResync || p.Committed == nil || *p.Committed + switch { + case p.Kind == PendingWriteRequestRecord || p.CommitRequest != nil: + return true + case p.Kind == PendingWriteResync: + return p.resyncActions == nil || !p.resyncActions.empty() + default: + return len(p.Events) > 0 + } } +// keepsEveryObject reports whether a write keeps the objects whose plan changed nothing: only a +// save does (docs/design/applier-echoes.md, decision 9). +func (p PendingWrite) keepsEveryObject() bool { return p.CommitRequest != nil || p.AllowEmpty } + func (p PendingWrite) path() string { if targetPath := p.Target().Path; targetPath != "" { return targetPath @@ -338,15 +352,18 @@ func (p PendingWrite) framedRequestMessage() (string, messageResolution) { return framed, messageResolutionRequestTemplate } +// executePendingWrite commits one write. Its first execution decides its actions on the tree it is +// committed on, and records them on the write (actionsDecided); every later one, a replay, applies +// those actions. func (w *BranchWorker) executePendingWrite( ctx context.Context, repo *gogit.Repository, worktree *gogit.Worktree, - pendingWrite PendingWrite, + pendingWrite *PendingWrite, ) (int, plumbing.Hash, messageResolution, error) { switch pendingWrite.Kind { case PendingWriteRequestRecord: - return w.executeRequestRecord(ctx, worktree, pendingWrite) + return w.executeRequestRecord(ctx, worktree, *pendingWrite) case PendingWriteResync: // Resync writes never carry a CommitRequest, so their commit hash is unused; // report ZeroHash to keep the per-write SHA bookkeeping uniform. @@ -360,12 +377,15 @@ func (w *BranchWorker) executePendingWrite( } } -// executeEventWrite applies a grouped window's or an atomic batch's events and commits them. +// executeEventWrite applies a grouped window's or an atomic batch's events and commits them. The +// first time, the write keeps only the events whose plan changed the tree: an object that changed +// nothing is no action, and a replay must not plan it again onto a newer tip, where it would revert +// what that tip changed (docs/design/applier-echoes.md, F2). A save keeps every object. func (w *BranchWorker) executeEventWrite( ctx context.Context, repo *gogit.Repository, worktree *gogit.Worktree, - pendingWrite PendingWrite, + pendingWrite *PendingWrite, ) (int, plumbing.Hash, messageResolution, error) { if len(pendingWrite.Events) == 0 { return 0, plumbing.ZeroHash, messageResolutionUnsupported, nil @@ -382,10 +402,17 @@ func (w *BranchWorker) executeEventWrite( fmt.Errorf("configure secret encryptor: %w", err) } - anyChanges, err := w.applyPendingWriteEvents(ctx, repo, worktree, pendingWrite.Events, pendingWrite.Targets) + anyChanges, changedBy, err := w.planPendingWriteEvents( + ctx, repo, worktree, pendingWrite.Events, pendingWrite.Targets) if err != nil { return 0, plumbing.ZeroHash, messageResolutionUnsupported, err } + if !pendingWrite.actionsDecided { + pendingWrite.actionsDecided = true + if !pendingWrite.keepsEveryObject() { + pendingWrite.Events = eventsThatChanged(pendingWrite.Events, changedBy) + } + } recordEmpty := !anyChanges && pendingWrite.AllowEmpty && !target.Suspend if !anyChanges && !recordEmpty { return 0, plumbing.ZeroHash, messageResolutionUnsupported, nil @@ -415,18 +442,37 @@ func (w *BranchWorker) executeEventWrite( return 1, hash, source, nil } -func (w *BranchWorker) applyPendingWriteEvents( +// eventsThatChanged is the events whose plan changed the tree, in order: events itself when every +// one did. +func eventsThatChanged(events []Event, changedBy []bool) []Event { + if !slices.Contains(changedBy, false) { + return events + } + kept := make([]Event, 0, len(events)) + for i := range events { + if changedBy[i] { + kept = append(kept, events[i]) + } + } + return kept +} + +// planPendingWriteEvents plans a write's events onto the worktree, one GitTarget base at a time, and +// reports whether the tree changed and, per event and in order, whether that event changed it. An +// event of a suspended target changes nothing. +func (w *BranchWorker) planPendingWriteEvents( ctx context.Context, repo *gogit.Repository, worktree *gogit.Worktree, events []Event, targets map[pendingTargetKey]ResolvedTargetMetadata, -) (bool, error) { +) (bool, []bool, error) { // Plan-then-flush each GitTarget subtree once: build the structure model, resolve // every event to a single-identity action, apply to hydrated file buffers, and // flush dirty/deleted files. A grouped window is single-target, so this is usually // one base path. - byBase := groupEventsByBase(events) + byBase, indexes := groupEventsByBaseWithIndex(events) + changedBy := make([]bool, len(events)) anyChanges := false for _, base := range sortedBaseKeys(byBase) { // A suspended target scans and publishes what it resolved, and writes nothing. The scan @@ -437,7 +483,7 @@ func (w *BranchWorker) applyPendingWriteEvents( // ones. if md, ok := targetForBase(targets, base); ok && md.Suspend { if err := w.refuseUnsafeWorktree(ctx, worktree, base, md); err != nil { - return false, err + return false, nil, err } log.FromContext(ctx).V(1).Info("live write suppressed: GitTarget is suspended", "gitTarget", md.Namespace+"/"+md.Name, "path", base, "events", len(byBase[base])) @@ -454,11 +500,11 @@ func (w *BranchWorker) applyPendingWriteEvents( // including one it did not author. The resync path already had this ordering. for _, event := range byBase[base] { if err := ensureBootstrapTemplateInPath(repo, base, event.BootstrapOptions); err != nil { - return false, err + return false, nil, err } } - changed, err := w.flushEventsToWorktree( + changed, baseChangedBy, err := w.planEventsToWorktree( ctx, worktree, base, @@ -468,13 +514,16 @@ func (w *BranchWorker) applyPendingWriteEvents( pruneModeForBase(targets, base), ) if err != nil { - return false, err + return false, nil, err } if changed { anyChanges = true } + for j, i := range indexes[base] { + changedBy[i] = baseChangedBy[j] + } } - return anyChanges, nil + return anyChanges, changedBy, nil } // executeRequestRecord commits a CommitRequest's message with an untouched tree: the record a diff --git a/internal/git/commit_executor_test.go b/internal/git/commit_executor_test.go index eb8f6c1e..e5730bb8 100644 --- a/internal/git/commit_executor_test.go +++ b/internal/git/commit_executor_test.go @@ -167,7 +167,7 @@ func TestExecutor_NoOpUnit_SkipsCommit(t *testing.T) { headBefore, err := repo.Head() require.NoError(t, err) - created, hash, _, err := worker.executePendingWrite(context.Background(), repo, worktree, PendingWrite{ + created, hash, _, err := worker.executePendingWrite(context.Background(), repo, worktree, &PendingWrite{ Kind: PendingWriteCommit, Events: []Event{event}, CommitConfig: ResolveCommitConfig(nil), @@ -210,7 +210,7 @@ func TestExecutor_AppliesEncryptionFromPendingWrite_NotFromWorker(t *testing.T) }, } - created, hash, _, err := worker.executePendingWrite(context.Background(), repo, worktree, pendingWrite) + created, hash, _, err := worker.executePendingWrite(context.Background(), repo, worktree, &pendingWrite) require.NoError(t, err) assert.Equal(t, 1, created) assert.False(t, hash.IsZero(), "a committed write reports its commit hash") diff --git a/internal/git/commit_request_attach.go b/internal/git/commit_request_attach.go index 6baf22e6..572ac668 100644 --- a/internal/git/commit_request_attach.go +++ b/internal/git/commit_request_attach.go @@ -20,6 +20,12 @@ var ErrFinalizeQueueFull = errors.New("branch worker event queue full; item drop // happen. var ErrResyncSuperseded = errors.New("resync superseded by a newer request for the same scope") +// ErrResyncDiscarded replies to a resync whose snapshot the worker did not use: it was collected +// while Git was ahead of the cluster, or across an apply, so an apply was about to make it stale +// (docs/design/applier-echoes.md, F3). It is NOT a failure: the worker asks for a fresh snapshot of +// the GitTarget once Git is no longer ahead. +var ErrResyncDiscarded = errors.New("resync snapshot discarded: an apply made it stale; a fresh one is requested") + // FinalizeOutcome is the terminal result of resolving a CommitRequest. type FinalizeOutcome string diff --git a/internal/git/commit_window_metrics_test.go b/internal/git/commit_window_metrics_test.go index 1794297d..640f6ef1 100644 --- a/internal/git/commit_window_metrics_test.go +++ b/internal/git/commit_window_metrics_test.go @@ -149,9 +149,9 @@ func TestCommitWindowMetrics_WindowsThatCommitNothingAreCountedToo(t *testing.T) require.True(t, loop.finalizeOpenWindow()) loop.pushPending() - // The same object again: a window with no diff. + // The same object again: a window with no diff, which settles at once. writeTo(loop, "present") - require.True(t, loop.finalizeOpenWindow()) + require.False(t, loop.finalizeOpenWindow()) // A window whose finalize is refused: the render fidelity gate closes while it is open. writeTo(loop, "blocked") diff --git a/internal/git/deleted_remote_branch_test.go b/internal/git/deleted_remote_branch_test.go index de85cadb..18abe232 100644 --- a/internal/git/deleted_remote_branch_test.go +++ b/internal/git/deleted_remote_branch_test.go @@ -16,7 +16,7 @@ package git // before the push, so the push simply re-created the branch. These tests are what keep that // recovery once the prefetch is conditional. // -// See docs/design/push-notification-and-reconcile-trigger.md §1.3 and §1.5. +// See docs/design/push-notification-and-reconcile-trigger.md, Part 1. import ( "context" diff --git a/internal/git/dirty_worktree_recovery_test.go b/internal/git/dirty_worktree_recovery_test.go index fda8bad5..27503fd0 100644 --- a/internal/git/dirty_worktree_recovery_test.go +++ b/internal/git/dirty_worktree_recovery_test.go @@ -11,7 +11,7 @@ package git // refuses a checkout that is not the projection of the retained writes, so a path that skipped // materialize fails rather than committing leftovers. // -// See docs/design/push-notification-and-reconcile-trigger.md §1.5. +// See docs/design/push-notification-and-reconcile-trigger.md, Part 1. import ( "errors" diff --git a/internal/git/git_request_ledger_test.go b/internal/git/git_request_ledger_test.go index 43a6a4f0..fb0a5c24 100644 --- a/internal/git/git_request_ledger_test.go +++ b/internal/git/git_request_ledger_test.go @@ -10,7 +10,7 @@ package git // every row carries request and response bytes beside its count, and the golden file publishes a // coarse size class for them — see sizeClass for why the class rather than the number. // -// See docs/design/push-notification-and-reconcile-trigger.md §1.6, which specifies this ledger. +// See docs/design/push-notification-and-reconcile-trigger.md, Part 1, which specifies this ledger. import ( "fmt" diff --git a/internal/git/git_roundtrip_ledger_test.go b/internal/git/git_roundtrip_ledger_test.go index 04df94eb..fb37bbbd 100644 --- a/internal/git/git_roundtrip_ledger_test.go +++ b/internal/git/git_roundtrip_ledger_test.go @@ -2,7 +2,7 @@ package git -// This is the measurement docs/design/push-notification-and-reconcile-trigger.md §1.6 asks for, before any of +// This is the measurement docs/design/push-notification-and-reconcile-trigger.md, Part 1, asks for, before any of // the changes that page argues for. Every connection count in §4 of that page is a prediction // derived from reading code; this file replaces reading with observing. // @@ -557,7 +557,7 @@ func renderLedger(rows []ledgerRow) string { var b strings.Builder b.WriteString("# Round trips to the Git host, per operation.\n") b.WriteString("# Generated by TestGitRoundTripLedger; specified by\n") - b.WriteString("# docs/design/push-notification-and-reconcile-trigger.md §1.6.\n") + b.WriteString("# docs/design/push-notification-and-reconcile-trigger.md, Part 1.\n") b.WriteString("#\n") b.WriteString("# Regenerate with: go test ./internal/git -run TestGitRoundTripLedger -update\n") b.WriteString("#\n") diff --git a/internal/git/intake.go b/internal/git/intake.go index b975e73c..7dfa8645 100644 --- a/internal/git/intake.go +++ b/internal/git/intake.go @@ -19,7 +19,7 @@ import ( // // What the budget counts is everything accepted and not yet published: the payload on the FIFO the // loop has not handled yet (charged at enqueue, so producers cannot fill the FIFO between two loop -// iterations), and what the loop holds (the open window, the pending writes, deferred heals, saves waiting for +// iterations), and what the loop holds (the open window, the pending writes, saves waiting for // a window). Every item is charged a fixed overhead beyond its payload, so the budget bounds how // many items are kept as well as their bytes. The charge is an estimate of the serialized YAML, not // of the heap: an object's in-memory form is several times larger. @@ -168,14 +168,11 @@ func (r *ResyncRequest) payloadSize() int64 { // charge is what a queued resync is charged: its snapshot, and the overhead of one kept item. func (r *ResyncRequest) charge() int64 { return r.payloadSize() + pendingWriteOverheadBytes } -// heldBytes is what the loop holds that is not published yet: the open window, the pending writes, the -// deferred heals' snapshots, and the saves waiting for a window. An attached save is in the charge -// of the window it is attached to, and then of the write that window became. +// heldBytes is what the loop holds that is not published yet: the open window, the pending writes, +// and the saves waiting for a window. An attached save is in the charge of the window it is attached +// to, and then of the write that window became. func (l *branchWorkerEventLoop) heldBytes() int64 { held := l.windowBytes + l.pendingWritesBytes - for _, heal := range l.deferredHeals { - held += heal.charge() - } for _, pcr := range l.pendingCRs { if !pcr.attached { held += attachCharge(pcr.message) diff --git a/internal/git/intake_test.go b/internal/git/intake_test.go index 240af2b6..7ab1e63d 100644 --- a/internal/git/intake_test.go +++ b/internal/git/intake_test.go @@ -69,8 +69,8 @@ func TestIntake_TheBudgetBoundsAcceptedWorkAtEnqueue(t *testing.T) { assert.LessOrEqual(t, loop.heldBytes(), budget, "everything accepted fits the budget once the loop holds it") } -// Everything the loop keeps counts against the budget: the open window, the pending writes, a deferred heal's -// snapshot, and a save registered while it waits for a window. +// Everything the loop keeps counts against the budget: the open window, the pending writes, and a +// save registered while it waits for a window. func TestIntake_EveryKindOfHeldWorkCounts(t *testing.T) { _, loop, _ := outageLoop(t, "intake-held-kinds") held := loop.heldBytes() @@ -84,16 +84,6 @@ func TestIntake_EveryKindOfHeldWorkCounts(t *testing.T) { "a registered save is charged for what it keeps") held = loop.heldBytes() - heal := &ResyncRequest{GitTargetName: ledgerTargetName, GitTargetNamespace: "default", Heal: true, - Desired: []manifestanalyzer.DesiredResource{{Object: bigConfigMap("deferred", 4096)}}, - Result: make(chan ResyncResult, 1)} - loop.openWindow = &openWindow{} // a heal waits for the window to close - loop.handleResyncRequest(heal) - loop.endWake(0) - require.Len(t, loop.deferredHeals, 1) - assert.GreaterOrEqual(t, loop.heldBytes()-held, int64(4096), "a deferred heal's snapshot is held") - held = loop.heldBytes() - loop.windowBytes = 2048 assert.Equal(t, held+2048, loop.heldBytes(), "and so is the open window") } diff --git a/internal/git/pending_writes.go b/internal/git/pending_writes.go index ffbfe882..426cc9bd 100644 --- a/internal/git/pending_writes.go +++ b/internal/git/pending_writes.go @@ -185,6 +185,7 @@ func (w *BranchWorker) resolveTargetMetadata( Namespaces: namespacePolicyFor(target.Spec, sourceNamespaces, wildcard), PruneMode: target.EffectivePruneMode(), SourceCluster: target.SourceCluster(), + Applied: target.Spec.ApplyTriggerRef != nil, Suspend: target.Spec.Suspend, }, nil } @@ -246,7 +247,7 @@ func resolvePlacementPolicy(spec *v1alpha3.GitTargetPlacementSpec) *manifestanal } // placementPolicyForBase finds the placement policy for the GitTarget that owns -// base among targets. base is already a sanitized subtree key (groupEventsByBase +// base among targets. base is already a sanitized subtree key (groupEventsByBaseWithIndex // runs it through sanitizePath), so md.Path must be sanitized the same way before // comparing — otherwise a root target, whose spec.path is "." but whose sanitized // base is "", would never match and would silently drop its declared placement on diff --git a/internal/git/pending_writes_loop.go b/internal/git/pending_writes_loop.go index 61426015..22861bd6 100644 --- a/internal/git/pending_writes_loop.go +++ b/internal/git/pending_writes_loop.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "slices" - "time" "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" itypes "github.com/ConfigButler/gitops-reverser/internal/types" @@ -31,7 +30,7 @@ import ( // waiting resync callers an answer (replyToDeferredResyncs). // // Settling an outcome never decides more work directly: a refused write leaves a reapply request -// due to its applier (reapplyForRefusal), which is sent once the branch is quiet, outside the +// due to its applier (reapplyForRefusal), which is sent after a flush, outside the // pending writes. // errWriteFailed marks a failure of the write itself, as opposed to reaching the remote or reading @@ -85,12 +84,8 @@ func retainedCharge(pendingWrite *PendingWrite) int64 { // A request riding it is held from here on, so the controller never fails a save whose write can // still land. Committing it is advance's job, at the end of the wake. func (l *branchWorkerEventLoop) decide(pendingWrite PendingWrite) { - // Every decided write restarts the quiet period the apply trigger waits out, wherever in the - // wake it is decided. A wake can open a window, close it, commit and push before it ends, and - // end quiet; reading "busy" only from the state a wake ends in would let that wake inherit the - // quiet period of the wakes before it and ask the applier to catch up at once, over the next - // edit. See serviceAppliers. - l.quietSince = time.Time{} + l.decisions++ + pendingWrite.decision = l.decisions pendingWrite.charge = retainedCharge(&pendingWrite) pendingWrite.decidedAt = l.w.now() l.pendingWrites = append(l.pendingWrites, pendingWrite) @@ -163,6 +158,7 @@ func (l *branchWorkerEventLoop) materialize() error { if i < len(l.pendingWrites) && l.pendingWrites[i].Kind == PendingWriteResync { refetch = fetchReasonForcedRecheck } + tipBefore := l.w.inbox.lastTip() if err := l.materializePrefix(refetch); err != nil { l.replyToDeferredResyncs(err) l.w.recordMaterializationFailure(err) @@ -173,6 +169,9 @@ func (l *branchWorkerEventLoop) materialize() error { if i == len(l.pendingWrites) { return nil } + if refetch != "" && l.discardIfFetchFoundForeign(i, tipBefore) { + continue + } // A write is planned under the policy in force when it is committed, exactly as a replay is: // an operator who tightened pruning since it was decided is obeyed. if err := l.w.tightenPendingPruneModes(l.w.ctx, l.pendingWrites[i:i+1]); err != nil { @@ -194,6 +193,26 @@ func (l *branchWorkerEventLoop) materialize() error { } } +// discardIfFetchFoundForeign discards the resync at i when the fetch it was planned behind found a +// commit the worker did not push: the cluster does not run it yet, and it holds what the snapshot is +// about to be judged against (docs/design/applier-echoes.md, F3). tipBefore is the tip observed before +// that fetch. The resync is decided and not committed, so nothing is committed on it. +func (l *branchWorkerEventLoop) discardIfFetchFoundForeign(i int, tipBefore string) bool { + req := l.pendingWrites[i].origin.resync + if req == nil || l.w.inbox.lastTip() == tipBefore || !l.w.inbox.foreignPending() || !l.snapshotsWait(req) { + return false + } + answered := l.pendingWrites[i].origin.answered + l.removeAt(i) + if answered { + // Its caller heard already that the remote held it back; it is only asked for again. + req = &ResyncRequest{GitTargetName: req.GitTargetName, GitTargetNamespace: req.GitTargetNamespace, + Scope: req.Scope} + } + l.discardSnapshot(req, snapshotForeignFetch) + return true +} + // materializePrefix rebuilds the writes committed before when the checkout is behind them: a write // failed part-way and could not be undone, a reset discarded their commits and the replay did not // finish, or their root was chosen under an older parent configuration. A parent change is read @@ -310,12 +329,17 @@ func (l *branchWorkerEventLoop) settleReplayRefusals(refused []PendingWrite) { } // settleCommitted acts on a write whose commit was made. A resync answers its caller with what it -// found, and one that committed nothing stops being pending: it is neither retained nor pushed. +// found. A write whose plan left no action, a resync or a window alike, stops being pending: it is +// neither retained nor pushed, so a replay can never plan it onto a newer tip +// (docs/design/applier-echoes.md, F2). func (l *branchWorkerEventLoop) settleCommitted(i int) { l.pendingWrites[i].committedOnce = true pendingWrite := l.pendingWrites[i] - if pendingWrite.origin.resync != nil { + switch { + case pendingWrite.origin.resync != nil: l.settleResyncApplied(i) + case !pendingWrite.retained(): + l.removeAt(i) } } @@ -325,6 +349,11 @@ func (l *branchWorkerEventLoop) settleResyncApplied(i int) { answered := l.pendingWrites[i].origin.answered stats := l.pendingWrites[i].ResyncStats committed := l.pendingWrites[i].retained() + if stats.Created > 0 { + // An object a resync adds to Git runs on the cluster already, and its applier never applied + // it: it is adopted (S2 in docs/design/applier-echoes.md). + l.adopt(req.GitTargetName, req.GitTargetNamespace, fmt.Sprintf("%d objects", stats.Created)) + } if !committed { l.removeAt(i) } diff --git a/internal/git/plan_flush.go b/internal/git/plan_flush.go index 21fe6ac4..ae0eb32d 100644 --- a/internal/git/plan_flush.go +++ b/internal/git/plan_flush.go @@ -27,7 +27,7 @@ import ( "github.com/ConfigButler/gitops-reverser/internal/typeset" ) -// flushEventsToWorktree is the plan-then-flush write path: model the subtree once, resolve each +// planEventsToWorktree is the plan-then-flush write path: model the subtree once, resolve each // event to a single-identity action, apply to commit-scoped buffers, flush only what changed. // // Every event is a single-identity intent and the writer NEVER mark-and-sweeps a batch. @@ -56,7 +56,10 @@ func clusterIDForEvents(events []Event) string { return "" } -func (w *BranchWorker) flushEventsToWorktree( +// planEventsToWorktree flushes one GitTarget base's events into the worktree, and reports whether +// the tree changed and, per event and in order, whether that event changed it: the write's actions +// (docs/design/applier-echoes.md, F2). +func (w *BranchWorker) planEventsToWorktree( ctx context.Context, worktree *gogit.Worktree, base string, @@ -64,11 +67,11 @@ func (w *BranchWorker) flushEventsToWorktree( policy *manifestanalyzer.PlacementPolicy, namespaces namespacePolicy, pruneMode v1alpha3.PruneMode, -) (bool, error) { +) (bool, []bool, error) { root := worktree.Filesystem().Root() scoped, err := scanRenderScope(root, base) if err != nil { - return false, err + return false, nil, err } // Every event in a base shares one GitTarget (events are grouped by base), so they share @@ -78,7 +81,7 @@ func (w *BranchWorker) flushEventsToWorktree( batch.pruneMode = pruneMode batch.target = placementTargetForEvents(events) if err := batch.refusal(); err != nil { - return false, err + return false, nil, err } // Publish what this folder's shape resolved to before folding anything in. It is a fact about // the scan, so it must not depend on the events applying — or on their being applied at all, @@ -90,7 +93,7 @@ func (w *BranchWorker) flushEventsToWorktree( // raised AFTER the layout is published: the folder is fine and its shape is still worth // reporting; what is wrong is the configuration pointed at it. if err := batch.sourceNamespaceRefusal(); err != nil { - return false, err + return false, nil, err } // Before the first event is folded in, not as they are reached: an upsert that refuses aborts // the batch, and a DELETE queued behind it would never be seen by the eligibility evidence the @@ -99,15 +102,19 @@ func (w *BranchWorker) flushEventsToWorktree( if batch.pruneMode.OrDefault().AppliesEventDeletes() { batch.declarePlannedRemovals(countDeleteEvents(events)) } - for _, event := range events { + changedBy := make([]bool, len(events)) + for i, event := range events { + batch.beginEvent() if err := batch.applyEvent(ctx, event); err != nil { - return false, err + return false, nil, err } + changedBy[i] = batch.eventChanged() } // The flush is anchored at renderBase — spec.path, or the common ancestor of spec.path // and every base it reads. The write jail (writeSubdir) is enforced inside the batch, so // a planned write outside spec.path is refused even though the scan reached past it. - return batch.flush(ctx, worktree, root, scoped.renderBase) + changed, err := batch.flush(ctx, worktree, root, scoped.renderBase) + return changed, changedBy, err } // countDeleteEvents counts the events in a batch that remove a document. @@ -210,6 +217,36 @@ type writeBatch struct { // (ADDED then MODIFIED, or back and gone again) read the pre-batch render and was wrong about // it. See rendersInBatch. rendered map[renderKey]bool + // touched records, while one event is applied, each file it touched with that file's bytes + // before the event. Nil when nothing is recorded. See beginEvent. + touched map[string]touchedFile +} + +// touchedFile is a file's bytes as they were before the event being applied touched it, and +// whether the file was there at all. +type touchedFile struct { + content []byte + present bool +} + +// beginEvent starts recording what the next event changes; eventChanged answers once it is applied. +// A write keeps only the objects whose plan changed its tree (docs/design/applier-echoes.md, F2), +// and the bytes are the one judge of that: every path an event takes, a field patch, an override +// edit or a kustomization entry, reaches its file through buffer. +func (wb *writeBatch) beginEvent() { wb.touched = map[string]touchedFile{} } + +// eventChanged reports whether the event applied since beginEvent changed any file's bytes, and +// stops recording. +func (wb *writeBatch) eventChanged() bool { + touched := wb.touched + wb.touched = nil + for rel, before := range touched { + after := wb.buffers[rel] + if before.present != (after.current != nil) || !bytes.Equal(before.content, after.current) { + return true + } + } + return false } // renderKey names one document of the store by where it lives and what it is. @@ -375,15 +412,18 @@ func (wb *writeBatch) declarePlannedRemovals(count int) { wb.plannedRemovals += // worktree bytes into Original/Current on first touch. A path with no worktree // bytes is a new file (Original nil). func (wb *writeBatch) buffer(rel string) *fileBuffer { - if b, ok := wb.buffers[rel]; ok { - return b + b, ok := wb.buffers[rel] + if !ok { + b = &fileBuffer{rel: rel} + if orig, found := wb.contentByPath[rel]; found { + b.original = orig + b.current = orig + } + wb.buffers[rel] = b } - b := &fileBuffer{rel: rel} - if orig, ok := wb.contentByPath[rel]; ok { - b.original = orig - b.current = orig + if _, seen := wb.touched[rel]; wb.touched != nil && !seen { + wb.touched[rel] = touchedFile{content: bytes.Clone(b.current), present: b.current != nil} } - wb.buffers[rel] = b return b } @@ -1964,16 +2004,19 @@ func loadWorktreeGitTargetIgnore(absBase string) (*manifestanalyzer.IgnoreMatche return manifestanalyzer.LoadGitTargetIgnore(content) } -// groupEventsByBase buckets events by their sanitized GitTarget base path, preserving -// arrival order within each bucket. A grouped commit window is single-target (one -// base) by construction; the grouping stays correct for any future multi-target batch. -func groupEventsByBase(events []Event) map[string][]Event { +// groupEventsByBaseWithIndex buckets events by their sanitized GitTarget base path, preserving +// arrival order within each bucket, and returns, per base, each event's index in events, so a +// result per event can be put back in order. A grouped commit window is single-target (one base) by +// construction; the grouping stays correct for any future multi-target batch. +func groupEventsByBaseWithIndex(events []Event) (map[string][]Event, map[string][]int) { byBase := map[string][]Event{} - for _, event := range events { + indexes := map[string][]int{} + for i, event := range events { base := sanitizePath(event.Path) byBase[base] = append(byBase[base], event) + indexes[base] = append(indexes[base], i) } - return byBase + return byBase, indexes } // sortedBufferKeys returns the buffer paths in lexicographic order so flushing is diff --git a/internal/git/planning_wrappers_test.go b/internal/git/planning_wrappers_test.go new file mode 100644 index 00000000..ae2d0505 --- /dev/null +++ b/internal/git/planning_wrappers_test.go @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "context" + + gogit "github.com/go-git/go-git/v6" + + "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" +) + +// The planners report per event, or with the actions a resync decided; most tests ask only whether +// the tree changed. These are those narrower answers. + +// flushEventsToWorktree is planEventsToWorktree without the per-event answer. +func (w *BranchWorker) flushEventsToWorktree( + ctx context.Context, + worktree *gogit.Worktree, + base string, + events []Event, + policy *manifestanalyzer.PlacementPolicy, + namespaces namespacePolicy, + pruneMode v1alpha3.PruneMode, +) (bool, error) { + changed, _, err := w.planEventsToWorktree(ctx, worktree, base, events, policy, namespaces, pruneMode) + return changed, err +} + +// applyPendingWriteEvents is planPendingWriteEvents without the per-event answer. +func (w *BranchWorker) applyPendingWriteEvents( + ctx context.Context, + repo *gogit.Repository, + worktree *gogit.Worktree, + events []Event, + targets map[pendingTargetKey]ResolvedTargetMetadata, +) (bool, error) { + changed, _, err := w.planPendingWriteEvents(ctx, repo, worktree, events, targets) + return changed, err +} + +// applyResyncToWorktree is planResyncToWorktree without the decided actions, for a GitTarget no +// other applier shares. +func (w *BranchWorker) applyResyncToWorktree( + ctx context.Context, + worktree *gogit.Worktree, + base string, + target ResolvedTargetMetadata, + desired []manifestanalyzer.DesiredResource, + scope *ResyncScope, +) (ResyncStats, bool, error) { + stats, changed, _, err := w.planResyncToWorktree(ctx, worktree, base, target, desired, scope, nil) + return stats, changed, err +} + +// groupEventsByBase is groupEventsByBaseWithIndex without the indexes. +func groupEventsByBase(events []Event) map[string][]Event { + byBase, _ := groupEventsByBaseWithIndex(events) + return byBase +} diff --git a/internal/git/refresh.go b/internal/git/refresh.go index 7da9fc52..7ef8f15a 100644 --- a/internal/git/refresh.go +++ b/internal/git/refresh.go @@ -76,7 +76,7 @@ func (w *BranchWorker) EnqueueRefresh(req *RefreshRequest) { } // noteBusyPushNotice records a push notice that reached a busy branch, so the wake closes the open -// window (closeWindowForForeignCommit): the publication it is about to make reads the remote anyway, +// window (closeWindowForPushHint): the publication it is about to make reads the remote anyway, // closing the window makes that happen now, and its rejection, if the branch moved, starts the // replay. A notice naming the tip already observed is the late report of a push already seen, // usually this worker's own, and is no reason to cut a window short. diff --git a/internal/git/refusal_reapply.go b/internal/git/refusal_reapply.go index 3f7b235c..9a7033ff 100644 --- a/internal/git/refusal_reapply.go +++ b/internal/git/refusal_reapply.go @@ -133,7 +133,7 @@ func (w *BranchWorker) refusalRateLimited(target itypes.ResourceReference) (bool } // reapplyForRefusal is the loop-side half: it turns a reported refusal into an unsent Refusal apply -// request for the target's applier, sent once the branch is quiet (serviceAppliers). +// request for the target's applier, sent after a flush (serviceAppliers). // // It runs on the event loop, after the refusal has already been reported, and it is deliberately // best-effort. A refusal is already surfaced as GitPathAccepted=False; failing to ask for a reapply @@ -309,7 +309,7 @@ func (l *branchWorkerEventLoop) refusalRecovered(target itypes.ResourceReference delete(l.refusalPending, key) } } - // An unsent refusal, decided and waiting for a quiet branch, is cancelled the same way. Keyed by + // An unsent refusal, decided and waiting for its flush, is cancelled the same way. Keyed by // target alone it could not be told apart from a sibling collection's, so it outlived the // recovery and asked for an apply for a refusal that no longer existed. for key := range l.unsentRefusals { @@ -440,7 +440,7 @@ func sortedRefusalKeys(pending map[refusalKey]pendingRefusalReapply) []refusalKe } // addUnsentRefusal records that the worker decided a Refusal request for the target's applier, -// which serviceAppliers sends once the branch is quiet. Several refusals for one target before then +// which serviceAppliers sends after a flush. Several refusals for one target before then // share one request. func (l *branchWorkerEventLoop) addUnsentRefusal(key refusalKey, refusal unsentRefusal, observation string) { // Recorded as covered now, when the request is decided, rather than when it is delivered: the @@ -452,7 +452,7 @@ func (l *branchWorkerEventLoop) addUnsentRefusal(key refusalKey, refusal unsentR l.unsentRefusals = map[refusalKey]unsentRefusal{} } l.unsentRefusals[key] = refusal - l.w.Log.Info("Refused edit: asking its applier to reapply once the branch is quiet", + l.w.Log.Info("Refused edit: asking its applier to reapply after a flush", "gitTarget", key.target.String(), "branch", l.w.Branch, "refused", refusal.detail) } diff --git a/internal/git/refusal_reapply_test.go b/internal/git/refusal_reapply_test.go index 5aec47b4..9567bd4e 100644 --- a/internal/git/refusal_reapply_test.go +++ b/internal/git/refusal_reapply_test.go @@ -173,22 +173,23 @@ func TestRefusalReapply_UnreadableTargetIsNotConsent(t *testing.T) { "a GitTarget that cannot be read must not be treated as opted in") } -// TestRefusalReapply_OwedRequestWaitsForAQuietBranch is the claim the action rests on: the request -// names the refusing target alone, and it is not sent while anything live is unpublished, because an -// apply then would revert that too. -func TestRefusalReapply_OwedRequestWaitsForAQuietBranch(t *testing.T) { +// TestRefusalReapply_TheRequestWaitsForAFlush is the claim the action rests on: the request names the +// refusing target alone, and it is sent only after a flush, so nothing live is left unpublished that +// the apply would revert too. +func TestRefusalReapply_TheRequestWaitsForAFlush(t *testing.T) { w := refusalTouchWorker(t, configv1alpha3.GitTargetSpec{OnRefusal: configv1alpha3.RefusalActionReapply}) requester := &recordingRequester{} w.applyRequester = requester loop := newBranchWorkerEventLoop(w, time.Minute) t.Cleanup(loop.stopTimers) - loop.openWindow = &openWindow{} + loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) + holdApplies(loop) loop.addUnsentRefusal(editingKey(), unsentRefusal{detail: "a base-owned field", uid: editingUID}, "observation-1") loop.serviceAppliers() - assert.Empty(t, requester.taken(), "nothing is sent while a commit window is open") + assert.Empty(t, requester.taken(), "nothing is sent while a save holds the flush") - loop.openWindow = nil + releaseApplies(loop) loop.serviceAppliers() requests := requester.taken() require.Len(t, requests, 1) @@ -495,11 +496,6 @@ func fluxTrigger(name string, mutate ...func(*configv1alpha3.FluxTrigger)) *conf return trigger } -// quietFor sets a FluxTrigger's quietPeriod. -func quietFor(d time.Duration) func(*configv1alpha3.FluxTrigger) { - return func(t *configv1alpha3.FluxTrigger) { t.Spec.QuietPeriod = &metav1.Duration{Duration: d} } -} - // testTriggersFor adds to objects the trigger each GitTarget among them names that objects does not // hold already: a FluxTrigger as fluxTrigger builds it, or an ArgoCDTrigger holding the Application // argocd/, each in the GitTarget's namespace. @@ -558,20 +554,12 @@ func createTestTrigger(t *testing.T, c client.Client, target *configv1alpha3.Git type recordingRequester struct { mu sync.Mutex requests []ApplyRequest - // prefetches are the PrefetchSource requests, in order. - prefetches []PrefetchRequest // counted keeps the reasons of the requests taken, so refusals still sees them. counted map[configv1alpha3.ApplyReason]int outcome ApplyOutcome manual bool } -func (r *recordingRequester) PrefetchSource(req PrefetchRequest) { - r.mu.Lock() - defer r.mu.Unlock() - r.prefetches = append(r.prefetches, req) -} - func (r *recordingRequester) RequestApply(req ApplyRequest) { r.mu.Lock() r.requests = append(r.requests, req) diff --git a/internal/git/replay_failure_test.go b/internal/git/replay_failure_test.go index d0e90eac..6ade57ec 100644 --- a/internal/git/replay_failure_test.go +++ b/internal/git/replay_failure_test.go @@ -18,7 +18,7 @@ package git // worktreeDirty is false throughout: the reset cleared it, and the worktree really is clean. It is // the retained writes that are stale, which is why this needs an invariant of its own. // -// See docs/design/push-notification-and-reconcile-trigger.md §1.5. +// See docs/design/push-notification-and-reconcile-trigger.md, Part 1. import ( "context" diff --git a/internal/git/reset_cleanliness_test.go b/internal/git/reset_cleanliness_test.go index 44c9c487..380cb200 100644 --- a/internal/git/reset_cleanliness_test.go +++ b/internal/git/reset_cleanliness_test.go @@ -12,7 +12,7 @@ package git // reaches the server, so the compare-and-swap caveat that forces other tests onto // startRealGitServer does not apply. // -// See docs/design/push-notification-and-reconcile-trigger.md §1.5. +// See docs/design/push-notification-and-reconcile-trigger.md, Part 1. import ( "context" diff --git a/internal/git/resync_flush.go b/internal/git/resync_flush.go index 08b4b4ed..356f5258 100644 --- a/internal/git/resync_flush.go +++ b/internal/git/resync_flush.go @@ -10,6 +10,8 @@ import ( "time" gogit "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/object" "sigs.k8s.io/controller-runtime/pkg/log" v1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" @@ -20,66 +22,8 @@ import ( itypes "github.com/ConfigButler/gitops-reverser/internal/types" ) -// handleResyncRequest applies one revision-pinned resync in order on the worker goroutine, or — -// for a HEAL resync that arrives while a commit window is open — defers it until the worker is idle -// so it never force-finalizes (steals) that window (Rec 1 / the 8f2ad84 regression). A non-heal -// resync, and a heal that arrives with no open window, applies immediately via applyResync. -func (l *branchWorkerEventLoop) handleResyncRequest(req *ResyncRequest) { - if req.Heal && l.openWindow != nil { - l.stashDeferredHeal(req) - return - } - l.applyResync(req) -} - -// stashDeferredHeal parks a heal resync until the commit window is idle (applyDeferredHeals drains -// it). It keeps one heal per (GitTarget, scope): a newer heal for the same key folds a fresher -// desired set, so the older one is superseded and its caller replied to immediately — otherwise the -// drainScopedResync goroutine waiting on the old request's channel would leak. -func (l *branchWorkerEventLoop) stashDeferredHeal(req *ResyncRequest) { - key := resyncHealKey(req) - for i := range l.deferredHeals { - if resyncHealKey(l.deferredHeals[i]) == key { - l.deferredHeals[i].reply(ResyncResult{}) - l.deferredHeals[i] = req - return - } - } - l.deferredHeals = append(l.deferredHeals, req) - l.w.Log.V(1).Info("heal resync deferred until the commit window is idle", - "scope", req.Scope.String(), - "gitTarget", req.GitTargetNamespace+"/"+req.GitTargetName, - "deferred", len(l.deferredHeals)) -} - -// applyDeferredHeals drains every parked heal once no commit window is open, so a heal never -// force-finalizes a window and never steals a sibling GitTarget's held CommitRequest window on a -// shared branch worker. A no-op while a window is still open or nothing is parked; the loop calls it -// at every idle boundary (silence timeout, identity switch, shutdown). -func (l *branchWorkerEventLoop) applyDeferredHeals() { - if l.openWindow != nil || len(l.deferredHeals) == 0 { - return - } - heals := l.deferredHeals - l.deferredHeals = nil - for _, req := range heals { - l.applyResync(req) - } -} - -// resyncHealKey identifies a deferred heal by the (GitTarget, scope) it corrects, so a re-stashed -// heal for the same target+type replaces rather than duplicates the parked one. -func resyncHealKey(req *ResyncRequest) healKey { - return healKey{ - name: req.GitTargetName, - namespace: req.GitTargetNamespace, - scope: req.Scope.String(), - } -} - // applyResync decides one revision-pinned resync in order on the worker goroutine. It mirrors the -// atomic-commit path: for a non-heal resync any open live window is finalized first so arrival order -// is preserved (a heal reaches here only when no window is open, so it finalizes nothing); the +// atomic-commit path: any open live window is finalized first so arrival order is preserved; the // resync is then decided like every other write, and its caller answered when its commit is made // or fails (settleResyncApplied, settleFailed), or as soon as the remote holds it back // (replyToDeferredResyncs). A resync that commits nothing stops being pending at once, so it never @@ -95,19 +39,16 @@ func resyncHealKey(req *ResyncRequest) healKey { // design removes is the PUBLICATION one, and a resync is not a publication. That fetch also serves // a forced GitTarget recheck, whose trigger is often "I changed Git; look again". func (l *branchWorkerEventLoop) applyResync(req *ResyncRequest) { + l.snapshotUsed(req) l.w.Log.Info("Handling resync request", "resources", len(req.Desired), "resourceVersion", req.ResourceVersion, "scope", req.Scope.String(), - "heal", req.Heal, "gitTarget", req.GitTargetNamespace+"/"+req.GitTargetName, "openWindow", l.openWindow != nil, "pendingWrites", len(l.pendingWrites)) - // A heal must never finalize a window (it only ever runs at idle); only a non-heal resync - // force-finalizes the open window to preserve arrival order before its mark-and-sweep. - if !req.Heal { - l.finalizeOpenWindowWithReason(windowFinalizeReasonResyncBeforeApply) - } + // A resync force-finalizes the open window to preserve arrival order before its mark-and-sweep. + l.finalizeOpenWindowWithReason(windowFinalizeReasonResyncBeforeApply) pendingWrite, err := l.w.buildResyncPendingWrite(l.w.ctx, req, &ResyncStats{}) if err != nil { @@ -115,6 +56,7 @@ func (l *branchWorkerEventLoop) applyResync(req *ResyncRequest) { req.reply(ResyncResult{Err: err}) return } + pendingWrite.Desired, pendingWrite.ownedElsewhere = l.withoutOwnedElsewhere(req) pendingWrite.Committed = new(bool) pendingWrite.origin.resync = req l.decide(*pendingWrite) @@ -186,11 +128,15 @@ func estimateDesiredSize(desired []manifestanalyzer.DesiredResource) int64 { // subtree's secret encryptor, folds the desired snapshot over the worktree, records // the plan stats on the caller's reply, and commits once when anything changed. A // resync that finds the mirror already in sync changes nothing and creates no commit. +// +// The snapshot is used once, to decide: the first execution records the upserts that changed the +// tree and the documents the sweep removed (resyncActions), and a replay onto a newer tip applies +// exactly those and never sweeps again (docs/design/applier-echoes.md, F2). func (w *BranchWorker) executeResyncPendingWrite( ctx context.Context, repo *gogit.Repository, worktree *gogit.Worktree, - pendingWrite PendingWrite, + pendingWrite *PendingWrite, ) (int, error) { target := pendingWrite.Target() base := sanitizePath(target.Path) @@ -213,14 +159,15 @@ func (w *BranchWorker) executeResyncPendingWrite( if target.Suspend { log.FromContext(ctx).V(1).Info("resync suppressed: GitTarget is suspended", "gitTarget", target.Namespace+"/"+target.Name, "path", base) + if !pendingWrite.actionsDecided { + // It decides nothing, so it is neither retained nor pushed. + pendingWrite.actionsDecided, pendingWrite.resyncActions = true, &resyncActions{} + } return 0, nil } - // Stage the path's bootstrap template (its directory and any .sops.yaml) before - // applying, exactly as the per-event path does via ensureBootstrapTemplateInPath. - // Without it a first resync into a fresh subtree has no directory for SOPS to chdir - // into when it encrypts a Secret. - if err := ensureBootstrapTemplateInPath(repo, base, target.BootstrapOptions); err != nil { + createsFolder, err := stageResyncFolder(repo, base, target) + if err != nil { return 0, err } @@ -236,19 +183,39 @@ func (w *BranchWorker) executeResyncPendingWrite( *pendingWrite.Committed = false } - stats, anyChanges, err := w.applyResyncToWorktree( - ctx, worktree, base, target, pendingWrite.Desired, pendingWrite.Scope, - ) + stats, anyChanges, err := w.decideOrReplayResync(ctx, worktree, base, target, pendingWrite, createsFolder) if err != nil { return 0, err } - if pendingWrite.ResyncStats != nil { - *pendingWrite.ResyncStats = stats - } - if !anyChanges { + if !anyChanges && (!createsFolder || !pendingWrite.resyncActions.bootstrap) { return 0, nil } + if err := commitResync(ctx, worktree, target, pendingWrite, stats); err != nil { + return 0, err + } + return 1, nil +} +// stageResyncFolder stages the path's bootstrap template (its directory and any .sops.yaml) before +// a resync applies, exactly as the per-event path does via ensureBootstrapTemplateInPath. Without it +// a first resync into a fresh subtree has no directory for SOPS to chdir into when it encrypts a +// Secret. It reports whether that creates a folder an applier applies: one must exist before the +// applier is asked to apply it, since Argo CD retries a sync of a missing path for minutes and Flux +// fails it, so the resync then commits it even with no object (resyncActions.bootstrap). +func stageResyncFolder(repo *gogit.Repository, base string, target ResolvedTargetMetadata) (bool, error) { + // Read before the bootstrap creates the directory. + createsFolder := target.Applied && base != "" && !headHasDir(repo, base) + if err := ensureBootstrapTemplateInPath(repo, base, target.BootstrapOptions); err != nil { + return false, err + } + return createsFolder, nil +} + +// commitResync commits what a resync changed in the worktree. +func commitResync( + ctx context.Context, worktree *gogit.Worktree, target ResolvedTargetMetadata, pendingWrite *PendingWrite, + stats ResyncStats, +) error { // Render the provider's reconcile commit template (e.g. a custom reconcile message), // counting the resources the resync changed and naming the scoped type + pinned // revision — the resync carries no events, so it cannot reuse the event-count path. @@ -258,15 +225,16 @@ func (w *BranchWorker) executeResyncPendingWrite( rendered, err := renderReconcileCommitMessage( changed, target.Name, pendingWrite.Scope, pendingWrite.ResourceVersion, pendingWrite.CommitConfig) if err != nil { - return 0, err + return err } - pendingWrite.CommitMessage = rendered - message, options, _, err := pendingWrite.commitMetadata() + committed := *pendingWrite + committed.CommitMessage = rendered + message, options, _, err := committed.commitMetadata() if err != nil { - return 0, err + return err } if _, err := worktree.Commit(message, options); err != nil { - return 0, fmt.Errorf("failed to create resync commit: %w", err) + return fmt.Errorf("failed to create resync commit: %w", err) } if pendingWrite.Committed != nil { *pendingWrite.Committed = true @@ -275,7 +243,62 @@ func (w *BranchWorker) executeResyncPendingWrite( "created", stats.Created, "updated", stats.Updated, "deleted", stats.Deleted, "skipped", stats.Skipped, "placementSkipped", stats.PlacementSkipped, "resourceVersion", pendingWrite.ResourceVersion) - return 1, nil + return nil +} + +// headHasDir reports whether the commit HEAD names, local commits included, holds the directory dir: +// a file under it. The directory on disk is no answer, since a reset can leave an empty one behind, +// and Git holds none. An unborn HEAD holds nothing. A HEAD that cannot be read says it holds it: the +// bootstrap commit is then left out, rather than risked as an empty one. +func headHasDir(repo *gogit.Repository, dir string) bool { + head, err := repo.Head() + if errors.Is(err, plumbing.ErrReferenceNotFound) { + return false + } + if err != nil { + return true + } + commit, err := repo.CommitObject(head.Hash()) + if err != nil { + return true + } + tree, err := commit.Tree() + if err != nil { + return true + } + _, err = tree.Tree(dir) + return !errors.Is(err, object.ErrDirectoryNotFound) +} + +// decideOrReplayResync folds a resync into the worktree: the first time, it plans the snapshot and +// records what it decided on the write, and the stats on the caller's reply; afterwards it applies +// those decided actions. +func (w *BranchWorker) decideOrReplayResync( + ctx context.Context, + worktree *gogit.Worktree, + base string, + target ResolvedTargetMetadata, + pendingWrite *PendingWrite, + createsFolder bool, +) (ResyncStats, bool, error) { + if pendingWrite.actionsDecided { + return w.applyResyncActionsToWorktree(ctx, worktree, base, target, pendingWrite.resyncActions) + } + stats, anyChanges, actions, err := w.planResyncToWorktree( + ctx, worktree, base, target, pendingWrite.Desired, pendingWrite.Scope, pendingWrite.ownedElsewhere, + ) + if err != nil { + return ResyncStats{}, false, err + } + if actions == nil { + actions = &resyncActions{} + } + actions.bootstrap = createsFolder + pendingWrite.actionsDecided, pendingWrite.resyncActions = true, actions + if pendingWrite.ResyncStats != nil { + *pendingWrite.ResyncStats = stats + } + return stats, anyChanges, nil } func (w *BranchWorker) refuseUnsafeWorktree( @@ -306,7 +329,7 @@ func (w *BranchWorker) refuseUnsafeWorktree( return nil } -// applyResyncToWorktree is the streaming mark-and-sweep resync apply (M8), described +// planResyncToWorktree is the streaming mark-and-sweep resync apply (M8), described // in docs/spec/reconcile-via-watchlist-mark-and-sweep.md ("Two Paths, One // Plan Type" — the Resync path). It folds the COMPLETE desired snapshot over the // content-derived store of the GitTarget subtree: @@ -332,23 +355,92 @@ func (w *BranchWorker) refuseUnsafeWorktree( // set is narrowed by a bad scope, an outage, or a controller that does not understand a newer // scope field updates and creates but removes nothing. `always` is the opt-in that restores // the full convergence described above. -func (w *BranchWorker) applyResyncToWorktree( +// +// It also returns what the resync decided: the actions a replay applies instead of planning the +// snapshot again. keep names, by objectKey, the objects another applier owns: they are not in +// desired, and their documents are not swept. +func (w *BranchWorker) planResyncToWorktree( ctx context.Context, worktree *gogit.Worktree, base string, target ResolvedTargetMetadata, desired []manifestanalyzer.DesiredResource, scope *ResyncScope, -) (ResyncStats, bool, error) { + keep map[string]bool, +) (ResyncStats, bool, *resyncActions, error) { // Normalize the prune mode ONCE, here, on this function's own copy of the metadata: everything // below asks the mode a question (may I sweep? what do I report?), and the empty string answers // "Never" to all of them while meaning "OnEvent". Doing it at the single entry point is why no // individual reader has to remember. target.PruneMode = target.PruneMode.OrDefault() + batch, scoped, err := w.resyncBatch(ctx, worktree, base, target) + if err != nil { + return ResyncStats{}, false, nil, err + } + // The store is built from the same files the planner reads, so the plan and the apply + // see identical bytes. The planner is the authoritative mark-and-sweep over the resolved + // resource-identity index; the upserts reuse the steady-state writer. A scoped resync + // (M12 per-type) restricts the sweep to one type so no sibling document is dropped. + plan := resyncPlan(batch.store, scoped.scan.YAMLFiles, desired, scope, target.PruneMode, + batch.unrenderedResources(), keep) + w.reportRetainedOrphans(ctx, plan, target, base, scope) + + stats, actions, err := batch.applyResyncPlan(ctx, desired, plan) + if err != nil { + return ResyncStats{}, false, nil, err + } + stats.PruneMode = target.PruneMode + // Anchored at renderBase; the write jail (writeSubdir) is enforced inside the flush. + changed, err := batch.flush(ctx, worktree, scoped.root, scoped.renderBase) + return stats, changed, actions, err +} + +// applyResyncActionsToWorktree replays what a resync decided onto a newer tree: each upsert is +// planned onto it, and each document the sweep removed is removed again where it is still there, +// while the target's prune mode still sweeps. Nothing else is planned, so an object that changed +// nothing, and a document the newer tip added, are left alone (docs/design/applier-echoes.md, F2). +func (w *BranchWorker) applyResyncActionsToWorktree( + ctx context.Context, + worktree *gogit.Worktree, + base string, + target ResolvedTargetMetadata, + actions *resyncActions, +) (ResyncStats, bool, error) { + target.PruneMode = target.PruneMode.OrDefault() + if actions == nil || actions.empty() { + return ResyncStats{PruneMode: target.PruneMode}, false, nil + } + batch, scoped, err := w.resyncBatch(ctx, worktree, base, target) + if err != nil { + return ResyncStats{}, false, err + } + stats, err := batch.applyResyncActions(ctx, actions, target.PruneMode) + if err != nil { + return ResyncStats{}, false, err + } + stats.PruneMode = target.PruneMode + changed, err := batch.flush(ctx, worktree, scoped.root, scoped.renderBase) + return stats, changed, err +} + +// resyncScoped is the scan a resync's batch was built from, and where its flush is anchored. +type resyncScoped struct { + root string + renderBase string + scan manifestanalyzer.FolderScan +} + +// resyncBatch builds the write batch a resync plans in, behind the gates every resync passes. +func (w *BranchWorker) resyncBatch( + ctx context.Context, + worktree *gogit.Worktree, + base string, + target ResolvedTargetMetadata, +) (*writeBatch, resyncScoped, error) { root := worktree.Filesystem().Root() scoped, err := scanRenderScope(root, base) if err != nil { - return ResyncStats{}, false, err + return nil, resyncScoped{}, err } batch := newWriteBatch( @@ -372,30 +464,15 @@ func (w *BranchWorker) applyResyncToWorktree( // so the watch layer surfaces it as a blocked stream instead of writing into a folder it // does not understand. if err := batch.refusal(); err != nil { - return ResyncStats{}, false, err + return nil, resyncScoped{}, err } // The same precondition the live path applies: a folder declared namespace-free that two // source namespaces reach cannot tell its own documents apart, and a resync writes exactly // what the live path writes. if err := batch.sourceNamespaceRefusal(); err != nil { - return ResyncStats{}, false, err + return nil, resyncScoped{}, err } - // The store is built from the same files the planner reads, so the plan and the apply - // see identical bytes. The planner is the authoritative mark-and-sweep over the resolved - // resource-identity index; the upserts reuse the steady-state writer. A scoped resync - // (M12 per-type) restricts the sweep to one type so no sibling document is dropped. - plan := resyncPlan(batch.store, scoped.scan.YAMLFiles, desired, scope, target.PruneMode, - batch.unrenderedResources()) - w.reportRetainedOrphans(ctx, plan, target, base, scope) - - stats, err := batch.applyResyncPlan(ctx, desired, plan) - if err != nil { - return ResyncStats{}, false, err - } - stats.PruneMode = target.PruneMode - // Anchored at renderBase; the write jail (writeSubdir) is enforced inside the flush. - changed, err := batch.flush(ctx, worktree, root, scoped.renderBase) - return stats, changed, err + return batch, resyncScoped{root: root, renderBase: scoped.renderBase, scan: scoped.scan}, nil } // applyResyncPlan folds the desired set and the plan's managed drops into the @@ -404,12 +481,16 @@ func (w *BranchWorker) applyResyncToWorktree( // re-derive each target's position from the live bytes, exactly as the steady-state // delete path does. An upsert error aborts before any flush, so a failed resync // writes nothing. +// +// It returns what the resync decided: every upsert that changed the tree, and every document the +// sweep removed (resyncActions). func (wb *writeBatch) applyResyncPlan( ctx context.Context, desired []manifestanalyzer.DesiredResource, plan manifestanalyzer.Plan, -) (ResyncStats, error) { +) (ResyncStats, *resyncActions, error) { var stats ResyncStats + actions := &resyncActions{} // Declared from the PLAN, before the first upsert: an upsert that refuses returns immediately, // and the sweep below it would never run, so the refusal would carry evidence saying this // flush removes nothing while the plan says otherwise. See writeBatch.plannedRemovals. @@ -423,9 +504,13 @@ func (wb *writeBatch) applyResyncPlan( // Count from what the upsert actually did, not from the plan: a sensitive // resource is PlanSkip in the plan but applyUpsert re-encrypts and changes it, // so plan-based stats would report a real commit as skipped. + wb.beginEvent() outcome, err := wb.applyUpsert(ctx, eventForDesired(dr)) if err != nil { - return ResyncStats{}, err + return ResyncStats{}, nil, err + } + if wb.eventChanged() { + actions.upserts = append(actions.upserts, dr) } switch outcome { case upsertCreated: @@ -454,10 +539,12 @@ func (wb *writeBatch) applyResyncPlan( } removed, err := wb.removeDocument(ctx, action.Resource, target, documentDeletedSweep) if err != nil { - return ResyncStats{}, err + return ResyncStats{}, nil, err } if removed { stats.Deleted++ + actions.deletes = append(actions.deletes, + resyncDelete{resource: action.Resource, identity: action.Identity}) } } // Skipped stays a plan view (documents present but not editable in place); it is @@ -468,9 +555,92 @@ func (wb *writeBatch) applyResyncPlan( // writer at all. stats.Retained = plan.RetainedOrphans wb.tallyPruneRetention(plan) + return stats, actions, nil +} + +// resyncActions is what a resync decided on the tree it was first committed on +// (docs/design/applier-echoes.md, F2): the snapshot's objects whose upsert changed that tree, and the +// documents its sweep removed, by identity. The snapshot's full membership is used once, to decide; +// a replay applies these and nothing else, so it never sweeps again. +type resyncActions struct { + upserts []manifestanalyzer.DesiredResource + deletes []resyncDelete + // bootstrap is a folder an applier applies that did not exist: the resync creates it with its + // bootstrap files, objects or not, so the applier is never asked to apply a missing path. + bootstrap bool +} + +// resyncDelete is one document a resync's sweep removed: its resource, and the manifest identity it +// was found by. +type resyncDelete struct { + resource itypes.ResourceIdentifier + identity manifestedit.Identity +} + +// empty reports whether the resync decided nothing: it changed nothing in Git, so it does not exist. +func (a *resyncActions) empty() bool { + return len(a.upserts) == 0 && len(a.deletes) == 0 && !a.bootstrap +} + +// applyResyncActions replays a resync's decided actions onto this batch's tree. An upsert is +// planned onto it as at first; a delete removes its document if the tree still holds it, located +// by manifest identity first and by resource identity otherwise, as a live delete is, and only +// while pruneMode still sweeps: a policy tightened since the resync was decided is obeyed. +func (wb *writeBatch) applyResyncActions( + ctx context.Context, + actions *resyncActions, + pruneMode v1alpha3.PruneMode, +) (ResyncStats, error) { + var stats ResyncStats + sweeps := pruneMode.SweepsOrphans() + if sweeps { + wb.declarePlannedRemovals(len(actions.deletes)) + } + for _, dr := range actions.upserts { + outcome, err := wb.applyUpsert(ctx, eventForDesired(dr)) + if err != nil { + return ResyncStats{}, err + } + switch outcome { + case upsertCreated: + stats.Created++ + case upsertUpdated: + stats.Updated++ + case upsertSkippedUnsafe: + stats.PlacementSkipped++ + case upsertNoChange: + } + } + if !sweeps { + return stats, nil + } + for _, deleted := range actions.deletes { + target, found := wb.resolveResyncDelete(deleted) + if !found { + continue + } + removed, err := wb.removeDocument(ctx, deleted.resource, target, documentDeletedSweep) + if err != nil { + return ResyncStats{}, err + } + if removed { + stats.Deleted++ + } + } return stats, nil } +// resolveResyncDelete locates, in this batch's tree, the document a resync's sweep removed. +func (wb *writeBatch) resolveResyncDelete(deleted resyncDelete) (deleteTarget, bool) { + if dm := wb.store.ByManifestIdentity[deleted.identity]; dm != nil { + return deleteTarget{ + filePath: wb.docLoc[dm].FilePath, + id: rawManifestIDForCurrentBytes(deleted.identity, dm), + }, true + } + return wb.resolveDelete(Event{Identifier: deleted.resource}) +} + // retentionLogInterval bounds how often ONE GitTarget subtree's suppressed sweep is reported at // default verbosity. A resync fires per watched type, and per namespace within a type, so an // unthrottled line would repeat for every one of them on every reconcile of a target that is @@ -577,7 +747,8 @@ func eventForDesired(dr manifestanalyzer.DesiredResource) Event { // // unrendered are the documents the folder holds but does not render (writeBatch.unrenderedResources). // They are outside what the target renders, so the sweep neither removes them again nor counts them -// as retained. +// as retained. keep, by objectKey, are the objects another applier owns (apply_ownership.go): the +// sweep leaves their documents alone too. func resyncPlan( store *manifestanalyzer.ManifestStore, files []manifestedit.FileContent, @@ -585,8 +756,11 @@ func resyncPlan( scope *ResyncScope, pruneMode v1alpha3.PruneMode, unrendered map[itypes.ResourceIdentifier]bool, + keep map[string]bool, ) manifestanalyzer.Plan { - inScope := func(ri itypes.ResourceIdentifier) bool { return scope.Matches(ri) && !unrendered[ri] } + inScope := func(ri itypes.ResourceIdentifier) bool { + return scope.Matches(ri) && !unrendered[ri] && !keep[objectKey(ri)] + } return manifestanalyzer.BuildScopedPlan(store, files, desired, resyncPlanPolicy(pruneMode), inScope) } diff --git a/internal/git/resync_heal_test.go b/internal/git/resync_heal_test.go deleted file mode 100644 index f865471e..00000000 --- a/internal/git/resync_heal_test.go +++ /dev/null @@ -1,160 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package git - -import ( - "testing" - "time" - - "github.com/go-git/go-git/v6/plumbing" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -// TestHandleResyncRequest_HealDefersWhileWindowOpenThenApplies pins Rec 1 (the 8f2ad84 fix): a HEAL -// resync that arrives while a commit window is open must be DEFERRED — it must not force-finalize -// (steal) the window — and must apply once the window finalizes on its own (a silence timeout, a -// CommitRequest). The held edit's commit therefore lands first, then the heal's, preserving arrival -// order without the steal. -func TestHandleResyncRequest_HealDefersWhileWindowOpenThenApplies(t *testing.T) { - worker, serverRepo, _ := setupCommitPushSplitWorker(t) - createPlainGitTarget(t, worker, "team-a", "team-a") - - loop := newBranchWorkerEventLoop(worker, time.Hour) - defer loop.stopTimers() - - initialRef, err := serverRepo.Reference(plumbing.NewBranchReferenceName("main"), true) - require.NoError(t, err) - initialHash := initialRef.Hash() - - // A live ConfigMap edit opens a window (commitWindow is an hour, so it stays open). - loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, - CommitMode: CommitModePerEvent, - }}) - require.NotNil(t, loop.openWindow, "the edit must open a window") - - // A HEAL resync for the same GitTarget arrives while the window is open. - scope := ResyncScopeFor(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, "") - healCh := make(chan ResyncResult, 1) - loop.handleResyncRequest(&ResyncRequest{ - GitTargetName: "team-a", - GitTargetNamespace: "default", - Scope: &scope, - Heal: true, - Desired: nil, - Result: healCh, - }) - loop.endWake(0) - - // It must be parked, not applied: the window is still open and the heal has not replied. - require.NotNil(t, loop.openWindow, "a heal must never force-finalize the open window") - require.Len(t, loop.deferredHeals, 1, "the heal is parked until the window is idle") - select { - case <-healCh: - t.Fatal("a deferred heal must not reply before it applies") - default: - } - afterDeferRef, err := serverRepo.Reference(plumbing.NewBranchReferenceName("main"), true) - require.NoError(t, err) - require.Equal(t, initialHash, afterDeferRef.Hash(), "nothing must commit while the heal is deferred") - - // The window finalizes on its own (silence boundary) and the loop drains the parked heal. - loop.finalizeOpenWindow() - loop.endWake(0) - - require.Empty(t, loop.deferredHeals, "the heal applies once the window is idle") - res := <-healCh - require.NoError(t, res.Err, "the deferred heal applies cleanly once it gets its turn") -} - -// TestHandleResyncRequest_AtomicDrainsDeferredHealFirst proves the atomic-ordering fix: when an -// atomic request finalizes an open window (an idle boundary), a heal parked behind that window — -// which arrived BEFORE the atomic — is drained at that boundary rather than being overtaken by the -// atomic write. -func TestHandleResyncRequest_AtomicDrainsDeferredHealFirst(t *testing.T) { - worker, _, _ := setupCommitPushSplitWorker(t) - createPlainGitTarget(t, worker, "team-a", "team-a") - - loop := newBranchWorkerEventLoop(worker, time.Hour) - loop.lastPushAt = time.Now() // avoid an immediate push; we only assert in-memory ordering - defer loop.stopTimers() - - // A live edit opens a window, then a heal (for a different type) parks behind it. - loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, - CommitMode: CommitModePerEvent, - }}) - require.NotNil(t, loop.openWindow) - scope := ResyncScopeFor(schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}, "") - healCh := make(chan ResyncResult, 1) - loop.handleResyncRequest(&ResyncRequest{ - GitTargetName: "team-a", GitTargetNamespace: "default", - Scope: &scope, Heal: true, Result: healCh, - }) - loop.endWake(0) - require.Len(t, loop.deferredHeals, 1, "the heal parks behind the open window") - - // An atomic request arrives: it finalizes the window, and the parked heal must drain at that - // boundary (it arrived first) — not be left behind the atomic. - loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("atomic", "reconciler", "team-a")}, - CommitMode: CommitModeAtomic, - }}) - - require.Empty(t, loop.deferredHeals, "the atomic finalize drained the parked heal at its idle boundary") - select { - case res := <-healCh: - require.NoError(t, res.Err, "the drained heal applied") - default: - t.Fatal("the heal must have applied when the atomic finalized the window") - } -} - -// TestHandleResyncRequest_HealDoesNotStealSiblingCommitRequestWindow is the shared-worker case of -// Rec 1: one BranchWorker serves N GitTargets and the commit window is a worker singleton, so a -// force-finalizing heal could finalize a DIFFERENT GitTarget's held window. A heal scoped to one -// GitTarget must leave another GitTarget's open CommitRequest window — and its pending request — -// untouched. -func TestHandleResyncRequest_HealDoesNotStealSiblingCommitRequestWindow(t *testing.T) { - worker, _, _ := setupCommitPushSplitWorker(t) - createPlainGitTarget(t, worker, crTarget, crTarget) - - loop := newBranchWorkerEventLoop(worker, time.Hour) - defer loop.stopTimers() - - // GitTarget crTarget opens a window holding alice's CommitRequest (a 60s grace keeps it open). - serviceAttach(loop, attachReq("alice", 60*time.Second)) - loop.handleQueueItem(WorkItem{Request: &WriteRequest{ - Events: []Event{configMapTargetEvent("held", "alice", crTarget)}, - CommitMode: CommitModePerEvent, - }}) - loop.endWake(0) - require.NotNil(t, loop.openWindow, "the edit must open a window") - require.NotNil(t, loop.openWindow.pendingCR, "the window must carry the attached CommitRequest") - - // A heal scoped to a DIFFERENT GitTarget arrives on the shared worker. - otherScope := ResyncScopeFor(schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}, "") - healCh := make(chan ResyncResult, 1) - loop.handleResyncRequest(&ResyncRequest{ - GitTargetName: "team-other", - GitTargetNamespace: "default", - Scope: &otherScope, - Heal: true, - Result: healCh, - }) - loop.endWake(0) - - // The sibling's window and its CommitRequest must be untouched, and the heal merely parked. - require.Len(t, loop.deferredHeals, 1, "the heal is parked, not applied") - require.NotNil(t, loop.openWindow, "another GitTarget's window must stay open") - require.NotNil(t, loop.openWindow.pendingCR, "the sibling's CommitRequest must stay attached") - _, resolved := outcome(t, worker) - assert.False(t, resolved, "a heal for one GitTarget must not resolve another GitTarget's CommitRequest") - select { - case <-healCh: - t.Fatal("a deferred heal must not reply while a sibling window is open") - default: - } -} diff --git a/internal/git/resync_scope_test.go b/internal/git/resync_scope_test.go index 790212a7..1860b0a5 100644 --- a/internal/git/resync_scope_test.go +++ b/internal/git/resync_scope_test.go @@ -104,35 +104,15 @@ func TestResyncScope_StringIsNilSafeAndNamesTheNamespace(t *testing.T) { allNamespaces := resyncScopePtr(configmapsGVRForScope, "") assert.Equal(t, "configmaps", allNamespaces.String(), "the scope renders its collection, which carries no served version: a version bump must not "+ - "split the deferred-heal key or the coalescing key for one sweep boundary") + "render one sweep boundary two ways") scoped := resyncScopePtr(configmapsGVRForScope, "team-a") assert.Contains(t, scoped.String(), "team-a", - "the namespace must appear in the scope string: it keys deferred heals, so two "+ - "namespaces of one type sharing a key would silently drop one heal") + "the namespace must appear in the scope string: it keys coalescing, so two "+ + "namespaces of one type sharing a key would silently drop one snapshot") assert.NotEqual(t, allNamespaces.String(), scoped.String()) } -// Deferred heals are keyed by (GitTarget, scope). Once one GitTarget can watch a type in -// several namespaces, the key must separate them — otherwise stashing team-b's heal -// replaces team-a's parked one and team-a's drift is never corrected. -func TestResyncHealKey_SeparatesNamespacesOfTheSameType(t *testing.T) { - req := func(ns string) *ResyncRequest { - return &ResyncRequest{ - GitTargetName: "team-a-config", - GitTargetNamespace: "default", - Scope: resyncScopePtr(configmapsGVRForScope, ns), - } - } - assert.NotEqual(t, resyncHealKey(req("team-a")), resyncHealKey(req("team-b")), - "two namespaces of one type must key separately") - assert.Equal(t, healKey{ - name: "team-a-config", namespace: "default", - scope: (resyncScopePtr(configmapsGVRForScope, "team-a")).String(), - }, resyncHealKey(req("team-a")), - "the key is stable for the same scope, so a re-stashed heal replaces rather than duplicates") -} - // THE test for this change. A replay covers one namespace, so its desired set names only // that namespace's objects. The sweep must therefore be confined to that namespace: a // GVR-only sweep would find team-b's document absent from desired and delete it, silently @@ -223,7 +203,7 @@ func TestResync_ClusterWideScopeStillSweepsEveryNamespace(t *testing.T) { // A scope's identity must round-trip to the boundary it sweeps. It did not: the scope carried // a full GVR while Matches compared group, resource and namespace only, so two served versions -// of one resource were two coalescing keys, two deferred-heal keys and two render-fidelity +// of one resource were two coalescing keys and two render-fidelity // scopes — over one sweep boundary. The version is now data on the scope, and the collection is the // identity. func TestResyncScope_ServedVersionIsDataNotIdentity(t *testing.T) { @@ -244,6 +224,4 @@ func TestResyncScope_ServedVersionIsDataNotIdentity(t *testing.T) { } assert.Equal(t, resyncKeyFor(request(v1)), resyncKeyFor(request(v1beta1)), "two snapshots of one boundary must coalesce, not queue two sweeps of each other's documents") - assert.Equal(t, resyncHealKey(request(v1)), resyncHealKey(request(v1beta1)), - "and park as one deferred heal") } diff --git a/internal/git/testdata/git-roundtrip-ledger.golden b/internal/git/testdata/git-roundtrip-ledger.golden index 1084d44f..5319eb66 100644 --- a/internal/git/testdata/git-roundtrip-ledger.golden +++ b/internal/git/testdata/git-roundtrip-ledger.golden @@ -1,6 +1,6 @@ # Round trips to the Git host, per operation. # Generated by TestGitRoundTripLedger; specified by -# docs/design/push-notification-and-reconcile-trigger.md §1.6. +# docs/design/push-notification-and-reconcile-trigger.md, Part 1. # # Regenerate with: go test ./internal/git -run TestGitRoundTripLedger -update # diff --git a/internal/git/types.go b/internal/git/types.go index bc2f4dec..5b132269 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -227,7 +227,7 @@ type CommitMode string const ( // CommitModePerEvent streams request events through the live commit window. // With commitWindow=0 each event finalizes immediately; otherwise events - // coalesce by author, target, and quiet-window boundaries. + // coalesce by author and target until the window's idle timeout or maximum duration. CommitModePerEvent CommitMode = "per_event" // CommitModeAtomic creates one commit for all events in the request. CommitModeAtomic CommitMode = "atomic" @@ -313,6 +313,11 @@ type ResolvedTargetMetadata struct { // against the right cluster's mapping. SourceCluster string + // Applied is whether the GitTarget names an apply trigger: an applier applies its folder, which + // must then exist before it is asked to. A resync into a folder that does not exist yet commits + // the folder's bootstrap files even when it holds no object (executeResyncPendingWrite). + Applied bool + // Suspend suppresses the WRITE only; the scan still runs, which keeps a suspended target's // status fresh. Being CAPTURED defines the cutover: a suspension arriving after this write was // planned does not retract it, and a commit already made locally is still pushed. Reading the @@ -382,6 +387,19 @@ type PendingWrite struct { // render failed at commit time. Zero means "not stamped", so messageSource() recomputes. committedMessageSource messageResolution + // actionsDecided is set once the write was planned on the tree it was first committed on. A + // write carries the actions that plan decided, never a picture of the cluster to plan again + // (docs/design/applier-echoes.md, F2): from then on Events holds only the objects whose plan + // changed that tree, except in a save (CommitRequest), which keeps every object it captured, + // and a resync carries resyncActions instead of its snapshot. A replay applies those actions to + // the newer tip, and plans nothing else again. Loop-goroutine only. + actionsDecided bool + // resyncActions is what a resync decided on its tree: the upserts that changed it and the + // documents its sweep removed. Nil until decided. See actionsDecided. + resyncActions *resyncActions + // ownedElsewhere names, by objectKey, the snapshot objects another applier owns, which a resync + // into a triggered folder neither upserts nor sweeps (apply_ownership.go). + ownedElsewhere map[string]bool // committedOnce is set once the loop has committed this write at least once. It stays set after // a reset discards that commit: whether the checkout holds it now is checkoutApplied's to say. A // decided write the loop could not commit yet (the remote was unreachable) stays pending with it @@ -397,6 +415,9 @@ type PendingWrite struct { // decidedAt is when the write was decided, which dates the oldest kept write. Loop-goroutine // only. decidedAt time.Time + // decision numbers the write in the order the loop decided it: the flush an apply waits for is the + // writes up to a mark (serviceAppliers). Loop-goroutine only. + decision uint64 // origin is what the write's outcome is settled against. See writeOrigin. origin writeOrigin @@ -465,7 +486,7 @@ func (s *ResyncScope) GVR() schema.GroupVersionResource { return schema.GroupVersionResource{Group: s.Collection.Group, Version: s.Version, Resource: s.Collection.Resource} } -// String renders the scope for logs and for the deferred-heal key. It is nil-safe: a nil +// String renders the scope for logs and status messages. It is nil-safe: a nil // scope is the whole-GitTarget resync and renders empty. func (s *ResyncScope) String() string { if s == nil { @@ -493,6 +514,16 @@ func (s *ResyncScope) Overlaps(other *ResyncScope) bool { return s.Collection.Overlaps(other.Collection) } +// ApplyState is where a branch worker's appliers stand, as a snapshot's collection reads it when it +// starts (docs/design/applier-echoes.md, F3). +type ApplyState struct { + // Epoch is raised at every send of an apply attempt and at every end of one. + Epoch uint64 + // GitAhead is whether Git holds a commit the cluster may not run yet: an applier on the branch + // has a ForeignCommit, Repair or Refusal pending, a converge due, or an attempt out. + GitAhead bool +} + // ResyncRequest is a synchronous resync of one GitTarget against a complete, // version-pinned desired snapshot (M8). It rides the worker queue so the single // git-mutating goroutine applies it in order with live events, and replies on @@ -510,12 +541,6 @@ type ResyncRequest struct { // removed type). Nil is a whole-GitTarget resync. See ResyncScope for the invariant // binding this to Desired. Scope *ResyncScope - // Heal marks a non-urgent drift-correcting resync the worker DEFERS while a commit window is - // open. One worker serves N GitTargets and the window is a worker singleton, so a - // force-finalizing heal can steal a DIFFERENT GitTarget's held CommitRequest window. Waiting - // for idle recurs on every silence timeout, so it never starves. A first-sync backfill is NOT - // a heal: it must establish initial state promptly. - Heal bool // SourceCollection names the target-watch collection that gathered this snapshot. Zero for a // whole-GitTarget resync, which speaks for no single collection. Diagnostic only: nothing // filters the queue on it. See source_collection.go. @@ -524,6 +549,19 @@ type ResyncRequest struct { // replay this is, for the scope Scope.Collection names. Zero for a resync no stream replayed. // The worker records that scope clean once the resync applies (recordReplayClean). RenderFidelityRevision uint64 + // Owners is the applier that owns each snapshot object some applier owns, read from the raw + // object (AppliedByOf). An object another applier owns is neither upserted into a triggered + // folder nor swept from it (apply_ownership.go). + Owners map[types.ResourceIdentifier]AppliedBy + // Initial marks the first snapshot of its collection for this GitTarget: none was applied for it + // before. It is the baseline the worker decides whatever Git holds, never held for an apply (F3 + // in docs/design/applier-echoes.md). + Initial bool + // CollectionStart is the worker's ApplyState read when the snapshot's collection started. A + // snapshot collected while Git was ahead of the cluster, or across an apply, holds a picture + // the apply is about to make stale, and is discarded (docs/design/applier-echoes.md, F3). Nil + // when the producer read none: only the worker's state on arrival is then checked. + CollectionStart *ApplyState // Result receives exactly one reply. It is buffered (cap 1) by the emitter so // the worker never blocks delivering it. Result chan ResyncResult @@ -672,6 +710,12 @@ type Event struct { // apply window reads it to tell an applier's write from an API edit (apply_window.go). LastFieldManagers []string + // AppliedBy is the applier that owns the object, read from its raw labels and annotations + // (AppliedByOf). PROVENANCE like LastFieldManagers, stamped by the live watch; zero when no + // applier owns it. An object another applier owns is not mirrored into a folder a trigger applies + // (apply_ownership.go). + AppliedBy AppliedBy + // LastFieldManagedAt is when those entries were written, to the second; zero with no // LastFieldManagers. The apply window compares it with the time of an API edit: a late apply // moves the applier's entry forward, and an undo that deletes the editor's entry does not. diff --git a/internal/git/worker_manager.go b/internal/git/worker_manager.go index 7fb31f9a..6b56fbc4 100644 --- a/internal/git/worker_manager.go +++ b/internal/git/worker_manager.go @@ -168,6 +168,12 @@ type WorkerManager struct { // for an apply while another worker on the same branch is busy writing. See apply_siblings.go. siblings *branchSiblings + // initialSnapshots and resyncRecheck are the watch side's half of the applier-echo rules every + // worker follows (apply_epoch.go). Set once at startup (SetSnapshotHooks); nil in the CLI and in + // tests. + initialSnapshots func(target types.ResourceReference) bool + resyncRecheck func(target types.ResourceReference) + // applyRequester delivers every worker's apply requests. Set once at startup // (SetApplyRequester) before any worker is created; nil in the CLI and in tests, which switches // the apply trigger off. @@ -227,6 +233,18 @@ func (m *WorkerManager) SetClusterMapper(resolver func(clusterID string) typeset m.clusterMapper = resolver } +// SetSnapshotHooks injects the watch side's half of docs/design/applier-echoes.md into every worker: +// settled reports whether every collection of a GitTarget has a settled snapshot, and recheck +// asks for fresh snapshots of a GitTarget whose snapshot a worker discarded as stale. Like SetMapper, +// it is called once at startup before any worker is created. +func (m *WorkerManager) SetSnapshotHooks( + settled func(target types.ResourceReference) bool, recheck func(target types.ResourceReference), +) { + m.mu.Lock() + defer m.mu.Unlock() + m.initialSnapshots, m.resyncRecheck = settled, recheck +} + // SetSSHHostKeyConfig injects the SSH host-key resolution config used by every worker's credential // reads. Like SetMapper, it is called once at startup before any worker is created. func (m *WorkerManager) SetSSHHostKeyConfig(cfg SSHHostKeyConfig) { @@ -405,6 +423,8 @@ func (m *WorkerManager) EnsureWorker( worker.scanAcceptance = m.scanAcceptance worker.applyRequester = m.applyRequester worker.triggerStatus = m.triggerStatus + worker.initialSnapshots = m.initialSnapshots + worker.resyncRecheck = m.resyncRecheck m.siblings.add(worker) worker.renderFidelityGate = m.renderFidelityGate worker.crOwners = m.commitRequests diff --git a/internal/telemetry/apply_requests.go b/internal/telemetry/apply_requests.go index 1a762325..b8e0fa31 100644 --- a/internal/telemetry/apply_requests.go +++ b/internal/telemetry/apply_requests.go @@ -8,23 +8,23 @@ import "go.opentelemetry.io/otel/metric" // object, labelled by {applier_kind, reason, outcome}. // // applier_kind names the applier: "kustomization" (a Flux Kustomization) or "application" (an Argo -// CD Application). reason is the attempt's ApplyReason (ForeignCommit, Refusal, Repair, CatchUp; -// never CatchUp for an application). outcome is a frozen enum: +// CD Application). reason is the attempt's ApplyReason (ForeignCommit, Refusal, Repair, CatchUp), +// for either kind. outcome is a frozen enum: // // sent — the applier was asked and the ask landed: a Kustomization was resumed with a -// requestedAt token or a sync operation set. Or it needed no request: a ForeignCommit or -// a CatchUp found the Kustomization suspended with the commit applied already, or the -// remote no longer carries the branch, so there is no commit to apply. +// requestedAt token or a sync operation set. What needs no apply (a converge of the commit +// the applier last applied, an empty tip) the branch worker never sends, so it is not +// counted. // failed — the attempt could not ask, including an applier that applies another repository or -// branch. The branch worker keeps the reason pending and decides a retry on a quiet branch, +// branch. The branch worker keeps the reason pending and decides a retry after a flush, // with a backoff of its own, so a failure that persists keeps counting. An // ApplyRequestFailed Event marks the first failure of a run. // dropped — the attempt asked nothing, because no GitTarget may still ask for its reason: each // was deleted or recreated, moved to another branch or applier, is no longer admitted by // its ClusterProvider, or withdrew its consent to a Refusal. // stale — the attempt asked nothing, because the decision no longer held when the write was due: -// the branch had moved on or become busy. The worker decides again at the next quiet -// moment. Frequent stale attempts mean edits keep arriving within an attempt's lifetime. +// the branch had moved on. The worker decides again at once, at the new tip. Frequent +// stale attempts mean the branch keeps moving within an attempt's lifetime. // // It counts attempts, so a request that fails twice and then lands is two "failed" and one "sent". // It says nothing about whether the applier APPLIED: the operation a sent request starts is diff --git a/internal/telemetry/apply_window.go b/internal/telemetry/apply_window.go index 12171c67..183107e3 100644 --- a/internal/telemetry/apply_window.go +++ b/internal/telemetry/apply_window.go @@ -16,8 +16,47 @@ import "go.opentelemetry.io/otel/metric" // the API edit itself. It was mirrored as ever, which cannot lose an edit; if it was the // apply, the revert is in Git as the applier's commit. // -// Any non-zero rate means edits are landing while an apply runs. A long quietPeriod makes the -// CatchUp windows rarer; the ForeignCommit and Refusal windows follow other people's pushes. +// Any non-zero rate means edits are landing while an apply runs: a withheld one is repaired, and the +// apply windows follow other people's pushes, refusals, repairs and API creates. // // Registered in registerCounters (exporter.go) with every other counter. var ApplyWindowCollisionsTotal metric.Int64Counter + +// ApplyEchoesTotal counts the applier's own writes the apply window judged: an apply GitOps Reverser +// asked for landing, object by object. Labelled by {provider_namespace, provider_name, branch, +// outcome}. outcome is a frozen enum: +// +// not_mirrored — the applier's managers alone wrote the object last, at or after the earliest send +// of the apply windows still open, and nobody edited it in the window: the apply +// itself. Its content is the commit's, which Git holds, so it is not mirrored. +// late — the applier's managers alone wrote the object last with no apply window open: an +// echo arriving after the window's grace, or an apply somebody else started. It is +// mirrored as an edit. A steady rate means applies keep landing outside the windows, +// and such a write can revert a newer Git-side change to the object. +// +// Registered in registerCounters (exporter.go) with every other counter. +var ApplyEchoesTotal metric.Int64Counter + +// ApplyOwnershipConflictsTotal counts objects another applier owns that a branch worker kept out of a +// folder an apply trigger applies: a live write not mirrored, or a snapshot object neither upserted +// nor swept (docs/design/applier-echoes.md, F4). Labelled by {provider_namespace, provider_name, +// branch}. Any non-zero rate is a misconfiguration: another Kustomization or Application applies +// objects into a namespace the folder mirrors, and its log line names both. +// +// Registered in registerCounters (exporter.go) with every other counter. +var ApplyOwnershipConflictsTotal metric.Int64Counter + +// StaleSnapshotsTotal counts resync snapshots a branch worker found stale: collected while Git was +// ahead of the cluster, across an apply, or arriving while Git is ahead, so an apply was about to make +// the picture they hold wrong (docs/design/applier-echoes.md, F3). Labelled by {provider_namespace, +// provider_name, branch, outcome}. outcome is a frozen enum: +// +// discarded — the snapshot was not decided, and a fresh one of its GitTarget is asked for once Git +// is no longer ahead. +// used — the GitTarget's snapshots had been held for the limit (five minutes) while an apply +// kept failing, so this one was decided anyway. Whatever the unapplied commit changed +// in the objects it holds is reverted, as the API-wins rule says. A non-zero rate means +// an applier is failing for minutes at a time. +// +// Registered in registerCounters (exporter.go) with every other counter. +var StaleSnapshotsTotal metric.Int64Counter diff --git a/internal/telemetry/exporter.go b/internal/telemetry/exporter.go index b4251c92..dcb0a58b 100644 --- a/internal/telemetry/exporter.go +++ b/internal/telemetry/exporter.go @@ -251,7 +251,8 @@ var ( // GitCommitWindowsTotal counts closed commit windows, labelled by {gittarget_namespace, // gittarget_name, close_reason, timer_source}. close_reason says what ended the window // (idle_timeout, max_duration, attach_next, identity_change, buffer_limit, resync_before_apply, - // atomic_before_apply, shutdown); timer_source says whose timers were in effect: the GitTarget's, + // atomic_before_apply, shutdown, foreign_commit for a Git host's push notice, apply_due for the + // flush before an apply); timer_source says whose timers were in effect: the GitTarget's, // or a CommitRequest's that attached and replaced them. Every window is counted exactly once, at // the start of its finalize, so a window that then produces no diff or fails to commit is counted // too. A push replay rebuilds commits and closes no window, and a CommitEmpty record has none. @@ -608,6 +609,9 @@ func registerCounters() error { {"gitopsreverser_apply_requests_total", &ApplyRequestsTotal}, {"gitopsreverser_apply_revision_mismatches_total", &ApplyRevisionMismatchesTotal}, {"gitopsreverser_apply_window_collisions_total", &ApplyWindowCollisionsTotal}, + {"gitopsreverser_apply_echoes_total", &ApplyEchoesTotal}, + {"gitopsreverser_stale_snapshots_total", &StaleSnapshotsTotal}, + {"gitopsreverser_apply_ownership_conflicts_total", &ApplyOwnershipConflictsTotal}, {"gitopsreverser_receiver_deliveries_total", &ReceiverDeliveriesTotal}, } for _, s := range counters { diff --git a/internal/watch/apply_state.go b/internal/watch/apply_state.go new file mode 100644 index 00000000..2c4be6b8 --- /dev/null +++ b/internal/watch/apply_state.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// The watch side's half of docs/design/applier-echoes.md: the branch worker judges a snapshot by +// what its appliers were doing when the snapshot's collection started (F3), and asks no applier for +// anything before a GitTarget's initial snapshots are in Git (S2). Both need what only the streams +// know. + +// applyStateAtCollectionStart reads the ApplyState of the GitTarget's branch worker, for a snapshot +// whose collection is about to start. Nil without a router, or when the worker cannot be resolved: +// the worker then judges the snapshot only by its state on arrival. +func (m *Manager) applyStateAtCollectionStart(ctx context.Context, gitDest types.ResourceReference) *git.ApplyState { + if m.EventRouter == nil { + return nil + } + return m.EventRouter.applyState(ctx, gitDest) +} + +// InitialSnapshotsSettled reports whether every collection the GitTarget's current plan watches has +// a settled snapshot: the branch worker applied, refused or failed one of it (markSnapshotSettled). +// The worker adds that it holds no resync of the GitTarget still to publish. +// +// A GitTarget with no planned collection has not settled them: its WatchRule may not be resolved +// yet (a CRD not discovered, a rule not reconciled), and its folder holds nothing of the cluster. It +// holds its applier back as a Blocked watch does, rather than let it apply a folder nothing has +// mirrored into. +// +// It is tracked per collection, not read from the stream's state: a stream streams once the worker +// took its snapshot, which may still be discarded. +func (m *Manager) InitialSnapshotsSettled(gitDest types.ResourceReference) bool { + table, ok := m.watchedTypeTableForGitDest(gitDest) + if !ok { + return false + } + collections := collectionsForWatchKeys(targetWatchKeys(table)) + return len(collections) > 0 && m.snapshotsSettled(gitDest, collections) +} + +// markSnapshotSettled records that a snapshot of the collection settled: a later one of it is no +// longer initial (git.ResyncRequest.Initial). +func (m *Manager) markSnapshotSettled(gitDest types.ResourceReference, collection types.CollectionKey) { + m.snapshotsMu.Lock() + defer m.snapshotsMu.Unlock() + if m.settledSnapshots == nil { + m.settledSnapshots = map[string]map[types.CollectionKey]bool{} + } + settled := m.settledSnapshots[gitDest.Key()] + if settled == nil { + settled = map[types.CollectionKey]bool{} + m.settledSnapshots[gitDest.Key()] = settled + } + settled[collection] = true +} + +// snapshotsSettled reports whether a snapshot of every one of the collections settled. +func (m *Manager) snapshotsSettled(gitDest types.ResourceReference, collections []types.CollectionKey) bool { + m.snapshotsMu.Lock() + defer m.snapshotsMu.Unlock() + settled := m.settledSnapshots[gitDest.Key()] + for _, collection := range collections { + if !settled[collection] { + return false + } + } + return true +} + +// forgetSettledSnapshots drops a torn-down GitTarget's record, so a same-name recreation starts with +// initial snapshots again. +func (m *Manager) forgetSettledSnapshots(gitDest types.ResourceReference) { + m.snapshotsMu.Lock() + defer m.snapshotsMu.Unlock() + delete(m.settledSnapshots, gitDest.Key()) +} diff --git a/internal/watch/apply_state_test.go b/internal/watch/apply_state_test.go new file mode 100644 index 00000000..1f3ab212 --- /dev/null +++ b/internal/watch/apply_state_test.go @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "errors" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// TestInitialSnapshots_TrackedPerCollection: a collection's snapshot is initial until one of it +// settled: the branch worker applied, refused or failed it. A discarded or superseded one leaves it +// initial, so the one that replaces it is still exempt from F3; tearing the GitTarget down forgets +// it, so a recreation starts over. +func TestInitialSnapshots_TrackedPerCollection(t *testing.T) { + configmaps := types.CollectionKeyFor(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, "") + secrets := types.CollectionKeyFor(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, "") + gitDest := types.NewResourceReference("alpha", "team-a") + m := &Manager{Log: logr.Discard()} + router := &EventRouter{WatchManager: m, Log: logr.Discard()} + + assert.True(t, m.snapshotsSettled(gitDest, nil), "a GitTarget with no collection has nothing to settle") + assert.True(t, router.initialSnapshot(gitDest, configmaps)) + + drain := func(collection types.CollectionKey, result git.ResyncResult) { + resultCh := make(chan git.ResyncResult, 1) + resultCh <- result + router.drainScopedResync(gitDest, collection, "reconcile", 0, resultCh) + } + drain(configmaps, git.ResyncResult{Err: git.ErrResyncDiscarded}) + assert.True(t, router.initialSnapshot(gitDest, configmaps), "a discarded snapshot did not settle") + drain(configmaps, git.ResyncResult{Err: git.ErrResyncSuperseded}) + assert.True(t, router.initialSnapshot(gitDest, configmaps), "nor did a superseded one") + + drain(configmaps, git.ResyncResult{}) + assert.False(t, router.initialSnapshot(gitDest, configmaps)) + assert.False(t, m.snapshotsSettled(gitDest, []types.CollectionKey{configmaps, secrets}), "secrets has none") + assert.True(t, router.initialSnapshot(gitDest, secrets)) + + drain(secrets, git.ResyncResult{Err: errors.New("push failed")}) + assert.True(t, m.snapshotsSettled(gitDest, []types.CollectionKey{configmaps, secrets})) + + m.forgetSettledSnapshots(gitDest) + assert.True(t, router.initialSnapshot(gitDest, configmaps), "a recreation starts over") +} + +// TestOwnerIsReadBeforeSanitize: the applier that owns an object is stamped on its live event and on +// its snapshot entry from the raw object, before sanitize strips the labels and annotations it is read +// from (docs/design/applier-echoes.md, F4). +func TestOwnerIsReadBeforeSanitize(t *testing.T) { + configmaps := schema.GroupVersionResource{Version: "v1", Resource: "configmaps"} + u := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", "kind": "ConfigMap", + "metadata": map[string]any{ + "name": "cfg", "namespace": "team-a", + "labels": map[string]any{ + "kustomize.toolkit.fluxcd.io/name": "infra", "kustomize.toolkit.fluxcd.io/namespace": "flux-system", + }, + }, + }} + want := git.AppliedBy{Kind: "Kustomization", Namespace: "flux-system", Name: "infra"} + + event := targetWatchGitEvent(configmaps, u, string(types.OperationUpdate)) + assert.Equal(t, want, event.AppliedBy) + + var snapshot replaySnapshot + snapshot.add(configmaps, u) + require.Len(t, snapshot.desired, 1) + assert.Equal(t, want, snapshot.owners[snapshot.desired[0].Resource]) +} diff --git a/internal/watch/desired_state_change_filter.go b/internal/watch/desired_state_change_filter.go index 66609773..68f3b190 100644 --- a/internal/watch/desired_state_change_filter.go +++ b/internal/watch/desired_state_change_filter.go @@ -119,6 +119,12 @@ func desiredStateHash(obj *unstructured.Unstructured) (string, bool) { type replaySnapshot struct { desired []manifestanalyzer.DesiredResource baselines map[k8stypes.UID]string + // start is the branch worker's ApplyState read when the collection started, nil when it could + // not be read. The worker discards a snapshot an apply made stale (git.ResyncRequest). + start *git.ApplyState + // owners is the applier that owns each object some applier owns, read from the raw object + // before sanitize strips it (git.AppliedByOf). + owners map[types.ResourceIdentifier]git.AppliedBy } // add folds one observed object into the snapshot. An object desiredFromObject excludes (a @@ -129,6 +135,12 @@ func (r *replaySnapshot) add(gvr schema.GroupVersionResource, u *unstructured.Un return } r.desired = append(r.desired, desired) + if owner := git.AppliedByOf(u); !owner.IsZero() { + if r.owners == nil { + r.owners = map[types.ResourceIdentifier]git.AppliedBy{} + } + r.owners[desired.Resource] = owner + } if r.baselines == nil { r.baselines = map[k8stypes.UID]string{} } diff --git a/internal/watch/event_router.go b/internal/watch/event_router.go index 33549f6f..d1f078f6 100644 --- a/internal/watch/event_router.go +++ b/internal/watch/event_router.go @@ -262,8 +262,7 @@ func (r *EventRouter) resolveWorkerForGitDest( // the buffered reply channel and whether the resync actually entered the FIFO. The scope // restricts the worker's mark-and-sweep to one type, and to one namespace when it names one, so // desired MUST carry exactly that scope's objects (empty for a sweep) — passing a scope wider -// than the gather deletes managed documents outside it. heal marks a drift-correcting resync the -// worker defers while a commit window is open. enqueued is false when the worker's queue was full +// than the gather deletes managed documents outside it. enqueued is false when the worker's queue was full // and dropped the request (its failure is still delivered on resultCh for the drain to record). // resourceVersion is the LIST's collection version the desired set is pinned to. func (r *EventRouter) enqueueScopedResync( @@ -271,9 +270,8 @@ func (r *EventRouter) enqueueScopedResync( gitDest types.ResourceReference, scope git.ResyncScope, sourceCollection types.CollectionKey, - desired []manifestanalyzer.DesiredResource, + snapshot replaySnapshot, resourceVersion string, - heal bool, renderFidelityRevision uint64, ) (chan git.ResyncResult, bool, error) { worker, err := r.resyncTarget(ctx, gitDest) @@ -282,21 +280,50 @@ func (r *EventRouter) enqueueScopedResync( } resultCh := make(chan git.ResyncResult, 1) enqueued := worker.EnqueueResync(&git.ResyncRequest{ - Desired: desired, + Desired: snapshot.desired, + Owners: snapshot.owners, ResourceVersion: resourceVersion, GitTargetName: gitDest.Name, GitTargetNamespace: gitDest.Namespace, Scope: &scope, SourceCollection: sourceCollection, - Heal: heal, // The worker records the scope clean itself once the resync applies, before the drain // below hears of it, so a live event queued behind the resync meets the gate it produced. RenderFidelityRevision: renderFidelityRevision, + CollectionStart: snapshot.start, + Initial: r.initialSnapshot(gitDest, scope.Collection), Result: resultCh, }) return resultCh, enqueued, nil } +// initialSnapshot reports whether no snapshot of the collection has settled yet: the worker then +// uses it whatever its appliers are doing, since it is what an apply waits for (git.ResyncRequest.Initial). +func (r *EventRouter) initialSnapshot(gitDest types.ResourceReference, collection types.CollectionKey) bool { + return r.WatchManager != nil && !r.WatchManager.snapshotsSettled(gitDest, []types.CollectionKey{collection}) +} + +// applyStateReader is the part of a branch worker that says where its appliers stand. See +// git.BranchWorker.ApplyState. +type applyStateReader interface { + ApplyState() git.ApplyState +} + +// applyState reads the ApplyState of a GitTarget's branch worker, or nil when the worker cannot be +// resolved or reports none. +func (r *EventRouter) applyState(ctx context.Context, gitDest types.ResourceReference) *git.ApplyState { + worker, err := r.resyncTarget(ctx, gitDest) + if err != nil { + return nil + } + reader, ok := worker.(applyStateReader) + if !ok { + return nil + } + state := reader.ApplyState() + return &state +} + // resyncTarget is the branch worker a GitTarget's scoped resyncs enter. func (r *EventRouter) resyncTarget(ctx context.Context, gitDest types.ResourceReference) (resyncEnqueuer, error) { if r.resyncWorker != nil { @@ -349,6 +376,9 @@ func (r *EventRouter) drainScopedResync( ) { select { case result := <-resultCh: + if settlesSnapshot(result.Err) && r.WatchManager != nil { + r.WatchManager.markSnapshotSettled(gitDest, collection) + } if result.Err != nil { r.handleScopedResyncError(gitDest, collection, kind, renderFidelityEpoch, result.Err) return @@ -377,11 +407,21 @@ func (r *EventRouter) drainScopedResync( ) } case <-time.After(resyncSignalTimeout): + if r.WatchManager != nil { + r.WatchManager.markSnapshotSettled(gitDest, collection) + } r.Log.Error(nil, "per-type "+kind+" timed out", "gitDest", gitDest.String(), "collection", collection.String()) r.recordBackgroundResyncFailure(gitDest) } } +// settlesSnapshot reports whether a resync's outcome settles its collection's snapshot: it was +// applied, refused or failed. A discarded one is asked for again, and a superseded one is replaced by +// the resync queued in its place; neither settles anything (docs/design/applier-echoes.md, S2). +func settlesSnapshot(err error) bool { + return !errors.Is(err, git.ErrResyncDiscarded) && !errors.Is(err, git.ErrResyncSuperseded) +} + // handleScopedResyncError classifies a failed resync. A path the acceptance gate refused // is not a transient write fault: nothing was committed, the human must clean the Git path, so // it is surfaced as target-level GitPathAccepted=False and is NOT counted as a background @@ -413,6 +453,14 @@ func (r *EventRouter) handleScopedResyncError( } return } + if errors.Is(err, git.ErrResyncDiscarded) { + // An apply made the snapshot stale before the worker could use it. Nothing failed: the + // worker asks for a fresh snapshot of the GitTarget once the apply is done, and that one + // reports. See docs/design/applier-echoes.md, F3. + r.Log.Info("per-type "+kind+" discarded as stale; a fresh snapshot is asked for", + "gitDest", gitDest.String(), "collection", collection.String(), "detail", err.Error()) + return + } if errors.Is(err, git.ErrResyncSuperseded) { // A newer resync for the same scope replaced this one while it was queued and runs in its // place, so no WRITE was missed. Its REPORTS are skipped, on the reasoning that the diff --git a/internal/watch/event_router_test.go b/internal/watch/event_router_test.go index 1cdd0418..5713829f 100644 --- a/internal/watch/event_router_test.go +++ b/internal/watch/event_router_test.go @@ -104,9 +104,8 @@ func TestEnqueueScopedResync_ReportsMissingWorker(t *testing.T) { types.NewResourceReference("team-a-config", "team-a"), git.ResyncScopeFor(configmapsGVR, ""), types.CollectionKey{}, - nil, + replaySnapshot{}, "12", - false, 0, ) @@ -138,9 +137,8 @@ func TestEnqueueScopedResync_ReportsGoneGitTargetAsTerminal(t *testing.T) { types.NewResourceReference("deleted-target", "team-a"), git.ResyncScopeFor(configmapsGVR, ""), types.CollectionKey{}, - nil, + replaySnapshot{}, "12", - false, 0, ) diff --git a/internal/watch/manager.go b/internal/watch/manager.go index cf0f51e2..f3174529 100644 --- a/internal/watch/manager.go +++ b/internal/watch/manager.go @@ -110,6 +110,11 @@ type Manager struct { // plane, which needs no provider, still works. SourceClusters SourceClusterResolver + // snapshotsMu guards settledSnapshots: per GitTarget key, the collections a snapshot of which + // the branch worker applied, refused or failed (apply_state.go). + snapshotsMu sync.Mutex + settledSnapshots map[string]map[types.CollectionKey]bool + // clusters holds one clusterContext per distinct cluster — its API catalog, type registry, // and clients. configPlaneClusterID is the operator's own cluster (always present, never a // source); every other key is a ClusterProvider name. See cluster_context.go. diff --git a/internal/watch/materialization.go b/internal/watch/materialization.go index caa4c17f..9564c717 100644 --- a/internal/watch/materialization.go +++ b/internal/watch/materialization.go @@ -102,6 +102,7 @@ func (m *Manager) tearDownGitTarget(gitDest types.ResourceReference) { m.forgetGitTargetUID(gitDest) m.forgetGitTargetCluster(gitDest) m.forgetGitTargetPruneMode(gitDest) + m.forgetSettledSnapshots(gitDest) m.mutateWatchPlane(func(s *watchPlaneState) bool { _, hadPass := s.passes[gitDest.Key()] _, hadLayout := s.layouts[gitDest.Key()] diff --git a/internal/watch/target_watch.go b/internal/watch/target_watch.go index b846e287..61e3b1df 100644 --- a/internal/watch/target_watch.go +++ b/internal/watch/target_watch.go @@ -742,6 +742,9 @@ func (m *Manager) targetWatchReplayAndStream( ) replaying := true replayStarted := time.Now() + // Read before the collection starts: the worker judges the snapshot by whether an apply could + // have changed what it saw since (docs/design/applier-echoes.md, F3). + start := m.applyStateAtCollectionStart(ctx, gitDest) w, err := m.openTargetWatch(ctx, m.clusterIDForGitTarget(gitDest), stream.key, opts) if err != nil { if watchListUnsupported(err) { @@ -768,7 +771,7 @@ func (m *Manager) targetWatchReplayAndStream( defer release() defer w.Stop() - return m.pumpTargetWatchSession(ctx, log, gitDest, stream, w.ResultChan(), replaying, replayStarted) + return m.pumpTargetWatchSession(ctx, log, gitDest, stream, w.ResultChan(), replaying, replayStarted, start) } // pumpTargetWatchSession drains one open session, folding replay events and then streaming live @@ -785,8 +788,9 @@ func (m *Manager) pumpTargetWatchSession( events <-chan watch.Event, replaying bool, replayStarted time.Time, + start *git.ApplyState, ) error { - var replay replaySnapshot + replay := replaySnapshot{start: start} for { select { case <-ctx.Done(): @@ -897,6 +901,7 @@ func (m *Manager) targetWatchListAndStream( buffered := make(chan watch.Event, targetWatchBufferCapacity) go bufferTargetWatchEvents(ctx, w.ResultChan(), buffered) + start := m.applyStateAtCollectionStart(ctx, gitDest) list, err := m.openTargetList(ctx, clusterID, stream.key, metav1.ListOptions{}) if err != nil { if ctx.Err() != nil { @@ -912,6 +917,7 @@ func (m *Manager) targetWatchListAndStream( return fmt.Errorf("list target watch snapshot %s/%q: %w", stream.key.GVR.String(), stream.key.Namespace, err) } snapshot := snapshotFromList(stream.key.GVR, list) + snapshot.start = start resourceVersion := list.GetResourceVersion() if err := m.enqueueReplayResync(ctx, log, gitDest, stream, snapshot, resourceVersion); err != nil { return err @@ -965,7 +971,7 @@ func (m *Manager) handleTargetWatchSessionEvent( return true, err } stream.markReplayed() - *replay = replaySnapshot{} + *replay = replaySnapshot{start: replay.start} m.markTargetStreamState( gitDest, stream.key.Collection(), @@ -1037,8 +1043,8 @@ func (m *Manager) enqueueReplayResync( // A retired stream's snapshot is dropped here, at the producer: see producerGate. if !stream.gate.enqueue(ctx, func() { resultCh, enqueued, err = m.EventRouter.enqueueScopedResync( - ctx, gitDest, resyncScopeForWatchKey(stream.key), stream.sourceCollection(), snapshot.desired, - resourceVersion, false, stream.revision) + ctx, gitDest, resyncScopeForWatchKey(stream.key), stream.sourceCollection(), snapshot, + resourceVersion, stream.revision) }) { return nil } @@ -1330,6 +1336,7 @@ func targetWatchGitEvent(gvr schema.GroupVersionResource, u *unstructured.Unstru } if op != string(types.OperationDelete) { event.LastFieldManagers, event.LastFieldManagedAt = lastFieldManagers(u) + event.AppliedBy = git.AppliedByOf(u) event.Object = sanitize.Sanitize(u) } return event diff --git a/test/e2e/Taskfile-e2e.yml b/test/e2e/Taskfile-e2e.yml index d44e97f3..32488b6d 100644 --- a/test/e2e/Taskfile-e2e.yml +++ b/test/e2e/Taskfile-e2e.yml @@ -321,8 +321,11 @@ tasks: # IceCreamOrder CRD groups), and none is Serial, so they run side by side. # The provenance spec drives no GitTarget and never patches argocd-secret, so # it shares the argocd namespace with the Argo CD spec without racing it. - BI_DIRECTIONAL_GINKGO_PROCS: '{{.BI_DIRECTIONAL_GINKGO_PROCS | default "3"}}' - BI_DIRECTIONAL_TIMEOUT: '{{.BI_DIRECTIONAL_TIMEOUT | default "25m"}}' + # The applier-echo scenarios (bi_directional_echoes_e2e_test.go) are two more long Ordered + # specs, and the restart ones run Serial at the end: four procs keep the leg inside the CI + # job's 45 minutes. + BI_DIRECTIONAL_GINKGO_PROCS: '{{.BI_DIRECTIONAL_GINKGO_PROCS | default "4"}}' + BI_DIRECTIONAL_TIMEOUT: '{{.BI_DIRECTIONAL_TIMEOUT | default "35m"}}' cmds: # Sequential, not deps: _prepare-e2e-ready restarts the k3d server node when # it injects the webhook TLS cert, which would break an Argo CD install diff --git a/test/e2e/apply_trigger.go b/test/e2e/apply_trigger.go index 0a0780e0..21374a59 100644 --- a/test/e2e/apply_trigger.go +++ b/test/e2e/apply_trigger.go @@ -19,14 +19,9 @@ const ( ) // createFluxTrigger creates the FluxTrigger name in namespace, holding the Kustomization -// flux-system/ in the operator's own cluster (the "default" ClusterProvider). An -// empty quietPeriod takes the default. -func createFluxTrigger(namespace, name, kustomization, quietPeriod string) { +// flux-system/ in the operator's own cluster (the "default" ClusterProvider). +func createFluxTrigger(namespace, name, kustomization string) { GinkgoHelper() - quiet := "" - if quietPeriod != "" { - quiet = fmt.Sprintf("\n quietPeriod: %q", quietPeriod) - } manifest := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 kind: FluxTrigger metadata: @@ -37,8 +32,8 @@ spec: name: default kustomizationRef: namespace: flux-system - name: %s%s -`, name, namespace, kustomization, quiet) + name: %s +`, name, namespace, kustomization) _, err := kubectlRunWithStdin(namespace, manifest, "apply", "-f", "-") Expect(err).NotTo(HaveOccurred(), "failed to create FluxTrigger %s/%s", namespace, name) } diff --git a/test/e2e/argocd_bi_directional_e2e_test.go b/test/e2e/argocd_bi_directional_e2e_test.go index 1fb45dc9..b08bcebc 100644 --- a/test/e2e/argocd_bi_directional_e2e_test.go +++ b/test/e2e/argocd_bi_directional_e2e_test.go @@ -51,7 +51,7 @@ const ( argoSyncTimeout = 120 * time.Second // A Git-side push reaches GitOps Reverser's receiver, which decides a - // ForeignCommit on a quiet branch, and Argo CD then syncs: seconds, far below the + // ForeignCommit after a flush, and Argo CD then syncs: seconds, far below the // 10m refresher, so this can only pass through the receiver. argoRequestedSyncTimeout = 90 * time.Second // Long enough for a sync GitOps Reverser should not have asked for to show up. @@ -164,9 +164,10 @@ var _ = Describe("Bi Directional (Argo CD)", Label("bi-directional", "argocd"), }) // A Git-side push reaches the cluster only through GitOps Reverser: its receiver - // hears of the push, the worker decides a ForeignCommit on a quiet branch, and the + // hears of the push, the worker decides a ForeignCommit after a flush, and the // sync it starts is pinned to that commit. Nothing GitOps Reverser pushes itself - // is synced. And what Argo CD stamps on the objects it applies stays out of Git. + // is synced; naming the Application converges it once. And what Argo CD stamps on + // the objects it applies stays out of Git. It("syncs a Git-side push through a pinned ForeignCommit, and keeps Argo CD's bookkeeping out of Git", func() { // SetupRepo leaves the Gitea repo EMPTY, so seed the branch first: the // GitTarget needs a main to bootstrap. Argo CD has nothing to sync until the @@ -178,11 +179,19 @@ var _ = Describe("Bi Directional (Argo CD)", Label("bi-directional", "argocd"), By("enabling gitops-reverser on the shared live path, naming the Application as the applier") receiverHookID = run.startReverserPipeline() + By("naming the Application converges it once, after the folder's initial snapshot is in Git") + Eventually(func(g Gomega) { + operation := run.triggerOperation(g) + g.Expect(operation.Reason).To(Equal("CatchUp")) + g.Expect(operation.Phase).To(Equal("Succeeded")) + }, argoRequestedSyncTimeout, biPollInterval).Should(Succeed()) + converged := run.triggerOperation(Default) + // Absorbs the reverser's bootstrap commit, so the counts below measure the // Git-side push and Argo CD's sync, and nothing else. baselineCommitCount := run.waitForStableRemoteCommitCount(biStableCountShortWait) - Expect(run.operationFinishedAt()).To(BeEmpty(), - "GitOps Reverser's own pushes and its start ask Argo CD for nothing") + Expect(run.triggerOperation(Default)).To(Equal(converged), + "GitOps Reverser's own pushes ask Argo CD for nothing more") By("pushing a clean IceCreamOrder from the Git side, and nothing else") run.writeLiveOrder(iceCreamOrderFile{ @@ -231,9 +240,9 @@ var _ = Describe("Bi Directional (Argo CD)", Label("bi-directional", "argocd"), // The race the apply trigger exists for (docs/design/apply-trigger.md): two API // edits to one field a few seconds apart. With automated sync, Argo CD synced the // first edit's commit over the second edit. Under the standard nothing syncs - // GitOps Reverser's own commits at all, so the later value survives, Git ends on - // it with no further commit, and Argo CD, refreshed by its webhook on each push, - // only reports the Application Synced. + // GitOps Reverser's own commits at all (an API edit asks for no apply), so the + // later value survives, Git ends on it with no further commit, and Argo CD, + // refreshed by its webhook on each push, only reports the Application Synced. It("keeps the later of two quick API edits, with no further commit and no sync", func() { By("committing a third IceCreamOrder from the Git side, synced by a ForeignCommit") run.writeLiveOrder(iceCreamOrderFile{ @@ -265,7 +274,7 @@ var _ = Describe("Bi Directional (Argo CD)", Label("bi-directional", "argocd"), g.Expect(count).To(Equal(baseline+2), "a sync of the first edit's commit over the second edit would show as more commits") g.Expect(run.operationFinishedAt()).To(Equal(lastSync), - "GitOps Reverser's own commits ask Argo CD for no sync: there is no catch-up") + "GitOps Reverser's own commits ask Argo CD for no sync: an API edit asks for no apply") }, 30*time.Second, 2*time.Second).Should(Succeed()) By("and Argo CD, which only compared, finds Git and the cluster in agreement") diff --git a/test/e2e/bi_directional_echoes_e2e_test.go b/test/e2e/bi_directional_echoes_e2e_test.go new file mode 100644 index 00000000..e3a2c443 --- /dev/null +++ b/test/e2e/bi_directional_echoes_e2e_test.go @@ -0,0 +1,507 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The applier echoes, end to end (docs/design/applier-echoes.md, "New end-to-end scenarios"). Each +// scenario runs for Flux and for Argo CD on a folder GitOps Reverser mirrors and the applier +// applies: the timing pipeline's shared folder, where a WatchRule mirrors the labeled ConfigMaps. +// The push race (E1) is bi_directional_push_race_e2e_test.go, the restart (E9) is +// bi_directional_restart_e2e_test.go, and the timing (E13) is bi_directional_timing_e2e_test.go. + +const ( + // echoApplyBudget is how long an apply GitOps Reverser asks for may take to show on the cluster. + echoApplyBudget = 2 * time.Minute + // echoHold is how long a state must hold to count as one nothing reverts. + echoHold = 20 * time.Second +) + +// echoApplier is what a scenario needs to know about the applier of the folder. +type echoApplier struct { + name string + // owns reports whether the applier owns the live ConfigMap: Flux's owner labels name its + // Kustomization, or Argo CD's tracking annotation names its Application. + owns func(g Gomega, testNs, configMap string) bool + // operation is the trigger's status.operation. + operation func(g Gomega) triggerOperation + // startOther starts another applier applying folder, which is not the trigger's, and returns its + // cleanup. + startOther func(folder string) func() +} + +// fluxEchoApplier is the FluxTrigger fluxName of testNs, holding the Kustomization of that name. +func fluxEchoApplier(testNs, fluxName string, repo *RepoArtifacts) echoApplier { + return echoApplier{ + name: "Flux", + owns: func(g Gomega, ns, configMap string) bool { + out, err := kubectlRunInNamespace(ns, "get", "configmap", configMap, + `-o=jsonpath={.metadata.labels.kustomize\.toolkit\.fluxcd\.io/name}`) + g.Expect(err).NotTo(HaveOccurred()) + return strings.TrimSpace(out) == fluxName + }, + operation: func(g Gomega) triggerOperation { + return readTriggerOperation(g, testNs, fluxTriggerResource, fluxName) + }, + startOther: func(folder string) func() { + other := "other-" + fluxName + applyFluxApplierWithPrune(testNs, repo, other, other+"-git", "main", "./"+folder, true) + return func() { + cleanupNamespacedResource("flux-system", "kustomization", other) + cleanupNamespacedResource("flux-system", "gitrepository", other) + cleanupNamespacedResource("flux-system", "secret", other+"-git") + } + }, + } +} + +// argoEchoApplier is the ArgoCDTrigger of run, holding its Application. +func argoEchoApplier(run argoBiDirectionalRun) echoApplier { + return echoApplier{ + name: "Argo CD", + owns: func(g Gomega, ns, configMap string) bool { + out, err := kubectlRunInNamespace(ns, "get", "configmap", configMap, + `-o=jsonpath={.metadata.annotations.argocd\.argoproj\.io/tracking-id}`) + g.Expect(err).NotTo(HaveOccurred()) + return strings.HasPrefix(strings.TrimSpace(out), run.appName+":") + }, + operation: run.triggerOperation, + startOther: func(folder string) func() { + other := run.baseAppConfig() + other.Name = "other-" + run.appName + other.Path = folder + other.Automated = true // not a trigger's: it syncs on its own + run.applyArgoApp(other) + return func() { cleanupNamespacedResource(run.argoNs, "application.argoproj.io", other.Name) } + }, + } +} + +// echoRun is one GitTarget mirroring the labeled ConfigMaps of testNs into the folder an applier +// applies. +type echoRun struct { + *timingPipeline + + applier echoApplier +} + +// cmFile is where the GitTarget writes the ConfigMap name. +func (r *echoRun) cmFile(name string) string { + return fmt.Sprintf("%s/%s/configmaps/%s.yaml", r.gitPath, r.testNs, name) +} + +// echoConfigMap is a labeled ConfigMap the WatchRule mirrors. +func echoConfigMap(namespace, name, value string) string { + return fmt.Sprintf(`apiVersion: v1 +kind: ConfigMap +metadata: + labels: + %s: "true" + name: %s + namespace: %s +data: + value: %s +`, timingLabelKey, name, namespace, value) +} + +// createThroughAPI creates the ConfigMap name holding value through the Kubernetes API. +func (r *echoRun) createThroughAPI(name, value string) { + GinkgoHelper() + _, err := kubectlRunWithStdin(r.testNs, echoConfigMap(r.testNs, name, value), "create", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "failed to create ConfigMap %s through the API", name) +} + +// editThroughAPI sets data.value of the ConfigMap name through the Kubernetes API. +func (r *echoRun) editThroughAPI(name, value string) { + GinkgoHelper() + _, err := kubectlRunInNamespace(r.testNs, "patch", "configmap", name, "--type=merge", "-p", + fmt.Sprintf(`{"data":{"value":%q}}`, value)) + Expect(err).NotTo(HaveOccurred(), "failed to edit ConfigMap %s through the API", name) +} + +// committedValue is data.value of the ConfigMap name as main holds it, and whether main holds it. +func (r *echoRun) committedValue(g Gomega, name string) (string, bool) { + _, err := gitRun(r.repo.CheckoutDir, "fetch", "origin", "main") + g.Expect(err).NotTo(HaveOccurred()) + out, err := gitRun(r.repo.CheckoutDir, "show", "origin/main:"+r.cmFile(name)) + if err != nil { + return "", false + } + match := timingValueLine.FindStringSubmatch(out) + g.Expect(match).NotTo(BeNil(), "the committed ConfigMap %s has no data.value", name) + return strings.TrimSpace(strings.TrimPrefix(match[0], match[1])), true +} + +// liveValue is data.value of the live ConfigMap name, and whether it exists. +func (r *echoRun) liveValue(g Gomega, name string) (string, bool) { + out, err := kubectlRunInNamespace(r.testNs, "get", "configmap", name, "--ignore-not-found", + "-o", "jsonpath={.metadata.name}|{.data.value}") + g.Expect(err).NotTo(HaveOccurred()) + gotName, value, _ := strings.Cut(strings.TrimSpace(out), "|") + return value, gotName != "" +} + +// pushFromOutside writes the ConfigMap name holding value into the folder and pushes it the way +// another writer would. +func (r *echoRun) pushFromOutside(name, value string) { + GinkgoHelper() + pushFileFromOutside(r.repo, r.cmFile(name), echoConfigMap(r.testNs, name, value), + fmt.Sprintf("e2e: a Git-side change sets %s to %s", name, value)) +} + +// deleteFromOutside deletes the ConfigMap name's file and pushes the delete the way another writer +// would. +func (r *echoRun) deleteFromOutside(name string) { + GinkgoHelper() + removeFileFromOutside(r.repo, r.cmFile(name), "e2e: a Git-side delete of "+name) +} + +// forceResync asks the GitTarget to re-anchor its streams, which replays each and resyncs its folder. +func (r *echoRun) forceResync() { + GinkgoHelper() + requestGitTargetReconcile(r.targetName, r.testNs) +} + +// eventuallyEverywhere waits until main and the cluster both hold value for the ConfigMap name. +func (r *echoRun) eventuallyEverywhere(name, value string, budget time.Duration) { + GinkgoHelper() + Eventually(func(g Gomega) { + committed, inGit := r.committedValue(g, name) + g.Expect(inGit).To(BeTrue(), "%s is not in Git", name) + g.Expect(committed).To(Equal(value), "Git") + live, exists := r.liveValue(g, name) + g.Expect(exists).To(BeTrue(), "%s is not on the cluster", name) + g.Expect(live).To(Equal(value), "the cluster") + }, budget, time.Second).Should(Succeed()) +} + +// consistentlyEverywhere holds that main and the cluster both keep value for the ConfigMap name. +func (r *echoRun) consistentlyEverywhere(name, value string) { + GinkgoHelper() + Consistently(func(g Gomega) { + committed, _ := r.committedValue(g, name) + g.Expect(committed).To(Equal(value), "Git must keep %s", value) + live, _ := r.liveValue(g, name) + g.Expect(live).To(Equal(value), "the cluster must keep %s", value) + }, echoHold, 2*time.Second).Should(Succeed()) +} + +// eventuallyAdopted waits until the applier owns the ConfigMap name, after an operation that +// succeeded. +func (r *echoRun) eventuallyAdopted(name string) { + GinkgoHelper() + Eventually(func(g Gomega) { + g.Expect(r.applier.owns(g, r.testNs, name)).To(BeTrue(), "the applier must own %s", name) + g.Expect(r.applier.operation(g).Phase).To(Equal("Succeeded")) + }, echoApplyBudget, 2*time.Second).Should(Succeed()) +} + +// eventuallyPruned waits until the ConfigMap name is gone from the cluster, and holds that it stays +// gone from Git. +func (r *echoRun) eventuallyPruned(name string) { + GinkgoHelper() + Eventually(func(g Gomega) { + _, exists := r.liveValue(g, name) + g.Expect(exists).To(BeFalse(), "the applier must prune %s, which it owns", name) + }, echoApplyBudget, 2*time.Second).Should(Succeed()) + Consistently(func(g Gomega) { + _, inGit := r.committedValue(g, name) + g.Expect(inGit).To(BeFalse(), "a resync must not write the pruned %s back", name) + }, echoHold, 2*time.Second).Should(Succeed()) +} + +// echoScenarios are the scenarios that run the same for both appliers, against run. +func echoScenarios(run func() *echoRun) { + It("adopts an object the API created, so a Git-side delete prunes it (E2)", func() { + r := run() + r.createThroughAPI("adopt-created", "c1") + r.eventuallyEverywhere("adopt-created", "c1", timingCommitBudget) + r.eventuallyAdopted("adopt-created") + + r.deleteFromOutside("adopt-created") + r.eventuallyPruned("adopt-created") + }) + + It("adopts an object deleted and created again with the same content (E3)", func() { + r := run() + r.pushFromOutside("recreated", "r1") + r.eventuallyEverywhere("recreated", "r1", echoApplyBudget) + r.eventuallyAdopted("recreated") + + By("deleting it through the API, and creating it again with identical content") + _, err := kubectlRunInNamespace(r.testNs, "delete", "configmap", "recreated", "--wait=true") + Expect(err).NotTo(HaveOccurred()) + r.createThroughAPI("recreated", "r1") + r.eventuallyEverywhere("recreated", "r1", echoApplyBudget) + r.eventuallyAdopted("recreated") + + r.deleteFromOutside("recreated") + r.eventuallyPruned("recreated") + }) + + It("keeps an edit made during an apply, in Git at once and on the cluster after the Repair (E5)", func() { + r := run() + r.pushFromOutside("during", "d0") + r.eventuallyEverywhere("during", "d0", echoApplyBudget) + + By("pushing a Git-side change to another object, and editing this one while that apply runs") + before := r.applier.operation(Default).Token + r.pushFromOutside("during-other", "o1") + Eventually(func(g Gomega) { + g.Expect(r.applier.operation(g).Token).NotTo(Equal(before), "the apply is asked for") + }, echoApplyBudget, 250*time.Millisecond).Should(Succeed()) + r.editThroughAPI("during", "d-edit") + + r.eventuallyEverywhere("during", "d-edit", echoApplyBudget) + r.eventuallyEverywhere("during-other", "o1", echoApplyBudget) + r.consistentlyEverywhere("during", "d-edit") + }) + + It("applies a Git-side push while edits keep arriving, and keeps every edit (E6)", func() { + r := run() + r.pushFromOutside("busy", "b0") + r.eventuallyEverywhere("busy", "b0", echoApplyBudget) + + By("editing an object every second while a Git-side push to another one waits") + stop := make(chan struct{}) + var wg sync.WaitGroup + last := "b0" + wg.Go(func() { + defer GinkgoRecover() + for i := 0; ; i++ { + select { + case <-stop: + return + case <-time.After(time.Second): + } + value := fmt.Sprintf("busy-%d", i) + r.editThroughAPI("busy", value) + last = value // read only after wg.Wait + } + }) + r.pushFromOutside("busy-push", "p1") + Eventually(func(g Gomega) { + live, _ := r.liveValue(g, "busy-push") + g.Expect(live).To(Equal("p1"), "the push is applied while edits keep arriving") + }, timingReceiverBudget, time.Second).Should(Succeed()) + close(stop) + wg.Wait() + + r.eventuallyEverywhere("busy", last, echoApplyBudget) + r.consistentlyEverywhere("busy", last) + }) + + It("keeps a Git-side push when a resync is forced before its apply finishes (E7)", func() { + r := run() + r.pushFromOutside("ahead", "a0") + r.eventuallyEverywhere("ahead", "a0", echoApplyBudget) + + r.pushFromOutside("ahead", "a1") + r.forceResync() + r.eventuallyEverywhere("ahead", "a1", echoApplyBudget) + r.consistentlyEverywhere("ahead", "a1") + }) + + It("keeps a Git-side addition across a resync under prune.mode Always (E8)", func() { + r := run() + _, err := kubectlRunInNamespace(r.testNs, "patch", "gittarget", r.targetName, "--type=merge", "-p", + `{"spec":{"prune":{"mode":"Always"}}}`) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + _, _ = kubectlRunInNamespace(r.testNs, "patch", "gittarget", r.targetName, "--type=merge", "-p", + `{"spec":{"prune":null}}`) + }) + + r.pushFromOutside("added-in-git", "g1") + r.forceResync() + r.eventuallyEverywhere("added-in-git", "g1", echoApplyBudget) + r.consistentlyEverywhere("added-in-git", "g1") + }) + + It("asks for no apply after an API edit, and keeps the later of two quick edits (E10)", func() { + r := run() + r.pushFromOutside("quick", "q0") + r.eventuallyEverywhere("quick", "q0", echoApplyBudget) + r.eventuallyAdopted("quick") + before := r.applier.operation(Default) + + r.editThroughAPI("quick", "q1") + r.editThroughAPI("quick", "q2") + r.eventuallyEverywhere("quick", "q2", timingCommitBudget) + Consistently(func(g Gomega) { + g.Expect(r.applier.operation(g).Token).To(Equal(before.Token), "an API edit asks for no apply") + live, _ := r.liveValue(g, "quick") + g.Expect(live).To(Equal("q2")) + }, echoHold, 2*time.Second).Should(Succeed()) + }) + + It("does not mirror or adopt an object another applier owns (E11)", func() { + r := run() + folder := "other/" + r.testNs + head := pushFileFromOutside(r.repo, folder+"/owned-elsewhere.yaml", + echoConfigMap(r.testNs, "owned-elsewhere", "e1"), "e2e: another applier's folder") + // The push is a foreign commit to the trigger's repository too, so it is applied first; what + // must not follow is an apply of the object the other applier creates. + var before triggerOperation + Eventually(func(g Gomega) { + before = r.applier.operation(g) + g.Expect(before.Commit).To(Equal(head), "the push is applied") + g.Expect(before.Phase).To(Equal("Succeeded")) + }, echoApplyBudget, 2*time.Second).Should(Succeed()) + stopOther := r.applier.startOther(folder) + DeferCleanup(stopOther) + + Eventually(func(g Gomega) { + live, exists := r.liveValue(g, "owned-elsewhere") + g.Expect(exists).To(BeTrue(), "the other applier applies it") + g.Expect(live).To(Equal("e1")) + }, echoApplyBudget, 2*time.Second).Should(Succeed()) + Consistently(func(g Gomega) { + _, inGit := r.committedValue(g, "owned-elsewhere") + g.Expect(inGit).To(BeFalse(), "it is not mirrored into the triggered folder") + g.Expect(r.applier.operation(g).Token).To(Equal(before.Token), "nor adopted") + }, echoHold, 2*time.Second).Should(Succeed()) + }) + + It("resolves a save made across a Git-side change from the push (E12)", func() { + r := run() + r.pushFromOutside("saved", "s0") + r.eventuallyEverywhere("saved", "s0", echoApplyBudget) + + By("saving an API edit with a CommitRequest while a Git-side push changes the same object") + name := applyCommitRequestWithGenerateName(r.testNs, "echo-save-", r.targetName, "e2e: save the edit") + r.editThroughAPI("saved", "s-save") + r.pushFromOutside("saved", "s-git") + + Eventually(func(g Gomega) { + g.Expect(commitRequestCondition(g, r.testNs, name, "Ready")).To(Equal("True")) + g.Expect(commitRequestField(g, r.testNs, name, `{.status.conditions[?(@.type=="Ready")].reason}`)). + To(BeElementOf(commitRequestReasonCommitted, "AlreadyPresent")) + }, echoApplyBudget, 2*time.Second).Should(Succeed()) + r.eventuallyEverywhere("saved", "s-save", echoApplyBudget) + r.consistentlyEverywhere("saved", "s-save") + }) +} + +// removeFileFromOutside deletes one file and pushes the delete to main the way another writer +// would. It retries, because GitOps Reverser may move main in between. +func removeFileFromOutside(repo *RepoArtifacts, path, message string) { + GinkgoHelper() + Eventually(func() error { + for _, args := range [][]string{ + {"fetch", "origin", "main"}, {"checkout", "-B", "main", "origin/main"}, {"reset", "--hard", "origin/main"}, + } { + if out, err := gitRun(repo.CheckoutDir, args...); err != nil { + return fmt.Errorf("git %v: %w: %s", args, err, out) + } + } + if _, err := os.Stat(filepath.Join(repo.CheckoutDir, path)); err != nil { + return fmt.Errorf("main holds no %s to delete: %w", path, err) + } + for _, args := range [][]string{{"rm", path}, {"commit", "-m", message}, {"push", "origin", "HEAD:main"}} { + if out, err := gitRun(repo.CheckoutDir, args...); err != nil { + return fmt.Errorf("git %v: %w: %s", args, err, out) + } + } + return nil + }, seedPushTimeout, seedPushInterval).Should(Succeed(), "the outside writer kept losing the push race") +} + +var _ = Describe("Bi Directional applier echoes (Flux)", Label("bi-directional", "flux", "applier-echoes"), Ordered, + func() { + var ( + testNs, fluxName string + r *echoRun + ) + + BeforeAll(func() { + skipUnlessBiDirectionalEnabled() + allowCrossNamespaceAppliers() + testNs = testNamespaceFor("echoes-flux") + _, _ = kubectlRun("create", "namespace", testNs) + fluxName = "echoes-" + testNs + + repo := SetupRepo(resolveE2EContext(), testNs, fmt.Sprintf("e2e-echoes-flux-%d", GinkgoRandomSeed())) + pipeline := newTimingPipeline(testNs, "flux-echoes", repo) + pipeline.start() + + applyFluxApplierWithPrune(testNs, repo, fluxName, fluxName+"-git", "main", "./"+pipeline.gitPath, true) + createFluxTrigger(testNs, fluxName, fluxName) + nameApplyTrigger(testNs, pipeline.targetName, "FluxTrigger", fluxName, "") + verifyResourceCondition("gittarget", pipeline.targetName, testNs, + "ApplyTriggerReady", "True", "Succeeded", "") + verifyResourceCondition(fluxTriggerResource, fluxName, testNs, "Held", "True", "Suspended", "") + + r = &echoRun{timingPipeline: pipeline, applier: fluxEchoApplier(testNs, fluxName, repo)} + }) + + AfterAll(func() { + if !biDirectionalEnabled() || r == nil { + return + } + r.cleanup() + cleanupNamespacedResource(testNs, fluxTriggerResource, fluxName) + cleanupNamespacedResource("flux-system", "kustomization", fluxName) + cleanupNamespacedResource("flux-system", "gitrepository", fluxName) + cleanupNamespacedResource("flux-system", "secret", fluxName+"-git") + cleanupNamespace(testNs) + }) + + echoScenarios(func() *echoRun { return r }) + }) + +var _ = Describe("Bi Directional applier echoes (Argo CD)", Label("bi-directional", "argocd", "applier-echoes"), + Ordered, func() { + var ( + testNs string + r *echoRun + run argoBiDirectionalRun + ) + + BeforeAll(func() { + skipUnlessBiDirectionalEnabled() + requireArgoCDInstalled() + allowCrossNamespaceAppliers() + testNs = testNamespaceFor("echoes-argo") + _, _ = kubectlRun("create", "namespace", testNs) + + repo := SetupRepo(resolveE2EContext(), testNs, fmt.Sprintf("e2e-echoes-argo-%d", GinkgoRandomSeed())) + pipeline := newTimingPipeline(testNs, "argo-echoes", repo) + run = newArgoBiDirectionalRun(testNs, repo, "argo-echoes") + pipeline.start() + + run.applyArgoRepoSecret() + cfg := run.baseAppConfig() + cfg.Path = pipeline.gitPath + run.applyArgoApp(cfg) + createArgoCDTrigger(testNs, run.appName, run.argoNs, run.appName) + nameApplyTrigger(testNs, pipeline.targetName, "ArgoCDTrigger", run.appName, "") + verifyResourceCondition("gittarget", pipeline.targetName, testNs, + "ApplyTriggerReady", "True", "Succeeded", "") + + r = &echoRun{timingPipeline: pipeline, applier: argoEchoApplier(run)} + }) + + AfterAll(func() { + if !biDirectionalEnabled() || r == nil { + return + } + cleanupNamespacedResource(run.argoNs, "application.argoproj.io", run.appName) + cleanupNamespacedResource(run.argoNs, "secret", run.repoSecretName) + cleanupNamespacedResource(testNs, argoCDTriggerResource, run.appName) + r.cleanup() + cleanupNamespace(testNs) + }) + + echoScenarios(func() *echoRun { return r }) + }) diff --git a/test/e2e/bi_directional_push_race_e2e_test.go b/test/e2e/bi_directional_push_race_e2e_test.go new file mode 100644 index 00000000..9527a707 --- /dev/null +++ b/test/e2e/bi_directional_push_race_e2e_test.go @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "fmt" + "strconv" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Two Git-side pushes in quick succession must both reach the cluster, and the later one must stay +// in Git. +// +// The race, found by the Argo CD timing spec in CI: GitOps Reverser asked the applier to apply push +// A, the applier's own write to the object came back through the watch inside the apply window, +// and it was mirrored as a pending write. Push B landed before that write was committed. The write +// was rebuilt on B and said A's value, so the commit it made reverted B, and the applier was then +// asked to apply the revert. The applier's own write in its apply window is not mirrored any more, +// and a write that changes nothing is no action (F1 and F2 of docs/design/applier-echoes.md). +// +// Each round pushes B the moment A is live, which is when A's echo is still pending. Several +// rounds, because the window is seconds wide and a slow round can miss it. +// +// The folder is shared, as in the timing spec: GitOps Reverser mirrors a labeled ConfigMap into +// the folder the applier applies. + +const ( + pushRaceRounds = 3 + // Long enough for a pending write to be committed and pushed, and for an apply of the revert + // to land: a revert shows as Git or the cluster leaving B within it. + pushRaceHold = 20 * time.Second +) + +// pushRace runs the rounds against a pipeline whose applier the receiver drives. +func pushRace(pipeline *timingPipeline) { + GinkgoHelper() + for i := range pushRaceRounds { + first := fmt.Sprintf("race-a-%d", i) + second := fmt.Sprintf("race-b-%d", i) + + By("round " + strconv.Itoa(i) + ": pushing A, and waiting until the applier made it live") + pipeline.pushFromGitSide(first) + Eventually(func(g Gomega) { + g.Expect(pipeline.liveValue(g)).To(Equal(first)) + }, timingReceiverBudget, 250*time.Millisecond).Should(Succeed()) + + By("pushing B at once, while A's own apply is still on its way back through the mirror") + pipeline.pushFromGitSide(second) + + By("B reaches the cluster") + Eventually(func(g Gomega) { + g.Expect(pipeline.liveValue(g)).To(Equal(second)) + }, timingReceiverBudget, time.Second).Should(Succeed(), + "the later Git-side push must be applied, not reverted by the echo of the earlier one") + + By("and nothing reverts it, in Git or in the cluster") + Consistently(func(g Gomega) { + g.Expect(pipeline.committedValue(g)).To(Equal(second), "Git must keep the later push") + g.Expect(pipeline.liveValue(g)).To(Equal(second)) + }, pushRaceHold, 2*time.Second).Should(Succeed()) + } +} + +var _ = Describe("Bi Directional push race (Flux)", Label("bi-directional", "flux", "push-race"), Ordered, + func() { + var ( + testNs, fluxName string + pipeline *timingPipeline + ) + + BeforeAll(func() { + skipUnlessBiDirectionalEnabled() + allowCrossNamespaceAppliers() + testNs = testNamespaceFor("push-race-flux") + _, _ = kubectlRun("create", "namespace", testNs) + fluxName = "push-race-" + testNs + + repo := SetupRepo(resolveE2EContext(), testNs, fmt.Sprintf("e2e-push-race-flux-%d", GinkgoRandomSeed())) + pipeline = newTimingPipeline(testNs, "flux-race", repo) + pipeline.start() + + applyFluxApplier(testNs, repo, fluxName, fluxName+"-git", "main", "./"+pipeline.gitPath) + createFluxTrigger(testNs, fluxName, fluxName) + nameApplyTrigger(testNs, pipeline.targetName, "FluxTrigger", fluxName, "") + verifyResourceCondition("gittarget", pipeline.targetName, testNs, + "ApplyTriggerReady", "True", "Succeeded", "") + verifyResourceCondition(fluxTriggerResource, fluxName, testNs, "Held", "True", "Suspended", "") + }) + + AfterAll(func() { + if !biDirectionalEnabled() { + return + } + if pipeline != nil { + pipeline.cleanup() + } + cleanupNamespacedResource(testNs, fluxTriggerResource, fluxName) + cleanupNamespacedResource("flux-system", "kustomization", fluxName) + cleanupNamespacedResource("flux-system", "gitrepository", fluxName) + cleanupNamespacedResource("flux-system", "secret", fluxName+"-git") + cleanupNamespace(testNs) + }) + + It("keeps a Git-side push made while the previous one is being applied", func() { + pushRace(pipeline) + }) + }) + +var _ = Describe("Bi Directional push race (Argo CD)", Label("bi-directional", "argocd", "push-race"), Ordered, + func() { + var ( + testNs string + pipeline *timingPipeline + run argoBiDirectionalRun + ) + + BeforeAll(func() { + skipUnlessBiDirectionalEnabled() + requireArgoCDInstalled() + allowCrossNamespaceAppliers() + testNs = testNamespaceFor("push-race-argo") + _, _ = kubectlRun("create", "namespace", testNs) + + repo := SetupRepo(resolveE2EContext(), testNs, fmt.Sprintf("e2e-push-race-argo-%d", GinkgoRandomSeed())) + pipeline = newTimingPipeline(testNs, "argo-race", repo) + run = newArgoBiDirectionalRun(testNs, repo, "argo-race") + pipeline.start() + + run.applyArgoRepoSecret() + cfg := run.baseAppConfig() + cfg.Path = pipeline.gitPath + run.applyArgoApp(cfg) + createArgoCDTrigger(testNs, run.appName, run.argoNs, run.appName) + nameApplyTrigger(testNs, pipeline.targetName, "ArgoCDTrigger", run.appName, "") + verifyResourceCondition("gittarget", pipeline.targetName, testNs, + "ApplyTriggerReady", "True", "Succeeded", "") + }) + + AfterAll(func() { + if !biDirectionalEnabled() { + return + } + cleanupNamespacedResource(run.argoNs, "application.argoproj.io", run.appName) + cleanupNamespacedResource(run.argoNs, "secret", run.repoSecretName) + cleanupNamespacedResource(testNs, argoCDTriggerResource, run.appName) + if pipeline != nil { + pipeline.cleanup() + } + cleanupNamespace(testNs) + }) + + It("keeps a Git-side push made while the previous one is being applied", func() { + pushRace(pipeline) + }) + }) diff --git a/test/e2e/bi_directional_restart_e2e_test.go b/test/e2e/bi_directional_restart_e2e_test.go new file mode 100644 index 00000000..f7d313a2 --- /dev/null +++ b/test/e2e/bi_directional_restart_e2e_test.go @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Restart with work made while the operator was down (E9 in docs/design/applier-echoes.md). While it +// is stopped, a mirrored ConfigMap is edited and another created through the API, and another writer +// pushes a ConfigMap the GitTarget does not watch into the folder the applier applies. When it starts +// again, the edit and the new object are mirrored before anything is applied, so nothing is reverted +// on the cluster; then the Git-side change is applied, and the new object is adopted. +// +// The initial snapshot is the startup baseline: it is decided whatever Git holds, so a Git-side +// change to a watched object made while the operator was down is the API-wins case of the +// collisions table (C2). The Git-side change here is to an object the GitTarget does not watch, which +// the snapshot leaves alone. +// +// Serial: scales the controller deployment, a singleton every concurrent spec depends on. See +// docs/spec/e2e-serial-registry.md. + +// restartWithWorkMadeWhileDown runs the scenario against r. +func restartWithWorkMadeWhileDown(r *echoRun) { + GinkgoHelper() + r.pushFromOutside("down-edited", "e0") + r.eventuallyEverywhere("down-edited", "e0", echoApplyBudget) + r.eventuallyAdopted("down-edited") + unwatched := r.gitPath + "/unwatched-" + r.testNs + ".yaml" + + By("stopping the operator") + deployment, err := controllerDeploymentName() + Expect(err).NotTo(HaveOccurred()) + scaleController(deployment, 0) + DeferCleanup(func() { scaleController(deployment, 1) }) + + By("editing a mirrored ConfigMap and creating another through the API, and pushing an unwatched one") + r.editThroughAPI("down-edited", "e-down") + r.createThroughAPI("down-created", "c-down") + pushFileFromOutside(r.repo, unwatched, fmt.Sprintf(`apiVersion: v1 +kind: ConfigMap +metadata: + name: down-unwatched + namespace: %s +data: + value: g-down +`, r.testNs), "e2e: another writer adds an object the GitTarget does not watch") + + By("starting the operator again") + scaleController(deployment, 1) + + r.eventuallyEverywhere("down-edited", "e-down", echoApplyBudget) + r.eventuallyEverywhere("down-created", "c-down", echoApplyBudget) + Eventually(func(g Gomega) { + out, getErr := kubectlRunInNamespace(r.testNs, "get", "configmap", "down-unwatched", + "--ignore-not-found", "-o", "jsonpath={.data.value}") + g.Expect(getErr).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(out)).To(Equal("g-down"), "the Git-side change is applied") + }, echoApplyBudget, 2*time.Second).Should(Succeed()) + r.eventuallyAdopted("down-created") + r.consistentlyEverywhere("down-edited", "e-down") +} + +// scaleController sets the controller deployment's replicas, and waits until it has that many ready. +func scaleController(deployment string, replicas int) { + GinkgoHelper() + _, err := kubectlRunInNamespace(namespace, "scale", "deployment", deployment, + fmt.Sprintf("--replicas=%d", replicas)) + Expect(err).NotTo(HaveOccurred(), "failed to scale %s to %d", deployment, replicas) + if replicas > 0 { + _, err = kubectlRunInNamespace(namespace, "rollout", "status", "deployment", deployment, "--timeout=180s") + Expect(err).NotTo(HaveOccurred(), "%s did not become ready", deployment) + return + } + Eventually(func(g Gomega) { + out, getErr := kubectlRunInNamespace(namespace, "get", "pods", "-o", "jsonpath={.items[*].metadata.name}") + g.Expect(getErr).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(out)).NotTo(ContainSubstring(deployment), "the operator's pods are gone") + }, 2*time.Minute, time.Second).Should(Succeed()) +} + +var _ = Describe("Bi Directional restart with work made while down (Flux)", + Label("bi-directional", "flux", "applier-echoes"), Serial, Ordered, func() { + var ( + testNs, fluxName string + r *echoRun + ) + + BeforeAll(func() { + skipUnlessBiDirectionalEnabled() + allowCrossNamespaceAppliers() + testNs = testNamespaceFor("echoes-restart-flux") + _, _ = kubectlRun("create", "namespace", testNs) + fluxName = "echoes-restart-" + testNs + + repo := SetupRepo(resolveE2EContext(), testNs, + fmt.Sprintf("e2e-echoes-restart-flux-%d", GinkgoRandomSeed())) + pipeline := newTimingPipeline(testNs, "flux-restart", repo) + pipeline.start() + applyFluxApplierWithPrune(testNs, repo, fluxName, fluxName+"-git", "main", "./"+pipeline.gitPath, true) + createFluxTrigger(testNs, fluxName, fluxName) + nameApplyTrigger(testNs, pipeline.targetName, "FluxTrigger", fluxName, "") + verifyResourceCondition("gittarget", pipeline.targetName, testNs, + "ApplyTriggerReady", "True", "Succeeded", "") + r = &echoRun{timingPipeline: pipeline, applier: fluxEchoApplier(testNs, fluxName, repo)} + }) + + AfterAll(func() { + if !biDirectionalEnabled() || r == nil { + return + } + r.cleanup() + cleanupNamespacedResource(testNs, fluxTriggerResource, fluxName) + cleanupNamespacedResource("flux-system", "kustomization", fluxName) + cleanupNamespacedResource("flux-system", "gitrepository", fluxName) + cleanupNamespacedResource("flux-system", "secret", fluxName+"-git") + cleanupNamespace(testNs) + }) + + It("mirrors the work made while down before it applies, and adopts what the API created", func() { + restartWithWorkMadeWhileDown(r) + }) + }) + +var _ = Describe("Bi Directional restart with work made while down (Argo CD)", + Label("bi-directional", "argocd", "applier-echoes"), Serial, Ordered, func() { + var ( + testNs string + r *echoRun + run argoBiDirectionalRun + ) + + BeforeAll(func() { + skipUnlessBiDirectionalEnabled() + requireArgoCDInstalled() + allowCrossNamespaceAppliers() + testNs = testNamespaceFor("echoes-restart-argo") + _, _ = kubectlRun("create", "namespace", testNs) + + repo := SetupRepo(resolveE2EContext(), testNs, + fmt.Sprintf("e2e-echoes-restart-argo-%d", GinkgoRandomSeed())) + pipeline := newTimingPipeline(testNs, "argo-restart", repo) + run = newArgoBiDirectionalRun(testNs, repo, "argo-restart") + pipeline.start() + run.applyArgoRepoSecret() + cfg := run.baseAppConfig() + cfg.Path = pipeline.gitPath + run.applyArgoApp(cfg) + createArgoCDTrigger(testNs, run.appName, run.argoNs, run.appName) + nameApplyTrigger(testNs, pipeline.targetName, "ArgoCDTrigger", run.appName, "") + verifyResourceCondition("gittarget", pipeline.targetName, testNs, + "ApplyTriggerReady", "True", "Succeeded", "") + r = &echoRun{timingPipeline: pipeline, applier: argoEchoApplier(run)} + }) + + AfterAll(func() { + if !biDirectionalEnabled() || r == nil { + return + } + cleanupNamespacedResource(run.argoNs, "application.argoproj.io", run.appName) + cleanupNamespacedResource(run.argoNs, "secret", run.repoSecretName) + cleanupNamespacedResource(testNs, argoCDTriggerResource, run.appName) + r.cleanup() + cleanupNamespace(testNs) + }) + + It("mirrors the work made while down before it applies, and adopts what the API created", func() { + restartWithWorkMadeWhileDown(r) + }) + }) diff --git a/test/e2e/bi_directional_timing_e2e_test.go b/test/e2e/bi_directional_timing_e2e_test.go index f9f7657d..29fdddd5 100644 --- a/test/e2e/bi_directional_timing_e2e_test.go +++ b/test/e2e/bi_directional_timing_e2e_test.go @@ -54,9 +54,8 @@ const ( // Measurement names, shared by both appliers so the two tables read side by side. const ( measureAPIToCommit = "API edit → commit on the branch" - measureCommitToApplied = "commit on the branch → Flux applied it" - measureAPIToApplied = "API edit → Flux applied it (includes the quiet period)" - measureAppliedToHeld = "applied → trigger holds the applier again" + measureCreateToCommit = "API create → commit on the branch" + measureCreateToAdopted = "API create → adopted by the applier" measurePushToLive = "Git-side push → live object (receiver)" measurePushToSyncDone = "Git-side push → Argo CD sync finished (receiver)" measurePushToFluxHeld = "Git-side push → Flux applied it and is held again (receiver)" @@ -74,18 +73,16 @@ func timingSamples() int { return timingDefaultSamples } -// timedUntil polls probe until it passes, at most budget after start, records the time from start -// to the first passing probe under name, and returns that duration. +// timedUntil polls probe until it passes, at most budget after start, and records the time from +// start to the first passing probe under name. func timedUntil( experiment *gmeasure.Experiment, name string, start time.Time, budget time.Duration, probe func(g Gomega), -) time.Duration { +) { GinkgoHelper() remaining := budget - time.Since(start) Expect(remaining).To(BeNumerically(">", 0), "%s: the budget of %s was spent before the probe began", name, budget) Eventually(probe, remaining, timingPoll).Should(Succeed(), "%s: over its budget of %s", name, budget) - elapsed := time.Since(start) - experiment.RecordDuration(name, elapsed, gmeasure.Precision(100*time.Millisecond)) - return elapsed + experiment.RecordDuration(name, time.Since(start), gmeasure.Precision(100*time.Millisecond)) } // timingPipeline is one GitTarget mirroring the labeled ConfigMap into the folder an applier @@ -265,9 +262,9 @@ var _ = Describe("Bi Directional timing (Flux)", Label("bi-directional", "flux", pipeline = newTimingPipeline(testNs, "flux-timing", repo) pipeline.start() - By("letting Flux apply the shared folder once, then naming a trigger with the default quiet period") + By("letting Flux apply the shared folder once, then naming a trigger") applyFluxApplier(testNs, repo, fluxName, fluxName+"-git", "main", "./"+pipeline.gitPath) - createFluxTrigger(testNs, fluxName, fluxName, "") + createFluxTrigger(testNs, fluxName, fluxName) nameApplyTrigger(testNs, pipeline.targetName, "FluxTrigger", fluxName, "") verifyResourceCondition("gittarget", pipeline.targetName, testNs, "ApplyTriggerReady", "True", "Succeeded", "") @@ -291,6 +288,7 @@ var _ = Describe("Bi Directional timing (Flux)", Label("bi-directional", "flux", // heldAgain passes once the trigger reports the operation for commit Succeeded, with the // Kustomization suspended again. + heldAgain := func(g Gomega, commit string) { operation := readTriggerOperation(g, testNs, fluxTriggerResource, fluxName) g.Expect(operation.Commit).To(Equal(commit)) @@ -298,46 +296,14 @@ var _ = Describe("Bi Directional timing (Flux)", Label("bi-directional", "flux", g.Expect(kustomizationSuspended(g, fluxName)).To(BeTrue()) } - // The API side: an edit is mirrored, the branch goes quiet, and the trigger asks Flux to catch - // up. The quiet period is the default, because that is the latency a user gets; it is part of - // the total by design, so the spec also proves it is honored: no apply lands before it ends. - It("times an API edit through Git to a Flux apply", func() { - experiment := gmeasure.NewExperiment("Flux: API edit round trip") - AddReportEntry(experiment.Name, experiment) - quiet := 30 * time.Second // DefaultApplierQuietPeriod; the trigger leaves spec.quietPeriod empty. - experiment.RecordNote(fmt.Sprintf("quietPeriod %s (the default), commit window 0s", quiet)) - - for i := range timingSamples() { - value := fmt.Sprintf(timingSampleValueFormat, "api", i) - By("sample " + strconv.Itoa(i) + ": editing the ConfigMap through the API") - start := pipeline.editThroughAPI(value) - - var head string - timedUntil(experiment, measureAPIToCommit, start, timingCommitBudget, func(g Gomega) { - g.Expect(pipeline.committedValue(g)).To(Equal(value)) - head = pipeline.remoteBranchHead(g) - }) - committedAt := time.Now() - - applied := timedUntil(experiment, measureAPIToApplied, start, timingCommitBudget+quiet+timingHeldBudget, - func(g Gomega) { - g.Expect(fluxLastAppliedRevision(g, fluxName)).To(ContainSubstring(head)) - }) - experiment.RecordDuration(measureCommitToApplied, time.Since(committedAt), - gmeasure.Precision(100*time.Millisecond)) - Expect(applied).To(BeNumerically(">=", quiet), - "Flux applied %s after the edit, before the branch had been quiet for %s", applied, quiet) - appliedAt := time.Now() - - timedUntil(experiment, measureAppliedToHeld, appliedAt, timingHeldBudget, func(g Gomega) { - heldAgain(g, head) - }) - Expect(pipeline.liveValue(Default)).To(Equal(value), "the catch-up must not revert the edit it applies") - } + // The API side: an edit is mirrored, and asks Flux for no apply; a create is mirrored, and + // adopted, so Flux owns it and prunes it once Git deletes it. + It("times an API edit to its commit, and an API create to its adoption", func() { + apiTimings(fluxEchoApplier(testNs, fluxName, pipeline.repo), pipeline, "Flux") }) // The Git side: another writer pushes, Gitea tells the receiver, and the trigger asks Flux to - // apply that commit. A ForeignCommit waits for no quiet period. + // apply that commit. A ForeignCommit goes out after a flush. It("times a Git-side push through the receiver to the live object", func() { experiment := gmeasure.NewExperiment("Flux: Git-side push") AddReportEntry(experiment.Name, experiment) @@ -405,9 +371,12 @@ var _ = Describe("Bi Directional timing (Argo CD)", Label("bi-directional", "arg cleanupNamespace(testNs) }) - // Argo CD is never asked to catch up with GitOps Reverser's own commits, so the API side has - // no apply to time: its latency is the commit, which the Flux table already measures. The Git - // side is a ForeignCommit sync pinned to the pushed commit. + // The API side, as for Flux: an edit asks for no sync, and a create is adopted by a CatchUp sync. + It("times an API edit to its commit, and an API create to its adoption", func() { + apiTimings(argoEchoApplier(run), pipeline, "Argo CD") + }) + + // The Git side is a ForeignCommit sync pinned to the pushed commit. It("times a Git-side push through the receiver to the live object", func() { experiment := gmeasure.NewExperiment("Argo CD: Git-side push") AddReportEntry(experiment.Name, experiment) @@ -431,3 +400,38 @@ var _ = Describe("Bi Directional timing (Argo CD)", Label("bi-directional", "arg } }) }) + +// apiTimings times the API side for an applier: an edit to its commit, which asks for no apply, and a +// create to its commit and to its adoption, the CatchUp that makes the applier own it. +func apiTimings(applier echoApplier, pipeline *timingPipeline, name string) { + GinkgoHelper() + r := &echoRun{timingPipeline: pipeline, applier: applier} + experiment := gmeasure.NewExperiment(name + ": API round trip") + AddReportEntry(experiment.Name, experiment) + experiment.RecordNote("commit window 0s") + + for i := range timingSamples() { + value := fmt.Sprintf(timingSampleValueFormat, "api", i) + By("sample " + strconv.Itoa(i) + ": editing the ConfigMap through the API") + start := pipeline.editThroughAPI(value) + timedUntil(experiment, measureAPIToCommit, start, timingCommitBudget, func(g Gomega) { + g.Expect(pipeline.committedValue(g)).To(Equal(value)) + }) + + created := fmt.Sprintf("timing-created-%d", i) + By("sample " + strconv.Itoa(i) + ": creating a ConfigMap through the API") + start = time.Now() + r.createThroughAPI(created, value) + timedUntil(experiment, measureCreateToCommit, start, timingCommitBudget, func(g Gomega) { + committed, inGit := r.committedValue(g, created) + g.Expect(inGit).To(BeTrue()) + g.Expect(committed).To(Equal(value)) + }) + timedUntil(experiment, measureCreateToAdopted, start, timingCommitBudget+timingReceiverBudget, + func(g Gomega) { + g.Expect(applier.owns(g, pipeline.testNs, created)).To(BeTrue()) + g.Expect(applier.operation(g).Phase).To(Equal("Succeeded")) + }) + Expect(pipeline.liveValue(Default)).To(Equal(value), "the adoption must not revert the edit") + } +} diff --git a/test/e2e/flux_applier_e2e_test.go b/test/e2e/flux_applier_e2e_test.go index a9439cac..fa5c0911 100644 --- a/test/e2e/flux_applier_e2e_test.go +++ b/test/e2e/flux_applier_e2e_test.go @@ -88,7 +88,7 @@ var _ = Describe("Manager Flux Applier", Label("manager", "flux-applier"), Order By("creating a trigger no GitTarget names: it is Ready, and does nothing") createGitTarget(destName, testNs, providerName, gitPath, "main") - createFluxTrigger(testNs, fluxName, fluxName, "") + createFluxTrigger(testNs, fluxName, fluxName) verifyResourceCondition(fluxTriggerResource, fluxName, testNs, "Ready", "True", "NotReferenced", "") verifyResourceCondition(fluxTriggerResource, fluxName, testNs, "Held", "False", "NotReferenced", "") Expect(kustomizationSuspended(Default, fluxName)).To(BeFalse(), "Flux applies it on its own") @@ -120,10 +120,13 @@ var _ = Describe("Manager Flux Applier", Label("manager", "flux-applier"), Order pushAppliedColorFromOutside(repo, appliedNs, "green") applyConfigMap(testNs, "applier-second-edit") - By("the ForeignCommit is refused at the patch, and the source on release is never stamped") + // Naming the trigger converges it once (a CatchUp), as soon as the branch has a tip, so that is + // the first ask refused; the ForeignCommit joins the same run of failures, and ApplyRequestFailed + // is recorded only for a run's first. + By("every ask is refused at the patch, and the source on release is never stamped") Eventually(func(g Gomega) { failed := gitTargetEventMessages(g, testNs, destName, "ApplyRequestFailed") - g.Expect(failed).To(ContainSubstring("(ForeignCommit)")) + g.Expect(failed).To(ContainSubstring("(CatchUp)")) g.Expect(failed).To(ContainSubstring(`fetches branch "release", not main`)) }, 3*time.Minute, 2*time.Second).Should(Succeed()) Consistently(func(g Gomega) { @@ -150,11 +153,10 @@ var _ = Describe("Manager Flux Applier", Label("manager", "flux-applier"), Order }, 5*time.Minute, 5*time.Second).Should(Succeed()) verifyResourceCondition(fluxTriggerResource, fluxName, testNs, "Held", "True", "Suspended", "") Expect(kustomizationSuspended(Default, fluxName)).To(BeTrue(), "the trigger holds its Kustomization") - prefetch, err := kubectlRunInNamespace(testNs, "get", fluxTriggerResource, fluxName, - "-o", "jsonpath={.status.source.prefetch}|{.status.source.otherConsumers}") + others, err := kubectlRunInNamespace(testNs, "get", fluxTriggerResource, fluxName, + "-o", "jsonpath={.status.source.otherConsumers}") Expect(err).NotTo(HaveOccurred()) - Expect(strings.TrimSpace(prefetch)).To(Equal("true|"), - "its Kustomization is the GitRepository's only consumer, so the source is prefetched") + Expect(strings.TrimSpace(others)).To(BeEmpty(), "its Kustomization is the GitRepository's only consumer") By("dropping the reference: the trigger hands the Kustomization back, and is idle again") _, err = kubectlRunInNamespace(testNs, "patch", "gittarget", destName, "--type=merge", "-p", @@ -299,6 +301,12 @@ const fluxApplierFolder = "flux/applier" // it, both on a 24h interval, and waits for the Kustomization's first apply, made before any trigger // holds it. The Flux specs share it. func applyFluxApplier(testNs string, repo *RepoArtifacts, name, secretName, branch, path string) { + GinkgoHelper() + applyFluxApplierWithPrune(testNs, repo, name, secretName, branch, path, false) +} + +// applyFluxApplierWithPrune is applyFluxApplier, with the Kustomization's spec.prune set to prune. +func applyFluxApplierWithPrune(testNs string, repo *RepoArtifacts, name, secretName, branch, path string, prune bool) { GinkgoHelper() // The in-cluster Gitea test credential, read from the cluster at runtime, base64-encoded as the // template's Secret expects. @@ -319,7 +327,7 @@ func applyFluxApplier(testNs string, repo *RepoArtifacts, name, secretName, bran HasTargetNS bool DecryptionProvider, DecryptionSecretName string }{ - Namespace: "flux-system", Name: name, Path: path, SourceName: name, Interval: "24h", + Namespace: "flux-system", Name: name, Path: path, SourceName: name, Interval: "24h", Prune: prune, }, "flux-system")).To(Succeed(), "failed to apply the Flux Kustomization") _, err := kubectlRunInNamespace("flux-system", "wait", "--for=condition=Ready", "kustomization/"+name, "--timeout=3m") diff --git a/test/e2e/flux_bi_directional_e2e_test.go b/test/e2e/flux_bi_directional_e2e_test.go index 4b0c7147..0505dc32 100644 --- a/test/e2e/flux_bi_directional_e2e_test.go +++ b/test/e2e/flux_bi_directional_e2e_test.go @@ -422,52 +422,18 @@ var _ = Describe("Bi Directional (Flux)", Label("bi-directional", "flux"), Order }, 2*time.Minute, 2*time.Second).Should(Succeed()) }) - // The race GitOps Reverser's apply trigger exists for, from docs/design/apply-trigger.md: two - // API edits to one field a few seconds apart. With Flux woken by every push, the first commit is - // applied over the second edit and the branch ping-pongs. With the Kustomization named as the GitTarget's applier, Flux is asked to catch up - // only once the branch has been quiet, so the later value survives and Flux still ends on the - // tip, without anybody reconciling it by hand. - It("asks Flux to apply only on a quiet branch, so the later of two quick API edits survives", func() { - By("naming a trigger that holds the live Kustomization") - createFluxTrigger(testNs, run.fluxLiveName, run.fluxLiveName, "5s") - nameApplyTrigger(testNs, run.gitTargetName, "FluxTrigger", run.fluxLiveName, "") - verifyResourceCondition("gittarget", run.gitTargetName, testNs, "ApplyTriggerReady", "True", "Succeeded", "") - run.waitForGitTargetReady() - - Expect(run.gitPull()).To(Succeed()) - baseline := run.waitForStableRemoteCommitCount(biStableCountMediumWait) - - By("editing the second order through the API, and again seconds after that edit is pushed") - _, err := kubectlRunInNamespace(testNs, "patch", iceCreamCRDName(crdGroupBiDirectional), - run.secondOrderName, "--type=merge", "-p", `{"spec":{"container":"Cone"}}`) - Expect(err).NotTo(HaveOccurred()) - run.waitForCommittedOrderSpec(run.secondOrderName, "Cone", "Strawberry", "WhippedCream") - _, err = kubectlRunInNamespace(testNs, "patch", iceCreamCRDName(crdGroupBiDirectional), - run.secondOrderName, "--type=merge", "-p", `{"spec":{"container":"WaffleBowl"}}`) - Expect(err).NotTo(HaveOccurred()) - run.waitForCommittedOrderSpec(run.secondOrderName, "WaffleBowl", "Strawberry", "WhippedCream") - head := run.gitHEAD() - - By("Flux catches up with those pushes on its own, once the branch is quiet") - run.waitForFluxKustomizationRevision(run.fluxLiveName, head) - - By("and the later value survives, with no commit after it") - Consistently(func(g Gomega) { - g.Expect(liveOrderContainer(g, run, run.secondOrderName)).To(Equal("WaffleBowl")) - g.Expect(run.gitPull()).To(Succeed()) - count, countErr := run.gitMainCommitCount() - g.Expect(countErr).NotTo(HaveOccurred()) - g.Expect(count).To(Equal(baseline+2), - "an apply of the first edit's commit over the second edit would show as more commits") - }, 30*time.Second, 2*time.Second).Should(Succeed()) - }) - // A Git-side push with an API edit on top of it, from // docs/design/apply-trigger.md. Nothing tells GitOps Reverser // about the push here, so it finds it the hard way: its own push is rejected, it replays the API // edit onto the Git-side commit, and only then asks Flux to apply, so the commit Flux applies // already carries the API side's answer. Nobody reconciles Flux by hand. It("applies a Git-side push only after replaying the API edit made on top of it", func() { + By("naming a trigger that holds the live Kustomization") + createFluxTrigger(testNs, run.fluxLiveName, run.fluxLiveName) + nameApplyTrigger(testNs, run.gitTargetName, "FluxTrigger", run.fluxLiveName, "") + verifyResourceCondition("gittarget", run.gitTargetName, testNs, "ApplyTriggerReady", "True", "Succeeded", "") + run.waitForGitTargetReady() + Expect(run.gitPull()).To(Succeed()) By("pushing a Git-side change to the first order") @@ -483,13 +449,16 @@ var _ = Describe("Bi Directional (Flux)", Label("bi-directional", "flux"), Order }) Expect(run.commitAllAndPush("bi-directional: a Git-side change to the first order")).To(Succeed()) + // Cone, because the spec before leaves the second order at Cup in Git and on the cluster: an + // edit to the value Git already holds changes nothing, so it is no write (F2 in + // docs/design/applier-echoes.md), nothing is pushed, and nothing finds the Git-side push. By("and editing the second order through the API before GitOps Reverser has heard of that push") _, err := kubectlRunInNamespace(testNs, "patch", iceCreamCRDName(crdGroupBiDirectional), - run.secondOrderName, "--type=merge", "-p", `{"spec":{"container":"Cup"}}`) + run.secondOrderName, "--type=merge", "-p", `{"spec":{"container":"Cone"}}`) Expect(err).NotTo(HaveOccurred()) By("its push is rejected, replayed onto the Git-side change, and Flux applies the result") - run.waitForCommittedOrderSpec(run.secondOrderName, "Cup", "Strawberry", "WhippedCream") + run.waitForCommittedOrderSpec(run.secondOrderName, "Cone", "Strawberry", "WhippedCream") run.waitForCommittedOrderSpec(run.firstOrderName, "Cup", run.reverseFlavor, run.reverseTopping) Expect(run.gitPull()).To(Succeed()) run.waitForFluxKustomizationRevision(run.fluxLiveName, run.gitHEAD()) @@ -497,7 +466,7 @@ var _ = Describe("Bi Directional (Flux)", Label("bi-directional", "flux"), Order By("and the API edit is not reverted by it") Consistently(func(g Gomega) { - g.Expect(liveOrderContainer(g, run, run.secondOrderName)).To(Equal("Cup")) + g.Expect(liveOrderContainer(g, run, run.secondOrderName)).To(Equal("Cone")) }, 20*time.Second, 2*time.Second).Should(Succeed()) }) }) diff --git a/test/e2e/flux_restart_recovery_e2e_test.go b/test/e2e/flux_restart_recovery_e2e_test.go index b8f9874f..f8d4b186 100644 --- a/test/e2e/flux_restart_recovery_e2e_test.go +++ b/test/e2e/flux_restart_recovery_e2e_test.go @@ -67,7 +67,7 @@ var _ = Describe("Flux Restart Recovery", Label("flux-restart-recovery"), Serial `{"spec":{"wait":true,"timeout":"5m"}}`) Expect(err).NotTo(HaveOccurred(), "the Kustomization waits for what it applies to be ready") createGitTarget(destName, testNs, providerName, gitPath, "main") - createFluxTrigger(testNs, fluxName, fluxName, "") + createFluxTrigger(testNs, fluxName, fluxName) nameApplyTrigger(testNs, destName, "FluxTrigger", fluxName, "") verifyResourceCondition("gittarget", destName, testNs, "ApplyTriggerReady", "True", "Succeeded", "") Expect(applyFromTemplate("test/e2e/templates/manager/watchrule-resources.tmpl", struct { diff --git a/test/e2e/new_branch_parent_e2e_test.go b/test/e2e/new_branch_parent_e2e_test.go index 1f5ece94..2ac4c757 100644 --- a/test/e2e/new_branch_parent_e2e_test.go +++ b/test/e2e/new_branch_parent_e2e_test.go @@ -294,14 +294,22 @@ data: By("a live event the parent already holds publishes nothing") pushesBefore := pushes() + windows := fmt.Sprintf(`sum(gitopsreverser_git_commit_windows_total{gittarget_namespace=%q,`+ + `gittarget_name=%q}) or vector(0)`, testNs, destName) + windowsBefore, err := queryPrometheus(windows) + Expect(err).NotTo(HaveOccurred()) applyConfigMap("standby-present") - // The no-diff window is still pushed, because only the remote can confirm "already - // present". Wait for that push cycle to finish before judging the branch. - waitForMetricWithTimeout(branchSeries("gitopsreverser_git_pushes_total", `,outcome="pushed"`), - func(v float64) bool { return v > pushesBefore }, - "the no-diff window reached the remote", 2*time.Minute) - Expect(remoteBranchHeadOf(Default, repo.CheckoutDir, writeBranch)).To(BeEmpty(), - "a write that commits nothing must not create the write branch") + // A write that changes nothing is no write: its window closes, and nothing is committed or + // pushed (docs/design/applier-echoes.md, F2). Wait for the window to close before judging. + waitForMetricWithTimeout(windows, func(v float64) bool { return v > windowsBefore }, + "the no-diff window closed", 2*time.Minute) + Consistently(func(g Gomega) { + g.Expect(remoteBranchHeadOf(g, repo.CheckoutDir, writeBranch)).To(BeEmpty(), + "a write that commits nothing must not create the write branch") + v, queryErr := queryPrometheus(branchSeries("gitopsreverser_git_pushes_total", `,outcome="pushed"`)) + g.Expect(queryErr).NotTo(HaveOccurred()) + g.Expect(v).To(Equal(pushesBefore), "and nothing is pushed") + }, 10*time.Second, 2*time.Second).Should(Succeed()) By(parent + " moves to B from outside, touching the target's folder") hashB := commitFilesToBranchFromOutside(repo, testNs, parent, "e2e: B moves the parent", map[string]string{ diff --git a/test/e2e/refusal_reapply_e2e_test.go b/test/e2e/refusal_reapply_e2e_test.go index 5058a582..b4e61157 100644 --- a/test/e2e/refusal_reapply_e2e_test.go +++ b/test/e2e/refusal_reapply_e2e_test.go @@ -96,7 +96,7 @@ var _ = Describe("Manager Refusal Reapply", Label("manager", "refusal-reapply"), By("creating the GitTarget on the OVERLAY, naming the Kustomization and asking it to reapply") createGitTarget(destName, testNs, providerName, gitPath, "main") - createFluxTrigger(testNs, fluxName, fluxName, "") + createFluxTrigger(testNs, fluxName, fluxName) nameApplyTrigger(testNs, destName, "FluxTrigger", fluxName, "Reapply") verifyResourceCondition("gittarget", destName, testNs, "ApplyTriggerReady", "True", "Succeeded", "")