From babd656a82859ec5cfa7a5c5b6e3a0c6168b7ff2 Mon Sep 17 00:00:00 2001 From: Sean Mooney Date: Mon, 24 Aug 2026 22:37:53 +0100 Subject: [PATCH 1/3] Add applied-input-hash tracking infrastructure Bump lib-common to a revision that provides the reusable deployment.IsReadyForInput / statefulset.IsReadyForInput helpers, which fetch a workload via an uncached reader, check readiness, and confirm the running pods carry a CONFIG_HASH matching the input just computed this reconcile. To use those helpers the reconcilers need an uncached client.Reader, so add APIReader to the shared ReconcilerBase, populated from mgr.GetAPIReader() in NewReconcilerBase. Nova, Cyborg and Placement all embed this struct, so the reader becomes available at every call site with no signature changes. Add an AppliedInputSecretHash field to the Status of every CR reconciled by this operator (Nova, NovaAPI, NovaScheduler, NovaConductor, NovaCell, NovaCompute, NovaMetadata, NovaNoVNCProxy; Cyborg, CyborgAPI, CyborgConductor; PlacementAPI) and regenerate the CRD manifests. This lets each controller report the hash of the specific input secret its workload has demonstrably rolled out with, so parents (and the external control-plane CR) can correlate readiness with applied input instead of trusting a possibly stale ReadyCondition during secret rotation. This commit only adds the field and infrastructure; the controllers are wired up in the following commits. Co-Authored-By: Claude Opus 4.8 --- api/bases/cyborg.openstack.org_cyborgapis.yaml | 8 ++++++++ api/bases/cyborg.openstack.org_cyborgconductors.yaml | 8 ++++++++ api/bases/cyborg.openstack.org_cyborgs.yaml | 9 +++++++++ api/bases/nova.openstack.org_nova.yaml | 9 +++++++++ api/bases/nova.openstack.org_novaapis.yaml | 8 ++++++++ api/bases/nova.openstack.org_novacells.yaml | 9 +++++++++ api/bases/nova.openstack.org_novacomputes.yaml | 8 ++++++++ api/bases/nova.openstack.org_novaconductors.yaml | 8 ++++++++ api/bases/nova.openstack.org_novametadata.yaml | 8 ++++++++ api/bases/nova.openstack.org_novanovncproxies.yaml | 8 ++++++++ api/bases/nova.openstack.org_novaschedulers.yaml | 8 ++++++++ api/bases/placement.openstack.org_placementapis.yaml | 8 ++++++++ api/cyborg/v1beta1/cyborg_types.go | 8 ++++++++ api/cyborg/v1beta1/cyborgapi_types.go | 7 +++++++ api/cyborg/v1beta1/cyborgconductor_types.go | 7 +++++++ api/nova/v1beta1/nova_types.go | 8 ++++++++ api/nova/v1beta1/novaapi_types.go | 7 +++++++ api/nova/v1beta1/novacell_types.go | 8 ++++++++ api/nova/v1beta1/novacompute_types.go | 7 +++++++ api/nova/v1beta1/novaconductor_types.go | 9 ++++++++- api/nova/v1beta1/novametadata_types.go | 7 +++++++ api/nova/v1beta1/novanovncproxy_types.go | 7 +++++++ api/nova/v1beta1/novascheduler_types.go | 7 +++++++ api/placement/v1beta1/api_types.go | 9 ++++++++- config/crd/bases/cyborg.openstack.org_cyborgapis.yaml | 8 ++++++++ .../crd/bases/cyborg.openstack.org_cyborgconductors.yaml | 8 ++++++++ config/crd/bases/cyborg.openstack.org_cyborgs.yaml | 9 +++++++++ config/crd/bases/nova.openstack.org_nova.yaml | 9 +++++++++ config/crd/bases/nova.openstack.org_novaapis.yaml | 8 ++++++++ config/crd/bases/nova.openstack.org_novacells.yaml | 9 +++++++++ config/crd/bases/nova.openstack.org_novacomputes.yaml | 8 ++++++++ config/crd/bases/nova.openstack.org_novaconductors.yaml | 8 ++++++++ config/crd/bases/nova.openstack.org_novametadata.yaml | 8 ++++++++ .../crd/bases/nova.openstack.org_novanovncproxies.yaml | 8 ++++++++ config/crd/bases/nova.openstack.org_novaschedulers.yaml | 8 ++++++++ .../crd/bases/placement.openstack.org_placementapis.yaml | 8 ++++++++ internal/common/reconciler.go | 7 ++++++- 37 files changed, 293 insertions(+), 3 deletions(-) diff --git a/api/bases/cyborg.openstack.org_cyborgapis.yaml b/api/bases/cyborg.openstack.org_cyborgapis.yaml index 55b0cef92..922e55f51 100644 --- a/api/bases/cyborg.openstack.org_cyborgapis.yaml +++ b/api/bases/cyborg.openstack.org_cyborgapis.yaml @@ -501,6 +501,14 @@ spec: status: description: CyborgAPIStatus defines the observed state of CyborgAPI. properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/cyborg.openstack.org_cyborgconductors.yaml b/api/bases/cyborg.openstack.org_cyborgconductors.yaml index 6db92c34c..b1a6eff98 100644 --- a/api/bases/cyborg.openstack.org_cyborgconductors.yaml +++ b/api/bases/cyborg.openstack.org_cyborgconductors.yaml @@ -321,6 +321,14 @@ spec: status: description: CyborgConductorStatus defines the observed state of CyborgConductor. properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/cyborg.openstack.org_cyborgs.yaml b/api/bases/cyborg.openstack.org_cyborgs.yaml index 8c18f1bc9..45dc23ca2 100644 --- a/api/bases/cyborg.openstack.org_cyborgs.yaml +++ b/api/bases/cyborg.openstack.org_cyborgs.yaml @@ -873,6 +873,15 @@ spec: finalizer. Tracked so the controller can remove its finalizer from the old secret when the openstack-operator rotates the reference. type: string + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the top-level input Secret that + both CyborgAPI and CyborgConductor have confirmed applied and rolled + out with. It is only updated once both report readiness for this + input, so a higher-level control-plane CR can correlate Cyborg's + readiness with a specific applied input instead of relying on a + possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_nova.yaml b/api/bases/nova.openstack.org_nova.yaml index bace7230f..471108c80 100644 --- a/api/bases/nova.openstack.org_nova.yaml +++ b/api/bases/nova.openstack.org_nova.yaml @@ -2944,6 +2944,15 @@ spec: finalizer. Tracked so the controller can remove its finalizer from the old secret when the openstack-operator rotates the reference. type: string + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the top-level input Secret that + all of NovaAPI, NovaScheduler, NovaMetadata, and cell0 have confirmed + applied and rolled out with. It is only updated once all of them + report readiness for this input, so a higher-level control-plane CR + can correlate Nova's readiness with a specific applied input instead + of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_novaapis.yaml b/api/bases/nova.openstack.org_novaapis.yaml index 57c4412ed..1ba0ea930 100644 --- a/api/bases/nova.openstack.org_novaapis.yaml +++ b/api/bases/nova.openstack.org_novaapis.yaml @@ -581,6 +581,14 @@ spec: status: description: NovaAPIStatus defines the observed state of NovaAPI properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_novacells.yaml b/api/bases/nova.openstack.org_novacells.yaml index 85c0f0d4c..695647da8 100644 --- a/api/bases/nova.openstack.org_novacells.yaml +++ b/api/bases/nova.openstack.org_novacells.yaml @@ -1617,6 +1617,15 @@ spec: status: description: NovaCellStatus defines the observed state of NovaCell properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the per-cell input Secret that + all of this cell's child services (Conductor, Metadata, NoVNCProxy, + Compute) have confirmed applied and rolled out with. It is only + updated once all of them report readiness for this input, so the + top-level Nova CR can correlate this cell's readiness with a specific + applied input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_novacomputes.yaml b/api/bases/nova.openstack.org_novacomputes.yaml index dd5f3f571..495e341aa 100644 --- a/api/bases/nova.openstack.org_novacomputes.yaml +++ b/api/bases/nova.openstack.org_novacomputes.yaml @@ -370,6 +370,14 @@ spec: status: description: NovaComputeStatus defines the observed state of NovaCompute properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_novaconductors.yaml b/api/bases/nova.openstack.org_novaconductors.yaml index 15b53520e..4a03d79f5 100644 --- a/api/bases/nova.openstack.org_novaconductors.yaml +++ b/api/bases/nova.openstack.org_novaconductors.yaml @@ -411,6 +411,14 @@ spec: status: description: NovaConductorStatus defines the observed state of NovaConductor properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_novametadata.yaml b/api/bases/nova.openstack.org_novametadata.yaml index 8c74082dc..44c5adab6 100644 --- a/api/bases/nova.openstack.org_novametadata.yaml +++ b/api/bases/nova.openstack.org_novametadata.yaml @@ -554,6 +554,14 @@ spec: status: description: NovaMetadataStatus defines the observed state of NovaMetadata properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_novanovncproxies.yaml b/api/bases/nova.openstack.org_novanovncproxies.yaml index 28bbc0816..45a97b100 100644 --- a/api/bases/nova.openstack.org_novanovncproxies.yaml +++ b/api/bases/nova.openstack.org_novanovncproxies.yaml @@ -537,6 +537,14 @@ spec: status: description: NovaNoVNCProxyStatus defines the observed state of NovaNoVNCProxy properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_novaschedulers.yaml b/api/bases/nova.openstack.org_novaschedulers.yaml index e3fa65b23..2af1285ab 100644 --- a/api/bases/nova.openstack.org_novaschedulers.yaml +++ b/api/bases/nova.openstack.org_novaschedulers.yaml @@ -387,6 +387,14 @@ spec: status: description: NovaSchedulerStatus defines the observed state of NovaScheduler properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/placement.openstack.org_placementapis.yaml b/api/bases/placement.openstack.org_placementapis.yaml index c66da0bd4..fd6374a14 100644 --- a/api/bases/placement.openstack.org_placementapis.yaml +++ b/api/bases/placement.openstack.org_placementapis.yaml @@ -562,6 +562,14 @@ spec: finalizer. Tracked so the controller can remove its finalizer from the old secret when the openstack-operator rotates the reference. type: string + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last Deployment rollout that has been confirmed ready. It is only + updated once the Deployment is observed to be running with this input, + so consumers can correlate readiness with a specific applied input + instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/cyborg/v1beta1/cyborg_types.go b/api/cyborg/v1beta1/cyborg_types.go index a50f91ed3..57241c7f4 100644 --- a/api/cyborg/v1beta1/cyborg_types.go +++ b/api/cyborg/v1beta1/cyborg_types.go @@ -137,6 +137,14 @@ type CyborgStatus struct { // finalizer. Tracked so the controller can remove its finalizer from the // old secret when the openstack-operator rotates the reference. ApplicationCredentialSecret string `json:"applicationCredentialSecret,omitempty"` + + // AppliedInputSecretHash is the hash of the top-level input Secret that + // both CyborgAPI and CyborgConductor have confirmed applied and rolled + // out with. It is only updated once both report readiness for this + // input, so a higher-level control-plane CR can correlate Cyborg's + // readiness with a specific applied input instead of relying on a + // possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/cyborg/v1beta1/cyborgapi_types.go b/api/cyborg/v1beta1/cyborgapi_types.go index 3b6e594f0..8240e4ae8 100644 --- a/api/cyborg/v1beta1/cyborgapi_types.go +++ b/api/cyborg/v1beta1/cyborgapi_types.go @@ -119,6 +119,13 @@ type CyborgAPIStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/cyborg/v1beta1/cyborgconductor_types.go b/api/cyborg/v1beta1/cyborgconductor_types.go index 3144fd2d5..1c80b2d1a 100644 --- a/api/cyborg/v1beta1/cyborgconductor_types.go +++ b/api/cyborg/v1beta1/cyborgconductor_types.go @@ -107,6 +107,13 @@ type CyborgConductorStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/nova/v1beta1/nova_types.go b/api/nova/v1beta1/nova_types.go index 4c21caf83..6abee79f7 100644 --- a/api/nova/v1beta1/nova_types.go +++ b/api/nova/v1beta1/nova_types.go @@ -191,6 +191,14 @@ type NovaStatus struct { // finalizer. Tracked so the controller can remove its finalizer from the // old secret when the openstack-operator rotates the reference. ApplicationCredentialSecret string `json:"applicationCredentialSecret,omitempty"` + + // AppliedInputSecretHash is the hash of the top-level input Secret that + // all of NovaAPI, NovaScheduler, NovaMetadata, and cell0 have confirmed + // applied and rolled out with. It is only updated once all of them + // report readiness for this input, so a higher-level control-plane CR + // can correlate Nova's readiness with a specific applied input instead + // of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/nova/v1beta1/novaapi_types.go b/api/nova/v1beta1/novaapi_types.go index 0ed53cda8..453674b72 100644 --- a/api/nova/v1beta1/novaapi_types.go +++ b/api/nova/v1beta1/novaapi_types.go @@ -207,6 +207,13 @@ type NovaAPIStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/nova/v1beta1/novacell_types.go b/api/nova/v1beta1/novacell_types.go index 7b35e9ddf..e0253b54b 100644 --- a/api/nova/v1beta1/novacell_types.go +++ b/api/nova/v1beta1/novacell_types.go @@ -280,6 +280,14 @@ type NovaCellStatus struct { // then the controller has not processed the latest changes injected by // the openstack-operator in the top-level CR (e.g. the ContainerImage) ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // AppliedInputSecretHash is the hash of the per-cell input Secret that + // all of this cell's child services (Conductor, Metadata, NoVNCProxy, + // Compute) have confirmed applied and rolled out with. It is only + // updated once all of them report readiness for this input, so the + // top-level Nova CR can correlate this cell's readiness with a specific + // applied input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/nova/v1beta1/novacompute_types.go b/api/nova/v1beta1/novacompute_types.go index 54a32cdd5..627886ddf 100644 --- a/api/nova/v1beta1/novacompute_types.go +++ b/api/nova/v1beta1/novacompute_types.go @@ -168,6 +168,13 @@ type NovaComputeStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } // NovaComputeCellStatus defines state of NovaCompute in cell diff --git a/api/nova/v1beta1/novaconductor_types.go b/api/nova/v1beta1/novaconductor_types.go index 0d4e7f2b1..731f29012 100644 --- a/api/nova/v1beta1/novaconductor_types.go +++ b/api/nova/v1beta1/novaconductor_types.go @@ -183,6 +183,13 @@ type NovaConductorStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true @@ -238,7 +245,7 @@ func NewNovaConductorSpec( Resources: novaCell.ConductorServiceTemplate.Resources, NetworkAttachments: novaCell.ConductorServiceTemplate.NetworkAttachments, }, - Override: novaCell.ConductorServiceTemplate.Override, + Override: novaCell.ConductorServiceTemplate.Override, KeystoneAuthURL: novaCell.KeystoneAuthURL, ServiceUser: novaCell.ServiceUser, Region: novaCell.Region, diff --git a/api/nova/v1beta1/novametadata_types.go b/api/nova/v1beta1/novametadata_types.go index 851f0315b..d91cf3804 100644 --- a/api/nova/v1beta1/novametadata_types.go +++ b/api/nova/v1beta1/novametadata_types.go @@ -225,6 +225,13 @@ type NovaMetadataStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/nova/v1beta1/novanovncproxy_types.go b/api/nova/v1beta1/novanovncproxy_types.go index 9b7575b33..165c8d265 100644 --- a/api/nova/v1beta1/novanovncproxy_types.go +++ b/api/nova/v1beta1/novanovncproxy_types.go @@ -201,6 +201,13 @@ type NovaNoVNCProxyStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/nova/v1beta1/novascheduler_types.go b/api/nova/v1beta1/novascheduler_types.go index c7e9721d4..5962769db 100644 --- a/api/nova/v1beta1/novascheduler_types.go +++ b/api/nova/v1beta1/novascheduler_types.go @@ -173,6 +173,13 @@ type NovaSchedulerStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/placement/v1beta1/api_types.go b/api/placement/v1beta1/api_types.go index 5a7e5b3d7..402a1c6e5 100644 --- a/api/placement/v1beta1/api_types.go +++ b/api/placement/v1beta1/api_types.go @@ -18,7 +18,7 @@ package v1beta1 import ( topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" - condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/probes" "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" @@ -191,6 +191,13 @@ type PlacementAPIStatus struct { // finalizer. Tracked so the controller can remove its finalizer from the // old secret when the openstack-operator rotates the reference. ApplicationCredentialSecret string `json:"applicationCredentialSecret,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last Deployment rollout that has been confirmed ready. It is only + // updated once the Deployment is observed to be running with this input, + // so consumers can correlate readiness with a specific applied input + // instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } // PlacementAPI is the Schema for the placementapis API diff --git a/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml b/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml index 55b0cef92..922e55f51 100644 --- a/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml +++ b/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml @@ -501,6 +501,14 @@ spec: status: description: CyborgAPIStatus defines the observed state of CyborgAPI. properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml b/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml index 6db92c34c..b1a6eff98 100644 --- a/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml +++ b/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml @@ -321,6 +321,14 @@ spec: status: description: CyborgConductorStatus defines the observed state of CyborgConductor. properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/cyborg.openstack.org_cyborgs.yaml b/config/crd/bases/cyborg.openstack.org_cyborgs.yaml index 8c18f1bc9..45dc23ca2 100644 --- a/config/crd/bases/cyborg.openstack.org_cyborgs.yaml +++ b/config/crd/bases/cyborg.openstack.org_cyborgs.yaml @@ -873,6 +873,15 @@ spec: finalizer. Tracked so the controller can remove its finalizer from the old secret when the openstack-operator rotates the reference. type: string + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the top-level input Secret that + both CyborgAPI and CyborgConductor have confirmed applied and rolled + out with. It is only updated once both report readiness for this + input, so a higher-level control-plane CR can correlate Cyborg's + readiness with a specific applied input instead of relying on a + possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_nova.yaml b/config/crd/bases/nova.openstack.org_nova.yaml index bace7230f..471108c80 100644 --- a/config/crd/bases/nova.openstack.org_nova.yaml +++ b/config/crd/bases/nova.openstack.org_nova.yaml @@ -2944,6 +2944,15 @@ spec: finalizer. Tracked so the controller can remove its finalizer from the old secret when the openstack-operator rotates the reference. type: string + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the top-level input Secret that + all of NovaAPI, NovaScheduler, NovaMetadata, and cell0 have confirmed + applied and rolled out with. It is only updated once all of them + report readiness for this input, so a higher-level control-plane CR + can correlate Nova's readiness with a specific applied input instead + of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_novaapis.yaml b/config/crd/bases/nova.openstack.org_novaapis.yaml index 57c4412ed..1ba0ea930 100644 --- a/config/crd/bases/nova.openstack.org_novaapis.yaml +++ b/config/crd/bases/nova.openstack.org_novaapis.yaml @@ -581,6 +581,14 @@ spec: status: description: NovaAPIStatus defines the observed state of NovaAPI properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_novacells.yaml b/config/crd/bases/nova.openstack.org_novacells.yaml index 85c0f0d4c..695647da8 100644 --- a/config/crd/bases/nova.openstack.org_novacells.yaml +++ b/config/crd/bases/nova.openstack.org_novacells.yaml @@ -1617,6 +1617,15 @@ spec: status: description: NovaCellStatus defines the observed state of NovaCell properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the per-cell input Secret that + all of this cell's child services (Conductor, Metadata, NoVNCProxy, + Compute) have confirmed applied and rolled out with. It is only + updated once all of them report readiness for this input, so the + top-level Nova CR can correlate this cell's readiness with a specific + applied input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_novacomputes.yaml b/config/crd/bases/nova.openstack.org_novacomputes.yaml index dd5f3f571..495e341aa 100644 --- a/config/crd/bases/nova.openstack.org_novacomputes.yaml +++ b/config/crd/bases/nova.openstack.org_novacomputes.yaml @@ -370,6 +370,14 @@ spec: status: description: NovaComputeStatus defines the observed state of NovaCompute properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_novaconductors.yaml b/config/crd/bases/nova.openstack.org_novaconductors.yaml index 15b53520e..4a03d79f5 100644 --- a/config/crd/bases/nova.openstack.org_novaconductors.yaml +++ b/config/crd/bases/nova.openstack.org_novaconductors.yaml @@ -411,6 +411,14 @@ spec: status: description: NovaConductorStatus defines the observed state of NovaConductor properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_novametadata.yaml b/config/crd/bases/nova.openstack.org_novametadata.yaml index 8c74082dc..44c5adab6 100644 --- a/config/crd/bases/nova.openstack.org_novametadata.yaml +++ b/config/crd/bases/nova.openstack.org_novametadata.yaml @@ -554,6 +554,14 @@ spec: status: description: NovaMetadataStatus defines the observed state of NovaMetadata properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_novanovncproxies.yaml b/config/crd/bases/nova.openstack.org_novanovncproxies.yaml index 28bbc0816..45a97b100 100644 --- a/config/crd/bases/nova.openstack.org_novanovncproxies.yaml +++ b/config/crd/bases/nova.openstack.org_novanovncproxies.yaml @@ -537,6 +537,14 @@ spec: status: description: NovaNoVNCProxyStatus defines the observed state of NovaNoVNCProxy properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_novaschedulers.yaml b/config/crd/bases/nova.openstack.org_novaschedulers.yaml index e3fa65b23..2af1285ab 100644 --- a/config/crd/bases/nova.openstack.org_novaschedulers.yaml +++ b/config/crd/bases/nova.openstack.org_novaschedulers.yaml @@ -387,6 +387,14 @@ spec: status: description: NovaSchedulerStatus defines the observed state of NovaScheduler properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/placement.openstack.org_placementapis.yaml b/config/crd/bases/placement.openstack.org_placementapis.yaml index c66da0bd4..fd6374a14 100644 --- a/config/crd/bases/placement.openstack.org_placementapis.yaml +++ b/config/crd/bases/placement.openstack.org_placementapis.yaml @@ -562,6 +562,14 @@ spec: finalizer. Tracked so the controller can remove its finalizer from the old secret when the openstack-operator rotates the reference. type: string + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last Deployment rollout that has been confirmed ready. It is only + updated once the Deployment is observed to be running with this input, + so consumers can correlate readiness with a specific applied input + instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/internal/common/reconciler.go b/internal/common/reconciler.go index 09b077779..968aa1119 100644 --- a/internal/common/reconciler.go +++ b/internal/common/reconciler.go @@ -28,7 +28,11 @@ import ( // ReconcilerBase provides a common set of clients scheme and loggers for all reconcilers. type ReconcilerBase struct { - Client client.Client + Client client.Client + // APIReader is an uncached client.Reader, used where reads must bypass + // the informer cache (e.g. to confirm a workload has actually rolled out + // with a specific input before reporting readiness). + APIReader client.Reader Kclient kubernetes.Interface Scheme *runtime.Scheme RequeueTimeout time.Duration @@ -51,6 +55,7 @@ func NewReconcilerBase( ) ReconcilerBase { return ReconcilerBase{ Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), Scheme: mgr.GetScheme(), Kclient: kclient, RequeueTimeout: time.Duration(5) * time.Second, From 182dffff1afa5784010f7357eb7f69bc4b037daf Mon Sep 17 00:00:00 2001 From: Sean Mooney Date: Mon, 24 Aug 2026 22:38:37 +0100 Subject: [PATCH 2/3] Correlate leaf-controller readiness with applied input Every Nova, Cyborg and Placement service controller previously marked its Deployment/StatefulSetReadyCondition true based only on ReadyCount == Replicas && Generation == ObservedGeneration. During a secret rotation that is racy: the workload's ReadyCondition can still read true from before the rotation while the pods have not yet rolled out with the new config, so a stale-true readiness can be reported to the parent. Gate each leaf's readiness on the new lib-common helper (deployment.IsReadyForInput / statefulset.IsReadyForInput) using the uncached APIReader, so the condition only goes true once the running pods carry a CONFIG_HASH matching the input computed this reconcile. At that same point each leaf records util.ObjectHash of the specific input secret it was given in its Status.AppliedInputSecretHash. PlacementAPI has no intermediate child CRs of its own, so this folds its Layer 1 and Layer 2 handling together: it sets AppliedInputSecretHash directly from its own IsReadyForInput confirmation for the external control-plane CR to correlate against. Co-Authored-By: Claude Opus 4.8 --- .../controller/cyborg/cyborgapi_controller.go | 11 ++++++++++- .../cyborg/cyborgconductor_controller.go | 11 ++++++++++- .../controller/nova/novaapi_controller.go | 19 +++++++++++++++++-- .../controller/nova/novacompute_controller.go | 19 +++++++++++++++++-- .../nova/novaconductor_controller.go | 19 +++++++++++++++++-- .../nova/novametadata_controller.go | 19 +++++++++++++++++-- .../nova/novanovncproxy_controller.go | 19 +++++++++++++++++-- .../nova/novascheduler_controller.go | 19 +++++++++++++++++-- .../controller/placement/api_controller.go | 15 ++++++++++++++- 9 files changed, 136 insertions(+), 15 deletions(-) diff --git a/internal/controller/cyborg/cyborgapi_controller.go b/internal/controller/cyborg/cyborgapi_controller.go index 1c4701b0e..09ce59ec5 100644 --- a/internal/controller/cyborg/cyborgapi_controller.go +++ b/internal/controller/cyborg/cyborgapi_controller.go @@ -300,7 +300,16 @@ func (r *CyborgAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( ssObj := ss.GetStatefulSet() instance.Status.ReadyCount = ssObj.Status.ReadyReplicas - if statefulset.IsReady(ssObj) { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: ssObj.Name, Namespace: ssObj.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + if ready { + instance.Status.AppliedInputSecretHash = secretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { instance.Status.Conditions.Set(condition.FalseCondition( diff --git a/internal/controller/cyborg/cyborgconductor_controller.go b/internal/controller/cyborg/cyborgconductor_controller.go index 95add889d..55f74b4cf 100644 --- a/internal/controller/cyborg/cyborgconductor_controller.go +++ b/internal/controller/cyborg/cyborgconductor_controller.go @@ -225,7 +225,16 @@ func (r *CyborgConductorReconciler) Reconcile(ctx context.Context, req ctrl.Requ ssObj := ss.GetStatefulSet() instance.Status.ReadyCount = ssObj.Status.ReadyReplicas - if statefulset.IsReady(ssObj) { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: ssObj.Name, Namespace: ssObj.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + if ready { + instance.Status.AppliedInputSecretHash = secretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { instance.Status.Conditions.Set(condition.FalseCondition( diff --git a/internal/controller/nova/novaapi_controller.go b/internal/controller/nova/novaapi_controller.go index b8d220d87..789fa67c5 100644 --- a/internal/controller/nova/novaapi_controller.go +++ b/internal/controller/nova/novaapi_controller.go @@ -335,7 +335,7 @@ func (r *NovaAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re return result, err } - result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached) + result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached, secret) if (err != nil || result != ctrl.Result{}) { return result, err } @@ -584,6 +584,7 @@ func (r *NovaAPIReconciler) ensureDeployment( inputHash string, annotations map[string]string, memcached *memcachedv1.Memcached, + secret corev1.Secret, ) (ctrl.Result, error) { Log := r.GetLogger(ctx) serviceLabels := getAPIServiceLabels() @@ -666,8 +667,22 @@ func (r *NovaAPIReconciler) ensureDeployment( return ctrl.Result{}, err } - if instance.Status.ReadyCount == *instance.Spec.Replicas && statefulSet.Generation == statefulSet.Status.ObservedGeneration { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: statefulSet.Name, Namespace: statefulSet.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + + if ready { Log.Info("Deployment is ready") + appliedSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.AppliedInputSecretHash = appliedSecretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { Log.Info("Deployment is not ready", "Status", ss.GetStatefulSet().Status) diff --git a/internal/controller/nova/novacompute_controller.go b/internal/controller/nova/novacompute_controller.go index dfa8f39a6..f0e63c922 100644 --- a/internal/controller/nova/novacompute_controller.go +++ b/internal/controller/nova/novacompute_controller.go @@ -260,7 +260,7 @@ func (r *NovaComputeReconciler) Reconcile(ctx context.Context, req ctrl.Request) return result, err } - result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations) + result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, secret) if (err != nil || result != ctrl.Result{}) { return result, err } @@ -407,6 +407,7 @@ func (r *NovaComputeReconciler) ensureDeployment( instance *novav1.NovaCompute, inputHash string, annotations map[string]string, + secret corev1.Secret, ) (ctrl.Result, error) { Log := r.GetLogger(ctx) serviceLabels := getComputeServiceLabels(instance.Spec.CellName) @@ -482,8 +483,22 @@ func (r *NovaComputeReconciler) ensureDeployment( return ctrl.Result{}, err } - if instance.Status.ReadyCount == *instance.Spec.Replicas && statefulSet.Generation == statefulSet.Status.ObservedGeneration { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: statefulSet.Name, Namespace: statefulSet.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + + if ready { Log.Info("Deployment is ready") + appliedSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.AppliedInputSecretHash = appliedSecretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { Log.Info("Deployment is not ready", "Status", ss.GetStatefulSet().Status) diff --git a/internal/controller/nova/novaconductor_controller.go b/internal/controller/nova/novaconductor_controller.go index 6a91d6bf6..8b9d4e18e 100644 --- a/internal/controller/nova/novaconductor_controller.go +++ b/internal/controller/nova/novaconductor_controller.go @@ -306,7 +306,7 @@ func (r *NovaConductorReconciler) Reconcile(ctx context.Context, req ctrl.Reques return result, err } - result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached) + result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached, secret) if (err != nil || result != ctrl.Result{}) { return result, err } @@ -571,6 +571,7 @@ func (r *NovaConductorReconciler) ensureDeployment( inputHash string, annotations map[string]string, memcached *memcachedv1.Memcached, + secret corev1.Secret, ) (ctrl.Result, error) { serviceLabels := map[string]string{ common.AppSelector: NovaConductorLabelPrefix, @@ -649,8 +650,22 @@ func (r *NovaConductorReconciler) ensureDeployment( return ctrl.Result{}, err } - if instance.Status.ReadyCount == *instance.Spec.Replicas && statefulSet.Generation == statefulSet.Status.ObservedGeneration { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: statefulSet.Name, Namespace: statefulSet.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + + if ready { Log.Info("Deployment is ready") + appliedSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.AppliedInputSecretHash = appliedSecretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { Log.Info("Deployment is not ready", "Status", ss.GetStatefulSet().Status) diff --git a/internal/controller/nova/novametadata_controller.go b/internal/controller/nova/novametadata_controller.go index f06c0f455..0174ed7aa 100644 --- a/internal/controller/nova/novametadata_controller.go +++ b/internal/controller/nova/novametadata_controller.go @@ -328,7 +328,7 @@ func (r *NovaMetadataReconciler) Reconcile(ctx context.Context, req ctrl.Request return result, err } - result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached) + result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached, secret) if (err != nil || result != ctrl.Result{}) { return result, err } @@ -590,6 +590,7 @@ func (r *NovaMetadataReconciler) ensureDeployment( inputHash string, annotations map[string]string, memcached *memcachedv1.Memcached, + secret corev1.Secret, ) (ctrl.Result, error) { Log := r.GetLogger(ctx) serviceLabels := getMetadataServiceLabels(instance.Spec.CellName) @@ -673,8 +674,22 @@ func (r *NovaMetadataReconciler) ensureDeployment( } statefulSet = ss.GetStatefulSet() - if instance.Status.ReadyCount == *instance.Spec.Replicas && statefulSet.Generation == statefulSet.Status.ObservedGeneration { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: statefulSet.Name, Namespace: statefulSet.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + + if ready { Log.Info("Deployment is ready") + appliedSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.AppliedInputSecretHash = appliedSecretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { Log.Info("Deployment is not ready", "Status", ss.GetStatefulSet().Status) diff --git a/internal/controller/nova/novanovncproxy_controller.go b/internal/controller/nova/novanovncproxy_controller.go index f742a92b3..2757343b5 100644 --- a/internal/controller/nova/novanovncproxy_controller.go +++ b/internal/controller/nova/novanovncproxy_controller.go @@ -350,7 +350,7 @@ func (r *NovaNoVNCProxyReconciler) Reconcile(ctx context.Context, req ctrl.Reque return result, err } - result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached) + result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached, secret) if (err != nil || result != ctrl.Result{}) { return result, err } @@ -547,6 +547,7 @@ func (r *NovaNoVNCProxyReconciler) ensureDeployment( inputHash string, annotations map[string]string, memcached *memcachedv1.Memcached, + secret corev1.Secret, ) (ctrl.Result, error) { Log := r.GetLogger(ctx) serviceLabels := getNoVNCProxyServiceLabels(instance.Spec.CellName) @@ -629,8 +630,22 @@ func (r *NovaNoVNCProxyReconciler) ensureDeployment( return ctrl.Result{}, err } - if instance.Status.ReadyCount == *instance.Spec.Replicas && statefulSet.Generation == statefulSet.Status.ObservedGeneration { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: statefulSet.Name, Namespace: statefulSet.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + + if ready { Log.Info("Deployment is ready") + appliedSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.AppliedInputSecretHash = appliedSecretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { Log.Info("Deployment is not ready", "Status", ss.GetStatefulSet().Status) diff --git a/internal/controller/nova/novascheduler_controller.go b/internal/controller/nova/novascheduler_controller.go index 57293653d..37626f77a 100644 --- a/internal/controller/nova/novascheduler_controller.go +++ b/internal/controller/nova/novascheduler_controller.go @@ -310,7 +310,7 @@ func (r *NovaSchedulerReconciler) Reconcile(ctx context.Context, req ctrl.Reques return result, err } - result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached) + result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached, secret) if (err != nil || result != ctrl.Result{}) { return result, err } @@ -629,6 +629,7 @@ func (r *NovaSchedulerReconciler) ensureDeployment( inputHash string, annotations map[string]string, memcached *memcachedv1.Memcached, + secret corev1.Secret, ) (ctrl.Result, error) { serviceLabels := map[string]string{ common.AppSelector: NovaSchedulerLabelPrefix, @@ -706,8 +707,22 @@ func (r *NovaSchedulerReconciler) ensureDeployment( return ctrl.Result{}, err } - if instance.Status.ReadyCount == *instance.Spec.Replicas && statefulSet.Generation == statefulSet.Status.ObservedGeneration { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: statefulSet.Name, Namespace: statefulSet.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + + if ready { Log.Info("Deployment is ready") + appliedSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.AppliedInputSecretHash = appliedSecretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { Log.Info("Deployment is not ready", "Status", ss.GetStatefulSet().Status) diff --git a/internal/controller/placement/api_controller.go b/internal/controller/placement/api_controller.go index 7d7e123a1..8ead3c6cc 100644 --- a/internal/controller/placement/api_controller.go +++ b/internal/controller/placement/api_controller.go @@ -408,6 +408,11 @@ func (r *PlacementAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request Log.Info("Waiting for the Deployment to become Ready before exposing the service in Keystone") return ctrl.Result{}, nil } + appliedSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.AppliedInputSecretHash = appliedSecretHash err = r.ensureKeystoneServiceUser(ctx, h, instance) if err != nil { return ctrl.Result{}, err @@ -1165,7 +1170,15 @@ func (r *PlacementAPIReconciler) ensureDeployment( // Replicas > ReadyReplicas. // In addition, make sure the controller sees the last Generation // by comparing it with the ObservedGeneration. - if deployment.IsReady(deploy) { + ready, err := deployment.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: deploy.Name, Namespace: deploy.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + if ready { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { Log.Info("Deployment is not ready") From 358e04070cd8899e1e578b82c48caaaa33c088e0 Mon Sep 17 00:00:00 2001 From: Sean Mooney Date: Mon, 24 Aug 2026 22:39:27 +0100 Subject: [PATCH 3/3] Correlate parent-CR readiness with children's applied input The Nova, NovaCell and Cyborg parent controllers aggregated child-CR readiness by mirroring the child's ReadyCondition up guarded only by a generation check (Cyborg did not even check that). That inherits the stale-true race: a child can look ready with a pre-rotation config while its pods are still rolling out the new secret, letting the parent report ready prematurely. Each parent now hashes the specific secret it created for a group of children and, for every child, additionally requires child.Status.AppliedInputSecretHash == that hash before treating the child as having applied the current input. Only once all of a parent's children have applied the current input does the parent record its own Status.AppliedInputSecretHash (hashed over its own externally-supplied input secret), so the external control-plane CR can correlate the same way. The Cyborg gates also gain the previously-missing generation check. Correlation is done purely in readiness reporting; the existing flow-control paths that gate config/secret propagation are left untouched, so a rotation still regenerates and propagates config to children (including NovaCell's compute-config regeneration gated on the NoVNCProxy) without being blocked by the applied-hash check. Add envtest rotation-race functional tests at each level (Nova top-level, NovaCell, Cyborg, PlacementAPI): rotate the input secret and assert AppliedInputSecretHash does not advance until every relevant workload has rolled out the new CONFIG_HASH, then assert it does. Co-Authored-By: Claude Opus 4.8 --- .../controller/cyborg/cyborg_controller.go | 48 +++++-- internal/controller/nova/nova_controller.go | 104 +++++++++++--- .../controller/nova/novacell_controller.go | 65 ++++++--- .../cyborg/cyborg_controller_test.go | 135 ++++++++++++++++++ test/functional/nova/cell_controller_test.go | 81 +++++++++++ test/functional/nova/reconfiguration_test.go | 81 +++++++++++ .../placement/api_controller_test.go | 44 ++++++ 7 files changed, 505 insertions(+), 53 deletions(-) diff --git a/internal/controller/cyborg/cyborg_controller.go b/internal/controller/cyborg/cyborg_controller.go index c5e9d98b6..c739cc530 100644 --- a/internal/controller/cyborg/cyborg_controller.go +++ b/internal/controller/cyborg/cyborg_controller.go @@ -327,7 +327,17 @@ func (r *CyborgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res // // Create sub-level secret with required configuration // - _, err = r.createSubLevelSecret(ctx, h, instance, transporturlSecret, inputSecret, db, acData, keystoneAuthURL, keystoneRegion) + subLevelSecretName, err := r.createSubLevelSecret(ctx, h, instance, transporturlSecret, inputSecret, db, acData, keystoneAuthURL, keystoneRegion) + if err != nil { + return ctrl.Result{}, err + } + subLevelSecretObj := &corev1.Secret{} + if err := r.APIReader.Get( + ctx, types.NamespacedName{Name: subLevelSecretName, Namespace: instance.Namespace}, subLevelSecretObj, + ); err != nil { + return ctrl.Result{}, err + } + subLevelSecretHash, err := util.ObjectHash(subLevelSecretObj.Data) if err != nil { return ctrl.Result{}, err } @@ -383,7 +393,7 @@ func (r *CyborgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res // // Create CyborgAPI sub-CR // - ctrlResult, err = r.ensureCyborgAPI(ctx, h, instance, serviceLabels) + ctrlResult, apiInputApplied, err := r.ensureCyborgAPI(ctx, h, instance, serviceLabels, subLevelSecretHash) if err != nil { return ctrl.Result{}, err } else if (ctrlResult != ctrl.Result{}) { @@ -393,13 +403,17 @@ func (r *CyborgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res // // Create CyborgConductor sub-CR // - ctrlResult, err = r.ensureConductor(ctx, h, instance, serviceLabels) + ctrlResult, conductorInputApplied, err := r.ensureConductor(ctx, h, instance, serviceLabels, subLevelSecretHash) if err != nil { return ctrl.Result{}, err } else if (ctrlResult != ctrl.Result{}) { return ctrlResult, nil } + if apiInputApplied && conductorInputApplied { + instance.Status.AppliedInputSecretHash = subLevelSecretHash + } + // // Remove finalizers from unused MariaDBAccount records // @@ -939,7 +953,8 @@ func (r *CyborgReconciler) ensureCyborgAPI( h *helper.Helper, instance *cyborgv1beta1.Cyborg, serviceLabels map[string]string, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) Log.Info(fmt.Sprintf("Reconciling CyborgAPI for '%s'", instance.Name)) @@ -979,7 +994,7 @@ func (r *CyborgReconciler) ensureCyborgAPI( condition.SeverityWarning, cyborgv1beta1.CyborgAPIReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { Log.Info(fmt.Sprintf("CyborgAPI CR %s - %s", apiName, op)) @@ -988,10 +1003,13 @@ func (r *CyborgReconciler) ensureCyborgAPI( apiObj := &cyborgv1beta1.CyborgAPI{} err = h.GetClient().Get(ctx, types.NamespacedName{Name: apiName, Namespace: instance.Namespace}, apiObj) if err != nil { - return ctrl.Result{}, err + return ctrl.Result{}, false, err } - if apiObj.IsReady() { + inputApplied := apiObj.Generation == apiObj.Status.ObservedGeneration && + apiObj.Status.AppliedInputSecretHash == expectedInputSecretHash && + apiObj.IsReady() + if inputApplied { instance.Status.APIServiceReadyCount = apiObj.Status.ReadyCount instance.Status.Conditions.MarkTrue(cyborgv1beta1.CyborgAPIReadyCondition, condition.DeploymentReadyMessage) } else { @@ -1002,7 +1020,7 @@ func (r *CyborgReconciler) ensureCyborgAPI( condition.DeploymentReadyRunningMessage)) } - return ctrl.Result{}, nil + return ctrl.Result{}, inputApplied, nil } func (r *CyborgReconciler) ensureConductor( @@ -1010,7 +1028,8 @@ func (r *CyborgReconciler) ensureConductor( h *helper.Helper, instance *cyborgv1beta1.Cyborg, serviceLabels map[string]string, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) Log.Info(fmt.Sprintf("Reconciling CyborgConductor for '%s'", instance.Name)) @@ -1053,7 +1072,7 @@ func (r *CyborgReconciler) ensureConductor( condition.SeverityWarning, cyborgv1beta1.CyborgConductorReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { Log.Info(fmt.Sprintf("CyborgConductor CR %s - %s", conductorName, op)) @@ -1062,10 +1081,13 @@ func (r *CyborgReconciler) ensureConductor( conductorObj := &cyborgv1beta1.CyborgConductor{} err = h.GetClient().Get(ctx, types.NamespacedName{Name: conductorName, Namespace: instance.Namespace}, conductorObj) if err != nil { - return ctrl.Result{}, err + return ctrl.Result{}, false, err } - if conductorObj.IsReady() { + inputApplied := conductorObj.Generation == conductorObj.Status.ObservedGeneration && + conductorObj.Status.AppliedInputSecretHash == expectedInputSecretHash && + conductorObj.IsReady() + if inputApplied { instance.Status.ConductorServiceReadyCount = conductorObj.Status.ReadyCount instance.Status.Conditions.MarkTrue(cyborgv1beta1.CyborgConductorReadyCondition, condition.DeploymentReadyMessage) } else { @@ -1076,7 +1098,7 @@ func (r *CyborgReconciler) ensureConductor( condition.DeploymentReadyRunningMessage)) } - return ctrl.Result{}, nil + return ctrl.Result{}, inputApplied, nil } // SetupWithManager sets up the controller with the Manager. diff --git a/internal/controller/nova/nova_controller.go b/internal/controller/nova/nova_controller.go index 86769f12e..7e02201d5 100644 --- a/internal/controller/nova/nova_controller.go +++ b/internal/controller/nova/nova_controller.go @@ -683,30 +683,42 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul return ctrl.Result{}, err } - result, err = r.ensureAPI( + topLevelSecretObj := &corev1.Secret{} + if err := r.APIReader.Get( + ctx, types.NamespacedName{Name: topLevelSecretName, Namespace: instance.Namespace}, topLevelSecretObj, + ); err != nil { + return ctrl.Result{}, err + } + topLevelSecretHash, err := util.ObjectHash(topLevelSecretObj.Data) + if err != nil { + return ctrl.Result{}, err + } + + result, apiInputApplied, err := r.ensureAPI( ctx, instance, cell0Template, cellDBs[novav1.Cell0Name].Database, apiDB, keystoneInternalAuthURL, keystonePublicAuthURL, region, - topLevelSecretName, + topLevelSecretName, topLevelSecretHash, ) if err != nil { return result, err } - result, err = r.ensureScheduler( + result, schedulerInputApplied, err := r.ensureScheduler( ctx, instance, cell0Template, cellDBs[novav1.Cell0Name].Database, apiDB, keystoneInternalAuthURL, region, - topLevelSecretName, + topLevelSecretName, topLevelSecretHash, ) if err != nil { return result, err } + metadataInputApplied := true if *instance.Spec.MetadataServiceTemplate.Enabled { - result, err = r.ensureMetadata( + result, metadataInputApplied, err = r.ensureMetadata( ctx, instance, cell0Template, cellDBs[novav1.Cell0Name].Database, apiDB, keystoneInternalAuthURL, region, - topLevelSecretName, + topLevelSecretName, topLevelSecretHash, ) if err != nil { return result, err @@ -722,6 +734,10 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul instance.Status.MetadataServiceReadyCount = 0 } + if apiInputApplied && schedulerInputApplied && metadataInputApplied { + instance.Status.AppliedInputSecretHash = topLevelSecretHash + } + // remove finalizers from unused MariaDBAccount records but ONLY if // ensureAPIDB finished if apiDBStatus == nova.DBCompleted { @@ -1301,6 +1317,17 @@ func (r *NovaReconciler) ensureCell( return nil, nova.CellDeploying, err } + cellSecretObj := &corev1.Secret{} + if err := r.APIReader.Get( + ctx, types.NamespacedName{Name: cellSecretName, Namespace: instance.Namespace}, cellSecretObj, + ); err != nil { + return nil, nova.CellFailed, err + } + cellSecretHash, err := util.ObjectHash(cellSecretObj.Data) + if err != nil { + return nil, nova.CellFailed, err + } + cellSpec := novav1.NovaCellSpec{ CellName: cellName, Secret: cellSecretName, @@ -1366,9 +1393,10 @@ func (r *NovaReconciler) ensureCell( Log.Info(fmt.Sprintf("NovaCell %s.", string(op)), "NovaCell.Name", cell.Name) } - if !cell.IsReady() || cell.Generation != cell.Status.ObservedGeneration { - // We wait for the cell to become Ready before we map it in the - // nova_api DB. + if !cell.IsReady() || cell.Generation != cell.Status.ObservedGeneration || + cell.Status.AppliedInputSecretHash != cellSecretHash { + // We wait for the cell to become Ready with this input before we map + // it in the nova_api DB. return cell, nova.CellDeploying, err } configHash, scriptName, configName, err := r.ensureNovaManageJobSecret(ctx, h, instance, @@ -1466,7 +1494,8 @@ func (r *NovaReconciler) ensureAPI( keystonePublicAuthURL string, region string, secretName string, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) // TODO(gibi): Pass down a narrowed secret that only hold @@ -1529,7 +1558,7 @@ func (r *NovaReconciler) ensureAPI( condition.SeverityError, novav1.NovaAPIReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { @@ -1541,12 +1570,23 @@ func (r *NovaReconciler) ensureAPI( // NOTE(gibi): it can be nil if the NovaAPI CR is created but no // reconciliation is run on it to initialize the ReadyCondition yet. if c != nil { + if c.Status == corev1.ConditionTrue && api.Status.AppliedInputSecretHash != expectedInputSecretHash { + // NovaAPI's own ReadyCondition may still read stale-true from + // before an input rotation. Don't propagate that until it has + // confirmed rollout of the currently expected input. + c.Status = corev1.ConditionFalse + c.Severity = condition.SeverityInfo + c.Reason = condition.RequestedReason + c.Message = condition.DeploymentReadyRunningMessage + } instance.Status.Conditions.Set(c) } instance.Status.APIServiceReadyCount = api.Status.ReadyCount } - return ctrl.Result{}, nil + inputApplied := api.Status.Conditions.IsTrue(condition.ReadyCondition) && + api.Status.AppliedInputSecretHash == expectedInputSecretHash + return ctrl.Result{}, inputApplied, nil } func (r *NovaReconciler) ensureScheduler( @@ -1558,7 +1598,8 @@ func (r *NovaReconciler) ensureScheduler( keystoneAuthURL string, region string, secretName string, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) // TODO(gibi): Pass down a narrowed secret that only hold // specific information but also holds user names @@ -1621,7 +1662,7 @@ func (r *NovaReconciler) ensureScheduler( condition.SeverityError, novav1.NovaSchedulerReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { @@ -1634,12 +1675,23 @@ func (r *NovaReconciler) ensureScheduler( // NOTE(gibi): it can be nil if the NovaScheduler CR is created but no // reconciliation is run on it to initialize the ReadyCondition yet. if c != nil { + if c.Status == corev1.ConditionTrue && scheduler.Status.AppliedInputSecretHash != expectedInputSecretHash { + // NovaScheduler's own ReadyCondition may still read stale-true + // from before an input rotation. Don't propagate that until it + // has confirmed rollout of the currently expected input. + c.Status = corev1.ConditionFalse + c.Severity = condition.SeverityInfo + c.Reason = condition.RequestedReason + c.Message = condition.DeploymentReadyRunningMessage + } instance.Status.Conditions.Set(c) } instance.Status.SchedulerServiceReadyCount = scheduler.Status.ReadyCount } - return ctrl.Result{}, nil + inputApplied := scheduler.Status.Conditions.IsTrue(condition.ReadyCondition) && + scheduler.Status.AppliedInputSecretHash == expectedInputSecretHash + return ctrl.Result{}, inputApplied, nil } func (r *NovaReconciler) ensureKeystoneServiceUser( @@ -1941,7 +1993,8 @@ func (r *NovaReconciler) ensureMetadata( keystoneAuthURL string, region string, secretName string, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) // There is a case when the user manually created a NovaMetadata while it // was disabled in the Nova and then tries to enable it in Nova. @@ -1958,7 +2011,7 @@ func (r *NovaReconciler) ensureMetadata( metadata := &novav1.NovaMetadata{} err := r.Client.Get(ctx, metadataName, metadata) if err != nil && !k8s_errors.IsNotFound(err) { - return ctrl.Result{}, err + return ctrl.Result{}, false, err } // If it is not created by us, we don't touch it @@ -1977,7 +2030,7 @@ func (r *NovaReconciler) ensureMetadata( novav1.NovaMetadataReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } // TODO(gibi): Pass down a narrowed secret that only hold @@ -2040,7 +2093,7 @@ func (r *NovaReconciler) ensureMetadata( condition.SeverityError, novav1.NovaMetadataReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { @@ -2052,11 +2105,22 @@ func (r *NovaReconciler) ensureMetadata( // NOTE(gibi): it can be nil if the NovaMetadata CR is created but no // reconciliation is run on it to initialize the ReadyCondition yet. if c != nil { + if c.Status == corev1.ConditionTrue && metadata.Status.AppliedInputSecretHash != expectedInputSecretHash { + // NovaMetadata's own ReadyCondition may still read stale-true + // from before an input rotation. Don't propagate that until it + // has confirmed rollout of the currently expected input. + c.Status = corev1.ConditionFalse + c.Severity = condition.SeverityInfo + c.Reason = condition.RequestedReason + c.Message = condition.DeploymentReadyRunningMessage + } instance.Status.Conditions.Set(c) } instance.Status.MetadataServiceReadyCount = metadata.Status.ReadyCount } - return ctrl.Result{}, nil + inputApplied := metadata.Status.Conditions.IsTrue(condition.ReadyCondition) && + metadata.Status.AppliedInputSecretHash == expectedInputSecretHash + return ctrl.Result{}, inputApplied, nil } // ensureCellMapped makes sure that the cell has a row in the diff --git a/internal/controller/nova/novacell_controller.go b/internal/controller/nova/novacell_controller.go index de611f63d..0ca9e51c4 100644 --- a/internal/controller/nova/novacell_controller.go +++ b/internal/controller/nova/novacell_controller.go @@ -166,13 +166,19 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r // all our input checks out so report InputReady instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) - result, err = r.ensureConductor(ctx, instance) + cellSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + + result, conductorInputApplied, err := r.ensureConductor(ctx, instance, cellSecretHash) if err != nil { return result, err } + metadataInputApplied := true if *instance.Spec.MetadataServiceTemplate.Enabled { - result, err = r.ensureMetadata(ctx, instance) + result, metadataInputApplied, err = r.ensureMetadata(ctx, instance, cellSecretHash) if err != nil { return result, err } @@ -193,7 +199,7 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r // to run discover job only when all computes are deployed and never discovered computeTemplatesHashMap := make(map[string]string) for computeName, computeTemplate := range instance.Spec.NovaComputeTemplates { - computeStatus := r.ensureNovaCompute(ctx, instance, computeTemplate, computeName) + computeStatus := r.ensureNovaCompute(ctx, instance, computeTemplate, computeName, cellSecretHash) instance.Status.NovaComputesStatus[computeName] = computeStatus // We hash the entire compute template to keep track of changes in the number of replicas, // allowing us to discover nodes accordingly @@ -214,6 +220,7 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r } // We need to check if all computes are deployed + allComputesApplied := true if len(instance.Spec.NovaComputeTemplates) == 0 { Log.Info("No nova compute ironic/fake driver service definition in cell") instance.Status.Conditions.Remove(novav1.NovaAllControlPlaneComputesReadyCondition) @@ -228,7 +235,8 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r failedComputes = append(failedComputes, computeName) } } - if len(instance.Spec.NovaComputeTemplates) == len(readyComputes) { + allComputesApplied = len(instance.Spec.NovaComputeTemplates) == len(readyComputes) + if allComputesApplied { instance.Status.Conditions.MarkTrue( novav1.NovaAllControlPlaneComputesReadyCondition, condition.ServiceConfigReadyMessage, ) @@ -247,8 +255,9 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r instance.Status.Hash[novav1.ComputeDiscoverHashKey] = computeTemplatesHash cellHasVNCService := (*instance.Spec.NoVNCProxyServiceTemplate.Enabled) + novncproxyInputApplied := true if cellHasVNCService { - result, err = r.ensureNoVNCProxy(ctx, instance) + result, novncproxyInputApplied, err = r.ensureNoVNCProxy(ctx, instance, cellSecretHash) if err != nil { return result, err } @@ -262,6 +271,10 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r instance.Status.Conditions.Remove(novav1.NovaNoVNCProxyReadyCondition) } + if conductorInputApplied && metadataInputApplied && novncproxyInputApplied && allComputesApplied { + instance.Status.AppliedInputSecretHash = cellSecretHash + } + // We need to wait for the NovaNoVNCProxy to become Ready before we can try // to generate the compute config secret as that needs the endpoint of the // proxy to be included. @@ -359,7 +372,8 @@ func (r *NovaCellReconciler) initConditions( func (r *NovaCellReconciler) ensureConductor( ctx context.Context, instance *novav1.NovaCell, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) conductorSpec := novav1.NewNovaConductorSpec(instance.Spec) @@ -386,7 +400,7 @@ func (r *NovaCellReconciler) ensureConductor( condition.SeverityError, novav1.NovaConductorReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { @@ -403,7 +417,9 @@ func (r *NovaCellReconciler) ensureConductor( } } - return ctrl.Result{}, nil + inputApplied := conductor.Status.Conditions.IsTrue(condition.ReadyCondition) && + conductor.Status.AppliedInputSecretHash == expectedInputSecretHash + return ctrl.Result{}, inputApplied, nil } func getNoVNCProxyName(instance *novav1.NovaCell) types.NamespacedName { @@ -413,7 +429,8 @@ func getNoVNCProxyName(instance *novav1.NovaCell) types.NamespacedName { func (r *NovaCellReconciler) ensureNoVNCProxy( ctx context.Context, instance *novav1.NovaCell, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) // There is a case when the user manually created a NoVNCProxy while it // was disabled in the cell and then tries to enable it in the cell. @@ -430,7 +447,7 @@ func (r *NovaCellReconciler) ensureNoVNCProxy( novncproxy := &novav1.NovaNoVNCProxy{} err := r.Client.Get(ctx, novncproxyName, novncproxy) if err != nil && !k8s_errors.IsNotFound(err) { - return ctrl.Result{}, err + return ctrl.Result{}, false, err } // If it is not created by us, we don't touch it @@ -449,7 +466,7 @@ func (r *NovaCellReconciler) ensureNoVNCProxy( novav1.NovaNoVNCProxyReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } // NoVNCProxy is either not exists, or it exists but owned by us so we can @@ -478,7 +495,7 @@ func (r *NovaCellReconciler) ensureNoVNCProxy( condition.SeverityError, novav1.NovaNoVNCProxyReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { @@ -495,7 +512,9 @@ func (r *NovaCellReconciler) ensureNoVNCProxy( } } - return ctrl.Result{}, nil + inputApplied := novncproxy.Status.Conditions.IsTrue(condition.ReadyCondition) && + novncproxy.Status.AppliedInputSecretHash == expectedInputSecretHash + return ctrl.Result{}, inputApplied, nil } func (r *NovaCellReconciler) ensureNoVNCProxyDeleted( @@ -535,7 +554,8 @@ func (r *NovaCellReconciler) ensureNoVNCProxyDeleted( func (r *NovaCellReconciler) ensureMetadata( ctx context.Context, instance *novav1.NovaCell, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) // There is a case when the user manually created a NovaMetadata while it // was disabled in the NovaCell and then tries to enable it in NovaCell. @@ -552,7 +572,7 @@ func (r *NovaCellReconciler) ensureMetadata( metadata := &novav1.NovaMetadata{} err := r.Client.Get(ctx, metadataName, metadata) if err != nil && !k8s_errors.IsNotFound(err) { - return ctrl.Result{}, err + return ctrl.Result{}, false, err } // If it is not created by us, we don't touch it @@ -571,7 +591,7 @@ func (r *NovaCellReconciler) ensureMetadata( novav1.NovaMetadataReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } metadataSpec := novav1.NewNovaMetadataSpec(instance.Spec) @@ -599,7 +619,7 @@ func (r *NovaCellReconciler) ensureMetadata( condition.SeverityError, novav1.NovaMetadataReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { @@ -616,7 +636,9 @@ func (r *NovaCellReconciler) ensureMetadata( } } - return ctrl.Result{}, nil + inputApplied := metadata.Status.Conditions.IsTrue(condition.ReadyCondition) && + metadata.Status.AppliedInputSecretHash == expectedInputSecretHash + return ctrl.Result{}, inputApplied, nil } // ensureComputeConfig ensures the the compute config Secret exists and up to @@ -689,6 +711,7 @@ func (r *NovaCellReconciler) ensureNovaCompute( instance *novav1.NovaCell, compute novav1.NovaComputeTemplate, computeName string, + expectedInputSecretHash string, ) novav1.NovaComputeCellStatus { Log := r.GetLogger(ctx) // There is a case when the user manually created a NovaCompute with selected name. @@ -706,6 +729,7 @@ func (r *NovaCellReconciler) ensureNovaCompute( err := r.Client.Get(ctx, fullComputeName, novacompute) var computeStatus novav1.NovaComputeCellStatus if computeStatus, ok := instance.Status.NovaComputesStatus[computeName]; ok { + computeStatus.Deployed = false computeStatus.Errors = false } else { computeStatus = novav1.NovaComputeCellStatus{Deployed: false, Errors: false} @@ -757,8 +781,9 @@ func (r *NovaCellReconciler) ensureNovaCompute( Log.Info(fmt.Sprintf("NovaCompute %s, NovaCompute.Name %s .", string(op), novacompute.Name)) } - if novacompute.Generation == novacompute.Status.ObservedGeneration && novacompute.IsReady() { - // We wait for the novacompute to become Ready before we map it deployed. + if novacompute.Generation == novacompute.Status.ObservedGeneration && novacompute.IsReady() && + novacompute.Status.AppliedInputSecretHash == expectedInputSecretHash { + // We wait for the novacompute to become Ready with this input before we map it deployed. computeStatus.Deployed = true } diff --git a/test/functional/cyborg/cyborg_controller_test.go b/test/functional/cyborg/cyborg_controller_test.go index 97738c441..d3082b7b8 100644 --- a/test/functional/cyborg/cyborg_controller_test.go +++ b/test/functional/cyborg/cyborg_controller_test.go @@ -409,6 +409,70 @@ var _ = Describe("Cyborg controller", func() { corev1.ConditionTrue, ) }) + + It("does not advance AppliedInputSecretHash until the API and conductor roll out the rotated input", func() { + // Bring Cyborg fully Ready first. + mariadb.SimulateMariaDBAccountCompleted(cyborgNames.MariaDBAccountName) + mariadb.SimulateMariaDBDatabaseCompleted(cyborgNames.MariaDBDatabaseName) + infra.SimulateTransportURLReady(cyborgNames.TransportURLName) + keystone.SimulateKeystoneServiceReady(cyborgNames.KeystoneServiceName) + th.SimulateJobSuccess(cyborgNames.DBSyncJobName) + th.SimulateStatefulSetReplicaReady(cyborgNames.APIStatefulSetName) + keystone.SimulateKeystoneEndpointReady(cyborgNames.KeystoneEndpointName) + th.SimulateStatefulSetReplicaReady(cyborgNames.ConductorStatefulSetName) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + + var oldHash string + Eventually(func(g Gomega) { + oldHash = GetCyborg(cyborgNames.CyborgName).Status.AppliedInputSecretHash + g.Expect(oldHash).NotTo(BeEmpty()) + }, timeout, interval).Should(Succeed()) + + apiSS := cyborgNames.APIStatefulSetName + condSS := cyborgNames.ConductorStatefulSetName + oldAPIHash := GetEnvVarValue( + th.GetStatefulSet(apiSS).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + oldCondHash := GetEnvVarValue( + th.GetStatefulSet(condSS).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + Expect(oldAPIHash).NotTo(BeEmpty()) + Expect(oldCondHash).NotTo(BeEmpty()) + + th.UpdateSecret( + types.NamespacedName{Namespace: cyborgNames.CyborgName.Namespace, Name: CyborgSecretName}, + CyborgPasswordSelectorValue, []byte("new-cyborg-password")) + + // The rotated input has not rolled out yet (envtest has no + // StatefulSet controller to bump ObservedGeneration), so the parent + // must keep reporting the pre-rotation hash. + Consistently(func(g Gomega) { + g.Expect(GetCyborg(cyborgNames.CyborgName).Status.AppliedInputSecretHash).To(Equal(oldHash)) + }, "2s", interval).Should(Succeed()) + + // Wait for the rotated secret to reach both StatefulSets, then + // simulate the rollouts completing. + Eventually(func(g Gomega) { + g.Expect(GetEnvVarValue( + th.GetStatefulSet(apiSS).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "")).NotTo(Equal(oldAPIHash)) + g.Expect(GetEnvVarValue( + th.GetStatefulSet(condSS).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "")).NotTo(Equal(oldCondHash)) + }, timeout, interval).Should(Succeed()) + th.SimulateStatefulSetReplicaReady(apiSS) + th.SimulateStatefulSetReplicaReady(condSS) + + // Both children confirmed the rollout, so the Cyborg CR advances its + // applied-input hash to the rotated value. + Eventually(func(g Gomega) { + newHash := GetCyborg(cyborgNames.CyborgName).Status.AppliedInputSecretHash + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(oldHash)) + }, timeout, interval).Should(Succeed()) + }) }) When("Cyborg CR is created with TLS and ApplicationCredentials", func() { @@ -601,6 +665,15 @@ var _ = Describe("Cyborg controller", func() { corev1.ConditionTrue, ) + apiStatefulSet := cyborgNames.APIStatefulSetName + conductorStatefulSet := cyborgNames.ConductorStatefulSetName + oldAPIHash := GetEnvVarValue( + th.GetStatefulSet(apiStatefulSet).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + oldConductorHash := GetEnvVarValue( + th.GetStatefulSet(conductorStatefulSet).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + Expect(oldAPIHash).NotTo(BeEmpty()) + Expect(oldConductorHash).NotTo(BeEmpty()) + newACSecretName := "ac-cyborg-x9y8z-secret" //nolint:gosec // G101 newAppCredSecret := th.CreateSecret( types.NamespacedName{Namespace: cyborgNames.CyborgName.Namespace, Name: newACSecretName}, @@ -636,6 +709,28 @@ var _ = Describe("Cyborg controller", func() { g.Expect(string(subSecret.Data["ACSecret"])).To(Equal(appCredSecretValue + "-rotated")) }, timeout, interval).Should(Succeed()) + // Wait until both sub-CRs propagate the rotated input to their + // StatefulSets before marking the new generations ready. + Eventually(func(g Gomega) { + apiHash := GetEnvVarValue( + th.GetStatefulSet(apiStatefulSet).Spec.Template.Spec.Containers[0].Env, + "CONFIG_HASH", "") + conductorHash := GetEnvVarValue( + th.GetStatefulSet(conductorStatefulSet).Spec.Template.Spec.Containers[0].Env, + "CONFIG_HASH", "") + g.Expect(apiHash).NotTo(BeEmpty()) + g.Expect(apiHash).NotTo(Equal(oldAPIHash)) + g.Expect(conductorHash).NotTo(BeEmpty()) + g.Expect(conductorHash).NotTo(Equal(oldConductorHash)) + }, timeout, interval).Should(Succeed()) + + // The old secret's finalizer must not be removed until the + // sub-CRs have actually rolled out with the rotated input. Since + // envtest has no StatefulSet controller, simulate that rollout + // now to unblock the finalizer removal below. + th.SimulateStatefulSetReplicaReady(conductorStatefulSet) + th.SimulateStatefulSetReplicaReady(apiStatefulSet) + // the finalizer is removed from the old secret Eventually(func(g Gomega) { secret := th.GetSecret(types.NamespacedName{ @@ -686,6 +781,15 @@ var _ = Describe("Cyborg controller", func() { corev1.ConditionTrue, ) + apiStatefulSet := cyborgNames.APIStatefulSetName + conductorStatefulSet := cyborgNames.ConductorStatefulSetName + oldAPIHash := GetEnvVarValue( + th.GetStatefulSet(apiStatefulSet).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + oldConductorHash := GetEnvVarValue( + th.GetStatefulSet(conductorStatefulSet).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + Expect(oldAPIHash).NotTo(BeEmpty()) + Expect(oldConductorHash).NotTo(BeEmpty()) + // clear the AC secret from the spec Eventually(func(g Gomega) { cyborg := GetCyborg(cyborgNames.CyborgName) @@ -693,6 +797,37 @@ var _ = Describe("Cyborg controller", func() { g.Expect(k8sClient.Update(ctx, cyborg)).Should(Succeed()) }, timeout, interval).Should(Succeed()) + // Wait for the sub-CRs to have picked up the updated sub-level + // secret (i.e. reconciled at least once past the spec change) + // before simulating their StatefulSet rollout below. + Eventually(func(g Gomega) { + subSecret := th.GetSecret(cyborgNames.SubLevelSecretName) + g.Expect(subSecret.Data).NotTo(HaveKey("ACID")) + g.Expect(subSecret.Data).NotTo(HaveKey("ACSecret")) + }, timeout, interval).Should(Succeed()) + + // Wait until both sub-CRs propagate the updated input to their + // StatefulSets before marking the new generations ready. + Eventually(func(g Gomega) { + apiHash := GetEnvVarValue( + th.GetStatefulSet(apiStatefulSet).Spec.Template.Spec.Containers[0].Env, + "CONFIG_HASH", "") + conductorHash := GetEnvVarValue( + th.GetStatefulSet(conductorStatefulSet).Spec.Template.Spec.Containers[0].Env, + "CONFIG_HASH", "") + g.Expect(apiHash).NotTo(BeEmpty()) + g.Expect(apiHash).NotTo(Equal(oldAPIHash)) + g.Expect(conductorHash).NotTo(BeEmpty()) + g.Expect(conductorHash).NotTo(Equal(oldConductorHash)) + }, timeout, interval).Should(Succeed()) + + // The old secret's finalizer must not be removed until the + // sub-CRs have actually rolled out with the updated input. Since + // envtest has no StatefulSet controller, simulate that rollout + // now to unblock the finalizer removal below. + th.SimulateStatefulSetReplicaReady(conductorStatefulSet) + th.SimulateStatefulSetReplicaReady(apiStatefulSet) + // the finalizer is removed from the secret Eventually(func(g Gomega) { secret := th.GetSecret(types.NamespacedName{ diff --git a/test/functional/nova/cell_controller_test.go b/test/functional/nova/cell_controller_test.go index 0a57fca94..971b2916f 100644 --- a/test/functional/nova/cell_controller_test.go +++ b/test/functional/nova/cell_controller_test.go @@ -32,6 +32,7 @@ import ( controllers "github.com/openstack-k8s-operators/nova-operator/internal/controller/nova" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -459,6 +460,86 @@ var _ = Describe("NovaCell controller", func() { Expect(cell.Status.NovaComputesStatus).To(HaveKey(ironicComputeName)) }) + It("does not advance AppliedInputSecretHash until all cell services roll out the rotated input", func() { + th.SimulateJobSuccess(cell1.DBSyncJobName) + th.SimulateStatefulSetReplicaReady(cell1.ConductorStatefulSetName) + th.SimulateStatefulSetReplicaReady(cell1.NovaComputeStatefulSetName) + th.SimulateStatefulSetReplicaReady(cell1.NoVNCProxyStatefulSetName) + th.SimulateStatefulSetReplicaReady(cell1.MetadataStatefulSetName) + + th.ExpectCondition( + cell1.CellCRName, + ConditionGetterFunc(NovaCellConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + + oldHash := GetNovaCell(cell1.CellCRName).Status.AppliedInputSecretHash + Expect(oldHash).NotTo(BeEmpty()) + + // conductor, novncproxy and metadata regenerate their config + // independently of readiness; the compute config is only + // regenerated once the NoVNCProxy is Ready again (it needs the + // proxy endpoint), so it rolls out last. + independentSS := []types.NamespacedName{ + cell1.ConductorStatefulSetName, + cell1.NoVNCProxyStatefulSetName, + cell1.MetadataStatefulSetName, + } + originalHashes := map[string]string{} + for _, ss := range append(independentSS, cell1.NovaComputeStatefulSetName) { + h := GetEnvVarValue( + th.GetStatefulSet(ss).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + Expect(h).NotTo(BeEmpty()) + originalHashes[ss.Name] = h + } + + th.UpdateSecret(cell1.InternalCellSecretName, "ServicePassword", []byte("new-service-password")) + + // The rotated input has not rolled out yet, so the cell must keep + // reporting the pre-rotation hash. + Consistently(func(g Gomega) { + g.Expect(GetNovaCell(cell1.CellCRName).Status.AppliedInputSecretHash).To(Equal(oldHash)) + }, "2s", interval).Should(Succeed()) + + for _, ss := range independentSS { + Eventually(func(g Gomega) { + newHash := GetEnvVarValue( + th.GetStatefulSet(ss).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(originalHashes[ss.Name])) + }, timeout, interval).Should(Succeed()) + th.SimulateStatefulSetReplicaReady(ss) + } + + // The compute still reports the previous rollout, so it must no + // longer count as deployed for the current cell input. + Eventually(func(g Gomega) { + cell := GetNovaCell(cell1.CellCRName) + g.Expect(cell.Status.NovaComputesStatus).To(HaveKey(ironicComputeName)) + g.Expect(cell.Status.NovaComputesStatus[ironicComputeName].Deployed).To(BeFalse()) + g.Expect(cell.Status.AppliedInputSecretHash).To(Equal(oldHash)) + }, timeout, interval).Should(Succeed()) + + // Now that the NoVNCProxy has rolled out, the compute config is + // regenerated with the rotated input and the compute rolls out. + Eventually(func(g Gomega) { + newHash := GetEnvVarValue( + th.GetStatefulSet(cell1.NovaComputeStatefulSetName).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(originalHashes[cell1.NovaComputeStatefulSetName.Name])) + }, timeout, interval).Should(Succeed()) + th.SimulateStatefulSetReplicaReady(cell1.NovaComputeStatefulSetName) + + // With every cell service confirming the rollout, the NovaCell + // advances its applied-input hash to the rotated value. + Eventually(func(g Gomega) { + newHash := GetNovaCell(cell1.CellCRName).Status.AppliedInputSecretHash + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(oldHash)) + }, timeout, interval).Should(Succeed()) + }) + It("deletes NoVNCProxy if it is disabled later", func() { th.SimulateJobSuccess(cell1.DBSyncJobName) th.SimulateStatefulSetReplicaReady(cell1.ConductorStatefulSetName) diff --git a/test/functional/nova/reconfiguration_test.go b/test/functional/nova/reconfiguration_test.go index 020e7dff9..0524ae9dc 100644 --- a/test/functional/nova/reconfiguration_test.go +++ b/test/functional/nova/reconfiguration_test.go @@ -866,6 +866,20 @@ var _ = Describe("Nova reconfiguration", func() { secretName := types.NamespacedName{Namespace: novaNames.NovaName.Namespace, Name: SecretName} th.UpdateSecret(secretName, "NovaPassword", []byte("new-service-password")) + // cell1 and cell2 need API DB access, which Nova only grants once + // cell0 has confirmed it rolled out this rotated input. Since + // envtest has no StatefulSet controller, cell0's conductor won't + // self-report that rollout until its StatefulSet is simulated + // ready at the new generation, so do that first to unblock the + // other cells' secret regeneration below. + Eventually(func(g Gomega) { + newHash := GetEnvVarValue( + th.GetStatefulSet(cell0.ConductorStatefulSetName).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(originalHashes[0])) + }, timeout, interval).Should(Succeed()) + th.SimulateStatefulSetReplicaReady(cell0.ConductorStatefulSetName) + // Assert that the config hash is updated in each stateful set for i, ss := range ssNames { Eventually(func(g Gomega) { @@ -876,6 +890,73 @@ var _ = Describe("Nova reconfiguration", func() { }, timeout, interval).Should(Succeed()) } }) + It("does not advance the top-level AppliedInputSecretHash until the services roll out the rotated input", func() { + // The top-level Nova CR only reports an applied-input hash once + // NovaAPI, NovaScheduler and NovaMetadata have each confirmed + // (via their own AppliedInputSecretHash) that their StatefulSets + // rolled out pods carrying the rotated secret. This is what lets a + // higher-level control-plane CR safely release a rotated secret's + // finalizer only after the credentials are truly in use. + oldHash := GetNova(novaNames.NovaName).Status.AppliedInputSecretHash + Expect(oldHash).NotTo(BeEmpty()) + + topLevelSS := []types.NamespacedName{ + novaNames.APIStatefulSetName, + novaNames.SchedulerStatefulSetName, + novaNames.MetadataStatefulSetName, + } + // Nova only progresses the top-level services once cell0 has + // confirmed it rolled out the rotated input (they depend on cell0's + // API DB access), so cell0's conductor must be resimulated first. + originalHashes := map[string]string{} + for _, ss := range append(topLevelSS, cell0.ConductorStatefulSetName) { + h := GetEnvVarValue( + th.GetStatefulSet(ss).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + Expect(h).NotTo(BeEmpty()) + originalHashes[ss.Name] = h + } + + secretName := types.NamespacedName{Namespace: novaNames.NovaName.Namespace, Name: SecretName} + th.UpdateSecret(secretName, "NovaPassword", []byte("new-service-password")) + + // Until the rotated input has demonstrably rolled out, the + // top-level CR must keep reporting the pre-rotation hash rather + // than optimistically advancing to the new one. + Consistently(func(g Gomega) { + g.Expect(GetNova(novaNames.NovaName).Status.AppliedInputSecretHash).To(Equal(oldHash)) + }, "2s", interval).Should(Succeed()) + + // cell0's conductor must confirm the rollout of the rotated input + // before Nova will reconcile the top-level services (envtest has no + // StatefulSet controller, so this must be simulated). + Eventually(func(g Gomega) { + newHash := GetEnvVarValue( + th.GetStatefulSet(cell0.ConductorStatefulSetName).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(originalHashes[cell0.ConductorStatefulSetName.Name])) + }, timeout, interval).Should(Succeed()) + th.SimulateStatefulSetReplicaReady(cell0.ConductorStatefulSetName) + + // Wait for the controller to propagate the rotated secret into each + // StatefulSet, then simulate the rollout completing. + for _, ss := range topLevelSS { + Eventually(func(g Gomega) { + newHash := GetEnvVarValue( + th.GetStatefulSet(ss).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(originalHashes[ss.Name])) + }, timeout, interval).Should(Succeed()) + th.SimulateStatefulSetReplicaReady(ss) + } + + // Now that every top-level service confirmed the rollout, the Nova + // CR advances its applied-input hash to the rotated value. + Eventually(func(g Gomega) { + newHash := GetNova(novaNames.NovaName).Status.AppliedInputSecretHash + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(oldHash)) + }, timeout, interval).Should(Succeed()) + }) }) It("deletes NovaMetadata if it is disabled", func() { Eventually(func(g Gomega) { diff --git a/test/functional/placement/api_controller_test.go b/test/functional/placement/api_controller_test.go index 871ac77e8..868074dc8 100644 --- a/test/functional/placement/api_controller_test.go +++ b/test/functional/placement/api_controller_test.go @@ -28,6 +28,7 @@ import ( keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" + util "github.com/openstack-k8s-operators/lib-common/modules/common/util" mariadb_test "github.com/openstack-k8s-operators/mariadb-operator/api/test/helpers" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" "github.com/openstack-k8s-operators/nova-operator/internal/placement" @@ -969,6 +970,49 @@ var _ = Describe("PlacementAPI controller", func() { }, timeout, interval).Should(Succeed()) }) + It("does not advance AppliedInputSecretHash until the rotated secret rolls out", func() { + // PlacementAPI has no intermediate child CR, so it folds the leaf + // and parent behaviour together: it records AppliedInputSecretHash + // only once IsReadyForInput confirms - via an uncached read - that + // the Deployment rolled out pods carrying the current CONFIG_HASH. + oldHash := GetPlacementAPI(names.PlacementAPIName).Status.AppliedInputSecretHash + Expect(oldHash).NotTo(BeEmpty()) + + oldConfigHash := GetEnvVarValue( + th.GetDeployment(names.DeploymentName).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + Expect(oldConfigHash).NotTo(BeEmpty()) + + th.UpdateSecret( + types.NamespacedName{Namespace: namespace, Name: SecretName}, + "PlacementPassword", []byte("foobar")) + rotatedSecret := th.GetSecret(types.NamespacedName{Namespace: namespace, Name: SecretName}) + expectedHash, err := util.ObjectHash(rotatedSecret.Data) + Expect(err).NotTo(HaveOccurred()) + + // The rotated input has not rolled out yet (envtest has no + // Deployment controller to bump ObservedGeneration), so the applied + // hash must not advance to the new value. + Consistently(func(g Gomega) { + g.Expect(GetPlacementAPI(names.PlacementAPIName).Status.AppliedInputSecretHash).To(Equal(oldHash)) + }, "2s", interval).Should(Succeed()) + + // Once the controller propagates the rotated secret into the + // Deployment and the rollout is simulated complete, the applied hash + // advances to the rotated value. + Eventually(func(g Gomega) { + newConfigHash := GetEnvVarValue( + th.GetDeployment(names.DeploymentName).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + g.Expect(newConfigHash).NotTo(BeEmpty()) + g.Expect(newConfigHash).NotTo(Equal(oldConfigHash)) + }, timeout, interval).Should(Succeed()) + th.SimulateDeploymentReplicaReady(names.DeploymentName) + + Eventually(func(g Gomega) { + newHash := GetPlacementAPI(names.PlacementAPIName).Status.AppliedInputSecretHash + g.Expect(newHash).To(Equal(expectedHash)) + }, timeout, interval).Should(Succeed()) + }) + It("updates the KeystoneAuthURL if keystone internal endpoint changes", func() { deployment := th.GetDeployment(names.DeploymentName) oldConfigHash := GetEnvVarValue(deployment.Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "")