diff --git a/api/bases/cyborg.openstack.org_cyborgapis.yaml b/api/bases/cyborg.openstack.org_cyborgapis.yaml index 55b0cef92..922e55f51 100644 --- a/api/bases/cyborg.openstack.org_cyborgapis.yaml +++ b/api/bases/cyborg.openstack.org_cyborgapis.yaml @@ -501,6 +501,14 @@ spec: status: description: CyborgAPIStatus defines the observed state of CyborgAPI. properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/cyborg.openstack.org_cyborgconductors.yaml b/api/bases/cyborg.openstack.org_cyborgconductors.yaml index 6db92c34c..b1a6eff98 100644 --- a/api/bases/cyborg.openstack.org_cyborgconductors.yaml +++ b/api/bases/cyborg.openstack.org_cyborgconductors.yaml @@ -321,6 +321,14 @@ spec: status: description: CyborgConductorStatus defines the observed state of CyborgConductor. properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/cyborg.openstack.org_cyborgs.yaml b/api/bases/cyborg.openstack.org_cyborgs.yaml index 8c18f1bc9..45dc23ca2 100644 --- a/api/bases/cyborg.openstack.org_cyborgs.yaml +++ b/api/bases/cyborg.openstack.org_cyborgs.yaml @@ -873,6 +873,15 @@ spec: finalizer. Tracked so the controller can remove its finalizer from the old secret when the openstack-operator rotates the reference. type: string + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the top-level input Secret that + both CyborgAPI and CyborgConductor have confirmed applied and rolled + out with. It is only updated once both report readiness for this + input, so a higher-level control-plane CR can correlate Cyborg's + readiness with a specific applied input instead of relying on a + possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_nova.yaml b/api/bases/nova.openstack.org_nova.yaml index bace7230f..471108c80 100644 --- a/api/bases/nova.openstack.org_nova.yaml +++ b/api/bases/nova.openstack.org_nova.yaml @@ -2944,6 +2944,15 @@ spec: finalizer. Tracked so the controller can remove its finalizer from the old secret when the openstack-operator rotates the reference. type: string + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the top-level input Secret that + all of NovaAPI, NovaScheduler, NovaMetadata, and cell0 have confirmed + applied and rolled out with. It is only updated once all of them + report readiness for this input, so a higher-level control-plane CR + can correlate Nova's readiness with a specific applied input instead + of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_novaapis.yaml b/api/bases/nova.openstack.org_novaapis.yaml index 57c4412ed..1ba0ea930 100644 --- a/api/bases/nova.openstack.org_novaapis.yaml +++ b/api/bases/nova.openstack.org_novaapis.yaml @@ -581,6 +581,14 @@ spec: status: description: NovaAPIStatus defines the observed state of NovaAPI properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_novacells.yaml b/api/bases/nova.openstack.org_novacells.yaml index 85c0f0d4c..695647da8 100644 --- a/api/bases/nova.openstack.org_novacells.yaml +++ b/api/bases/nova.openstack.org_novacells.yaml @@ -1617,6 +1617,15 @@ spec: status: description: NovaCellStatus defines the observed state of NovaCell properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the per-cell input Secret that + all of this cell's child services (Conductor, Metadata, NoVNCProxy, + Compute) have confirmed applied and rolled out with. It is only + updated once all of them report readiness for this input, so the + top-level Nova CR can correlate this cell's readiness with a specific + applied input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_novacomputes.yaml b/api/bases/nova.openstack.org_novacomputes.yaml index dd5f3f571..495e341aa 100644 --- a/api/bases/nova.openstack.org_novacomputes.yaml +++ b/api/bases/nova.openstack.org_novacomputes.yaml @@ -370,6 +370,14 @@ spec: status: description: NovaComputeStatus defines the observed state of NovaCompute properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_novaconductors.yaml b/api/bases/nova.openstack.org_novaconductors.yaml index 15b53520e..4a03d79f5 100644 --- a/api/bases/nova.openstack.org_novaconductors.yaml +++ b/api/bases/nova.openstack.org_novaconductors.yaml @@ -411,6 +411,14 @@ spec: status: description: NovaConductorStatus defines the observed state of NovaConductor properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_novametadata.yaml b/api/bases/nova.openstack.org_novametadata.yaml index 8c74082dc..44c5adab6 100644 --- a/api/bases/nova.openstack.org_novametadata.yaml +++ b/api/bases/nova.openstack.org_novametadata.yaml @@ -554,6 +554,14 @@ spec: status: description: NovaMetadataStatus defines the observed state of NovaMetadata properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_novanovncproxies.yaml b/api/bases/nova.openstack.org_novanovncproxies.yaml index 28bbc0816..45a97b100 100644 --- a/api/bases/nova.openstack.org_novanovncproxies.yaml +++ b/api/bases/nova.openstack.org_novanovncproxies.yaml @@ -537,6 +537,14 @@ spec: status: description: NovaNoVNCProxyStatus defines the observed state of NovaNoVNCProxy properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/nova.openstack.org_novaschedulers.yaml b/api/bases/nova.openstack.org_novaschedulers.yaml index e3fa65b23..2af1285ab 100644 --- a/api/bases/nova.openstack.org_novaschedulers.yaml +++ b/api/bases/nova.openstack.org_novaschedulers.yaml @@ -387,6 +387,14 @@ spec: status: description: NovaSchedulerStatus defines the observed state of NovaScheduler properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/bases/placement.openstack.org_placementapis.yaml b/api/bases/placement.openstack.org_placementapis.yaml index c66da0bd4..fd6374a14 100644 --- a/api/bases/placement.openstack.org_placementapis.yaml +++ b/api/bases/placement.openstack.org_placementapis.yaml @@ -562,6 +562,14 @@ spec: finalizer. Tracked so the controller can remove its finalizer from the old secret when the openstack-operator rotates the reference. type: string + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last Deployment rollout that has been confirmed ready. It is only + updated once the Deployment is observed to be running with this input, + so consumers can correlate readiness with a specific applied input + instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/api/cyborg/v1beta1/cyborg_types.go b/api/cyborg/v1beta1/cyborg_types.go index a50f91ed3..57241c7f4 100644 --- a/api/cyborg/v1beta1/cyborg_types.go +++ b/api/cyborg/v1beta1/cyborg_types.go @@ -137,6 +137,14 @@ type CyborgStatus struct { // finalizer. Tracked so the controller can remove its finalizer from the // old secret when the openstack-operator rotates the reference. ApplicationCredentialSecret string `json:"applicationCredentialSecret,omitempty"` + + // AppliedInputSecretHash is the hash of the top-level input Secret that + // both CyborgAPI and CyborgConductor have confirmed applied and rolled + // out with. It is only updated once both report readiness for this + // input, so a higher-level control-plane CR can correlate Cyborg's + // readiness with a specific applied input instead of relying on a + // possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/cyborg/v1beta1/cyborgapi_types.go b/api/cyborg/v1beta1/cyborgapi_types.go index 3b6e594f0..8240e4ae8 100644 --- a/api/cyborg/v1beta1/cyborgapi_types.go +++ b/api/cyborg/v1beta1/cyborgapi_types.go @@ -119,6 +119,13 @@ type CyborgAPIStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/cyborg/v1beta1/cyborgconductor_types.go b/api/cyborg/v1beta1/cyborgconductor_types.go index 3144fd2d5..1c80b2d1a 100644 --- a/api/cyborg/v1beta1/cyborgconductor_types.go +++ b/api/cyborg/v1beta1/cyborgconductor_types.go @@ -107,6 +107,13 @@ type CyborgConductorStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/nova/v1beta1/nova_types.go b/api/nova/v1beta1/nova_types.go index 4c21caf83..6abee79f7 100644 --- a/api/nova/v1beta1/nova_types.go +++ b/api/nova/v1beta1/nova_types.go @@ -191,6 +191,14 @@ type NovaStatus struct { // finalizer. Tracked so the controller can remove its finalizer from the // old secret when the openstack-operator rotates the reference. ApplicationCredentialSecret string `json:"applicationCredentialSecret,omitempty"` + + // AppliedInputSecretHash is the hash of the top-level input Secret that + // all of NovaAPI, NovaScheduler, NovaMetadata, and cell0 have confirmed + // applied and rolled out with. It is only updated once all of them + // report readiness for this input, so a higher-level control-plane CR + // can correlate Nova's readiness with a specific applied input instead + // of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/nova/v1beta1/novaapi_types.go b/api/nova/v1beta1/novaapi_types.go index 0ed53cda8..453674b72 100644 --- a/api/nova/v1beta1/novaapi_types.go +++ b/api/nova/v1beta1/novaapi_types.go @@ -207,6 +207,13 @@ type NovaAPIStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/nova/v1beta1/novacell_types.go b/api/nova/v1beta1/novacell_types.go index 7b35e9ddf..e0253b54b 100644 --- a/api/nova/v1beta1/novacell_types.go +++ b/api/nova/v1beta1/novacell_types.go @@ -280,6 +280,14 @@ type NovaCellStatus struct { // then the controller has not processed the latest changes injected by // the openstack-operator in the top-level CR (e.g. the ContainerImage) ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // AppliedInputSecretHash is the hash of the per-cell input Secret that + // all of this cell's child services (Conductor, Metadata, NoVNCProxy, + // Compute) have confirmed applied and rolled out with. It is only + // updated once all of them report readiness for this input, so the + // top-level Nova CR can correlate this cell's readiness with a specific + // applied input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/nova/v1beta1/novacompute_types.go b/api/nova/v1beta1/novacompute_types.go index 54a32cdd5..627886ddf 100644 --- a/api/nova/v1beta1/novacompute_types.go +++ b/api/nova/v1beta1/novacompute_types.go @@ -168,6 +168,13 @@ type NovaComputeStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } // NovaComputeCellStatus defines state of NovaCompute in cell diff --git a/api/nova/v1beta1/novaconductor_types.go b/api/nova/v1beta1/novaconductor_types.go index 0d4e7f2b1..731f29012 100644 --- a/api/nova/v1beta1/novaconductor_types.go +++ b/api/nova/v1beta1/novaconductor_types.go @@ -183,6 +183,13 @@ type NovaConductorStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true @@ -238,7 +245,7 @@ func NewNovaConductorSpec( Resources: novaCell.ConductorServiceTemplate.Resources, NetworkAttachments: novaCell.ConductorServiceTemplate.NetworkAttachments, }, - Override: novaCell.ConductorServiceTemplate.Override, + Override: novaCell.ConductorServiceTemplate.Override, KeystoneAuthURL: novaCell.KeystoneAuthURL, ServiceUser: novaCell.ServiceUser, Region: novaCell.Region, diff --git a/api/nova/v1beta1/novametadata_types.go b/api/nova/v1beta1/novametadata_types.go index 851f0315b..d91cf3804 100644 --- a/api/nova/v1beta1/novametadata_types.go +++ b/api/nova/v1beta1/novametadata_types.go @@ -225,6 +225,13 @@ type NovaMetadataStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/nova/v1beta1/novanovncproxy_types.go b/api/nova/v1beta1/novanovncproxy_types.go index 9b7575b33..165c8d265 100644 --- a/api/nova/v1beta1/novanovncproxy_types.go +++ b/api/nova/v1beta1/novanovncproxy_types.go @@ -201,6 +201,13 @@ type NovaNoVNCProxyStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/nova/v1beta1/novascheduler_types.go b/api/nova/v1beta1/novascheduler_types.go index c7e9721d4..5962769db 100644 --- a/api/nova/v1beta1/novascheduler_types.go +++ b/api/nova/v1beta1/novascheduler_types.go @@ -173,6 +173,13 @@ type NovaSchedulerStatus struct { // LastAppliedTopology - the last applied Topology LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last StatefulSet rollout that has been confirmed ready. It is only + // updated once the StatefulSet is observed to be running with this + // input, so a parent CR can correlate readiness with a specific applied + // input instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/placement/v1beta1/api_types.go b/api/placement/v1beta1/api_types.go index 5a7e5b3d7..402a1c6e5 100644 --- a/api/placement/v1beta1/api_types.go +++ b/api/placement/v1beta1/api_types.go @@ -18,7 +18,7 @@ package v1beta1 import ( topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" - condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/probes" "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" @@ -191,6 +191,13 @@ type PlacementAPIStatus struct { // finalizer. Tracked so the controller can remove its finalizer from the // old secret when the openstack-operator rotates the reference. ApplicationCredentialSecret string `json:"applicationCredentialSecret,omitempty"` + + // AppliedInputSecretHash is the hash of the input Secret consumed by the + // last Deployment rollout that has been confirmed ready. It is only + // updated once the Deployment is observed to be running with this input, + // so consumers can correlate readiness with a specific applied input + // instead of relying on a possibly stale ReadyCondition. + AppliedInputSecretHash string `json:"appliedInputSecretHash,omitempty"` } // PlacementAPI is the Schema for the placementapis API diff --git a/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml b/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml index 55b0cef92..922e55f51 100644 --- a/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml +++ b/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml @@ -501,6 +501,14 @@ spec: status: description: CyborgAPIStatus defines the observed state of CyborgAPI. properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml b/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml index 6db92c34c..b1a6eff98 100644 --- a/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml +++ b/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml @@ -321,6 +321,14 @@ spec: status: description: CyborgConductorStatus defines the observed state of CyborgConductor. properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/cyborg.openstack.org_cyborgs.yaml b/config/crd/bases/cyborg.openstack.org_cyborgs.yaml index 8c18f1bc9..45dc23ca2 100644 --- a/config/crd/bases/cyborg.openstack.org_cyborgs.yaml +++ b/config/crd/bases/cyborg.openstack.org_cyborgs.yaml @@ -873,6 +873,15 @@ spec: finalizer. Tracked so the controller can remove its finalizer from the old secret when the openstack-operator rotates the reference. type: string + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the top-level input Secret that + both CyborgAPI and CyborgConductor have confirmed applied and rolled + out with. It is only updated once both report readiness for this + input, so a higher-level control-plane CR can correlate Cyborg's + readiness with a specific applied input instead of relying on a + possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_nova.yaml b/config/crd/bases/nova.openstack.org_nova.yaml index bace7230f..471108c80 100644 --- a/config/crd/bases/nova.openstack.org_nova.yaml +++ b/config/crd/bases/nova.openstack.org_nova.yaml @@ -2944,6 +2944,15 @@ spec: finalizer. Tracked so the controller can remove its finalizer from the old secret when the openstack-operator rotates the reference. type: string + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the top-level input Secret that + all of NovaAPI, NovaScheduler, NovaMetadata, and cell0 have confirmed + applied and rolled out with. It is only updated once all of them + report readiness for this input, so a higher-level control-plane CR + can correlate Nova's readiness with a specific applied input instead + of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_novaapis.yaml b/config/crd/bases/nova.openstack.org_novaapis.yaml index 57c4412ed..1ba0ea930 100644 --- a/config/crd/bases/nova.openstack.org_novaapis.yaml +++ b/config/crd/bases/nova.openstack.org_novaapis.yaml @@ -581,6 +581,14 @@ spec: status: description: NovaAPIStatus defines the observed state of NovaAPI properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_novacells.yaml b/config/crd/bases/nova.openstack.org_novacells.yaml index 85c0f0d4c..695647da8 100644 --- a/config/crd/bases/nova.openstack.org_novacells.yaml +++ b/config/crd/bases/nova.openstack.org_novacells.yaml @@ -1617,6 +1617,15 @@ spec: status: description: NovaCellStatus defines the observed state of NovaCell properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the per-cell input Secret that + all of this cell's child services (Conductor, Metadata, NoVNCProxy, + Compute) have confirmed applied and rolled out with. It is only + updated once all of them report readiness for this input, so the + top-level Nova CR can correlate this cell's readiness with a specific + applied input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_novacomputes.yaml b/config/crd/bases/nova.openstack.org_novacomputes.yaml index dd5f3f571..495e341aa 100644 --- a/config/crd/bases/nova.openstack.org_novacomputes.yaml +++ b/config/crd/bases/nova.openstack.org_novacomputes.yaml @@ -370,6 +370,14 @@ spec: status: description: NovaComputeStatus defines the observed state of NovaCompute properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_novaconductors.yaml b/config/crd/bases/nova.openstack.org_novaconductors.yaml index 15b53520e..4a03d79f5 100644 --- a/config/crd/bases/nova.openstack.org_novaconductors.yaml +++ b/config/crd/bases/nova.openstack.org_novaconductors.yaml @@ -411,6 +411,14 @@ spec: status: description: NovaConductorStatus defines the observed state of NovaConductor properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_novametadata.yaml b/config/crd/bases/nova.openstack.org_novametadata.yaml index 8c74082dc..44c5adab6 100644 --- a/config/crd/bases/nova.openstack.org_novametadata.yaml +++ b/config/crd/bases/nova.openstack.org_novametadata.yaml @@ -554,6 +554,14 @@ spec: status: description: NovaMetadataStatus defines the observed state of NovaMetadata properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_novanovncproxies.yaml b/config/crd/bases/nova.openstack.org_novanovncproxies.yaml index 28bbc0816..45a97b100 100644 --- a/config/crd/bases/nova.openstack.org_novanovncproxies.yaml +++ b/config/crd/bases/nova.openstack.org_novanovncproxies.yaml @@ -537,6 +537,14 @@ spec: status: description: NovaNoVNCProxyStatus defines the observed state of NovaNoVNCProxy properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/nova.openstack.org_novaschedulers.yaml b/config/crd/bases/nova.openstack.org_novaschedulers.yaml index e3fa65b23..2af1285ab 100644 --- a/config/crd/bases/nova.openstack.org_novaschedulers.yaml +++ b/config/crd/bases/nova.openstack.org_novaschedulers.yaml @@ -387,6 +387,14 @@ spec: status: description: NovaSchedulerStatus defines the observed state of NovaScheduler properties: + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last StatefulSet rollout that has been confirmed ready. It is only + updated once the StatefulSet is observed to be running with this + input, so a parent CR can correlate readiness with a specific applied + input instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/config/crd/bases/placement.openstack.org_placementapis.yaml b/config/crd/bases/placement.openstack.org_placementapis.yaml index c66da0bd4..fd6374a14 100644 --- a/config/crd/bases/placement.openstack.org_placementapis.yaml +++ b/config/crd/bases/placement.openstack.org_placementapis.yaml @@ -562,6 +562,14 @@ spec: finalizer. Tracked so the controller can remove its finalizer from the old secret when the openstack-operator rotates the reference. type: string + appliedInputSecretHash: + description: |- + AppliedInputSecretHash is the hash of the input Secret consumed by the + last Deployment rollout that has been confirmed ready. It is only + updated once the Deployment is observed to be running with this input, + so consumers can correlate readiness with a specific applied input + instead of relying on a possibly stale ReadyCondition. + type: string conditions: description: Conditions items: diff --git a/internal/common/reconciler.go b/internal/common/reconciler.go index 09b077779..968aa1119 100644 --- a/internal/common/reconciler.go +++ b/internal/common/reconciler.go @@ -28,7 +28,11 @@ import ( // ReconcilerBase provides a common set of clients scheme and loggers for all reconcilers. type ReconcilerBase struct { - Client client.Client + Client client.Client + // APIReader is an uncached client.Reader, used where reads must bypass + // the informer cache (e.g. to confirm a workload has actually rolled out + // with a specific input before reporting readiness). + APIReader client.Reader Kclient kubernetes.Interface Scheme *runtime.Scheme RequeueTimeout time.Duration @@ -51,6 +55,7 @@ func NewReconcilerBase( ) ReconcilerBase { return ReconcilerBase{ Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), Scheme: mgr.GetScheme(), Kclient: kclient, RequeueTimeout: time.Duration(5) * time.Second, diff --git a/internal/controller/cyborg/cyborg_controller.go b/internal/controller/cyborg/cyborg_controller.go index c5e9d98b6..c739cc530 100644 --- a/internal/controller/cyborg/cyborg_controller.go +++ b/internal/controller/cyborg/cyborg_controller.go @@ -327,7 +327,17 @@ func (r *CyborgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res // // Create sub-level secret with required configuration // - _, err = r.createSubLevelSecret(ctx, h, instance, transporturlSecret, inputSecret, db, acData, keystoneAuthURL, keystoneRegion) + subLevelSecretName, err := r.createSubLevelSecret(ctx, h, instance, transporturlSecret, inputSecret, db, acData, keystoneAuthURL, keystoneRegion) + if err != nil { + return ctrl.Result{}, err + } + subLevelSecretObj := &corev1.Secret{} + if err := r.APIReader.Get( + ctx, types.NamespacedName{Name: subLevelSecretName, Namespace: instance.Namespace}, subLevelSecretObj, + ); err != nil { + return ctrl.Result{}, err + } + subLevelSecretHash, err := util.ObjectHash(subLevelSecretObj.Data) if err != nil { return ctrl.Result{}, err } @@ -383,7 +393,7 @@ func (r *CyborgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res // // Create CyborgAPI sub-CR // - ctrlResult, err = r.ensureCyborgAPI(ctx, h, instance, serviceLabels) + ctrlResult, apiInputApplied, err := r.ensureCyborgAPI(ctx, h, instance, serviceLabels, subLevelSecretHash) if err != nil { return ctrl.Result{}, err } else if (ctrlResult != ctrl.Result{}) { @@ -393,13 +403,17 @@ func (r *CyborgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res // // Create CyborgConductor sub-CR // - ctrlResult, err = r.ensureConductor(ctx, h, instance, serviceLabels) + ctrlResult, conductorInputApplied, err := r.ensureConductor(ctx, h, instance, serviceLabels, subLevelSecretHash) if err != nil { return ctrl.Result{}, err } else if (ctrlResult != ctrl.Result{}) { return ctrlResult, nil } + if apiInputApplied && conductorInputApplied { + instance.Status.AppliedInputSecretHash = subLevelSecretHash + } + // // Remove finalizers from unused MariaDBAccount records // @@ -939,7 +953,8 @@ func (r *CyborgReconciler) ensureCyborgAPI( h *helper.Helper, instance *cyborgv1beta1.Cyborg, serviceLabels map[string]string, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) Log.Info(fmt.Sprintf("Reconciling CyborgAPI for '%s'", instance.Name)) @@ -979,7 +994,7 @@ func (r *CyborgReconciler) ensureCyborgAPI( condition.SeverityWarning, cyborgv1beta1.CyborgAPIReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { Log.Info(fmt.Sprintf("CyborgAPI CR %s - %s", apiName, op)) @@ -988,10 +1003,13 @@ func (r *CyborgReconciler) ensureCyborgAPI( apiObj := &cyborgv1beta1.CyborgAPI{} err = h.GetClient().Get(ctx, types.NamespacedName{Name: apiName, Namespace: instance.Namespace}, apiObj) if err != nil { - return ctrl.Result{}, err + return ctrl.Result{}, false, err } - if apiObj.IsReady() { + inputApplied := apiObj.Generation == apiObj.Status.ObservedGeneration && + apiObj.Status.AppliedInputSecretHash == expectedInputSecretHash && + apiObj.IsReady() + if inputApplied { instance.Status.APIServiceReadyCount = apiObj.Status.ReadyCount instance.Status.Conditions.MarkTrue(cyborgv1beta1.CyborgAPIReadyCondition, condition.DeploymentReadyMessage) } else { @@ -1002,7 +1020,7 @@ func (r *CyborgReconciler) ensureCyborgAPI( condition.DeploymentReadyRunningMessage)) } - return ctrl.Result{}, nil + return ctrl.Result{}, inputApplied, nil } func (r *CyborgReconciler) ensureConductor( @@ -1010,7 +1028,8 @@ func (r *CyborgReconciler) ensureConductor( h *helper.Helper, instance *cyborgv1beta1.Cyborg, serviceLabels map[string]string, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) Log.Info(fmt.Sprintf("Reconciling CyborgConductor for '%s'", instance.Name)) @@ -1053,7 +1072,7 @@ func (r *CyborgReconciler) ensureConductor( condition.SeverityWarning, cyborgv1beta1.CyborgConductorReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { Log.Info(fmt.Sprintf("CyborgConductor CR %s - %s", conductorName, op)) @@ -1062,10 +1081,13 @@ func (r *CyborgReconciler) ensureConductor( conductorObj := &cyborgv1beta1.CyborgConductor{} err = h.GetClient().Get(ctx, types.NamespacedName{Name: conductorName, Namespace: instance.Namespace}, conductorObj) if err != nil { - return ctrl.Result{}, err + return ctrl.Result{}, false, err } - if conductorObj.IsReady() { + inputApplied := conductorObj.Generation == conductorObj.Status.ObservedGeneration && + conductorObj.Status.AppliedInputSecretHash == expectedInputSecretHash && + conductorObj.IsReady() + if inputApplied { instance.Status.ConductorServiceReadyCount = conductorObj.Status.ReadyCount instance.Status.Conditions.MarkTrue(cyborgv1beta1.CyborgConductorReadyCondition, condition.DeploymentReadyMessage) } else { @@ -1076,7 +1098,7 @@ func (r *CyborgReconciler) ensureConductor( condition.DeploymentReadyRunningMessage)) } - return ctrl.Result{}, nil + return ctrl.Result{}, inputApplied, nil } // SetupWithManager sets up the controller with the Manager. diff --git a/internal/controller/cyborg/cyborgapi_controller.go b/internal/controller/cyborg/cyborgapi_controller.go index 1c4701b0e..09ce59ec5 100644 --- a/internal/controller/cyborg/cyborgapi_controller.go +++ b/internal/controller/cyborg/cyborgapi_controller.go @@ -300,7 +300,16 @@ func (r *CyborgAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( ssObj := ss.GetStatefulSet() instance.Status.ReadyCount = ssObj.Status.ReadyReplicas - if statefulset.IsReady(ssObj) { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: ssObj.Name, Namespace: ssObj.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + if ready { + instance.Status.AppliedInputSecretHash = secretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { instance.Status.Conditions.Set(condition.FalseCondition( diff --git a/internal/controller/cyborg/cyborgconductor_controller.go b/internal/controller/cyborg/cyborgconductor_controller.go index 95add889d..55f74b4cf 100644 --- a/internal/controller/cyborg/cyborgconductor_controller.go +++ b/internal/controller/cyborg/cyborgconductor_controller.go @@ -225,7 +225,16 @@ func (r *CyborgConductorReconciler) Reconcile(ctx context.Context, req ctrl.Requ ssObj := ss.GetStatefulSet() instance.Status.ReadyCount = ssObj.Status.ReadyReplicas - if statefulset.IsReady(ssObj) { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: ssObj.Name, Namespace: ssObj.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + if ready { + instance.Status.AppliedInputSecretHash = secretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { instance.Status.Conditions.Set(condition.FalseCondition( diff --git a/internal/controller/nova/nova_controller.go b/internal/controller/nova/nova_controller.go index 86769f12e..7e02201d5 100644 --- a/internal/controller/nova/nova_controller.go +++ b/internal/controller/nova/nova_controller.go @@ -683,30 +683,42 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul return ctrl.Result{}, err } - result, err = r.ensureAPI( + topLevelSecretObj := &corev1.Secret{} + if err := r.APIReader.Get( + ctx, types.NamespacedName{Name: topLevelSecretName, Namespace: instance.Namespace}, topLevelSecretObj, + ); err != nil { + return ctrl.Result{}, err + } + topLevelSecretHash, err := util.ObjectHash(topLevelSecretObj.Data) + if err != nil { + return ctrl.Result{}, err + } + + result, apiInputApplied, err := r.ensureAPI( ctx, instance, cell0Template, cellDBs[novav1.Cell0Name].Database, apiDB, keystoneInternalAuthURL, keystonePublicAuthURL, region, - topLevelSecretName, + topLevelSecretName, topLevelSecretHash, ) if err != nil { return result, err } - result, err = r.ensureScheduler( + result, schedulerInputApplied, err := r.ensureScheduler( ctx, instance, cell0Template, cellDBs[novav1.Cell0Name].Database, apiDB, keystoneInternalAuthURL, region, - topLevelSecretName, + topLevelSecretName, topLevelSecretHash, ) if err != nil { return result, err } + metadataInputApplied := true if *instance.Spec.MetadataServiceTemplate.Enabled { - result, err = r.ensureMetadata( + result, metadataInputApplied, err = r.ensureMetadata( ctx, instance, cell0Template, cellDBs[novav1.Cell0Name].Database, apiDB, keystoneInternalAuthURL, region, - topLevelSecretName, + topLevelSecretName, topLevelSecretHash, ) if err != nil { return result, err @@ -722,6 +734,10 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul instance.Status.MetadataServiceReadyCount = 0 } + if apiInputApplied && schedulerInputApplied && metadataInputApplied { + instance.Status.AppliedInputSecretHash = topLevelSecretHash + } + // remove finalizers from unused MariaDBAccount records but ONLY if // ensureAPIDB finished if apiDBStatus == nova.DBCompleted { @@ -1301,6 +1317,17 @@ func (r *NovaReconciler) ensureCell( return nil, nova.CellDeploying, err } + cellSecretObj := &corev1.Secret{} + if err := r.APIReader.Get( + ctx, types.NamespacedName{Name: cellSecretName, Namespace: instance.Namespace}, cellSecretObj, + ); err != nil { + return nil, nova.CellFailed, err + } + cellSecretHash, err := util.ObjectHash(cellSecretObj.Data) + if err != nil { + return nil, nova.CellFailed, err + } + cellSpec := novav1.NovaCellSpec{ CellName: cellName, Secret: cellSecretName, @@ -1366,9 +1393,10 @@ func (r *NovaReconciler) ensureCell( Log.Info(fmt.Sprintf("NovaCell %s.", string(op)), "NovaCell.Name", cell.Name) } - if !cell.IsReady() || cell.Generation != cell.Status.ObservedGeneration { - // We wait for the cell to become Ready before we map it in the - // nova_api DB. + if !cell.IsReady() || cell.Generation != cell.Status.ObservedGeneration || + cell.Status.AppliedInputSecretHash != cellSecretHash { + // We wait for the cell to become Ready with this input before we map + // it in the nova_api DB. return cell, nova.CellDeploying, err } configHash, scriptName, configName, err := r.ensureNovaManageJobSecret(ctx, h, instance, @@ -1466,7 +1494,8 @@ func (r *NovaReconciler) ensureAPI( keystonePublicAuthURL string, region string, secretName string, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) // TODO(gibi): Pass down a narrowed secret that only hold @@ -1529,7 +1558,7 @@ func (r *NovaReconciler) ensureAPI( condition.SeverityError, novav1.NovaAPIReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { @@ -1541,12 +1570,23 @@ func (r *NovaReconciler) ensureAPI( // NOTE(gibi): it can be nil if the NovaAPI CR is created but no // reconciliation is run on it to initialize the ReadyCondition yet. if c != nil { + if c.Status == corev1.ConditionTrue && api.Status.AppliedInputSecretHash != expectedInputSecretHash { + // NovaAPI's own ReadyCondition may still read stale-true from + // before an input rotation. Don't propagate that until it has + // confirmed rollout of the currently expected input. + c.Status = corev1.ConditionFalse + c.Severity = condition.SeverityInfo + c.Reason = condition.RequestedReason + c.Message = condition.DeploymentReadyRunningMessage + } instance.Status.Conditions.Set(c) } instance.Status.APIServiceReadyCount = api.Status.ReadyCount } - return ctrl.Result{}, nil + inputApplied := api.Status.Conditions.IsTrue(condition.ReadyCondition) && + api.Status.AppliedInputSecretHash == expectedInputSecretHash + return ctrl.Result{}, inputApplied, nil } func (r *NovaReconciler) ensureScheduler( @@ -1558,7 +1598,8 @@ func (r *NovaReconciler) ensureScheduler( keystoneAuthURL string, region string, secretName string, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) // TODO(gibi): Pass down a narrowed secret that only hold // specific information but also holds user names @@ -1621,7 +1662,7 @@ func (r *NovaReconciler) ensureScheduler( condition.SeverityError, novav1.NovaSchedulerReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { @@ -1634,12 +1675,23 @@ func (r *NovaReconciler) ensureScheduler( // NOTE(gibi): it can be nil if the NovaScheduler CR is created but no // reconciliation is run on it to initialize the ReadyCondition yet. if c != nil { + if c.Status == corev1.ConditionTrue && scheduler.Status.AppliedInputSecretHash != expectedInputSecretHash { + // NovaScheduler's own ReadyCondition may still read stale-true + // from before an input rotation. Don't propagate that until it + // has confirmed rollout of the currently expected input. + c.Status = corev1.ConditionFalse + c.Severity = condition.SeverityInfo + c.Reason = condition.RequestedReason + c.Message = condition.DeploymentReadyRunningMessage + } instance.Status.Conditions.Set(c) } instance.Status.SchedulerServiceReadyCount = scheduler.Status.ReadyCount } - return ctrl.Result{}, nil + inputApplied := scheduler.Status.Conditions.IsTrue(condition.ReadyCondition) && + scheduler.Status.AppliedInputSecretHash == expectedInputSecretHash + return ctrl.Result{}, inputApplied, nil } func (r *NovaReconciler) ensureKeystoneServiceUser( @@ -1941,7 +1993,8 @@ func (r *NovaReconciler) ensureMetadata( keystoneAuthURL string, region string, secretName string, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) // There is a case when the user manually created a NovaMetadata while it // was disabled in the Nova and then tries to enable it in Nova. @@ -1958,7 +2011,7 @@ func (r *NovaReconciler) ensureMetadata( metadata := &novav1.NovaMetadata{} err := r.Client.Get(ctx, metadataName, metadata) if err != nil && !k8s_errors.IsNotFound(err) { - return ctrl.Result{}, err + return ctrl.Result{}, false, err } // If it is not created by us, we don't touch it @@ -1977,7 +2030,7 @@ func (r *NovaReconciler) ensureMetadata( novav1.NovaMetadataReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } // TODO(gibi): Pass down a narrowed secret that only hold @@ -2040,7 +2093,7 @@ func (r *NovaReconciler) ensureMetadata( condition.SeverityError, novav1.NovaMetadataReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { @@ -2052,11 +2105,22 @@ func (r *NovaReconciler) ensureMetadata( // NOTE(gibi): it can be nil if the NovaMetadata CR is created but no // reconciliation is run on it to initialize the ReadyCondition yet. if c != nil { + if c.Status == corev1.ConditionTrue && metadata.Status.AppliedInputSecretHash != expectedInputSecretHash { + // NovaMetadata's own ReadyCondition may still read stale-true + // from before an input rotation. Don't propagate that until it + // has confirmed rollout of the currently expected input. + c.Status = corev1.ConditionFalse + c.Severity = condition.SeverityInfo + c.Reason = condition.RequestedReason + c.Message = condition.DeploymentReadyRunningMessage + } instance.Status.Conditions.Set(c) } instance.Status.MetadataServiceReadyCount = metadata.Status.ReadyCount } - return ctrl.Result{}, nil + inputApplied := metadata.Status.Conditions.IsTrue(condition.ReadyCondition) && + metadata.Status.AppliedInputSecretHash == expectedInputSecretHash + return ctrl.Result{}, inputApplied, nil } // ensureCellMapped makes sure that the cell has a row in the diff --git a/internal/controller/nova/novaapi_controller.go b/internal/controller/nova/novaapi_controller.go index b8d220d87..789fa67c5 100644 --- a/internal/controller/nova/novaapi_controller.go +++ b/internal/controller/nova/novaapi_controller.go @@ -335,7 +335,7 @@ func (r *NovaAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re return result, err } - result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached) + result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached, secret) if (err != nil || result != ctrl.Result{}) { return result, err } @@ -584,6 +584,7 @@ func (r *NovaAPIReconciler) ensureDeployment( inputHash string, annotations map[string]string, memcached *memcachedv1.Memcached, + secret corev1.Secret, ) (ctrl.Result, error) { Log := r.GetLogger(ctx) serviceLabels := getAPIServiceLabels() @@ -666,8 +667,22 @@ func (r *NovaAPIReconciler) ensureDeployment( return ctrl.Result{}, err } - if instance.Status.ReadyCount == *instance.Spec.Replicas && statefulSet.Generation == statefulSet.Status.ObservedGeneration { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: statefulSet.Name, Namespace: statefulSet.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + + if ready { Log.Info("Deployment is ready") + appliedSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.AppliedInputSecretHash = appliedSecretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { Log.Info("Deployment is not ready", "Status", ss.GetStatefulSet().Status) diff --git a/internal/controller/nova/novacell_controller.go b/internal/controller/nova/novacell_controller.go index de611f63d..0ca9e51c4 100644 --- a/internal/controller/nova/novacell_controller.go +++ b/internal/controller/nova/novacell_controller.go @@ -166,13 +166,19 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r // all our input checks out so report InputReady instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) - result, err = r.ensureConductor(ctx, instance) + cellSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + + result, conductorInputApplied, err := r.ensureConductor(ctx, instance, cellSecretHash) if err != nil { return result, err } + metadataInputApplied := true if *instance.Spec.MetadataServiceTemplate.Enabled { - result, err = r.ensureMetadata(ctx, instance) + result, metadataInputApplied, err = r.ensureMetadata(ctx, instance, cellSecretHash) if err != nil { return result, err } @@ -193,7 +199,7 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r // to run discover job only when all computes are deployed and never discovered computeTemplatesHashMap := make(map[string]string) for computeName, computeTemplate := range instance.Spec.NovaComputeTemplates { - computeStatus := r.ensureNovaCompute(ctx, instance, computeTemplate, computeName) + computeStatus := r.ensureNovaCompute(ctx, instance, computeTemplate, computeName, cellSecretHash) instance.Status.NovaComputesStatus[computeName] = computeStatus // We hash the entire compute template to keep track of changes in the number of replicas, // allowing us to discover nodes accordingly @@ -214,6 +220,7 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r } // We need to check if all computes are deployed + allComputesApplied := true if len(instance.Spec.NovaComputeTemplates) == 0 { Log.Info("No nova compute ironic/fake driver service definition in cell") instance.Status.Conditions.Remove(novav1.NovaAllControlPlaneComputesReadyCondition) @@ -228,7 +235,8 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r failedComputes = append(failedComputes, computeName) } } - if len(instance.Spec.NovaComputeTemplates) == len(readyComputes) { + allComputesApplied = len(instance.Spec.NovaComputeTemplates) == len(readyComputes) + if allComputesApplied { instance.Status.Conditions.MarkTrue( novav1.NovaAllControlPlaneComputesReadyCondition, condition.ServiceConfigReadyMessage, ) @@ -247,8 +255,9 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r instance.Status.Hash[novav1.ComputeDiscoverHashKey] = computeTemplatesHash cellHasVNCService := (*instance.Spec.NoVNCProxyServiceTemplate.Enabled) + novncproxyInputApplied := true if cellHasVNCService { - result, err = r.ensureNoVNCProxy(ctx, instance) + result, novncproxyInputApplied, err = r.ensureNoVNCProxy(ctx, instance, cellSecretHash) if err != nil { return result, err } @@ -262,6 +271,10 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r instance.Status.Conditions.Remove(novav1.NovaNoVNCProxyReadyCondition) } + if conductorInputApplied && metadataInputApplied && novncproxyInputApplied && allComputesApplied { + instance.Status.AppliedInputSecretHash = cellSecretHash + } + // We need to wait for the NovaNoVNCProxy to become Ready before we can try // to generate the compute config secret as that needs the endpoint of the // proxy to be included. @@ -359,7 +372,8 @@ func (r *NovaCellReconciler) initConditions( func (r *NovaCellReconciler) ensureConductor( ctx context.Context, instance *novav1.NovaCell, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) conductorSpec := novav1.NewNovaConductorSpec(instance.Spec) @@ -386,7 +400,7 @@ func (r *NovaCellReconciler) ensureConductor( condition.SeverityError, novav1.NovaConductorReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { @@ -403,7 +417,9 @@ func (r *NovaCellReconciler) ensureConductor( } } - return ctrl.Result{}, nil + inputApplied := conductor.Status.Conditions.IsTrue(condition.ReadyCondition) && + conductor.Status.AppliedInputSecretHash == expectedInputSecretHash + return ctrl.Result{}, inputApplied, nil } func getNoVNCProxyName(instance *novav1.NovaCell) types.NamespacedName { @@ -413,7 +429,8 @@ func getNoVNCProxyName(instance *novav1.NovaCell) types.NamespacedName { func (r *NovaCellReconciler) ensureNoVNCProxy( ctx context.Context, instance *novav1.NovaCell, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) // There is a case when the user manually created a NoVNCProxy while it // was disabled in the cell and then tries to enable it in the cell. @@ -430,7 +447,7 @@ func (r *NovaCellReconciler) ensureNoVNCProxy( novncproxy := &novav1.NovaNoVNCProxy{} err := r.Client.Get(ctx, novncproxyName, novncproxy) if err != nil && !k8s_errors.IsNotFound(err) { - return ctrl.Result{}, err + return ctrl.Result{}, false, err } // If it is not created by us, we don't touch it @@ -449,7 +466,7 @@ func (r *NovaCellReconciler) ensureNoVNCProxy( novav1.NovaNoVNCProxyReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } // NoVNCProxy is either not exists, or it exists but owned by us so we can @@ -478,7 +495,7 @@ func (r *NovaCellReconciler) ensureNoVNCProxy( condition.SeverityError, novav1.NovaNoVNCProxyReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { @@ -495,7 +512,9 @@ func (r *NovaCellReconciler) ensureNoVNCProxy( } } - return ctrl.Result{}, nil + inputApplied := novncproxy.Status.Conditions.IsTrue(condition.ReadyCondition) && + novncproxy.Status.AppliedInputSecretHash == expectedInputSecretHash + return ctrl.Result{}, inputApplied, nil } func (r *NovaCellReconciler) ensureNoVNCProxyDeleted( @@ -535,7 +554,8 @@ func (r *NovaCellReconciler) ensureNoVNCProxyDeleted( func (r *NovaCellReconciler) ensureMetadata( ctx context.Context, instance *novav1.NovaCell, -) (ctrl.Result, error) { + expectedInputSecretHash string, +) (ctrl.Result, bool, error) { Log := r.GetLogger(ctx) // There is a case when the user manually created a NovaMetadata while it // was disabled in the NovaCell and then tries to enable it in NovaCell. @@ -552,7 +572,7 @@ func (r *NovaCellReconciler) ensureMetadata( metadata := &novav1.NovaMetadata{} err := r.Client.Get(ctx, metadataName, metadata) if err != nil && !k8s_errors.IsNotFound(err) { - return ctrl.Result{}, err + return ctrl.Result{}, false, err } // If it is not created by us, we don't touch it @@ -571,7 +591,7 @@ func (r *NovaCellReconciler) ensureMetadata( novav1.NovaMetadataReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } metadataSpec := novav1.NewNovaMetadataSpec(instance.Spec) @@ -599,7 +619,7 @@ func (r *NovaCellReconciler) ensureMetadata( condition.SeverityError, novav1.NovaMetadataReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrl.Result{}, false, err } if op != controllerutil.OperationResultNone { @@ -616,7 +636,9 @@ func (r *NovaCellReconciler) ensureMetadata( } } - return ctrl.Result{}, nil + inputApplied := metadata.Status.Conditions.IsTrue(condition.ReadyCondition) && + metadata.Status.AppliedInputSecretHash == expectedInputSecretHash + return ctrl.Result{}, inputApplied, nil } // ensureComputeConfig ensures the the compute config Secret exists and up to @@ -689,6 +711,7 @@ func (r *NovaCellReconciler) ensureNovaCompute( instance *novav1.NovaCell, compute novav1.NovaComputeTemplate, computeName string, + expectedInputSecretHash string, ) novav1.NovaComputeCellStatus { Log := r.GetLogger(ctx) // There is a case when the user manually created a NovaCompute with selected name. @@ -706,6 +729,7 @@ func (r *NovaCellReconciler) ensureNovaCompute( err := r.Client.Get(ctx, fullComputeName, novacompute) var computeStatus novav1.NovaComputeCellStatus if computeStatus, ok := instance.Status.NovaComputesStatus[computeName]; ok { + computeStatus.Deployed = false computeStatus.Errors = false } else { computeStatus = novav1.NovaComputeCellStatus{Deployed: false, Errors: false} @@ -757,8 +781,9 @@ func (r *NovaCellReconciler) ensureNovaCompute( Log.Info(fmt.Sprintf("NovaCompute %s, NovaCompute.Name %s .", string(op), novacompute.Name)) } - if novacompute.Generation == novacompute.Status.ObservedGeneration && novacompute.IsReady() { - // We wait for the novacompute to become Ready before we map it deployed. + if novacompute.Generation == novacompute.Status.ObservedGeneration && novacompute.IsReady() && + novacompute.Status.AppliedInputSecretHash == expectedInputSecretHash { + // We wait for the novacompute to become Ready with this input before we map it deployed. computeStatus.Deployed = true } diff --git a/internal/controller/nova/novacompute_controller.go b/internal/controller/nova/novacompute_controller.go index dfa8f39a6..f0e63c922 100644 --- a/internal/controller/nova/novacompute_controller.go +++ b/internal/controller/nova/novacompute_controller.go @@ -260,7 +260,7 @@ func (r *NovaComputeReconciler) Reconcile(ctx context.Context, req ctrl.Request) return result, err } - result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations) + result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, secret) if (err != nil || result != ctrl.Result{}) { return result, err } @@ -407,6 +407,7 @@ func (r *NovaComputeReconciler) ensureDeployment( instance *novav1.NovaCompute, inputHash string, annotations map[string]string, + secret corev1.Secret, ) (ctrl.Result, error) { Log := r.GetLogger(ctx) serviceLabels := getComputeServiceLabels(instance.Spec.CellName) @@ -482,8 +483,22 @@ func (r *NovaComputeReconciler) ensureDeployment( return ctrl.Result{}, err } - if instance.Status.ReadyCount == *instance.Spec.Replicas && statefulSet.Generation == statefulSet.Status.ObservedGeneration { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: statefulSet.Name, Namespace: statefulSet.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + + if ready { Log.Info("Deployment is ready") + appliedSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.AppliedInputSecretHash = appliedSecretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { Log.Info("Deployment is not ready", "Status", ss.GetStatefulSet().Status) diff --git a/internal/controller/nova/novaconductor_controller.go b/internal/controller/nova/novaconductor_controller.go index 6a91d6bf6..8b9d4e18e 100644 --- a/internal/controller/nova/novaconductor_controller.go +++ b/internal/controller/nova/novaconductor_controller.go @@ -306,7 +306,7 @@ func (r *NovaConductorReconciler) Reconcile(ctx context.Context, req ctrl.Reques return result, err } - result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached) + result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached, secret) if (err != nil || result != ctrl.Result{}) { return result, err } @@ -571,6 +571,7 @@ func (r *NovaConductorReconciler) ensureDeployment( inputHash string, annotations map[string]string, memcached *memcachedv1.Memcached, + secret corev1.Secret, ) (ctrl.Result, error) { serviceLabels := map[string]string{ common.AppSelector: NovaConductorLabelPrefix, @@ -649,8 +650,22 @@ func (r *NovaConductorReconciler) ensureDeployment( return ctrl.Result{}, err } - if instance.Status.ReadyCount == *instance.Spec.Replicas && statefulSet.Generation == statefulSet.Status.ObservedGeneration { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: statefulSet.Name, Namespace: statefulSet.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + + if ready { Log.Info("Deployment is ready") + appliedSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.AppliedInputSecretHash = appliedSecretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { Log.Info("Deployment is not ready", "Status", ss.GetStatefulSet().Status) diff --git a/internal/controller/nova/novametadata_controller.go b/internal/controller/nova/novametadata_controller.go index f06c0f455..0174ed7aa 100644 --- a/internal/controller/nova/novametadata_controller.go +++ b/internal/controller/nova/novametadata_controller.go @@ -328,7 +328,7 @@ func (r *NovaMetadataReconciler) Reconcile(ctx context.Context, req ctrl.Request return result, err } - result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached) + result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached, secret) if (err != nil || result != ctrl.Result{}) { return result, err } @@ -590,6 +590,7 @@ func (r *NovaMetadataReconciler) ensureDeployment( inputHash string, annotations map[string]string, memcached *memcachedv1.Memcached, + secret corev1.Secret, ) (ctrl.Result, error) { Log := r.GetLogger(ctx) serviceLabels := getMetadataServiceLabels(instance.Spec.CellName) @@ -673,8 +674,22 @@ func (r *NovaMetadataReconciler) ensureDeployment( } statefulSet = ss.GetStatefulSet() - if instance.Status.ReadyCount == *instance.Spec.Replicas && statefulSet.Generation == statefulSet.Status.ObservedGeneration { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: statefulSet.Name, Namespace: statefulSet.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + + if ready { Log.Info("Deployment is ready") + appliedSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.AppliedInputSecretHash = appliedSecretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { Log.Info("Deployment is not ready", "Status", ss.GetStatefulSet().Status) diff --git a/internal/controller/nova/novanovncproxy_controller.go b/internal/controller/nova/novanovncproxy_controller.go index f742a92b3..2757343b5 100644 --- a/internal/controller/nova/novanovncproxy_controller.go +++ b/internal/controller/nova/novanovncproxy_controller.go @@ -350,7 +350,7 @@ func (r *NovaNoVNCProxyReconciler) Reconcile(ctx context.Context, req ctrl.Reque return result, err } - result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached) + result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached, secret) if (err != nil || result != ctrl.Result{}) { return result, err } @@ -547,6 +547,7 @@ func (r *NovaNoVNCProxyReconciler) ensureDeployment( inputHash string, annotations map[string]string, memcached *memcachedv1.Memcached, + secret corev1.Secret, ) (ctrl.Result, error) { Log := r.GetLogger(ctx) serviceLabels := getNoVNCProxyServiceLabels(instance.Spec.CellName) @@ -629,8 +630,22 @@ func (r *NovaNoVNCProxyReconciler) ensureDeployment( return ctrl.Result{}, err } - if instance.Status.ReadyCount == *instance.Spec.Replicas && statefulSet.Generation == statefulSet.Status.ObservedGeneration { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: statefulSet.Name, Namespace: statefulSet.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + + if ready { Log.Info("Deployment is ready") + appliedSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.AppliedInputSecretHash = appliedSecretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { Log.Info("Deployment is not ready", "Status", ss.GetStatefulSet().Status) diff --git a/internal/controller/nova/novascheduler_controller.go b/internal/controller/nova/novascheduler_controller.go index 57293653d..37626f77a 100644 --- a/internal/controller/nova/novascheduler_controller.go +++ b/internal/controller/nova/novascheduler_controller.go @@ -310,7 +310,7 @@ func (r *NovaSchedulerReconciler) Reconcile(ctx context.Context, req ctrl.Reques return result, err } - result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached) + result, err = r.ensureDeployment(ctx, h, instance, inputHash, serviceAnnotations, memcached, secret) if (err != nil || result != ctrl.Result{}) { return result, err } @@ -629,6 +629,7 @@ func (r *NovaSchedulerReconciler) ensureDeployment( inputHash string, annotations map[string]string, memcached *memcachedv1.Memcached, + secret corev1.Secret, ) (ctrl.Result, error) { serviceLabels := map[string]string{ common.AppSelector: NovaSchedulerLabelPrefix, @@ -706,8 +707,22 @@ func (r *NovaSchedulerReconciler) ensureDeployment( return ctrl.Result{}, err } - if instance.Status.ReadyCount == *instance.Spec.Replicas && statefulSet.Generation == statefulSet.Status.ObservedGeneration { + ready, err := statefulset.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: statefulSet.Name, Namespace: statefulSet.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + + if ready { Log.Info("Deployment is ready") + appliedSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.AppliedInputSecretHash = appliedSecretHash instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { Log.Info("Deployment is not ready", "Status", ss.GetStatefulSet().Status) diff --git a/internal/controller/placement/api_controller.go b/internal/controller/placement/api_controller.go index 7d7e123a1..8ead3c6cc 100644 --- a/internal/controller/placement/api_controller.go +++ b/internal/controller/placement/api_controller.go @@ -408,6 +408,11 @@ func (r *PlacementAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request Log.Info("Waiting for the Deployment to become Ready before exposing the service in Keystone") return ctrl.Result{}, nil } + appliedSecretHash, err := util.ObjectHash(secret.Data) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.AppliedInputSecretHash = appliedSecretHash err = r.ensureKeystoneServiceUser(ctx, h, instance) if err != nil { return ctrl.Result{}, err @@ -1165,7 +1170,15 @@ func (r *PlacementAPIReconciler) ensureDeployment( // Replicas > ReadyReplicas. // In addition, make sure the controller sees the last Generation // by comparing it with the ObservedGeneration. - if deployment.IsReady(deploy) { + ready, err := deployment.IsReadyForInput( + ctx, r.APIReader, + types.NamespacedName{Name: deploy.Name, Namespace: deploy.Namespace}, + inputHash, + ) + if err != nil { + return ctrl.Result{}, err + } + if ready { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { Log.Info("Deployment is not ready") diff --git a/test/functional/cyborg/cyborg_controller_test.go b/test/functional/cyborg/cyborg_controller_test.go index 97738c441..d3082b7b8 100644 --- a/test/functional/cyborg/cyborg_controller_test.go +++ b/test/functional/cyborg/cyborg_controller_test.go @@ -409,6 +409,70 @@ var _ = Describe("Cyborg controller", func() { corev1.ConditionTrue, ) }) + + It("does not advance AppliedInputSecretHash until the API and conductor roll out the rotated input", func() { + // Bring Cyborg fully Ready first. + mariadb.SimulateMariaDBAccountCompleted(cyborgNames.MariaDBAccountName) + mariadb.SimulateMariaDBDatabaseCompleted(cyborgNames.MariaDBDatabaseName) + infra.SimulateTransportURLReady(cyborgNames.TransportURLName) + keystone.SimulateKeystoneServiceReady(cyborgNames.KeystoneServiceName) + th.SimulateJobSuccess(cyborgNames.DBSyncJobName) + th.SimulateStatefulSetReplicaReady(cyborgNames.APIStatefulSetName) + keystone.SimulateKeystoneEndpointReady(cyborgNames.KeystoneEndpointName) + th.SimulateStatefulSetReplicaReady(cyborgNames.ConductorStatefulSetName) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + + var oldHash string + Eventually(func(g Gomega) { + oldHash = GetCyborg(cyborgNames.CyborgName).Status.AppliedInputSecretHash + g.Expect(oldHash).NotTo(BeEmpty()) + }, timeout, interval).Should(Succeed()) + + apiSS := cyborgNames.APIStatefulSetName + condSS := cyborgNames.ConductorStatefulSetName + oldAPIHash := GetEnvVarValue( + th.GetStatefulSet(apiSS).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + oldCondHash := GetEnvVarValue( + th.GetStatefulSet(condSS).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + Expect(oldAPIHash).NotTo(BeEmpty()) + Expect(oldCondHash).NotTo(BeEmpty()) + + th.UpdateSecret( + types.NamespacedName{Namespace: cyborgNames.CyborgName.Namespace, Name: CyborgSecretName}, + CyborgPasswordSelectorValue, []byte("new-cyborg-password")) + + // The rotated input has not rolled out yet (envtest has no + // StatefulSet controller to bump ObservedGeneration), so the parent + // must keep reporting the pre-rotation hash. + Consistently(func(g Gomega) { + g.Expect(GetCyborg(cyborgNames.CyborgName).Status.AppliedInputSecretHash).To(Equal(oldHash)) + }, "2s", interval).Should(Succeed()) + + // Wait for the rotated secret to reach both StatefulSets, then + // simulate the rollouts completing. + Eventually(func(g Gomega) { + g.Expect(GetEnvVarValue( + th.GetStatefulSet(apiSS).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "")).NotTo(Equal(oldAPIHash)) + g.Expect(GetEnvVarValue( + th.GetStatefulSet(condSS).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "")).NotTo(Equal(oldCondHash)) + }, timeout, interval).Should(Succeed()) + th.SimulateStatefulSetReplicaReady(apiSS) + th.SimulateStatefulSetReplicaReady(condSS) + + // Both children confirmed the rollout, so the Cyborg CR advances its + // applied-input hash to the rotated value. + Eventually(func(g Gomega) { + newHash := GetCyborg(cyborgNames.CyborgName).Status.AppliedInputSecretHash + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(oldHash)) + }, timeout, interval).Should(Succeed()) + }) }) When("Cyborg CR is created with TLS and ApplicationCredentials", func() { @@ -601,6 +665,15 @@ var _ = Describe("Cyborg controller", func() { corev1.ConditionTrue, ) + apiStatefulSet := cyborgNames.APIStatefulSetName + conductorStatefulSet := cyborgNames.ConductorStatefulSetName + oldAPIHash := GetEnvVarValue( + th.GetStatefulSet(apiStatefulSet).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + oldConductorHash := GetEnvVarValue( + th.GetStatefulSet(conductorStatefulSet).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + Expect(oldAPIHash).NotTo(BeEmpty()) + Expect(oldConductorHash).NotTo(BeEmpty()) + newACSecretName := "ac-cyborg-x9y8z-secret" //nolint:gosec // G101 newAppCredSecret := th.CreateSecret( types.NamespacedName{Namespace: cyborgNames.CyborgName.Namespace, Name: newACSecretName}, @@ -636,6 +709,28 @@ var _ = Describe("Cyborg controller", func() { g.Expect(string(subSecret.Data["ACSecret"])).To(Equal(appCredSecretValue + "-rotated")) }, timeout, interval).Should(Succeed()) + // Wait until both sub-CRs propagate the rotated input to their + // StatefulSets before marking the new generations ready. + Eventually(func(g Gomega) { + apiHash := GetEnvVarValue( + th.GetStatefulSet(apiStatefulSet).Spec.Template.Spec.Containers[0].Env, + "CONFIG_HASH", "") + conductorHash := GetEnvVarValue( + th.GetStatefulSet(conductorStatefulSet).Spec.Template.Spec.Containers[0].Env, + "CONFIG_HASH", "") + g.Expect(apiHash).NotTo(BeEmpty()) + g.Expect(apiHash).NotTo(Equal(oldAPIHash)) + g.Expect(conductorHash).NotTo(BeEmpty()) + g.Expect(conductorHash).NotTo(Equal(oldConductorHash)) + }, timeout, interval).Should(Succeed()) + + // The old secret's finalizer must not be removed until the + // sub-CRs have actually rolled out with the rotated input. Since + // envtest has no StatefulSet controller, simulate that rollout + // now to unblock the finalizer removal below. + th.SimulateStatefulSetReplicaReady(conductorStatefulSet) + th.SimulateStatefulSetReplicaReady(apiStatefulSet) + // the finalizer is removed from the old secret Eventually(func(g Gomega) { secret := th.GetSecret(types.NamespacedName{ @@ -686,6 +781,15 @@ var _ = Describe("Cyborg controller", func() { corev1.ConditionTrue, ) + apiStatefulSet := cyborgNames.APIStatefulSetName + conductorStatefulSet := cyborgNames.ConductorStatefulSetName + oldAPIHash := GetEnvVarValue( + th.GetStatefulSet(apiStatefulSet).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + oldConductorHash := GetEnvVarValue( + th.GetStatefulSet(conductorStatefulSet).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + Expect(oldAPIHash).NotTo(BeEmpty()) + Expect(oldConductorHash).NotTo(BeEmpty()) + // clear the AC secret from the spec Eventually(func(g Gomega) { cyborg := GetCyborg(cyborgNames.CyborgName) @@ -693,6 +797,37 @@ var _ = Describe("Cyborg controller", func() { g.Expect(k8sClient.Update(ctx, cyborg)).Should(Succeed()) }, timeout, interval).Should(Succeed()) + // Wait for the sub-CRs to have picked up the updated sub-level + // secret (i.e. reconciled at least once past the spec change) + // before simulating their StatefulSet rollout below. + Eventually(func(g Gomega) { + subSecret := th.GetSecret(cyborgNames.SubLevelSecretName) + g.Expect(subSecret.Data).NotTo(HaveKey("ACID")) + g.Expect(subSecret.Data).NotTo(HaveKey("ACSecret")) + }, timeout, interval).Should(Succeed()) + + // Wait until both sub-CRs propagate the updated input to their + // StatefulSets before marking the new generations ready. + Eventually(func(g Gomega) { + apiHash := GetEnvVarValue( + th.GetStatefulSet(apiStatefulSet).Spec.Template.Spec.Containers[0].Env, + "CONFIG_HASH", "") + conductorHash := GetEnvVarValue( + th.GetStatefulSet(conductorStatefulSet).Spec.Template.Spec.Containers[0].Env, + "CONFIG_HASH", "") + g.Expect(apiHash).NotTo(BeEmpty()) + g.Expect(apiHash).NotTo(Equal(oldAPIHash)) + g.Expect(conductorHash).NotTo(BeEmpty()) + g.Expect(conductorHash).NotTo(Equal(oldConductorHash)) + }, timeout, interval).Should(Succeed()) + + // The old secret's finalizer must not be removed until the + // sub-CRs have actually rolled out with the updated input. Since + // envtest has no StatefulSet controller, simulate that rollout + // now to unblock the finalizer removal below. + th.SimulateStatefulSetReplicaReady(conductorStatefulSet) + th.SimulateStatefulSetReplicaReady(apiStatefulSet) + // the finalizer is removed from the secret Eventually(func(g Gomega) { secret := th.GetSecret(types.NamespacedName{ diff --git a/test/functional/nova/cell_controller_test.go b/test/functional/nova/cell_controller_test.go index 0a57fca94..971b2916f 100644 --- a/test/functional/nova/cell_controller_test.go +++ b/test/functional/nova/cell_controller_test.go @@ -32,6 +32,7 @@ import ( controllers "github.com/openstack-k8s-operators/nova-operator/internal/controller/nova" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -459,6 +460,86 @@ var _ = Describe("NovaCell controller", func() { Expect(cell.Status.NovaComputesStatus).To(HaveKey(ironicComputeName)) }) + It("does not advance AppliedInputSecretHash until all cell services roll out the rotated input", func() { + th.SimulateJobSuccess(cell1.DBSyncJobName) + th.SimulateStatefulSetReplicaReady(cell1.ConductorStatefulSetName) + th.SimulateStatefulSetReplicaReady(cell1.NovaComputeStatefulSetName) + th.SimulateStatefulSetReplicaReady(cell1.NoVNCProxyStatefulSetName) + th.SimulateStatefulSetReplicaReady(cell1.MetadataStatefulSetName) + + th.ExpectCondition( + cell1.CellCRName, + ConditionGetterFunc(NovaCellConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + + oldHash := GetNovaCell(cell1.CellCRName).Status.AppliedInputSecretHash + Expect(oldHash).NotTo(BeEmpty()) + + // conductor, novncproxy and metadata regenerate their config + // independently of readiness; the compute config is only + // regenerated once the NoVNCProxy is Ready again (it needs the + // proxy endpoint), so it rolls out last. + independentSS := []types.NamespacedName{ + cell1.ConductorStatefulSetName, + cell1.NoVNCProxyStatefulSetName, + cell1.MetadataStatefulSetName, + } + originalHashes := map[string]string{} + for _, ss := range append(independentSS, cell1.NovaComputeStatefulSetName) { + h := GetEnvVarValue( + th.GetStatefulSet(ss).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + Expect(h).NotTo(BeEmpty()) + originalHashes[ss.Name] = h + } + + th.UpdateSecret(cell1.InternalCellSecretName, "ServicePassword", []byte("new-service-password")) + + // The rotated input has not rolled out yet, so the cell must keep + // reporting the pre-rotation hash. + Consistently(func(g Gomega) { + g.Expect(GetNovaCell(cell1.CellCRName).Status.AppliedInputSecretHash).To(Equal(oldHash)) + }, "2s", interval).Should(Succeed()) + + for _, ss := range independentSS { + Eventually(func(g Gomega) { + newHash := GetEnvVarValue( + th.GetStatefulSet(ss).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(originalHashes[ss.Name])) + }, timeout, interval).Should(Succeed()) + th.SimulateStatefulSetReplicaReady(ss) + } + + // The compute still reports the previous rollout, so it must no + // longer count as deployed for the current cell input. + Eventually(func(g Gomega) { + cell := GetNovaCell(cell1.CellCRName) + g.Expect(cell.Status.NovaComputesStatus).To(HaveKey(ironicComputeName)) + g.Expect(cell.Status.NovaComputesStatus[ironicComputeName].Deployed).To(BeFalse()) + g.Expect(cell.Status.AppliedInputSecretHash).To(Equal(oldHash)) + }, timeout, interval).Should(Succeed()) + + // Now that the NoVNCProxy has rolled out, the compute config is + // regenerated with the rotated input and the compute rolls out. + Eventually(func(g Gomega) { + newHash := GetEnvVarValue( + th.GetStatefulSet(cell1.NovaComputeStatefulSetName).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(originalHashes[cell1.NovaComputeStatefulSetName.Name])) + }, timeout, interval).Should(Succeed()) + th.SimulateStatefulSetReplicaReady(cell1.NovaComputeStatefulSetName) + + // With every cell service confirming the rollout, the NovaCell + // advances its applied-input hash to the rotated value. + Eventually(func(g Gomega) { + newHash := GetNovaCell(cell1.CellCRName).Status.AppliedInputSecretHash + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(oldHash)) + }, timeout, interval).Should(Succeed()) + }) + It("deletes NoVNCProxy if it is disabled later", func() { th.SimulateJobSuccess(cell1.DBSyncJobName) th.SimulateStatefulSetReplicaReady(cell1.ConductorStatefulSetName) diff --git a/test/functional/nova/reconfiguration_test.go b/test/functional/nova/reconfiguration_test.go index 020e7dff9..0524ae9dc 100644 --- a/test/functional/nova/reconfiguration_test.go +++ b/test/functional/nova/reconfiguration_test.go @@ -866,6 +866,20 @@ var _ = Describe("Nova reconfiguration", func() { secretName := types.NamespacedName{Namespace: novaNames.NovaName.Namespace, Name: SecretName} th.UpdateSecret(secretName, "NovaPassword", []byte("new-service-password")) + // cell1 and cell2 need API DB access, which Nova only grants once + // cell0 has confirmed it rolled out this rotated input. Since + // envtest has no StatefulSet controller, cell0's conductor won't + // self-report that rollout until its StatefulSet is simulated + // ready at the new generation, so do that first to unblock the + // other cells' secret regeneration below. + Eventually(func(g Gomega) { + newHash := GetEnvVarValue( + th.GetStatefulSet(cell0.ConductorStatefulSetName).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(originalHashes[0])) + }, timeout, interval).Should(Succeed()) + th.SimulateStatefulSetReplicaReady(cell0.ConductorStatefulSetName) + // Assert that the config hash is updated in each stateful set for i, ss := range ssNames { Eventually(func(g Gomega) { @@ -876,6 +890,73 @@ var _ = Describe("Nova reconfiguration", func() { }, timeout, interval).Should(Succeed()) } }) + It("does not advance the top-level AppliedInputSecretHash until the services roll out the rotated input", func() { + // The top-level Nova CR only reports an applied-input hash once + // NovaAPI, NovaScheduler and NovaMetadata have each confirmed + // (via their own AppliedInputSecretHash) that their StatefulSets + // rolled out pods carrying the rotated secret. This is what lets a + // higher-level control-plane CR safely release a rotated secret's + // finalizer only after the credentials are truly in use. + oldHash := GetNova(novaNames.NovaName).Status.AppliedInputSecretHash + Expect(oldHash).NotTo(BeEmpty()) + + topLevelSS := []types.NamespacedName{ + novaNames.APIStatefulSetName, + novaNames.SchedulerStatefulSetName, + novaNames.MetadataStatefulSetName, + } + // Nova only progresses the top-level services once cell0 has + // confirmed it rolled out the rotated input (they depend on cell0's + // API DB access), so cell0's conductor must be resimulated first. + originalHashes := map[string]string{} + for _, ss := range append(topLevelSS, cell0.ConductorStatefulSetName) { + h := GetEnvVarValue( + th.GetStatefulSet(ss).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + Expect(h).NotTo(BeEmpty()) + originalHashes[ss.Name] = h + } + + secretName := types.NamespacedName{Namespace: novaNames.NovaName.Namespace, Name: SecretName} + th.UpdateSecret(secretName, "NovaPassword", []byte("new-service-password")) + + // Until the rotated input has demonstrably rolled out, the + // top-level CR must keep reporting the pre-rotation hash rather + // than optimistically advancing to the new one. + Consistently(func(g Gomega) { + g.Expect(GetNova(novaNames.NovaName).Status.AppliedInputSecretHash).To(Equal(oldHash)) + }, "2s", interval).Should(Succeed()) + + // cell0's conductor must confirm the rollout of the rotated input + // before Nova will reconcile the top-level services (envtest has no + // StatefulSet controller, so this must be simulated). + Eventually(func(g Gomega) { + newHash := GetEnvVarValue( + th.GetStatefulSet(cell0.ConductorStatefulSetName).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(originalHashes[cell0.ConductorStatefulSetName.Name])) + }, timeout, interval).Should(Succeed()) + th.SimulateStatefulSetReplicaReady(cell0.ConductorStatefulSetName) + + // Wait for the controller to propagate the rotated secret into each + // StatefulSet, then simulate the rollout completing. + for _, ss := range topLevelSS { + Eventually(func(g Gomega) { + newHash := GetEnvVarValue( + th.GetStatefulSet(ss).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(originalHashes[ss.Name])) + }, timeout, interval).Should(Succeed()) + th.SimulateStatefulSetReplicaReady(ss) + } + + // Now that every top-level service confirmed the rollout, the Nova + // CR advances its applied-input hash to the rotated value. + Eventually(func(g Gomega) { + newHash := GetNova(novaNames.NovaName).Status.AppliedInputSecretHash + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(oldHash)) + }, timeout, interval).Should(Succeed()) + }) }) It("deletes NovaMetadata if it is disabled", func() { Eventually(func(g Gomega) { diff --git a/test/functional/placement/api_controller_test.go b/test/functional/placement/api_controller_test.go index 871ac77e8..868074dc8 100644 --- a/test/functional/placement/api_controller_test.go +++ b/test/functional/placement/api_controller_test.go @@ -28,6 +28,7 @@ import ( keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" + util "github.com/openstack-k8s-operators/lib-common/modules/common/util" mariadb_test "github.com/openstack-k8s-operators/mariadb-operator/api/test/helpers" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" "github.com/openstack-k8s-operators/nova-operator/internal/placement" @@ -969,6 +970,49 @@ var _ = Describe("PlacementAPI controller", func() { }, timeout, interval).Should(Succeed()) }) + It("does not advance AppliedInputSecretHash until the rotated secret rolls out", func() { + // PlacementAPI has no intermediate child CR, so it folds the leaf + // and parent behaviour together: it records AppliedInputSecretHash + // only once IsReadyForInput confirms - via an uncached read - that + // the Deployment rolled out pods carrying the current CONFIG_HASH. + oldHash := GetPlacementAPI(names.PlacementAPIName).Status.AppliedInputSecretHash + Expect(oldHash).NotTo(BeEmpty()) + + oldConfigHash := GetEnvVarValue( + th.GetDeployment(names.DeploymentName).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + Expect(oldConfigHash).NotTo(BeEmpty()) + + th.UpdateSecret( + types.NamespacedName{Namespace: namespace, Name: SecretName}, + "PlacementPassword", []byte("foobar")) + rotatedSecret := th.GetSecret(types.NamespacedName{Namespace: namespace, Name: SecretName}) + expectedHash, err := util.ObjectHash(rotatedSecret.Data) + Expect(err).NotTo(HaveOccurred()) + + // The rotated input has not rolled out yet (envtest has no + // Deployment controller to bump ObservedGeneration), so the applied + // hash must not advance to the new value. + Consistently(func(g Gomega) { + g.Expect(GetPlacementAPI(names.PlacementAPIName).Status.AppliedInputSecretHash).To(Equal(oldHash)) + }, "2s", interval).Should(Succeed()) + + // Once the controller propagates the rotated secret into the + // Deployment and the rollout is simulated complete, the applied hash + // advances to the rotated value. + Eventually(func(g Gomega) { + newConfigHash := GetEnvVarValue( + th.GetDeployment(names.DeploymentName).Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "") + g.Expect(newConfigHash).NotTo(BeEmpty()) + g.Expect(newConfigHash).NotTo(Equal(oldConfigHash)) + }, timeout, interval).Should(Succeed()) + th.SimulateDeploymentReplicaReady(names.DeploymentName) + + Eventually(func(g Gomega) { + newHash := GetPlacementAPI(names.PlacementAPIName).Status.AppliedInputSecretHash + g.Expect(newHash).To(Equal(expectedHash)) + }, timeout, interval).Should(Succeed()) + }) + It("updates the KeystoneAuthURL if keystone internal endpoint changes", func() { deployment := th.GetDeployment(names.DeploymentName) oldConfigHash := GetEnvVarValue(deployment.Spec.Template.Spec.Containers[0].Env, "CONFIG_HASH", "")