Skip to content

Commit 54b6029

Browse files
lmicciniclaude
andcommitted
Make cell ordering deterministic to stabilize input secret hash
Spec.CellTemplates is a map, so ranging it to build orderedCellNames yielded a random order each reconcile. expectedInputHash is derived from that order via util.ObjectHash(secretNames), so it flip-flopped between reconciles, churning every child's AppliedInputSecretHash and their Ready conditions. Sort the non-cell0 tail so the ordering, and thus the hash, is stable across reconciles. Harden the secret-rotation functional tests against a convergence race: rotating or clearing any consumed secret changes expectedInputHash, which bumps every child StatefulSet generation. envtest does not advance ObservedGeneration on its own, so a one-shot SimulateReadyOfNovaTopServices could run before the hash-change reconcile landed and leave the children stuck not-ready, so allServicesReady never became true and the old consumer finalizer was never released. Re-simulate the current generation on every poll and allow 2*timeout, matching the robust cell-transport test. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent b1b5500 commit 54b6029

8 files changed

Lines changed: 88 additions & 36 deletions

File tree

api/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.26.3
44

55
require (
66
github.com/google/go-cmp v0.7.0
7-
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260827143655-1b22d997cce7
7+
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce
88
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4
99
github.com/robfig/cron/v3 v3.0.1
1010
k8s.io/api v0.33.13

api/go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,8 @@ github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw
101101
github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
102102
github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I=
103103
github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
104-
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260827143655-1b22d997cce7 h1:0KwtvLJYDUz6Hpl1ILUmkLK7jnuCsJ5/WLUevlPgJLE=
105-
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260827143655-1b22d997cce7/go.mod h1:2nZqLpDUKJR2HzPJPuWGGiXS/nH4PPBwwTuNeKY7QT0=
104+
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce h1:XLbNNe5V6mxfJF+ETXlwgOuMAwVpidXNyBU5HD35AAQ=
105+
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce/go.mod h1:XCImRHgk9LhAWyGEYihqOPubYCNwosqe/toIylOaFAc=
106106
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4 h1:zDxe4n0GPQ+U8VFZyyCoQed+9Xauz4G22J/X6B4gq20=
107107
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:vDcTSyaI7R6tjtZOqonr0hzq8JMOfBhJCWZy0tVjQPw=
108108
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

go.mod

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,13 @@ require (
1010
github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.7
1111
github.com/onsi/ginkgo/v2 v2.32.1
1212
github.com/onsi/gomega v1.42.1
13-
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260827143655-1b22d997cce7
14-
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260822072640-7bd662ad0284
13+
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce
14+
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260805073337-c70c88f59cab
1515
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4
1616
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260824142420-113dc64ecfd4
1717
github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260824142420-113dc64ecfd4
1818
github.com/openstack-k8s-operators/lib-common/modules/users v0.0.0-20260824142420-113dc64ecfd4
19-
github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260822071633-1e127e728e62
19+
github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260803090920-52e0ba098701
2020
github.com/openstack-k8s-operators/nova-operator/api v0.0.0-00010101000000-000000000000
2121
go.uber.org/zap v1.28.0
2222
golang.org/x/exp v0.0.0-20260611194520-c48552f49976
@@ -72,7 +72,7 @@ require (
7272
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
7373
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
7474
github.com/openshift/api v3.9.0+incompatible // indirect
75-
github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260818072803-e18950de3098 // indirect
75+
github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260824142420-113dc64ecfd4 // indirect
7676
github.com/pkg/errors v0.9.1 // indirect
7777
github.com/prometheus/client_golang v1.22.0 // indirect
7878
github.com/prometheus/client_model v0.6.2 // indirect
@@ -147,5 +147,3 @@ replace k8s.io/code-generator => k8s.io/code-generator v0.33.13 //allow-merging
147147
replace k8s.io/component-base => k8s.io/component-base v0.33.13 //allow-merging
148148

149149
replace k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20250627150254-e9823e99808e //allow-merging
150-
151-
replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/lmiccini/lib-common/modules/common v0.0.0-20260819124656-03fdfe41f952 //allow-merging

go.sum

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -122,8 +122,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
122122
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
123123
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
124124
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
125-
github.com/lmiccini/lib-common/modules/common v0.0.0-20260819124656-03fdfe41f952 h1:y9o0jNE889zIW4PIVAl+zu/G/cpbal4A0yfPoKPp91w=
126-
github.com/lmiccini/lib-common/modules/common v0.0.0-20260819124656-03fdfe41f952/go.mod h1:vDcTSyaI7R6tjtZOqonr0hzq8JMOfBhJCWZy0tVjQPw=
127125
github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
128126
github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
129127
github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE=
@@ -142,20 +140,22 @@ github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I=
142140
github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
143141
github.com/openshift/api v0.0.0-20260710141509-36dec0bfafe4 h1:YVY7Mb0AdJBbNgC/Zz9R/ZwhixOmZNA8P+6ZPaDW/0M=
144142
github.com/openshift/api v0.0.0-20260710141509-36dec0bfafe4/go.mod h1:SPLf21TYPipzCO67BURkCfK6dcIIxx0oNRVWaOyRcXM=
145-
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260827143655-1b22d997cce7 h1:0KwtvLJYDUz6Hpl1ILUmkLK7jnuCsJ5/WLUevlPgJLE=
146-
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260827143655-1b22d997cce7/go.mod h1:2nZqLpDUKJR2HzPJPuWGGiXS/nH4PPBwwTuNeKY7QT0=
147-
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260822072640-7bd662ad0284 h1:ERZkiJP+trC/2tiWZ7cVcQSFc3K16mhdVlV+nVy4Vhk=
148-
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260822072640-7bd662ad0284/go.mod h1:UCwCYAGhumXDqnViXuvwd6N+BC4fE5KTpbcJq+PFd40=
143+
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce h1:XLbNNe5V6mxfJF+ETXlwgOuMAwVpidXNyBU5HD35AAQ=
144+
github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260803113523-879ab83a55ce/go.mod h1:XCImRHgk9LhAWyGEYihqOPubYCNwosqe/toIylOaFAc=
145+
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260805073337-c70c88f59cab h1:r1O+mkLoivl3yqljVPJtHYsQ3SeN6ftm7deIt+PR62w=
146+
github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260805073337-c70c88f59cab/go.mod h1:yPuK2VNA98W6kQSo2Qc1lTjGVWJz7iKJveAOlVrgXQQ=
147+
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4 h1:zDxe4n0GPQ+U8VFZyyCoQed+9Xauz4G22J/X6B4gq20=
148+
github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:vDcTSyaI7R6tjtZOqonr0hzq8JMOfBhJCWZy0tVjQPw=
149149
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260824142420-113dc64ecfd4 h1:cYQFMrD6zcVsyCXExm07a6wuVLMJmXSNQNaBK23Lu3M=
150150
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:Q5wcK4PnlDJ55H+M38CFNLzL53OHYiwq81mR/gKtJXs=
151-
github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260818072803-e18950de3098 h1:1yHjBoRwSKTHVDDhgPdAPpBF9kSzw/ued/X+5KCkHyM=
152-
github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260818072803-e18950de3098/go.mod h1:fJ9CCRKpo8mQODfv4UDJ3tjUTbyCznYslXQwCkDaGg0=
151+
github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260824142420-113dc64ecfd4 h1:atoyW+P7wLeLpKdfWG4vtBBErRVg23+0l3ZAM+cHibo=
152+
github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:fJ9CCRKpo8mQODfv4UDJ3tjUTbyCznYslXQwCkDaGg0=
153153
github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260824142420-113dc64ecfd4 h1:n1ve1cxcG8ZAXCHb8SuNOA/+hXm+GYOl2Wp6oig6Mbo=
154154
github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260824142420-113dc64ecfd4/go.mod h1:tT5uXteekJ3lRgFO5/C3i6Q5A7dj40SDy6XbOJhnRuw=
155155
github.com/openstack-k8s-operators/lib-common/modules/users v0.0.0-20260824142420-113dc64ecfd4 h1:zvCfemkF/LjcRNUCbRlnU4tOu71u+TTwf78RrGJTWZU=
156156
github.com/openstack-k8s-operators/lib-common/modules/users v0.0.0-20260824142420-113dc64ecfd4/go.mod h1:jmVVRfwuFQ3fHLIOAkUrtLgOwEL6j45POzS3PBJuwyM=
157-
github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260822071633-1e127e728e62 h1:kPodlXj+hrF6Q3m0VpEz9gBU8QcQXXuRkQbTcDwI00E=
158-
github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260822071633-1e127e728e62/go.mod h1:A1yXU4F8Qs+F29KcgaI3zBvPJjY3OUzlCt7QCY73wAI=
157+
github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260803090920-52e0ba098701 h1:rYJayTvo7sE3F1NsjLqbRzd/F9RjymfpCGvqYZrj/q8=
158+
github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20260803090920-52e0ba098701/go.mod h1:e/G0+fuofGEMmE1v6RWNVn9cImJosyQriG3j4NABZwQ=
159159
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
160160
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
161161
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

internal/controller/nova/nova_controller.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"errors"
2222
"fmt"
23+
"sort"
2324
"strings"
2425
"time"
2526

@@ -368,12 +369,22 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul
368369
// We need to create a list of cellNames to iterate on and as the map
369370
// iteration order is undefined we need to make sure that cell0 is the
370371
// first to allow dependency handling during ensureCell calls.
372+
// Build a deterministic cell ordering: cell0 first, then the remaining
373+
// cells sorted by name. Spec.CellTemplates is a map, so ranging it directly
374+
// yields a random order each reconcile. Several derived values depend on
375+
// this order being stable across reconciles - most importantly the input
376+
// secret hash (util.ObjectHash(secretNames)) that gates secret rotation and
377+
// child readiness. A non-deterministic order makes that hash flip-flop and
378+
// churns the child Ready conditions.
371379
orderedCellNames := []string{novav1.Cell0Name}
380+
otherCellNames := []string{}
372381
for cellName := range instance.Spec.CellTemplates {
373382
if cellName != novav1.Cell0Name {
374-
orderedCellNames = append(orderedCellNames, cellName)
383+
otherCellNames = append(otherCellNames, cellName)
375384
}
376385
}
386+
sort.Strings(otherCellNames)
387+
orderedCellNames = append(orderedCellNames, otherCellNames...)
377388

378389
// Create the Cell DBs. Note that we are not returning on error or if the
379390
// DB creation is still in progress. We move forward with whatever we can

internal/controller/nova/novacell_controller.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -422,6 +422,7 @@ func (r *NovaCellReconciler) ensureConductor(
422422
// out a new input, keep its last reported condition instead of
423423
// flipping it, matching nova's original behavior for the True case.
424424
if c.Status != corev1.ConditionTrue ||
425+
expectedHash == "" ||
425426
conductor.Status.AppliedInputSecretHash == expectedHash {
426427
instance.Status.ConductorServiceReadyCount = conductor.Status.ReadyCount
427428
instance.Status.Conditions.Set(c)
@@ -532,6 +533,7 @@ func (r *NovaCellReconciler) ensureNoVNCProxy(
532533
// out a new input, keep its last reported condition instead of
533534
// flipping it, matching nova's original behavior for the True case.
534535
if c.Status != corev1.ConditionTrue ||
536+
expectedHash == "" ||
535537
novncproxy.Status.AppliedInputSecretHash == expectedHash {
536538
instance.Status.NoVNCPRoxyServiceReadyCount = novncproxy.Status.ReadyCount
537539
instance.Status.Conditions.Set(c)
@@ -671,6 +673,7 @@ func (r *NovaCellReconciler) ensureMetadata(
671673
// out a new input, keep its last reported condition instead of
672674
// flipping it, matching nova's original behavior for the True case.
673675
if c.Status != corev1.ConditionTrue ||
676+
expectedHash == "" ||
674677
metadata.Status.AppliedInputSecretHash == expectedHash {
675678
instance.Status.MetadataServiceReadyCount = metadata.Status.ReadyCount
676679
instance.Status.Conditions.Set(c)
@@ -831,7 +834,7 @@ func (r *NovaCellReconciler) ensureNovaCompute(
831834
}
832835

833836
if novacompute.Generation == novacompute.Status.ObservedGeneration &&
834-
novacompute.Status.AppliedInputSecretHash == expectedHash &&
837+
(expectedHash == "" || novacompute.Status.AppliedInputSecretHash == expectedHash) &&
835838
novacompute.IsReady() {
836839
computeStatus.Deployed = true
837840
}

test/functional/nova/nova_controller_test.go

Lines changed: 36 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2331,15 +2331,23 @@ var _ = Describe("application credentials", func() {
23312331
ContainElement(nova.ACConsumerFinalizer))
23322332
}, timeout, interval).Should(Succeed())
23332333

2334-
SimulateReadyOfNovaTopServices()
2334+
// Rotating the AC secret changes the input secret hash, which bumps
2335+
// the child generations and drives them not-ready until their
2336+
// statefulsets roll out the new config. Re-simulate on every poll so
2337+
// the workloads converge and allServicesReady lets
2338+
// FinalizeSecretRotation release the old AC consumer finalizer.
23352339
Eventually(func(g Gomega) {
2340+
th.SimulateStatefulSetReplicaReady(novaNames.APIStatefulSetName)
2341+
th.SimulateStatefulSetReplicaReady(novaNames.SchedulerStatefulSetName)
2342+
th.SimulateStatefulSetReplicaReady(novaNames.MetadataStatefulSetName)
2343+
th.SimulateStatefulSetReplicaReady(cell0.ConductorStatefulSetName)
23362344
secret := th.GetSecret(types.NamespacedName{
23372345
Namespace: novaNames.NovaName.Namespace,
23382346
Name: acSecretName,
23392347
})
23402348
g.Expect(secret.Finalizers).NotTo(
23412349
ContainElement(nova.ACConsumerFinalizer))
2342-
}, timeout, interval).Should(Succeed())
2350+
}, 2*timeout, interval).Should(Succeed())
23432351

23442352
Eventually(func(g Gomega) {
23452353
n := GetNova(novaNames.NovaName)
@@ -2390,16 +2398,23 @@ var _ = Describe("application credentials", func() {
23902398
g.Expect(k8sClient.Update(ctx, n)).Should(Succeed())
23912399
}, timeout, interval).Should(Succeed())
23922400

2393-
SimulateReadyOfNovaTopServices()
2394-
2401+
// Clearing the AC secret changes the input secret hash, which bumps
2402+
// the child generations and drives them not-ready until their
2403+
// statefulsets roll out the new config. Re-simulate on every poll so
2404+
// the workloads converge and allServicesReady lets
2405+
// FinalizeSecretRotation release the AC consumer finalizer.
23952406
Eventually(func(g Gomega) {
2407+
th.SimulateStatefulSetReplicaReady(novaNames.APIStatefulSetName)
2408+
th.SimulateStatefulSetReplicaReady(novaNames.SchedulerStatefulSetName)
2409+
th.SimulateStatefulSetReplicaReady(novaNames.MetadataStatefulSetName)
2410+
th.SimulateStatefulSetReplicaReady(cell0.ConductorStatefulSetName)
23962411
secret := th.GetSecret(types.NamespacedName{
23972412
Namespace: novaNames.NovaName.Namespace,
23982413
Name: acSecretName,
23992414
})
24002415
g.Expect(secret.Finalizers).NotTo(
24012416
ContainElement(nova.ACConsumerFinalizer))
2402-
}, timeout, interval).Should(Succeed())
2417+
}, 2*timeout, interval).Should(Succeed())
24032418

24042419
Eventually(func(g Gomega) {
24052420
n := GetNova(novaNames.NovaName)
@@ -2611,8 +2626,11 @@ var _ = Describe("Nova controller - transport URL secret rotation", func() {
26112626

26122627
// The old secret's finalizer is only released once every Nova
26132628
// sub-service has rolled out the new transport config and reports
2614-
// ready (the allServicesReady guard). Re-simulate each StatefulSet
2615-
// as ready and poke a reconcile so the guarded release can fire.
2629+
// ready (the allServicesReady guard). The re-render bumps each
2630+
// StatefulSet generation and envtest does not advance
2631+
// ObservedGeneration on its own, and the re-render can lag behind
2632+
// the first poll, so re-simulate the current generation and poke a
2633+
// reconcile on every iteration until the finalizer has moved.
26162634
Eventually(func(g Gomega) {
26172635
th.SimulateStatefulSetReplicaReady(novaNames.APIStatefulSetName)
26182636
th.SimulateStatefulSetReplicaReady(novaNames.SchedulerStatefulSetName)
@@ -2624,18 +2642,16 @@ var _ = Describe("Nova controller - transport URL secret rotation", func() {
26242642
}
26252643
n.Annotations["test-reconcile-trigger"] = fmt.Sprintf("%d", time.Now().UnixNano())
26262644
g.Expect(k8sClient.Update(ctx, n)).To(Succeed())
2627-
}, timeout, interval).Should(Succeed())
26282645

2629-
Eventually(func(g Gomega) {
26302646
secret := th.GetSecret(types.NamespacedName{
26312647
Namespace: novaNames.NovaName.Namespace,
26322648
Name: oldSecretName,
26332649
})
26342650
g.Expect(secret.Finalizers).NotTo(
26352651
ContainElement(nova.TransportConsumerFinalizer))
2636-
n := GetNova(novaNames.NovaName)
2652+
n = GetNova(novaNames.NovaName)
26372653
g.Expect(n.Status.TransportURLSecret).To(Equal(newSecretName))
2638-
}, timeout, interval).Should(Succeed())
2654+
}, 2*timeout, interval).Should(Succeed())
26392655
})
26402656

26412657
It("should move the finalizer from the old to the new secret on notification transport rotation", func() {
@@ -2697,6 +2713,13 @@ var _ = Describe("Nova controller - transport URL secret rotation", func() {
26972713
ContainElement(nova.TransportConsumerFinalizer))
26982714
}, timeout, interval).Should(Succeed())
26992715

2716+
// Rotating the notifications secret re-renders every top-level
2717+
// service config, bumping each StatefulSet generation. envtest does
2718+
// not advance ObservedGeneration on its own, so the services only
2719+
// report they have applied the new input once we simulate the new
2720+
// generation as Ready. The re-render may lag behind the first poll,
2721+
// so re-simulate the current generation on every iteration until the
2722+
// consumer finalizer has moved to the rotated secret.
27002723
Eventually(func(g Gomega) {
27012724
th.SimulateStatefulSetReplicaReady(novaNames.APIStatefulSetName)
27022725
th.SimulateStatefulSetReplicaReady(novaNames.SchedulerStatefulSetName)
@@ -2708,18 +2731,16 @@ var _ = Describe("Nova controller - transport URL secret rotation", func() {
27082731
}
27092732
n.Annotations["test-reconcile-trigger"] = fmt.Sprintf("%d", time.Now().UnixNano())
27102733
g.Expect(k8sClient.Update(ctx, n)).To(Succeed())
2711-
}, timeout, interval).Should(Succeed())
27122734

2713-
Eventually(func(g Gomega) {
27142735
secret := th.GetSecret(types.NamespacedName{
27152736
Namespace: novaNames.NovaName.Namespace,
27162737
Name: oldNotifSecretName,
27172738
})
27182739
g.Expect(secret.Finalizers).NotTo(
27192740
ContainElement(nova.TransportConsumerFinalizer))
2720-
n := GetNova(novaNames.NovaName)
2741+
n = GetNova(novaNames.NovaName)
27212742
g.Expect(n.Status.NotificationsTransportURLSecret).To(Equal(newNotifSecretName))
2722-
}, timeout, interval).Should(Succeed())
2743+
}, 2*timeout, interval).Should(Succeed())
27232744
})
27242745

27252746
})

test/functional/nova/reconfiguration_test.go

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -818,6 +818,25 @@ var _ = Describe("Nova reconfiguration", func() {
818818
secretName := types.NamespacedName{Namespace: novaNames.NovaName.Namespace, Name: SecretName}
819819
th.UpdateSecret(secretName, "NovaPassword", []byte("new-service-password"))
820820

821+
// Changing the password regenerates the NoVNCProxy config which
822+
// bumps its StatefulSet generation, so the proxy reports it has not
823+
// yet applied the new input. The cell only regenerates the
824+
// (dataplane) compute-config once the proxy has rolled out the new
825+
// input, so first wait for the proxy config to pick up the new
826+
// password and then simulate the StatefulSet rollout (mirroring the
827+
// transport-URL rotation test).
828+
for _, cmName := range []types.NamespacedName{
829+
cell1.CellNoVNCProxyNameConfigDataName,
830+
cell2.CellNoVNCProxyNameConfigDataName,
831+
} {
832+
Eventually(func(g Gomega) {
833+
conf := string(th.GetSecret(cmName).Data["01-nova.conf"])
834+
g.Expect(conf).Should(ContainSubstring("password = new-service-password"))
835+
}, timeout, interval).Should(Succeed(), fmt.Sprintf("Failed on %s", cmName))
836+
}
837+
th.SimulateStatefulSetReplicaReady(cell1.NoVNCProxyStatefulSetName)
838+
th.SimulateStatefulSetReplicaReady(cell2.NoVNCProxyStatefulSetName)
839+
821840
// Expect that every service config is updated with the new service password
822841
for _, cmName := range []types.NamespacedName{
823842
cell0.ConductorConfigDataName,

0 commit comments

Comments
 (0)