fix(helm): detect invalid indexes per server pod

.github/workflows/helm-chart-ci.yml

@@ -349,6 +349,21 @@ jobs:
           expect "$(printf 'review\tsecret')" 'must be printable ASCII'
           expect 'café' 'must be printable ASCII'
 
+      - name: helm test query assertions and shell quoting
+        run: |
+          # Execute the rendered query with the same Groovy version as Server.
+          GROOVY_VERSION=$(python3 - <<'PY_VERSION'
+          import xml.etree.ElementTree as ET
+          pom = ET.parse('pom.xml')
+          print(pom.find('{*}properties/{*}groovy.version').text)
+          PY_VERSION
+          )
+          GROOVY_URL="https://repo.maven.apache.org/maven2/org/apache/groovy/groovy/${GROOVY_VERSION}/groovy-${GROOVY_VERSION}.jar"
+          curl -fsSLo /tmp/groovy.jar "${GROOVY_URL}"
+          curl -fsSLo /tmp/groovy.jar.sha1 "${GROOVY_URL}.sha1"
+          printf '%s  /tmp/groovy.jar\n' "$(cat /tmp/groovy.jar.sha1)" | sha1sum -c -
+          GROOVY_JAR=/tmp/groovy.jar python3 helm/test_helm_hook.py
+
       - name: helm test hook behaviour
         shell: bash
         run: |

helm/hugegraph/README.md

@@ -231,9 +231,16 @@ an existing account; follow [credential rotation](docs/operations.md#credentials
 The chart checks persisted Raft identities and storage/membership changes during
 upgrades; those changes require a coordinated procedure.
 
-`helm --wait` checks readiness; `helm test` checks graph-bound Gremlin on each
-Server Pod. Also verify a fresh Hubble login and graph query. Templates and
-linting alone do not prove live volume, network or application behavior.
+`helm --wait` checks readiness. `helm test` checks graph binding and visible
+associated indexes for `CREATED` status on each Server Pod. Run it after index
+creation/rebuild jobs finish, with no schema changes during the test. On HStore,
+reading an unfinished index can cache its temporary status on other Pods;
+retries do not refresh that state, and indexed queries may keep failing until
+those Pods restart after the jobs finish. It does not prove index query results; add
+[business assertions](docs/operations.md#business-query-assertions)
+with `--set-file server.testGremlin=assertions.groovy`. Also verify a fresh Hubble
+login and graph query. Templates and linting alone do not prove live volume,
+network or application behavior.
 
 Website guides: [Helm deployment](https://hugegraph.apache.org/docs/quickstart/hugegraph/hugegraph-helm/)
 and [Kubernetes operations](https://hugegraph.apache.org/docs/quickstart/hugegraph/hugegraph-helm-operations/).

helm/hugegraph/docs/operations.md

@@ -283,6 +283,51 @@ and `q=1.` are equivalent to `q=0` and `q=1`. Explicit gzip exclusion overrides
 Older images can still force gzip; use a Server image built from current source.
 Updating chart values alone does not change an existing image's behavior.
 
+## Business query assertions
+
+The default `helm test` checks the `server.testGraphAlias` binding (`DEFAULT-hugegraph` by default)
+and requires `CREATED` indexes
+associated with the vertex/edge labels visible on each Pod. It reads those
+indexes by ID, using the same local cache path as queries; it does not refresh
+the full index cache. This does not detect missing/stale label associations,
+orphaned indexes or incorrect index data, and does not prove range-query results.
+Run it after index creation/rebuild jobs finish, with no schema changes during
+the test. Like indexed queries, the check rejects every other status, including
+`CREATING`, `REBUILDING` and `INVALID`. On HStore, reading an unfinished index can
+cache its temporary status on Pods other than the job runner. Later attempts
+reuse that cache; the retry window does not repair it. Indexed queries on those
+Pods may keep failing after the job succeeds; replace affected Server Pods only
+after the jobs finish. This check does not fix cross-Server schema cache consistency.
+
+For a graph with existing test data, provide a read-only Gremlin script that
+asserts the expected result. For example, with 40 existing `person` vertices
+(ages 0–39), save this as `assertions.groovy`, adapting names/results to your data:
+
+```groovy
+def n = graph.traversal().V().hasLabel('person')
+             .has('age', org.apache.tinkerpop.gremlin.process.traversal.P.gte(20))
+             .count().next()
+assert n == 20 : "Expected 20 adults, got ${n}"
+```
+
+```bash
+helm upgrade hugegraph ./helm/hugegraph -n hugegraph --reuse-values \
+  --set-file server.testGremlin=assertions.groovy
+helm test hugegraph -n hugegraph --logs --timeout 5m
+```
+
+In each attempt, the hook checks binding/index state on every Ready Server Pod,
+then runs optional business assertions on Pods where that check succeeds. A failing
+request or assertion reports its Pod IP, check stage and response, and fails
+the hook if no complete attempt succeeds before its deadline. Use
+`kubectl -n hugegraph get pods -o wide` to map IPs to Pod names. Returning a value
+or `false` alone is not a failure: use `assert` or throw an exception. Without a
+script, the hook explicitly reports business assertions as skipped. The hook
+creates no schema/data by default. Custom scripts use the hook's Server credential
+(`admin` when authentication is enabled); read-only execution is not enforced,
+so supply trusted, read-only assertions. The hook does not guarantee the first request
+after startup succeeds; its retries allow transient elections to settle.
+
 ## Troubleshooting
 
 | Symptom | First checks |

helm/hugegraph/templates/tests/test-connection.yaml

@@ -16,6 +16,18 @@
 #
 
 {{ include "hugegraph.resolveValues" . }}
+{{- $smoke := `def ids = [] as Set;
+(graph.schema().getVertexLabels() + graph.schema().getEdgeLabels()).each {
+    ids.addAll(it.indexLabels())
+};
+ids.each {
+    def index = graph.indexLabel(it);
+    if (!index.status().ok()) {
+        throw new IllegalStateException('Index ' + index.name() + ' is ' +
+                                        index.status().name() + ' (CREATED expected)')
+    }
+};
+graph.traversal().V().limit(1).count()` }}
 # Official Helm chart test hook — only runs with: helm test <release>
 # Not an install-time hook. Safe with --wait (no init Job in no-init design).
 apiVersion: v1
@@ -78,7 +90,12 @@ spec:
           PORT="{{ .Values.server.port }}"
           FLOOR="{{ include "hugegraph.server.replicaFloor" . }}"
           DEADLINE_S="${HG_TEST_DEADLINE_S:-150}"
-          BODY={{ dict "gremlin" "graph.traversal().V().limit(1).count()" "aliases" (dict "graph" (.Values.server.testGraphAlias | default "DEFAULT-hugegraph")) | toJson | squote }}
+          BODY='{{ dict "gremlin" $smoke "aliases" (dict "graph" (.Values.server.testGraphAlias | default "DEFAULT-hugegraph")) | toJson | replace "'" "'\"'\"'" }}'
+          {{- if .Values.server.testGremlin }}
+          BUSINESS_BODY='{{ dict "gremlin" .Values.server.testGremlin "aliases" (dict "graph" (.Values.server.testGraphAlias | default "DEFAULT-hugegraph")) | toJson | replace "'" "'\"'\"'" }}'
+          {{- else }}
+          BUSINESS_BODY=""
+          {{- end }}
           {{- if .Values.server.auth.enabled }}
           AUTH_HEADER=$(printf 'admin:%s' "${PASSWORD}" | base64 | tr -d '\r\n')
           request() {
@@ -88,7 +105,7 @@ spec:
           gremlin() {
             printf 'header = "Authorization: Basic %s"\n' "${AUTH_HEADER}" |
               curl --config - -sS --compressed --connect-timeout 3 --max-time 20 \
-                -H 'Content-Type: application/json' -X POST -d "${BODY}" \
+                -H 'Content-Type: application/json' -X POST -d "$2" \
                 -w '\n%{http_code}\n' "$1" 2>&1
           }
           {{- else }}
@@ -97,10 +114,21 @@ spec:
           }
           gremlin() {
             curl -sS --compressed --connect-timeout 3 --max-time 20 \
-              -H 'Content-Type: application/json' -X POST -d "${BODY}" \
+              -H 'Content-Type: application/json' -X POST -d "$2" \
               -w '\n%{http_code}\n' "$1" 2>&1
           }
           {{- end }}
+          check_gremlin() {
+            OUT=$(gremlin "$1" "$3") || true
+            CODE=$(printf '%s\n' "${OUT}" | grep -E '^[0-9][0-9][0-9]$' | tail -1) || CODE=""
+            if [ "${CODE}" = "200" ]; then
+              echo "helm test: $2 OK on $1"
+              return 0
+            fi
+            printf 'helm test: %s failed on %s code=%s body: %.400s\n' \
+              "$2" "$1" "${CODE:-no-response}" "${OUT}"
+            return 1
+          }
           echo "helm test: GET http://${SVC}:${PORT}/versions"
           request "http://${SVC}:${PORT}/versions"
           echo
@@ -115,6 +143,9 @@ spec:
           # Pods, retrying until the deadline so a transient PD election is
           # not misread as a broken Pod.
           echo "helm test: Gremlin on every Server Pod via ${HEADLESS} (floor ${FLOOR})"
+          if [ -z "${BUSINESS_BODY}" ]; then
+            echo "helm test: business assertions skipped (server.testGremlin is empty)"
+          fi
           START=$(date +%s)
           while :; do
             # The headless Service answers with every Ready Pod's addresses,
@@ -135,18 +166,16 @@ spec:
             FAILED=""
             for IP in ${IPS}; do
               case "${IP}" in *:*) TARGET="[${IP}]" ;; *) TARGET="${IP}" ;; esac
-              OUT=$(gremlin "http://${TARGET}:${PORT}/gremlin") || true
-              CODE=$(printf '%s\n' "${OUT}" | grep -E '^[0-9][0-9][0-9]$' | tail -1) || CODE=""
-              if [ "${CODE}" = "200" ]; then
-                echo "helm test: Gremlin OK on ${IP}"
-              else
+              URL="http://${TARGET}:${PORT}/gremlin"
+              if ! check_gremlin "${URL}" "Gremlin binding/index-state" "${BODY}"; then
+                FAILED="${FAILED}${IP}(${CODE:-no-response}) "
+              elif [ -n "${BUSINESS_BODY}" ] &&
+                   ! check_gremlin "${URL}" "business assertion" "${BUSINESS_BODY}"; then
                 FAILED="${FAILED}${IP}(${CODE:-no-response}) "
-                printf 'helm test: Gremlin failed on %s code=%s body: %.400s\n' \
-                  "${IP}" "${CODE:-no-response}" "${OUT}"
               fi
             done
             if [ -z "${FAILED}" ] && [ "${COUNT}" -ge "${FLOOR}" ]; then
-              echo "helm test: OK (${COUNT} Server Pods, all Gremlin-bound)"
+              echo "helm test: OK (${COUNT} Server Pods, all configured checks passed)"
               exit 0
             fi
             if [ $(( $(date +%s) - START )) -ge "${DEADLINE_S}" ]; then

helm/hugegraph/tests/test_hook_gremlin_test.yaml

@@ -91,6 +91,19 @@ tests:
           path: spec.containers[0].command[2]
           pattern: '"graph":"DEFAULT-Demo"'
 
+  - it: defaults a null alias for both index and business checks
+    template: tests/test-connection.yaml
+    set:
+      server.testGraphAlias: null
+      server.testGremlin: "assert graph.traversal().V().count().next() == 40"
+    asserts:
+      - matchRegex:
+          path: spec.containers[0].command[2]
+          pattern: '(?m)^\s*BODY=.*"graph":"DEFAULT-hugegraph"'
+      - matchRegex:
+          path: spec.containers[0].command[2]
+          pattern: '(?m)^\s*BUSINESS_BODY=.*"graph":"DEFAULT-hugegraph"'
+
   - it: floors the per-Pod count at the HPA minimum when HPA is enabled
     template: tests/test-connection.yaml
     set:
@@ -102,6 +115,40 @@ tests:
           path: spec.containers[0].command[2]
           pattern: "FLOOR=\"5\""
 
+  - it: checks linked indexes by ID without refreshing the full index cache
+    template: tests/test-connection.yaml
+    asserts:
+      - matchRegex:
+          path: spec.containers[0].command[2]
+          pattern: "getVertexLabels.*getEdgeLabels"
+      - matchRegex:
+          path: spec.containers[0].command[2]
+          pattern: "graph.indexLabel"
+      - matchRegex:
+          path: spec.containers[0].command[2]
+          pattern: "index.status.*ok"
+      - matchRegex:
+          path: spec.containers[0].command[2]
+          pattern: "CREATED expected"
+      - notMatchRegex:
+          path: spec.containers[0].command[2]
+          pattern: "getIndexLabels"
+      - matchRegex:
+          path: spec.containers[0].command[2]
+          pattern: "business assertions skipped"
+
+  - it: sends optional business assertions on each Pod
+    template: tests/test-connection.yaml
+    set:
+      server.testGremlin: "assert graph.traversal().V().count().next() == 40"
+    asserts:
+      - matchRegex:
+          path: spec.containers[0].command[2]
+          pattern: "assert graph.traversal"
+      - matchRegex:
+          path: spec.containers[0].command[2]
+          pattern: "business assertion"
+
   - it: sends no credentials when authentication is disabled
     template: tests/test-connection.yaml
     set:

helm/hugegraph/values.schema.json

@@ -1173,6 +1173,10 @@
         "testResources": {
           "$ref": "#/definitions/resources"
         },
+        "testGremlin": {
+          "type": "string",
+          "description": "Optional Gremlin assertions run with the hook's Server credential on each Pod. Supply trusted read-only scripts; read-only execution is not enforced."
+        },
         "service": {
           "$ref": "#/definitions/service"
         },

helm/hugegraph/values.yaml

@@ -349,6 +349,10 @@ server:
   # Existing Gremlin graph binding checked on every Server Pod. Select the
   # target graph explicitly; graphSpace does not create or infer this binding.
   testGraphAlias: DEFAULT-hugegraph
+  # Optional Gremlin assertions, run with the hook's Server credential on every Pod.
+  # Supply trusted read-only scripts; read-only execution is not enforced.
+  # Use --set-file server.testGremlin=/path/to/assertions.groovy; assert expected results.
+  testGremlin: ""
   # Resources for the Helm test hook container. Bounded by default so the hook
   # cannot run unlimited on a restricted or quota-managed namespace.
   testResources: