From b3f10cfbd919009ed43d2202f856b81adc867d98 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 13 Oct 2026 10:50:05 -0700 Subject: [PATCH 1/5] feat(kubernetes): add operator allowlist for sandbox pod annotations Add allowed_pod_annotations to the Kubernetes driver config. Sandbox template annotations must match an exact key or a /* entry, and the default empty list rejects all of them. Keys in the openshell.ai and agents.x-k8s.io domains are always rejected. Expose the setting through the server.allowedPodAnnotations Helm value. Signed-off-by: Matthew Grossman --- .../openshell-driver-kubernetes/src/config.rs | 156 +++++++++++++++++- .../openshell-driver-kubernetes/src/driver.rs | 109 ++++++++++++ .../openshell-driver-kubernetes/src/main.rs | 8 + deploy/helm/openshell/README.md | 1 + .../openshell/templates/gateway-config.yaml | 3 + .../openshell/tests/gateway_config_test.yaml | 18 ++ deploy/helm/openshell/values.yaml | 5 + docs/how-it-works/gateways/configuration.mdx | 4 + docs/how-it-works/sandboxes/runtimes.mdx | 1 + 9 files changed, 304 insertions(+), 1 deletion(-) diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 6bd7d93af3..6adaea20f7 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -263,6 +263,11 @@ pub struct KubernetesComputeConfig { /// Applied when a `CreateSandbox` request does not specify one. /// Empty string (default) = omit the field, using the cluster default. pub default_runtime_class_name: String, + /// Pod annotation keys that sandbox templates may set. Each entry is an + /// exact key (`example.com/team`) or a prefix wildcard (`example.com/*`). + /// Empty (default) rejects every caller-supplied pod annotation. Keys in + /// `OpenShell` and Agent Sandbox domains are always rejected. + pub allowed_pod_annotations: Vec, /// Lifetime (seconds) of the projected `ServiceAccount` token kubelet /// writes into each sandbox pod. Used only for the one-shot /// `IssueSandboxToken` bootstrap exchange — the gateway-minted JWT @@ -364,6 +369,7 @@ impl Default for KubernetesComputeConfig { workspace_default_storage_size: DEFAULT_WORKSPACE_STORAGE_SIZE.to_string(), workspace_storage_class: String::new(), default_runtime_class_name: String::new(), + allowed_pod_annotations: Vec::new(), sa_token_ttl_secs: 3600, provider_spiffe_workload_api_socket_path: String::new(), sandbox_uid: None, @@ -379,7 +385,59 @@ impl KubernetesComputeConfig { self.validate_provider_spiffe_workload_api_socket_path()?; self.validate_sandbox_identity_config()?; self.validate_proxy_uid()?; - self.validate_upstream_proxy_config() + self.validate_upstream_proxy_config()?; + self.validate_allowed_pod_annotations() + } + + pub fn validate_allowed_pod_annotations(&self) -> Result<(), String> { + for entry in &self.allowed_pod_annotations { + let key = entry.strip_suffix("/*").unwrap_or(entry); + let wildcard = key.len() != entry.len(); + let (prefix, name) = match key.split_once('/') { + Some((prefix, name)) if !wildcard => (Some(prefix), Some(name)), + Some(_) => { + return Err(format!( + "allowed_pod_annotations entry {entry:?}: wildcard entries must be `/*`" + )); + } + None if wildcard => (Some(key), None), + None => (None, Some(key)), + }; + if let Some(prefix) = prefix + && !is_dns1123_subdomain(prefix) + { + return Err(format!( + "allowed_pod_annotations entry {entry:?}: invalid key prefix {prefix:?}" + )); + } + if let Some(name) = name + && !is_annotation_name(name) + { + return Err(format!( + "allowed_pod_annotations entry {entry:?}: invalid key name {name:?}" + )); + } + if prefix.is_some_and(is_reserved_annotation_domain) { + return Err(format!( + "allowed_pod_annotations entry {entry:?}: domain is reserved for OpenShell and Agent Sandbox" + )); + } + } + Ok(()) + } + + /// Whether a caller-supplied pod annotation key passes the operator allowlist. + #[must_use] + pub fn pod_annotation_allowed(&self, key: &str) -> bool { + let prefix = key.split_once('/').map(|(prefix, _)| prefix); + if prefix.is_some_and(is_reserved_annotation_domain) { + return false; + } + self.allowed_pod_annotations.iter().any(|entry| { + entry + .strip_suffix("/*") + .map_or_else(|| entry == key, |domain| prefix == Some(domain)) + }) } /// Clamp `sa_token_ttl_secs` into the `[MIN_SA_TOKEN_TTL_SECS, @@ -788,6 +846,32 @@ fn is_dns1123_subdomain(value: &str) -> bool { }) } +/// Annotation domains owned by `OpenShell` and Agent Sandbox. Sandbox templates +/// can never set keys in these domains or their subdomains. +const RESERVED_POD_ANNOTATION_DOMAINS: &[&str] = &["openshell.ai", "agents.x-k8s.io"]; + +fn is_reserved_annotation_domain(domain: &str) -> bool { + RESERVED_POD_ANNOTATION_DOMAINS.iter().any(|reserved| { + domain == *reserved + || domain + .strip_suffix(reserved) + .is_some_and(|rest| rest.ends_with('.')) + }) +} + +/// Kubernetes qualified-name segment: at most 63 alphanumeric, `-`, `_`, or +/// `.` characters, beginning and ending with an alphanumeric character. +fn is_annotation_name(value: &str) -> bool { + let bytes = value.as_bytes(); + !bytes.is_empty() + && bytes.len() <= 63 + && bytes.first().is_some_and(u8::is_ascii_alphanumeric) + && bytes.last().is_some_and(u8::is_ascii_alphanumeric) + && bytes + .iter() + .all(|b| b.is_ascii_alphanumeric() || b"-_.".contains(b)) +} + fn validate_provider_spiffe_workload_api_socket_path_value( socket_path: &str, ) -> Result<(), String> { @@ -961,6 +1045,76 @@ mod tests { assert_eq!(cfg.default_runtime_class_name, "nvidia"); } + #[test] + fn allowed_pod_annotations_defaults_to_empty() { + let cfg = KubernetesComputeConfig::default(); + assert!(cfg.allowed_pod_annotations.is_empty()); + assert!(!cfg.pod_annotation_allowed("example.com/owner")); + } + + #[test] + fn allowed_pod_annotations_accepts_exact_and_wildcard_entries() { + let cfg: KubernetesComputeConfig = serde_json::from_value(serde_json::json!({ + "allowed_pod_annotations": ["example.com/*", "team.example.org/owner", "owner"] + })) + .unwrap(); + cfg.validate_allowed_pod_annotations().unwrap(); + assert!(cfg.pod_annotation_allowed("example.com/anything")); + assert!(!cfg.pod_annotation_allowed("sub.example.com/anything")); + assert!(cfg.pod_annotation_allowed("team.example.org/owner")); + assert!(!cfg.pod_annotation_allowed("team.example.org/other")); + assert!(cfg.pod_annotation_allowed("owner")); + } + + #[test] + fn allowed_pod_annotations_rejects_invalid_entries() { + for entry in [ + "*", + "", + "example.com/", + "Example.com/owner", + "example.com/a/*", + "example.com/*/owner", + "-bad", + ] { + let cfg = KubernetesComputeConfig { + allowed_pod_annotations: vec![entry.to_string()], + ..KubernetesComputeConfig::default() + }; + assert!( + cfg.validate_allowed_pod_annotations().is_err(), + "{entry:?} should be rejected" + ); + } + } + + #[test] + fn allowed_pod_annotations_rejects_reserved_domains() { + for entry in [ + "openshell.ai/*", + "openshell.ai/sandbox-id", + "agents.x-k8s.io/pod-name", + "extensions.agents.x-k8s.io/*", + ] { + let cfg = KubernetesComputeConfig { + allowed_pod_annotations: vec![entry.to_string()], + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_allowed_pod_annotations().unwrap_err(); + assert!(err.contains("reserved"), "{entry:?}: {err}"); + } + } + + #[test] + fn reserved_annotation_domains_are_never_allowed() { + let cfg = KubernetesComputeConfig { + allowed_pod_annotations: vec!["openshell.ai/sandbox-id".into()], + ..KubernetesComputeConfig::default() + }; + assert!(!cfg.pod_annotation_allowed("openshell.ai/sandbox-id")); + assert!(!cfg.pod_annotation_allowed("x.agents.x-k8s.io/pod-name")); + } + #[test] fn default_runtime_class_name_is_empty() { let cfg = KubernetesComputeConfig::default(); diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 42f75abaa6..645bba4e55 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -758,6 +758,9 @@ impl KubernetesComputeDriver { config .validate_upstream_proxy_config() .map_err(KubernetesDriverError::Precondition)?; + config + .validate_allowed_pod_annotations() + .map_err(KubernetesDriverError::Precondition)?; // Read the bundle once at startup so an unreadable path or a // certificate-free file fails the gateway here, rather than as a // repeated sandbox-create failure. The authoritative read still @@ -1412,6 +1415,17 @@ impl KubernetesComputeDriver { kubernetes_driver_config_for_spec(sandbox.spec.as_ref()) } + fn validate_pod_annotations_for_sandbox(&self, sandbox: &Sandbox) -> Result<(), String> { + let Some(template) = sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + else { + return Ok(()); + }; + validate_pod_annotations(template, &self.config) + } + fn agent_sandbox_api( client: Client, sandbox_api_version: &str, @@ -1604,6 +1618,8 @@ impl KubernetesComputeDriver { .map_err(tonic::Status::failed_precondition)?; let _ = Self::validate_driver_config_for_sandbox(sandbox) .map_err(tonic::Status::invalid_argument)?; + self.validate_pod_annotations_for_sandbox(sandbox) + .map_err(tonic::Status::invalid_argument)?; match self.config.workspace_mode { WorkspaceMode::Shared => { validate_kube_resource_name_length(&sandbox.workspace, &sandbox.name)?; @@ -1859,6 +1875,8 @@ impl KubernetesComputeDriver { sandbox, ) .map_err(admission_error)?; + self.validate_pod_annotations_for_sandbox(sandbox) + .map_err(KubernetesDriverError::InvalidArgument)?; let gpu_requirements = sandbox .spec .as_ref() @@ -6738,6 +6756,31 @@ fn platform_config_string(template: &SandboxTemplate, key: &str) -> Option Result<(), String> { + let Some(serde_json::Value::Object(annotations)) = + platform_config_struct(template, "annotations") + else { + return Ok(()); + }; + let rejected: Vec<&str> = annotations + .keys() + .map(String::as_str) + .filter(|key| !config.pod_annotation_allowed(key)) + .collect(); + if rejected.is_empty() { + return Ok(()); + } + Err(format!( + "pod annotations not permitted by the gateway's allowed_pod_annotations: {}", + rejected.join(", ") + )) +} + /// Extract a nested Struct value from the template's `platform_config`, /// converting it to `serde_json::Value`. fn platform_config_struct(template: &SandboxTemplate, key: &str) -> Option { @@ -9535,6 +9578,72 @@ mod tests { assert!(err.message().contains("gpu count must be greater than 0")); } + fn template_with_pod_annotations(keys: &[&str]) -> SandboxTemplate { + let annotations = keys + .iter() + .map(|key| { + ( + (*key).to_string(), + Value { + kind: Some(Kind::StringValue("value".to_string())), + }, + ) + }) + .collect(); + SandboxTemplate { + platform_config: Some(Struct { + fields: std::iter::once(( + "annotations".to_string(), + Value { + kind: Some(Kind::StructValue(Struct { + fields: annotations, + })), + }, + )) + .collect(), + }), + ..SandboxTemplate::default() + } + } + + #[test] + fn pod_annotations_rejected_by_default() { + let template = template_with_pod_annotations(&["example.com/owner"]); + let err = + validate_pod_annotations(&template, &KubernetesComputeConfig::default()).unwrap_err(); + assert!(err.contains("example.com/owner"), "{err}"); + assert!(err.contains("allowed_pod_annotations"), "{err}"); + } + + #[test] + fn pod_annotations_without_template_annotations_pass() { + validate_pod_annotations( + &SandboxTemplate::default(), + &KubernetesComputeConfig::default(), + ) + .unwrap(); + } + + #[test] + fn pod_annotations_allowlist_accepts_exact_and_wildcard_entries() { + let config = KubernetesComputeConfig { + allowed_pod_annotations: vec!["example.com/*".into(), "team".into()], + ..KubernetesComputeConfig::default() + }; + let template = template_with_pod_annotations(&["example.com/owner", "team"]); + validate_pod_annotations(&template, &config).unwrap(); + + let template = template_with_pod_annotations(&[ + "example.com/owner", + "sub.example.com/owner", + "other.example.net/owner", + ]); + let err = validate_pod_annotations(&template, &config).unwrap_err(); + assert!(err.contains("sub.example.com/owner"), "{err}"); + assert!(err.contains("other.example.net/owner"), "{err}"); + assert!(!err.contains("example.com/owner,"), "{err}"); + } + #[test] fn kube_pulling_event_adds_image_progress_metadata() { let mut metadata = std::collections::HashMap::new(); diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index d70f706f5f..dca69bd0b3 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -87,6 +87,13 @@ struct Args { )] sandbox_image_pull_secrets: Vec, + #[arg( + long, + env = "OPENSHELL_K8S_ALLOWED_POD_ANNOTATIONS", + value_delimiter = ',' + )] + allowed_pod_annotations: Vec, + #[arg(long, env = "OPENSHELL_MANAGED_SSH_INGRESS_ENABLED")] managed_ssh_ingress_enabled: bool, @@ -288,6 +295,7 @@ async fn main() -> Result<()> { .unwrap_or_default(), default_runtime_class_name: std::env::var("OPENSHELL_K8S_DEFAULT_RUNTIME_CLASS_NAME") .unwrap_or_default(), + allowed_pod_annotations: args.allowed_pod_annotations, sa_token_ttl_secs: args.sa_token_ttl_secs, provider_spiffe_workload_api_socket_path: args .provider_spiffe_workload_api_socket_path diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 4b534bc0be..b036a43970 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -424,6 +424,7 @@ discovery endpoint or its TLS CA. | securityContext.capabilities.drop | list | `["ALL"]` | Linux capabilities dropped from the gateway container. | | securityContext.runAsNonRoot | bool | `true` | Require the gateway container to run as a non-root user. | | securityContext.runAsUser | int | `1000` | UID assigned to the gateway container. | +| server.allowedPodAnnotations | list | `[]` | Pod annotation keys that sandbox templates may set (`allowed_pod_annotations`). Each entry is an exact key (`example.com/team`) or a prefix wildcard (`example.com/*`). Empty (default) rejects every caller-supplied pod annotation. | | server.auth.allowUnauthenticatedUsers | bool | `false` | UNSAFE: accept unauthenticated CLI/user requests as a local developer principal. Intended only for trusted local Skaffold/k3d development or a fully trusted fronting proxy. Leave false for shared or production clusters. | | server.credentialDrivers.kubernetesSecrets.createNamespace | bool | `false` | Create the credential namespace. Requires a namespace other than the release namespace. The Namespace is retained on uninstall so stored credentials survive; an existing Namespace not owned by this release is left untouched. | | server.credentialDrivers.kubernetesSecrets.enabled | bool | `false` | Enable the in-tree Kubernetes Secret credential driver. WARNING: The RBAC Role grants read/write access to ALL Secrets in the configured namespace. Use a dedicated namespace to limit blast radius. | diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index c6eea2b6a1..4f71eeb4d8 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -247,6 +247,9 @@ data: {{- if .Values.server.defaultRuntimeClassName }} default_runtime_class_name = {{ .Values.server.defaultRuntimeClassName | quote }} {{- end }} + {{- with .Values.server.allowedPodAnnotations }} + allowed_pod_annotations = [{{- range $i, $key := . }}{{ if $i }}, {{ end }}{{ $key | quote }}{{- end }}] + {{- end }} {{- with include "openshell.sandboxId" (dict "name" "server.sandboxUid" "value" .Values.server.sandboxUid) }} sandbox_uid = {{ . }} {{- end }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 16ccfaee6b..c62b127b3a 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -722,6 +722,24 @@ tests: path: data["gateway.toml"] pattern: 'image_pull_secrets\s*=' + - it: renders allowed pod annotations for the Kubernetes driver + template: templates/gateway-config.yaml + set: + server.allowedPodAnnotations: + - example.com/* + - team.example.org/owner + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?allowed_pod_annotations\s*=\s*\["example\.com/\*", "team\.example\.org/owner"\]' + + - it: omits allowed pod annotations by default + template: templates/gateway-config.yaml + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'allowed_pod_annotations\s*=' + - it: does not render local mTLS user auth for Kubernetes deployments template: templates/gateway-config.yaml asserts: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index d0612cd563..57716ad5fb 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -372,6 +372,11 @@ server: # Set to a RuntimeClass name (e.g. "kata-containers", "nvidia") to apply it # to all sandboxes that don't explicitly override it. defaultRuntimeClassName: "" + # -- Pod annotation keys that sandbox templates may set + # (`allowed_pod_annotations`). Each entry is an exact key + # (`example.com/team`) or a prefix wildcard (`example.com/*`). + # Empty (default) rejects every caller-supplied pod annotation. + allowedPodAnnotations: [] # -- UID for sandbox pods (`sandbox_uid`). Empty (default) = use the OpenShift # SCC namespace annotation if present, otherwise the driver default. Must be # an integer between 1 and 4294967294. diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index a2b72381a1..0caef54285 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -845,6 +845,10 @@ workspace_default_storage_size = "10Gi" # Kubernetes RuntimeClass applied to sandbox pods when the API request does # not specify one. Empty (default) = omit the field, using the cluster default. # default_runtime_class_name = "kata-containers" +# Pod annotation keys that sandbox templates may set. Each entry is an exact +# key or a `/*` wildcard. Empty (default) rejects every template +# annotation. Keys under openshell.ai and agents.x-k8s.io are always rejected. +# allowed_pod_annotations = ["example.com/*", "team.example.org/owner"] # Kubelet clamps projected tokens below 600 seconds. The driver caps values at 86400. sa_token_ttl_secs = 3600 # Optional SPIFFE Workload API socket mounted into sandbox pods for dynamic diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index b3b9fa0511..52154d866c 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -246,6 +246,7 @@ Only the OpenShell gateway and the Agent Sandbox controller should be able to ma | `supervisor_image` | `supervisor.image.*` | Override the supervisor image. | | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Default workspace PVC size. | | `workspace_storage_class` | `server.workspaceStorageClass` | `StorageClass` for workspace PVCs. Set this if the cluster has no default `StorageClass`. | +| `allowed_pod_annotations` | `server.allowedPodAnnotations` | Pod annotation keys that sandbox templates may set, as exact keys or `/*` wildcards. Empty (default) rejects all template annotations. | | `sandbox_uid` / `sandbox_gid` | `server.sandboxUid` / `server.sandboxGid` | Sandbox pod UID and GID. Takes priority over OpenShift SCC namespace annotations. `sandbox_gid` defaults to the UID. | | `https_proxy` | `upstreamProxy.url` | Corporate proxy for sandbox egress. | | `no_proxy` | `upstreamProxy.noProxy` | Destinations that bypass the corporate proxy. | From 3cf8e82d5c56aad2114ac1f2d993d40f876d416c Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 13 Oct 2026 12:29:20 -0700 Subject: [PATCH 2/5] test(kubernetes): cover pod annotation rejection on sandbox create Assert the exact rejected annotation keys, exercise create_sandbox_inner with a disallowed annotation, and point the rejection error at the Helm gatewayConfig path. Signed-off-by: Matthew Grossman --- .../openshell-driver-kubernetes/src/driver.rs | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 00428ac1a5..48c1568374 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -6849,7 +6849,8 @@ fn validate_pod_annotations( return Ok(()); } Err(format!( - "pod annotations not permitted by the gateway's allowed_pod_annotations: {}", + "pod annotations not permitted by the gateway's allowed_pod_annotations \ + (Helm: gatewayConfig.openshell.drivers.kubernetes.allowed_pod_annotations): {}", rejected.join(", ") )) } @@ -9762,9 +9763,24 @@ mod tests { "other.example.net/owner", ]); let err = validate_pod_annotations(&template, &config).unwrap_err(); - assert!(err.contains("sub.example.com/owner"), "{err}"); - assert!(err.contains("other.example.net/owner"), "{err}"); - assert!(!err.contains("example.com/owner,"), "{err}"); + let (_, rejected) = err.rsplit_once(": ").unwrap(); + assert_eq!(rejected, "other.example.net/owner, sub.example.com/owner"); + } + + #[tokio::test] + async fn create_sandbox_rejects_disallowed_pod_annotations() { + let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig::default()); + let sandbox = Sandbox { + spec: Some(SandboxSpec { + template: Some(template_with_pod_annotations(&["example.com/owner"])), + ..Default::default() + }), + ..Default::default() + }; + assert!(matches!( + Box::pin(driver.create_sandbox_inner(&sandbox)).await, + Err(KubernetesDriverError::InvalidArgument(message)) if message.contains("example.com/owner") + )); } #[test] From 061731e0319286c15c3c251f6af7ccb899dd9ad9 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 13 Oct 2026 12:40:33 -0700 Subject: [PATCH 3/5] fix(kubernetes): drop Helm path from pod annotation rejection error The rejection error reaches API callers, not operators, so name only the gateway config key. Spell the Helm path in the runtimes docs with the quoted table key to avoid an invalid nested gatewayConfig rendering. Signed-off-by: Matthew Grossman --- crates/openshell-driver-kubernetes/src/driver.rs | 3 +-- docs/how-it-works/sandboxes/runtimes.mdx | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 48c1568374..dad8a6b62a 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -6849,8 +6849,7 @@ fn validate_pod_annotations( return Ok(()); } Err(format!( - "pod annotations not permitted by the gateway's allowed_pod_annotations \ - (Helm: gatewayConfig.openshell.drivers.kubernetes.allowed_pod_annotations): {}", + "pod annotations not permitted by the gateway's allowed_pod_annotations: {}", rejected.join(", ") )) } diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index fc45112cc2..8fbe904312 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -253,7 +253,7 @@ Only the OpenShell gateway and the Agent Sandbox controller should be able to ma | `supervisor_image` | `supervisor.image.*` | Override the supervisor image. | | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Default workspace PVC size. | | `workspace_storage_class` | `server.workspaceStorageClass` | `StorageClass` for workspace PVCs. Set this if the cluster has no default `StorageClass`. | -| `allowed_pod_annotations` | `gatewayConfig.openshell.drivers.kubernetes.allowed_pod_annotations` | Pod annotation keys that sandbox templates may set, as exact keys or `/*` wildcards. Empty (default) rejects all template annotations. | +| `allowed_pod_annotations` | `gatewayConfig["openshell.drivers.kubernetes"].allowed_pod_annotations` | Pod annotation keys that sandbox templates may set, as exact keys or `/*` wildcards. Empty (default) rejects all template annotations. | | `sandbox_uid` / `sandbox_gid` | `server.sandboxUid` / `server.sandboxGid` | Sandbox pod UID and GID. Takes priority over OpenShift SCC namespace annotations. `sandbox_gid` defaults to the UID. | | `https_proxy` | `upstreamProxy.url` | Corporate proxy for sandbox egress. | | `no_proxy` | `upstreamProxy.noProxy` | Destinations that bypass the corporate proxy. | From bf237fb8d15a80e53d93ce66105d45ca92d8c3f2 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 13 Oct 2026 12:57:12 -0700 Subject: [PATCH 4/5] test(kubernetes): condense pod annotation allowlist tests Fold the allowlist matching and validation cases into table-driven tests, reuse the json_struct helper for templates, and document why the openshell.ai and agents.x-k8s.io domains are reserved. Signed-off-by: Matthew Grossman --- .../openshell-driver-kubernetes/src/config.rs | 95 ++++++++----------- .../openshell-driver-kubernetes/src/driver.rs | 62 +++--------- docs/how-it-works/gateways/configuration.mdx | 5 +- docs/how-it-works/sandboxes/runtimes.mdx | 2 +- 4 files changed, 60 insertions(+), 104 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 6adaea20f7..6bf07fa301 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -265,8 +265,12 @@ pub struct KubernetesComputeConfig { pub default_runtime_class_name: String, /// Pod annotation keys that sandbox templates may set. Each entry is an /// exact key (`example.com/team`) or a prefix wildcard (`example.com/*`). - /// Empty (default) rejects every caller-supplied pod annotation. Keys in - /// `OpenShell` and Agent Sandbox domains are always rejected. + /// Empty (default) rejects every caller-supplied pod annotation. + /// + /// Keys in `openshell.ai` and `agents.x-k8s.io` (and their subdomains) + /// are always rejected and cannot be allowlisted. `OpenShell` writes sandbox + /// identity under `openshell.ai`, and the Agent Sandbox controller owns + /// `agents.x-k8s.io` and strips those keys from pod templates. pub allowed_pod_annotations: Vec, /// Lifetime (seconds) of the projected `ServiceAccount` token kubelet /// writes into each sandbox pod. Used only for the one-shot @@ -846,8 +850,10 @@ fn is_dns1123_subdomain(value: &str) -> bool { }) } -/// Annotation domains owned by `OpenShell` and Agent Sandbox. Sandbox templates -/// can never set keys in these domains or their subdomains. +/// Annotation domains owned by `OpenShell` (sandbox identity such as +/// `openshell.ai/sandbox-id`) and the Agent Sandbox controller (which strips +/// them from pod templates). Sandbox templates can never set keys in these +/// domains or their subdomains, regardless of `allowed_pod_annotations`. const RESERVED_POD_ANNOTATION_DOMAINS: &[&str] = &["openshell.ai", "agents.x-k8s.io"]; fn is_reserved_annotation_domain(domain: &str) -> bool { @@ -1045,29 +1051,38 @@ mod tests { assert_eq!(cfg.default_runtime_class_name, "nvidia"); } - #[test] - fn allowed_pod_annotations_defaults_to_empty() { - let cfg = KubernetesComputeConfig::default(); - assert!(cfg.allowed_pod_annotations.is_empty()); - assert!(!cfg.pod_annotation_allowed("example.com/owner")); + fn with_allowed_pod_annotations(entries: &[&str]) -> KubernetesComputeConfig { + KubernetesComputeConfig { + allowed_pod_annotations: entries.iter().map(ToString::to_string).collect(), + ..KubernetesComputeConfig::default() + } } #[test] - fn allowed_pod_annotations_accepts_exact_and_wildcard_entries() { - let cfg: KubernetesComputeConfig = serde_json::from_value(serde_json::json!({ - "allowed_pod_annotations": ["example.com/*", "team.example.org/owner", "owner"] - })) - .unwrap(); - cfg.validate_allowed_pod_annotations().unwrap(); - assert!(cfg.pod_annotation_allowed("example.com/anything")); - assert!(!cfg.pod_annotation_allowed("sub.example.com/anything")); - assert!(cfg.pod_annotation_allowed("team.example.org/owner")); - assert!(!cfg.pod_annotation_allowed("team.example.org/other")); - assert!(cfg.pod_annotation_allowed("owner")); + fn pod_annotation_allowed_matches_exact_and_domain_entries() { + assert!(!KubernetesComputeConfig::default().pod_annotation_allowed("example.com/a")); + + let cfg = with_allowed_pod_annotations(&["example.com/*", "team.org/owner", "owner"]); + for (key, allowed) in [ + ("example.com/a", true), + ("sub.example.com/a", false), + ("team.org/owner", true), + ("team.org/other", false), + ("owner", true), + ] { + assert_eq!(cfg.pod_annotation_allowed(key), allowed, "{key}"); + } + + // Reserved domains stay blocked even if validation was bypassed. + let cfg = with_allowed_pod_annotations(&["openshell.ai/sandbox-id"]); + assert!(!cfg.pod_annotation_allowed("openshell.ai/sandbox-id")); } #[test] - fn allowed_pod_annotations_rejects_invalid_entries() { + fn validate_allowed_pod_annotations_rejects_malformed_and_reserved_entries() { + with_allowed_pod_annotations(&["example.com/*", "team.org/owner", "owner"]) + .validate_allowed_pod_annotations() + .unwrap(); for entry in [ "*", "", @@ -1076,45 +1091,19 @@ mod tests { "example.com/a/*", "example.com/*/owner", "-bad", - ] { - let cfg = KubernetesComputeConfig { - allowed_pod_annotations: vec![entry.to_string()], - ..KubernetesComputeConfig::default() - }; - assert!( - cfg.validate_allowed_pod_annotations().is_err(), - "{entry:?} should be rejected" - ); - } - } - - #[test] - fn allowed_pod_annotations_rejects_reserved_domains() { - for entry in [ "openshell.ai/*", - "openshell.ai/sandbox-id", "agents.x-k8s.io/pod-name", "extensions.agents.x-k8s.io/*", ] { - let cfg = KubernetesComputeConfig { - allowed_pod_annotations: vec![entry.to_string()], - ..KubernetesComputeConfig::default() - }; - let err = cfg.validate_allowed_pod_annotations().unwrap_err(); - assert!(err.contains("reserved"), "{entry:?}: {err}"); + assert!( + with_allowed_pod_annotations(&[entry]) + .validate_allowed_pod_annotations() + .is_err(), + "{entry:?}" + ); } } - #[test] - fn reserved_annotation_domains_are_never_allowed() { - let cfg = KubernetesComputeConfig { - allowed_pod_annotations: vec!["openshell.ai/sandbox-id".into()], - ..KubernetesComputeConfig::default() - }; - assert!(!cfg.pod_annotation_allowed("openshell.ai/sandbox-id")); - assert!(!cfg.pod_annotation_allowed("x.agents.x-k8s.io/pod-name")); - } - #[test] fn default_runtime_class_name_is_empty() { let cfg = KubernetesComputeConfig::default(); diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index dad8a6b62a..a115dd18d0 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -9702,68 +9702,32 @@ mod tests { } fn template_with_pod_annotations(keys: &[&str]) -> SandboxTemplate { - let annotations = keys + let annotations: serde_json::Map<_, _> = keys .iter() - .map(|key| { - ( - (*key).to_string(), - Value { - kind: Some(Kind::StringValue("value".to_string())), - }, - ) - }) + .map(|key| ((*key).to_string(), "v".into())) .collect(); SandboxTemplate { - platform_config: Some(Struct { - fields: std::iter::once(( - "annotations".to_string(), - Value { - kind: Some(Kind::StructValue(Struct { - fields: annotations, - })), - }, - )) - .collect(), - }), + platform_config: Some(json_struct( + serde_json::json!({ "annotations": annotations }), + )), ..SandboxTemplate::default() } } #[test] - fn pod_annotations_rejected_by_default() { - let template = template_with_pod_annotations(&["example.com/owner"]); - let err = - validate_pod_annotations(&template, &KubernetesComputeConfig::default()).unwrap_err(); - assert!(err.contains("example.com/owner"), "{err}"); - assert!(err.contains("allowed_pod_annotations"), "{err}"); - } - - #[test] - fn pod_annotations_without_template_annotations_pass() { - validate_pod_annotations( - &SandboxTemplate::default(), - &KubernetesComputeConfig::default(), - ) - .unwrap(); - } - - #[test] - fn pod_annotations_allowlist_accepts_exact_and_wildcard_entries() { + fn validate_pod_annotations_reports_keys_outside_allowlist() { let config = KubernetesComputeConfig { - allowed_pod_annotations: vec!["example.com/*".into(), "team".into()], + allowed_pod_annotations: vec!["example.com/*".into()], ..KubernetesComputeConfig::default() }; - let template = template_with_pod_annotations(&["example.com/owner", "team"]); - validate_pod_annotations(&template, &config).unwrap(); + validate_pod_annotations(&SandboxTemplate::default(), &config).unwrap(); + validate_pod_annotations(&template_with_pod_annotations(&["example.com/a"]), &config) + .unwrap(); - let template = template_with_pod_annotations(&[ - "example.com/owner", - "sub.example.com/owner", - "other.example.net/owner", - ]); + let template = + template_with_pod_annotations(&["example.com/a", "sub.example.com/a", "other.net/a"]); let err = validate_pod_annotations(&template, &config).unwrap_err(); - let (_, rejected) = err.rsplit_once(": ").unwrap(); - assert_eq!(rejected, "other.example.net/owner, sub.example.com/owner"); + assert!(err.ends_with(": other.net/a, sub.example.com/a"), "{err}"); } #[tokio::test] diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 04ba7c3bb4..d9ca28eef4 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -862,7 +862,10 @@ workspace_default_storage_size = "10Gi" # default_runtime_class_name = "kata-containers" # Pod annotation keys that sandbox templates may set. Each entry is an exact # key or a `/*` wildcard. Empty (default) rejects every template -# annotation. Keys under openshell.ai and agents.x-k8s.io are always rejected. +# annotation. Keys under openshell.ai (OpenShell sandbox identity) and +# agents.x-k8s.io (owned by the Agent Sandbox controller), including +# subdomains, are reserved: templates cannot set them and the allowlist +# cannot include them. # allowed_pod_annotations = ["example.com/*", "team.example.org/owner"] # Kubelet clamps projected tokens below 600 seconds. The driver caps values at 86400. sa_token_ttl_secs = 3600 diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 8fbe904312..48524780c4 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -253,7 +253,7 @@ Only the OpenShell gateway and the Agent Sandbox controller should be able to ma | `supervisor_image` | `supervisor.image.*` | Override the supervisor image. | | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Default workspace PVC size. | | `workspace_storage_class` | `server.workspaceStorageClass` | `StorageClass` for workspace PVCs. Set this if the cluster has no default `StorageClass`. | -| `allowed_pod_annotations` | `gatewayConfig["openshell.drivers.kubernetes"].allowed_pod_annotations` | Pod annotation keys that sandbox templates may set, as exact keys or `/*` wildcards. Empty (default) rejects all template annotations. | +| `allowed_pod_annotations` | `gatewayConfig["openshell.drivers.kubernetes"].allowed_pod_annotations` | Pod annotation keys that sandbox templates may set, as exact keys or `/*` wildcards. Empty (default) rejects all template annotations. Keys under `openshell.ai` and `agents.x-k8s.io` are reserved and can't be allowlisted. | | `sandbox_uid` / `sandbox_gid` | `server.sandboxUid` / `server.sandboxGid` | Sandbox pod UID and GID. Takes priority over OpenShift SCC namespace annotations. `sandbox_gid` defaults to the UID. | | `https_proxy` | `upstreamProxy.url` | Corporate proxy for sandbox egress. | | `no_proxy` | `upstreamProxy.noProxy` | Destinations that bypass the corporate proxy. | From 09d5580e8d0f0c5be17708ec6c223e601ebdf014 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 13 Oct 2026 14:08:34 -0700 Subject: [PATCH 5/5] feat(kubernetes): add operator allowlist for sandbox pod labels Sandbox template labels were copied onto workload pods unfiltered. Labels select pods for NetworkPolicy rules and Services, so a caller could place a sandbox under rules meant for other workloads or set OpenShell-owned labels. Add allowed_pod_labels, sharing the allowed_pod_annotations entry format, validation, and reserved openshell.ai / agents.x-k8s.io domains. Empty by default, which rejects every template label. Signed-off-by: Matthew Grossman --- .../openshell-driver-kubernetes/src/config.rs | 154 ++++++++++++------ .../openshell-driver-kubernetes/src/driver.rs | 116 +++++++++---- .../openshell-driver-kubernetes/src/main.rs | 4 + .../openshell/tests/gateway_config_test.yaml | 12 +- docs/how-it-works/gateways/configuration.mdx | 5 + docs/how-it-works/sandboxes/runtimes.mdx | 1 + 6 files changed, 204 insertions(+), 88 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 6bf07fa301..acbd0d006f 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -272,6 +272,10 @@ pub struct KubernetesComputeConfig { /// identity under `openshell.ai`, and the Agent Sandbox controller owns /// `agents.x-k8s.io` and strips those keys from pod templates. pub allowed_pod_annotations: Vec, + /// Pod label keys that sandbox templates may set, using the same entry + /// format and reserved domains as `allowed_pod_annotations`. Empty + /// (default) rejects every caller-supplied pod label. + pub allowed_pod_labels: Vec, /// Lifetime (seconds) of the projected `ServiceAccount` token kubelet /// writes into each sandbox pod. Used only for the one-shot /// `IssueSandboxToken` bootstrap exchange — the gateway-minted JWT @@ -374,6 +378,7 @@ impl Default for KubernetesComputeConfig { workspace_storage_class: String::new(), default_runtime_class_name: String::new(), allowed_pod_annotations: Vec::new(), + allowed_pod_labels: Vec::new(), sa_token_ttl_secs: 3600, provider_spiffe_workload_api_socket_path: String::new(), sandbox_uid: None, @@ -390,58 +395,28 @@ impl KubernetesComputeConfig { self.validate_sandbox_identity_config()?; self.validate_proxy_uid()?; self.validate_upstream_proxy_config()?; - self.validate_allowed_pod_annotations() + self.validate_allowed_pod_annotations()?; + self.validate_allowed_pod_labels() } pub fn validate_allowed_pod_annotations(&self) -> Result<(), String> { - for entry in &self.allowed_pod_annotations { - let key = entry.strip_suffix("/*").unwrap_or(entry); - let wildcard = key.len() != entry.len(); - let (prefix, name) = match key.split_once('/') { - Some((prefix, name)) if !wildcard => (Some(prefix), Some(name)), - Some(_) => { - return Err(format!( - "allowed_pod_annotations entry {entry:?}: wildcard entries must be `/*`" - )); - } - None if wildcard => (Some(key), None), - None => (None, Some(key)), - }; - if let Some(prefix) = prefix - && !is_dns1123_subdomain(prefix) - { - return Err(format!( - "allowed_pod_annotations entry {entry:?}: invalid key prefix {prefix:?}" - )); - } - if let Some(name) = name - && !is_annotation_name(name) - { - return Err(format!( - "allowed_pod_annotations entry {entry:?}: invalid key name {name:?}" - )); - } - if prefix.is_some_and(is_reserved_annotation_domain) { - return Err(format!( - "allowed_pod_annotations entry {entry:?}: domain is reserved for OpenShell and Agent Sandbox" - )); - } - } - Ok(()) + validate_pod_metadata_allowlist("allowed_pod_annotations", &self.allowed_pod_annotations) + } + + pub fn validate_allowed_pod_labels(&self) -> Result<(), String> { + validate_pod_metadata_allowlist("allowed_pod_labels", &self.allowed_pod_labels) } /// Whether a caller-supplied pod annotation key passes the operator allowlist. #[must_use] pub fn pod_annotation_allowed(&self, key: &str) -> bool { - let prefix = key.split_once('/').map(|(prefix, _)| prefix); - if prefix.is_some_and(is_reserved_annotation_domain) { - return false; - } - self.allowed_pod_annotations.iter().any(|entry| { - entry - .strip_suffix("/*") - .map_or_else(|| entry == key, |domain| prefix == Some(domain)) - }) + pod_metadata_key_allowed(&self.allowed_pod_annotations, key) + } + + /// Whether a caller-supplied pod label key passes the operator allowlist. + #[must_use] + pub fn pod_label_allowed(&self, key: &str) -> bool { + pod_metadata_key_allowed(&self.allowed_pod_labels, key) } /// Clamp `sa_token_ttl_secs` into the `[MIN_SA_TOKEN_TTL_SECS, @@ -850,14 +825,67 @@ fn is_dns1123_subdomain(value: &str) -> bool { }) } -/// Annotation domains owned by `OpenShell` (sandbox identity such as -/// `openshell.ai/sandbox-id`) and the Agent Sandbox controller (which strips -/// them from pod templates). Sandbox templates can never set keys in these -/// domains or their subdomains, regardless of `allowed_pod_annotations`. -const RESERVED_POD_ANNOTATION_DOMAINS: &[&str] = &["openshell.ai", "agents.x-k8s.io"]; +/// Validate an `allowed_pod_annotations` or `allowed_pod_labels` list. Each +/// entry is an exact qualified key or a `/*` wildcard outside the +/// reserved domains. +fn validate_pod_metadata_allowlist(field: &str, entries: &[String]) -> Result<(), String> { + for entry in entries { + let key = entry.strip_suffix("/*").unwrap_or(entry); + let wildcard = key.len() != entry.len(); + let (prefix, name) = match key.split_once('/') { + Some((prefix, name)) if !wildcard => (Some(prefix), Some(name)), + Some(_) => { + return Err(format!( + "{field} entry {entry:?}: wildcard entries must be `/*`" + )); + } + None if wildcard => (Some(key), None), + None => (None, Some(key)), + }; + if let Some(prefix) = prefix + && !is_dns1123_subdomain(prefix) + { + return Err(format!( + "{field} entry {entry:?}: invalid key prefix {prefix:?}" + )); + } + if let Some(name) = name + && !is_qualified_name_segment(name) + { + return Err(format!( + "{field} entry {entry:?}: invalid key name {name:?}" + )); + } + if prefix.is_some_and(is_reserved_pod_metadata_domain) { + return Err(format!( + "{field} entry {entry:?}: domain is reserved for OpenShell and Agent Sandbox" + )); + } + } + Ok(()) +} + +fn pod_metadata_key_allowed(entries: &[String], key: &str) -> bool { + let prefix = key.split_once('/').map(|(prefix, _)| prefix); + if prefix.is_some_and(is_reserved_pod_metadata_domain) { + return false; + } + entries.iter().any(|entry| { + entry + .strip_suffix("/*") + .map_or_else(|| entry == key, |domain| prefix == Some(domain)) + }) +} + +/// Pod annotation and label domains owned by `OpenShell` (sandbox identity +/// such as `openshell.ai/sandbox-id` and boundary labels) and the Agent +/// Sandbox controller (which strips them from pod templates). Sandbox +/// templates can never set keys in these domains or their subdomains, +/// regardless of `allowed_pod_annotations` or `allowed_pod_labels`. +const RESERVED_POD_METADATA_DOMAINS: &[&str] = &["openshell.ai", "agents.x-k8s.io"]; -fn is_reserved_annotation_domain(domain: &str) -> bool { - RESERVED_POD_ANNOTATION_DOMAINS.iter().any(|reserved| { +fn is_reserved_pod_metadata_domain(domain: &str) -> bool { + RESERVED_POD_METADATA_DOMAINS.iter().any(|reserved| { domain == *reserved || domain .strip_suffix(reserved) @@ -867,7 +895,7 @@ fn is_reserved_annotation_domain(domain: &str) -> bool { /// Kubernetes qualified-name segment: at most 63 alphanumeric, `-`, `_`, or /// `.` characters, beginning and ending with an alphanumeric character. -fn is_annotation_name(value: &str) -> bool { +fn is_qualified_name_segment(value: &str) -> bool { let bytes = value.as_bytes(); !bytes.is_empty() && bytes.len() <= 63 @@ -1104,6 +1132,28 @@ mod tests { } } + #[test] + fn pod_label_allowlist_is_separate_from_annotation_allowlist() { + let cfg = KubernetesComputeConfig { + allowed_pod_annotations: vec!["annotations.example.com/*".into()], + allowed_pod_labels: vec!["app".into(), "labels.example.com/*".into()], + ..KubernetesComputeConfig::default() + }; + assert!(cfg.pod_label_allowed("app")); + assert!(cfg.pod_label_allowed("labels.example.com/team")); + assert!(!cfg.pod_label_allowed("annotations.example.com/team")); + assert!(!cfg.pod_annotation_allowed("labels.example.com/team")); + assert!(!KubernetesComputeConfig::default().pod_label_allowed("app")); + + let cfg = KubernetesComputeConfig { + allowed_pod_labels: vec!["openshell.ai/managed-by".into()], + ..KubernetesComputeConfig::default() + }; + assert!(!cfg.pod_label_allowed("openshell.ai/managed-by")); + let err = cfg.validate_allowed_pod_labels().unwrap_err(); + assert!(err.starts_with("allowed_pod_labels entry"), "{err}"); + } + #[test] fn default_runtime_class_name_is_empty() { let cfg = KubernetesComputeConfig::default(); diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index a115dd18d0..f6546e7b8f 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -778,6 +778,9 @@ impl KubernetesComputeDriver { config .validate_allowed_pod_annotations() .map_err(KubernetesDriverError::Precondition)?; + config + .validate_allowed_pod_labels() + .map_err(KubernetesDriverError::Precondition)?; // Read the bundle once at startup so an unreadable path or a // certificate-free file fails the gateway here, rather than as a // repeated sandbox-create failure. The authoritative read still @@ -1431,7 +1434,7 @@ impl KubernetesComputeDriver { kubernetes_driver_config_for_spec(sandbox.spec.as_ref()) } - fn validate_pod_annotations_for_sandbox(&self, sandbox: &Sandbox) -> Result<(), String> { + fn validate_pod_metadata_for_sandbox(&self, sandbox: &Sandbox) -> Result<(), String> { let Some(template) = sandbox .spec .as_ref() @@ -1439,7 +1442,7 @@ impl KubernetesComputeDriver { else { return Ok(()); }; - validate_pod_annotations(template, &self.config) + validate_pod_metadata(template, &self.config) } fn agent_sandbox_api( @@ -1634,7 +1637,7 @@ impl KubernetesComputeDriver { .map_err(tonic::Status::failed_precondition)?; let _ = Self::validate_driver_config_for_sandbox(sandbox) .map_err(tonic::Status::invalid_argument)?; - self.validate_pod_annotations_for_sandbox(sandbox) + self.validate_pod_metadata_for_sandbox(sandbox) .map_err(tonic::Status::invalid_argument)?; match self.config.workspace_mode { WorkspaceMode::Shared => { @@ -1891,7 +1894,7 @@ impl KubernetesComputeDriver { sandbox, ) .map_err(admission_error)?; - self.validate_pod_annotations_for_sandbox(sandbox) + self.validate_pod_metadata_for_sandbox(sandbox) .map_err(KubernetesDriverError::InvalidArgument)?; let gpu_requirements = sandbox .spec @@ -6829,27 +6832,43 @@ fn platform_config_string(template: &SandboxTemplate, key: &str) -> Option Result<(), String> { - let Some(serde_json::Value::Object(annotations)) = + if let Some(serde_json::Value::Object(annotations)) = platform_config_struct(template, "annotations") - else { - return Ok(()); - }; - let rejected: Vec<&str> = annotations - .keys() - .map(String::as_str) - .filter(|key| !config.pod_annotation_allowed(key)) - .collect(); + { + reject_disallowed_pod_metadata_keys( + "annotations", + "allowed_pod_annotations", + annotations.keys().map(String::as_str), + |key| config.pod_annotation_allowed(key), + )?; + } + reject_disallowed_pod_metadata_keys( + "labels", + "allowed_pod_labels", + template.labels.keys().map(String::as_str), + |key| config.pod_label_allowed(key), + ) +} + +fn reject_disallowed_pod_metadata_keys<'a>( + kind: &str, + field: &str, + keys: impl Iterator, + allowed: impl Fn(&str) -> bool, +) -> Result<(), String> { + let mut rejected: Vec<&str> = keys.filter(|key| !allowed(key)).collect(); if rejected.is_empty() { return Ok(()); } + rejected.sort_unstable(); Err(format!( - "pod annotations not permitted by the gateway's allowed_pod_annotations: {}", + "pod {kind} not permitted by the gateway's {field}: {}", rejected.join(", ") )) } @@ -9714,36 +9733,65 @@ mod tests { } } + fn template_with_pod_labels(keys: &[&str]) -> SandboxTemplate { + SandboxTemplate { + labels: keys + .iter() + .map(|key| ((*key).to_string(), "v".to_string())) + .collect(), + ..SandboxTemplate::default() + } + } + #[test] - fn validate_pod_annotations_reports_keys_outside_allowlist() { + fn validate_pod_metadata_reports_keys_outside_allowlists() { let config = KubernetesComputeConfig { allowed_pod_annotations: vec!["example.com/*".into()], + allowed_pod_labels: vec!["app".into()], ..KubernetesComputeConfig::default() }; - validate_pod_annotations(&SandboxTemplate::default(), &config).unwrap(); - validate_pod_annotations(&template_with_pod_annotations(&["example.com/a"]), &config) - .unwrap(); + validate_pod_metadata(&SandboxTemplate::default(), &config).unwrap(); + validate_pod_metadata(&template_with_pod_annotations(&["example.com/a"]), &config).unwrap(); + validate_pod_metadata(&template_with_pod_labels(&["app"]), &config).unwrap(); let template = template_with_pod_annotations(&["example.com/a", "sub.example.com/a", "other.net/a"]); - let err = validate_pod_annotations(&template, &config).unwrap_err(); - assert!(err.ends_with(": other.net/a, sub.example.com/a"), "{err}"); + let err = validate_pod_metadata(&template, &config).unwrap_err(); + assert!( + err.ends_with("allowed_pod_annotations: other.net/a, sub.example.com/a"), + "{err}" + ); + + let template = template_with_pod_labels(&["app", "team", "openshell.ai/managed-by"]); + let err = validate_pod_metadata(&template, &config).unwrap_err(); + assert!( + err.ends_with("allowed_pod_labels: openshell.ai/managed-by, team"), + "{err}" + ); } #[tokio::test] - async fn create_sandbox_rejects_disallowed_pod_annotations() { + async fn create_sandbox_rejects_disallowed_pod_metadata() { let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig::default()); - let sandbox = Sandbox { - spec: Some(SandboxSpec { - template: Some(template_with_pod_annotations(&["example.com/owner"])), + for (template, key) in [ + ( + template_with_pod_annotations(&["example.com/owner"]), + "example.com/owner", + ), + (template_with_pod_labels(&["app"]), "app"), + ] { + let sandbox = Sandbox { + spec: Some(SandboxSpec { + template: Some(template), + ..Default::default() + }), ..Default::default() - }), - ..Default::default() - }; - assert!(matches!( - Box::pin(driver.create_sandbox_inner(&sandbox)).await, - Err(KubernetesDriverError::InvalidArgument(message)) if message.contains("example.com/owner") - )); + }; + assert!(matches!( + Box::pin(driver.create_sandbox_inner(&sandbox)).await, + Err(KubernetesDriverError::InvalidArgument(message)) if message.contains(key) + )); + } } #[test] diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index dca69bd0b3..448147af6c 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -94,6 +94,9 @@ struct Args { )] allowed_pod_annotations: Vec, + #[arg(long, env = "OPENSHELL_K8S_ALLOWED_POD_LABELS", value_delimiter = ',')] + allowed_pod_labels: Vec, + #[arg(long, env = "OPENSHELL_MANAGED_SSH_INGRESS_ENABLED")] managed_ssh_ingress_enabled: bool, @@ -296,6 +299,7 @@ async fn main() -> Result<()> { default_runtime_class_name: std::env::var("OPENSHELL_K8S_DEFAULT_RUNTIME_CLASS_NAME") .unwrap_or_default(), allowed_pod_annotations: args.allowed_pod_annotations, + allowed_pod_labels: args.allowed_pod_labels, sa_token_ttl_secs: args.sa_token_ttl_secs, provider_spiffe_workload_api_socket_path: args .provider_spiffe_workload_api_socket_path diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index bc78ca0109..faf2b07a7e 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -725,7 +725,7 @@ tests: path: data["gateway.toml"] pattern: 'image_pull_secrets\s*=' - - it: renders allowed pod annotations for the Kubernetes driver + - it: renders allowed pod annotations and labels for the Kubernetes driver template: templates/gateway-config.yaml set: gatewayConfig: @@ -733,17 +733,25 @@ tests: allowed_pod_annotations: - example.com/* - team.example.org/owner + allowed_pod_labels: + - app asserts: - matchRegex: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?allowed_pod_annotations\s*=\s*\[\s*"example\.com/\*",\s*"team\.example\.org/owner",?\s*\]' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?allowed_pod_labels\s*=\s*\[\s*"app",?\s*\]' - - it: omits allowed pod annotations by default + - it: omits allowed pod annotations and labels by default template: templates/gateway-config.yaml asserts: - notMatchRegex: path: data["gateway.toml"] pattern: 'allowed_pod_annotations\s*=' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'allowed_pod_labels\s*=' - it: does not render local mTLS user auth for Kubernetes deployments template: templates/gateway-config.yaml diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index d9ca28eef4..38f873413e 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -867,6 +867,11 @@ workspace_default_storage_size = "10Gi" # subdomains, are reserved: templates cannot set them and the allowlist # cannot include them. # allowed_pod_annotations = ["example.com/*", "team.example.org/owner"] +# Pod label keys that sandbox templates may set, in the same format and with +# the same reserved domains. Empty (default) rejects every template label. +# Labels select pods for NetworkPolicy rules and Services, so only allow keys +# that no other rule in the sandbox namespace selects on. +# allowed_pod_labels = ["app", "team.example.org/*"] # Kubelet clamps projected tokens below 600 seconds. The driver caps values at 86400. sa_token_ttl_secs = 3600 # Optional SPIFFE Workload API socket mounted into sandbox pods for dynamic diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 48524780c4..cc76cf98d2 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -254,6 +254,7 @@ Only the OpenShell gateway and the Agent Sandbox controller should be able to ma | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Default workspace PVC size. | | `workspace_storage_class` | `server.workspaceStorageClass` | `StorageClass` for workspace PVCs. Set this if the cluster has no default `StorageClass`. | | `allowed_pod_annotations` | `gatewayConfig["openshell.drivers.kubernetes"].allowed_pod_annotations` | Pod annotation keys that sandbox templates may set, as exact keys or `/*` wildcards. Empty (default) rejects all template annotations. Keys under `openshell.ai` and `agents.x-k8s.io` are reserved and can't be allowlisted. | +| `allowed_pod_labels` | `gatewayConfig["openshell.drivers.kubernetes"].allowed_pod_labels` | Pod label keys that sandbox templates may set, in the same format as `allowed_pod_annotations`. Empty (default) rejects all template labels. Labels select pods for NetworkPolicy rules and Services, so only allow keys that no other rule in the sandbox namespace selects on. | | `sandbox_uid` / `sandbox_gid` | `server.sandboxUid` / `server.sandboxGid` | Sandbox pod UID and GID. Takes priority over OpenShift SCC namespace annotations. `sandbox_gid` defaults to the UID. | | `https_proxy` | `upstreamProxy.url` | Corporate proxy for sandbox egress. | | `no_proxy` | `upstreamProxy.noProxy` | Destinations that bypass the corporate proxy. |