diff --git a/crates/openshell-core/src/settings.rs b/crates/openshell-core/src/settings.rs index a8c13ae482..c3db1f731c 100644 --- a/crates/openshell-core/src/settings.rs +++ b/crates/openshell-core/src/settings.rs @@ -116,7 +116,7 @@ pub const OCSF_SCHEMA_VERSION_VALUES: &[&str] = &["", "1.1", "1.3"]; pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[ // When true the supervisor emits OCSF-JSON records to stderr and writes - // JSONL to `/var/log/openshell-ocsf*.log` when file logging is available. + // JSONL to `/var/log/openshell-ocsf*.jsonl` when file logging is available. // Defaults to false; shorthand output remains unchanged. RegisteredSetting { key: "ocsf_json_enabled", diff --git a/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs b/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs index bc8aa60c74..7921d0dd56 100644 --- a/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs +++ b/crates/openshell-ocsf/src/tracing_layers/event_bridge.rs @@ -31,7 +31,7 @@ pub fn clone_current_event() -> Option { /// Emit an `OcsfEvent` through the tracing subscriber. /// /// The OCSF layers (`OcsfShorthandLayer`, `OcsfJsonlLayer`) format it -/// as shorthand (`openshell.log`) and JSONL (`openshell-ocsf.log`). +/// as human-readable shorthand and structured JSONL for their configured sinks. /// /// Both layers receive the event — `clone_current_event()` is non-consuming. pub fn emit_ocsf_event(event: OcsfEvent) { diff --git a/crates/openshell-supervisor-network/src/policy_local.rs b/crates/openshell-supervisor-network/src/policy_local.rs index 73c855c5fd..c3bb88b4ed 100644 --- a/crates/openshell-supervisor-network/src/policy_local.rs +++ b/crates/openshell-supervisor-network/src/policy_local.rs @@ -75,11 +75,8 @@ const MAX_DENIALS_LIMIT: usize = 100; /// denials. const DENIAL_LOG_FILES_TO_SCAN: usize = 2; const LOG_DIR: &str = "/var/log"; -/// Shorthand log filenames are `openshell.YYYY-MM-DD.log`. The trailing dot in -/// the prefix is intentional: it disambiguates from the OCSF JSONL appender's -/// `openshell-ocsf.YYYY-MM-DD.log`, which we never want to surface here (the -/// JSONL is opt-in via `ocsf_json_enabled` and not the source of truth for -/// `/v1/denials`). +/// `/v1/denials` reads `openshell.YYYY-MM-DD.log`, not the OCSF `.jsonl` files, +/// which are opt-in via `ocsf_json_enabled`. const SHORTHAND_LOG_PREFIX: &str = "openshell."; /// Defensive cap on per-line length returned to the agent so a pathological /// log entry (very long URL path, etc.) cannot blow up the response. @@ -471,8 +468,7 @@ fn collect_shorthand_log_files(log_dir: &Path, max_files: usize) -> std::io::Res let path = entry.path(); let name = entry.file_name(); let name = name.to_string_lossy(); - // `openshell.YYYY-MM-DD.log` only — the trailing dot in the prefix - // disambiguates from `openshell-ocsf.YYYY-MM-DD.log`. + // `/v1/denials` reads `openshell.YYYY-MM-DD.log`, not the OCSF `.jsonl` files. if !name.starts_with(SHORTHAND_LOG_PREFIX) || !name.ends_with(".log") { return None; } @@ -1654,11 +1650,11 @@ mod tests { #[tokio::test] async fn recent_denials_skips_jsonl_log_files() { - // The shorthand reader must not surface `openshell-ocsf.*.log` content + // The shorthand reader must not surface `openshell-ocsf.*.jsonl` content // even if a deny-looking line is present, so the response stays // independent of the JSONL appender's enabled state. let dir = tempfile::tempdir().unwrap(); - let jsonl = dir.path().join("openshell-ocsf.2026-05-06.log"); + let jsonl = dir.path().join("openshell-ocsf.2026-05-06.jsonl"); std::fs::write( &jsonl, r#"{"class_uid":4002,"action_id":2,"message":"DENIED","time":1}"#, diff --git a/crates/openshell-supervisor-process/src/skills/policy_advisor.md b/crates/openshell-supervisor-process/src/skills/policy_advisor.md index 8d792aaedb..d8facc839b 100644 --- a/crates/openshell-supervisor-process/src/skills/policy_advisor.md +++ b/crates/openshell-supervisor-process/src/skills/policy_advisor.md @@ -224,6 +224,6 @@ behavior: - `/var/log/openshell.YYYY-MM-DD.log` — shorthand log of sandbox activity. This is what `/v1/denials` reads from. -- `/var/log/openshell-ocsf.YYYY-MM-DD.log` — full OCSF JSON events, only +- `/var/log/openshell-ocsf.YYYY-MM-DD.jsonl` — full OCSF JSON events, only written when the `ocsf_json_enabled` setting is on. Not used by `/v1/denials`; useful for SIEM ingestion. diff --git a/crates/openshell-supervisor/src/logging.rs b/crates/openshell-supervisor/src/logging.rs index 15cd33783b..89b313c3dd 100644 --- a/crates/openshell-supervisor/src/logging.rs +++ b/crates/openshell-supervisor/src/logging.rs @@ -36,11 +36,11 @@ where .thread_name("openshell-console") .finish(console); let mut guards = vec![console_guard]; - let shorthand_file = file_writer(directory, "openshell", &mut guards); + let shorthand_file = file_writer(directory, "openshell", "log", &mut guards); let file_available = shorthand_file.is_some(); // Preserve the existing file-sink behavior while making stderr independent. let json_file = if file_available { - file_writer(directory, "openshell-ocsf", &mut guards) + file_writer(directory, "openshell-ocsf", "jsonl", &mut guards) } else { None }; @@ -73,12 +73,13 @@ where fn file_writer( directory: &Path, prefix: &str, + suffix: &str, guards: &mut Vec, ) -> Option { let roller = RollingFileAppender::builder() .rotation(Rotation::DAILY) .filename_prefix(prefix) - .filename_suffix("log") + .filename_suffix(suffix) .max_log_files(3) .build(directory) .ok()?; @@ -192,6 +193,7 @@ mod tests { .starts_with("openshell-ocsf.") }) .unwrap(); + assert_eq!(path.extension().unwrap(), "jsonl"); let file = std::fs::read_to_string(path).unwrap(); assert_eq!( serde_json::from_str::(file.trim()).unwrap(), diff --git a/docs/observability/accessing-logs.mdx b/docs/observability/accessing-logs.mdx index a2eca9172d..b17841b20f 100644 --- a/docs/observability/accessing-logs.mdx +++ b/docs/observability/accessing-logs.mdx @@ -78,7 +78,7 @@ The gateway merges the log and platform event sources before emitting, so events ## Direct Filesystem Access -The supervisor writes shorthand logs to `/var/log/openshell.YYYY-MM-DD.log` and, when enabled, OCSF JSONL to `/var/log/openshell-ocsf.YYYY-MM-DD.log` in its own filesystem. `openshell sandbox exec` runs in the workload and does not give access to the supervisor's files. +The supervisor writes shorthand logs to `/var/log/openshell.YYYY-MM-DD.log` and, when enabled, OCSF JSONL to `/var/log/openshell-ocsf.YYYY-MM-DD.jsonl` in its own filesystem. `openshell sandbox exec` runs in the workload and does not give access to the supervisor's files. | Compute driver | Supervisor log location | Local storage lifetime | |---|---|---| diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index cddc35fa3d..366546a8b6 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -261,9 +261,12 @@ The supervisor writes logs to `/var/log/` in its own filesystem, separate from t | File | Format | Rotation | |---|---|---| | `openshell.YYYY-MM-DD.log` | Shorthand + standard tracing | Daily | -| `openshell-ocsf.YYYY-MM-DD.log` | OCSF JSONL when enabled | Daily | +| `openshell-ocsf.YYYY-MM-DD.jsonl` | OCSF JSONL when enabled | Daily | -Each supervisor file appender is configured with a three-file limit. The shorthand appender's filename prefix also matches JSONL files, so its cleanup can delete JSONL history on filesystems that expose file creation timestamps. Do not rely on independent three-file retention for each format. +Each format rotates daily with an independent three-file limit. + +Configure supervisor OCSF collectors to match `openshell-ocsf.*.jsonl`. +Shorthand filenames and Windows MXC gateway filenames are unchanged. File output requires a writable `/var/log` and uses non-blocking writers that can drop events under pressure. If file logging cannot start, the supervisor falls back to stderr-only output. Enabling `ocsf_json_enabled` adds full `OCSF-JSON` records to stderr even without a file sink; see [OCSF JSON collection](/observability/ocsf-json-export#output-location). diff --git a/docs/observability/ocsf-json-export.mdx b/docs/observability/ocsf-json-export.mdx index 7e19d9d812..32abecd9df 100644 --- a/docs/observability/ocsf-json-export.mdx +++ b/docs/observability/ocsf-json-export.mdx @@ -88,11 +88,11 @@ kubectl -n logs -c supervisor --follow JSON stderr output works independently of writable log storage and the diagnostic log filter. Disabling the setting stops new JSON records. The console writer uses a bounded queue of 1,024 complete lines shared by diagnostics, shorthand, and JSON; when the queue is full, new lines are dropped rather than blocking enforcement. Delivery is best-effort, and retention depends on your runtime and external collector. The gateway log API and `openshell logs` continue to return shorthand. -When file logging is available, supervisors also write plain JSONL to `/var/log/openshell-ocsf.YYYY-MM-DD.log` in their own filesystem, separate from the workload. `openshell sandbox exec` cannot read those files. File output requires a writable `/var/log`; a missing file sink does not disable JSON stderr output. +When file logging is available, supervisors also write plain JSONL to `/var/log/openshell-ocsf.YYYY-MM-DD.jsonl` in their own filesystem, separate from the workload. `openshell sandbox exec` cannot read those files. File output requires a writable `/var/log`; a missing file sink does not disable JSON stderr output. This file contains supervisor-originated events, not the [sandbox-runtime events](/observability/accessing-logs#main-process-output) emitted into workload container logs. -Supervisor files rotate daily with a configured three-file limit, but cleanup can remove JSONL history earlier. See [supervisor file retention](/observability/logging#log-file-location) for the limits. Docker and Podman store them on tmpfs; Kubernetes stores them in the supervisor pod's `emptyDir`. For durable retention, arrange external collection before rotation or teardown. See [supervisor file access](/observability/accessing-logs#direct-filesystem-access) for collection paths and best-effort limits. +Supervisor files rotate daily with an independent three-file limit per format. Docker and Podman store them on tmpfs; Kubernetes stores them in the supervisor pod's `emptyDir`. For durable retention, arrange external collection before rotation or teardown. See [supervisor file access](/observability/accessing-logs#direct-filesystem-access) for collection paths and best-effort limits. Windows MXC gateway records use `%PROGRAMDATA%\OpenShell\logs` or `OPENSHELL_OCSF_LOG_DIR`. That sink also rotates daily and retains the three most recent files. The separate `[openshell.gateway.ocsf_log]` sink uses its configured path and retention. @@ -296,14 +296,14 @@ The JSONL file can be shipped to any tool that accepts OCSF-formatted data: After [copying supervisor files](/observability/accessing-logs#direct-filesystem-access) into `./supervisor-logs`, use `jq` to extract all denied connections: ```shell -jq -c 'select(.action == "Denied")' ./supervisor-logs/openshell-ocsf.*.log +jq -c 'select(.action == "Denied")' ./supervisor-logs/openshell-ocsf.*.jsonl ``` ## Relationship to Shorthand Logs Sandbox events identify the sandbox environment with `device.type_id: 99` (Other) and `device.type: "Sandbox"`. The operating system is reported separately in `device.os.name`; the device type does not classify the underlying host or compute backend. -The shorthand format in `openshell.YYYY-MM-DD.log` and the JSON format in `openshell-ocsf.YYYY-MM-DD.log` are derived from the same OCSF events. The shorthand is a human-readable projection; the JSON is the full structured record when no schema downgrade is configured. When `ocsf_schema_version` is set, the JSON export is a lossy projection of the internal event model. Both formats are generated at the same time from the same event data. +The shorthand format in `openshell.YYYY-MM-DD.log` and the JSON format in `openshell-ocsf.YYYY-MM-DD.jsonl` are derived from the same OCSF events. The shorthand is a human-readable projection; the JSON is the full structured record when no schema downgrade is configured. When `ocsf_schema_version` is set, the JSON export is a lossy projection of the internal event model. Both formats are generated at the same time from the same event data. Shorthand stderr output follows the diagnostic log filter; its local file is available when file logging starts successfully. JSON stderr and file export are opt-in through `ocsf_json_enabled`. File export requires the supervisor's JSONL sink to start successfully; stderr export is independent of that sink.