From 82bb60bdf95b311c39f3c768c488ba8ebe857a23 Mon Sep 17 00:00:00 2001 From: ABHAY PANDEY Date: Wed, 21 Oct 2026 08:37:03 +0530 Subject: [PATCH 1/4] feat(admin): trust and automation for operator console Add audit log API and Trust UI, signed webhook events for probe/mirror/SSL, OpenTelemetry probe and mirror metrics, redacted diagnostics export, and manual NIP-66 probe from the admin console. Closes #919 --- .changeset/admin-trust-automation-919.md | 5 + resources/admin/assets/admin-trust.js | 207 ++++++++++++++++++ resources/admin/assets/dashboard.js | 67 +++++- resources/admin/assets/notifications.js | 29 ++- resources/admin/dashboard.html | 76 ++++++- resources/default-settings.yaml | 4 + src/@types/admin-audit.ts | 20 ++ src/@types/operator-notifications.ts | 10 + src/admin/password-admin-auth-provider.ts | 10 +- src/app/relay-monitor-worker.ts | 5 + src/app/static-mirroring-worker.ts | 9 + .../admin/get-admin-audit-log-controller.ts | 19 ++ ...get-admin-diagnostics-export-controller.ts | 23 ++ .../admin/patch-notifications-controller.ts | 2 + .../admin/patch-settings-controller.ts | 4 + ...ost-admin-network-health-run-controller.ts | 103 +++++++++ .../post-ops-nip05-reverify-controller.ts | 2 + .../admin/post-settings-restore-controller.ts | 2 + src/factories/admin-auth-provider-factory.ts | 5 +- .../get-admin-audit-log-controller-factory.ts | 10 + ...n-diagnostics-export-controller-factory.ts | 7 + ...n-network-health-run-controller-factory.ts | 21 ++ src/routes/admin/index.ts | 21 ++ src/schemas/admin-notifications-schema.ts | 5 + src/services/notification-channels.ts | 12 +- src/telemetry/metrics.ts | 2 +- src/telemetry/ops-metrics.ts | 56 +++++ src/utils/admin-audit-store.ts | 68 ++++++ src/utils/admin-diagnostics-export.ts | 64 ++++++ src/utils/admin-notifications-settings.ts | 10 + src/utils/ops-trust-notifications.ts | 120 ++++++++++ src/utils/record-admin-audit.ts | 26 +++ test/unit/utils/admin-audit-store.spec.ts | 50 +++++ 33 files changed, 1066 insertions(+), 8 deletions(-) create mode 100644 .changeset/admin-trust-automation-919.md create mode 100644 resources/admin/assets/admin-trust.js create mode 100644 src/@types/admin-audit.ts create mode 100644 src/controllers/admin/get-admin-audit-log-controller.ts create mode 100644 src/controllers/admin/get-admin-diagnostics-export-controller.ts create mode 100644 src/controllers/admin/post-admin-network-health-run-controller.ts create mode 100644 src/factories/controllers/get-admin-audit-log-controller-factory.ts create mode 100644 src/factories/controllers/get-admin-diagnostics-export-controller-factory.ts create mode 100644 src/factories/controllers/post-admin-network-health-run-controller-factory.ts create mode 100644 src/telemetry/ops-metrics.ts create mode 100644 src/utils/admin-audit-store.ts create mode 100644 src/utils/admin-diagnostics-export.ts create mode 100644 src/utils/ops-trust-notifications.ts create mode 100644 src/utils/record-admin-audit.ts create mode 100644 test/unit/utils/admin-audit-store.spec.ts diff --git a/.changeset/admin-trust-automation-919.md b/.changeset/admin-trust-automation-919.md new file mode 100644 index 000000000..9b1fc3fc6 --- /dev/null +++ b/.changeset/admin-trust-automation-919.md @@ -0,0 +1,5 @@ +--- +"nostream": minor +--- + +Add admin trust and automation: Redis-backed audit log API and UI, signed HTTP webhooks for probe/mirror/SSL events, OpenTelemetry probe and mirror metrics, and redacted diagnostics export. diff --git a/resources/admin/assets/admin-trust.js b/resources/admin/assets/admin-trust.js new file mode 100644 index 000000000..e436916c8 --- /dev/null +++ b/resources/admin/assets/admin-trust.js @@ -0,0 +1,207 @@ +(() => { + const config = window.__ADMIN_DASHBOARD__ || { pathPrefix: '' } + const adminBase = `${config.pathPrefix || ''}/admin` + + const auditBody = document.getElementById('trust-audit-body') + if (!auditBody) { + return + } + + const elements = { + auditSync: document.getElementById('trust-audit-sync'), + auditRefresh: document.getElementById('trust-audit-refresh'), + auditBody, + diagnosticsDownload: document.getElementById('trust-diagnostics-download'), + diagnosticsPreviewButton: document.getElementById('trust-diagnostics-preview-button'), + diagnosticsPreviewWrap: document.getElementById('trust-diagnostics-preview-wrap'), + diagnosticsPreviewClose: document.getElementById('trust-diagnostics-preview-close'), + diagnosticsPreviewPlaceholder: document.getElementById('trust-diagnostics-preview-placeholder'), + diagnosticsPreviewBody: document.getElementById('trust-diagnostics-preview-body'), + diagnosticsStatus: document.getElementById('trust-diagnostics-status'), + error: document.getElementById('trust-error'), + } + + const escapeHtml = (value) => + String(value) + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", ''') + + const formatTime = (iso) => { + if (!iso) { + return '—' + } + const date = new Date(iso) + if (Number.isNaN(date.getTime())) { + return iso + } + return date.toISOString().replace('T', ' ').replace(/\.\d{3}Z$/, ' UTC') + } + + const adminFetch = async (path, options = {}) => { + const response = await fetch(`${adminBase}${path}`, { + credentials: 'same-origin', + ...options, + }) + if (response.status === 401) { + throw new Error('Unauthorized') + } + return response + } + + const showError = (message) => { + if (!elements.error) { + return + } + elements.error.textContent = message + elements.error.classList.remove('d-none') + } + + const hideError = () => { + elements.error?.classList.add('d-none') + } + + const renderAuditEntries = (entries) => { + elements.auditBody.replaceChildren() + if (!Array.isArray(entries) || entries.length === 0) { + elements.auditBody.innerHTML = 'No audit entries yet.' + return + } + + entries.forEach((entry) => { + const row = document.createElement('tr') + const detail = + entry.detail && Object.keys(entry.detail).length > 0 + ? `${escapeHtml(JSON.stringify(entry.detail))}` + : '—' + row.innerHTML = ` + ${escapeHtml(formatTime(entry.at))} + ${escapeHtml(entry.action ?? '')} + ${escapeHtml(entry.remoteAddress ?? '—')} + ${detail} + ` + elements.auditBody.appendChild(row) + }) + } + + const loadAuditLog = async () => { + hideError() + if (elements.auditSync) { + elements.auditSync.textContent = '> audit: loading…' + } + + try { + const response = await adminFetch('/audit-log?limit=100') + if (!response.ok) { + showError('Failed to load audit log') + if (elements.auditSync) { + elements.auditSync.textContent = '> audit: load failed' + } + return + } + + const body = await response.json() + renderAuditEntries(body.entries ?? []) + if (elements.auditSync) { + elements.auditSync.textContent = `> audit: ${body.entries?.length ?? 0} entries` + } + } catch (error) { + if (error instanceof Error && error.message === 'Unauthorized') { + globalThis.dispatchEvent(new CustomEvent('admin-unauthorized')) + return + } + showError('Network error while loading audit log') + } + } + + const downloadDiagnostics = async () => { + hideError() + if (elements.diagnosticsStatus) { + elements.diagnosticsStatus.textContent = 'Preparing export…' + } + + try { + const response = await adminFetch('/diagnostics/export?download=1') + if (!response.ok) { + showError('Diagnostics export failed') + return + } + + const blob = await response.blob() + const url = URL.createObjectURL(blob) + const anchor = document.createElement('a') + anchor.href = url + anchor.download = `nostream-diagnostics-${new Date().toISOString().replace(/[:.]/g, '-')}.json` + anchor.click() + URL.revokeObjectURL(url) + if (elements.diagnosticsStatus) { + elements.diagnosticsStatus.textContent = 'Download started (secrets redacted).' + } + } catch { + showError('Network error during diagnostics export') + } + } + + const closeDiagnosticsPreview = () => { + elements.diagnosticsPreviewWrap?.classList.add('d-none') + elements.diagnosticsPreviewPlaceholder?.classList.remove('d-none') + if (elements.diagnosticsPreviewBody) { + elements.diagnosticsPreviewBody.textContent = '' + } + if (elements.diagnosticsStatus) { + elements.diagnosticsStatus.innerHTML = + '> diagnostics: idle' + } + } + + const showDiagnosticsPreview = (payload) => { + elements.diagnosticsPreviewPlaceholder?.classList.add('d-none') + elements.diagnosticsPreviewWrap?.classList.remove('d-none') + if (elements.diagnosticsPreviewBody) { + elements.diagnosticsPreviewBody.textContent = JSON.stringify(payload, null, 2) + } + if (elements.diagnosticsStatus) { + elements.diagnosticsStatus.textContent = `Generated ${payload.generatedAt ?? ''} · version ${payload.version ?? 'unknown'}` + } + } + + const previewDiagnostics = async () => { + hideError() + if (elements.diagnosticsStatus) { + elements.diagnosticsStatus.textContent = 'Loading preview…' + } + + try { + const response = await adminFetch('/diagnostics/export') + if (!response.ok) { + showError('Diagnostics preview failed') + return + } + + const payload = await response.json() + showDiagnosticsPreview(payload) + } catch { + showError('Network error during diagnostics preview') + } + } + + elements.auditRefresh?.addEventListener('click', () => { + void loadAuditLog() + }) + elements.diagnosticsDownload?.addEventListener('click', () => { + void downloadDiagnostics() + }) + elements.diagnosticsPreviewButton?.addEventListener('click', () => { + void previewDiagnostics() + }) + elements.diagnosticsPreviewClose?.addEventListener('click', () => { + closeDiagnosticsPreview() + }) + + window.__ADMIN_TRUST__ = { + load: loadAuditLog, + closeDiagnosticsPreview, + } +})() diff --git a/resources/admin/assets/dashboard.js b/resources/admin/assets/dashboard.js index b008c7b36..0674e012f 100644 --- a/resources/admin/assets/dashboard.js +++ b/resources/admin/assets/dashboard.js @@ -41,6 +41,7 @@ const networkHealthRunAt = document.getElementById('network-health-run-at') const networkHealthTimelineWrap = document.getElementById('network-health-timeline-wrap') const networkHealthTimeline = document.getElementById('network-health-timeline') + const networkHealthRunButton = document.getElementById('network-health-run-button') let settingsLoaded = false let settingsLoading = false @@ -220,11 +221,22 @@ streamStatus.appendChild(document.createTextNode(label)) } - const updateSystemAlert = (snapshot) => { + let activeDashboardView = 'metrics-view' + + const hideSystemAlert = () => { if (!systemAlert) { return } + systemAlert.className = 'system-alert d-none' + systemAlert.textContent = '' + } + + const updateSystemAlert = (snapshot) => { + if (!systemAlert || activeDashboardView !== 'metrics-view') { + return + } + if (lastSnapshot?.timestamp && Date.now() - lastSnapshot.timestamp > staleThresholdMs) { systemAlert.textContent = '[WARN] Metrics stale — waiting for backend update.' systemAlert.className = 'system-alert degraded' @@ -725,6 +737,42 @@ }) } + const runNetworkHealthProbe = async () => { + if (!networkHealthRunButton) { + return + } + + networkHealthRunButton.disabled = true + setNetworkHealthSyncLine('manual probe running…') + + try { + const response = await fetch(`${adminBase}/network-health/run`, { + method: 'POST', + credentials: 'include', + }) + + if (response.status === 401) { + showLogin() + return + } + + if (!response.ok) { + setNetworkHealthSyncLine('manual probe failed') + return + } + + networkHealthHistoryPinned = false + selectedNetworkHealthRunAt = null + await refreshNetworkHealth() + await refreshNetworkHealthHistory() + setNetworkHealthSyncLine('manual probe completed') + } catch { + setNetworkHealthSyncLine('network error during manual probe') + } finally { + networkHealthRunButton.disabled = false + } + } + const refreshNetworkHealth = async () => { try { const response = await fetch(`${adminBase}/network-health`, { @@ -1476,6 +1524,11 @@ } const showView = (viewId) => { + activeDashboardView = viewId + if (viewId !== 'metrics-view') { + hideSystemAlert() + } + dashboardViews.forEach((view) => { view.classList.toggle('d-none', view.id !== viewId) }) @@ -1509,6 +1562,14 @@ return } + if (viewId === 'trust-view') { + stopNetworkHealthPolling() + stopMetricsStream() + window.__ADMIN_OPS__?.stop?.() + window.__ADMIN_TRUST__?.load?.() + return + } + if (viewId === 'metrics-view') { stopNetworkHealthPolling() window.__ADMIN_OPS__?.stop?.() @@ -1797,6 +1858,10 @@ void loadSettingsData(true) }) + networkHealthRunButton?.addEventListener('click', () => { + void runNetworkHealthProbe() + }) + if (liveClock) { liveClock.textContent = formatClock() setInterval(() => { diff --git a/resources/admin/assets/notifications.js b/resources/admin/assets/notifications.js index 48ab7fe28..0642baa4d 100644 --- a/resources/admin/assets/notifications.js +++ b/resources/admin/assets/notifications.js @@ -60,6 +60,10 @@ 'admission.invoice.failed': true, 'settings.changed': true, 'relay.restarted': false, + 'probe.failed': true, + 'probe.recovered': true, + 'ssl.expiring': true, + 'mirror.disconnected': true, }) const newTargetId = () => { @@ -74,6 +78,7 @@ type: 'http', enabled: true, url: '', + secret: '', botToken: '', chatId: '', }) @@ -109,6 +114,7 @@ const type = card.querySelector('[data-target-field="type"]')?.value ?? 'http' const enabled = card.querySelector('[data-target-field="enabled"]')?.checked ?? true const urlInput = card.querySelector('[data-target-field="url"]') + const secretInput = card.querySelector('[data-target-field="secret"]') const botTokenInput = card.querySelector('[data-target-field="botToken"]') const chatIdInput = card.querySelector('[data-target-field="chatId"]') @@ -121,6 +127,13 @@ target.url = REDACTED } + const secret = secretInput?.value?.trim() ?? '' + if (secret && secret !== REDACTED) { + target.secret = secret + } else if (secret === REDACTED) { + target.secret = REDACTED + } + const botToken = botTokenInput?.value?.trim() ?? '' if (botToken && botToken !== REDACTED) { target.botToken = botToken @@ -230,6 +243,20 @@ urlInput.value = target.url ?? '' urlCol.append(urlLabel, urlInput) + const secretCol = document.createElement('div') + secretCol.className = 'col-12 col-md-6 col-lg-3' + const secretLabel = document.createElement('label') + secretLabel.className = 'form-label field-label small mb-1' + secretLabel.textContent = 'Signing secret' + const secretInput = document.createElement('input') + secretInput.type = 'password' + secretInput.autocomplete = 'off' + secretInput.className = 'form-control console-input form-control-sm' + secretInput.dataset.targetField = 'secret' + secretInput.placeholder = 'HTTP HMAC (optional)' + secretInput.value = target.secret ?? '' + secretCol.append(secretLabel, secretInput) + const botCol = document.createElement('div') botCol.className = 'col-12 col-md-6 col-lg-3' const botLabel = document.createElement('label') @@ -256,7 +283,7 @@ chatInput.value = target.chatId ?? '' chatCol.append(chatLabel, chatInput) - fieldsRow.append(urlCol, botCol, chatCol) + fieldsRow.append(urlCol, secretCol, botCol, chatCol) card.append(header, statusEl, fieldsRow) removeButton.addEventListener('click', () => { diff --git a/resources/admin/dashboard.html b/resources/admin/dashboard.html index 2b514deac..808ea93e6 100644 --- a/resources/admin/dashboard.html +++ b/resources/admin/dashboard.html @@ -66,6 +66,7 @@

Sign in

> Navigation

+ @@ -248,7 +249,10 @@

Worker ops

-

Tor / I2P & network probes

+
+

Tor / I2P & network probes

+ +

> probes: waiting for snapshot…

No external probe results yet. Enable NIP-66 relay monitoring to populate this panel.
@@ -411,6 +415,30 @@

Operator notifications

+
+
+ + +
+
+
+
+ + +
+
+
+
+ + +
+
+
+
+ + +
+
@@ -480,6 +508,51 @@

Delivery history

+
+
+

Diagnostics export

+

Download a redacted support bundle: version, health, settings, workers, mirrors, and latest probe snapshot.

+ +
+ + +
+

> diagnostics: idle

+

Preview not loaded.

+
+
+

JSON Diagnostics preview

+ +
+ 

+            

+          

+
+          

+            

+              
Admin audit log

+              
+            

+            
> audit: not loaded

+            

+              
+                
+                
+                  
+                    
+                    
+                    
+                    
+                  
+                
+                
+                  
+                
+              
Time (UTC)ActionRemoteDetail
No entries loaded.

+            

+          

+        

+
         

           

             
Configuration

@@ -527,5 +600,6 @@ 
Preview diff

     
     
     
+    
   
 
diff --git a/resources/default-settings.yaml b/resources/default-settings.yaml
index d4d9f02a5..a94c23ef2 100755
--- a/resources/default-settings.yaml
+++ b/resources/default-settings.yaml
@@ -384,6 +384,10 @@ admin:
       admission.invoice.failed: true
       settings.changed: true
       relay.restarted: false
+      probe.failed: true
+      probe.recovered: true
+      ssl.expiring: true
+      mirror.disconnected: true
     retry:
       maxAttempts: 5
       baseDelayMs: 1000
diff --git a/src/@types/admin-audit.ts b/src/@types/admin-audit.ts
new file mode 100644
index 000000000..d258594c0
--- /dev/null
+++ b/src/@types/admin-audit.ts
@@ -0,0 +1,20 @@
+export type AdminAuditAction =
+  | 'admin.login'
+  | 'admin.logout'
+  | 'settings.updated'
+  | 'settings.restored'
+  | 'notifications.updated'
+  | 'probe.manual_run'
+  | 'nip05.reverify'
+
+export interface AdminAuditEntry {
+  id: string
+  at: string
+  action: AdminAuditAction
+  remoteAddress?: string
+  detail?: Record
+}
+
+export const ADMIN_AUDIT_LOG_KEY = 'admin:audit:log'
+export const MAX_ADMIN_AUDIT_ENTRIES = 2000
+export const MAX_ADMIN_AUDIT_API_LIMIT = 500
diff --git a/src/@types/operator-notifications.ts b/src/@types/operator-notifications.ts
index 037963b82..4a7debfec 100644
--- a/src/@types/operator-notifications.ts
+++ b/src/@types/operator-notifications.ts
@@ -4,6 +4,10 @@ export enum OperatorNotificationEventType {
   ADMISSION_INVOICE_FAILED = 'admission.invoice.failed',
   SETTINGS_CHANGED = 'settings.changed',
   RELAY_RESTARTED = 'relay.restarted',
+  PROBE_FAILED = 'probe.failed',
+  PROBE_RECOVERED = 'probe.recovered',
+  SSL_EXPIRING = 'ssl.expiring',
+  MIRROR_DISCONNECTED = 'mirror.disconnected',
 }
 
 export type OperatorNotificationChannelType = 'http' | 'discord' | 'slack' | 'telegram'
@@ -13,6 +17,8 @@ export interface OperatorNotificationTarget {
   type: OperatorNotificationChannelType
   enabled: boolean
   url?: string
+  /** HMAC-SHA256 signing secret for HTTP webhook payloads (optional). */
+  secret?: string
   botToken?: string
   chatId?: string
 }
@@ -23,6 +29,10 @@ export interface OperatorNotificationEventToggles {
   'admission.invoice.failed'?: boolean
   'settings.changed'?: boolean
   'relay.restarted'?: boolean
+  'probe.failed'?: boolean
+  'probe.recovered'?: boolean
+  'ssl.expiring'?: boolean
+  'mirror.disconnected'?: boolean
 }
 
 export interface OperatorNotificationRetrySettings {
diff --git a/src/admin/password-admin-auth-provider.ts b/src/admin/password-admin-auth-provider.ts
index 9111934ae..9ac041546 100644
--- a/src/admin/password-admin-auth-provider.ts
+++ b/src/admin/password-admin-auth-provider.ts
@@ -15,8 +15,13 @@ import {
 } from '../utils/admin-session'
 import { validateSchema } from '../utils/validation'
 
+export type AdminAuthAuditHook = (event: 'login' | 'logout', request: Request) => void | Promise
+
 export class PasswordAdminAuthProvider implements IAdminAuthProvider {
-  public constructor(private readonly settings: () => Settings) {}
+  public constructor(
+    private readonly settings: () => Settings,
+    private readonly auditHook?: AdminAuthAuditHook,
+  ) {}
 
   public async handleLogin(request: Request, response: Response): Promise {
     const validation = validateSchema(adminLoginBodySchema)(request.body)
@@ -37,6 +42,8 @@ export class PasswordAdminAuthProvider implements IAdminAuthProvider {
     try {
       const token = createAdminSessionToken(expiresAt)
 
+      void this.auditHook?.('login', request)
+
       response
         .status(200)
         .setHeader('content-type', 'application/json')
@@ -49,6 +56,7 @@ export class PasswordAdminAuthProvider implements IAdminAuthProvider {
 
   public handleLogout(request: Request, response: Response): void {
     const currentSettings = this.settings()
+    void this.auditHook?.('logout', request)
 
     response
       .status(200)
diff --git a/src/app/relay-monitor-worker.ts b/src/app/relay-monitor-worker.ts
index 95aa2fda3..57a178ea5 100644
--- a/src/app/relay-monitor-worker.ts
+++ b/src/app/relay-monitor-worker.ts
@@ -5,7 +5,9 @@ import { IRelayProbeSnapshotStore, RelayProbeRunSnapshot } from '../@types/relay
 import { Settings } from '../@types/settings'
 import { createLogger } from '../factories/logger-factory'
 import { INip66EventPublisher } from '../services/nip66-event-publisher'
+import { recordProbeRunMetrics } from '../telemetry/ops-metrics'
 import { shutdownMetricsTelemetry } from '../telemetry/metrics'
+import { notifyProbeRunTransition } from '../utils/ops-trust-notifications'
 import { ADMIN_OPS_HEARTBEAT_KEEPALIVE_MS, publishAdminOpsWorkerHeartbeatSafely } from '../utils/admin-ops-store'
 import { getMonitorPrivateKey } from '../utils/monitor-identity'
 import { getEffectiveProbeIntervalSeconds, getNip66HistoryRetention, getProbeIntervalMs } from '../utils/nip66-schedule'
@@ -138,6 +140,7 @@ export class RelayMonitorWorker implements IRunnable {
   }
 
   private async onSchedule(): Promise {
+    const probeRunStartedMs = Date.now()
     const lastRunStartedAt = new Date().toISOString()
     await this.publishOpsHeartbeat({ lastRunStartedAt })
 
@@ -203,6 +206,8 @@ export class RelayMonitorWorker implements IRunnable {
     const expirySeconds = getEffectiveProbeIntervalSeconds(currentSettings) * 2
 
     await this.snapshotStore.saveLatest(snapshot, expirySeconds)
+    recordProbeRunMetrics(snapshot.status, Date.now() - probeRunStartedMs)
+    await notifyProbeRunTransition(this.opsCache, snapshot)
     if (this.historyStore) {
       try {
         await this.historyStore.append(snapshot, getNip66HistoryRetention(currentSettings))
diff --git a/src/app/static-mirroring-worker.ts b/src/app/static-mirroring-worker.ts
index 0e5fac5b3..2fc1aa01c 100644
--- a/src/app/static-mirroring-worker.ts
+++ b/src/app/static-mirroring-worker.ts
@@ -12,7 +12,9 @@ import { EventLimits, FeeSchedule, Mirror, Settings } from '../@types/settings'
 import { WebSocketServerAdapterEvent } from '../constants/adapter'
 import { EventExpirationTimeMetadataKey } from '../constants/base'
 import { createLogger } from '../factories/logger-factory'
+import { recordMirrorReconnectMetric } from '../telemetry/ops-metrics'
 import { shutdownMetricsTelemetry } from '../telemetry/metrics'
+import { notifyMirrorDisconnected } from '../utils/ops-trust-notifications'
 import {
   ADMIN_OPS_HEARTBEAT_KEEPALIVE_MS,
   publishAdminOpsMirrorHeartbeatSafely,
@@ -198,6 +200,13 @@ export class StaticMirroringWorker implements IRunnable {
           logger(`disconnected (${code}): ${reason.toString()}`)
           this.connectionState = 'closed'
           this.reconnectCount += 1
+          recordMirrorReconnectMetric(this.mirrorIndex)
+          void notifyMirrorDisconnected(this.opsCache, {
+            index: this.mirrorIndex,
+            targetUrl: config.address,
+            code,
+            reason: reason.toString(),
+          })
           void this.publishOpsHeartbeats()
 
           setTimeout(() => {
diff --git a/src/controllers/admin/get-admin-audit-log-controller.ts b/src/controllers/admin/get-admin-audit-log-controller.ts
new file mode 100644
index 000000000..a08bdadea
--- /dev/null
+++ b/src/controllers/admin/get-admin-audit-log-controller.ts
@@ -0,0 +1,19 @@
+import { Request, Response } from 'express'
+
+import { MAX_ADMIN_AUDIT_API_LIMIT } from '../../@types/admin-audit'
+import { IController } from '../../@types/controllers'
+import { AdminAuditStore } from '../../utils/admin-audit-store'
+
+export class GetAdminAuditLogController implements IController {
+  public constructor(private readonly auditStore: AdminAuditStore) {}
+
+  public async handleRequest(request: Request, response: Response): Promise {
+    const limitCandidate = Number(request.query.limit ?? 100)
+    const limit = Number.isFinite(limitCandidate)
+      ? Math.min(Math.max(Math.floor(limitCandidate), 1), MAX_ADMIN_AUDIT_API_LIMIT)
+      : 100
+
+    const entries = await this.auditStore.list(limit)
+    response.status(200).setHeader('content-type', 'application/json').send({ entries })
+  }
+}
diff --git a/src/controllers/admin/get-admin-diagnostics-export-controller.ts b/src/controllers/admin/get-admin-diagnostics-export-controller.ts
new file mode 100644
index 000000000..94fae45c5
--- /dev/null
+++ b/src/controllers/admin/get-admin-diagnostics-export-controller.ts
@@ -0,0 +1,23 @@
+import { Request, Response } from 'express'
+
+import { IController } from '../../@types/controllers'
+import { Settings } from '../../@types/settings'
+import { buildAdminDiagnosticsExport } from '../../utils/admin-diagnostics-export'
+import { loadMergedSettings } from '../../utils/settings-config'
+
+export class GetAdminDiagnosticsExportController implements IController {
+  public constructor(private readonly getSettings: () => Settings = loadMergedSettings) {}
+
+  public async handleRequest(request: Request, response: Response): Promise {
+    const payload = await buildAdminDiagnosticsExport(this.getSettings)
+    const download = request.query.download === '1' || request.query.download === 'true'
+
+    response.status(200).setHeader('content-type', 'application/json')
+    if (download) {
+      const stamp = payload.generatedAt.replace(/[:.]/g, '-')
+      response.setHeader('content-disposition', `attachment; filename="nostream-diagnostics-${stamp}.json"`)
+    }
+
+    response.send(payload)
+  }
+}
diff --git a/src/controllers/admin/patch-notifications-controller.ts b/src/controllers/admin/patch-notifications-controller.ts
index fbfa5f794..2e3545d49 100644
--- a/src/controllers/admin/patch-notifications-controller.ts
+++ b/src/controllers/admin/patch-notifications-controller.ts
@@ -21,6 +21,7 @@ import {
   validateSettings,
 } from '../../utils/settings-config'
 import { validateSchema } from '../../utils/validation'
+import { recordAdminAuditFromRequest } from '../../utils/record-admin-audit'
 
 const logger = createLogger('patch-admin-notifications-controller')
 
@@ -60,6 +61,7 @@ export class PatchAdminNotificationsController implements IController {
       changes: [{ path: 'admin.notifications', reload: 'hot-reload' }],
       remoteAddress: request.ip,
     })
+    await recordAdminAuditFromRequest('notifications.updated', request)
 
     try {
       await this.notificationOutboxRepository.enqueue(OperatorNotificationEventType.SETTINGS_CHANGED, {
diff --git a/src/controllers/admin/patch-settings-controller.ts b/src/controllers/admin/patch-settings-controller.ts
index 804af2215..f67eb099e 100644
--- a/src/controllers/admin/patch-settings-controller.ts
+++ b/src/controllers/admin/patch-settings-controller.ts
@@ -22,6 +22,7 @@ import { INotificationOutboxRepository } from '../../@types/repositories'
 import { OperatorNotificationEventType } from '../../@types/operator-notifications'
 import { redactSettingsSecrets } from '../../utils/settings-redaction'
 import { createLogger } from '../../factories/logger-factory'
+import { recordAdminAuditFromRequest } from '../../utils/record-admin-audit'
 
 const logger = createLogger('patch-admin-settings-controller')
 
@@ -80,6 +81,9 @@ export class PatchAdminSettingsController implements IController {
       changes: updatedChanges.map(({ path, reload }) => ({ path, reload })),
       remoteAddress: request.ip,
     })
+    await recordAdminAuditFromRequest('settings.updated', request, {
+      changes: updatedChanges.map(({ path, reload }) => ({ path, reload })),
+    })
 
     try {
       await this.notificationOutboxRepository.enqueue(OperatorNotificationEventType.SETTINGS_CHANGED, {
diff --git a/src/controllers/admin/post-admin-network-health-run-controller.ts b/src/controllers/admin/post-admin-network-health-run-controller.ts
new file mode 100644
index 000000000..525b76b19
--- /dev/null
+++ b/src/controllers/admin/post-admin-network-health-run-controller.ts
@@ -0,0 +1,103 @@
+import { Request, Response } from 'express'
+
+import { ICacheAdapter } from '../../@types/adapters'
+import { IController } from '../../@types/controllers'
+import { IRelayProbeSnapshotStore, RelayProbeRunSnapshot } from '../../@types/relay-probe-snapshot'
+import { Settings } from '../../@types/settings'
+import { buildProbeOptions, RunProbeFn } from '../../app/relay-monitor-worker'
+import { createLogger } from '../../factories/logger-factory'
+import { recordProbeRunMetrics } from '../../telemetry/ops-metrics'
+import { recordAdminAuditFromRequest } from '../../utils/record-admin-audit'
+import { notifyProbeRunTransition } from '../../utils/ops-trust-notifications'
+import { getEffectiveProbeIntervalSeconds, getNip66HistoryRetention } from '../../utils/nip66-schedule'
+import { runProbe } from '../../utils/relay-probe'
+import { RelayProbeHistoryStore } from '../../utils/relay-probe-history'
+import { deriveRelayProbeRunStatus, serializeProbeResults } from '../../utils/relay-probe-snapshot'
+import {
+  filterValidProbeTargets,
+  resolveProbeTargets,
+  resolvePublicProbeTargetKeys,
+} from '../../utils/relay-probe-targets'
+import { loadMergedSettings } from '../../utils/settings-config'
+import { ProbeResult } from '../../utils/relay-probe/types'
+
+const logger = createLogger('post-admin-network-health-run')
+
+export class PostAdminNetworkHealthRunController implements IController {
+  public constructor(
+    private readonly snapshotStore: IRelayProbeSnapshotStore,
+    private readonly getSettings: () => Settings = loadMergedSettings,
+    private readonly probeRunner: RunProbeFn = runProbe,
+    private readonly historyStore?: RelayProbeHistoryStore,
+    private readonly opsCache?: ICacheAdapter,
+  ) {}
+
+  public async handleRequest(request: Request, response: Response): Promise {
+    const started = Date.now()
+    const currentSettings = this.getSettings()
+
+    if (!currentSettings.nip66?.enabled) {
+      response.status(409).setHeader('content-type', 'application/json').send({ error: 'NIP-66 monitoring is disabled' })
+      return
+    }
+
+    const configuredTargets = resolveProbeTargets(currentSettings)
+    const { valid, invalid } = filterValidProbeTargets(configuredTargets)
+
+    if (valid.length === 0) {
+      response
+        .status(400)
+        .setHeader('content-type', 'application/json')
+        .send({ error: 'No valid probe targets configured', invalid })
+      return
+    }
+
+    const probeOptions = buildProbeOptions(currentSettings)
+    const results: ProbeResult[] = []
+
+    for (const target of valid) {
+      try {
+        results.push(await this.probeRunner(target, probeOptions))
+      } catch (error) {
+        logger.error('manual probe failed for %s: %o', target, error)
+      }
+    }
+
+    if (results.length === 0) {
+      response.status(502).setHeader('content-type', 'application/json').send({ error: 'Probe run produced no results' })
+      return
+    }
+
+    const publicTargetKeys = resolvePublicProbeTargetKeys(currentSettings)
+    const snapshot: RelayProbeRunSnapshot = {
+      runAt: new Date().toISOString(),
+      targets: valid,
+      results: serializeProbeResults(results),
+      status: deriveRelayProbeRunStatus(results),
+      probeContext: {
+        configuredRelayUrl: currentSettings.info?.relay_url,
+        publicTargetKeys: [...publicTargetKeys],
+      },
+    }
+
+    const expirySeconds = getEffectiveProbeIntervalSeconds(currentSettings) * 2
+    await this.snapshotStore.saveLatest(snapshot, expirySeconds)
+
+    if (this.historyStore) {
+      try {
+        await this.historyStore.append(snapshot, getNip66HistoryRetention(currentSettings))
+      } catch (error) {
+        logger.error('failed to save probe history: %o', error)
+      }
+    }
+
+    recordProbeRunMetrics(snapshot.status, Date.now() - started)
+    await notifyProbeRunTransition(this.opsCache, snapshot)
+    await recordAdminAuditFromRequest('probe.manual_run', request, {
+      status: snapshot.status,
+      targets: valid,
+    })
+
+    response.status(200).setHeader('content-type', 'application/json').send({ snapshot })
+  }
+}
diff --git a/src/controllers/admin/post-ops-nip05-reverify-controller.ts b/src/controllers/admin/post-ops-nip05-reverify-controller.ts
index ccc332a78..57adc09d7 100644
--- a/src/controllers/admin/post-ops-nip05-reverify-controller.ts
+++ b/src/controllers/admin/post-ops-nip05-reverify-controller.ts
@@ -7,6 +7,7 @@ import { getNip05VerifyExpirationMs, toAdminNip05Row } from '../../utils/admin-o
 import { reverifyNip05Verification } from '../../utils/nip05-reverification'
 import { loadMergedSettings } from '../../utils/settings-config'
 import { pubkeySchema } from '../../schemas/base-schema'
+import { recordAdminAuditFromRequest } from '../../utils/record-admin-audit'
 
 export class PostAdminOpsNip05ReverifyController implements IController {
   public constructor(
@@ -35,6 +36,7 @@ export class PostAdminOpsNip05ReverifyController implements IController {
 
     try {
       const updated = await reverifyNip05Verification(existing, this.nip05VerificationRepository)
+      await recordAdminAuditFromRequest('nip05.reverify', request, { pubkey: pubkeyResult.data })
       const verifyExpirationMs = getNip05VerifyExpirationMs(settings)
       response
         .status(200)
diff --git a/src/controllers/admin/post-settings-restore-controller.ts b/src/controllers/admin/post-settings-restore-controller.ts
index aadcca090..aded18e40 100644
--- a/src/controllers/admin/post-settings-restore-controller.ts
+++ b/src/controllers/admin/post-settings-restore-controller.ts
@@ -4,6 +4,7 @@ import { IController } from '../../@types/controllers'
 import { adminSettingsRestoreBodySchema } from '../../schemas/admin-settings-schema'
 import { appendSettingsAuditLog, restoreSettingsBackup } from '../../utils/settings-config'
 import { validateSchema } from '../../utils/validation'
+import { recordAdminAuditFromRequest } from '../../utils/record-admin-audit'
 
 export class PostAdminSettingsRestoreController implements IController {
   public async handleRequest(request: Request, response: Response): Promise {
@@ -20,6 +21,7 @@ export class PostAdminSettingsRestoreController implements IController {
         filename: validation.value.filename,
         remoteAddress: request.ip,
       })
+      await recordAdminAuditFromRequest('settings.restored', request, { filename: validation.value.filename })
 
       response.status(200).setHeader('content-type', 'application/json').send({ ok: true, filename: validation.value.filename })
     } catch (error) {
diff --git a/src/factories/admin-auth-provider-factory.ts b/src/factories/admin-auth-provider-factory.ts
index 162670231..501f95abe 100644
--- a/src/factories/admin-auth-provider-factory.ts
+++ b/src/factories/admin-auth-provider-factory.ts
@@ -1,7 +1,10 @@
 import { PasswordAdminAuthProvider } from '../admin/password-admin-auth-provider'
 import { IAdminAuthProvider } from '../@types/admin'
+import { recordAdminAuditFromRequest } from '../utils/record-admin-audit'
 import { createSettings } from './settings-factory'
 
 export const createAdminAuthProvider = (): IAdminAuthProvider => {
-  return new PasswordAdminAuthProvider(createSettings)
+  return new PasswordAdminAuthProvider(createSettings, async (event, request) => {
+    await recordAdminAuditFromRequest(event === 'login' ? 'admin.login' : 'admin.logout', request)
+  })
 }
diff --git a/src/factories/controllers/get-admin-audit-log-controller-factory.ts b/src/factories/controllers/get-admin-audit-log-controller-factory.ts
new file mode 100644
index 000000000..df7a5c9db
--- /dev/null
+++ b/src/factories/controllers/get-admin-audit-log-controller-factory.ts
@@ -0,0 +1,10 @@
+import { RedisAdapter } from '../../adapters/redis-adapter'
+import { IController } from '../../@types/controllers'
+import { getCacheClient } from '../../cache/client'
+import { GetAdminAuditLogController } from '../../controllers/admin/get-admin-audit-log-controller'
+import { AdminAuditStore } from '../../utils/admin-audit-store'
+
+export const createGetAdminAuditLogController = (): IController => {
+  const auditStore = new AdminAuditStore(new RedisAdapter(getCacheClient()))
+  return new GetAdminAuditLogController(auditStore)
+}
diff --git a/src/factories/controllers/get-admin-diagnostics-export-controller-factory.ts b/src/factories/controllers/get-admin-diagnostics-export-controller-factory.ts
new file mode 100644
index 000000000..eb630c141
--- /dev/null
+++ b/src/factories/controllers/get-admin-diagnostics-export-controller-factory.ts
@@ -0,0 +1,7 @@
+import { IController } from '../../@types/controllers'
+import { GetAdminDiagnosticsExportController } from '../../controllers/admin/get-admin-diagnostics-export-controller'
+import { createSettings } from '../settings-factory'
+
+export const createGetAdminDiagnosticsExportController = (): IController => {
+  return new GetAdminDiagnosticsExportController(createSettings)
+}
diff --git a/src/factories/controllers/post-admin-network-health-run-controller-factory.ts b/src/factories/controllers/post-admin-network-health-run-controller-factory.ts
new file mode 100644
index 000000000..783b86f6b
--- /dev/null
+++ b/src/factories/controllers/post-admin-network-health-run-controller-factory.ts
@@ -0,0 +1,21 @@
+import { RedisAdapter } from '../../adapters/redis-adapter'
+import { IController } from '../../@types/controllers'
+import { getCacheClient } from '../../cache/client'
+import { PostAdminNetworkHealthRunController } from '../../controllers/admin/post-admin-network-health-run-controller'
+import { createSettings } from '../settings-factory'
+import { RelayProbeHistoryStore } from '../../utils/relay-probe-history'
+import { RelayProbeSnapshotStore } from '../../utils/relay-probe-snapshot'
+
+export const createPostAdminNetworkHealthRunController = (): IController => {
+  const cacheAdapter = new RedisAdapter(getCacheClient())
+  const snapshotStore = new RelayProbeSnapshotStore(cacheAdapter)
+  const historyStore = new RelayProbeHistoryStore(cacheAdapter)
+
+  return new PostAdminNetworkHealthRunController(
+    snapshotStore,
+    createSettings,
+    undefined,
+    historyStore,
+    cacheAdapter,
+  )
+}
diff --git a/src/routes/admin/index.ts b/src/routes/admin/index.ts
index 0449675e5..97269f9fc 100644
--- a/src/routes/admin/index.ts
+++ b/src/routes/admin/index.ts
@@ -22,6 +22,9 @@ import { createGetAdminOpsWorkersController } from '../../factories/controllers/
 import { createGetAdminOpsNip05VerificationsController } from '../../factories/controllers/get-admin-ops-nip05-verifications-controller-factory'
 import { createGetAdminOpsStorageCountsController } from '../../factories/controllers/get-admin-ops-storage-counts-controller-factory'
 import { createPostAdminOpsNip05ReverifyController } from '../../factories/controllers/post-admin-ops-nip05-reverify-controller-factory'
+import { createGetAdminAuditLogController } from '../../factories/controllers/get-admin-audit-log-controller-factory'
+import { createGetAdminDiagnosticsExportController } from '../../factories/controllers/get-admin-diagnostics-export-controller-factory'
+import { createPostAdminNetworkHealthRunController } from '../../factories/controllers/post-admin-network-health-run-controller-factory'
 import { adminAuthGateMiddleware, adminAuthMiddleware } from '../../handlers/request-handlers/admin-auth-middleware'
 import { adminEnabledMiddleware } from '../../handlers/request-handlers/admin-enabled-middleware'
 import { adminJsonBodyMiddleware } from '../../handlers/request-handlers/admin-json-body-middleware'
@@ -79,6 +82,24 @@ router.get(
   adminAuthMiddleware,
   withAdminController(createGetAdminNetworkHealthHistoryController),
 )
+router.post(
+  '/network-health/run',
+  adminRateLimitMiddleware,
+  adminAuthMiddleware,
+  withAdminController(createPostAdminNetworkHealthRunController),
+)
+router.get(
+  '/audit-log',
+  adminRateLimitMiddleware,
+  adminAuthMiddleware,
+  withAdminController(createGetAdminAuditLogController),
+)
+router.get(
+  '/diagnostics/export',
+  adminRateLimitMiddleware,
+  adminAuthMiddleware,
+  withAdminController(createGetAdminDiagnosticsExportController),
+)
 router.get(
   '/settings',
   adminRateLimitMiddleware,
diff --git a/src/schemas/admin-notifications-schema.ts b/src/schemas/admin-notifications-schema.ts
index c7d05c0ac..0cb4f5c7d 100644
--- a/src/schemas/admin-notifications-schema.ts
+++ b/src/schemas/admin-notifications-schema.ts
@@ -6,6 +6,7 @@ const targetSchema = z
     type: z.enum(['http', 'discord', 'slack', 'telegram']),
     enabled: z.boolean(),
     url: z.string().optional(),
+    secret: z.string().optional(),
     botToken: z.string().optional(),
     chatId: z.string().optional(),
   })
@@ -22,6 +23,10 @@ export const adminNotificationsPatchBodySchema = z
         'admission.invoice.failed': z.boolean().optional(),
         'settings.changed': z.boolean().optional(),
         'relay.restarted': z.boolean().optional(),
+        'probe.failed': z.boolean().optional(),
+        'probe.recovered': z.boolean().optional(),
+        'ssl.expiring': z.boolean().optional(),
+        'mirror.disconnected': z.boolean().optional(),
       })
       .strict()
       .optional(),
diff --git a/src/services/notification-channels.ts b/src/services/notification-channels.ts
index dde8cc4cc..7f49ec8f5 100644
--- a/src/services/notification-channels.ts
+++ b/src/services/notification-channels.ts
@@ -1,4 +1,5 @@
 import axios from 'axios'
+import { createHmac } from 'crypto'
 
 import {
   OperatorNotificationChannelType,
@@ -31,9 +32,16 @@ const deliverHttp = async (target: OperatorNotificationTarget, envelope: Operato
     throw new Error('HTTP notification target requires url')
   }
 
-  await axios.post(target.url, envelope, {
+  const body = JSON.stringify(envelope)
+  const headers: Record = { 'content-type': 'application/json' }
+  if (target.secret) {
+    const digest = createHmac('sha256', target.secret).update(body).digest('hex')
+    headers['x-nostream-signature'] = `sha256=${digest}`
+  }
+
+  await axios.post(target.url, body, {
     timeout: REQUEST_TIMEOUT_MS,
-    headers: { 'content-type': 'application/json' },
+    headers,
     validateStatus: (status) => status >= 200 && status < 300,
   })
 }
diff --git a/src/telemetry/metrics.ts b/src/telemetry/metrics.ts
index 75d3f143a..3b9b84205 100644
--- a/src/telemetry/metrics.ts
+++ b/src/telemetry/metrics.ts
@@ -20,7 +20,7 @@ export interface RelayMetricInstruments {
 let relayMetricInstruments: RelayMetricInstruments | undefined
 let meterProvider: MeterProvider | undefined
 
-const getWorkerAttributes = () => {
+export const getWorkerAttributes = () => {
   const workerType = process.env.WORKER_TYPE ?? (cluster.isPrimary ? 'primary' : 'worker')
   const workerIndex = process.env.WORKER_INDEX
 
diff --git a/src/telemetry/ops-metrics.ts b/src/telemetry/ops-metrics.ts
new file mode 100644
index 000000000..27f8eaafd
--- /dev/null
+++ b/src/telemetry/ops-metrics.ts
@@ -0,0 +1,56 @@
+import { metrics, type Counter, type Histogram } from '@opentelemetry/api'
+
+import { createLogger } from '../factories/logger-factory'
+import { getWorkerAttributes } from './metrics'
+
+const logger = createLogger('telemetry-ops-metrics')
+
+export interface OpsMetricInstruments {
+  probeRunsTotal: Counter
+  probeRunDurationMs: Histogram
+  mirrorReconnectsTotal: Counter
+}
+
+let opsMetricInstruments: OpsMetricInstruments | undefined
+
+const createOpsMetricInstruments = (): OpsMetricInstruments => {
+  const meter = metrics.getMeter('nostream-ops')
+
+  return {
+    probeRunsTotal: meter.createCounter('nostream.probe.runs_total', {
+      description: 'NIP-66 probe runs by outcome',
+    }),
+    probeRunDurationMs: meter.createHistogram('nostream.probe.run_duration_ms', {
+      description: 'Wall-clock duration of a probe run in milliseconds',
+      unit: 'ms',
+    }),
+    mirrorReconnectsTotal: meter.createCounter('nostream.mirror.reconnects_total', {
+      description: 'Static mirror websocket reconnect attempts',
+    }),
+  }
+}
+
+export const getOpsMetricInstruments = (): OpsMetricInstruments => {
+  if (!opsMetricInstruments) {
+    try {
+      opsMetricInstruments = createOpsMetricInstruments()
+    } catch (error) {
+      logger.warn('unable to initialize ops metrics: %o', error)
+      opsMetricInstruments = createOpsMetricInstruments()
+    }
+  }
+
+  return opsMetricInstruments
+}
+
+export const recordProbeRunMetrics = (status: string, durationMs: number): void => {
+  const instruments = getOpsMetricInstruments()
+  const attributes = { ...getWorkerAttributes(), status }
+  instruments.probeRunsTotal.add(1, attributes)
+  instruments.probeRunDurationMs.record(durationMs, attributes)
+}
+
+export const recordMirrorReconnectMetric = (mirrorIndex: string): void => {
+  const instruments = getOpsMetricInstruments()
+  instruments.mirrorReconnectsTotal.add(1, { ...getWorkerAttributes(), mirror_index: mirrorIndex })
+}
diff --git a/src/utils/admin-audit-store.ts b/src/utils/admin-audit-store.ts
new file mode 100644
index 000000000..ee5a8ef52
--- /dev/null
+++ b/src/utils/admin-audit-store.ts
@@ -0,0 +1,68 @@
+import { randomUUID } from 'crypto'
+
+import { ICacheAdapter } from '../@types/adapters'
+import {
+  ADMIN_AUDIT_LOG_KEY,
+  AdminAuditAction,
+  AdminAuditEntry,
+  MAX_ADMIN_AUDIT_API_LIMIT,
+  MAX_ADMIN_AUDIT_ENTRIES,
+} from '../@types/admin-audit'
+import { createLogger } from '../factories/logger-factory'
+
+const logger = createLogger('admin-audit-store')
+
+export class AdminAuditStore {
+  public constructor(private readonly cache: ICacheAdapter) {}
+
+  public async append(
+    action: AdminAuditAction,
+    fields: Omit = {},
+  ): Promise {
+    const entry: AdminAuditEntry = {
+      id: randomUUID(),
+      at: new Date().toISOString(),
+      action,
+      ...fields,
+    }
+
+    const score = Date.parse(entry.at)
+    if (!Number.isFinite(score)) {
+      throw new Error('Invalid audit timestamp')
+    }
+
+    await this.cache.addToSortedSet(ADMIN_AUDIT_LOG_KEY, { [JSON.stringify(entry)]: String(score) })
+    await this.trim()
+    return entry
+  }
+
+  public async list(limit: number): Promise {
+    await this.trim()
+    const clamped = Math.min(Math.max(Math.floor(limit), 1), MAX_ADMIN_AUDIT_API_LIMIT)
+    const raw = await this.cache.getRangeFromSortedSet(ADMIN_AUDIT_LOG_KEY, -clamped, -1)
+
+    return raw
+      .map((payload) => JSON.parse(payload) as AdminAuditEntry)
+      .reverse()
+  }
+
+  private async trim(): Promise {
+    await this.cache.removeRangeByRankFromSortedSet(ADMIN_AUDIT_LOG_KEY, 0, -(MAX_ADMIN_AUDIT_ENTRIES + 1))
+  }
+}
+
+export const appendAdminAuditEntrySafely = async (
+  cache: ICacheAdapter | undefined,
+  action: AdminAuditAction,
+  fields: Omit = {},
+): Promise => {
+  if (!cache) {
+    return
+  }
+
+  try {
+    await new AdminAuditStore(cache).append(action, fields)
+  } catch (error) {
+    logger.warn('failed to append admin audit entry %s: %o', action, error)
+  }
+}
diff --git a/src/utils/admin-diagnostics-export.ts b/src/utils/admin-diagnostics-export.ts
new file mode 100644
index 000000000..edf63809c
--- /dev/null
+++ b/src/utils/admin-diagnostics-export.ts
@@ -0,0 +1,64 @@
+import { readFileSync } from 'fs'
+import { join } from 'path'
+
+import { Settings } from '../@types/settings'
+import { RedisAdapter } from '../adapters/redis-adapter'
+import { getCacheClient } from '../cache/client'
+import { AdminOpsStore } from './admin-ops-store'
+import { collectAdminHealthSnapshot } from './admin-health'
+import { getRedactedAdminNotifications } from './admin-notifications-settings'
+import { RelayProbeSnapshotStore } from './relay-probe-snapshot'
+import { loadMergedSettings } from './settings-config'
+import { redactSettingsSecrets } from './settings-redaction'
+
+export interface AdminDiagnosticsExport {
+  generatedAt: string
+  version: string
+  health: Awaited>
+  settings: Settings
+  notifications: ReturnType
+  networkHealth: {
+    latest: Awaited>
+  }
+  operations: {
+    workers: Awaited>
+    mirrors: Awaited>
+  }
+}
+
+const readPackageVersion = (): string => {
+  try {
+    const raw = readFileSync(join(process.cwd(), 'package.json'), 'utf-8')
+    const parsed = JSON.parse(raw) as { version?: string }
+    return parsed.version ?? 'unknown'
+  } catch {
+    return process.env.npm_package_version ?? 'unknown'
+  }
+}
+
+export const buildAdminDiagnosticsExport = async (
+  getSettings: () => Settings = loadMergedSettings,
+): Promise => {
+  const cache = new RedisAdapter(getCacheClient())
+  const opsStore = new AdminOpsStore(cache)
+  const snapshotStore = new RelayProbeSnapshotStore(cache)
+
+  const [health, workers, mirrors, latest] = await Promise.all([
+    collectAdminHealthSnapshot(),
+    opsStore.listWorkerHeartbeats(),
+    opsStore.listMirrorHeartbeats(),
+    snapshotStore.getLatest(),
+  ])
+
+  const settings = redactSettingsSecrets(getSettings()) as Settings
+
+  return {
+    generatedAt: new Date().toISOString(),
+    version: readPackageVersion(),
+    health,
+    settings,
+    notifications: getRedactedAdminNotifications(),
+    networkHealth: { latest },
+    operations: { workers, mirrors },
+  }
+}
diff --git a/src/utils/admin-notifications-settings.ts b/src/utils/admin-notifications-settings.ts
index 1d801a14d..5b0917ae7 100644
--- a/src/utils/admin-notifications-settings.ts
+++ b/src/utils/admin-notifications-settings.ts
@@ -48,6 +48,16 @@ const mergeTargetSecrets = (
     merged.url = existing.url
   }
 
+  if (isRedactedSecret(incoming.secret)) {
+    if (existing?.secret) {
+      merged.secret = existing.secret
+    } else {
+      delete merged.secret
+    }
+  } else if (incoming.secret === undefined && existing?.secret) {
+    merged.secret = existing.secret
+  }
+
   if (isRedactedSecret(incoming.botToken)) {
     if (existing?.botToken) {
       merged.botToken = existing.botToken
diff --git a/src/utils/ops-trust-notifications.ts b/src/utils/ops-trust-notifications.ts
new file mode 100644
index 000000000..10c5b9d7e
--- /dev/null
+++ b/src/utils/ops-trust-notifications.ts
@@ -0,0 +1,120 @@
+import { ICacheAdapter } from '../@types/adapters'
+import { OperatorNotificationEventType } from '../@types/operator-notifications'
+import { RelayProbeRunSnapshot } from '../@types/relay-probe-snapshot'
+import { createLogger } from '../factories/logger-factory'
+import { enqueueOperatorNotification } from './operator-notification-enqueue'
+
+const logger = createLogger('ops-trust-notifications')
+
+const PROBE_STATUS_KEY = 'admin:trust:probe:last_status'
+const SSL_ALERT_PREFIX = 'admin:trust:ssl:alert:'
+const MIRROR_DISCONNECT_PREFIX = 'admin:trust:mirror:disconnect:'
+
+const SSL_EXPIRY_WARNING_DAYS = 14
+const DEDUPE_TTL_SECONDS = 6 * 3600
+
+const isFailedStatus = (status: RelayProbeRunSnapshot['status']): boolean =>
+  status === 'failed' || status === 'partial'
+
+export const notifyProbeRunTransition = async (
+  cache: ICacheAdapter | undefined,
+  snapshot: RelayProbeRunSnapshot,
+): Promise => {
+  if (!cache) {
+    return
+  }
+
+  const previous = (await cache.getKey(PROBE_STATUS_KEY)) || ''
+  const current = snapshot.status
+
+  try {
+    if (previous && isFailedStatus(previous as RelayProbeRunSnapshot['status']) && current === 'ok') {
+      await enqueueOperatorNotification(OperatorNotificationEventType.PROBE_RECOVERED, {
+        runAt: snapshot.runAt,
+        status: current,
+        targets: snapshot.targets,
+      })
+    } else if (current !== 'ok' && previous !== current) {
+      await enqueueOperatorNotification(OperatorNotificationEventType.PROBE_FAILED, {
+        runAt: snapshot.runAt,
+        status: current,
+        targets: snapshot.targets,
+      })
+    }
+
+    await cache.setKey(PROBE_STATUS_KEY, current, 86400 * 30)
+  } catch (error) {
+    logger.error('probe transition notification failed: %o', error)
+  }
+
+  await notifySslExpiringFromSnapshot(cache, snapshot)
+}
+
+const notifySslExpiringFromSnapshot = async (
+  cache: ICacheAdapter,
+  snapshot: RelayProbeRunSnapshot,
+): Promise => {
+  const warningThresholdMs = SSL_EXPIRY_WARNING_DAYS * 86400 * 1000
+  const now = Date.now()
+
+  for (const result of snapshot.results) {
+    const relayUrl = result.target.relayUrl
+    const expiresAt = result.tls.data?.expiresAt
+    if (!relayUrl || !expiresAt) {
+      continue
+    }
+
+    const expiryMs = new Date(expiresAt).getTime()
+    if (!Number.isFinite(expiryMs) || expiryMs <= now) {
+      continue
+    }
+
+    const remainingMs = expiryMs - now
+    if (remainingMs > warningThresholdMs) {
+      continue
+    }
+
+    const dedupeKey = `${SSL_ALERT_PREFIX}${relayUrl}`
+    const alreadySent = await cache.hasKey(dedupeKey)
+    if (alreadySent) {
+      continue
+    }
+
+    try {
+      await enqueueOperatorNotification(OperatorNotificationEventType.SSL_EXPIRING, {
+        relayUrl,
+        expiresAt,
+        daysRemaining: Math.ceil(remainingMs / 86400000),
+      })
+      await cache.setKey(dedupeKey, '1', DEDUPE_TTL_SECONDS)
+    } catch (error) {
+      logger.error('ssl expiring notification failed for %s: %o', relayUrl, error)
+    }
+  }
+}
+
+export const notifyMirrorDisconnected = async (
+  cache: ICacheAdapter | undefined,
+  fields: { index: string; targetUrl: string; code?: number; reason?: string },
+): Promise => {
+  if (!cache) {
+    return
+  }
+
+  const dedupeKey = `${MIRROR_DISCONNECT_PREFIX}${fields.index}`
+  if (await cache.hasKey(dedupeKey)) {
+    return
+  }
+
+  try {
+    await enqueueOperatorNotification(OperatorNotificationEventType.MIRROR_DISCONNECTED, {
+      index: fields.index,
+      targetUrl: fields.targetUrl,
+      code: fields.code,
+      reason: fields.reason,
+    })
+    await cache.setKey(dedupeKey, '1', DEDUPE_TTL_SECONDS)
+  } catch (error) {
+    logger.error('mirror disconnected notification failed: %o', error)
+  }
+}
diff --git a/src/utils/record-admin-audit.ts b/src/utils/record-admin-audit.ts
new file mode 100644
index 000000000..8876ab4d2
--- /dev/null
+++ b/src/utils/record-admin-audit.ts
@@ -0,0 +1,26 @@
+import { Request } from 'express'
+
+import { RedisAdapter } from '../adapters/redis-adapter'
+import { AdminAuditAction } from '../@types/admin-audit'
+import { getCacheClient } from '../cache/client'
+import { appendAdminAuditEntrySafely } from './admin-audit-store'
+
+const getAdminAuditCache = () => new RedisAdapter(getCacheClient())
+
+export const recordAdminAuditFromRequest = async (
+  action: AdminAuditAction,
+  request: Request,
+  detail?: Record,
+): Promise => {
+  await appendAdminAuditEntrySafely(getAdminAuditCache(), action, {
+    remoteAddress: request.ip,
+    ...(detail ? { detail } : {}),
+  })
+}
+
+export const recordAdminAudit = async (
+  action: AdminAuditAction,
+  fields: { remoteAddress?: string; detail?: Record } = {},
+): Promise => {
+  await appendAdminAuditEntrySafely(getAdminAuditCache(), action, fields)
+}
diff --git a/test/unit/utils/admin-audit-store.spec.ts b/test/unit/utils/admin-audit-store.spec.ts
new file mode 100644
index 000000000..4134d6708
--- /dev/null
+++ b/test/unit/utils/admin-audit-store.spec.ts
@@ -0,0 +1,50 @@
+import { expect } from 'chai'
+import sinon from 'sinon'
+
+import { AdminAuditStore } from '../../../src/utils/admin-audit-store'
+import { ICacheAdapter } from '../../../src/@types/adapters'
+
+describe('AdminAuditStore', () => {
+  let cache: sinon.SinonStubbedInstance
+
+  beforeEach(() => {
+    cache = {
+      getKey: sinon.stub(),
+      hasKey: sinon.stub(),
+      setKey: sinon.stub(),
+      setKeyIfNotExists: sinon.stub(),
+      addToSortedSet: sinon.stub().resolves(1),
+      removeRangeByScoreFromSortedSet: sinon.stub().resolves(0),
+      removeRangeByRankFromSortedSet: sinon.stub().resolves(0),
+      getRangeFromSortedSet: sinon.stub(),
+      setKeyExpiry: sinon.stub(),
+      deleteKey: sinon.stub(),
+      getHKey: sinon.stub(),
+      setHKey: sinon.stub(),
+      addToSet: sinon.stub(),
+      getSetMembers: sinon.stub(),
+      eval: sinon.stub(),
+    }
+  })
+
+  it('appends entries to the sorted set and trims', async () => {
+    const store = new AdminAuditStore(cache)
+    const entry = await store.append('admin.login', { remoteAddress: '127.0.0.1' })
+
+    expect(entry.action).to.equal('admin.login')
+    expect(entry.remoteAddress).to.equal('127.0.0.1')
+    expect(cache.addToSortedSet.calledOnce).to.equal(true)
+    expect(cache.removeRangeByRankFromSortedSet.called).to.equal(true)
+  })
+
+  it('lists entries newest-first', async () => {
+    const older = JSON.stringify({ id: '1', at: '2020-01-01T00:00:00.000Z', action: 'admin.logout' })
+    const newer = JSON.stringify({ id: '2', at: '2020-01-02T00:00:00.000Z', action: 'admin.login' })
+    cache.getRangeFromSortedSet.resolves([older, newer])
+
+    const store = new AdminAuditStore(cache)
+    const entries = await store.list(10)
+
+    expect(entries.map((entry) => entry.id)).to.deep.equal(['2', '1'])
+  })
+})

From 8f491bd1887fb74c82820e7ace266711f0d7fc54 Mon Sep 17 00:00:00 2001
From: ABHAY PANDEY 
Date: Wed, 21 Oct 2026 12:26:26 +0530
Subject: [PATCH 2/4] fix(admin): harden trust notifications and address review
 feedback

Wrap Redis dedupe in try/catch, enqueue before marking alerts sent, atomize
probe transitions, allow clearing webhook secrets, and fix Trust 401 handling.
---
 resources/admin/assets/admin-trust.js         |  26 +++--
 resources/admin/assets/dashboard.js           |   6 +
 resources/admin/assets/notifications.js       |   6 +-
 src/utils/admin-notifications-settings.ts     |   4 +-
 src/utils/operator-notification-enqueue.ts    |   4 +-
 src/utils/ops-trust-notifications.ts          | 103 +++++++++++++-----
 .../services/notification-channels.spec.ts    |  35 +++++-
 test/unit/utils/admin-audit-store.spec.ts     |  58 +++++-----
 .../admin-notifications-settings.spec.ts      |  35 ++++++
 .../utils/ops-trust-notifications.spec.ts     |  70 ++++++++++++
 10 files changed, 275 insertions(+), 72 deletions(-)
 create mode 100644 test/unit/utils/ops-trust-notifications.spec.ts

diff --git a/resources/admin/assets/admin-trust.js b/resources/admin/assets/admin-trust.js
index e436916c8..c11668fa8 100644
--- a/resources/admin/assets/admin-trust.js
+++ b/resources/admin/assets/admin-trust.js
@@ -63,6 +63,18 @@
     elements.error?.classList.add('d-none')
   }
 
+  const handleUnauthorized = () => {
+    globalThis.dispatchEvent(new CustomEvent('admin-unauthorized'))
+  }
+
+  const handleFetchError = (error, fallbackMessage) => {
+    if (error instanceof Error && error.message === 'Unauthorized') {
+      handleUnauthorized()
+      return
+    }
+    showError(fallbackMessage)
+  }
+
   const renderAuditEntries = (entries) => {
     elements.auditBody.replaceChildren()
     if (!Array.isArray(entries) || entries.length === 0) {
@@ -108,11 +120,7 @@
         elements.auditSync.textContent = `> audit: ${body.entries?.length ?? 0} entries`
       }
     } catch (error) {
-      if (error instanceof Error && error.message === 'Unauthorized') {
-        globalThis.dispatchEvent(new CustomEvent('admin-unauthorized'))
-        return
-      }
-      showError('Network error while loading audit log')
+      handleFetchError(error, 'Network error while loading audit log')
     }
   }
 
@@ -139,8 +147,8 @@
       if (elements.diagnosticsStatus) {
         elements.diagnosticsStatus.textContent = 'Download started (secrets redacted).'
       }
-    } catch {
-      showError('Network error during diagnostics export')
+    } catch (error) {
+      handleFetchError(error, 'Network error during diagnostics export')
     }
   }
 
@@ -182,8 +190,8 @@
 
       const payload = await response.json()
       showDiagnosticsPreview(payload)
-    } catch {
-      showError('Network error during diagnostics preview')
+    } catch (error) {
+      handleFetchError(error, 'Network error during diagnostics preview')
     }
   }
 
diff --git a/resources/admin/assets/dashboard.js b/resources/admin/assets/dashboard.js
index 0674e012f..4b82595b8 100644
--- a/resources/admin/assets/dashboard.js
+++ b/resources/admin/assets/dashboard.js
@@ -371,6 +371,7 @@
   }
 
   const showDashboard = () => {
+    activeDashboardView = 'metrics-view'
     loginPanel.classList.add('d-none')
     dashboardPanel.classList.remove('d-none')
     dashboardActions.classList.remove('d-none')
@@ -1869,6 +1870,11 @@
     }, 1000)
   }
 
+  document.addEventListener('admin-unauthorized', () => {
+    window.__ADMIN_TRUST__?.closeDiagnosticsPreview?.()
+    showLogin()
+  })
+
   applyTheme(getTheme())
   checkSession()
 })()
diff --git a/resources/admin/assets/notifications.js b/resources/admin/assets/notifications.js
index 0642baa4d..e76e97ac9 100644
--- a/resources/admin/assets/notifications.js
+++ b/resources/admin/assets/notifications.js
@@ -128,10 +128,10 @@
       }
 
       const secret = secretInput?.value?.trim() ?? ''
-      if (secret && secret !== REDACTED) {
-        target.secret = secret
-      } else if (secret === REDACTED) {
+      if (secret === REDACTED) {
         target.secret = REDACTED
+      } else if (secretInput) {
+        target.secret = secret
       }
 
       const botToken = botTokenInput?.value?.trim() ?? ''
diff --git a/src/utils/admin-notifications-settings.ts b/src/utils/admin-notifications-settings.ts
index 5b0917ae7..c09fff249 100644
--- a/src/utils/admin-notifications-settings.ts
+++ b/src/utils/admin-notifications-settings.ts
@@ -48,7 +48,9 @@ const mergeTargetSecrets = (
     merged.url = existing.url
   }
 
-  if (isRedactedSecret(incoming.secret)) {
+  if (incoming.secret === '') {
+    delete merged.secret
+  } else if (isRedactedSecret(incoming.secret)) {
     if (existing?.secret) {
       merged.secret = existing.secret
     } else {
diff --git a/src/utils/operator-notification-enqueue.ts b/src/utils/operator-notification-enqueue.ts
index f570153aa..bfc209266 100644
--- a/src/utils/operator-notification-enqueue.ts
+++ b/src/utils/operator-notification-enqueue.ts
@@ -22,10 +22,12 @@ export const enqueueOperatorNotification = async (
   eventType: OperatorNotificationEventType | string,
   payload: NotificationOutboxPayload,
   client?: DatabaseClient,
-): Promise => {
+): Promise => {
   try {
     await getOutboxRepository().enqueue(eventType, payload, client)
+    return true
   } catch (error) {
     logger.error('Unable to enqueue operator notification %s', eventType, error)
+    return false
   }
 }
diff --git a/src/utils/ops-trust-notifications.ts b/src/utils/ops-trust-notifications.ts
index 10c5b9d7e..889f7d41b 100644
--- a/src/utils/ops-trust-notifications.ts
+++ b/src/utils/ops-trust-notifications.ts
@@ -12,9 +12,61 @@ const MIRROR_DISCONNECT_PREFIX = 'admin:trust:mirror:disconnect:'
 
 const SSL_EXPIRY_WARNING_DAYS = 14
 const DEDUPE_TTL_SECONDS = 6 * 3600
+const PROBE_STATUS_TTL_SECONDS = 86400 * 30
+
+/** Atomically read previous probe status, persist the new status, return alert action. */
+const PROBE_TRANSITION_SCRIPT = `
+local prev = redis.call('GET', KEYS[1]) or ''
+local current = ARGV[1]
+local ttl = tonumber(ARGV[2])
+local action = ''
+if prev ~= '' and (prev == 'failed' or prev == 'partial') and current == 'ok' then
+  action = 'recovered'
+elseif current ~= 'ok' and prev ~= current then
+  action = 'failed'
+end
+redis.call('SET', KEYS[1], current, 'EX', ttl)
+return action
+`
+
+const enqueueTrustNotification = async (
+  eventType: OperatorNotificationEventType,
+  payload: Record,
+): Promise => {
+  return enqueueOperatorNotification(eventType, payload)
+}
+
+const sendDedupedTrustNotification = async (
+  cache: ICacheAdapter,
+  dedupeKey: string,
+  eventType: OperatorNotificationEventType,
+  payload: Record,
+): Promise => {
+  let claimed = false
+  try {
+    claimed = await cache.setKeyIfNotExists(dedupeKey, 'pending', DEDUPE_TTL_SECONDS)
+    if (!claimed) {
+      return
+    }
 
-const isFailedStatus = (status: RelayProbeRunSnapshot['status']): boolean =>
-  status === 'failed' || status === 'partial'
+    const enqueued = await enqueueTrustNotification(eventType, payload)
+    if (!enqueued) {
+      await cache.deleteKey(dedupeKey)
+      return
+    }
+
+    await cache.setKey(dedupeKey, '1', DEDUPE_TTL_SECONDS)
+  } catch (error) {
+    if (claimed) {
+      try {
+        await cache.deleteKey(dedupeKey)
+      } catch {
+        // best-effort release so a later attempt can retry
+      }
+    }
+    throw error
+  }
+}
 
 export const notifyProbeRunTransition = async (
   cache: ICacheAdapter | undefined,
@@ -24,30 +76,38 @@ export const notifyProbeRunTransition = async (
     return
   }
 
-  const previous = (await cache.getKey(PROBE_STATUS_KEY)) || ''
-  const current = snapshot.status
-
   try {
-    if (previous && isFailedStatus(previous as RelayProbeRunSnapshot['status']) && current === 'ok') {
-      await enqueueOperatorNotification(OperatorNotificationEventType.PROBE_RECOVERED, {
+    const action = String(
+      (await cache.eval(PROBE_TRANSITION_SCRIPT, [PROBE_STATUS_KEY], [
+        snapshot.status,
+        String(PROBE_STATUS_TTL_SECONDS),
+      ])) ?? '',
+    )
+
+    if (action === 'recovered') {
+      const enqueued = await enqueueTrustNotification(OperatorNotificationEventType.PROBE_RECOVERED, {
         runAt: snapshot.runAt,
-        status: current,
+        status: snapshot.status,
         targets: snapshot.targets,
       })
-    } else if (current !== 'ok' && previous !== current) {
-      await enqueueOperatorNotification(OperatorNotificationEventType.PROBE_FAILED, {
+      if (!enqueued) {
+        logger.error('probe recovered notification was not enqueued')
+      }
+    } else if (action === 'failed') {
+      const enqueued = await enqueueTrustNotification(OperatorNotificationEventType.PROBE_FAILED, {
         runAt: snapshot.runAt,
-        status: current,
+        status: snapshot.status,
         targets: snapshot.targets,
       })
+      if (!enqueued) {
+        logger.error('probe failed notification was not enqueued')
+      }
     }
 
-    await cache.setKey(PROBE_STATUS_KEY, current, 86400 * 30)
+    await notifySslExpiringFromSnapshot(cache, snapshot)
   } catch (error) {
     logger.error('probe transition notification failed: %o', error)
   }
-
-  await notifySslExpiringFromSnapshot(cache, snapshot)
 }
 
 const notifySslExpiringFromSnapshot = async (
@@ -75,18 +135,12 @@ const notifySslExpiringFromSnapshot = async (
     }
 
     const dedupeKey = `${SSL_ALERT_PREFIX}${relayUrl}`
-    const alreadySent = await cache.hasKey(dedupeKey)
-    if (alreadySent) {
-      continue
-    }
-
     try {
-      await enqueueOperatorNotification(OperatorNotificationEventType.SSL_EXPIRING, {
+      await sendDedupedTrustNotification(cache, dedupeKey, OperatorNotificationEventType.SSL_EXPIRING, {
         relayUrl,
         expiresAt,
         daysRemaining: Math.ceil(remainingMs / 86400000),
       })
-      await cache.setKey(dedupeKey, '1', DEDUPE_TTL_SECONDS)
     } catch (error) {
       logger.error('ssl expiring notification failed for %s: %o', relayUrl, error)
     }
@@ -102,18 +156,13 @@ export const notifyMirrorDisconnected = async (
   }
 
   const dedupeKey = `${MIRROR_DISCONNECT_PREFIX}${fields.index}`
-  if (await cache.hasKey(dedupeKey)) {
-    return
-  }
-
   try {
-    await enqueueOperatorNotification(OperatorNotificationEventType.MIRROR_DISCONNECTED, {
+    await sendDedupedTrustNotification(cache, dedupeKey, OperatorNotificationEventType.MIRROR_DISCONNECTED, {
       index: fields.index,
       targetUrl: fields.targetUrl,
       code: fields.code,
       reason: fields.reason,
     })
-    await cache.setKey(dedupeKey, '1', DEDUPE_TTL_SECONDS)
   } catch (error) {
     logger.error('mirror disconnected notification failed: %o', error)
   }
diff --git a/test/unit/services/notification-channels.spec.ts b/test/unit/services/notification-channels.spec.ts
index 3213f6296..de25af2e0 100644
--- a/test/unit/services/notification-channels.spec.ts
+++ b/test/unit/services/notification-channels.spec.ts
@@ -1,10 +1,13 @@
-import axios from 'axios'
 import chai from 'chai'
 import Sinon from 'sinon'
 import sinonChai from 'sinon-chai'
 
-import { OperatorNotificationEventType } from '../../../src/@types/operator-notifications'
-import { deliverToTarget, validateTargetConfig } from '../../../src/services/notification-channels'
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const axios = require('axios')
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { OperatorNotificationEventType } = require('../../../src/@types/operator-notifications')
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { deliverToTarget, validateTargetConfig } = require('../../../src/services/notification-channels')
 
 chai.use(sinonChai)
 
@@ -35,7 +38,31 @@ describe('notification-channels', () => {
       envelope,
     )
 
-    expect(axios.post).to.have.been.calledOnceWith('https://example.com/hook', envelope, Sinon.match.object)
+    const body = JSON.stringify(envelope)
+    expect(axios.post).to.have.been.calledOnceWith(
+      'https://example.com/hook',
+      body,
+      Sinon.match({
+        headers: { 'content-type': 'application/json' },
+      }),
+    )
+  })
+
+  it('signs HTTP webhook payloads when a secret is configured', async () => {
+    await deliverToTarget(
+      {
+        id: 'http-1',
+        type: 'http',
+        enabled: true,
+        url: 'https://example.com/hook',
+        secret: 'test-secret',
+      },
+      envelope,
+    )
+
+    expect(axios.post).to.have.been.calledOnce
+    const [, , config] = axios.post.getCall(0).args
+    expect(config.headers['x-nostream-signature']).to.match(/^sha256=[a-f0-9]{64}$/)
   })
 
   it('delivers Slack webhook payloads', async () => {
diff --git a/test/unit/utils/admin-audit-store.spec.ts b/test/unit/utils/admin-audit-store.spec.ts
index 4134d6708..45710c194 100644
--- a/test/unit/utils/admin-audit-store.spec.ts
+++ b/test/unit/utils/admin-audit-store.spec.ts
@@ -1,46 +1,50 @@
-import { expect } from 'chai'
-import sinon from 'sinon'
+import chai from 'chai'
 
-import { AdminAuditStore } from '../../../src/utils/admin-audit-store'
 import { ICacheAdapter } from '../../../src/@types/adapters'
+import { AdminAuditStore } from '../../../src/utils/admin-audit-store'
+
+const { expect } = chai
+
+const createCacheStub = (): ICacheAdapter => ({
+  getKey: async () => '',
+  hasKey: async () => false,
+  setKey: async () => true,
+  setKeyIfNotExists: async () => true,
+  addToSortedSet: async () => 1,
+  removeRangeByScoreFromSortedSet: async () => 0,
+  removeRangeByRankFromSortedSet: async () => 0,
+  getRangeFromSortedSet: async () => [],
+  setKeyExpiry: async () => {},
+  deleteKey: async () => 1,
+  getHKey: async () => '',
+  setHKey: async () => true,
+  addToSet: async () => 1,
+  getSetMembers: async () => [],
+  eval: async () => '',
+})
 
 describe('AdminAuditStore', () => {
-  let cache: sinon.SinonStubbedInstance
-
-  beforeEach(() => {
-    cache = {
-      getKey: sinon.stub(),
-      hasKey: sinon.stub(),
-      setKey: sinon.stub(),
-      setKeyIfNotExists: sinon.stub(),
-      addToSortedSet: sinon.stub().resolves(1),
-      removeRangeByScoreFromSortedSet: sinon.stub().resolves(0),
-      removeRangeByRankFromSortedSet: sinon.stub().resolves(0),
-      getRangeFromSortedSet: sinon.stub(),
-      setKeyExpiry: sinon.stub(),
-      deleteKey: sinon.stub(),
-      getHKey: sinon.stub(),
-      setHKey: sinon.stub(),
-      addToSet: sinon.stub(),
-      getSetMembers: sinon.stub(),
-      eval: sinon.stub(),
+  it('appends entries to the sorted set and trims', async () => {
+    const cache = createCacheStub()
+    let addCalls = 0
+    cache.addToSortedSet = async () => {
+      addCalls += 1
+      return 1
     }
-  })
 
-  it('appends entries to the sorted set and trims', async () => {
     const store = new AdminAuditStore(cache)
     const entry = await store.append('admin.login', { remoteAddress: '127.0.0.1' })
 
     expect(entry.action).to.equal('admin.login')
     expect(entry.remoteAddress).to.equal('127.0.0.1')
-    expect(cache.addToSortedSet.calledOnce).to.equal(true)
-    expect(cache.removeRangeByRankFromSortedSet.called).to.equal(true)
+    expect(addCalls).to.equal(1)
   })
 
   it('lists entries newest-first', async () => {
     const older = JSON.stringify({ id: '1', at: '2020-01-01T00:00:00.000Z', action: 'admin.logout' })
     const newer = JSON.stringify({ id: '2', at: '2020-01-02T00:00:00.000Z', action: 'admin.login' })
-    cache.getRangeFromSortedSet.resolves([older, newer])
+    const cache = createCacheStub()
+    cache.getRangeFromSortedSet = async () => [older, newer]
 
     const store = new AdminAuditStore(cache)
     const entries = await store.list(10)
diff --git a/test/unit/utils/admin-notifications-settings.spec.ts b/test/unit/utils/admin-notifications-settings.spec.ts
index 27997a927..df7ec084d 100644
--- a/test/unit/utils/admin-notifications-settings.spec.ts
+++ b/test/unit/utils/admin-notifications-settings.spec.ts
@@ -69,6 +69,41 @@ describe('admin-notifications-settings', () => {
     expect(next.targets[0].botToken).to.equal(undefined)
   })
 
+  it('clears signing secret when patch sends an empty string', () => {
+    const current = getMergedAdminNotifications({
+      admin: {
+        notifications: {
+          enabled: true,
+          targets: [
+            {
+              id: 'http-1',
+              type: 'http',
+              enabled: true,
+              url: 'https://example.com/hook',
+              secret: 'old-secret',
+            },
+          ],
+          events: {},
+          retry: { maxAttempts: 5, baseDelayMs: 1000 },
+        },
+      },
+    } as any)
+
+    const next = mergeAdminNotificationsPatch(current, {
+      targets: [
+        {
+          id: 'http-1',
+          type: 'http',
+          enabled: true,
+          url: 'https://example.com/hook',
+          secret: '',
+        },
+      ],
+    })
+
+    expect(next.targets[0].secret).to.equal(undefined)
+  })
+
   it('does not persist redacted webhook url for new http targets', () => {
     const current = getMergedAdminNotifications({
       admin: {
diff --git a/test/unit/utils/ops-trust-notifications.spec.ts b/test/unit/utils/ops-trust-notifications.spec.ts
new file mode 100644
index 000000000..22e56a762
--- /dev/null
+++ b/test/unit/utils/ops-trust-notifications.spec.ts
@@ -0,0 +1,70 @@
+import chai from 'chai'
+import Sinon from 'sinon'
+
+import { ICacheAdapter } from '../../../src/@types/adapters'
+import { OperatorNotificationEventType } from '../../../src/@types/operator-notifications'
+import { notifyMirrorDisconnected, notifyProbeRunTransition } from '../../../src/utils/ops-trust-notifications'
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const enqueueModule = require('../../../src/utils/operator-notification-enqueue')
+
+const { expect } = chai
+
+const createCacheStub = (overrides: Partial = {}): ICacheAdapter => ({
+  getKey: async () => '',
+  hasKey: async () => false,
+  setKey: async () => true,
+  setKeyIfNotExists: async () => true,
+  addToSortedSet: async () => 1,
+  removeRangeByScoreFromSortedSet: async () => 0,
+  removeRangeByRankFromSortedSet: async () => 0,
+  getRangeFromSortedSet: async () => [],
+  setKeyExpiry: async () => {},
+  deleteKey: async () => 1,
+  getHKey: async () => '',
+  setHKey: async () => true,
+  addToSet: async () => 1,
+  getSetMembers: async () => [],
+  eval: async () => '',
+  ...overrides,
+})
+
+describe('ops-trust-notifications', () => {
+  afterEach(() => {
+    Sinon.restore()
+  })
+
+  it('does not mark mirror dedupe key when enqueue fails', async () => {
+    const deleteKey = Sinon.stub().resolves(1)
+    const setKey = Sinon.stub().resolves(true)
+    const cache = createCacheStub({
+      setKeyIfNotExists: async () => true,
+      deleteKey,
+      setKey,
+    })
+
+    Sinon.stub(enqueueModule, 'enqueueOperatorNotification').resolves(false)
+
+    await notifyMirrorDisconnected(cache, { index: '0', targetUrl: 'wss://mirror.example' })
+
+    expect(deleteKey.called).to.equal(true)
+    expect(setKey.called).to.equal(false)
+  })
+
+  it('enqueues probe failed alerts from atomic transition action', async () => {
+    const enqueue = Sinon.stub(enqueueModule, 'enqueueOperatorNotification').resolves(true)
+    const cache = createCacheStub({
+      eval: async () => 'failed',
+    })
+
+    await notifyProbeRunTransition(cache, {
+      runAt: new Date().toISOString(),
+      targets: ['wss://relay.example'],
+      results: [],
+      status: 'failed',
+    })
+
+    expect(enqueue.calledOnce).to.equal(true)
+    expect(enqueue.firstCall.args[0]).to.equal(OperatorNotificationEventType.PROBE_FAILED)
+  })
+})

From bee82462166aa27b86ffadb77c96b15dfd2f96b7 Mon Sep 17 00:00:00 2001
From: ABHAY PANDEY 
Date: Wed, 21 Oct 2026 12:40:19 +0530
Subject: [PATCH 3/4] fix(redis): retry eval on NOSCRIPT for trust probe
 transitions

Reload cached script SHAs after Redis flushes scripts and isolate SSL
expiry notifications from probe eval failures.
---
 src/adapters/redis-adapter.ts            | 32 +++++++++++++++++++++---
 src/utils/ops-trust-notifications.ts     |  6 ++++-
 test/unit/adapters/redis-adapter.spec.ts | 13 ++++++++++
 3 files changed, 46 insertions(+), 5 deletions(-)

diff --git a/src/adapters/redis-adapter.ts b/src/adapters/redis-adapter.ts
index 4ec094f45..d997a9101 100644
--- a/src/adapters/redis-adapter.ts
+++ b/src/adapters/redis-adapter.ts
@@ -140,12 +140,36 @@ export class RedisAdapter implements ICacheAdapter {
     return this.client.sMembers(key)
   }
 
+  private isNoScriptError(error: unknown): boolean {
+    if (!error || typeof error !== 'object') {
+      return false
+    }
+
+    const candidate = error as { code?: string; message?: string }
+    return candidate.code === 'NOSCRIPT' || (candidate.message?.includes('NOSCRIPT') ?? false)
+  }
+
   public async eval(script: string, keys: string[], args: string[]): Promise {
     await this.connection
-    if (!this.scriptShas.has(script)) {
-      const sha = await this.client.scriptLoad(script)
-      this.scriptShas.set(script, sha)
+
+    const runEvalSha = async (): Promise => {
+      if (!this.scriptShas.has(script)) {
+        const sha = await this.client.scriptLoad(script)
+        this.scriptShas.set(script, sha)
+      }
+
+      return this.client.evalSha(this.scriptShas.get(script)!, { keys, arguments: args })
+    }
+
+    try {
+      return await runEvalSha()
+    } catch (error) {
+      if (!this.isNoScriptError(error)) {
+        throw error
+      }
+
+      this.scriptShas.delete(script)
+      return runEvalSha()
     }
-    return await this.client.evalSha(this.scriptShas.get(script)!, { keys, arguments: args })
   }
 }
diff --git a/src/utils/ops-trust-notifications.ts b/src/utils/ops-trust-notifications.ts
index 889f7d41b..633b587c4 100644
--- a/src/utils/ops-trust-notifications.ts
+++ b/src/utils/ops-trust-notifications.ts
@@ -103,10 +103,14 @@ export const notifyProbeRunTransition = async (
         logger.error('probe failed notification was not enqueued')
       }
     }
+  } catch (error) {
+    logger.error('probe transition notification failed: %o', error)
+  }
 
+  try {
     await notifySslExpiringFromSnapshot(cache, snapshot)
   } catch (error) {
-    logger.error('probe transition notification failed: %o', error)
+    logger.error('ssl expiring notifications failed: %o', error)
   }
 }
 
diff --git a/test/unit/adapters/redis-adapter.spec.ts b/test/unit/adapters/redis-adapter.spec.ts
index ca5e9972b..8054c7cef 100644
--- a/test/unit/adapters/redis-adapter.spec.ts
+++ b/test/unit/adapters/redis-adapter.spec.ts
@@ -319,5 +319,18 @@ describe('RedisAdapter', () => {
       expect(client.scriptLoad).to.have.been.calledOnce
       expect(client.evalSha).to.have.been.calledTwice
     })
+
+    it('reloads script and retries after NOSCRIPT', async () => {
+      client.scriptLoad.onFirstCall().resolves('stale-sha')
+      client.scriptLoad.onSecondCall().resolves('fresh-sha')
+      client.evalSha.onFirstCall().rejects(Object.assign(new Error('NOSCRIPT No matching script'), { code: 'NOSCRIPT' }))
+      client.evalSha.onSecondCall().resolves('recovered')
+
+      const result = await adapter.eval('local x = 1', ['key1'], ['arg1'])
+
+      expect(client.scriptLoad).to.have.been.calledTwice
+      expect(client.evalSha).to.have.been.calledTwice
+      expect(result).to.equal('recovered')
+    })
   })
 })

From e5720937b1df95752b694296be617a4fa832e909 Mon Sep 17 00:00:00 2001
From: ABHAY PANDEY 
Date: Wed, 21 Oct 2026 16:47:47 +0530
Subject: [PATCH 4/4] fix(admin): skip audit Redis when cache is not configured

Avoid constructing an invalid Redis URL during admin login and settings
updates when REDIS_* env vars are unset (e.g. unit test CI).
---
 src/cache/client.ts             |  8 ++++++++
 src/utils/record-admin-audit.ts | 10 ++++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/cache/client.ts b/src/cache/client.ts
index c00a4b4b7..2e4866f75 100644
--- a/src/cache/client.ts
+++ b/src/cache/client.ts
@@ -21,6 +21,14 @@ const redactRedisUrlCredentials = (url: string): string => {
   }
 }
 
+export const isCacheConfigured = (): boolean => {
+  if (process.env.REDIS_URI?.trim()) {
+    return true
+  }
+
+  return Boolean(process.env.REDIS_HOST?.trim() && process.env.REDIS_PORT?.trim())
+}
+
 export const getCacheConfig = (): RedisClientOptions => {
   const password = process.env.REDIS_PASSWORD
 
diff --git a/src/utils/record-admin-audit.ts b/src/utils/record-admin-audit.ts
index 8876ab4d2..8c6cefafc 100644
--- a/src/utils/record-admin-audit.ts
+++ b/src/utils/record-admin-audit.ts
@@ -2,10 +2,16 @@ import { Request } from 'express'
 
 import { RedisAdapter } from '../adapters/redis-adapter'
 import { AdminAuditAction } from '../@types/admin-audit'
-import { getCacheClient } from '../cache/client'
+import { getCacheClient, isCacheConfigured } from '../cache/client'
 import { appendAdminAuditEntrySafely } from './admin-audit-store'
 
-const getAdminAuditCache = () => new RedisAdapter(getCacheClient())
+const getAdminAuditCache = (): RedisAdapter | undefined => {
+  if (!isCacheConfigured()) {
+    return undefined
+  }
+
+  return new RedisAdapter(getCacheClient())
+}
 
 export const recordAdminAuditFromRequest = async (
   action: AdminAuditAction,