From a724676c187534e03645261ad968c41c6d2e6704 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 2 Oct 2026 20:44:37 +0000 Subject: [PATCH 01/40] fix(otel): export fallback root before invocation return --- otel-plugin/README.md | 13 ++ .../durable/otel/ExecutionOtelPlugin.java | 2 + .../durable/otel/ExecutionTraceContext.java | 5 +- .../durable/otel/InvocationOtelPlugin.java | 2 + .../durable/otel/OtelPluginSupport.java | 32 ++++ .../lambda/durable/otel/SpanAttributes.java | 2 + .../durable/otel/ExecutionOtelPluginTest.java | 19 +- .../durable/otel/ExecutionRootTest.java | 174 ++++++++++++++++++ .../otel/InvocationOtelPluginTest.java | 44 ++--- 9 files changed, 260 insertions(+), 33 deletions(-) create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java diff --git a/otel-plugin/README.md b/otel-plugin/README.md index c51032aaa..c680a4ca7 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -40,6 +40,19 @@ If you configure your own `SdkTracerProviderBuilder`, add the OpenTelemetry SDK ``` +## Fallback execution roots + +When the backend supplies no complete remote parent, both views export an `DurableExecutionRoot` anchor before the first +invocation returns, including when it suspends with `PENDING` or fails with `RETRYING`. +The anchor is marked `durable.execution.synthetic_root=true`. Its trace and span IDs are deterministic and its start +and end timestamps are the checkpointed execution start. It does not report execution status or duration; `Workflow` +continues to report those at terminal completion. Complete remote parents remain externally owned and are never exported. + +Every invocation may re-export the anchor to recover from an earlier interrupted or lost export. Re-exports retain the +same identity, timestamps, and attributes. The existing provider resource still applies. Each execution ARN owns its +own anchor even when multiple executions share a propagated trace ID without a parent; they are not collapsed into one +execution. Upstream sampling and configured fallback sampling apply to anchors and their descendants together. + ## Quick Start using X-Ray/CloudWatch Tracing (ADOT Java Agent) 1. Add the ADOT Lambda Layer to your function diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index 09214055c..fa9016557 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -332,6 +332,8 @@ public void onInvocationEnd(InvocationEndInfo info) { } workflowSpan.end(); } + OtelPluginSupport.exportExecutionRoot( + tracer, idGenerator, executionAncestor, durableExecutionArn, executionStartTime, samplingIntent); workflowSpanContext = null; executionAncestor = null; executionStartTime = null; diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionTraceContext.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionTraceContext.java index cf8a3dc4b..b3e7eaea5 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionTraceContext.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionTraceContext.java @@ -39,8 +39,9 @@ * so a remote parent left unsampled would make a parent-based sampler drop every child span; deferring to the sampler * avoids that. * - *

The ancestor is a non-recording context: it is either the external backend server span or a synthetic root the SDK - * does not export. + *

The ancestor is a non-recording context: it is either the external backend server span or an SDK-owned synthetic + * root. Both plugins export the synthetic root before each invocation returns, including suspension, with its stable + * execution-start timestamp and zero duration. They never export a remote parent. * * @param executionAncestor the common parent context for the Workflow and Invocation spans */ diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index e89804bdc..16b5e74f3 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -338,6 +338,8 @@ public void onInvocationEnd(InvocationEndInfo info) { } workflowSpan.end(); } + OtelPluginSupport.exportExecutionRoot( + tracer, idGenerator, executionAncestor, durableExecutionArn, executionStartTime, samplingIntent); workflowSpanContext = null; executionAncestor = null; executionStartTime = null; diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 63fbec908..8b9aaa5bc 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -2,9 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.otel; +import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_ARN; +import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_SYNTHETIC_ROOT; + import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.api.trace.TracerProvider; @@ -13,6 +17,7 @@ import io.opentelemetry.sdk.trace.samplers.SamplingResult; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; import java.util.Collections; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,6 +25,33 @@ /** Shared utilities for OTel plugin default constructor support (ADOT Java agent SPI path). */ final class OtelPluginSupport { + /** + * Exports the SDK-owned fallback ancestor before each invocation returns, including PENDING. It is a zero-duration + * identity anchor at the checkpointed execution start, not a completion summary. Recovery re-exports use identical + * identity and timestamps; Workflow alone carries execution duration and outcome. Remote parents are never owned. + */ + static void exportExecutionRoot( + Tracer tracer, + DeterministicIdGenerator idGenerator, + SpanContext ancestor, + String arn, + Instant start, + DurableSamplingDecision.Intent intent) { + if (ancestor == null || ancestor.isRemote()) { + return; + } + var builder = tracer.spanBuilder("DurableExecutionRoot") + .setSpanKind(SpanKind.INTERNAL) + .setParent(DurableSamplingDecision.store(Context.root(), intent)) + .setAttribute(DURABLE_EXECUTION_ARN, arn) + .setAttribute(DURABLE_EXECUTION_SYNTHETIC_ROOT, true) + .setStartTimestamp(start); + try (var ignored = DurableSamplingDecision.openScope(intent)) { + var root = idGenerator.startSpan(builder, ancestor.getTraceId(), ancestor.getSpanId()); + root.end(start); + } + } + private static final Logger logger = LoggerFactory.getLogger(OtelPluginSupport.class); private OtelPluginSupport() {} diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java index 62510c547..c669c86b9 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java @@ -10,6 +10,8 @@ final class SpanAttributes { private SpanAttributes() {} static final AttributeKey DURABLE_EXECUTION_ARN = AttributeKey.stringKey("durable.execution.arn"); + static final AttributeKey DURABLE_EXECUTION_SYNTHETIC_ROOT = + AttributeKey.booleanKey("durable.execution.synthetic_root"); static final AttributeKey DURABLE_EXECUTION_STATUS = AttributeKey.stringKey("durable.execution.status"); static final AttributeKey DURABLE_OPERATION_ID = AttributeKey.stringKey("durable.operation.id"); static final AttributeKey DURABLE_OPERATION_TYPE = AttributeKey.stringKey("durable.operation.type"); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java index 8b84efc10..9ce54d3ef 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java @@ -141,7 +141,7 @@ void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() { new InvocationEndInfo("req-enabled", "arn:enabled", true, InvocationStatus.SUCCEEDED, null)); var spans = globalExporter.getFinishedSpanItems(); - assertEquals(3, spans.size()); + assertEquals(4, spans.size()); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("enabled-step"))); assertFalse(spans.stream().anyMatch(span -> span.getName().equals("disabled-step"))); } @@ -178,8 +178,8 @@ void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var spans = globalExporter.getFinishedSpanItems(); - // Workflow + Invocation + operation = 3 - assertEquals(3, spans.size()); + // Workflow + Invocation + operation + DurableExecutionRoot = 4 + assertEquals(4, spans.size()); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("Workflow"))); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("Invocation"))); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step"))); @@ -206,7 +206,7 @@ void terminalInvocation_exportsWorkflowAndInvocationSpans() { plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(2, spans.size(), "Terminal invocation should export the Workflow span and the invocation span"); + assertEquals(3, spans.size(), "Terminal invocation exports Workflow, Invocation, and DurableExecutionRoot"); var workflowSpan = spanByName(spans, "Workflow"); var invocationSpan = spanByName(spans, "Invocation"); @@ -402,8 +402,8 @@ void nonTerminalInvocation_doesNotExportWorkflowSpan() { plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null)); var spans = spanExporter.getFinishedSpanItems(); - // Only the invocation span is exported; the Workflow span is not ended on non-terminal status. - assertEquals(1, spans.size()); + // Invocation and DurableExecutionRoot are exported; Workflow remains deferred until terminal status. + assertEquals(2, spans.size()); assertEquals("Invocation", spans.get(0).getName()); assertEquals(StatusCode.OK, spans.get(0).getStatus().getStatusCode(), "PENDING invocation span maps to OK"); } @@ -449,7 +449,7 @@ void retryingInvocation_invocationSpanUnset_workflowNotExported() { "req-1", ARN, true, InvocationStatus.RETRYING, new RuntimeException("transient"))); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(1, spans.size(), "RETRYING is non-terminal — Workflow span not exported"); + assertEquals(2, spans.size(), "RETRYING is non-terminal — Workflow span not exported"); var invocationSpan = spans.get(0); assertEquals("Invocation", invocationSpan.getName()); assertEquals( @@ -941,10 +941,11 @@ void operationNotCompleted_notEndedAtInvocationEnd() { plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null)); var spans = spanExporter.getFinishedSpanItems(); - // Only the invocation span is exported. A still-open operation has no recording span (creation is deferred to + // Invocation and DurableExecutionRoot are exported. A still-open operation has no recording span (creation is + // deferred to // onOperationEnd), so there is nothing to abandon, and the Workflow span is not exported on a non-terminal // invocation. - assertEquals(1, spans.size()); + assertEquals(2, spans.size()); assertEquals("Invocation", spans.get(0).getName()); assertTrue( spans.stream().noneMatch(s -> s.getName().equals("my-wait")), diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java new file mode 100644 index 000000000..acf3ca0ea --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java @@ -0,0 +1,174 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; +import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_ARN; +import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_STATUS; +import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_SYNTHETIC_ROOT; + +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class ExecutionRootTest { + private static final Instant START = Instant.parse("2026-10-02T00:00:00Z"); + private static final String TRACE = "6955b900123456789012345678901234"; + private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id"; + + @ParameterizedTest + @CsvSource({"true,true", "true,false", "false,true", "false,false"}) + void anchorIsExportedOnFirstPendingInvocationThenConnectsTerminalWorkflow(boolean executionView, boolean success) { + var exporter = InMemorySpanExporter.create(); + var plugin = plugin(executionView, exporter, null, Sampler.alwaysOn()); + var effects = new AtomicInteger(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> { + ctx.step("once", Integer.class, stepCtx -> effects.incrementAndGet()); + ctx.wait("pause", Duration.ofSeconds(10)); + if (!success) throw new IllegalArgumentException("expected"); + return input; + }, + DurableConfig.builder().withPlugins(plugin).build()); + assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); + var first = exporter.getFinishedSpanItems(); + var anchor = named(first, "DurableExecutionRoot"); + assertEquals(true, anchor.getAttributes().get(DURABLE_EXECUTION_SYNTHETIC_ROOT)); + assertFalse(anchor.getParentSpanContext().isValid()); + assertEquals(anchor.getStartEpochNanos(), anchor.getEndEpochNanos()); + assertEquals(StatusCode.UNSET, anchor.getStatus().getStatusCode()); + assertNull(anchor.getAttributes().get(DURABLE_EXECUTION_STATUS)); + assertTrue(first.stream().noneMatch(span -> span.getName().equals("Workflow"))); + assertEquals(anchor.getSpanId(), named(first, "Invocation").getParentSpanId()); + runner.advanceTime(); + assertEquals( + success ? ExecutionStatus.SUCCEEDED : ExecutionStatus.FAILED, + runner.runUntilComplete("input").getStatus()); + assertEquals(1, effects.get()); + var all = exporter.getFinishedSpanItems(); + assertEquals(anchor.getSpanId(), named(all, "Workflow").getParentSpanId()); + var anchors = all.stream() + .filter(span -> span.getName().equals("DurableExecutionRoot")) + .toList(); + assertEquals(2, anchors.size()); + assertStable(anchor, anchors.get(1)); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void fallbackRecoveryKeepsIdentityAndRemoteParentsRemainExternal(boolean executionView) { + var cases = new ExtractedContext[] { + null, + new ExtractedContext("invalid", "bad", ExtractedContext.Sampling.UNDECIDED), + new ExtractedContext(TRACE, null, ExtractedContext.Sampling.SAMPLED), + new ExtractedContext(TRACE, "0000000000000000", ExtractedContext.Sampling.SAMPLED) + }; + for (var extracted : cases) { + var exporter = InMemorySpanExporter.create(); + var plugin = plugin(executionView, exporter, extracted, Sampler.alwaysOn()); + invoke(plugin, ARN, true, InvocationStatus.RETRYING); + var root = named(exporter.getFinishedSpanItems(), "DurableExecutionRoot"); + invoke(plugin, ARN, true, InvocationStatus.PENDING); // redelivery after a failed invocation + invoke(plugin, ARN, false, InvocationStatus.SUCCEEDED); + var roots = exporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals("DurableExecutionRoot")) + .toList(); + assertEquals(3, roots.size()); + roots.forEach(span -> assertStable(root, span)); + assertEquals(new DeterministicIdGenerator().generateExecutionRootSpanId(ARN), root.getSpanId()); + assertEquals(START.getEpochSecond() * 1_000_000_000L, root.getStartEpochNanos()); + if (extracted != null && extracted.hasValidTraceId()) assertEquals(TRACE, root.getTraceId()); + } + var exporter = InMemorySpanExporter.create(); + var plugin = plugin( + executionView, + exporter, + new ExtractedContext(TRACE, "1234567890123456", ExtractedContext.Sampling.SAMPLED), + Sampler.alwaysOn()); + invoke(plugin, ARN, true, InvocationStatus.SUCCEEDED); + assertEquals(2, exporter.getFinishedSpanItems().size()); + assertTrue(exporter.getFinishedSpanItems().stream() + .allMatch(span -> span.getParentSpanId().equals("1234567890123456"))); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void anchorsHonorSamplingAndAreOwnedByExecutionOnSharedTrace(boolean executionView) { + for (var extracted : + new ExtractedContext[] {null, new ExtractedContext(TRACE, null, ExtractedContext.Sampling.NOT_SAMPLED) + }) { + var exporter = InMemorySpanExporter.create(); + var plugin = plugin( + executionView, exporter, extracted, extracted == null ? Sampler.alwaysOff() : Sampler.alwaysOn()); + invoke(plugin, ARN, true, InvocationStatus.PENDING); + invoke(plugin, ARN, false, InvocationStatus.SUCCEEDED); + assertTrue(exporter.getFinishedSpanItems().isEmpty()); + } + var exporter = InMemorySpanExporter.create(); + var plugin = plugin( + executionView, + exporter, + new ExtractedContext(TRACE, null, ExtractedContext.Sampling.SAMPLED), + Sampler.alwaysOff()); + invoke(plugin, ARN, true, InvocationStatus.SUCCEEDED); + invoke(plugin, ARN + "-other", true, InvocationStatus.SUCCEEDED); + var roots = exporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals("DurableExecutionRoot")) + .toList(); + assertEquals(2, roots.size()); + assertTrue(roots.stream().allMatch(span -> span.getTraceId().equals(TRACE))); + assertNotEquals(roots.get(0).getSpanId(), roots.get(1).getSpanId()); + assertNotEquals( + roots.get(0).getAttributes().get(DURABLE_EXECUTION_ARN), + roots.get(1).getAttributes().get(DURABLE_EXECUTION_ARN)); + } + + private static void assertStable(SpanData first, SpanData replay) { + assertEquals(first.getSpanContext(), replay.getSpanContext()); + assertEquals(first.getStartEpochNanos(), replay.getStartEpochNanos()); + assertEquals(first.getEndEpochNanos(), replay.getEndEpochNanos()); + assertEquals(first.getAttributes(), replay.getAttributes()); + assertEquals(first.getResource(), replay.getResource()); + } + + private static SpanData named(List spans, String name) { + return spans.stream() + .filter(span -> span.getName().equals(name)) + .findFirst() + .orElseThrow(); + } + + private static void invoke(DurableExecutionPlugin plugin, String arn, boolean first, InvocationStatus status) { + plugin.onInvocationStart(new InvocationInfo("request", arn, first, START)); + plugin.onInvocationEnd(new InvocationEndInfo("request", arn, first, status, null)); + } + + private static DurableExecutionPlugin plugin( + boolean executionView, InMemorySpanExporter exporter, ExtractedContext extracted, Sampler sampler) { + var builder = + SdkTracerProvider.builder().setSampler(sampler).addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var config = OtelPluginConfig.builder() + .contextExtractor(() -> extracted) + .enableMdc(false) + .build(); + return executionView ? new ExecutionOtelPlugin(builder, config) : new InvocationOtelPlugin(builder, config); + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java index d8d7e7f43..314363eb1 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java @@ -121,7 +121,7 @@ void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() { new InvocationEndInfo("req-enabled", "arn:enabled", true, InvocationStatus.SUCCEEDED, null)); var spans = globalExporter.getFinishedSpanItems(); - assertEquals(3, spans.size()); + assertEquals(4, spans.size()); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("enabled-step"))); assertFalse(spans.stream().anyMatch(span -> span.getName().equals("disabled-step"))); } @@ -159,7 +159,7 @@ void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { var spans = globalExporter.getFinishedSpanItems(); // Plugin creates Workflow + Invocation + operation spans - assertEquals(3, spans.size()); + assertEquals(4, spans.size()); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step"))); } @@ -208,7 +208,7 @@ public ContextPropagators getPropagators() { var spans = globalExporter.getFinishedSpanItems(); // Plugin creates Workflow + Invocation + operation spans - assertEquals(3, spans.size()); + assertEquals(4, spans.size()); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step"))); var expectedIds = new DeterministicIdGenerator(); expectedIds.setDurableExecutionArn("arn:exec1"); @@ -330,7 +330,7 @@ void invocationStart_and_end_createsSpan() { null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(2, spans.size()); // invocation + Workflow + assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot var span = spans.get(0); assertEquals("Invocation", span.getName()); @@ -582,7 +582,7 @@ void invocationEnd_withFailure_setsErrorStatus() { "req-123", "arn:exec1", true, InvocationStatus.FAILED, new RuntimeException("boom"))); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(2, spans.size()); // invocation + Workflow + assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot assertEquals(StatusCode.ERROR, spans.get(0).getStatus().getStatusCode()); } @@ -593,7 +593,7 @@ void invocationEnd_withRetrying_leavesStatusUnset() { "req-123", "arn:exec1", true, InvocationStatus.RETRYING, new RuntimeException("transient"))); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(1, spans.size()); + assertEquals(2, spans.size()); assertEquals( StatusCode.UNSET, spans.get(0).getStatus().getStatusCode(), @@ -618,7 +618,7 @@ void operationStart_createsSpan_operationEnd_endsIt() { plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(3, spans.size()); // operation + invocation + Workflow + assertEquals(4, spans.size()); // operation + invocation + Workflow + DurableExecutionRoot var operationSpan = spans.stream() .filter(s -> s.getName().contains("step")) @@ -650,7 +650,7 @@ void userFunctionStart_and_end_createsAttemptSpan() { plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(3, spans.size()); // attempt + invocation + Workflow + assertEquals(4, spans.size()); // attempt + invocation + Workflow + DurableExecutionRoot var attemptSpan = spans.stream() .filter(s -> s.getName().contains("attempt")) @@ -941,7 +941,7 @@ void fullLifecycle_producesCorrectSpanHierarchy() { var spans = spanExporter.getFinishedSpanItems(); // 2 attempt spans + 2 operation spans + 1 invocation span + 1 Workflow span = 6 - assertEquals(6, spans.size()); + assertEquals(7, spans.size()); var workflowTraceId = spanByName("Workflow").getTraceId(); var invocationTraceId = spanByName("Invocation").getTraceId(); @@ -988,7 +988,7 @@ void operationNotCompleted_spanEndedAtInvocationEnd() { var spans = spanExporter.getFinishedSpanItems(); // Should have: operation span (ended at invocation end) + invocation span - assertEquals(2, spans.size()); + assertEquals(3, spans.size()); var operationSpan = spans.stream() .filter(s -> s.getName().contains("wait")) @@ -1110,7 +1110,7 @@ void xrayExtraction_withoutParentDoesNotForceTraceId() { xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(2, spans.size()); // invocation + Workflow + assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot var invocationSpan = spanByName("Invocation"); var workflowSpan = spanByName("Workflow"); // Remote trace but no parent → a synthetic execution root on the remote trace ID anchors the execution. Both @@ -1230,7 +1230,7 @@ void xrayExtraction_withoutParentSpanId_invocationSpanParentsOntoSyntheticRoot() xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(2, spans.size()); // invocation + Workflow + assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot // Remote trace, no parent → a synthetic execution root on the remote trace ID anchors the execution. The // Invocation span has a valid parent (that synthetic root) and joins the remote trace. @@ -1318,7 +1318,7 @@ void xrayExtraction_nullExtractor_sharesArnDerivedExecutionTrace() { noXrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", arn, true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(2, spans.size()); // invocation + Workflow + assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot var invocationTraceId = spanByName("Invocation").getTraceId(); var workflowTraceId = spanByName("Workflow").getTraceId(); @@ -1385,7 +1385,7 @@ void operationEnd_withoutMatchingStart_createsContinuationSpanWithLink() { plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(3, spans.size()); // continuation + invocation + Workflow + assertEquals(4, spans.size()); // continuation + invocation + Workflow + DurableExecutionRoot var continuationSpan = spans.stream() .filter(s -> s.getName().contains("wait")) @@ -1622,9 +1622,9 @@ void multiInvocation_stepWaitStep_producesCorrectSpans() { new OperationInfo("op-2", "pause", "WAIT", "Wait", null, Instant.now(), null, null, false)); plugin.onInvocationEnd(new InvocationEndInfo("req-1", arn, true, InvocationStatus.PENDING, null)); - // Invocation 1 should have: step op + step attempt + wait (PENDING) + invocation = 4 + // Step op + step attempt + wait (PENDING) + Invocation + DurableExecutionRoot = 5 var inv1Spans = spanExporter.getFinishedSpanItems(); - assertEquals(4, inv1Spans.size()); + assertEquals(5, inv1Spans.size()); var inv1TraceId = inv1Spans.stream() .filter(span -> span.getName().equals("Invocation")) .findFirst() @@ -1685,8 +1685,8 @@ void multiInvocation_stepWaitStep_producesCorrectSpans() { plugin.onInvocationEnd(new InvocationEndInfo("req-2", arn, false, InvocationStatus.SUCCEEDED, null)); var inv2Spans = spanExporter.getFinishedSpanItems(); - // wait continuation + step-B op + step-B attempt + invocation + Workflow = 5 - assertEquals(5, inv2Spans.size()); + // Wait continuation + step-B op + step-B attempt + Invocation + Workflow + DurableExecutionRoot = 6 + assertEquals(6, inv2Spans.size()); // Includes DurableExecutionRoot. var inv2TraceId = inv2Spans.stream() .filter(span -> span.getName().equals("Invocation")) @@ -1761,8 +1761,8 @@ void crossInvocation_stepRetry_attemptsParentedToRespectiveInvocations() { plugin.onInvocationEnd(new InvocationEndInfo("req-1", arn, true, InvocationStatus.PENDING, null)); var inv1Spans = spanExporter.getFinishedSpanItems(); - // operation span (PENDING) + attempt 1 span + invocation span = 3 - assertEquals(3, inv1Spans.size()); + // Operation (PENDING) + attempt 1 + Invocation + DurableExecutionRoot = 4 + assertEquals(4, inv1Spans.size()); var inv1OperationSpan = inv1Spans.stream() .filter(s -> @@ -1817,8 +1817,8 @@ void crossInvocation_stepRetry_attemptsParentedToRespectiveInvocations() { plugin.onInvocationEnd(new InvocationEndInfo("req-2", arn, false, InvocationStatus.SUCCEEDED, null)); var inv2Spans = spanExporter.getFinishedSpanItems(); - // operation span + attempt 2 span + invocation span + Workflow span = 4 - assertEquals(4, inv2Spans.size()); + // Operation + attempt 2 + Invocation + Workflow + DurableExecutionRoot = 5 + assertEquals(5, inv2Spans.size()); // Includes DurableExecutionRoot. var inv2OperationSpan = inv2Spans.stream() .filter(s -> From 5c79837019fad6f9566fd0f4c4e225b7c678da52 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 2 Oct 2026 20:54:39 +0000 Subject: [PATCH 02/40] test(otel): cover fallback root recovery across environments --- otel-plugin/README.md | 20 ++- .../durable/otel/ExecutionRootTest.java | 132 ++++++++++++++++-- 2 files changed, 140 insertions(+), 12 deletions(-) diff --git a/otel-plugin/README.md b/otel-plugin/README.md index c680a4ca7..999c6037e 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -42,17 +42,31 @@ If you configure your own `SdkTracerProviderBuilder`, add the OpenTelemetry SDK ## Fallback execution roots -When the backend supplies no complete remote parent, both views export an `DurableExecutionRoot` anchor before the first +When the backend supplies no complete remote parent, both views export a `DurableExecutionRoot` anchor before the first invocation returns, including when it suspends with `PENDING` or fails with `RETRYING`. The anchor is marked `durable.execution.synthetic_root=true`. Its trace and span IDs are deterministic and its start and end timestamps are the checkpointed execution start. It does not report execution status or duration; `Workflow` continues to report those at terminal completion. Complete remote parents remain externally owned and are never exported. +Java's `InvocationInfo` requires a non-null execution start timestamp from the initial checkpointed execution operation. +Missing timestamps are rejected before the plugin runs; anchor timestamps never fall back to the current wall clock. Every invocation may re-export the anchor to recover from an earlier interrupted or lost export. Re-exports retain the -same identity, timestamps, and attributes. The existing provider resource still applies. Each execution ARN owns its -own anchor even when multiple executions share a propagated trace ID without a parent; they are not collapsed into one +same span fields under stable sampling. The existing provider resource still applies: if a later invocation runs in another +execution environment, resource attributes such as `faas.instance` can differ. Backends that deduplicate by span identity +may retain either copy's resource. Each execution ARN owns its own anchor even when multiple executions share a +propagated trace ID without a parent; they are not collapsed into one execution. Upstream sampling and configured fallback sampling apply to anchors and their descendants together. +After the first invocation's export and flush succeed, its anchor remains available even if the execution is stopped +or times out while suspended and never invokes the plugin again. An invocation killed before its end hook or flush can +still lose its spans; a later invocation, including terminal completion, attempts to export the anchor again. Export +and flush failures do not provide a delivery guarantee. + +For a consistent hierarchy across invocations, preserve an explicit upstream sampling decision or use a deterministic +sampling policy based on the stable trace ID. A non-deterministic sampler can export an anchor without a Workflow, or +a Workflow without its anchor. Any sampler-supplied attributes and trace state must also stay stable for identical +anchor re-exports. + ## Quick Start using X-Ray/CloudWatch Tracing (ADOT Java Agent) 1. Add the ADOT Lambda Layer to your function diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java index acf3ca0ea..b03ad29ce 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java @@ -7,14 +7,23 @@ import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_STATUS; import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_SYNTHETIC_ROOT; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.export.SpanExporter; import io.opentelemetry.sdk.trace.samplers.Sampler; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.params.ParameterizedTest; @@ -123,13 +132,16 @@ void anchorsHonorSamplingAndAreOwnedByExecutionOnSharedTrace(boolean executionVi assertTrue(exporter.getFinishedSpanItems().isEmpty()); } var exporter = InMemorySpanExporter.create(); - var plugin = plugin( - executionView, - exporter, - new ExtractedContext(TRACE, null, ExtractedContext.Sampling.SAMPLED), - Sampler.alwaysOff()); - invoke(plugin, ARN, true, InvocationStatus.SUCCEEDED); - invoke(plugin, ARN + "-other", true, InvocationStatus.SUCCEEDED); + var extracted = new ExtractedContext(TRACE, null, ExtractedContext.Sampling.SAMPLED); + for (var service : List.of("caller", "callee")) { + var builder = SdkTracerProvider.builder() + .setResource(Resource.create(Attributes.of(AttributeKey.stringKey("service.name"), service))) + .setSampler(Sampler.alwaysOff()) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var plugin = plugin(executionView, builder, extracted); + var start = service.equals("caller") ? START : START.plusSeconds(10); + invoke(plugin, ARN + "-" + service, true, InvocationStatus.SUCCEEDED, start); + } var roots = exporter.getFinishedSpanItems().stream() .filter(span -> span.getName().equals("DurableExecutionRoot")) .toList(); @@ -139,14 +151,106 @@ void anchorsHonorSamplingAndAreOwnedByExecutionOnSharedTrace(boolean executionVi assertNotEquals( roots.get(0).getAttributes().get(DURABLE_EXECUTION_ARN), roots.get(1).getAttributes().get(DURABLE_EXECUTION_ARN)); + assertNotEquals(roots.get(0).getStartEpochNanos(), roots.get(1).getStartEpochNanos()); + assertNotEquals(roots.get(0).getResource(), roots.get(1).getResource()); + for (var root : roots) { + var workflow = exporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals("Workflow")) + .filter(span -> span.getAttributes() + .get(DURABLE_EXECUTION_ARN) + .equals(root.getAttributes().get(DURABLE_EXECUTION_ARN))) + .findFirst() + .orElseThrow(); + assertEquals(root.getSpanContext(), workflow.getParentSpanContext()); + } + } + + @ParameterizedTest + @CsvSource({"true,true", "true,false", "false,true", "false,false"}) + void failedFirstFlushIsRecoveredInAnotherEnvironment(boolean executionView, boolean success) { + var first = failedSuspendedAnchor(executionView); + var exporter = InMemorySpanExporter.create(); + var builder = SdkTracerProvider.builder() + .setResource(environmentResource("resumed")) + .setSampler(Sampler.alwaysOn()) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)); + invoke( + plugin(executionView, builder, null), + ARN, + false, + success ? InvocationStatus.SUCCEEDED : InvocationStatus.FAILED); + var recovered = named(exporter.getFinishedSpanItems(), "DurableExecutionRoot"); + assertStableSpanFields(first, recovered); + assertNotEquals(first.getResource(), recovered.getResource()); + assertEquals( + recovered.getSpanContext(), + named(exporter.getFinishedSpanItems(), "Workflow").getParentSpanContext()); + } + + private static SpanData failedSuspendedAnchor(boolean executionView) { + var attempted = new ArrayList(); + try (var processor = BatchSpanProcessor.builder(failingExporter(attempted)) + .setScheduleDelay(Duration.ofDays(1)) + .build()) { + var builder = SdkTracerProvider.builder() + .setResource(environmentResource("first")) + .setSampler(Sampler.alwaysOn()) + .addSpanProcessor(processor); + invoke(plugin(executionView, builder, null), ARN, true, InvocationStatus.PENDING); + // The end hook's forceFlush attempts the anchor export before returning, even without a terminal hook. + assertTrue(attempted.stream().noneMatch(span -> span.getName().equals("Workflow"))); + assertEquals( + START.getEpochSecond() * 1_000_000_000L, + named(attempted, "DurableExecutionRoot").getStartEpochNanos()); + } + return named(attempted, "DurableExecutionRoot"); + } + + private static SpanExporter failingExporter(List attempted) { + return new SpanExporter() { + @Override + public CompletableResultCode export(Collection spans) { + attempted.addAll(spans); + return CompletableResultCode.ofFailure(); + } + + @Override + public CompletableResultCode flush() { + return CompletableResultCode.ofFailure(); + } + + @Override + public CompletableResultCode shutdown() { + return CompletableResultCode.ofSuccess(); + } + }; + } + + private static Resource environmentResource(String instance) { + return Resource.create(Attributes.of(AttributeKey.stringKey("faas.instance"), instance)); } private static void assertStable(SpanData first, SpanData replay) { + assertStableSpanFields(first, replay); + assertEquals(first.getResource(), replay.getResource()); + } + + private static void assertStableSpanFields(SpanData first, SpanData replay) { assertEquals(first.getSpanContext(), replay.getSpanContext()); + assertEquals(first.getParentSpanContext(), replay.getParentSpanContext()); + assertEquals(first.getName(), replay.getName()); + assertEquals(first.getKind(), replay.getKind()); assertEquals(first.getStartEpochNanos(), replay.getStartEpochNanos()); assertEquals(first.getEndEpochNanos(), replay.getEndEpochNanos()); assertEquals(first.getAttributes(), replay.getAttributes()); - assertEquals(first.getResource(), replay.getResource()); + assertEquals(first.getStatus(), replay.getStatus()); + assertEquals(first.getEvents(), replay.getEvents()); + assertEquals(first.getLinks(), replay.getLinks()); + assertEquals(first.hasEnded(), replay.hasEnded()); + assertEquals(first.getTotalAttributeCount(), replay.getTotalAttributeCount()); + assertEquals(first.getTotalRecordedEvents(), replay.getTotalRecordedEvents()); + assertEquals(first.getTotalRecordedLinks(), replay.getTotalRecordedLinks()); + assertEquals(first.getInstrumentationScopeInfo(), replay.getInstrumentationScopeInfo()); } private static SpanData named(List spans, String name) { @@ -157,7 +261,12 @@ private static SpanData named(List spans, String name) { } private static void invoke(DurableExecutionPlugin plugin, String arn, boolean first, InvocationStatus status) { - plugin.onInvocationStart(new InvocationInfo("request", arn, first, START)); + invoke(plugin, arn, first, status, START); + } + + private static void invoke( + DurableExecutionPlugin plugin, String arn, boolean first, InvocationStatus status, Instant start) { + plugin.onInvocationStart(new InvocationInfo("request", arn, first, start)); plugin.onInvocationEnd(new InvocationEndInfo("request", arn, first, status, null)); } @@ -165,6 +274,11 @@ private static DurableExecutionPlugin plugin( boolean executionView, InMemorySpanExporter exporter, ExtractedContext extracted, Sampler sampler) { var builder = SdkTracerProvider.builder().setSampler(sampler).addSpanProcessor(SimpleSpanProcessor.create(exporter)); + return plugin(executionView, builder, extracted); + } + + private static DurableExecutionPlugin plugin( + boolean executionView, SdkTracerProviderBuilder builder, ExtractedContext extracted) { var config = OtelPluginConfig.builder() .contextExtractor(() -> extracted) .enableMdc(false) From 2cefa300de10354094c25d1cb3d400e51743f09c Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 2 Oct 2026 21:11:35 +0000 Subject: [PATCH 03/40] merge: preserve remote root recovery test ancestry --- .../durable/otel/OtelPluginSupport.java | 6 +- .../durable/otel/ExecutionRootTest.java | 60 +++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 8b9aaa5bc..43ef7aeb7 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -46,10 +46,12 @@ static void exportExecutionRoot( .setAttribute(DURABLE_EXECUTION_ARN, arn) .setAttribute(DURABLE_EXECUTION_SYNTHETIC_ROOT, true) .setStartTimestamp(start); + Span root; try (var ignored = DurableSamplingDecision.openScope(intent)) { - var root = idGenerator.startSpan(builder, ancestor.getTraceId(), ancestor.getSpanId()); - root.end(start); + root = idGenerator.startSpan(builder, ancestor.getTraceId(), ancestor.getSpanId()); } + // End processors/exporters may create unrelated spans; they must not inherit this sampling override. + root.end(start); } private static final Logger logger = LoggerFactory.getLogger(OtelPluginSupport.class); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java index b03ad29ce..30d79ff38 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java @@ -9,12 +9,17 @@ import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.Context; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.ReadWriteSpan; +import io.opentelemetry.sdk.trace.ReadableSpan; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import io.opentelemetry.sdk.trace.SpanProcessor; import io.opentelemetry.sdk.trace.data.SpanData; import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; @@ -26,6 +31,7 @@ import java.util.Collection; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; @@ -230,6 +236,60 @@ private static Resource environmentResource(String instance) { return Resource.create(Attributes.of(AttributeKey.stringKey("faas.instance"), instance)); } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void processorSpansDoNotInheritDurableSamplingOverride(boolean executionView) { + var exporter = InMemorySpanExporter.create(); + var observed = new AtomicReference(); + try (var unrelated = SdkTracerProvider.builder() + .setSampler(DurableSampler.wrap(Sampler.alwaysOff())) + .build()) { + var builder = SdkTracerProvider.builder() + .setSampler(Sampler.alwaysOff()) + .addSpanProcessor(new SamplingObserver(unrelated, observed)) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var config = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(() -> new ExtractedContext(TRACE, null, ExtractedContext.Sampling.SAMPLED)) + .build(); + DurableExecutionPlugin plugin = executionView + ? new ExecutionOtelPlugin(builder, config) + : new InvocationOtelPlugin(builder, config); + invoke(plugin, ARN, true, InvocationStatus.PENDING); + assertTrue(named(exporter.getFinishedSpanItems(), "DurableExecutionRoot") + .getSpanContext() + .isSampled()); + assertNotNull(observed.get(), "Root end processor must have run"); + assertFalse(observed.get().isSampled(), "Processor's unrelated span must retain its always-off policy"); + } + } + + private record SamplingObserver(SdkTracerProvider provider, AtomicReference observed) + implements SpanProcessor { + @Override + public void onStart(Context parent, ReadWriteSpan span) {} + + @Override + public boolean isStartRequired() { + return false; + } + + @Override + public void onEnd(ReadableSpan span) { + if (span.getName().equals("DurableExecutionRoot")) { + var callback = + provider.get("processor").spanBuilder("unrelated").startSpan(); + observed.set(callback.getSpanContext()); + callback.end(); + } + } + + @Override + public boolean isEndRequired() { + return true; + } + } + private static void assertStable(SpanData first, SpanData replay) { assertStableSpanFields(first, replay); assertEquals(first.getResource(), replay.getResource()); From 7c967444dfbe100074942168c2c5dfd00c9b3882 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 2 Oct 2026 21:29:31 +0000 Subject: [PATCH 04/40] fix(otel): consume sampling bridge before start processors --- .../lambda/durable/otel/DurableSampler.java | 2 +- .../durable/otel/DurableSamplingDecision.java | 22 ++++++++++----- .../durable/otel/DurableSamplerTest.java | 26 ++++++++++++++++++ ...urableSamplingDecisionClassLoaderTest.java | 5 +++- .../durable/otel/ExecutionRootTest.java | 27 +++++++++++++------ 5 files changed, 66 insertions(+), 16 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java index 7ce304a54..34e222ab3 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java @@ -113,7 +113,7 @@ public SamplingResult shouldSample( SpanKind spanKind, Attributes attributes, List parentLinks) { - var intent = DurableSamplingDecision.get(parentContext); + var intent = DurableSamplingDecision.consume(parentContext); if (intent == null) { // Not a durable span: the customer's sampler governs it unchanged. return delegate.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java index c43d8eb02..930351292 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java @@ -32,15 +32,16 @@ * the full {@link SamplingResult}, including any attributes a custom sampler attached. *

  • Thread-scoped system property — a cross-class-loader fallback modelled on * {@link DeterministicIdGenerator}'s scoped-ID bridge. The intent is published on the thread that creates the - * durable span for the synchronous duration of {@code startSpan()} (the sampler runs on that same thread), keyed - * by thread ID under a bootstrap-visible {@link System} property so both class loaders read the same value. A - * resolved decision bridges its {@link SamplingDecision} name (the three built-in decisions carry no attributes, - * so reconstructing them is faithful); a deferral bridges a sentinel plus the canonical trace ID. + * durable span until its sampler consumes the value (before synchronous span processors run), keyed by thread ID + * under a bootstrap-visible {@link System} property so both class loaders read the same value. A resolved + * decision bridges its {@link SamplingDecision} name (the three built-in decisions carry no attributes, so + * reconstructing them is faithful); a deferral bridges a sentinel plus the canonical trace ID. * * *

    The scope is opened immediately around each durable {@code startSpan()} call and closed right after, so the - * property never leaks beyond the span it applies to. Nothing is persisted across invocations; cross-invocation - * consistency comes from recomputing the intent from stable inputs, not from sharing state. + * property is consumed once by the sampler before callbacks can create unrelated spans. Nothing is persisted across + * invocations; cross-invocation consistency comes from recomputing the intent from stable inputs, not from sharing + * state. */ final class DurableSamplingDecision { @@ -109,6 +110,15 @@ static Intent get(Context context) { return fromScopedProperty(); } + /** Resolves this span's intent and consumes the cross-loader bridge before sampler/processor callbacks run. */ + static Intent consume(Context context) { + try { + return get(context); + } finally { + System.clearProperty(scopedProperty()); + } + } + private static String encode(Intent intent) { return intent.isDeferred() ? DEFERRED_PREFIX + intent.deferredTraceId() diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java index 6fea5784e..21b977739 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java @@ -62,6 +62,32 @@ void tearDown() { OtelPluginAutoConfigurationState.resetInstalledForTest(); } + @Test + void consumingContextIntentAlsoClearsCrossLoaderFallback() { + var delegate = new CountingSampler(Sampler.alwaysOff()); + var sampler = DurableSampler.wrap(delegate); + var parent = DurableSamplingDecision.store( + Context.root(), DurableSamplingDecision.Intent.resolved(SamplingResult.recordOnly())); + try (var ignored = DurableSamplingDecision.openScope( + DurableSamplingDecision.Intent.resolved(SamplingResult.recordAndSample()))) { + assertEquals( + SamplingDecision.RECORD_ONLY, + sampler.shouldSample(parent, TRACE_ID, "durable", SpanKind.INTERNAL, Attributes.empty(), List.of()) + .getDecision()); + assertEquals( + SamplingDecision.DROP, + sampler.shouldSample( + Context.root(), + TRACE_ID, + "callback", + SpanKind.INTERNAL, + Attributes.empty(), + List.of()) + .getDecision()); + assertEquals(1, delegate.count(), "The unrelated callback must use its own sampler"); + } + } + // ─── Unit tests for the wrapper ────────────────────────────────────── @Test diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java index 121f4c8fb..bcb9e6c32 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java @@ -53,13 +53,16 @@ void decisionCrossesClassLoaderBoundary_viaScopedProperty() throws Exception { openScope.setAccessible(true); var get = agentDecision.getDeclaredMethod("get", Context.class); get.setAccessible(true); + var consume = agentDecision.getDeclaredMethod("consume", Context.class); + consume.setAccessible(true); // The application-side loader publishes the intent on this thread; the agent-side loader reads it back from // a ROOT context (its context key would be a different instance and would miss), reconstructing its own // Intent from the bridged value. var scope = (AutoCloseable) openScope.invoke(null, appIntent); try { - var crossLoaderIntent = get.invoke(null, Context.root()); + var crossLoaderIntent = consume.invoke(null, Context.root()); + assertNull(get.invoke(null, Context.root()), "The bridge is consumed before onStart processors run"); assertNotNull( crossLoaderIntent, "The agent-side loader must read the intent published by the app side"); // Its Intent type is the agent loader's copy; read the resolved SamplingResult reflectively. diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java index 30d79ff38..2a00910d5 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java @@ -237,8 +237,8 @@ private static Resource environmentResource(String instance) { } @ParameterizedTest - @ValueSource(booleans = {true, false}) - void processorSpansDoNotInheritDurableSamplingOverride(boolean executionView) { + @CsvSource({"true,true", "true,false", "false,true", "false,false"}) + void processorSpansDoNotInheritDurableSamplingOverride(boolean executionView, boolean observeStart) { var exporter = InMemorySpanExporter.create(); var observed = new AtomicReference(); try (var unrelated = SdkTracerProvider.builder() @@ -246,7 +246,7 @@ void processorSpansDoNotInheritDurableSamplingOverride(boolean executionView) { .build()) { var builder = SdkTracerProvider.builder() .setSampler(Sampler.alwaysOff()) - .addSpanProcessor(new SamplingObserver(unrelated, observed)) + .addSpanProcessor(new SamplingObserver(unrelated, observed, observeStart)) .addSpanProcessor(SimpleSpanProcessor.create(exporter)); var config = OtelPluginConfig.builder() .enableMdc(false) @@ -259,23 +259,34 @@ void processorSpansDoNotInheritDurableSamplingOverride(boolean executionView) { assertTrue(named(exporter.getFinishedSpanItems(), "DurableExecutionRoot") .getSpanContext() .isSampled()); - assertNotNull(observed.get(), "Root end processor must have run"); + assertNotNull(observed.get(), "Root processor callback must have run"); assertFalse(observed.get().isSampled(), "Processor's unrelated span must retain its always-off policy"); } } - private record SamplingObserver(SdkTracerProvider provider, AtomicReference observed) + private record SamplingObserver( + SdkTracerProvider provider, AtomicReference observed, boolean observeStart) implements SpanProcessor { @Override - public void onStart(Context parent, ReadWriteSpan span) {} + public void onStart(Context parent, ReadWriteSpan span) { + if (observeStart) { + observe(span); + } + } @Override public boolean isStartRequired() { - return false; + return observeStart; } @Override public void onEnd(ReadableSpan span) { + if (!observeStart) { + observe(span); + } + } + + private void observe(ReadableSpan span) { if (span.getName().equals("DurableExecutionRoot")) { var callback = provider.get("processor").spanBuilder("unrelated").startSpan(); @@ -286,7 +297,7 @@ public void onEnd(ReadableSpan span) { @Override public boolean isEndRequired() { - return true; + return !observeStart; } } From cf3541b4b91c08d0ed7207e4e0d229b456a83626 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 2 Oct 2026 21:41:32 +0000 Subject: [PATCH 05/40] test(otel): verify root start processor sampling and identity --- .../otel/RootStartProcessorIsolationTest.java | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java new file mode 100644 index 000000000..83b6aa8e4 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java @@ -0,0 +1,173 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.trace.ReadWriteSpan; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; + +class RootStartProcessorIsolationTest { + private static final String TRACE = "6955b900123456789012345678901234"; + private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id"; + private static final Instant START = Instant.parse("2026-10-02T00:00:00Z"); + + static Stream cases() { + return Stream.of(true, false) + .flatMap(view -> Stream.of(true, false) + .flatMap(sampled -> Stream.of(true, false) + .flatMap(shared -> Stream.of(true, false) + .map(recordOnly -> Arguments.of(view, sampled, shared, recordOnly))))); + } + + @ParameterizedTest + @MethodSource("cases") + void rootStartCallbacksRetainTheirOwnSamplingAndRandomIds( + boolean executionView, boolean sampled, boolean sharedGenerator, boolean rootRecordOnly) { + var observer = new StartObserver(); + var policy = sampled ? Sampler.alwaysOn() : Sampler.alwaysOff(); + var builder = SdkTracerProvider.builder() + .setSampler(rootSampler(rootRecordOnly, policy)) + .addSpanProcessor(observer); + var config = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(() -> new ExtractedContext( + TRACE, + null, + rootRecordOnly ? ExtractedContext.Sampling.UNDECIDED : ExtractedContext.Sampling.SAMPLED)) + .build(); + DurableExecutionPlugin plugin = + executionView ? new ExecutionOtelPlugin(builder, config) : new InvocationOtelPlugin(builder, config); + // Rebuilding this configured builder shares the plugin-installed generator; the other branch exercises the + // same thread-scoped bridge through an independent generator instance. + try (var unrelated = sharedGenerator + ? builder.build() + : SdkTracerProvider.builder() + .setSampler(DurableSampler.wrap(policy)) + .setIdGenerator(new DeterministicIdGenerator()) + .build()) { + observer.tracer = unrelated.get("processor"); + var original = Context.current(); + assertControl(observer.tracer, sampled); + invoke(plugin, true); + invoke(plugin, false); + assertControl(observer.tracer, sampled); + assertSame(original, Context.current()); + assertObservations(observer, sampled); + } + } + + private static void invoke(DurableExecutionPlugin plugin, boolean first) { + plugin.onInvocationStart(new InvocationInfo("request", ARN, first, START)); + plugin.onInvocationEnd(new InvocationEndInfo("request", ARN, first, InvocationStatus.PENDING, null)); + } + + private static void assertControl(Tracer tracer, boolean sampled) { + var span = tracer.spanBuilder("control").setNoParent().startSpan(); + assertEquals(sampled, span.getSpanContext().isSampled()); + assertEquals(sampled, span.isRecording()); + span.end(); + } + + private static void assertObservations(StartObserver observer, boolean sampled) { + assertEquals(2, observer.roots.size()); + assertEquals(observer.roots.get(0), observer.roots.get(1)); + var root = observer.roots.get(0); + assertEquals(new DeterministicIdGenerator().generateExecutionRootSpanId(ARN), root.getSpanId()); + assertEquals(2, observer.callbacks.size()); + assertEquals(List.of(sampled, sampled), observer.recording); + for (var callback : observer.callbacks) { + assertEquals(sampled, callback.isSampled()); + assertTrue(callback.isValid()); + assertNotEquals(root.getTraceId(), callback.getTraceId()); + assertNotEquals(root.getSpanId(), callback.getSpanId()); + } + assertEquals( + 2, + observer.callbacks.stream() + .map(SpanContext::getTraceId) + .distinct() + .count()); + assertEquals( + 2, + observer.callbacks.stream() + .map(SpanContext::getSpanId) + .distinct() + .count()); + } + + private static Sampler rootSampler(boolean recordOnly, Sampler policy) { + if (!recordOnly) return policy; + return new Sampler() { + @Override + public SamplingResult shouldSample( + Context parent, + String trace, + String name, + SpanKind kind, + Attributes attributes, + List links) { + return name.equals("Workflow") + ? SamplingResult.recordOnly() + : policy.shouldSample(parent, trace, name, kind, attributes, links); + } + + @Override + public String getDescription() { + return "RecordOnlyWorkflow"; + } + }; + } + + private static final class StartObserver implements SpanProcessor { + private Tracer tracer; + private final List roots = new ArrayList<>(); + private final List callbacks = new ArrayList<>(); + private final List recording = new ArrayList<>(); + + @Override + public void onStart(Context parent, ReadWriteSpan span) { + // The name guard permits exactly one unrelated span per root and prevents processor recursion. + if (!span.getName().equals("DurableExecutionRoot")) return; + roots.add(span.getSpanContext()); + var callback = tracer.spanBuilder("unrelated-onStart").setNoParent().startSpan(); + callbacks.add(callback.getSpanContext()); + recording.add(callback.isRecording()); + callback.end(); + } + + @Override + public boolean isStartRequired() { + return true; + } + + @Override + public void onEnd(ReadableSpan span) {} + + @Override + public boolean isEndRequired() { + return false; + } + } +} From 0284eda29565cd6ef4ecb67ba1caa4a7d32db446 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 2 Oct 2026 22:19:53 +0000 Subject: [PATCH 06/40] fix(otel): consume parent context sampling intent once --- .../durable/otel/DurableSamplingDecision.java | 21 ++++++++++++------- .../durable/otel/DurableSamplerTest.java | 5 +++-- .../otel/RootStartProcessorIsolationTest.java | 21 +++++++++++++++---- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java index 930351292..9c72c39bd 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java @@ -6,6 +6,7 @@ import io.opentelemetry.context.ContextKey; import io.opentelemetry.sdk.trace.samplers.SamplingDecision; import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import java.util.concurrent.atomic.AtomicReference; /** * Carries the durable execution's sampling intent to {@link DurableSampler} for one durable span. @@ -66,7 +67,7 @@ boolean isDeferred() { } } - private static final ContextKey KEY = + private static final ContextKey> KEY = ContextKey.named("software.amazon.lambda.durable.otel.durable-sampling-decision"); private static final String SCOPED_PROPERTY_PREFIX = "software.amazon.lambda.durable.otel.scopedSamplingDecision."; @@ -75,9 +76,9 @@ boolean isDeferred() { private DurableSamplingDecision() {} - /** Returns a context carrying the durable sampling intent (same-class-loader carrier), derived from the given. */ + /** Returns a context carrying a one-shot intent for one span, preserving the full SamplingResult. */ static Context store(Context context, Intent intent) { - return context.with(KEY, intent); + return context.with(KEY, new AtomicReference<>(intent)); } /** @@ -99,21 +100,25 @@ static Scope openScope(Intent intent) { } /** - * Returns the durable sampling intent for a span, or {@code null} when none is present. Prefers the full-fidelity - * context key (same class loader) and falls back to the thread-scoped system property (cross class loader). + * Inspects the unconsumed durable sampling intent for a span, or {@code null} when none is present. Prefers the + * full-fidelity context key (same class loader) and falls back to the thread-scoped system property (cross class + * loader). */ static Intent get(Context context) { - var fromContext = context.get(KEY); + var holder = context.get(KEY); + var fromContext = holder != null ? holder.get() : null; if (fromContext != null) { return fromContext; } return fromScopedProperty(); } - /** Resolves this span's intent and consumes the cross-loader bridge before sampler/processor callbacks run. */ + /** Consumes both carriers before callbacks can reuse the parent context or cross-loader bridge. */ static Intent consume(Context context) { try { - return get(context); + var holder = context.get(KEY); + var fromContext = holder != null ? holder.getAndSet(null) : null; + return fromContext != null ? fromContext : fromScopedProperty(); } finally { System.clearProperty(scopedProperty()); } diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java index 21b977739..f6126492e 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java @@ -150,9 +150,10 @@ void deferredIntent_evaluatesDelegateOncePerExecution() { // A stateful/quota delegate must be consulted once per execution (trace ID), not per durable span. var delegate = new CountingSampler(Sampler.alwaysOn()); var sampler = DurableSampler.wrap(delegate); - var parent = DurableSamplingDecision.store(Context.root(), DurableSamplingDecision.Intent.deferred(TRACE_ID)); - for (var i = 0; i < 4; i++) { + // Each SDK-owned span receives its own one-shot carrier, while the resolved decision stays execution-wide. + var parent = + DurableSamplingDecision.store(Context.root(), DurableSamplingDecision.Intent.deferred(TRACE_ID)); sampler.shouldSample(parent, TRACE_ID, "op" + i, SpanKind.INTERNAL, Attributes.empty(), List.of()); } diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java index 83b6aa8e4..5c26b8671 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java @@ -38,14 +38,20 @@ static Stream cases() { .flatMap(view -> Stream.of(true, false) .flatMap(sampled -> Stream.of(true, false) .flatMap(shared -> Stream.of(true, false) - .map(recordOnly -> Arguments.of(view, sampled, shared, recordOnly))))); + .flatMap(recordOnly -> Stream.of(true, false) + .map(forwardParent -> Arguments.of( + view, sampled, shared, recordOnly, forwardParent)))))); } @ParameterizedTest @MethodSource("cases") void rootStartCallbacksRetainTheirOwnSamplingAndRandomIds( - boolean executionView, boolean sampled, boolean sharedGenerator, boolean rootRecordOnly) { - var observer = new StartObserver(); + boolean executionView, + boolean sampled, + boolean sharedGenerator, + boolean rootRecordOnly, + boolean forwardParent) { + var observer = new StartObserver(forwardParent); var policy = sampled ? Sampler.alwaysOn() : Sampler.alwaysOff(); var builder = SdkTracerProvider.builder() .setSampler(rootSampler(rootRecordOnly, policy)) @@ -142,6 +148,12 @@ public String getDescription() { private static final class StartObserver implements SpanProcessor { private Tracer tracer; + private final boolean forwardParent; + + private StartObserver(boolean forwardParent) { + this.forwardParent = forwardParent; + } + private final List roots = new ArrayList<>(); private final List callbacks = new ArrayList<>(); private final List recording = new ArrayList<>(); @@ -151,7 +163,8 @@ public void onStart(Context parent, ReadWriteSpan span) { // The name guard permits exactly one unrelated span per root and prevents processor recursion. if (!span.getName().equals("DurableExecutionRoot")) return; roots.add(span.getSpanContext()); - var callback = tracer.spanBuilder("unrelated-onStart").setNoParent().startSpan(); + var builder = tracer.spanBuilder("unrelated-onStart"); + var callback = (forwardParent ? builder.setParent(parent) : builder.setNoParent()).startSpan(); callbacks.add(callback.getSpanContext()); recording.add(callback.isRecording()); callback.end(); From 6b3e260db351b6c0a8db10c25562d442de963581 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 6 Oct 2026 22:42:23 +0000 Subject: [PATCH 07/40] Preserve pending conformance and E2E CI runs --- .github/workflows/conformance-tests.yml | 1 + .github/workflows/e2e-tests.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/conformance-tests.yml b/.github/workflows/conformance-tests.yml index 06baa03f5..c066bf277 100644 --- a/.github/workflows/conformance-tests.yml +++ b/.github/workflows/conformance-tests.yml @@ -25,6 +25,7 @@ concurrency: # same stack -- mirrors e2e-tests.yml. group: conformance-tests cancel-in-progress: false + queue: max permissions: contents: read diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index e57070144..5dd246d57 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -26,6 +26,7 @@ on: concurrency: group: e2e-tests cancel-in-progress: false + queue: max # permission can be added at job level or workflow level permissions: From efa70dc8a1faffc935c40d2ac61c13264ef1bc19 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:39:09 -0700 Subject: [PATCH 08/40] ci: serialize revised OTel callers without replacing pending work --- .github/workflows/otel-conformance-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/otel-conformance-tests.yml b/.github/workflows/otel-conformance-tests.yml index 68b3bc3f2..43a3e693f 100644 --- a/.github/workflows/otel-conformance-tests.yml +++ b/.github/workflows/otel-conformance-tests.yml @@ -53,6 +53,11 @@ on: permissions: {} +concurrency: + group: otel-conformance-tests + cancel-in-progress: false + queue: max + jobs: opentelemetry: # 1. Run for non-PR events, such as scheduled runs and manual invocations From e779bc4a377e09565d3c30b1fbdbbd539b23f9d6 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 01:34:41 +0000 Subject: [PATCH 09/40] ci: run OTel conformance on validated CodeBuild Java 21 --- .github/workflows/otel-conformance-tests.yml | 33 +++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/.github/workflows/otel-conformance-tests.yml b/.github/workflows/otel-conformance-tests.yml index 43a3e693f..4f948b298 100644 --- a/.github/workflows/otel-conformance-tests.yml +++ b/.github/workflows/otel-conformance-tests.yml @@ -67,8 +67,9 @@ jobs: actions: write contents: read id-token: write - uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@43872f8d5dd917fe3ee6d01a6c953c1ff67c5fe4 + uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@a66037abbbfa55fde97f714e30f0bc262edefd63 with: + runs_on: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'ubuntu-latest' || format('codebuild-github-actions-runner-{0}-{1}', github.run_id, github.run_attempt) }} language: java resource_prefix: j sdk_repository: aws/aws-durable-execution-sdk-java @@ -80,13 +81,37 @@ jobs: # checked out (.build/durable-sdk). examples_dir: .build/durable-sdk/conformance-tests-otel setup_command: | - if [ -z "${JAVA_HOME_21_X64:-}" ]; then - echo "The runner does not provide Java 21" + set -euo pipefail + if [ -n "${JAVA_HOME_21_X64:-}" ]; then + export JAVA_HOME="$JAVA_HOME_21_X64" + elif [ -x /usr/lib/jvm/java-21-amazon-corretto/bin/java ]; then + export JAVA_HOME=/usr/lib/jvm/java-21-amazon-corretto + elif [ -z "${JAVA_HOME:-}" ]; then + export JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v java)")")")" + fi + if [ ! -x "$JAVA_HOME/bin/java" ] || [ ! -x "$JAVA_HOME/bin/javac" ]; then + echo "Selected JAVA_HOME=$JAVA_HOME is not a complete JDK" exit 1 fi - export JAVA_HOME="$JAVA_HOME_21_X64" export PATH="$JAVA_HOME/bin:$PATH" + java_spec=$(java -XshowSettings:properties -version 2>&1 | awk '$1 == "java.specification.version" {print $3}') + if [ "$java_spec" != "21" ]; then + echo "Java 21 is required; selected JAVA_HOME=$JAVA_HOME reports specification version $java_spec" + exit 1 + fi + javac_version=$(javac -version 2>&1) + if [[ ! "$javac_version" =~ ^javac[[:space:]]21([.]|[[:space:]]|$) ]]; then + echo "Java 21 javac is required; selected compiler reports $javac_version" + exit 1 + fi java -version + javac -version + if [ -n "${GITHUB_ENV:-}" ]; then + echo "JAVA_HOME=$JAVA_HOME" >> "$GITHUB_ENV" + fi + if [ -n "${GITHUB_PATH:-}" ]; then + echo "$JAVA_HOME/bin" >> "$GITHUB_PATH" + fi prepare_command: | JAVA_SDK_VERSION=$( mvn -B -q \ From 9e9bdbb347f0f2c4e831f94e3b6daf397db2c3b0 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 02:50:05 +0000 Subject: [PATCH 10/40] ci: run Build for OpenTelemetry plugin changes --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a3e02cc11..8f0ab1157 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,6 +29,7 @@ on: - 'sdk-testing/**' - 'sdk-integration-tests/**' - 'insight-plugin/**' + - 'otel-plugin/**' - 'examples/**' - 'pom.xml' push: @@ -42,6 +43,7 @@ on: - 'sdk-testing/**' - 'sdk-integration-tests/**' - 'insight-plugin/**' + - 'otel-plugin/**' - 'examples/**' - 'pom.xml' From 652dfa577d4c5c93b87bd3b651b537ef439cb5f8 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 03:04:55 +0000 Subject: [PATCH 11/40] fix: isolate durable sampling across agent class loaders --- .../durable/otel/DurableSamplingDecision.java | 5 +- .../durable/otel/ExecutionOtelPlugin.java | 10 +- .../durable/otel/InvocationOtelPlugin.java | 10 +- .../durable/otel/OtelPluginSupport.java | 5 +- ...urableSamplingDecisionClassLoaderTest.java | 118 ++++++++++++++++++ 5 files changed, 140 insertions(+), 8 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java index 9c72c39bd..2341f8bdb 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java @@ -29,8 +29,9 @@ * identity, so a key created in one loader is not equal to the key created in the other. To bridge this: * *

      - *
    1. Context key — used when both sides share a class loader (an application-owned provider). It preserves - * the full {@link SamplingResult}, including any attributes a custom sampler attached. + *
    2. Context key — used when both sides share a class loader (an application-owned provider). Agent-backed + * spans omit this carrier because another loader cannot consume its key. It preserves the full + * {@link SamplingResult}, including any attributes a custom sampler attached. *
    3. Thread-scoped system property — a cross-class-loader fallback modelled on * {@link DeterministicIdGenerator}'s scoped-ID bridge. The intent is published on the thread that creates the * durable span until its sampler consumes the value (before synchronous span processors run), keyed by thread ID diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index fa9016557..a6882079d 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -333,7 +333,13 @@ public void onInvocationEnd(InvocationEndInfo info) { workflowSpan.end(); } OtelPluginSupport.exportExecutionRoot( - tracer, idGenerator, executionAncestor, durableExecutionArn, executionStartTime, samplingIntent); + tracer, + idGenerator, + executionAncestor, + durableExecutionArn, + executionStartTime, + samplingIntent, + sdkTracerProvider != null); workflowSpanContext = null; executionAncestor = null; executionStartTime = null; @@ -634,7 +640,7 @@ private Context resolveParentContext(String parentId) { */ private Context withDurableDecision(Context context) { var intent = samplingIntent; - return intent != null ? DurableSamplingDecision.store(context, intent) : context; + return intent != null && sdkTracerProvider != null ? DurableSamplingDecision.store(context, intent) : context; } private TraceFlags effectiveTraceFlags() { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index 16b5e74f3..aefe15f4c 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -339,7 +339,13 @@ public void onInvocationEnd(InvocationEndInfo info) { workflowSpan.end(); } OtelPluginSupport.exportExecutionRoot( - tracer, idGenerator, executionAncestor, durableExecutionArn, executionStartTime, samplingIntent); + tracer, + idGenerator, + executionAncestor, + durableExecutionArn, + executionStartTime, + samplingIntent, + sdkTracerProvider != null); workflowSpanContext = null; executionAncestor = null; executionStartTime = null; @@ -652,7 +658,7 @@ private Context resolveParentContext(String parentId) { */ private Context withDurableDecision(Context context) { var intent = samplingIntent; - return intent != null ? DurableSamplingDecision.store(context, intent) : context; + return intent != null && sdkTracerProvider != null ? DurableSamplingDecision.store(context, intent) : context; } /** diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 43ef7aeb7..573c35a4f 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -36,13 +36,14 @@ static void exportExecutionRoot( SpanContext ancestor, String arn, Instant start, - DurableSamplingDecision.Intent intent) { + DurableSamplingDecision.Intent intent, + boolean useContextCarrier) { if (ancestor == null || ancestor.isRemote()) { return; } var builder = tracer.spanBuilder("DurableExecutionRoot") .setSpanKind(SpanKind.INTERNAL) - .setParent(DurableSamplingDecision.store(Context.root(), intent)) + .setParent(useContextCarrier ? DurableSamplingDecision.store(Context.root(), intent) : Context.root()) .setAttribute(DURABLE_EXECUTION_ARN, arn) .setAttribute(DURABLE_EXECUTION_SYNTHETIC_ROOT, true) .setStartTimestamp(start); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java index bcb9e6c32..a4aaa5104 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java @@ -3,17 +3,38 @@ package software.amazon.lambda.durable.otel; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertNull; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerProvider; import io.opentelemetry.context.Context; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.sdk.trace.IdGenerator; +import io.opentelemetry.sdk.trace.ReadWriteSpan; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.samplers.Sampler; import io.opentelemetry.sdk.trace.samplers.SamplingDecision; import io.opentelemetry.sdk.trace.samplers.SamplingResult; import java.net.URL; import java.net.URLClassLoader; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; /** * Verifies the durable sampling decision crosses the application/Java-agent class-loader boundary. @@ -82,6 +103,103 @@ void decisionCrossesClassLoaderBoundary_viaScopedProperty() throws Exception { } } + @ParameterizedTest + @ValueSource(strings = {"InvocationOtelPlugin", "ExecutionOtelPlugin"}) + void agentProcessorForwardingParentCannotReuseApplicationSamplingIntent(String pluginName) throws Exception { + var previousHeader = System.getProperty("com.amazonaws.xray.traceHeader"); + GlobalOpenTelemetry.resetForTest(); + OtelPluginAutoConfigurationState.markInstalled(); + System.setProperty("com.amazonaws.xray.traceHeader", "Root=1-6955b900-123456789012345678901234;Sampled=1"); + try (var appLoader = pluginClassLoader(); + var agentLoader = pluginClassLoader()) { + var appSamplerType = Class.forName(DurableSampler.class.getName(), true, appLoader); + var appWrap = appSamplerType.getDeclaredMethod("wrap", Sampler.class); + appWrap.setAccessible(true); + var agentSamplerType = Class.forName(DurableSampler.class.getName(), true, agentLoader); + var agentWrap = agentSamplerType.getDeclaredMethod("wrap", Sampler.class); + agentWrap.setAccessible(true); + var agentIdType = Class.forName(DeterministicIdGenerator.class.getName(), true, agentLoader); + var callbacks = new AtomicInteger(); + var leakedSampling = new AtomicBoolean(); + try (var appProvider = SdkTracerProvider.builder() + .setSampler((Sampler) appWrap.invoke(null, Sampler.alwaysOff())) + .build(); + var agentProvider = SdkTracerProvider.builder() + .setSampler((Sampler) agentWrap.invoke(null, Sampler.alwaysOff())) + .setIdGenerator( + (IdGenerator) agentIdType.getConstructor().newInstance()) + .addSpanProcessor(new SpanProcessor() { + @Override + public void onStart(Context parent, ReadWriteSpan span) { + if (!span.getName().equals("DurableExecutionRoot")) return; + var unrelated = appProvider + .get("processor") + .spanBuilder("unrelated-forwarded-parent") + .setParent(parent) + .startSpan(); + callbacks.incrementAndGet(); + leakedSampling.compareAndSet(false, unrelated.isRecording()); + unrelated.end(); + } + + @Override + public boolean isStartRequired() { + return true; + } + + @Override + public void onEnd(ReadableSpan span) {} + + @Override + public boolean isEndRequired() { + return false; + } + }) + .build()) { + var hiddenAgentProvider = new TracerProvider() { + @Override + public Tracer get(String name) { + return agentProvider.get(name); + } + + @Override + public Tracer get(String name, String version) { + return agentProvider.get(name, version); + } + }; + GlobalOpenTelemetry.set(new OpenTelemetry() { + @Override + public TracerProvider getTracerProvider() { + return hiddenAgentProvider; + } + + @Override + public ContextPropagators getPropagators() { + return ContextPropagators.noop(); + } + }); + var plugin = (DurableExecutionPlugin) + Class.forName("software.amazon.lambda.durable.otel." + pluginName, true, appLoader) + .getConstructor() + .newInstance(); + var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id"; + for (var first : new boolean[] {true, false}) { + plugin.onInvocationStart(new InvocationInfo("request", arn, first, Instant.EPOCH)); + plugin.onInvocationEnd( + new InvocationEndInfo("request", arn, first, InvocationStatus.PENDING, null)); + } + assertEquals(2, callbacks.get(), "Both invocation roots must reach the real agent-side processor"); + assertFalse( + leakedSampling.get(), "The supplied parent must not override the app provider's DROP policy"); + } + } finally { + GlobalOpenTelemetry.resetForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); + if (previousHeader == null) System.clearProperty("com.amazonaws.xray.traceHeader"); + else System.setProperty("com.amazonaws.xray.traceHeader", previousHeader); + } + } + /** * A child-first class loader that loads {@code software.amazon.lambda.durable.otel.*} itself (so each instance * holds its own copies, mirroring the two plugin class loaders) while delegating OpenTelemetry and JDK classes to From 6edd3e4da24acf36dcc5e868a08c37b8d0835177 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:16:27 -0700 Subject: [PATCH 12/40] test: return one checkpoint state per operation in unit mocks (cherry picked from commit 5e171cbab84a00c121535cd5771eed055ccaa437) --- .../amazon/lambda/durable/TestUtils.java | 9 ++-- .../amazon/lambda/durable/TestUtilsTest.java | 51 +++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/TestUtilsTest.java diff --git a/sdk/src/test/java/software/amazon/lambda/durable/TestUtils.java b/sdk/src/test/java/software/amazon/lambda/durable/TestUtils.java index 1aaba5d4c..44f0078a9 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/TestUtils.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/TestUtils.java @@ -5,7 +5,7 @@ import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; -import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.UUID; import software.amazon.awssdk.services.lambda.model.*; @@ -18,7 +18,8 @@ public static DurableExecutionClient createMockClient() { var client = mock(DurableExecutionClient.class); when(client.checkpoint(any(), any(), any())).thenAnswer(invocation -> { var updates = (List) invocation.getArgument(2); - var responseOperations = new ArrayList(); + // A checkpoint returns the latest state of each operation, not one entry per update. + var responseOperations = new LinkedHashMap(); if (updates != null) { for (var update : updates) { @@ -52,14 +53,14 @@ public static DurableExecutionClient createMockClient() { opBuilder.contextDetails(contexDetail.build()); } } - responseOperations.add(opBuilder.build()); + responseOperations.put(update.id(), opBuilder.build()); } } return CheckpointDurableExecutionResponse.builder() .checkpointToken("new-token") .newExecutionState(CheckpointUpdatedExecutionState.builder() - .operations(responseOperations) + .operations(responseOperations.values()) .build()) .build(); }); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/TestUtilsTest.java b/sdk/src/test/java/software/amazon/lambda/durable/TestUtilsTest.java new file mode 100644 index 000000000..1a2864de9 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/TestUtilsTest.java @@ -0,0 +1,51 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.OperationAction; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.awssdk.services.lambda.model.OperationUpdate; +import software.amazon.lambda.durable.plugin.PluginInfoConverter; + +class TestUtilsTest { + @Test + void batchedUpdatesReturnLatestStatePerOperation() { + var client = TestUtils.createMockClient(); + var response = client.checkpoint( + "arn:execution", + "token", + List.of( + update("first", OperationAction.START, null), + update("second", OperationAction.START, null), + update("first", OperationAction.SUCCEED, "\"result\""))); + var operations = response.newExecutionState().operations(); + + // Duplicate IDs make the real SDK callback reject a successful checkpoint. + var change = assertDoesNotThrow(() -> PluginInfoConverter.toOperationChangeInfo( + "request", "arn:execution", operations, operations, Set.of())); + assertEquals(2, operations.size()); + assertEquals(Set.of("first", "second"), change.updatedOperations().keySet()); + assertEquals( + OperationStatus.SUCCEEDED, + change.updatedOperations().get("first").status()); + assertEquals("\"result\"", change.updatedOperations().get("first").result()); + assertEquals( + OperationStatus.STARTED, + change.updatedOperations().get("second").status()); + } + + private static OperationUpdate update(String id, OperationAction action, String payload) { + return OperationUpdate.builder() + .id(id) + .type(OperationType.STEP) + .action(action) + .payload(payload) + .build(); + } +} From 277c23fdf0c55d4f10b6fba245a1694ee1c4f919 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 03:24:52 +0000 Subject: [PATCH 13/40] test: show failure details in executor reuse assertions --- .../amazon/lambda/durable/execution/DurableExecutionTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/DurableExecutionTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/DurableExecutionTest.java index e0773aaeb..0297b1546 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/execution/DurableExecutionTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/DurableExecutionTest.java @@ -411,7 +411,7 @@ void testExecutorNotShutdownAfterMultipleHandlerInvocations() { (userInput, ctx) -> ctx.step("test1", String.class, stepCtx -> "Result 1: " + userInput), config); - assertEquals(ExecutionStatus.SUCCEEDED, output1.status()); + assertEquals(ExecutionStatus.SUCCEEDED, output1.status(), String.valueOf(output1.error())); assertFalse(sharedExecutor.isShutdown(), "Executor should not be shutdown after first execution"); // Create second input with different execution operation @@ -440,7 +440,7 @@ void testExecutorNotShutdownAfterMultipleHandlerInvocations() { (userInput, ctx) -> ctx.step("test2", String.class, stepCtx -> "Result 2: " + userInput), config); - assertEquals(ExecutionStatus.SUCCEEDED, output2.status()); + assertEquals(ExecutionStatus.SUCCEEDED, output2.status(), String.valueOf(output2.error())); assertFalse(sharedExecutor.isShutdown(), "Executor should not be shutdown after second execution"); // Verify both executions completed successfully and used the same executor From 6401d6966cacce76f4366d6ebe21ce4ae4e5e260 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:24:17 -0700 Subject: [PATCH 14/40] docs: align trace hierarchy with exported execution roots --- otel-plugin/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/otel-plugin/README.md b/otel-plugin/README.md index 999c6037e..5e6bf798b 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -199,14 +199,14 @@ Remote backend server span (Root / Parent) When no valid remote parent can be constructed, a synthetic execution root anchors the trace instead and both spans parent onto it: ``` -Synthetic execution root +DurableExecutionRoot (materialized and re-exported each invocation) ├── Workflow ├── Invocation 1 ├── Invocation 2 └── Invocation N ``` -- **Execution ancestor** — the common parent both the Workflow and Invocation spans resolve onto. A valid remote server span (`Root` and `Parent`) is used directly, whether or not `Sampled` is present; only when a valid remote parent cannot be constructed does a synthetic execution root take its place. It is a non-recording context, not an exported span. +- **Execution ancestor** — the common parent both the Workflow and Invocation spans resolve onto. A valid remote server span (`Root` and `Parent`) is used directly, whether or not `Sampled` is present; only when a valid remote parent cannot be constructed does a synthetic execution root take its place. The remote ancestor is used as a non-recording context; the synthetic ancestor is materialized as `DurableExecutionRoot` and re-exported on each invocation with its stable span ID and execution start time, subject to sampling. - **Workflow span** — one logical span per durable execution, joining the execution trace with a stable span ID derived from the ARN. Exported only on the terminal invocation (SUCCEEDED/FAILED). - **Invocation span** — one per Lambda invocation, parented to the ambient span only when it is on the execution trace, otherwise to the execution ancestor - **Operation span** — one per durable operation, named after your step/wait names @@ -216,7 +216,7 @@ Operation and attempt spans link to the Workflow span. `ExecutionOtelPlugin` rev ### Sampling -The plugin decides sampling once per invocation and applies that single decision to every durable span (Workflow, Invocation, operation, attempt), so the configured sampler is not re-invoked per span and the full decision — including `RECORD_ONLY` — is preserved. The decision follows this precedence, highest first: +The plugin decides sampling once per invocation and applies that single decision to every durable span (DurableExecutionRoot, Workflow, Invocation, operation, attempt), so the configured sampler is not re-invoked per span and the full decision — including `RECORD_ONLY` — is preserved. The decision follows this precedence, highest first: 1. **Backend decision** — `Sampled=1` / `Sampled=0` in the propagated header is authoritative and always preserved, regardless of the configured sampler. 2. **Same-trace ambient span** — when the header carries no usable `Sampled` value but a valid ambient span (for example an auto-instrumentation Lambda handler span) is already on the execution's trace, the plugin follows that span's decision: sampled → sampled; unsampled but still recording → `RECORD_ONLY`; unsampled and not recording → dropped. From cca1b684263c10269cd0848df68740e2835f3ba8 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 05:08:38 +0000 Subject: [PATCH 15/40] fix: retain READY wait-for-condition progress across checkpoints --- ...tForConditionReadinessIntegrationTest.java | 445 ++++++++++++++++++ .../operation/WaitForConditionOperation.java | 125 +++-- 2 files changed, 525 insertions(+), 45 deletions(-) create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionReadinessIntegrationTest.java diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionReadinessIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionReadinessIntegrationTest.java new file mode 100644 index 000000000..111c3c984 --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionReadinessIntegrationTest.java @@ -0,0 +1,445 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.management.ManagementFactory; +import java.lang.reflect.Method; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Deque; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.concurrent.locks.LockSupport; +import java.util.function.IntFunction; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.MDC; +import org.slf4j.spi.MDCAdapter; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.config.WaitForConditionConfig; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.DurableExecutionOutput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.model.WaitForConditionResult; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +class WaitForConditionReadinessIntegrationTest { + @ParameterizedTest + @ValueSource(ints = {2, 25}) + void readyInRetryResponseContinuesWithoutSuspensionAndReplaysStoredResult(int threshold) throws Exception { + try (var run = new Run( + new ImmediateReadyClient(), + Executors.newCachedThreadPool(), + value -> value >= threshold + ? WaitForConditionResult.stopPolling(value) + : WaitForConditionResult.continuePolling(value + 1))) { + var first = run.start().get(5, TimeUnit.SECONDS); + assertEquals(ExecutionStatus.SUCCEEDED, first.status(), "READY work must not produce an invalid PENDING"); + assertEquals("\"" + threshold + "\"", first.result()); + assertEquals(threshold, run.checks.get()); + var updates = run.client.getOperationUpdates().size(); + var replay = run.start().get(5, TimeUnit.SECONDS); + assertEquals(first, replay); + assertEquals(threshold, run.checks.get(), "Completed checks must not be repeated on replay"); + assertEquals(updates, run.client.getOperationUpdates().size()); + } + } + + @Test + void immediateReadyFailureRemainsCheckpointedOnReplay() throws Exception { + try (var run = new Run(new ImmediateReadyClient(), Executors.newCachedThreadPool(), value -> { + if (value == 2) throw new IllegalStateException("predicate failure"); + return WaitForConditionResult.continuePolling(value + 1); + })) { + var first = run.start().get(5, TimeUnit.SECONDS); + assertEquals(ExecutionStatus.FAILED, first.status()); + assertEquals("predicate failure", first.error().errorMessage()); + var updates = run.client.getOperationUpdates().size(); + var replay = run.start().get(5, TimeUnit.SECONDS); + assertEquals(ExecutionStatus.FAILED, replay.status()); + assertEquals(first.error().errorType(), replay.error().errorType()); + assertEquals(first.error().errorMessage(), replay.error().errorMessage()); + assertEquals(2, run.checks.get()); + assertEquals(updates, run.client.getOperationUpdates().size()); + } + } + + @Test + void pendingRetryStillSuspendsAndResumesWhenBackendBecomesReady() throws Exception { + try (var run = new Run( + new LocalMemoryExecutionClient(), + Executors.newCachedThreadPool(), + value -> value == 2 + ? WaitForConditionResult.stopPolling(value) + : WaitForConditionResult.continuePolling(value + 1))) { + var first = run.start().get(5, TimeUnit.SECONDS); + assertEquals(ExecutionStatus.PENDING, first.status()); + assertEquals( + OperationStatus.PENDING, + run.client.getOperationByName("condition").status()); + assertEquals(1, run.checks.get()); + assertTrue(run.client.advanceTime()); + var resumed = run.start().get(5, TimeUnit.SECONDS); + assertEquals(ExecutionStatus.SUCCEEDED, resumed.status()); + assertEquals("\"2\"", resumed.result()); + assertEquals(2, run.checks.get()); + assertEquals(resumed, run.start().get(5, TimeUnit.SECONDS)); + assertEquals(2, run.checks.get()); + } + } + + @ParameterizedTest + @CsvSource({"true,false", "false,true", "true,true"}) + void asynchronousReadyHandoffDoesNotDeadlockOrLoseItsActivityLease( + boolean holdWorkerExit, boolean holdExecutorReturn) throws Exception { + var gate = new HandoffGate(holdWorkerExit, holdExecutorReturn); + var client = new DelayedReadyClient(gate); + try (var mdc = new MdcClearGate(gate); + var run = new Run(client, gate, value -> { + if (value == 1) { + gate.firstWorker.set(Thread.currentThread()); + return WaitForConditionResult.continuePolling(2); + } + gate.nextCheckEntered.countDown(); + await(gate.releaseNextCheck); + return WaitForConditionResult.stopPolling(value); + })) { + try { + var result = run.start(); + await(gate.pollEntered); + if (holdWorkerExit) await(gate.workerExitEntered); + else await(gate.executorReturnEntered); + assertTrue(client.advanceTime()); + gate.releaseReadyResponse.countDown(); + await(gate.readyResponseReturned); + awaitCheckpointHandoff(gate.pollThread.get()); + assertThrows(TimeoutException.class, () -> result.get(100, TimeUnit.MILLISECONDS)); + assertEquals(1, run.checks.get()); + gate.releaseWorkerExit.countDown(); + if (holdExecutorReturn) await(gate.executorReturnEntered); + gate.releaseExecutorReturn.countDown(); + await(gate.nextCheckEntered); + assertThrows( + TimeoutException.class, + () -> result.get(100, TimeUnit.MILLISECONDS), + "The next check must stay active after the old worker deregisters and the checkpoint returns"); + gate.releaseNextCheck.countDown(); + assertEquals( + ExecutionStatus.SUCCEEDED, + result.get(5, TimeUnit.SECONDS).status()); + assertEquals(2, run.checks.get()); + assertEquals(4, run.stateReads.get(), "In-process continuation must reuse normalized state"); + assertEquals( + ExecutionStatus.SUCCEEDED, + run.start().get(5, TimeUnit.SECONDS).status()); + assertEquals(2, run.checks.get()); + assertEquals(5, run.stateReads.get(), "Completed replay only decodes the stored result"); + } finally { + gate.releaseAll(); + } + } + } + + private static void awaitCheckpointHandoff(Thread checkpoint) { + var deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (System.nanoTime() < deadline) { + var stack = checkpoint.getStackTrace(); + if (checkpoint.getState() == Thread.State.WAITING + && Arrays.stream(stack) + .anyMatch(frame -> frame.getClassName().endsWith("WaitForConditionOperation")) + && Arrays.stream(stack) + .anyMatch(frame -> frame.getClassName().equals(CompletableFuture.class.getName()) + && frame.getMethodName().equals("join"))) { + var bean = ManagementFactory.getThreadMXBean(); + if (bean.isObjectMonitorUsageSupported()) { + var info = bean.getThreadInfo(new long[] {checkpoint.getId()}, true, true)[0]; + System.out.println( + "READY_HANDOFF checkpoint monitors=" + Arrays.toString(info.getLockedMonitors())); + } + return; + } + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)); + } + fail("The READY callback did not reach the controlled old-worker handoff window"); + } + + private static void await(CountDownLatch latch) { + try { + assertTrue(latch.await(5, TimeUnit.SECONDS), "Controlled readiness gate was not released"); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new AssertionError(error); + } + } + + private static final class ImmediateReadyClient extends LocalMemoryExecutionClient { + @Override + public CheckpointDurableExecutionResponse checkpoint(String arn, String token, List updates) { + var applied = super.checkpoint(arn, token, updates); + // Advance the test backend's virtual time before materializing the response. The configured + // one-second delay is valid under Lambda's minimum; no zero-delay service request is assumed. + advanceTime(); + var ready = super.checkpoint(arn, applied.checkpointToken(), List.of()); + var states = new LinkedHashMap(); + applied.newExecutionState().operations().forEach(op -> states.put(op.id(), op)); + ready.newExecutionState().operations().forEach(op -> states.put(op.id(), op)); + return ready.toBuilder() + .newExecutionState(CheckpointUpdatedExecutionState.builder() + .operations(states.values()) + .build()) + .build(); + } + } + + private static final class DelayedReadyClient extends LocalMemoryExecutionClient { + private final HandoffGate gate; + private final AtomicBoolean firstPoll = new AtomicBoolean(true); + + private DelayedReadyClient(HandoffGate gate) { + this.gate = gate; + } + + @Override + public CheckpointDurableExecutionResponse checkpoint(String arn, String token, List updates) { + if (updates.isEmpty() && firstPoll.compareAndSet(true, false)) { + gate.pollThread.set(Thread.currentThread()); + gate.pollEntered.countDown(); + await(gate.releaseReadyResponse); + var response = super.checkpoint(arn, token, updates); + gate.readyResponseReturned.countDown(); + return response; + } + return super.checkpoint(arn, token, updates); + } + } + + private static final class HandoffGate extends AbstractExecutorService { + private final ExecutorService delegate = Executors.newCachedThreadPool(); + private final AtomicInteger submitted = new AtomicInteger(); + private final AtomicBoolean exitHeld = new AtomicBoolean(); + private final AtomicReference firstWorker = new AtomicReference<>(); + private final AtomicReference pollThread = new AtomicReference<>(); + private final CountDownLatch pollEntered = new CountDownLatch(1); + private final CountDownLatch releaseReadyResponse = new CountDownLatch(1); + private final CountDownLatch readyResponseReturned = new CountDownLatch(1); + private final CountDownLatch workerExitEntered = new CountDownLatch(1); + private final CountDownLatch releaseWorkerExit = new CountDownLatch(1); + private final CountDownLatch executorReturnEntered = new CountDownLatch(1); + private final CountDownLatch releaseExecutorReturn = new CountDownLatch(1); + private final CountDownLatch nextCheckEntered = new CountDownLatch(1); + private final CountDownLatch releaseNextCheck = new CountDownLatch(1); + private final boolean holdWorkerExit; + private final boolean holdExecutorReturn; + + private HandoffGate(boolean holdWorkerExit, boolean holdExecutorReturn) { + this.holdWorkerExit = holdWorkerExit; + this.holdExecutorReturn = holdExecutorReturn; + } + + @Override + public void execute(Runnable task) { + var number = submitted.incrementAndGet(); + var finished = new CountDownLatch(1); + delegate.execute(() -> { + try { + task.run(); + } finally { + finished.countDown(); + } + }); + // Completing work before execute() returns is permitted by Executor's contract. + if (holdExecutorReturn && number == 2) { + await(finished); + executorReturnEntered.countDown(); + await(releaseExecutorReturn); + } + } + + private void releaseAll() { + releaseReadyResponse.countDown(); + releaseWorkerExit.countDown(); + releaseExecutorReturn.countDown(); + releaseNextCheck.countDown(); + } + + @Override + public void shutdown() { + delegate.shutdown(); + } + + @Override + public List shutdownNow() { + releaseAll(); + return delegate.shutdownNow(); + } + + @Override + public boolean isShutdown() { + return delegate.isShutdown(); + } + + @Override + public boolean isTerminated() { + return delegate.isTerminated(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return delegate.awaitTermination(timeout, unit); + } + } + + private static final class MdcClearGate implements AutoCloseable { + private final MDCAdapter previous = MDC.getMDCAdapter(); + private final Method setter = MDC.class.getDeclaredMethod("setMDCAdapter", MDCAdapter.class); + + private MdcClearGate(HandoffGate gate) throws Exception { + setter.setAccessible(true); + setter.invoke(null, new MDCAdapter() { + public void clear() { + if (gate.holdWorkerExit + && Thread.currentThread() == gate.firstWorker.get() + && gate.exitHeld.compareAndSet(false, true)) { + gate.workerExitEntered.countDown(); + await(gate.releaseWorkerExit); + } + previous.clear(); + } + + public void put(String key, String value) { + previous.put(key, value); + } + + public String get(String key) { + return previous.get(key); + } + + public void remove(String key) { + previous.remove(key); + } + + public Map getCopyOfContextMap() { + return previous.getCopyOfContextMap(); + } + + public void setContextMap(Map values) { + previous.setContextMap(values); + } + + public void pushByKey(String key, String value) { + previous.pushByKey(key, value); + } + + public String popByKey(String key) { + return previous.popByKey(key); + } + + public Deque getCopyOfDequeByKey(String key) { + return previous.getCopyOfDequeByKey(key); + } + + public void clearDequeByKey(String key) { + previous.clearDequeByKey(key); + } + }); + } + + public void close() throws Exception { + setter.invoke(null, previous); + } + } + + private static final class Run implements AutoCloseable { + private final LocalMemoryExecutionClient client; + private final ExecutorService workers; + private final ExecutorService caller = Executors.newSingleThreadExecutor(); + private final AtomicInteger checks = new AtomicInteger(); + private final AtomicInteger stateReads = new AtomicInteger(); + private final SerDes stateSerDes = new SerDes() { + private final JacksonSerDes delegate = new JacksonSerDes(); + + public String serialize(Object value) { + return delegate.serialize(value); + } + + public T deserialize(String value, TypeToken type) { + stateReads.incrementAndGet(); + return delegate.deserialize(value, type); + } + }; + private final IntFunction> check; + private final DurableConfig config; + private final WaitForConditionConfig wait = WaitForConditionConfig.builder() + .initialState(1) + .serDes(stateSerDes) + .waitStrategy((value, attempt) -> Duration.ofSeconds(1)) + .build(); + + private Run( + LocalMemoryExecutionClient client, + ExecutorService workers, + IntFunction> check) { + this.client = client; + this.workers = workers; + this.check = check; + config = DurableConfig.builder() + .withExecutorService(workers) + .withDurableExecutionClient(client) + .withCheckpointDelay(Duration.ZERO) + .withPollingStrategy(attempt -> Duration.ZERO) + .build(); + } + + private Future start() { + var execution = Operation.builder() + .id("execution") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + var history = new ArrayList<>(client.getAllOperations()); + history.add(0, execution); + var input = new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/ready/execution", + "token", + CheckpointUpdatedExecutionState.builder() + .operations(history) + .build()); + return caller.submit(() -> DurableExecutor.execute( + input, + null, + TypeToken.get(String.class), + (value, ctx) -> String.valueOf(ctx.waitForCondition( + "condition", + Integer.class, + (state, step) -> { + checks.incrementAndGet(); + return check.apply(state); + }, + wait)), + config)); + } + + @Override + public void close() throws Exception { + caller.shutdownNow(); + workers.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java index 72eb65b8e..42b511a9f 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java @@ -4,6 +4,8 @@ import java.time.Duration; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationAction; @@ -108,68 +110,101 @@ private void resumeCheckLoop(Operation existing) { } private CompletableFuture pollReadyAndResumeCheckLoop(Operation existing) { - return pollForOperationUpdates() - .thenCompose(op -> op.status() == OperationStatus.READY - ? CompletableFuture.completedFuture(op) - : pollForOperationUpdates()) - .thenAccept(this::resumeCheckLoop); + return pollUntilReady().thenAccept(op -> { + if (!isOperationCompleted() && op.status() == OperationStatus.READY) resumeCheckLoop(op); + }); + } + + private CompletableFuture pollUntilReady() { + var known = getOperation(); + if (isReadyOrTerminal(known)) return CompletableFuture.completedFuture(known); + var update = pollForOperationUpdates(); + // Register before re-reading: another checkpoint may already have delivered READY before this poll existed. + known = getOperation(); + if (isReadyOrTerminal(known)) { + update.complete(known); + return CompletableFuture.completedFuture(known); + } + return update.thenCompose( + op -> isReadyOrTerminal(op) ? CompletableFuture.completedFuture(op) : pollUntilReady()); + } + + private static boolean isReadyOrTerminal(Operation operation) { + return operation != null + && (operation.status() == OperationStatus.READY + || operation.status() == OperationStatus.SUCCEEDED + || operation.status() == OperationStatus.FAILED); } private void executeCheckLogic(T currentState, int attempt) { - Runnable userHandler = () -> { + var publishedWorker = new CompletableFuture>(); + runUserHandler(() -> runCheckLoop(currentState, attempt, publishedWorker), ThreadType.STEP); + publishedWorker.complete(getRunningUserHandler()); + } + + private void runCheckLoop(T currentState, int attempt, CompletableFuture> publishedWorker) { + while (!isOperationCompleted()) { var stepContext = getContext().createStepContext(getOperationId(), getName(), attempt); BaseContextImpl.setCurrentContext(stepContext); try (var ignored = DurableLogger.attachContext()) { try { - // Checkpoint START if not already started var existing = getOperation(); if (existing == null || existing.status() != OperationStatus.STARTED) { - var startUpdate = OperationUpdate.builder().action(OperationAction.START); - sendOperationUpdateAsync(startUpdate); + sendOperationUpdateAsync(OperationUpdate.builder().action(OperationAction.START)); } - - // Execute check function inside the plugin hook boundary so a failure is reported - // through onUserFunctionEnd; checkpoint/poll handling stays outside the boundary. - WaitForConditionResult result = - runUserFunction(attempt, () -> checkFunc.apply(currentState, stepContext)); - - // Normalize the value through SerDes so first execution matches replay. + var stateForCheck = currentState; + var result = runUserFunction(attempt, () -> checkFunc.apply(stateForCheck, stepContext)); var serializedState = serializeAndDeserializeResult(result.value()); - T deserializedValue = serializedState.deserialized(); - + var deserializedValue = serializedState.deserialized(); if (result.isDone()) { - // Condition met — checkpoint SUCCEED - var successUpdate = OperationUpdate.builder() + sendOperationUpdate(OperationUpdate.builder() .action(OperationAction.SUCCEED) - .payload(serializedState.serialized()); - sendOperationUpdate(successUpdate); - } else { - // Compute delay from strategy - Duration delay = config.waitStrategy().evaluate(deserializedValue, attempt); - - // Checkpoint RETRY with delay - var retryUpdate = OperationUpdate.builder() - .action(OperationAction.RETRY) - .payload(serializedState.serialized()) - .stepOptions(StepOptions.builder() - .nextAttemptDelaySeconds(Math.toIntExact(delay.toSeconds())) - .build()); - sendOperationUpdate(retryUpdate); - - // Poll for READY, then continue the loop - pollForOperationUpdates() - .thenCompose(op -> op.status() == OperationStatus.READY - ? CompletableFuture.completedFuture(op) - : pollForOperationUpdates()) - .thenRun(() -> executeCheckLogic(deserializedValue, attempt + 1)); + .payload(serializedState.serialized())); + return; } - } catch (Throwable e) { - handleCheckFailure(e); + Duration delay = config.waitStrategy().evaluate(deserializedValue, attempt); + sendOperationUpdate(OperationUpdate.builder() + .action(OperationAction.RETRY) + .payload(serializedState.serialized()) + .stepOptions(StepOptions.builder() + .nextAttemptDelaySeconds(Math.toIntExact(delay.toSeconds())) + .build())); + var inlineReady = + continueInlineOrAfterCurrentWorker(publishedWorker, deserializedValue, attempt + 1); + if (inlineReady == null || inlineReady.status() != OperationStatus.READY) return; + // READY can be present in the RETRY response. Keep this worker active and continue without + // recursively starting an overlapping handler or suspending executable work. + currentState = deserializedValue; + attempt++; + } catch (Throwable failure) { + handleCheckFailure(failure); + return; } } - }; + } + } - runUserHandler(userHandler, ThreadType.STEP); + private Operation continueInlineOrAfterCurrentWorker( + CompletableFuture> publishedWorker, T nextState, int nextAttempt) { + var owner = Thread.currentThread(); + var inline = new AtomicReference(); + var acceptingInline = new AtomicBoolean(true); + try { + pollUntilReady().thenAccept(op -> { + if (Thread.currentThread() == owner && acceptingInline.get()) { + inline.set(op); + } else { + // A checkpoint callback can observe READY before this attempt's worker exits. Its checkpoint + // processing lease stays active through this handoff, preventing a false quiescence window. + publishedWorker.join().join(); + if (!isOperationCompleted() && op.status() == OperationStatus.READY) + executeCheckLogic(nextState, nextAttempt); + } + }); + } finally { + acceptingInline.set(false); + } + return inline.get(); } private void handleCheckFailure(Throwable exception) { From 5c123250c9a9674e1c11cf9db40c7b32d735fed6 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:27:14 -0700 Subject: [PATCH 16/40] fix: stop wait-for-condition polling for all terminal states --- .../operation/WaitForConditionOperation.java | 14 ++- .../WaitForConditionTerminalPollingTest.java | 119 ++++++++++++++++++ 2 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionTerminalPollingTest.java diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java index 42b511a9f..afff3deb8 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java @@ -20,6 +20,7 @@ import software.amazon.lambda.durable.exception.DurableOperationException; import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; import software.amazon.lambda.durable.exception.WaitForConditionFailedException; +import software.amazon.lambda.durable.execution.ExecutionManager; import software.amazon.lambda.durable.execution.SuspendExecutionException; import software.amazon.lambda.durable.execution.ThreadType; import software.amazon.lambda.durable.logging.DurableLogger; @@ -129,11 +130,14 @@ private CompletableFuture pollUntilReady() { op -> isReadyOrTerminal(op) ? CompletableFuture.completedFuture(op) : pollUntilReady()); } - private static boolean isReadyOrTerminal(Operation operation) { - return operation != null - && (operation.status() == OperationStatus.READY - || operation.status() == OperationStatus.SUCCEEDED - || operation.status() == OperationStatus.FAILED); + private boolean isReadyOrTerminal(Operation operation) { + if (operation == null) return false; + var status = operation.status(); + if (status == null || status == OperationStatus.UNKNOWN_TO_SDK_VERSION) { + throw terminateExecutionWithIllegalDurableOperationException( + "Unexpected waitForCondition status: " + operation.statusAsString()); + } + return status == OperationStatus.READY || ExecutionManager.isTerminalStatus(status); } private void executeCheckLogic(T currentState, int attempt) { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionTerminalPollingTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionTerminalPollingTest.java new file mode 100644 index 000000000..ecac9a10b --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionTerminalPollingTest.java @@ -0,0 +1,119 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.operation; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.WaitForConditionConfig; +import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.exception.IllegalDurableOperationException; +import software.amazon.lambda.durable.execution.ExecutionManager; +import software.amazon.lambda.durable.model.OperationIdentifier; +import software.amazon.lambda.durable.model.OperationSubType; +import software.amazon.lambda.durable.model.WaitForConditionResult; + +class WaitForConditionTerminalPollingTest { + @ParameterizedTest + @CsvSource({ + "CANCELLED,false", + "CANCELLED,true", + "TIMED_OUT,false", + "TIMED_OUT,true", + "STOPPED,false", + "STOPPED,true" + }) + void terminalObservationNeverRegistersAnotherPoll(String status, boolean alreadyKnown) throws Exception { + var fixture = new Fixture(alreadyKnown ? status : "PENDING"); + var result = fixture.poll(); + if (!alreadyKnown) fixture.deliver(status); + assertEquals(status, result.get(1, TimeUnit.SECONDS).statusAsString()); + assertEquals(alreadyKnown ? 0 : 1, fixture.polls.size()); + } + + @ParameterizedTest + @ValueSource(strings = {"NEW_UNKNOWN_STATUS"}) + @NullAndEmptySource + void malformedStatusTerminatesInsteadOfPollingForever(String status) throws Exception { + var fixture = new Fixture("PENDING"); + var result = fixture.poll(); + fixture.deliver(status); + var failure = assertThrows(ExecutionException.class, () -> result.get(1, TimeUnit.SECONDS)); + assertInstanceOf(IllegalDurableOperationException.class, failure.getCause()); + verify(fixture.manager).terminateExecution((IllegalDurableOperationException) failure.getCause()); + assertEquals(1, fixture.polls.size()); + } + + @Test + void nonterminalUpdatesStillWaitForReady() throws Exception { + var fixture = new Fixture("PENDING"); + var result = fixture.poll(); + fixture.deliver("STARTED"); + assertFalse(result.isDone()); + fixture.deliver("PENDING"); + assertFalse(result.isDone()); + fixture.deliver("READY"); + assertEquals("READY", result.get(1, TimeUnit.SECONDS).statusAsString()); + assertEquals(3, fixture.polls.size()); + } + + private static final class Fixture { + final ExecutionManager manager = mock(ExecutionManager.class); + final AtomicReference known; + final List> polls = new ArrayList<>(); + final WaitForConditionOperation operation; + + Fixture(String status) { + known = new AtomicReference<>(snapshot(status)); + var context = mock(DurableContextImpl.class); + when(context.getExecutionManager()).thenReturn(manager); + when(context.getDurableConfig()).thenReturn(DurableConfig.builder().build()); + when(manager.getOperationAndUpdateReplayState("condition")).thenAnswer(call -> known.get()); + when(manager.pollForOperationUpdates("condition")).thenAnswer(call -> { + var poll = new CompletableFuture(); + polls.add(poll); + return poll; + }); + operation = new WaitForConditionOperation<>( + OperationIdentifier.of("condition", "condition", OperationSubType.WAIT_FOR_CONDITION), + (state, contextIgnored) -> WaitForConditionResult.stopPolling(state), + TypeToken.get(Integer.class), + WaitForConditionConfig.builder().initialState(1).build(), + context); + } + + @SuppressWarnings("unchecked") + CompletableFuture poll() throws Exception { + var method = WaitForConditionOperation.class.getDeclaredMethod("pollUntilReady"); + method.setAccessible(true); + return (CompletableFuture) method.invoke(operation); + } + + void deliver(String status) { + var update = snapshot(status); + known.set(update); + polls.get(polls.size() - 1).complete(update); + } + + private static Operation snapshot(String status) { + return Operation.builder() + .id("condition") + .type(OperationType.STEP) + .status(status) + .build(); + } + } +} From 6603e2143497b1ce692e34ff44158aa8a928f86f Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 19:43:59 +0000 Subject: [PATCH 17/40] fix: replay terminal wait-for-condition outcomes safely --- ...onditionTerminalReplayIntegrationTest.java | 188 ++++++++++++++++++ .../operation/WaitForConditionOperation.java | 5 +- 2 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionTerminalReplayIntegrationTest.java diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionTerminalReplayIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionTerminalReplayIntegrationTest.java new file mode 100644 index 000000000..544f9106a --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionTerminalReplayIntegrationTest.java @@ -0,0 +1,188 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.services.lambda.model.CheckpointDurableExecutionResponse; +import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; +import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.awssdk.services.lambda.model.ExecutionDetails; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationAction; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.awssdk.services.lambda.model.OperationUpdate; +import software.amazon.awssdk.services.lambda.model.StepDetails; +import software.amazon.lambda.durable.config.WaitForConditionConfig; +import software.amazon.lambda.durable.exception.WaitForConditionFailedException; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.DurableExecutionOutput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.model.WaitForConditionResult; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +class WaitForConditionTerminalReplayIntegrationTest { + private static final String CACHED_ERROR = "cached terminal error"; + + static Stream terminals() { + return Stream.of( + OperationStatus.FAILED, + OperationStatus.CANCELLED, + OperationStatus.TIMED_OUT, + OperationStatus.STOPPED) + .flatMap(status -> Stream.of("absent", "empty", "stored") + .flatMap(details -> Stream.of(false, true).map(live -> Arguments.of(status, details, live)))); + } + + @ParameterizedTest + @MethodSource("terminals") + void terminalSnapshotsReplayWithoutRepeatingChecksOrCheckpoints( + OperationStatus status, String details, boolean deliverWhileRunning) throws Exception { + var terminal = new AtomicReference(); + var client = new LocalMemoryExecutionClient() { + @Override + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List updates) { + var response = super.checkpoint(arn, token, updates); + if (!deliverWhileRunning || updates.stream().noneMatch(u -> u.action() == OperationAction.RETRY)) + return response; + // Exercise an actual checkpoint-completion boundary with the backend's terminal snapshot. + var condition = getOperationByName("condition"); + terminal.set(terminalSnapshot(condition, status, details)); + return response.toBuilder() + .newExecutionState(CheckpointUpdatedExecutionState.builder() + .operations(terminal.get()) + .build()) + .build(); + } + }; + var workers = Executors.newCachedThreadPool(); + var caller = Executors.newSingleThreadExecutor(); + var checks = new AtomicInteger(); + try { + var config = DurableConfig.builder() + .withExecutorService(workers) + .withDurableExecutionClient(client) + .withCheckpointDelay(Duration.ZERO) + .withPollingStrategy(attempt -> Duration.ZERO) + .build(); + var first = caller.submit(() -> execute(List.of(), config, checks, status, details)) + .get(5, TimeUnit.SECONDS); + if (deliverWhileRunning) assertHandled(first, status, details); + else { + assertEquals(ExecutionStatus.PENDING, first.status()); + terminal.set(terminalSnapshot(client.getOperationByName("condition"), status, details)); + } + assertEquals(1, checks.get()); + var updates = client.getOperationUpdates().size(); + var history = List.of(terminal.get()); + var replay = caller.submit(() -> execute(history, config, checks, status, details)) + .get(5, TimeUnit.SECONDS); + assertHandled(replay, status, details); + var repeated = caller.submit(() -> execute(history, config, checks, status, details)) + .get(5, TimeUnit.SECONDS); + assertEquals(replay, repeated); + assertEquals(1, checks.get(), "Neither cancelled polling nor completed replay may re-enter the predicate"); + assertEquals(updates, client.getOperationUpdates().size(), "Replay must not rewrite a terminal checkpoint"); + } finally { + caller.shutdownNow(); + workers.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + private static DurableExecutionOutput execute( + List history, + DurableConfig config, + AtomicInteger checks, + OperationStatus status, + String details) { + var operations = new ArrayList(); + operations.add(Operation.builder() + .id("execution") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build()); + operations.addAll(history); + var input = new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/terminal/execution", + "token", + CheckpointUpdatedExecutionState.builder().operations(operations).build()); + return DurableExecutor.execute( + input, + null, + TypeToken.get(String.class), + (value, context) -> { + try { + context.waitForCondition( + "condition", + Integer.class, + (state, step) -> { + checks.incrementAndGet(); + return WaitForConditionResult.continuePolling(1); + }, + WaitForConditionConfig.builder() + .initialState(0) + .waitStrategy((state, attempt) -> Duration.ofSeconds(1)) + .build()); + return "unexpected success"; + } catch (WaitForConditionFailedException failure) { + assertNotEquals("stored", details); + assertEquals(status, failure.getOperationStatus()); + assertEquals("condition", failure.getOperation().name()); + return status + ":no error details"; + } catch (IllegalStateException failure) { + assertEquals("stored", details); + assertEquals(CACHED_ERROR, failure.getMessage()); + return CACHED_ERROR; + } + }, + config); + } + + private static void assertHandled(DurableExecutionOutput output, OperationStatus status, String details) { + assertEquals( + ExecutionStatus.SUCCEEDED, + output.status(), + () -> output.error() == null + ? "No error payload" + : output.error().errorType() + ": " + output.error().errorMessage()); + assertEquals( + new JacksonSerDes().serialize(details.equals("stored") ? CACHED_ERROR : status + ":no error details"), + output.result()); + } + + private static Operation terminalSnapshot(Operation pending, OperationStatus status, String details) { + StepDetails snapshot = null; + if (details.equals("empty")) snapshot = StepDetails.builder().attempt(1).build(); + else if (details.equals("stored")) + snapshot = StepDetails.builder() + .attempt(1) + .error(ErrorObject.builder() + .errorType(IllegalStateException.class.getName()) + .errorMessage(CACHED_ERROR) + .errorData(new JacksonSerDes().serialize(new IllegalStateException(CACHED_ERROR))) + .build()) + .build(); + return pending.toBuilder().status(status).stepDetails(snapshot).build(); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java index afff3deb8..aaed6dbbe 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java @@ -66,7 +66,7 @@ protected void start() { @Override protected void replay(Operation existing) { switch (existing.status()) { - case SUCCEEDED, FAILED -> markAlreadyCompleted(); // Check if already completed / failed + case SUCCEEDED, FAILED, CANCELLED, TIMED_OUT, STOPPED -> markAlreadyCompleted(); case PENDING -> pollReadyAndResumeCheckLoop(existing); // Check if pending retry case STARTED, READY -> resumeCheckLoop(existing); default -> @@ -84,7 +84,8 @@ public T get() { var result = (stepDetails != null) ? stepDetails.result() : null; return deserializeResult(result); } else { - var errorObject = op.stepDetails().error(); + var stepDetails = op.stepDetails(); + var errorObject = stepDetails != null ? stepDetails.error() : null; // Attempt to reconstruct and throw the original exception Throwable original = deserializeException(errorObject); From c6317030617aaa82333094e4fb49e47f9c86df20 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 22:50:33 +0000 Subject: [PATCH 18/40] ci: pin OTel queue workflow to published squash commit --- .github/workflows/otel-conformance-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/otel-conformance-tests.yml b/.github/workflows/otel-conformance-tests.yml index 452db089d..cfb82bbe7 100644 --- a/.github/workflows/otel-conformance-tests.yml +++ b/.github/workflows/otel-conformance-tests.yml @@ -68,7 +68,7 @@ jobs: actions: write contents: read id-token: write - uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@a66037abbbfa55fde97f714e30f0bc262edefd63 + uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@f18bd0b5f28c5c90e288d0fb8bca08a849b51863 with: runs_on: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'ubuntu-latest' || format('codebuild-github-actions-runner-{0}-{1}', github.run_id, github.run_attempt) }} language: java From 361c25bdd0acc0c1e10edb8cb252fbd1b4865065 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 8 Oct 2026 04:31:08 +0000 Subject: [PATCH 19/40] fix: resume condition polling outside the checkpoint batcher Port of 880c672b694dd6e9859596872447c03fb779d522 after independent public-API deadlock reproduction; preserve this branch workflow, OTel topology and invocation lifecycle. --- ...tForConditionReadinessIntegrationTest.java | 175 ++++++++++++++++-- .../durable/execution/ExecutionManager.java | 38 +++- .../operation/WaitForConditionOperation.java | 22 ++- .../execution/ExecutionManagerTest.java | 113 +++++++++++ .../WaitForConditionOperationTest.java | 2 + 5 files changed, 329 insertions(+), 21 deletions(-) diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionReadinessIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionReadinessIntegrationTest.java index 111c3c984..4bf216f38 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionReadinessIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionReadinessIntegrationTest.java @@ -6,6 +6,8 @@ import java.lang.management.ManagementFactory; import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -19,6 +21,7 @@ import java.util.concurrent.locks.LockSupport; import java.util.function.IntFunction; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; @@ -36,6 +39,146 @@ import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; class WaitForConditionReadinessIntegrationTest { + @TempDir + Path subprocessLogs; + + @ParameterizedTest + @ValueSource(strings = {"async", "direct", "submit-and-wait"}) + void delayedReadyResumesWithSynchronousOperationExecutors(String mode) throws Exception { + var log = subprocessLogs.resolve(mode + ".log"); + var process = new ProcessBuilder( + Path.of(System.getProperty("java.home"), "bin", "java").toString(), + "-cp", + System.getProperty("java.class.path"), + SynchronousProbe.class.getName(), + mode) + .redirectErrorStream(true) + .redirectOutput(log.toFile()) + .start(); + try { + assertTrue(process.waitFor(15, TimeUnit.SECONDS), "The isolated READY probe exceeded its cleanup bound"); + var output = Files.readString(log); + System.out.println("SYNC_READY_PROBE mode=" + mode + " exit=" + process.exitValue() + "\n" + output); + assertEquals(0, process.exitValue(), output); + assertTrue(output.contains("SYNC_READY_SUCCESS checks=2 replayChecks=2"), output); + } finally { + if (process.isAlive()) process.destroyForcibly(); + assertTrue(process.waitFor(5, TimeUnit.SECONDS)); + } + } + + /** A separate process bounds cleanup of an old-code deadlock without altering the observed protocol. */ + public static final class SynchronousProbe { + public static void main(String[] args) throws Exception { + var gate = new HandoffGate(true, false); + var workers = new SynchronousContinuationExecutor(args[0]); + var client = new DelayedReadyClient(gate); + var run = new Run(client, workers, value -> { + if (value == 1) gate.firstWorker.set(Thread.currentThread()); + return value == 2 + ? WaitForConditionResult.stopPolling(value) + : WaitForConditionResult.continuePolling(2); + }); + try (var mdc = new MdcClearGate(gate)) { + var first = run.start(); + await(gate.pollEntered); + await(gate.workerExitEntered); + assertTrue(client.advanceTime()); + gate.releaseReadyResponse.countDown(); + await(gate.readyResponseReturned); + gate.releaseWorkerExit.countDown(); + assertEquals( + ExecutionStatus.SUCCEEDED, + first.get(3, TimeUnit.SECONDS).status()); + assertEquals(2, run.checks.get()); + assertEquals( + ExecutionStatus.SUCCEEDED, + run.start().get(3, TimeUnit.SECONDS).status()); + assertEquals(2, run.checks.get()); + assertTrue(workers.continuationDispatches.get() > 0); + System.out.println("SYNC_READY_SUCCESS checks=2 replayChecks=2 mode=" + args[0]); + run.close(); + gate.shutdownNow(); + } catch (Throwable failure) { + System.out.println("SYNC_READY_BLOCKED mode=" + args[0] + " failure=" + + failure.getClass().getName()); + failure.printStackTrace(System.out); + var bean = ManagementFactory.getThreadMXBean(); + Thread.getAllStackTraces().forEach((thread, stack) -> { + if (!thread.getName().startsWith("durable-sdk-internal") + && !thread.getName().startsWith("sync-ready-worker")) return; + var info = bean.getThreadInfo(new long[] {thread.getId()}, true, true)[0]; + System.out.println("THREAD " + thread.getName() + " " + thread.getState() + " monitors=" + + (info == null ? "[]" : Arrays.toString(info.getLockedMonitors()))); + Arrays.stream(stack).forEach(frame -> System.out.println(" " + frame)); + }); + // The probe owns no cloud resources; exiting the child JVM releases only this fixture's blocked + // threads. + System.exit(2); + } + } + } + + private static final class SynchronousContinuationExecutor extends AbstractExecutorService { + private final String mode; + private final AtomicInteger submissions = new AtomicInteger(); + private final AtomicInteger continuationDispatches = new AtomicInteger(); + private final ExecutorService delegate = Executors.newCachedThreadPool(task -> { + var thread = new Thread(task, "sync-ready-worker"); + thread.setDaemon(true); + return thread; + }); + + private SynchronousContinuationExecutor(String mode) { + this.mode = mode; + } + + @Override + public void execute(Runnable task) { + if (submissions.incrementAndGet() <= 2) { + delegate.execute(task); // Root handler and first check retain their normal asynchronous setup. + return; + } + continuationDispatches.incrementAndGet(); + if (mode.equals("direct")) task.run(); + else if (mode.equals("submit-and-wait")) { + try { + delegate.submit(task).get(); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new AssertionError(failure); + } catch (ExecutionException failure) { + throw new AssertionError(failure.getCause()); + } + } else delegate.execute(task); + } + + @Override + public void shutdown() { + delegate.shutdown(); + } + + @Override + public List shutdownNow() { + return delegate.shutdownNow(); + } + + @Override + public boolean isShutdown() { + return delegate.isShutdown(); + } + + @Override + public boolean isTerminated() { + return delegate.isTerminated(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return delegate.awaitTermination(timeout, unit); + } + } + @ParameterizedTest @ValueSource(ints = {2, 25}) void readyInRetryResponseContinuesWithoutSuspensionAndReplaysStoredResult(int threshold) throws Exception { @@ -155,24 +298,34 @@ void asynchronousReadyHandoffDoesNotDeadlockOrLoseItsActivityLease( private static void awaitCheckpointHandoff(Thread checkpoint) { var deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); while (System.nanoTime() < deadline) { - var stack = checkpoint.getStackTrace(); - if (checkpoint.getState() == Thread.State.WAITING - && Arrays.stream(stack) - .anyMatch(frame -> frame.getClassName().endsWith("WaitForConditionOperation")) - && Arrays.stream(stack) - .anyMatch(frame -> frame.getClassName().equals(CompletableFuture.class.getName()) - && frame.getMethodName().equals("join"))) { + for (var entry : Thread.getAllStackTraces().entrySet()) { + var continuation = entry.getKey(); + var stack = entry.getValue(); + if (continuation.getState() != Thread.State.WAITING + || !continuation.getName().startsWith("durable-sdk-internal-") + || Arrays.stream(stack) + .noneMatch(frame -> frame.getClassName().endsWith("WaitForConditionOperation")) + || Arrays.stream(stack) + .noneMatch(frame -> frame.getClassName().equals(CompletableFuture.class.getName()) + && frame.getMethodName().equals("join"))) continue; + assertNotSame(checkpoint, continuation, "Worker publication must not block the checkpoint callback"); + assertTrue(Arrays.stream(stack) + .noneMatch(frame -> frame.getClassName().endsWith("ApiRequestDelayedBatcher"))); + if (Arrays.stream(checkpoint.getStackTrace()) + .anyMatch(frame -> frame.getClassName().endsWith("CheckpointManager") + && frame.getMethodName().equals("checkpointBatch"))) continue; var bean = ManagementFactory.getThreadMXBean(); if (bean.isObjectMonitorUsageSupported()) { - var info = bean.getThreadInfo(new long[] {checkpoint.getId()}, true, true)[0]; - System.out.println( - "READY_HANDOFF checkpoint monitors=" + Arrays.toString(info.getLockedMonitors())); + var info = bean.getThreadInfo(new long[] {continuation.getId()}, true, true)[0]; + assertEquals(0, info.getLockedMonitors().length); + System.out.println("READY_HANDOFF continuation=" + continuation.getName() + " checkpoint=" + + checkpoint.getName() + " continuationMonitors=[] checkpointBatchReturned=true"); } return; } LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)); } - fail("The READY callback did not reach the controlled old-worker handoff window"); + fail("The independent READY continuation did not reach the controlled old-worker handoff window"); } private static void await(CountDownLatch latch) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java index 4c7feeda4..9a61c0224 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java @@ -15,6 +15,7 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -75,7 +76,7 @@ public class ExecutionManager implements SafeCloseable { private final Set activeThreads = Collections.synchronizedSet(new HashSet<>()); private static final ThreadLocal currentThreadContext = new ThreadLocal<>(); private final CompletableFuture executionExceptionFuture = new CompletableFuture<>(); - // Guarded by activeThreads so starting a checkpoint request is atomic with the last-thread suspension decision. + // Requests and polling continuations; guarded by activeThreads so admission is atomic with last-thread suspension. private int checkpointRequestsInFlight; /** @@ -544,6 +545,41 @@ void finishCheckpointProcessing() { } } + /** + * Runs a polling continuation away from the serialized checkpoint batcher. Keeps checkpoint processing active from + * receiving the callback until the continuation has registered its next worker (or finished inline). Canceling the + * returned future affects observation only; queued or running work still owns its activity lease. + * + * @param continuation operation work to dispatch after receiving a polling update + * @return completion of the continuation, or an already-completed future when execution has stopped + */ + public CompletableFuture runCheckpointContinuation(Runnable continuation) { + return runCheckpointContinuation(continuation, InternalExecutor.INSTANCE); + } + + CompletableFuture runCheckpointContinuation(Runnable continuation, Executor coordinator) { + if (!tryStartCheckpointProcessing()) return CompletableFuture.completedFuture(null); + try { + var completion = new CompletableFuture(); + coordinator.execute(() -> { + try { + try { + continuation.run(); + } finally { + finishCheckpointProcessing(); + } + completion.complete(null); + } catch (Throwable failure) { + completion.completeExceptionally(failure); + } + }); + return completion; + } catch (RuntimeException | Error failure) { + finishCheckpointProcessing(); + throw failure; + } + } + private boolean shouldSuspendExecution() { return activeThreads.isEmpty() && checkpointRequestsInFlight == 0 && !executionExceptionFuture.isDone(); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java index aaed6dbbe..1dbf5ba5b 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java @@ -112,9 +112,10 @@ private void resumeCheckLoop(Operation existing) { } private CompletableFuture pollReadyAndResumeCheckLoop(Operation existing) { - return pollUntilReady().thenAccept(op -> { - if (!isOperationCompleted() && op.status() == OperationStatus.READY) resumeCheckLoop(op); - }); + return pollUntilReady() + .thenCompose(op -> executionManager.runCheckpointContinuation(() -> { + if (!isOperationCompleted() && op.status() == OperationStatus.READY) resumeCheckLoop(op); + })); } private CompletableFuture pollUntilReady() { @@ -195,15 +196,18 @@ private Operation continueInlineOrAfterCurrentWorker( var inline = new AtomicReference(); var acceptingInline = new AtomicBoolean(true); try { - pollUntilReady().thenAccept(op -> { + pollUntilReady().thenCompose(op -> { if (Thread.currentThread() == owner && acceptingInline.get()) { inline.set(op); + return CompletableFuture.completedFuture(null); } else { - // A checkpoint callback can observe READY before this attempt's worker exits. Its checkpoint - // processing lease stays active through this handoff, preventing a false quiescence window. - publishedWorker.join().join(); - if (!isOperationCompleted() && op.status() == OperationStatus.READY) - executeCheckLogic(nextState, nextAttempt); + // A configured executor may run or wait for the next check inline. Leave the serialized + // checkpoint callback before dispatch, retaining activity through worker publication/registration. + return executionManager.runCheckpointContinuation(() -> { + publishedWorker.join().join(); + if (!isOperationCompleted() && op.status() == OperationStatus.READY) + executeCheckLogic(nextState, nextAttempt); + }); } }); } finally { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java index 1552d2b21..dfb67639e 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java @@ -14,10 +14,15 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; import software.amazon.awssdk.services.lambda.model.GetDurableExecutionStateResponse; import software.amazon.awssdk.services.lambda.model.Operation; @@ -372,6 +377,114 @@ void deferredSuspensionOccursWhenCheckpointFinishesWithoutReactivatingThread() { assertTrue(manager.isExecutionCompletedExceptionally()); } + @ParameterizedTest + @ValueSource(strings = {"complete", "register-worker", "fail"}) + void pollingContinuationRetainsActivityUntilItFinishesDispatch(String outcome) throws Exception { + var manager = createManager(List.of(executionOp(), stepOp("step", OperationStatus.PENDING))); + var entered = new CountDownLatch(1); + var release = new CountDownLatch(1); + var caller = Thread.currentThread(); + var failure = new IllegalStateException("continuation failure"); + manager.registerActiveThread("root"); + var completion = manager.runCheckpointContinuation(() -> { + assertNotSame(caller, Thread.currentThread()); + entered.countDown(); + try { + assertTrue(release.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new AssertionError(interrupted); + } + if (outcome.equals("register-worker")) manager.registerActiveThread("next"); + if (outcome.equals("fail")) throw failure; + }); + try { + assertTrue(entered.await(3, TimeUnit.SECONDS)); + manager.deregisterActiveThread("root"); + assertFalse(manager.isExecutionCompletedExceptionally()); + assertFalse(completion.isDone()); + } finally { + release.countDown(); + } + if (outcome.equals("fail")) { + assertSame( + failure, + assertThrows(ExecutionException.class, () -> completion.get(3, TimeUnit.SECONDS)) + .getCause()); + } else completion.get(3, TimeUnit.SECONDS); + if (outcome.equals("register-worker")) { + assertFalse(manager.isExecutionCompletedExceptionally()); + assertThrows(SuspendExecutionException.class, () -> manager.deregisterActiveThread("next")); + } + assertTrue(manager.isExecutionCompletedExceptionally()); + manager.runCheckpointContinuation(() -> fail("No continuation may start after suspension")) + .get(3, TimeUnit.SECONDS); + } + + @Test + void cancellingQueuedContinuationDoesNotSkipItsWorkOrLeakActivity() { + var manager = createManager(List.of(executionOp(), stepOp("step", OperationStatus.PENDING))); + var queued = new AtomicReference(); + var calls = new AtomicInteger(); + manager.registerActiveThread("root"); + var completion = manager.runCheckpointContinuation(calls::incrementAndGet, queued::set); + assertTrue(completion.cancel(false)); + manager.deregisterActiveThread("root"); + assertFalse(manager.isExecutionCompletedExceptionally(), "Queued work still owns the activity lease"); + queued.get().run(); + assertAll( + () -> assertEquals(1, calls.get()), + () -> assertTrue(manager.isExecutionCompletedExceptionally(), "The actual runnable releases its lease"), + () -> assertTrue(completion.isCancelled())); + } + + @Test + void cancellingRunningContinuationDoesNotReleaseActivityBeforeItsCleanup() throws Exception { + var manager = createManager(List.of(executionOp(), stepOp("step", OperationStatus.PENDING))); + var queued = new AtomicReference(); + var entered = new CountDownLatch(1); + var release = new CountDownLatch(1); + manager.registerActiveThread("root"); + var completion = manager.runCheckpointContinuation( + () -> { + entered.countDown(); + try { + assertTrue(release.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new AssertionError(interrupted); + } + }, + queued::set); + var actualWorker = CompletableFuture.runAsync(queued.get()); + try { + assertTrue(entered.await(3, TimeUnit.SECONDS)); + assertTrue(completion.cancel(false)); + manager.deregisterActiveThread("root"); + assertFalse(manager.isExecutionCompletedExceptionally(), "Cancellation must not release running work"); + } finally { + release.countDown(); + } + actualWorker.get(3, TimeUnit.SECONDS); + assertTrue(manager.isExecutionCompletedExceptionally()); + assertTrue(completion.isCancelled()); + } + + @Test + void rejectedContinuationReleasesActivityAndPreservesTheRejection() { + var manager = createManager(List.of(executionOp(), stepOp("step", OperationStatus.PENDING))); + var rejection = new RejectedExecutionException("coordinator unavailable"); + manager.registerActiveThread("root"); + assertSame( + rejection, + assertThrows( + RejectedExecutionException.class, + () -> manager.runCheckpointContinuation(() -> fail("Rejected work must not run"), task -> { + throw rejection; + }))); + assertThrows(SuspendExecutionException.class, () -> manager.deregisterActiveThread("root")); + } + @Test void checkpointDeliveryIsAtomicWithOperationRegistration() throws Exception { var manager = createManager(List.of(executionOp(), stepOp("step", OperationStatus.PENDING))); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java index 6c923994f..c4df8aac5 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java @@ -284,6 +284,8 @@ void getFailedWithNullErrorDataThrowsStepFailedException() { @Test void replayPendingPollsAndResumesCheckLoop() throws Exception { + when(executionManager.runCheckpointContinuation(any())) + .thenAnswer(call -> CompletableFuture.runAsync(call.getArgument(0))); var pendingOp = Operation.builder() .id(OPERATION_ID) .name(OPERATION_NAME) From 1e27f85a6c4497b8f3fe731e489184a5a9f1e3fa Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 8 Oct 2026 04:59:29 +0000 Subject: [PATCH 20/40] fix: yield READY checks to queued user operations --- ...tForConditionReadinessIntegrationTest.java | 96 ++++++++++++++++--- .../operation/WaitForConditionOperation.java | 44 +++------ 2 files changed, 96 insertions(+), 44 deletions(-) diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionReadinessIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionReadinessIntegrationTest.java index 4bf216f38..9ed8cd295 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionReadinessIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/WaitForConditionReadinessIntegrationTest.java @@ -19,6 +19,7 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.*; import java.util.concurrent.locks.LockSupport; +import java.util.function.BiFunction; import java.util.function.IntFunction; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -45,12 +46,22 @@ class WaitForConditionReadinessIntegrationTest { @ParameterizedTest @ValueSource(strings = {"async", "direct", "submit-and-wait"}) void delayedReadyResumesWithSynchronousOperationExecutors(String mode) throws Exception { - var log = subprocessLogs.resolve(mode + ".log"); + assertProbe(SynchronousProbe.class, mode, "SYNC_READY_SUCCESS checks=2 replayChecks=2"); + } + + @ParameterizedTest + @ValueSource(strings = {"cached", "bounded"}) + void immediateReadyYieldsToTheSiblingThatUnlocksItsCondition(String mode) throws Exception { + assertProbe(FairnessProbe.class, mode, "FAIR_READY_SUCCESS siblingCalls=1 replayStable=true"); + } + + private void assertProbe(Class probe, String mode, String successMarker) throws Exception { + var log = subprocessLogs.resolve(probe.getSimpleName() + "-" + mode + ".log"); var process = new ProcessBuilder( Path.of(System.getProperty("java.home"), "bin", "java").toString(), "-cp", System.getProperty("java.class.path"), - SynchronousProbe.class.getName(), + probe.getName(), mode) .redirectErrorStream(true) .redirectOutput(log.toFile()) @@ -60,13 +71,73 @@ void delayedReadyResumesWithSynchronousOperationExecutors(String mode) throws Ex var output = Files.readString(log); System.out.println("SYNC_READY_PROBE mode=" + mode + " exit=" + process.exitValue() + "\n" + output); assertEquals(0, process.exitValue(), output); - assertTrue(output.contains("SYNC_READY_SUCCESS checks=2 replayChecks=2"), output); + assertTrue(output.contains(successMarker), output); } finally { if (process.isAlive()) process.destroyForcibly(); assertTrue(process.waitFor(5, TimeUnit.SECONDS)); } } + public static final class FairnessProbe { + public static void main(String[] args) throws Exception { + var workers = args[0].equals("bounded") ? Executors.newFixedThreadPool(2) : Executors.newCachedThreadPool(); + var siblingQueued = new CountDownLatch(1); + var unlocked = new AtomicBoolean(); + var checks = new AtomicInteger(); + var siblings = new AtomicInteger(); + var run = new Run(new ImmediateReadyClient(), workers, value -> { + throw new AssertionError("The explicit public handler supplies its own condition"); + }); + run.customHandler = (input, context) -> { + var condition = context.waitForConditionAsync( + "condition", + Integer.class, + (state, step) -> { + checks.incrementAndGet(); + await(siblingQueued); + return unlocked.get() + ? WaitForConditionResult.stopPolling(state) + : WaitForConditionResult.continuePolling(state + 1); + }, + run.wait); + var sibling = context.stepAsync("unlock-condition", String.class, step -> { + siblings.incrementAndGet(); + unlocked.set(true); + return "unlocked"; + }); + siblingQueued.countDown(); + var result = condition.get(); + assertEquals("unlocked", sibling.get()); + return String.valueOf(result); + }; + try { + var first = run.start().get(3, TimeUnit.SECONDS); + assertEquals(ExecutionStatus.SUCCEEDED, first.status()); + assertEquals(1, siblings.get()); + var completedChecks = checks.get(); + var checkpoints = run.client.getOperationUpdates().size(); + assertEquals(first, run.start().get(3, TimeUnit.SECONDS)); + assertEquals(completedChecks, checks.get()); + assertEquals(1, siblings.get()); + assertEquals(checkpoints, run.client.getOperationUpdates().size()); + System.out.println("FAIR_READY_SUCCESS siblingCalls=1 replayStable=true checks=" + completedChecks); + run.close(); + } catch (Throwable failure) { + System.out.println("FAIR_READY_FAILURE mode=" + args[0] + " checks=" + checks.get() + " siblingCalls=" + + siblings.get() + " queued=" + + ((ThreadPoolExecutor) workers).getQueue().size()); + failure.printStackTrace(System.out); + Thread.getAllStackTraces().forEach((thread, stack) -> { + if (!thread.getName().startsWith("durable-sdk-internal") + && !thread.getName().startsWith("pool-")) return; + System.out.println("THREAD " + thread.getName() + " " + thread.getState()); + Arrays.stream(stack).forEach(frame -> System.out.println(" " + frame)); + }); + System.exit(2); + } + } + } + /** A separate process bounds cleanup of an old-code deadlock without altering the observed protocol. */ public static final class SynchronousProbe { public static void main(String[] args) throws Exception { @@ -533,6 +604,7 @@ public T deserialize(String value, TypeToken type) { } }; private final IntFunction> check; + private BiFunction customHandler; private final DurableConfig config; private final WaitForConditionConfig wait = WaitForConditionConfig.builder() .initialState(1) @@ -576,14 +648,16 @@ private Future start() { input, null, TypeToken.get(String.class), - (value, ctx) -> String.valueOf(ctx.waitForCondition( - "condition", - Integer.class, - (state, step) -> { - checks.incrementAndGet(); - return check.apply(state); - }, - wait)), + customHandler != null + ? customHandler + : (value, ctx) -> String.valueOf(ctx.waitForCondition( + "condition", + Integer.class, + (state, step) -> { + checks.incrementAndGet(); + return check.apply(state); + }, + wait)), config)); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java index 1dbf5ba5b..36e9e986f 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java @@ -4,8 +4,6 @@ import java.time.Duration; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationAction; @@ -149,7 +147,7 @@ private void executeCheckLogic(T currentState, int attempt) { } private void runCheckLoop(T currentState, int attempt, CompletableFuture> publishedWorker) { - while (!isOperationCompleted()) { + if (!isOperationCompleted()) { var stepContext = getContext().createStepContext(getOperationId(), getName(), attempt); BaseContextImpl.setCurrentContext(stepContext); try (var ignored = DurableLogger.attachContext()) { @@ -175,13 +173,9 @@ private void runCheckLoop(T currentState, int attempt, CompletableFuture> publishedWorker, T nextState, int nextAttempt) { - var owner = Thread.currentThread(); - var inline = new AtomicReference(); - var acceptingInline = new AtomicBoolean(true); - try { - pollUntilReady().thenCompose(op -> { - if (Thread.currentThread() == owner && acceptingInline.get()) { - inline.set(op); - return CompletableFuture.completedFuture(null); - } else { - // A configured executor may run or wait for the next check inline. Leave the serialized - // checkpoint callback before dispatch, retaining activity through worker publication/registration. - return executionManager.runCheckpointContinuation(() -> { - publishedWorker.join().join(); - if (!isOperationCompleted() && op.status() == OperationStatus.READY) - executeCheckLogic(nextState, nextAttempt); - }); - } - }); - } finally { - acceptingInline.set(false); - } - return inline.get(); + pollUntilReady() + .thenCompose(op -> executionManager.runCheckpointContinuation(() -> { + publishedWorker.join().join(); + if (!isOperationCompleted() && op.status() == OperationStatus.READY) + executeCheckLogic(nextState, nextAttempt); + })); } private void handleCheckFailure(Throwable exception) { From bc9787ccc6f176079c61dbc270fb1b4c98f70115 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 8 Oct 2026 04:59:29 +0000 Subject: [PATCH 21/40] fix: match sampling context carriers to their sampler class copy --- .../durable/otel/ExecutionOtelPlugin.java | 6 +- .../durable/otel/InvocationOtelPlugin.java | 6 +- .../durable/otel/OtelPluginSupport.java | 5 ++ ...urableSamplingDecisionClassLoaderTest.java | 59 +++++++++++++++---- 4 files changed, 61 insertions(+), 15 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index bf47ff0f0..96eb283f6 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -341,7 +341,7 @@ public void onInvocationEnd(InvocationEndInfo info) { durableExecutionArn, executionStartTime, samplingIntent, - sdkTracerProvider != null); + OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider)); workflowSpanContext = null; executionAncestor = null; executionStartTime = null; @@ -642,7 +642,9 @@ private Context resolveParentContext(String parentId) { */ private Context withDurableDecision(Context context) { var intent = samplingIntent; - return intent != null && sdkTracerProvider != null ? DurableSamplingDecision.store(context, intent) : context; + return intent != null && OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider) + ? DurableSamplingDecision.store(context, intent) + : context; } private TraceFlags effectiveTraceFlags() { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index 8ca4102eb..29d8f20d9 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -347,7 +347,7 @@ public void onInvocationEnd(InvocationEndInfo info) { durableExecutionArn, executionStartTime, samplingIntent, - sdkTracerProvider != null); + OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider)); workflowSpanContext = null; executionAncestor = null; executionStartTime = null; @@ -660,7 +660,9 @@ private Context resolveParentContext(String parentId) { */ private Context withDurableDecision(Context context) { var intent = samplingIntent; - return intent != null && sdkTracerProvider != null ? DurableSamplingDecision.store(context, intent) : context; + return intent != null && OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider) + ? DurableSamplingDecision.store(context, intent) + : context; } /** diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 9d2184fa8..9f608277b 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -59,6 +59,11 @@ static void exportExecutionRoot( private OtelPluginSupport() {} + /** Only the same DurableSampler class copy can consume this loader's parent-context holder. */ + static boolean usesLocalDurableSampler(SdkTracerProvider provider) { + return provider != null && provider.getSampler() instanceof DurableSampler; + } + /** Creates a new DeterministicIdGenerator for the application-side state bridge. */ static DeterministicIdGenerator createDefaultIdGenerator() { return new DeterministicIdGenerator(); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java index a4aaa5104..204547389 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java @@ -7,9 +7,11 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.api.trace.TracerProvider; import io.opentelemetry.context.Context; @@ -30,7 +32,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.params.provider.CsvSource; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; @@ -104,8 +106,22 @@ void decisionCrossesClassLoaderBoundary_viaScopedProperty() throws Exception { } @ParameterizedTest - @ValueSource(strings = {"InvocationOtelPlugin", "ExecutionOtelPlugin"}) - void agentProcessorForwardingParentCannotReuseApplicationSamplingIntent(String pluginName) throws Exception { + @CsvSource({ + "InvocationOtelPlugin,true,false,DurableExecutionRoot", + "InvocationOtelPlugin,false,false,DurableExecutionRoot", + "InvocationOtelPlugin,false,true,DurableExecutionRoot", + "ExecutionOtelPlugin,true,false,DurableExecutionRoot", + "ExecutionOtelPlugin,false,false,DurableExecutionRoot", + "ExecutionOtelPlugin,false,true,DurableExecutionRoot", + "InvocationOtelPlugin,true,false,Invocation", + "InvocationOtelPlugin,false,false,Invocation", + "InvocationOtelPlugin,false,true,Invocation", + "ExecutionOtelPlugin,true,false,Invocation", + "ExecutionOtelPlugin,false,false,Invocation", + "ExecutionOtelPlugin,false,true,Invocation" + }) + void agentProcessorForwardingParentCannotReuseApplicationSamplingIntent( + String pluginName, boolean hideProvider, boolean localSampler, String observedSpan) throws Exception { var previousHeader = System.getProperty("com.amazonaws.xray.traceHeader"); GlobalOpenTelemetry.resetForTest(); OtelPluginAutoConfigurationState.markInstalled(); @@ -118,6 +134,12 @@ void agentProcessorForwardingParentCannotReuseApplicationSamplingIntent(String p var agentSamplerType = Class.forName(DurableSampler.class.getName(), true, agentLoader); var agentWrap = agentSamplerType.getDeclaredMethod("wrap", Sampler.class); agentWrap.setAccessible(true); + var appDecision = Class.forName(DurableSamplingDecision.class.getName(), true, appLoader); + var appGet = appDecision.getDeclaredMethod("get", Context.class); + appGet.setAccessible(true); + var agentDecision = Class.forName(DurableSamplingDecision.class.getName(), true, agentLoader); + var agentGet = agentDecision.getDeclaredMethod("get", Context.class); + agentGet.setAccessible(true); var agentIdType = Class.forName(DeterministicIdGenerator.class.getName(), true, agentLoader); var callbacks = new AtomicInteger(); var leakedSampling = new AtomicBoolean(); @@ -125,13 +147,14 @@ void agentProcessorForwardingParentCannotReuseApplicationSamplingIntent(String p .setSampler((Sampler) appWrap.invoke(null, Sampler.alwaysOff())) .build(); var agentProvider = SdkTracerProvider.builder() - .setSampler((Sampler) agentWrap.invoke(null, Sampler.alwaysOff())) + .setSampler( + (Sampler) (localSampler ? appWrap : agentWrap).invoke(null, Sampler.alwaysOff())) .setIdGenerator( (IdGenerator) agentIdType.getConstructor().newInstance()) .addSpanProcessor(new SpanProcessor() { @Override public void onStart(Context parent, ReadWriteSpan span) { - if (!span.getName().equals("DurableExecutionRoot")) return; + if (!span.getName().equals(observedSpan)) return; var unrelated = appProvider .get("processor") .spanBuilder("unrelated-forwarded-parent") @@ -156,6 +179,7 @@ public boolean isEndRequired() { } }) .build()) { + assertEquals(localSampler, agentProvider.getSampler().getClass() == appSamplerType); var hiddenAgentProvider = new TracerProvider() { @Override public Tracer get(String name) { @@ -170,7 +194,7 @@ public Tracer get(String name, String version) { GlobalOpenTelemetry.set(new OpenTelemetry() { @Override public TracerProvider getTracerProvider() { - return hiddenAgentProvider; + return hideProvider ? hiddenAgentProvider : agentProvider; } @Override @@ -183,12 +207,25 @@ public ContextPropagators getPropagators() { .getConstructor() .newInstance(); var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id"; - for (var first : new boolean[] {true, false}) { - plugin.onInvocationStart(new InvocationInfo("request", arn, first, Instant.EPOCH)); - plugin.onInvocationEnd( - new InvocationEndInfo("request", arn, first, InvocationStatus.PENDING, null)); + var previousAmbient = Span.current(); + var ambient = appProvider + .get("ambient") + .spanBuilder("ambient-control") + .startSpan(); + try (var ambientScope = ambient.makeCurrent()) { + for (var first : new boolean[] {true, false}) { + plugin.onInvocationStart(new InvocationInfo("request", arn, first, Instant.EPOCH)); + plugin.onInvocationEnd( + new InvocationEndInfo("request", arn, first, InvocationStatus.PENDING, null)); + assertSame(ambient, Span.current(), "Plugin cleanup must preserve the caller's ambient span"); + assertNull(appGet.invoke(null, Context.root()), "Application sampling intent must be cleared"); + assertNull(agentGet.invoke(null, Context.root()), "Agent sampling intent must be cleared"); + } + } finally { + ambient.end(); } - assertEquals(2, callbacks.get(), "Both invocation roots must reach the real agent-side processor"); + assertSame(previousAmbient, Span.current()); + assertEquals(2, callbacks.get(), "The selected span in both invocations must reach the real processor"); assertFalse( leakedSampling.get(), "The supplied parent must not override the app provider's DROP policy"); } From a08bdb46f84f7338e91d458504c6fd162f05de8c Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 8 Oct 2026 05:53:04 +0000 Subject: [PATCH 22/40] fix: drain owned condition continuations before manager shutdown --- .../ContinuationShutdownIntegrationTest.java | 431 ++++++++++++++++++ .../durable/execution/ExecutionManager.java | 110 ++++- .../operation/WaitForConditionOperation.java | 4 +- .../execution/ExecutionManagerTest.java | 61 +++ .../WaitForConditionOperationTest.java | 4 +- 5 files changed, 590 insertions(+), 20 deletions(-) create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/execution/ContinuationShutdownIntegrationTest.java diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/execution/ContinuationShutdownIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/execution/ContinuationShutdownIntegrationTest.java new file mode 100644 index 000000000..94b3489f8 --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/execution/ContinuationShutdownIntegrationTest.java @@ -0,0 +1,431 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.execution; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.config.CompletionConfig; +import software.amazon.lambda.durable.config.ParallelConfig; +import software.amazon.lambda.durable.config.WaitForConditionConfig; +import software.amazon.lambda.durable.context.BaseContext; +import software.amazon.lambda.durable.context.BaseContextImpl; +import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.WaitForConditionResult; +import software.amazon.lambda.durable.operation.BaseDurableOperation; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +class ContinuationShutdownIntegrationTest { + private ControlledManager initialManager; + private BaseContext previousContext; + private ThreadContext previousThreadContext; + + private void rememberContext(ControlledManager manager) { + if (initialManager != null) return; + initialManager = manager; + previousContext = BaseContext.getCurrentContext(); + previousThreadContext = manager.getCurrentThreadContext(); + } + + @AfterEach + void restoreCallerContext() { + if (initialManager == null) return; + BaseContextImpl.setCurrentContext(previousContext); + initialManager.setCurrentThreadContext(previousThreadContext); + } + + @Test + void runningPredicateFinishesItsCheckpointAndReplaysWithoutRepeatingWork() throws Exception { + var closed = new AtomicBoolean(); + var late = new AtomicInteger(); + var client = readyClient(closed, late); + var queue = new LinkedBlockingQueue(); + var workers = Executors.newCachedThreadPool(); + var started = new CountDownLatch(1); + var release = new CountDownLatch(1); + var calls = new AtomicInteger(); + var setup = setup(client, workers, queue, closed); + try { + setup.context.waitForConditionAsync( + "condition", + Integer.class, + (state, step) -> { + calls.incrementAndGet(); + if (state == 1) return WaitForConditionResult.continuePolling(2); + started.countDown(); + await(release); + return WaitForConditionResult.stopPolling(state); + }, + waitConfig()); + var task = queue.poll(3, TimeUnit.SECONDS); + assertNotNull(task); + CompletableFuture.runAsync(task).get(3, TimeUnit.SECONDS); + assertTrue(started.await(3, TimeUnit.SECONDS)); + var closing = CompletableFuture.runAsync(setup.manager::close); + assertTrue(setup.manager.closeEntered.await(3, TimeUnit.SECONDS)); + assertThrows(TimeoutException.class, () -> closing.get(150, TimeUnit.MILLISECONDS)); + release.countDown(); + closing.get(3, TimeUnit.SECONDS); + assertEquals(2, calls.get()); + assertEquals( + OperationStatus.SUCCEEDED, + client.getOperationByName("condition").status()); + var checkpoints = client.getOperationUpdates().size(); + var replay = setup(client, workers, new LinkedBlockingQueue<>(), new AtomicBoolean()); + try { + assertEquals( + 2, + replay.context.waitForCondition( + "condition", + Integer.class, + (state, step) -> { + calls.incrementAndGet(); + throw new AssertionError("A completed predicate must not run on replay"); + }, + waitConfig())); + assertEquals(2, calls.get()); + assertEquals(checkpoints, client.getOperationUpdates().size()); + assertEquals(0, late.get()); + } finally { + replay.manager.close(); + } + setup.manager + .runCheckpointContinuation(() -> fail("New work must be rejected after close")) + .get(3, TimeUnit.SECONDS); + assertTrue(queue.isEmpty()); + } finally { + release.countDown(); + setup.manager.close(); + workers.shutdownNow(); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + } + } + + @Test + void earlyParallelCompletionStopsOnlyItsQueuedConditionAndUnblocksTheWaitingBranch() throws Exception { + var closed = new AtomicBoolean(); + var late = new AtomicInteger(); + var client = readyClient(closed, late); + var queue = new LinkedBlockingQueue(); + var workers = Executors.newCachedThreadPool(); + var created = new CountDownLatch(1); + var calls = new AtomicInteger(); + var condition = new AtomicReference(); + var setup = setup(client, workers, queue, closed); + try { + var parallel = setup.context.parallel( + "early", + ParallelConfig.builder() + .completionConfig(CompletionConfig.minSuccessful(1)) + .build()); + var task = new AtomicReference(); + try (parallel) { + parallel.branch("waiting", Integer.class, branch -> { + var future = branch.waitForConditionAsync( + "condition", + Integer.class, + (state, step) -> { + calls.incrementAndGet(); + return WaitForConditionResult.continuePolling(state + 1); + }, + waitConfig()); + condition.set((BaseDurableOperation) future); + created.countDown(); + return future.get(); + }); + parallel.branch("winner", String.class, branch -> { + await(created); + try { + task.set(queue.poll(3, TimeUnit.SECONDS)); + assertNotNull(task.get()); + condition.get().getRunningUserHandler().get(3, TimeUnit.SECONDS); + } catch (Exception failure) { + throw new AssertionError(failure); + } + return "winner"; + }); + } + var result = parallel.get(); + assertEquals(1, result.succeeded()); + assertFalse(condition.get().getCompletionFuture().isDone()); + var checkpointCount = client.getOperationUpdates().size(); + var closing = CompletableFuture.runAsync(setup.manager::close); + assertTrue(setup.manager.closeEntered.await(3, TimeUnit.SECONDS)); + assertThrows(TimeoutException.class, () -> closing.get(150, TimeUnit.MILLISECONDS)); + CompletableFuture.runAsync(task.get()).get(3, TimeUnit.SECONDS); + closing.get(3, TimeUnit.SECONDS); + assertTrue(condition.get().getCompletionFuture().isCompletedExceptionally()); + assertEquals(1, calls.get()); + assertEquals(0, late.get()); + assertEquals(checkpointCount, client.getOperationUpdates().size()); + assertEquals(1, result.succeeded(), "Cleanup must not rewrite the stored early result"); + } finally { + assertThrows(SuspendExecutionException.class, setup.manager::suspendExecution); + Runnable pending; + while ((pending = queue.poll()) != null) pending.run(); + workers.shutdownNow(); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + setup.manager.close(); + } + } + + private record Setup(ControlledManager manager, DurableContextImpl context) {} + + private Setup setup( + LocalMemoryExecutionClient client, + ExecutorService workers, + BlockingQueue queue, + AtomicBoolean closed) { + var config = DurableConfig.builder() + .withDurableExecutionClient(client) + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withPollingStrategy(attempt -> Duration.ZERO) + .build(); + var manager = new ControlledManager(input(client), config, queue, closed); + rememberContext(manager); + manager.registerActiveThread(null); + manager.setCurrentThreadContext(new ThreadContext(null, ThreadType.CONTEXT)); + var context = DurableContextImpl.createRootContext(manager, config, null); + BaseContextImpl.setCurrentContext(context); + return new Setup(manager, context); + } + + private static WaitForConditionConfig waitConfig() { + return WaitForConditionConfig.builder() + .initialState(1) + .waitStrategy((state, attempt) -> Duration.ofSeconds(1)) + .build(); + } + + private static void await(CountDownLatch latch) { + try { + assertTrue(latch.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new AssertionError(failure); + } + } + + private static LocalMemoryExecutionClient readyClient(AtomicBoolean closed, AtomicInteger late) { + return new LocalMemoryExecutionClient() { + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List updates) { + if (closed.get()) late.incrementAndGet(); + var applied = super.checkpoint(arn, token, updates); + advanceTime(); + var ready = super.checkpoint(arn, applied.checkpointToken(), List.of()); + var states = new LinkedHashMap(); + applied.newExecutionState().operations().forEach(op -> states.put(op.id(), op)); + ready.newExecutionState().operations().forEach(op -> states.put(op.id(), op)); + return ready.toBuilder() + .newExecutionState(CheckpointUpdatedExecutionState.builder() + .operations(states.values()) + .build()) + .build(); + } + }; + } + + @ParameterizedTest + @ValueSource(strings = {"queued", "cancelled-observation", "after-old-worker-join", "already-suspended"}) + void normalCloseQuiescesAdmittedReadyWork(String mode) throws Exception { + var closed = new AtomicBoolean(); + var lateBackend = new AtomicInteger(); + var predicates = new AtomicInteger(); + var queue = new LinkedBlockingQueue(); + var workers = new GateExecutor(mode.equals("after-old-worker-join")); + var client = new LocalMemoryExecutionClient() { + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List updates) { + if (closed.get()) lateBackend.incrementAndGet(); + var applied = super.checkpoint(arn, token, updates); + advanceTime(); + var ready = super.checkpoint(arn, applied.checkpointToken(), List.of()); + var states = new LinkedHashMap(); + applied.newExecutionState().operations().forEach(op -> states.put(op.id(), op)); + ready.newExecutionState().operations().forEach(op -> states.put(op.id(), op)); + return ready.toBuilder() + .newExecutionState(CheckpointUpdatedExecutionState.builder() + .operations(states.values()) + .build()) + .build(); + } + }; + var config = DurableConfig.builder() + .withDurableExecutionClient(client) + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withPollingStrategy(attempt -> Duration.ZERO) + .build(); + var manager = new ControlledManager(input(client), config, queue, closed); + rememberContext(manager); + manager.registerActiveThread(null); + manager.setCurrentThreadContext(new ThreadContext(null, ThreadType.CONTEXT)); + var context = DurableContextImpl.createRootContext(manager, config, null); + BaseContextImpl.setCurrentContext(context); + var body = new CompletableFuture(); + var selected = manager.runUntilCompleteOrSuspend(body); + try { + context.waitForConditionAsync( + "condition", + Integer.class, + (state, step) -> { + predicates.incrementAndGet(); + return state == 1 + ? WaitForConditionResult.continuePolling(2) + : WaitForConditionResult.stopPolling(state); + }, + WaitForConditionConfig.builder() + .initialState(1) + .waitStrategy((state, attempt) -> Duration.ofSeconds(1)) + .build()); + var task = queue.poll(3, TimeUnit.SECONDS); + assertNotNull(task); + assertTrue(workers.firstFinished.await(3, TimeUnit.SECONDS)); + if (mode.equals("cancelled-observation")) assertTrue(manager.observation.cancel(false)); + CompletableFuture taskRun = null; + if (mode.equals("after-old-worker-join")) { + taskRun = CompletableFuture.runAsync(task); + assertTrue(workers.nextDispatchEntered.await(3, TimeUnit.SECONDS)); + } + var stopped = mode.equals("already-suspended"); + if (stopped) assertThrows(SuspendExecutionException.class, manager::suspendExecution); + else { + body.complete("root-done"); + assertEquals("root-done", selected.get(3, TimeUnit.SECONDS)); + } + var closing = CompletableFuture.runAsync(manager::close); + assertTrue(manager.closeEntered.await(3, TimeUnit.SECONDS)); + boolean returnedWhileHeld; + try { + closing.get(150, TimeUnit.MILLISECONDS); + returnedWhileHeld = true; + } catch (TimeoutException expected) { + returnedWhileHeld = false; + } + workers.releaseNextDispatch.countDown(); + if (taskRun == null) taskRun = CompletableFuture.runAsync(task); + taskRun.get(3, TimeUnit.SECONDS); + closing.get(3, TimeUnit.SECONDS); + workers.shutdown(); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + var returnedEarly = returnedWhileHeld; + System.out.println("CONTINUATION_CLOSE mode=" + mode + " returnedWhileHeld=" + returnedEarly + + " predicateCalls=" + predicates.get() + " lateCheckpointAttempts=" + + manager.lateCheckpointAttempts.get() + " lateBackendCalls=" + lateBackend.get()); + assertAll( + () -> { + if (!stopped) assertFalse(returnedEarly, "Close must wait for the actual admitted task"); + }, + () -> assertEquals(1, predicates.get(), "No new predicate may run after teardown"), + () -> assertEquals(0, manager.lateCheckpointAttempts.get()), + () -> assertEquals(0, lateBackend.get())); + if (!stopped) assertEquals("root-done", selected.join()); + } finally { + workers.releaseNextDispatch.countDown(); + Runnable pending; + while ((pending = queue.poll()) != null) pending.run(); + workers.shutdownNow(); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + manager.close(); + } + } + + private static final class ControlledManager extends ExecutionManager { + private final BlockingQueue queue; + private final AtomicBoolean closed; + private final CountDownLatch closeEntered = new CountDownLatch(1); + private final AtomicInteger lateCheckpointAttempts = new AtomicInteger(); + private volatile CompletableFuture observation; + + private ControlledManager( + DurableExecutionInput input, + DurableConfig config, + BlockingQueue queue, + AtomicBoolean closed) { + super(input, config, null); + this.queue = queue; + this.closed = closed; + } + + public CompletableFuture runCheckpointContinuation(BaseDurableOperation owner, Runnable task) { + observation = super.runCheckpointContinuation(owner, task, queue::add); + return observation; + } + + public CompletableFuture sendOperationUpdate(OperationUpdate update) { + if (closed.get()) lateCheckpointAttempts.incrementAndGet(); + return super.sendOperationUpdate(update); + } + + public void close() { + closeEntered.countDown(); + super.close(); + closed.set(true); + } + } + + private static final class GateExecutor extends ThreadPoolExecutor { + private final AtomicInteger submitted = new AtomicInteger(); + private final AtomicInteger finished = new AtomicInteger(); + private final boolean hold; + private final CountDownLatch firstFinished = new CountDownLatch(1); + private final CountDownLatch nextDispatchEntered = new CountDownLatch(1); + private final CountDownLatch releaseNextDispatch = new CountDownLatch(1); + + private GateExecutor(boolean hold) { + super(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); + this.hold = hold; + } + + public void execute(Runnable task) { + if (submitted.incrementAndGet() == 2 && hold) { + nextDispatchEntered.countDown(); + try { + assertTrue(releaseNextDispatch.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new AssertionError(failure); + } + } + super.execute(task); + } + + protected void afterExecute(Runnable task, Throwable failure) { + if (finished.incrementAndGet() == 1) firstFinished.countDown(); + } + } + + private static DurableExecutionInput input(LocalMemoryExecutionClient client) { + var states = new ArrayList<>(client.getAllOperations()); + states.add( + 0, + Operation.builder() + .id("execution") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails(ExecutionDetails.builder() + .inputPayload("\"input\"") + .build()) + .build()); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/close/execution", + "token", + CheckpointUpdatedExecutionState.builder().operations(states).build()); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java index 9a61c0224..9f260a688 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java @@ -7,6 +7,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -78,6 +79,9 @@ public class ExecutionManager implements SafeCloseable { private final CompletableFuture executionExceptionFuture = new CompletableFuture<>(); // Requests and polling continuations; guarded by activeThreads so admission is atomic with last-thread suspension. private int checkpointRequestsInFlight; + // Guarded by activeThreads; an observation future can be cancelled independently of its actual task. + private final Map checkpointContinuations = new HashMap<>(); + private boolean closing; /** * Per-wait state used to coordinate the caller thread with the future completion callback. @@ -554,34 +558,85 @@ void finishCheckpointProcessing() { * @return completion of the continuation, or an already-completed future when execution has stopped */ public CompletableFuture runCheckpointContinuation(Runnable continuation) { - return runCheckpointContinuation(continuation, InternalExecutor.INSTANCE); + return runCheckpointContinuation(null, continuation, InternalExecutor.INSTANCE); + } + + /** Schedules an operation continuation whose unfinished waiters are stopped during manager cleanup. */ + public CompletableFuture runCheckpointContinuation(BaseDurableOperation owner, Runnable continuation) { + return runCheckpointContinuation(owner, continuation, InternalExecutor.INSTANCE); } CompletableFuture runCheckpointContinuation(Runnable continuation, Executor coordinator) { - if (!tryStartCheckpointProcessing()) return CompletableFuture.completedFuture(null); + return runCheckpointContinuation(null, continuation, coordinator); + } + + CompletableFuture runCheckpointContinuation( + BaseDurableOperation owner, Runnable continuation, Executor coordinator) { + var registration = registerCheckpointContinuation(owner); + if (registration == null) { + if (isClosing()) stopContinuationOwner(owner); + return CompletableFuture.completedFuture(null); + } try { var completion = new CompletableFuture(); - coordinator.execute(() -> { - try { - try { - continuation.run(); - } finally { - finishCheckpointProcessing(); - } - completion.complete(null); - } catch (Throwable failure) { - completion.completeExceptionally(failure); - } - }); + coordinator.execute(() -> completeCheckpointContinuation(registration, continuation, completion)); return completion; } catch (RuntimeException | Error failure) { - finishCheckpointProcessing(); + finishCheckpointContinuation(registration); throw failure; } } + private void completeCheckpointContinuation( + Object registration, Runnable continuation, CompletableFuture completion) { + try { + try { + if (!isClosing()) continuation.run(); + } finally { + finishCheckpointContinuation(registration); + } + completion.complete(null); + } catch (Throwable failure) { + completion.completeExceptionally(failure); + } + } + + private Object registerCheckpointContinuation(BaseDurableOperation owner) { + synchronized (activeThreads) { + if (closing || executionExceptionFuture.isDone()) return null; + var registration = new Object(); + checkpointContinuations.put(registration, owner); + checkpointRequestsInFlight++; + return registration; + } + } + + private void finishCheckpointContinuation(Object registration) { + synchronized (activeThreads) { + checkpointContinuations.remove(registration); + try { + finishCheckpointProcessing(); + } finally { + activeThreads.notifyAll(); + } + } + } + + private boolean isClosing() { + synchronized (activeThreads) { + return closing; + } + } + + private static void stopContinuationOwner(BaseDurableOperation owner) { + if (owner != null) owner.getCompletionFuture().completeExceptionally(new SuspendExecutionException()); + } + private boolean shouldSuspendExecution() { - return activeThreads.isEmpty() && checkpointRequestsInFlight == 0 && !executionExceptionFuture.isDone(); + return !closing + && activeThreads.isEmpty() + && checkpointRequestsInFlight == 0 + && !executionExceptionFuture.isDone(); } private void preSuspendCheck() { @@ -631,11 +686,34 @@ public CompletableFuture pollForOperationUpdates(String operationId, /** Shutdown the checkpoint batcher. */ @Override public void close() { + stopCheckpointContinuations(); validateRunningThreads(); checkpointManager.shutdown(); } + private void stopCheckpointContinuations() { + List owners; + synchronized (activeThreads) { + closing = true; + owners = new ArrayList<>(checkpointContinuations.values()); + } + owners.forEach(ExecutionManager::stopContinuationOwner); + synchronized (activeThreads) { + while (!checkpointContinuations.isEmpty()) { + try { + // wait releases this monitor so tasks can finish and release their registrations. No user work + // or future join is performed while holding the coordination monitor. + activeThreads.wait(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException( + "Interrupted while waiting for checkpoint continuations", interrupted); + } + } + } + } + private void validateRunningThreads() { // This will detect stuck user thread and thread leaks in the thread pool for (BaseDurableOperation op : registeredOperations.values()) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java index 36e9e986f..40eb517ec 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java @@ -111,7 +111,7 @@ private void resumeCheckLoop(Operation existing) { private CompletableFuture pollReadyAndResumeCheckLoop(Operation existing) { return pollUntilReady() - .thenCompose(op -> executionManager.runCheckpointContinuation(() -> { + .thenCompose(op -> executionManager.runCheckpointContinuation(this, () -> { if (!isOperationCompleted() && op.status() == OperationStatus.READY) resumeCheckLoop(op); })); } @@ -187,7 +187,7 @@ private void runCheckLoop(T currentState, int attempt, CompletableFuture> publishedWorker, T nextState, int nextAttempt) { pollUntilReady() - .thenCompose(op -> executionManager.runCheckpointContinuation(() -> { + .thenCompose(op -> executionManager.runCheckpointContinuation(this, () -> { publishedWorker.join().join(); if (!isOperationCompleted() && op.status() == OperationStatus.READY) executeCheckLogic(nextState, nextAttempt); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java index dfb67639e..65d961f36 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java @@ -485,6 +485,67 @@ void rejectedContinuationReleasesActivityAndPreservesTheRejection() { assertThrows(SuspendExecutionException.class, () -> manager.deregisterActiveThread("root")); } + @ParameterizedTest + @ValueSource(strings = {"queued", "cancelled-observation", "rejected"}) + void admissionRacingCloseDrainsActualWorkAndPreservesUnrelatedOperations(String mode) throws Exception { + var manager = createManager(List.of(executionOp(), stepOp("step", OperationStatus.PENDING))); + manager.registerActiveThread("root"); + var owner = mock(BaseDurableOperation.class); + var ownerCompletion = new CompletableFuture(); + when(owner.getCompletionFuture()).thenReturn(ownerCompletion); + var unrelated = mock(BaseDurableOperation.class); + var unrelatedCompletion = new CompletableFuture(); + when(unrelated.getOperationId()).thenReturn("unrelated"); + when(unrelated.getCompletionFuture()).thenReturn(unrelatedCompletion); + manager.registerOperation(unrelated); + var dispatchEntered = new CountDownLatch(1); + var releaseDispatch = new CountDownLatch(1); + var queued = new AtomicReference(); + var rejection = new RejectedExecutionException("rejected during close"); + var admission = CompletableFuture.supplyAsync(() -> + manager.runCheckpointContinuation(owner, () -> fail("Closing must stop this admitted body"), task -> { + dispatchEntered.countDown(); + try { + assertTrue(releaseDispatch.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new AssertionError(interrupted); + } + if (mode.equals("rejected")) throw rejection; + queued.set(task); + })); + assertTrue(dispatchEntered.await(3, TimeUnit.SECONDS)); + var closing = CompletableFuture.runAsync(manager::close); + try { + assertInstanceOf( + SuspendExecutionException.class, + assertThrows(ExecutionException.class, () -> ownerCompletion.get(3, TimeUnit.SECONDS)) + .getCause()); + assertThrows(TimeoutException.class, () -> closing.get(100, TimeUnit.MILLISECONDS)); + } finally { + releaseDispatch.countDown(); + } + if (mode.equals("rejected")) { + assertSame( + rejection, + assertThrows(ExecutionException.class, () -> admission.get(3, TimeUnit.SECONDS)) + .getCause()); + } else { + var observation = admission.get(3, TimeUnit.SECONDS); + if (mode.equals("cancelled-observation")) assertTrue(observation.cancel(false)); + assertThrows(TimeoutException.class, () -> closing.get(100, TimeUnit.MILLISECONDS)); + queued.get().run(); + } + closing.get(3, TimeUnit.SECONDS); + assertFalse(unrelatedCompletion.isDone(), "Normal close must not apply global stopAllOperations"); + assertFalse(manager.isExecutionCompletedExceptionally(), "Cleanup must not replace the selected root outcome"); + manager.runCheckpointContinuation( + () -> fail("Post-close admission must be rejected"), + task -> fail("Post-close work must not reach the executor")) + .get(3, TimeUnit.SECONDS); + assertDoesNotThrow(() -> manager.deregisterActiveThread("root")); + } + @Test void checkpointDeliveryIsAtomicWithOperationRegistration() throws Exception { var manager = createManager(List.of(executionOp(), stepOp("step", OperationStatus.PENDING))); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java index c4df8aac5..39ea05e8d 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java @@ -284,8 +284,8 @@ void getFailedWithNullErrorDataThrowsStepFailedException() { @Test void replayPendingPollsAndResumesCheckLoop() throws Exception { - when(executionManager.runCheckpointContinuation(any())) - .thenAnswer(call -> CompletableFuture.runAsync(call.getArgument(0))); + when(executionManager.runCheckpointContinuation(any(), any(Runnable.class))) + .thenAnswer(call -> CompletableFuture.runAsync(call.getArgument(1))); var pendingOp = Operation.builder() .id(OPERATION_ID) .name(OPERATION_NAME) From 9760d9b5666794256da34e4b8a1b523b84a71571 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 8 Oct 2026 06:32:38 +0000 Subject: [PATCH 23/40] fix: retain live operation parents for native span clocks --- .../durable/otel/InvocationOtelPlugin.java | 5 + .../otel/InvocationClockContainmentTest.java | 145 ++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationClockContainmentTest.java diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index 29d8f20d9..3e9b8160f 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -641,6 +641,11 @@ private Context invocationParentContext(ExecutionTraceContext execCtx, String ca private Context resolveParentContext(String parentId) { if (parentId != null) { + var parentSpan = operationSpans.get(parentId); + if (parentSpan != null) { + // Retain the provider's live span, including its clock, while this parent is open. + return withDurableDecision(Context.current().with(parentSpan)); + } var parentSpanContext = operationContexts.get(parentId); if (parentSpanContext != null) { return withDurableDecision(Context.current().with(Span.wrap(parentSpanContext))); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationClockContainmentTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationClockContainmentTest.java new file mode 100644 index 000000000..da378fd86 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationClockContainmentTest.java @@ -0,0 +1,145 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.sdk.common.Clock; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicLong; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.plugin.OperationEndInfo; +import software.amazon.lambda.durable.plugin.OperationInfo; + +class InvocationClockContainmentTest { + @ParameterizedTest + @EnumSource(InvocationStatus.class) + void openChildAndParentShareTheConfiguredSdkClock(InvocationStatus status) { + var wall = new AtomicLong(100_000_000_000L); + var monotonic = new AtomicLong(); + var clock = new Clock() { + public long now() { + return wall.addAndGet(1_000_000); + } + + public long nanoTime() { + return monotonic.addAndGet(1_000); + } + }; + try (var exporter = InMemorySpanExporter.create()) { + var plugin = new InvocationOtelPlugin( + SdkTracerProvider.builder().setClock(clock).addSpanProcessor(SimpleSpanProcessor.create(exporter)), + OtelPluginConfig.builder() + .contextExtractor(() -> null) + .enableMdc(false) + .build()); + var start = Instant.ofEpochSecond(10); + var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/clock/execution"; + plugin.onInvocationStart(new InvocationInfo("request", arn, true, start)); + plugin.onOperationStart(new OperationInfo( + "parent", "parent", "CONTEXT", "WaitForCallback", null, start, null, null, false)); + plugin.onOperationStart( + new OperationInfo("child", "child", "CALLBACK", "Callback", "parent", start, null, null, false)); + plugin.onInvocationEnd(new InvocationEndInfo( + "request", + arn, + true, + status, + status == InvocationStatus.FAILED ? new IllegalStateException("failure") : null)); + var spans = exporter.getFinishedSpanItems(); + var child = spans.stream() + .filter(s -> s.getName().equals("child")) + .findFirst() + .orElseThrow(); + var parent = spans.stream() + .filter(s -> s.getName().equals("parent")) + .findFirst() + .orElseThrow(); + var invocation = spans.stream() + .filter(s -> s.getName().equals("Invocation")) + .findFirst() + .orElseThrow(); + assertEquals(parent.getSpanId(), child.getParentSpanId()); + assertEquals( + parent.getSpanContext(), + child.getParentSpanContext(), + "Live parent keeps the cached trace ID, span ID, flags and trace state"); + assertEquals(invocation.getSpanId(), parent.getParentSpanId()); + assertTrue(spans.indexOf(child) < spans.indexOf(parent), "The plugin already drains children first"); + assertTrue( + child.getEndEpochNanos() <= parent.getEndEpochNanos(), + "Separate anchor clocks must not invert child-first end timestamps"); + assertTrue(parent.getEndEpochNanos() <= invocation.getEndEpochNanos()); + assertTrue(child.getStartEpochNanos() >= parent.getStartEpochNanos()); + assertTrue( + invocation.getEndEpochNanos() < 101_000_000_000L, + "Keep the configured clock; do not substitute system time or relax containment"); + } + } + + @Test + void endedParentRetainsItsCachedContextForAReplayedChildWithoutReexport() { + try (var exporter = InMemorySpanExporter.create()) { + var plugin = new InvocationOtelPlugin( + SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)), + OtelPluginConfig.builder() + .contextExtractor(() -> null) + .enableMdc(false) + .build()); + var start = Instant.now(); + var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/fallback/execution"; + plugin.onInvocationStart(new InvocationInfo("request", arn, false, start)); + plugin.onOperationStart(new OperationInfo( + "parent", "parent", "CONTEXT", "RunInChildContext", null, start, null, null, true)); + plugin.onOperationEnd(new OperationEndInfo( + "parent", + "parent", + "CONTEXT", + "RunInChildContext", + null, + start, + Instant.now(), + "SUCCEEDED", + null, + true, + null, + null)); + var parent = exporter.getFinishedSpanItems().stream() + .filter(s -> s.getName().equals("parent")) + .findFirst() + .orElseThrow(); + plugin.onOperationEnd(new OperationEndInfo( + "child", + "child", + "CALLBACK", + "Callback", + "parent", + start, + Instant.now(), + "SUCCEEDED", + null, + true, + null, + null)); + plugin.onInvocationEnd(new InvocationEndInfo("request", arn, false, InvocationStatus.PENDING, null)); + var spans = exporter.getFinishedSpanItems(); + var child = spans.stream() + .filter(s -> s.getName().equals("child")) + .findFirst() + .orElseThrow(); + assertEquals(parent.getSpanContext(), child.getParentSpanContext()); + assertEquals( + 1, spans.stream().filter(s -> s.getName().equals("parent")).count()); + assertEquals( + 1, spans.stream().filter(s -> s.getName().equals("child")).count()); + } + } +} From ef2c0fe0075e6f38ffc899114460bd7d4a976d63 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 8 Oct 2026 06:43:40 +0000 Subject: [PATCH 24/40] ci: use bounded layer lookup retries for existing 20 OTel cases --- .github/workflows/otel-conformance-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/otel-conformance-tests.yml b/.github/workflows/otel-conformance-tests.yml index cfb82bbe7..bd7a4a9f6 100644 --- a/.github/workflows/otel-conformance-tests.yml +++ b/.github/workflows/otel-conformance-tests.yml @@ -68,8 +68,9 @@ jobs: actions: write contents: read id-token: write - uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@f18bd0b5f28c5c90e288d0fb8bca08a849b51863 + uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@1768f32d61d958b974ce9e66748df72e23bc19b2 with: + case_count: 20 runs_on: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'ubuntu-latest' || format('codebuild-github-actions-runner-{0}-{1}', github.run_id, github.run_attempt) }} language: java resource_prefix: j From 274807ea0a542e860cf4b4f4cd86ac6b5abd28e4 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 8 Oct 2026 06:57:40 +0000 Subject: [PATCH 25/40] fix: defer custom sampling results to their owning sampler --- .../durable/otel/OtelPluginSupport.java | 5 +- ...urableSamplingDecisionClassLoaderTest.java | 112 ++++++++++++++++++ 2 files changed, 115 insertions(+), 2 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 9f608277b..8fd7ca97b 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -147,8 +147,9 @@ static SamplingResult resolveSamplingResult( } return ambientSpan.isRecording() ? SamplingResult.recordOnly() : SamplingResult.drop(); } - // 3. An application-owned provider exposes the real sampler: evaluate it once, preserving its full result. - if (sdkTracerProvider != null) { + // 3. Only the local sampler can consume a full result from this loader's context carrier. A visible foreign + // sampler must defer too: the shared-property carrier cannot encode custom attributes or trace-state updates. + if (usesLocalDurableSampler(sdkTracerProvider)) { return sdkTracerProvider .getSampler() .shouldSample( diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java index 204547389..3d6409660 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java @@ -11,22 +11,30 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.api.trace.TracerProvider; import io.opentelemetry.context.Context; import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.IdGenerator; import io.opentelemetry.sdk.trace.ReadWriteSpan; import io.opentelemetry.sdk.trace.ReadableSpan; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import io.opentelemetry.sdk.trace.samplers.Sampler; import io.opentelemetry.sdk.trace.samplers.SamplingDecision; import io.opentelemetry.sdk.trace.samplers.SamplingResult; import java.net.URL; import java.net.URLClassLoader; import java.time.Instant; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.AfterEach; @@ -49,6 +57,110 @@ * then asserts the thread-scoped system-property bridge carries the decision from one loader to the other. */ class DurableSamplingDecisionClassLoaderTest { + @ParameterizedTest + @CsvSource({ + "InvocationOtelPlugin,local", + "InvocationOtelPlugin,foreign", + "InvocationOtelPlugin,opaque", + "ExecutionOtelPlugin,local", + "ExecutionOtelPlugin,foreign", + "ExecutionOtelPlugin,opaque" + }) + void customSamplerMetadataSurvivesProviderOwnershipBoundaries(String pluginName, String topology) throws Exception { + var previousHeader = System.getProperty("com.amazonaws.xray.traceHeader"); + GlobalOpenTelemetry.resetForTest(); + OtelPluginAutoConfigurationState.markInstalled(); + System.setProperty("com.amazonaws.xray.traceHeader", "Root=1-6955b900-123456789012345678901234"); + var evaluations = new AtomicInteger(); + var key = AttributeKey.stringKey("sampler.extra"); + var result = new SamplingResult() { + public SamplingDecision getDecision() { + return SamplingDecision.RECORD_AND_SAMPLE; + } + + public Attributes getAttributes() { + return Attributes.of(key, "kept"); + } + + public TraceState getUpdatedTraceState(TraceState parent) { + return parent.toBuilder().put("vendor", "kept").build(); + } + }; + var delegate = new Sampler() { + public SamplingResult shouldSample( + Context parent, + String traceId, + String name, + SpanKind kind, + Attributes attributes, + List links) { + evaluations.incrementAndGet(); + return result; + } + + public String getDescription() { + return "custom-metadata"; + } + }; + try (var appLoader = pluginClassLoader(); + var agentLoader = pluginClassLoader(); + var exporter = InMemorySpanExporter.create()) { + var loader = topology.equals("local") ? appLoader : agentLoader; + var samplerType = Class.forName(DurableSampler.class.getName(), true, loader); + var wrap = samplerType.getDeclaredMethod("wrap", Sampler.class); + wrap.setAccessible(true); + var idType = Class.forName(DeterministicIdGenerator.class.getName(), true, loader); + try (var provider = SdkTracerProvider.builder() + .setSampler((Sampler) wrap.invoke(null, delegate)) + .setIdGenerator((IdGenerator) idType.getConstructor().newInstance()) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build()) { + var hidden = new TracerProvider() { + public Tracer get(String name) { + return provider.get(name); + } + + public Tracer get(String name, String version) { + return provider.get(name, version); + } + }; + GlobalOpenTelemetry.set(new OpenTelemetry() { + public TracerProvider getTracerProvider() { + return topology.equals("opaque") ? hidden : provider; + } + + public ContextPropagators getPropagators() { + return ContextPropagators.noop(); + } + }); + var plugin = (DurableExecutionPlugin) + Class.forName("software.amazon.lambda.durable.otel." + pluginName, true, appLoader) + .getConstructor() + .newInstance(); + var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/custom/id"; + for (var first : new boolean[] {true, false}) { + plugin.onInvocationStart(new InvocationInfo("request", arn, first, Instant.ofEpochSecond(10))); + plugin.onInvocationEnd( + new InvocationEndInfo("request", arn, first, InvocationStatus.PENDING, null)); + } + var spans = exporter.getFinishedSpanItems().stream() + .filter(s -> + s.getName().equals("Invocation") || s.getName().equals("DurableExecutionRoot")) + .toList(); + assertEquals(4, spans.size()); + for (var span : spans) { + assertEquals("kept", span.getAttributes().get(key), span.getName()); + assertEquals("kept", span.getSpanContext().getTraceState().get("vendor"), span.getName()); + } + assertEquals(topology.equals("local") ? 2 : 1, evaluations.get()); + } + } finally { + GlobalOpenTelemetry.resetForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); + if (previousHeader == null) System.clearProperty("com.amazonaws.xray.traceHeader"); + else System.setProperty("com.amazonaws.xray.traceHeader", previousHeader); + } + } @AfterEach void clearBridge() { From 22304e1e110e68190a9ac4670f2f7a3a08d07359 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 8 Oct 2026 16:55:40 +0000 Subject: [PATCH 26/40] fix: isolate deferred sampling decisions by execution --- otel-plugin/README.md | 2 + .../lambda/durable/otel/DurableSampler.java | 17 ++-- .../durable/otel/DurableSamplingDecision.java | 8 +- .../durable/otel/OtelPluginSupport.java | 2 +- ...urableSamplingDecisionClassLoaderTest.java | 82 ++++++++++++------- .../durable/otel/ExecutionOtelPluginTest.java | 6 +- 6 files changed, 72 insertions(+), 45 deletions(-) diff --git a/otel-plugin/README.md b/otel-plugin/README.md index 9bd7ff8cb..19f94bcb1 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -300,6 +300,8 @@ For precise, provider-independent control, set an explicit `Sampled` value upstr | `durable.attempt.number` | 1-based attempt number | | `durable.attempt.outcome` | SUCCEEDED (span status `OK`), FAILED (`ERROR`), or INCOMPLETE (`UNSET`) | +Deferred sampler results are isolated by execution ARN and canonical trace ID in the existing 256-entry LRU cache. The delegate is reused while an entry is resident; eviction can cause another evaluation. The delegate still receives the canonical trace ID. + ## Log Correlation (MDC) When `enableMdc` is true (default), the plugin injects these fields into SLF4J MDC during user function execution: diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java index 34e222ab3..78dc3d84e 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java @@ -44,13 +44,15 @@ final class DurableSampler implements Sampler { private static final int MAX_CACHED_DEFERRED_DECISIONS = 256; private final Sampler delegate; - // Caches the delegate's decision for a deferred durable execution, keyed by canonical trace ID, so a stateful or - // quota-based delegate is consulted once per execution rather than per span. Access-ordered LRU, size-capped and - // synchronized (contention is low: at most one miss per execution). - private final Map deferredDecisions = + // Caches the delegate's decision for a deferred durable execution, keyed by execution ARN and canonical trace ID, + // so a stateful or quota-based delegate is reused while that execution entry remains resident. + // Access-ordered LRU, size-capped and synchronized; eviction permits another delegate evaluation. + private record ExecutionKey(String traceId, String executionArn) {} + + private final Map deferredDecisions = Collections.synchronizedMap(new LinkedHashMap<>(16, 0.75f, true) { @Override - protected boolean removeEldestEntry(Map.Entry eldest) { + protected boolean removeEldestEntry(Map.Entry eldest) { return size() > MAX_CACHED_DEFERRED_DECISIONS; } }); @@ -126,8 +128,9 @@ public SamplingResult shouldSample( // Deferred (agent path, real sampler not reproducible here): evaluate the actual delegate once per execution // and reuse it, so an installed drop/rate-limit policy is honored and consulted only once. return deferredDecisions.computeIfAbsent( - intent.deferredTraceId(), - key -> delegate.shouldSample(Context.root(), key, name, spanKind, attributes, Collections.emptyList())); + new ExecutionKey(intent.deferredTraceId(), attributes.get(SpanAttributes.DURABLE_EXECUTION_ARN)), + key -> delegate.shouldSample( + Context.root(), key.traceId(), name, spanKind, attributes, Collections.emptyList())); } @Override diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java index 2341f8bdb..1d2de1217 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java @@ -19,8 +19,8 @@ * preserving the full three-way decision (including {@code RECORD_ONLY}) and consulting no delegate; or *
    4. a deferral marker (carrying the canonical trace ID) — used on the Java-agent path when the real sampler * (a remote/custom/file-only policy) cannot be reproduced here. {@link DurableSampler} then evaluates its actual - * delegate once per execution, caches the result by trace ID, and reuses it for the execution's remaining durable - * spans, so an installed drop/rate-limit policy is honored and consulted only once. + * delegate once per execution, caches the result by execution ARN and trace ID, and reuses it for the execution's + * remaining durable spans, so an installed drop/rate-limit policy is honored and consulted only once. * * *

      Two carriers, because the plugin runs across two class loaders. Under the documented ADOT setup the plugin @@ -49,10 +49,10 @@ final class DurableSamplingDecision { /** * The durable sampling intent for a span: either a resolved {@link SamplingResult}, or a deferral to the agent-side - * sampler's own delegate keyed by the canonical trace ID. + * sampler's own delegate. The sampler uses the execution ARN already attached to the span with this trace ID. * * @param resolved the resolved decision, or null when deferring - * @param deferredTraceId the canonical trace ID to key the agent-side delegate cache on, or null when resolved + * @param deferredTraceId the canonical trace ID passed to the agent-side delegate, or null when resolved */ record Intent(SamplingResult resolved, String deferredTraceId) { static Intent resolved(SamplingResult result) { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 8fd7ca97b..ac91e27c6 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -96,7 +96,7 @@ static DeterministicIdGenerator createDefaultIdGenerator() { * pipeline finally installs, and another extension's customizer can wrap or replace a recognized configured * sampler, so a reconstruction could disagree with the real delegate. Deferring routes the decision to the * agent-installed {@link DurableSampler}, which consults its actual delegate once per execution, caches the - * result by trace ID, and reuses it for the execution's remaining durable spans (see + * result by execution ARN and trace ID, and reuses it for the execution's remaining durable spans (see * {@link DurableSampler#shouldSample}). The delegate's decision is honored in full — including a * {@code DROP}/rate-limited outcome — so durable spans are not force-sampled. *

    diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java index 3d6409660..17bbff153 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java @@ -59,33 +59,21 @@ class DurableSamplingDecisionClassLoaderTest { @ParameterizedTest @CsvSource({ - "InvocationOtelPlugin,local", - "InvocationOtelPlugin,foreign", - "InvocationOtelPlugin,opaque", - "ExecutionOtelPlugin,local", - "ExecutionOtelPlugin,foreign", - "ExecutionOtelPlugin,opaque" + "InvocationOtelPlugin,local,false", "InvocationOtelPlugin,local,true", + "InvocationOtelPlugin,foreign,false", "InvocationOtelPlugin,foreign,true", + "InvocationOtelPlugin,opaque,false", "InvocationOtelPlugin,opaque,true", + "ExecutionOtelPlugin,local,false", "ExecutionOtelPlugin,local,true", + "ExecutionOtelPlugin,foreign,false", "ExecutionOtelPlugin,foreign,true", + "ExecutionOtelPlugin,opaque,false", "ExecutionOtelPlugin,opaque,true" }) - void customSamplerMetadataSurvivesProviderOwnershipBoundaries(String pluginName, String topology) throws Exception { + void customSamplerMetadataSurvivesProviderOwnershipBoundaries( + String pluginName, String topology, boolean sharedTraceExecutions) throws Exception { var previousHeader = System.getProperty("com.amazonaws.xray.traceHeader"); GlobalOpenTelemetry.resetForTest(); OtelPluginAutoConfigurationState.markInstalled(); System.setProperty("com.amazonaws.xray.traceHeader", "Root=1-6955b900-123456789012345678901234"); var evaluations = new AtomicInteger(); var key = AttributeKey.stringKey("sampler.extra"); - var result = new SamplingResult() { - public SamplingDecision getDecision() { - return SamplingDecision.RECORD_AND_SAMPLE; - } - - public Attributes getAttributes() { - return Attributes.of(key, "kept"); - } - - public TraceState getUpdatedTraceState(TraceState parent) { - return parent.toBuilder().put("vendor", "kept").build(); - } - }; var delegate = new Sampler() { public SamplingResult shouldSample( Context parent, @@ -95,7 +83,24 @@ public SamplingResult shouldSample( Attributes attributes, List links) { evaluations.incrementAndGet(); - return result; + var second = + attributes.get(SpanAttributes.DURABLE_EXECUTION_ARN).contains("/second/"); + var metadata = sharedTraceExecutions ? (second ? "second" : "first") : "kept"; + return new SamplingResult() { + public SamplingDecision getDecision() { + return sharedTraceExecutions && !second + ? SamplingDecision.RECORD_ONLY + : SamplingDecision.RECORD_AND_SAMPLE; + } + + public Attributes getAttributes() { + return Attributes.of(key, metadata); + } + + public TraceState getUpdatedTraceState(TraceState parentState) { + return parentState.toBuilder().put("vendor", metadata).build(); + } + }; } public String getDescription() { @@ -113,7 +118,9 @@ public String getDescription() { try (var provider = SdkTracerProvider.builder() .setSampler((Sampler) wrap.invoke(null, delegate)) .setIdGenerator((IdGenerator) idType.getConstructor().newInstance()) - .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .addSpanProcessor(SimpleSpanProcessor.builder(exporter) + .setExportUnsampledSpans(true) + .build()) .build()) { var hidden = new TracerProvider() { public Tracer get(String name) { @@ -137,22 +144,35 @@ public ContextPropagators getPropagators() { Class.forName("software.amazon.lambda.durable.otel." + pluginName, true, appLoader) .getConstructor() .newInstance(); - var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/custom/id"; - for (var first : new boolean[] {true, false}) { - plugin.onInvocationStart(new InvocationInfo("request", arn, first, Instant.ofEpochSecond(10))); - plugin.onInvocationEnd( - new InvocationEndInfo("request", arn, first, InvocationStatus.PENDING, null)); + var executionCount = sharedTraceExecutions ? 2 : 1; + for (var index = 0; index < executionCount; index++) { + var executionName = index == 0 ? "first" : "second"; + var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/" + executionName + + "/id"; + for (var first : new boolean[] {true, false}) { + plugin.onInvocationStart(new InvocationInfo("request", arn, first, Instant.ofEpochSecond(10))); + plugin.onInvocationEnd( + new InvocationEndInfo("request", arn, first, InvocationStatus.PENDING, null)); + } } var spans = exporter.getFinishedSpanItems().stream() .filter(s -> s.getName().equals("Invocation") || s.getName().equals("DurableExecutionRoot")) .toList(); - assertEquals(4, spans.size()); + assertEquals(executionCount * 4, spans.size()); for (var span : spans) { - assertEquals("kept", span.getAttributes().get(key), span.getName()); - assertEquals("kept", span.getSpanContext().getTraceState().get("vendor"), span.getName()); + var second = span.getAttributes() + .get(SpanAttributes.DURABLE_EXECUTION_ARN) + .contains("/second/"); + var expected = sharedTraceExecutions ? (second ? "second" : "first") : "kept"; + assertEquals(expected, span.getAttributes().get(key), span.getName()); + assertEquals(expected, span.getSpanContext().getTraceState().get("vendor"), span.getName()); + assertEquals( + !sharedTraceExecutions || second, + span.getSpanContext().isSampled()); + assertEquals("6955b900123456789012345678901234", span.getTraceId()); } - assertEquals(topology.equals("local") ? 2 : 1, evaluations.get()); + assertEquals(executionCount * (topology.equals("local") ? 2 : 1), evaluations.get()); } } finally { GlobalOpenTelemetry.resetForTest(); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java index 9ce54d3ef..b8b37d43a 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java @@ -280,7 +280,9 @@ void workflowAndInvocationSpans_shareExecutionTrace_withoutAmbientContext() { void invocationStart_joinsAmbientTrace_whenAmbientIsOnExecutionTrace() { // Drive an invocation to learn the canonical execution trace ID, then start a fresh invocation with an ambient // span on that same trace: the Invocation span joins the ambient span directly. - plugin.onInvocationStart(new InvocationInfo("req-0", ARN, true, Instant.now())); + // The backend execution start is identical across invocations, even when wall time crosses a second. + var executionStartTime = Instant.parse("2026-01-01T00:00:00Z"); + plugin.onInvocationStart(new InvocationInfo("req-0", ARN, true, executionStartTime)); plugin.onInvocationEnd(new InvocationEndInfo("req-0", ARN, true, InvocationStatus.SUCCEEDED, null)); var canonicalTraceId = spanByName(spanExporter.getFinishedSpanItems(), "Workflow").getTraceId(); @@ -290,7 +292,7 @@ void invocationStart_joinsAmbientTrace_whenAmbientIsOnExecutionTrace() { var ambient = SpanContext.create(canonicalTraceId, ambientSpanId, TraceFlags.getSampled(), TraceState.getDefault()); try (var ignored = Span.wrap(ambient).makeCurrent()) { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, false, Instant.now())); + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, false, executionStartTime)); } plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, false, InvocationStatus.SUCCEEDED, null)); From fd53f8bff7532c22cc79f5f19b950986e3e3f913 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 8 Oct 2026 17:20:16 +0000 Subject: [PATCH 27/40] fix: isolate replacement sampler callbacks in both views --- .../lambda/durable/otel/ExecutionOtelPlugin.java | 7 ++++--- .../lambda/durable/otel/InvocationOtelPlugin.java | 7 ++++--- .../lambda/durable/otel/OtelPluginSupport.java | 15 +++++++++++++-- .../DurableSamplingDecisionClassLoaderTest.java | 15 ++++++++++++--- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index 96eb283f6..683235443 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -341,7 +341,8 @@ public void onInvocationEnd(InvocationEndInfo info) { durableExecutionArn, executionStartTime, samplingIntent, - OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider)); + OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider), + OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)); workflowSpanContext = null; executionAncestor = null; executionStartTime = null; @@ -669,7 +670,7 @@ private TraceState effectiveTraceState() { */ private Span startDurableSpan(SpanBuilder spanBuilder) { var intent = samplingIntent; - if (intent == null) { + if (intent == null || !OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)) { return spanBuilder.startSpan(); } try (var ignored = DurableSamplingDecision.openScope(intent)) { @@ -680,7 +681,7 @@ private Span startDurableSpan(SpanBuilder spanBuilder) { /** Starts a durable span with a forced span ID, publishing the sampling intent as in {@link #startDurableSpan}. */ private Span startDurableSpan(SpanBuilder spanBuilder, String traceId, String spanId) { var intent = samplingIntent; - if (intent == null) { + if (intent == null || !OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)) { return idGenerator.startSpan(spanBuilder, traceId, spanId); } try (var ignored = DurableSamplingDecision.openScope(intent)) { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index 3e9b8160f..6796b43d1 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -347,7 +347,8 @@ public void onInvocationEnd(InvocationEndInfo info) { durableExecutionArn, executionStartTime, samplingIntent, - OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider)); + OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider), + OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)); workflowSpanContext = null; executionAncestor = null; executionStartTime = null; @@ -678,7 +679,7 @@ private Context withDurableDecision(Context context) { */ private Span startDurableSpan(SpanBuilder spanBuilder) { var intent = samplingIntent; - if (intent == null) { + if (intent == null || !OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)) { return spanBuilder.startSpan(); } try (var ignored = DurableSamplingDecision.openScope(intent)) { @@ -689,7 +690,7 @@ private Span startDurableSpan(SpanBuilder spanBuilder) { /** Starts a durable span with a forced span ID, publishing the sampling intent as in {@link #startDurableSpan}. */ private Span startDurableSpan(SpanBuilder spanBuilder, String traceId, String spanId) { var intent = samplingIntent; - if (intent == null) { + if (intent == null || !OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)) { return idGenerator.startSpan(spanBuilder, traceId, spanId); } try (var ignored = DurableSamplingDecision.openScope(intent)) { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index ac91e27c6..9a2b73a59 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -37,7 +37,8 @@ static void exportExecutionRoot( String arn, Instant start, DurableSamplingDecision.Intent intent, - boolean useContextCarrier) { + boolean useContextCarrier, + boolean useThreadCarrier) { if (ancestor == null || ancestor.isRemote()) { return; } @@ -48,7 +49,11 @@ static void exportExecutionRoot( .setAttribute(DURABLE_EXECUTION_SYNTHETIC_ROOT, true) .setStartTimestamp(start); Span root; - try (var ignored = DurableSamplingDecision.openScope(intent)) { + if (useThreadCarrier) { + try (var ignored = DurableSamplingDecision.openScope(intent)) { + root = idGenerator.startSpan(builder, ancestor.getTraceId(), ancestor.getSpanId()); + } + } else { root = idGenerator.startSpan(builder, ancestor.getTraceId(), ancestor.getSpanId()); } // End processors/exporters may create unrelated spans; they must not inherit this sampling override. @@ -59,6 +64,12 @@ static void exportExecutionRoot( private OtelPluginSupport() {} + /** A visible replacement sampler cannot consume our bridge; an opaque agent provider may still need it. */ + static boolean usesDurableSamplingBridge(SdkTracerProvider provider) { + // This identifies only the cross-loader wire consumer; context-key ownership still requires instanceof. + return provider == null || provider.getSampler().getClass().getName().equals(DurableSampler.class.getName()); + } + /** Only the same DurableSampler class copy can consume this loader's parent-context holder. */ static boolean usesLocalDurableSampler(SdkTracerProvider provider) { return provider != null && provider.getSampler() instanceof DurableSampler; diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java index 17bbff153..f60c36949 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java @@ -250,10 +250,16 @@ void decisionCrossesClassLoaderBoundary_viaScopedProperty() throws Exception { "InvocationOtelPlugin,false,true,Invocation", "ExecutionOtelPlugin,true,false,Invocation", "ExecutionOtelPlugin,false,false,Invocation", - "ExecutionOtelPlugin,false,true,Invocation" + "ExecutionOtelPlugin,false,true,Invocation", + "InvocationOtelPlugin,false,false,PlainInvocation", + "ExecutionOtelPlugin,false,false,PlainInvocation", + "InvocationOtelPlugin,false,false,PlainDurableExecutionRoot", + "ExecutionOtelPlugin,false,false,PlainDurableExecutionRoot" }) void agentProcessorForwardingParentCannotReuseApplicationSamplingIntent( String pluginName, boolean hideProvider, boolean localSampler, String observedSpan) throws Exception { + var plainSampler = observedSpan.startsWith("Plain"); + var spanName = plainSampler ? observedSpan.substring("Plain".length()) : observedSpan; var previousHeader = System.getProperty("com.amazonaws.xray.traceHeader"); GlobalOpenTelemetry.resetForTest(); OtelPluginAutoConfigurationState.markInstalled(); @@ -280,13 +286,16 @@ void agentProcessorForwardingParentCannotReuseApplicationSamplingIntent( .build(); var agentProvider = SdkTracerProvider.builder() .setSampler( - (Sampler) (localSampler ? appWrap : agentWrap).invoke(null, Sampler.alwaysOff())) + plainSampler + ? Sampler.alwaysOn() + : (Sampler) (localSampler ? appWrap : agentWrap) + .invoke(null, Sampler.alwaysOff())) .setIdGenerator( (IdGenerator) agentIdType.getConstructor().newInstance()) .addSpanProcessor(new SpanProcessor() { @Override public void onStart(Context parent, ReadWriteSpan span) { - if (!span.getName().equals(observedSpan)) return; + if (!span.getName().equals(spanName)) return; var unrelated = appProvider .get("processor") .spanBuilder("unrelated-forwarded-parent") From d220dd3cf94da77da260b0dc7cf293256985f291 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 8 Oct 2026 18:22:01 +0000 Subject: [PATCH 28/40] fix: retain visible replacement sampler root policy --- otel-plugin/README.md | 46 +++++-- .../durable/otel/OtelPluginSupport.java | 8 +- .../VisibleReplacementSdkSpanPolicyTest.java | 129 ++++++++++++++++++ 3 files changed, 169 insertions(+), 14 deletions(-) create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/VisibleReplacementSdkSpanPolicyTest.java diff --git a/otel-plugin/README.md b/otel-plugin/README.md index 19f94bcb1..af047bdb0 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -55,7 +55,8 @@ same span fields under stable sampling. The existing provider resource still app execution environment, resource attributes such as `faas.instance` can differ. Backends that deduplicate by span identity may retain either copy's resource. Each execution ARN owns its own anchor even when multiple executions share a propagated trace ID without a parent; they are not collapsed into one -execution. Upstream sampling and configured fallback sampling apply to anchors and their descendants together. +execution. With the SDK sampler retained in the provider, upstream sampling and configured fallback sampling apply +to anchors and their descendants together. After the first invocation's export and flush succeed, its anchor remains available even if the execution is stopped or times out while suspended and never invokes the plugin again. An invocation killed before its end hook or flush can @@ -258,14 +259,39 @@ Operation and attempt spans link to the Workflow span. `ExecutionOtelPlugin` rev ### Sampling -The plugin decides sampling once per invocation and applies that single decision to every durable span (DurableExecutionRoot, Workflow, Invocation, operation, attempt), so the configured sampler is not re-invoked per span and the full decision — including `RECORD_ONLY` — is preserved. The decision follows this precedence, highest first: - -1. **Backend decision** — `Sampled=1` / `Sampled=0` in the propagated header is authoritative and always preserved, regardless of the configured sampler. -2. **Same-trace ambient span** — when the header carries no usable `Sampled` value but a valid ambient span (for example an auto-instrumentation Lambda handler span) is already on the execution's trace, the plugin follows that span's decision: sampled → sampled; unsampled but still recording → `RECORD_ONLY`; unsampled and not recording → dropped. -3. **Configured sampler (application-owned provider)** — when you pass a `SdkTracerProvider` to the plugin, its sampler is read directly and evaluated once with the trace ID, span name, and attributes. A trace-ID-ratio sampler therefore produces a stable decision across reinvocations (the trace ID is stable). -4. **Installed sampler (Java-agent path)** — when the agent owns the provider, it is behind a classloader boundary and its *effective* sampler (which another agent extension may have wrapped or replaced) cannot be reliably read at decision time. Rather than guess, the plugin **defers**: it installs a delegating sampler through the agent's autoconfiguration and lets that wrapper consult the agent's real sampler. The delegate's decision is honored in full — if your configured policy is `always_off`, a rate limiter, or a remote sampler (`xray`, `jaeger_remote`) that returns drop, the durable spans are dropped; they are **not** force-sampled. To avoid consuming a stateful or quota-based sampler once per span, the wrapper consults the delegate once per execution (keyed by trace ID) and reuses that decision for the execution's remaining durable spans within the invocation. - -For precise, provider-independent control, set an explicit `Sampled` value upstream (for example by enabling X-Ray active tracing) — that backend decision takes precedence over everything else. +The SDK-wide sampling guarantees below require `DurableSampler` itself to be the final installed sampler. The +[builder constructors](#configuration) and [ADOT extension setup](#1-adot-lambda-layer) install it automatically; +configure its delegate through these documented paths and retain `DurableSampler` as the final sampler. An +unrecognized outer wrapper is treated as a plain replacement. The SDK wrapper applies its sampling intent to every durable +span (DurableExecutionRoot, Workflow, Invocation, operation and attempt), preserving the full result, including `RECORD_ONLY`, sampler attributes +and trace state. + +The supported SDK sampler follows this precedence, highest first: + +1. **Backend decision** — with `DurableSampler` installed, `Sampled=1` / `Sampled=0` in the propagated header is + authoritative, regardless of the wrapped delegate's policy. +2. **Same-trace ambient span** — without an explicit header decision, a valid ambient span already on the execution + trace supplies its decision: sampled → sampled; unsampled and recording → `RECORD_ONLY`; unsampled and + non-recording → dropped. +3. **Same-copy durable sampler** — the visible SDK sampler is evaluated against root context once per invocation + using the canonical trace ID, span name and attributes. Its complete result is carried in that loader's context. +4. **Foreign or opaque SDK sampler** — resolution is deferred to the installed wrapper's actual delegate. Its full + result is cached by execution ARN and canonical trace ID in a 256-entry LRU cache and reused while resident. + Eviction can cause another evaluation. The delegate still receives the canonical trace ID; drop and + `RECORD_ONLY` decisions, attributes and updated trace state are retained. + +A visible plain replacement sampler supplies its root policy for execution-ancestor flags, then keeps its normal +per-span sampling behavior. It receives no SDK sampling carrier and does not provide the SDK decision-reuse +or full-result override guarantees above. Opaque providers cannot expose a later sampler replacement to the +application, so retain the wrapper through the documented customization setup. + +For explicit execution-level control, keep `DurableSampler` as the final installed sampler and set `Sampled` upstream, +for example through X-Ray +active tracing. The SDK sampling wrapper preserves that decision. + +Parentless fallback-root exports also use the provider's sampler. If the SDK wrapper is replaced by a plain +sampler, those anchors can follow its normal root policy independently of the flags used by their descendants; +full upstream sampling overrides require the installed SDK wrapper. ## Span Attributes @@ -300,8 +326,6 @@ For precise, provider-independent control, set an explicit `Sampled` value upstr | `durable.attempt.number` | 1-based attempt number | | `durable.attempt.outcome` | SUCCEEDED (span status `OK`), FAILED (`ERROR`), or INCOMPLETE (`UNSET`) | -Deferred sampler results are isolated by execution ARN and canonical trace ID in the existing 256-entry LRU cache. The delegate is reused while an entry is resident; eviction can cause another evaluation. The delegate still receives the canonical trace ID. - ## Log Correlation (MDC) When `enableMdc` is true (default), the plugin injects these fields into SLF4J MDC during user function execution: diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 9a2b73a59..68963ded4 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -158,9 +158,11 @@ static SamplingResult resolveSamplingResult( } return ambientSpan.isRecording() ? SamplingResult.recordOnly() : SamplingResult.drop(); } - // 3. Only the local sampler can consume a full result from this loader's context carrier. A visible foreign - // sampler must defer too: the shared-property carrier cannot encode custom attributes or trace-state updates. - if (usesLocalDurableSampler(sdkTracerProvider)) { + // 3. Resolve a local durable sampler's full result, or a visible replacement's root policy for ancestor flags. + // A foreign DurableSampler still resolves in its own loader to retain full attributes and trace state. + // A plain replacement receives no carrier; the provider keeps its normal per-span sampling behavior. + if (sdkTracerProvider != null + && (usesLocalDurableSampler(sdkTracerProvider) || !usesDurableSamplingBridge(sdkTracerProvider))) { return sdkTracerProvider .getSampler() .shouldSample( diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/VisibleReplacementSdkSpanPolicyTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/VisibleReplacementSdkSpanPolicyTest.java new file mode 100644 index 000000000..d22efaa82 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/VisibleReplacementSdkSpanPolicyTest.java @@ -0,0 +1,129 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.TracerProvider; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class VisibleReplacementSdkSpanPolicyTest { + @ParameterizedTest + @CsvSource({ + "invocation,false,root,false", "execution,false,root,false", + "invocation,true,root,true", "execution,true,root,true", + "invocation,false,sampled,true", "execution,false,sampled,true", + "invocation,true,unsampled,false", "execution,true,unsampled,false" + }) + void actualSdkSpansHonorVisibleReplacementRootPolicy( + String view, boolean rootSamples, String upstream, boolean expectedSampled) { + var oldHeader = System.getProperty("com.amazonaws.xray.traceHeader"); + var rootPolicySnapshots = new AtomicInteger(); + var invocations = new AtomicInteger(); + var sideEffects = new AtomicInteger(); + var header = "Root=1-6955b900-123456789012345678901234"; + if (upstream.equals("sampled")) header += ";Sampled=1"; + else if (upstream.equals("unsampled")) header += ";Sampled=0"; + var root = new Sampler() { + public SamplingResult shouldSample( + Context parent, + String traceId, + String name, + SpanKind kind, + Attributes attributes, + List links) { + assertFalse(Span.fromContext(parent).getSpanContext().isValid()); + // Count the SDK's root-policy snapshot, excluding normal provider sampling for a real synthetic-root + // export on branches that implement it. No exactly-once promise is made for a plain provider. + if (!name.equals("DurableExecutionRoot") + && attributes.get(AttributeKey.stringKey("durable.execution.arn")) != null) { + rootPolicySnapshots.incrementAndGet(); + } + return rootSamples ? SamplingResult.recordAndSample() : SamplingResult.drop(); + } + + public String getDescription() { + return "counted-visible-root-policy"; + } + }; + GlobalOpenTelemetry.resetForTest(); + OtelPluginAutoConfigurationState.markInstalled(); + System.setProperty("com.amazonaws.xray.traceHeader", header); + try (var exporter = InMemorySpanExporter.create(); + var provider = SdkTracerProvider.builder() + .setSampler(Sampler.parentBased(root)) + .setIdGenerator(new DeterministicIdGenerator()) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build()) { + GlobalOpenTelemetry.set(new OpenTelemetry() { + public TracerProvider getTracerProvider() { + return provider; + } + + public ContextPropagators getPropagators() { + return ContextPropagators.noop(); + } + }); + DurableExecutionPlugin plugin = + view.equals("invocation") ? new InvocationOtelPlugin() : new ExecutionOtelPlugin(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + invocations.incrementAndGet(); + context.step("saved", String.class, step -> { + sideEffects.incrementAndGet(); + return "saved"; + }); + context.wait("pause", Duration.ofSeconds(1)); + return "done"; + }, + DurableConfig.builder().withPlugins(plugin).build()); + assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); + assertEquals( + ExecutionStatus.SUCCEEDED, runner.runUntilComplete("input").getStatus()); + assertTrue(invocations.get() >= 2); + var sdkSpans = exporter.getFinishedSpanItems().stream() + .filter(span -> + span.getInstrumentationScopeInfo().getName().equals("aws-durable-execution-sdk-java")) + // A plain provider keeps its native policy for parentless synthetic-root exports. Full SDK + // sampling overrides require DurableSampler; this regression concerns regular SDK descendants. + .filter(span -> !span.getName().equals("DurableExecutionRoot")) + .toList(); + if (expectedSampled) { + assertFalse(sdkSpans.isEmpty()); + assertTrue( + sdkSpans.stream().allMatch(span -> span.getSpanContext().isSampled())); + } else assertTrue(sdkSpans.isEmpty(), sdkSpans.toString()); + assertEquals(upstream.equals("root") ? invocations.get() : 0, rootPolicySnapshots.get()); + assertEquals(1, sideEffects.get()); + } finally { + GlobalOpenTelemetry.resetForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); + DurableSamplingDecision.clearSharedStateForTest(); + if (oldHeader == null) System.clearProperty("com.amazonaws.xray.traceHeader"); + else System.setProperty("com.amazonaws.xray.traceHeader", oldHeader); + } + } +} From 960a394a6d2bffbccf26fbdf456e395f50b5b392 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 8 Oct 2026 20:48:58 +0000 Subject: [PATCH 29/40] fix(otel): resolve root sampling context before descendants --- otel-plugin/README.md | 5 + .../durable/otel/ExecutionOtelPlugin.java | 49 ++++-- .../durable/otel/InvocationOtelPlugin.java | 49 ++++-- .../durable/otel/OtelPluginSupport.java | 45 ++++- ...urableSamplingDecisionClassLoaderTest.java | 135 +++++++++++++++ .../ExecutionOtelPluginIntegrationTest.java | 24 +-- .../otel/ExecutionRootLifecycleTest.java | 154 ++++++++++++++++++ 7 files changed, 401 insertions(+), 60 deletions(-) create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java diff --git a/otel-plugin/README.md b/otel-plugin/README.md index af047bdb0..e50681722 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -44,6 +44,11 @@ If you configure your own `SdkTracerProviderBuilder`, add the OpenTelemetry SDK When the backend supplies no complete remote parent, both views export a `DurableExecutionRoot` anchor before the first invocation returns, including when it suspends with `PENDING` or fails with `RETRYING`. +The anchor is started before descendants so their parent context includes its provider-resolved trace flags and +trace state. It ends at invocation end, before flushing, with the same fixed execution-start timestamp. For a +deferred sampler, the first actual fallback span is now `DurableExecutionRoot`; custom policies that depend on the +first span name or attributes can observe that ordering. No arbitrary name/input invariance is promised. + The anchor is marked `durable.execution.synthetic_root=true`. Its trace and span IDs are deterministic and its start and end timestamps are the checkpointed execution start. It does not report execution status or duration; `Workflow` continues to report those at terminal completion. Complete remote parents remain externally owned and are never exported. diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index 683235443..b9f042b66 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -105,6 +105,7 @@ public class ExecutionOtelPlugin implements DurableExecutionPlugin { // Per-invocation state private volatile boolean tracingEnabled; private volatile Span invocationSpan; + private volatile Span executionRootSpan; private volatile String durableExecutionArn; // Trace ID and flags of the execution trace, published together as one snapshot so readers never pair a trace ID @@ -238,9 +239,22 @@ public void onInvocationStart(InvocationInfo info) { : DurableSamplingDecision.Intent.deferred(canonicalTraceId); var sampled = OtelPluginSupport.isSampled(decision); var execCtx = ExecutionTraceContext.resolve(extracted, canonicalTraceId, arn(), idGenerator, () -> sampled); + executionStartTime = info.executionStartTime(); + executionRootSpan = OtelPluginSupport.startExecutionRoot( + tracer, + idGenerator, + execCtx.executionAncestor(), + info.durableExecutionArn(), + executionStartTime, + samplingIntent, + OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider), + OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)); + // A visible plain replacement keeps its documented ancestor-flag fallback, without SDK-wide overrides. + if (executionRootSpan != null && OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)) { + execCtx = new ExecutionTraceContext(executionRootSpan.getSpanContext()); + } executionTrace = new ExecutionTrace(canonicalTraceId, execCtx.traceFlags()); executionAncestor = execCtx.executionAncestor(); - executionStartTime = info.executionStartTime(); // Invocation span — child of the ambient Lambda span when it is on the execution trace, otherwise a child of // the execution ancestor so it stays within the same trace. @@ -275,6 +289,22 @@ public void onInvocationStart(InvocationInfo info) { @Override public void onInvocationEnd(InvocationEndInfo info) { + var root = executionRootSpan; + var rootTimestamp = executionStartTime; + var provider = sdkTracerProvider; + var shouldFlush = tracingEnabled || root != null; + executionRootSpan = null; // One-shot release even if an end processor throws or re-enters. + OtelPluginSupport.finishWithRootCleanup(() -> finishInvocationEnd(info), () -> { + if (root != null) root.end(rootTimestamp); + if (shouldFlush && provider != null) { + var flushResult = provider.forceFlush().join(5, TimeUnit.SECONDS); + if (!flushResult.isSuccess()) + logger.warn("OTel span flush failed or timed out — some spans may be lost"); + } + }); + } + + private void finishInvocationEnd(InvocationEndInfo info) { if (!tracingEnabled) { return; } @@ -334,27 +364,10 @@ public void onInvocationEnd(InvocationEndInfo info) { } workflowSpan.end(); } - OtelPluginSupport.exportExecutionRoot( - tracer, - idGenerator, - executionAncestor, - durableExecutionArn, - executionStartTime, - samplingIntent, - OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider), - OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)); workflowSpanContext = null; executionAncestor = null; executionStartTime = null; samplingIntent = null; - - // Flush spans before Lambda freezes - if (sdkTracerProvider != null) { - var flushResult = sdkTracerProvider.forceFlush().join(5, TimeUnit.SECONDS); - if (!flushResult.isSuccess()) { - logger.warn("OTel span flush failed or timed out — some spans may be lost"); - } - } } // ─── Operation hooks ───────────────────────────────────────────────── diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index 6796b43d1..eabccc4f8 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -92,6 +92,7 @@ public class InvocationOtelPlugin implements DurableExecutionPlugin { // Per-invocation state private volatile boolean tracingEnabled; private volatile Span invocationSpan; + private volatile Span executionRootSpan; private volatile String durableExecutionArn; // Trace ID and flags of the execution trace, published together as one snapshot so readers never pair a trace ID // with mismatched flags. @@ -232,9 +233,22 @@ public void onInvocationStart(InvocationInfo info) { var sampled = OtelPluginSupport.isSampled(decision); var execCtx = ExecutionTraceContext.resolve( extracted, canonicalTraceId, info.durableExecutionArn(), idGenerator, () -> sampled); + executionStartTime = info.executionStartTime(); + executionRootSpan = OtelPluginSupport.startExecutionRoot( + tracer, + idGenerator, + execCtx.executionAncestor(), + info.durableExecutionArn(), + executionStartTime, + samplingIntent, + OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider), + OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)); + // A visible plain replacement keeps its documented ancestor-flag fallback, without SDK-wide overrides. + if (executionRootSpan != null && OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)) { + execCtx = new ExecutionTraceContext(executionRootSpan.getSpanContext()); + } executionTrace = new ExecutionTrace(canonicalTraceId, execCtx.traceFlags()); executionAncestor = execCtx.executionAncestor(); - executionStartTime = info.executionStartTime(); // Invocation span parent — the same-trace ambient span when available, then the execution ancestor, so the // Invocation span stays on the execution trace. @@ -272,6 +286,22 @@ public void onInvocationStart(InvocationInfo info) { @Override public void onInvocationEnd(InvocationEndInfo info) { + var root = executionRootSpan; + var rootTimestamp = executionStartTime; + var provider = sdkTracerProvider; + var shouldFlush = tracingEnabled || root != null; + executionRootSpan = null; // One-shot release even if an end processor throws or re-enters. + OtelPluginSupport.finishWithRootCleanup(() -> finishInvocationEnd(info), () -> { + if (root != null) root.end(rootTimestamp); + if (shouldFlush && provider != null) { + var flushResult = provider.forceFlush().join(5, TimeUnit.SECONDS); + if (!flushResult.isSuccess()) + logger.warn("OTel span flush failed or timed out — some spans may be lost"); + } + }); + } + + private void finishInvocationEnd(InvocationEndInfo info) { if (!tracingEnabled) { return; } @@ -340,27 +370,10 @@ public void onInvocationEnd(InvocationEndInfo info) { } workflowSpan.end(); } - OtelPluginSupport.exportExecutionRoot( - tracer, - idGenerator, - executionAncestor, - durableExecutionArn, - executionStartTime, - samplingIntent, - OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider), - OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)); workflowSpanContext = null; executionAncestor = null; executionStartTime = null; samplingIntent = null; - - if (sdkTracerProvider != null) { - // Flush spans before Lambda freezes - var flushResult = sdkTracerProvider.forceFlush().join(5, TimeUnit.SECONDS); - if (!flushResult.isSuccess()) { - logger.warn("OTel span flush failed or timed out — some spans may be lost"); - } - } } // ─── Operation hooks ───────────────────────────────────────────────── diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 68963ded4..0f81e1d18 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -26,11 +26,12 @@ final class OtelPluginSupport { /** - * Exports the SDK-owned fallback ancestor before each invocation returns, including PENDING. It is a zero-duration - * identity anchor at the checkpointed execution start, not a completion summary. Recovery re-exports use identical - * identity and timestamps; Workflow alone carries execution duration and outcome. Remote parents are never owned. + * Starts the SDK-owned fallback ancestor before descendants so they inherit the provider's actual sampling + * metadata. The caller ends this identity anchor at the same checkpointed start timestamp before invocation return. + * Recovery re-exports retain identity and timestamps; Workflow carries duration and outcome. Remote parents are + * never owned. */ - static void exportExecutionRoot( + static Span startExecutionRoot( Tracer tracer, DeterministicIdGenerator idGenerator, SpanContext ancestor, @@ -40,7 +41,7 @@ static void exportExecutionRoot( boolean useContextCarrier, boolean useThreadCarrier) { if (ancestor == null || ancestor.isRemote()) { - return; + return null; } var builder = tracer.spanBuilder("DurableExecutionRoot") .setSpanKind(SpanKind.INTERNAL) @@ -56,8 +57,38 @@ static void exportExecutionRoot( } else { root = idGenerator.startSpan(builder, ancestor.getTraceId(), ancestor.getSpanId()); } - // End processors/exporters may create unrelated spans; they must not inherit this sampling override. - root.end(start); + return root; + } + + /** Always releases an invocation's new root resource without hiding an earlier failure or a later JVM fatal. */ + static void finishWithRootCleanup(Runnable finish, Runnable cleanup) { + Throwable primary = null; + try { + finish.run(); + } catch (Throwable failure) { + primary = failure; + throw failure; + } finally { + try { + cleanup.run(); + } catch (Throwable failure) { + if (primary == null) throw failure; + if (primary != failure) { + if (endFailurePriority(failure) > endFailurePriority(primary)) { + failure.addSuppressed(primary); + throw failure; + } + primary.addSuppressed(failure); + } + } + } + } + + @SuppressWarnings("removal") + private static int endFailurePriority(Throwable failure) { + if (failure instanceof VirtualMachineError || failure instanceof ThreadDeath) return 3; + if (failure instanceof Error && !(failure instanceof LinkageError)) return 2; + return 1; } private static final Logger logger = LoggerFactory.getLogger(OtelPluginSupport.class); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java index f60c36949..563b39aab 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java @@ -57,6 +57,141 @@ * then asserts the thread-scoped system-property bridge carries the decision from one loader to the other. */ class DurableSamplingDecisionClassLoaderTest { + @ParameterizedTest + @CsvSource({ + "InvocationOtelPlugin,local,false", "InvocationOtelPlugin,local,true", + "InvocationOtelPlugin,foreign,false", "InvocationOtelPlugin,foreign,true", + "InvocationOtelPlugin,opaque,false", "InvocationOtelPlugin,opaque,true", + "ExecutionOtelPlugin,local,false", "ExecutionOtelPlugin,local,true", + "ExecutionOtelPlugin,foreign,false", "ExecutionOtelPlugin,foreign,true", + "ExecutionOtelPlugin,opaque,false", "ExecutionOtelPlugin,opaque,true" + }) + void materializedRootSuppliesParentSamplingMetadata(String pluginName, String topology, boolean recordOnly) + throws Exception { + var previousHeader = System.getProperty("com.amazonaws.xray.traceHeader"); + GlobalOpenTelemetry.resetForTest(); + OtelPluginAutoConfigurationState.markInstalled(); + System.setProperty("com.amazonaws.xray.traceHeader", "Root=1-6955b900-123456789012345678901234"); + var evaluations = new AtomicInteger(); + var key = AttributeKey.stringKey("sampler.extra"); + var result = new SamplingResult() { + public SamplingDecision getDecision() { + return recordOnly ? SamplingDecision.RECORD_ONLY : SamplingDecision.RECORD_AND_SAMPLE; + } + + public Attributes getAttributes() { + return Attributes.of(key, "kept"); + } + + public TraceState getUpdatedTraceState(TraceState parent) { + var depth = parent.get("depth"); + return parent.toBuilder() + .put("vendor", "kept") + .put("depth", Integer.toString(depth == null ? 1 : Integer.parseInt(depth) + 1)) + .build(); + } + }; + var delegate = new Sampler() { + public SamplingResult shouldSample( + Context parent, + String traceId, + String name, + SpanKind kind, + Attributes attributes, + List links) { + evaluations.incrementAndGet(); + return result; + } + + public String getDescription() { + return "custom-metadata"; + } + }; + try (var appLoader = pluginClassLoader(); + var agentLoader = pluginClassLoader(); + var exporter = InMemorySpanExporter.create()) { + var loader = topology.equals("local") ? appLoader : agentLoader; + var samplerType = Class.forName(DurableSampler.class.getName(), true, loader); + var wrap = samplerType.getDeclaredMethod("wrap", Sampler.class); + wrap.setAccessible(true); + var idType = Class.forName(DeterministicIdGenerator.class.getName(), true, loader); + try (var provider = SdkTracerProvider.builder() + .setSampler((Sampler) wrap.invoke(null, delegate)) + .setIdGenerator((IdGenerator) idType.getConstructor().newInstance()) + .addSpanProcessor(SimpleSpanProcessor.builder(exporter) + .setExportUnsampledSpans(true) + .build()) + .build()) { + var hidden = new TracerProvider() { + public Tracer get(String name) { + return provider.get(name); + } + + public Tracer get(String name, String version) { + return provider.get(name, version); + } + }; + GlobalOpenTelemetry.set(new OpenTelemetry() { + public TracerProvider getTracerProvider() { + return topology.equals("opaque") ? hidden : provider; + } + + public ContextPropagators getPropagators() { + return ContextPropagators.noop(); + } + }); + var plugin = (DurableExecutionPlugin) + Class.forName("software.amazon.lambda.durable.otel." + pluginName, true, appLoader) + .getConstructor() + .newInstance(); + var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/custom/id"; + for (var first : new boolean[] {true, false}) { + plugin.onInvocationStart(new InvocationInfo("request", arn, first, Instant.ofEpochSecond(10))); + plugin.onInvocationEnd(new InvocationEndInfo( + "request", + arn, + first, + first ? InvocationStatus.PENDING : InvocationStatus.SUCCEEDED, + null)); + } + var spans = exporter.getFinishedSpanItems(); + var roots = spans.stream() + .filter(span -> span.getName().equals("DurableExecutionRoot")) + .toList(); + assertEquals(2, roots.size()); + var rootContext = roots.get(0).getSpanContext(); + for (var anchor : roots) { + assertEquals( + rootContext, anchor.getSpanContext(), "Re-export keeps the complete stable root context"); + assertEquals(10_000_000_000L, anchor.getStartEpochNanos()); + assertEquals(anchor.getStartEpochNanos(), anchor.getEndEpochNanos()); + assertEquals("1", anchor.getSpanContext().getTraceState().get("depth")); + assertEquals(!recordOnly, anchor.getSpanContext().isSampled()); + assertEquals("kept", anchor.getAttributes().get(key)); + } + var descendants = spans.stream() + .filter(span -> span.getName().equals("Invocation") + || span.getName().equals("Workflow")) + .toList(); + assertEquals(3, descendants.size()); + for (var child : descendants) { + assertEquals( + rootContext, + child.getParentSpanContext(), + "A child must record the materialized parent's flags and trace state: " + child.getName()); + assertEquals("2", child.getSpanContext().getTraceState().get("depth")); + assertEquals("kept", child.getAttributes().get(key)); + } + assertEquals(topology.equals("local") ? 2 : 1, evaluations.get()); + } + } finally { + GlobalOpenTelemetry.resetForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); + if (previousHeader == null) System.clearProperty("com.amazonaws.xray.traceHeader"); + else System.setProperty("com.amazonaws.xray.traceHeader", previousHeader); + } + } + @ParameterizedTest @CsvSource({ "InvocationOtelPlugin,local,false", "InvocationOtelPlugin,local,true", diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java index 129a1b045..175e7ea9d 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java @@ -144,17 +144,15 @@ void waitAcrossInvocations_sharesOneTrace_andExportsWorkflowOnceOnTerminal() { } @Test - void wrappedAgentProvider_withRootDroppingSampler_keepsExecutionTreeConsistentlySampled() { - // The agent provider is hidden behind a classloader wrapper, so the plugin cannot reach the sampler and the - // bridge published nothing. With no reachable decision, the synthetic execution root is treated as sampled so - // the execution root and everything parented onto it are sampled together — no orphan operation spans exported - // under a dropped root. + void wrappedAgentProvider_withInstalledRootDroppingSampler_keepsExecutionTreeDropped() { + // The documented agent customizer retains DurableSampler as the final sampler. Its actual delegate's + // root DROP decision must apply to the fallback anchor and descendants, including behind an opaque provider. OtelPluginAutoConfigurationState.markInstalled(); GlobalOpenTelemetry.resetForTest(); var globalExporter = InMemorySpanExporter.create(); var sdkTracerProvider = SdkTracerProvider.builder() .setIdGenerator(new DeterministicIdGenerator()) - .setSampler(Sampler.parentBased(Sampler.alwaysOff())) + .setSampler(DurableSampler.wrap(Sampler.parentBased(Sampler.alwaysOff()))) .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) .build(); var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(sdkTracerProvider); @@ -180,18 +178,10 @@ public ContextPropagators getPropagators() { var result = runner.runUntilComplete("World"); assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); - var spans = globalExporter.getFinishedSpanItems(); - assertSpanExists(spans, "Workflow"); - assertSpanExists(spans, "wrapped-step"); - var workflowTraceId = spans.stream() - .filter(s -> s.getName().equals("Workflow")) - .findFirst() - .orElseThrow() - .getTraceId(); - // Every exported span is on the one execution trace: the root and its children are sampled consistently. assertTrue( - spans.stream().allMatch(s -> s.getTraceId().equals(workflowTraceId)), - "All spans share the sampled execution trace"); + globalExporter.getFinishedSpanItems().isEmpty(), + "The installed sampler's root DROP policy must not force-sample an execution tree"); + sdkTracerProvider.close(); } // Helpers diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java new file mode 100644 index 000000000..6092a4a0a --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java @@ -0,0 +1,154 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.ReadWriteSpan; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; + +class ExecutionRootLifecycleTest { + private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/root/id"; + private static final Instant START = Instant.ofEpochSecond(10); + + @ParameterizedTest + @CsvSource({ + "false,start-failure", "true,start-failure", + "false,end-failure", "true,end-failure", + "false,fatal-root-cleanup", "true,fatal-root-cleanup", + "false,reentrant-end", "true,reentrant-end" + }) + void rootIsClosedOnceAcrossPartialStartEndFailureReentryAndReuse(boolean executionView, String mode) { + var original = Context.current(); + var plugin = new AtomicReference(); + var failOnce = new AtomicBoolean(true); + var rootFatalOnce = new AtomicBoolean(true); + var inRootCallback = new AtomicBoolean(); + var rootsEnded = new AtomicInteger(); + var flushes = new AtomicInteger(); + var starts = new ArrayList(); + var roots = new ArrayList(); + var primary = new AssertionError("invocation processor failure"); + var cleanupFatal = new InternalError("root cleanup fatal"); + var observer = new SpanProcessor() { + public void onStart(Context parent, ReadWriteSpan span) { + starts.add(span.getName()); + if (span.getName().equals("DurableExecutionRoot")) roots.add(span); + if (span.getName().equals("Invocation") + && mode.equals("start-failure") + && failOnce.compareAndSet(true, false)) throw primary; + } + + public boolean isStartRequired() { + return true; + } + + public void onEnd(ReadableSpan span) { + if (span.getName().equals("Invocation") + && mode.contains("failure") + && !mode.equals("start-failure") + && failOnce.compareAndSet(true, false)) throw primary; + if (span.getName().equals("Invocation") + && mode.equals("fatal-root-cleanup") + && failOnce.compareAndSet(true, false)) throw primary; + if (!span.getName().equals("DurableExecutionRoot")) return; + rootsEnded.incrementAndGet(); + inRootCallback.set(true); + try { + if (mode.equals("reentrant-end")) plugin.get().onInvocationEnd(end(false)); + if (mode.equals("fatal-root-cleanup") && rootFatalOnce.compareAndSet(true, false)) + throw cleanupFatal; + } finally { + inRootCallback.set(false); + } + } + + public boolean isEndRequired() { + return true; + } + + public CompletableResultCode forceFlush() { + assertFalse(inRootCallback.get(), "Reentrant End must not flush recursively from root end callbacks"); + flushes.incrementAndGet(); + return CompletableResultCode.ofSuccess(); + } + }; + try (var exporter = InMemorySpanExporter.create()) { + var builder = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .addSpanProcessor(observer); + var config = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(() -> null) + .build(); + plugin.set( + executionView + ? new ExecutionOtelPlugin(builder, config) + : new InvocationOtelPlugin(builder, config)); + var first = new InvocationInfo("first", ARN, true, START); + if (mode.equals("start-failure")) + assertSame( + primary, + assertThrows(AssertionError.class, () -> plugin.get().onInvocationStart(first))); + else plugin.get().onInvocationStart(first); + assertEquals("DurableExecutionRoot", starts.get(0)); + assertTrue(roots.get(0).isRecording(), "The root is retained until invocation end"); + if (mode.equals("end-failure")) + assertSame( + primary, + assertThrows(AssertionError.class, () -> plugin.get().onInvocationEnd(end(true)))); + else if (mode.equals("fatal-root-cleanup")) { + assertSame( + cleanupFatal, + assertThrows(InternalError.class, () -> plugin.get().onInvocationEnd(end(true)))); + assertEquals(List.of(primary), List.of(cleanupFatal.getSuppressed())); + } else plugin.get().onInvocationEnd(end(true)); + assertEquals(1, rootsEnded.get()); + assertFalse(roots.get(0).isRecording()); + assertEquals( + mode.equals("fatal-root-cleanup") ? 0 : 1, + flushes.get(), + "Normal and partial-start cleanup flush only after the root ends; fatal root cleanup escapes"); + var beforeDuplicateEnd = flushes.get(); + plugin.get().onInvocationEnd(end(true)); + assertEquals(beforeDuplicateEnd, flushes.get()); + assertEquals(1, rootsEnded.get()); + + plugin.get().onInvocationStart(new InvocationInfo("second", ARN, false, START)); + plugin.get().onInvocationEnd(end(false)); + assertEquals(2, rootsEnded.get()); + assertEquals(roots.get(0).getSpanContext(), roots.get(1).getSpanContext()); + var exportedRoots = exporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals("DurableExecutionRoot")) + .toList(); + assertEquals(2, exportedRoots.size()); + for (var root : exportedRoots) { + assertEquals(10_000_000_000L, root.getStartEpochNanos()); + assertEquals(root.getStartEpochNanos(), root.getEndEpochNanos()); + } + assertSame(original, Context.current()); + } + } + + private static InvocationEndInfo end(boolean first) { + return new InvocationEndInfo("request", ARN, first, InvocationStatus.PENDING, null); + } +} From fdc1d6d9d9c0069ddd31c8ce93839daa2d8ac6b7 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 8 Oct 2026 21:27:28 +0000 Subject: [PATCH 30/40] fix: preserve owned continuation failures and rejected admission --- otel-plugin/README.md | 12 + .../ContinuationFailurePublicProbeTest.java | 241 ++++++++++++++++++ .../ContinuationFatalPublicProbeTest.java | 200 +++++++++++++++ ...nrecoverableDurableExecutionException.java | 7 +- .../durable/execution/ExecutionManager.java | 40 ++- .../operation/BaseDurableOperation.java | 17 +- .../execution/ExecutionManagerTest.java | 159 ++++++++++++ 7 files changed, 670 insertions(+), 6 deletions(-) create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFailurePublicProbeTest.java create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFatalPublicProbeTest.java diff --git a/otel-plugin/README.md b/otel-plugin/README.md index e50681722..f2cec58fe 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -13,6 +13,18 @@ OpenTelemetry instrumentation plugin for the AWS Lambda Durable Execution SDK fo - **ADOT Java Agent Integration**: `new InvocationOtelPlugin()` late-binds the ADOT Java agent's global provider with no handler-side OpenTelemetry initialization - **Lambda Layer Discovery**: `DURABLE_EXECUTION_PLUGINS` loads either OTel plugin from a JAR under a layer's `java/lib` directory +## Checkpoint continuation failures + +An operation's SDK checkpoint continuation reports resumption/deserialization and dispatch failures through +retryable invocation control before releasing its activity lease. End reports `RETRYING`, the caller receives +`UnrecoverableDurableExecutionException` with the original cause, and persisted operation state remains available +for a later invocation. Rejected worker admission releases its activity registration. Direct `VirtualMachineError` +and `ThreadDeath` also settle the continuation observation before escaping its coordinator worker. + +Ordinary unowned helper failures retain their observation-only behavior. Normal manager closing does not replace +the selected outcome or stop unrelated operations. This boundary does not change handler/predicate failure +classification, hook threading, trace topology, or add general wrapped-fatal classification. + ## Installation ```xml diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFailurePublicProbeTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFailurePublicProbeTest.java new file mode 100644 index 000000000..7837527b5 --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFailurePublicProbeTest.java @@ -0,0 +1,241 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.function.BiFunction; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.config.WaitForConditionConfig; +import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.execution.ExecutionManager; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.DurableExecutionOutput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.model.WaitForConditionResult; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +class ContinuationFailurePublicProbeTest { + @SuppressWarnings("removal") + @ParameterizedTest + @CsvSource({"false,false", "false,true", "true,false", "true,true"}) + void readyResumeFailureDoesNotDisappearOrHang(boolean rejection, boolean withPlugin) throws Exception { + RuntimeException failure = rejection + ? new RejectedExecutionException("resumed worker rejected") + : new IllegalArgumentException("resumed state cannot deserialize"); + var managerForCleanup = new AtomicReference(); + var armed = new AtomicBoolean(); + var injected = new CountDownLatch(1); + var injectionOwner = new AtomicReference(); + var checks = new AtomicInteger(); + var ends = new CopyOnWriteArrayList(); + var pollEntered = new CountDownLatch(1); + var releaseReady = new CountDownLatch(1); + var firstArmedPoll = new AtomicBoolean(true); + var onCoordinator = new AtomicBoolean(); + var pendingCall = new AtomicReference>(); + var serde = new SerDes() { + private final JacksonSerDes delegate = new JacksonSerDes(); + + public String serialize(Object value) { + return delegate.serialize(value); + } + + public T deserialize(String value, TypeToken type) { + if (!rejection + && armed.get() + && Thread.currentThread().getName().startsWith("durable-sdk-internal-")) { + injectionOwner.set(Thread.currentThread().getName()); + onCoordinator.set(Arrays.stream(Thread.currentThread().getStackTrace()) + .anyMatch(frame -> frame.getMethodName().equals("completeCheckpointContinuation"))); + injected.countDown(); + throw failure; + } + return delegate.deserialize(value, type); + } + }; + var client = new LocalMemoryExecutionClient() { + @Override + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List updates) { + if (armed.get() && updates.isEmpty() && firstArmedPoll.compareAndSet(true, false)) { + pollEntered.countDown(); + try { + assertTrue(releaseReady.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException interrupted) { + throw new AssertionError(interrupted); + } + } + return super.checkpoint(arn, token, updates); + } + }; + var workers = + new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue()) { + @Override + public void execute(Runnable task) { + if (rejection + && armed.get() + && Thread.currentThread().getName().startsWith("durable-sdk-internal-")) { + injectionOwner.set(Thread.currentThread().getName()); + onCoordinator.set(Arrays.stream( + Thread.currentThread().getStackTrace()) + .anyMatch(frame -> frame.getMethodName().equals("completeCheckpointContinuation"))); + injected.countDown(); + throw failure; + } + super.execute(task); + } + }; + var caller = Executors.newSingleThreadExecutor(); + var builder = DurableConfig.builder() + .withExecutorService(workers) + .withDurableExecutionClient(client) + .withCheckpointDelay(Duration.ZERO) + .withPollingStrategy(attempt -> Duration.ZERO); + if (withPlugin) + builder.withPlugins(new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + ends.add(info); + } + }); + var config = builder.build(); + var condition = WaitForConditionConfig.builder() + .initialState(1) + .serDes(serde) + .waitStrategy((state, attempt) -> Duration.ofSeconds(1)) + .build(); + BiFunction handler = (input, context) -> { + managerForCleanup.set( + ((software.amazon.lambda.durable.context.BaseContextImpl) context).getExecutionManager()); + var waiting = context.waitForConditionAsync( + "condition", + Integer.class, + (state, step) -> { + checks.incrementAndGet(); + return state == 2 + ? WaitForConditionResult.stopPolling(state) + : WaitForConditionResult.continuePolling(2); + }, + condition); + if (armed.get()) { + try { + assertTrue(pollEntered.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException interrupted) { + throw new AssertionError(interrupted); + } + } + return String.valueOf(waiting.get()); + }; + try { + var first = caller.submit(() -> DurableExecutor.execute( + input(List.of()), null, TypeToken.get(String.class), handler, config)) + .get(3, TimeUnit.SECONDS); + assertEquals(ExecutionStatus.PENDING, first.status()); + var cachedPending = new ArrayList<>(client.getAllOperations()); + assertEquals( + OperationStatus.PENDING, + client.getOperationByName("condition").status()); + assertTrue(client.advanceTime()); + armed.set(true); + var second = caller.submit(() -> + DurableExecutor.execute(input(cachedPending), null, TypeToken.get(String.class), handler, config)); + pendingCall.set(second); + assertTrue(pollEntered.await(3, TimeUnit.SECONDS)); + releaseReady.countDown(); + assertTrue(injected.await(3, TimeUnit.SECONDS)); + assertTrue(onCoordinator.get(), "Fault must run in the actual WFC coordinator continuation"); + DurableExecutionOutput output = null; + Throwable callerFailure = null; + try { + output = second.get(3, TimeUnit.SECONDS); + } catch (ExecutionException observationFailure) { + callerFailure = observationFailure.getCause(); + } + System.out.println("CONTINUATION_ORDINARY_PUBLIC owner=" + injectionOwner.get() + + " caller=" + + (output != null + ? output.status() + : callerFailure.getClass().getName()) + + " backend=" + client.getOperationByName("condition").status() + + " checks=" + checks.get()); + assertEquals(1, checks.get(), "A failed resumed state read must not rerun the condition body"); + assertNull(output, "An SDK continuation failure must not produce a durable response"); + var retry = assertInstanceOf(UnrecoverableDurableExecutionException.class, callerFailure); + assertTrue(retry.isRetryable()); + assertSame(failure, retry.getCause()); + assertEquals( + OperationStatus.READY, + client.getOperationByName("condition").status()); + var activeField = ExecutionManager.class.getDeclaredField("activeThreads"); + activeField.setAccessible(true); + assertFalse( + ((Set) activeField.get(managerForCleanup.get())) + .contains(client.getOperationByName("condition").id()), + "A rejected operation worker must not leave a phantom activity registration"); + if (withPlugin) + assertEquals( + List.of(InvocationStatus.PENDING, InvocationStatus.RETRYING), + ends.stream().map(InvocationEndInfo::invocationStatus).toList()); + else assertTrue(ends.isEmpty()); + armed.set(false); + var third = caller.submit(() -> DurableExecutor.execute( + input(client.getAllOperations()), null, TypeToken.get(String.class), handler, config)); + pendingCall.set(third); + assertEquals( + ExecutionStatus.SUCCEEDED, third.get(3, TimeUnit.SECONDS).status()); + assertEquals(2, checks.get(), "Only the original and successful resumed predicates run"); + } finally { + releaseReady.countDown(); + // Cleanup-only escape hatch AFTER the real public invocation observation/timeout; never test behavior. + var manager = managerForCleanup.get(); + if (pendingCall.get() != null + && !pendingCall.get().isDone() + && manager != null + && !manager.isExecutionCompletedExceptionally()) { + try { + manager.terminateExecution(new UnrecoverableDurableExecutionException( + ErrorObject.builder().errorMessage("probe cleanup").build(), true)); + } catch (UnrecoverableDurableExecutionException expected) { + } + } + caller.shutdownNow(); + workers.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + private static DurableExecutionInput input(List operations) { + var execution = Operation.builder() + .id("execution") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + var history = new ArrayList(); + history.add(execution); + history.addAll(operations); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/fatal/execution", + "token", + CheckpointUpdatedExecutionState.builder().operations(history).build()); + } +} diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFatalPublicProbeTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFatalPublicProbeTest.java new file mode 100644 index 000000000..770f99579 --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationFatalPublicProbeTest.java @@ -0,0 +1,200 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.function.BiFunction; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.config.WaitForConditionConfig; +import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.DurableExecutionOutput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.model.WaitForConditionResult; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +class ContinuationFatalPublicProbeTest { + @SuppressWarnings("removal") + @ParameterizedTest + @CsvSource({"false,false", "false,true", "true,false", "true,true"}) + void readyResumeSerdeFatalDoesNotDisappearIntoPending(boolean death, boolean withPlugin) throws Exception { + Error fatal = death ? new ThreadDeath() : new InternalError("resume serde fatal"); + var armed = new AtomicBoolean(); + var injected = new CountDownLatch(1); + var escaped = new CountDownLatch(1); + var injectionOwner = new AtomicReference(); + var ownerFailure = new AtomicReference(); + var checks = new AtomicInteger(); + var ends = new CopyOnWriteArrayList(); + var pollEntered = new CountDownLatch(1); + var releaseReady = new CountDownLatch(1); + var firstArmedPoll = new AtomicBoolean(true); + var onCoordinator = new AtomicBoolean(); + var previousHandler = Thread.getDefaultUncaughtExceptionHandler(); + Thread.setDefaultUncaughtExceptionHandler((thread, failure) -> { + if (failure == fatal) { + ownerFailure.set(failure); + escaped.countDown(); + } else if (previousHandler != null) previousHandler.uncaughtException(thread, failure); + }); + var serde = new SerDes() { + private final JacksonSerDes delegate = new JacksonSerDes(); + + public String serialize(Object value) { + return delegate.serialize(value); + } + + public T deserialize(String value, TypeToken type) { + if (armed.get() && Thread.currentThread().getName().startsWith("durable-sdk-internal-")) { + injectionOwner.set(Thread.currentThread().getName()); + onCoordinator.set(Arrays.stream(Thread.currentThread().getStackTrace()) + .anyMatch(frame -> frame.getMethodName().equals("completeCheckpointContinuation"))); + injected.countDown(); + throw fatal; + } + return delegate.deserialize(value, type); + } + }; + var client = new LocalMemoryExecutionClient() { + @Override + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List updates) { + if (armed.get() && updates.isEmpty() && firstArmedPoll.compareAndSet(true, false)) { + pollEntered.countDown(); + try { + assertTrue(releaseReady.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException failure) { + throw new AssertionError(failure); + } + } + return super.checkpoint(arn, token, updates); + } + }; + var workers = Executors.newCachedThreadPool(); + var caller = Executors.newSingleThreadExecutor(); + var builder = DurableConfig.builder() + .withExecutorService(workers) + .withDurableExecutionClient(client) + .withCheckpointDelay(Duration.ZERO) + .withPollingStrategy(attempt -> Duration.ZERO); + if (withPlugin) + builder.withPlugins(new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + ends.add(info); + } + }); + var config = builder.build(); + var condition = WaitForConditionConfig.builder() + .initialState(1) + .serDes(serde) + .waitStrategy((state, attempt) -> Duration.ofSeconds(1)) + .build(); + BiFunction handler = (input, context) -> { + var waiting = context.waitForConditionAsync( + "condition", + Integer.class, + (state, step) -> { + checks.incrementAndGet(); + return state == 2 + ? WaitForConditionResult.stopPolling(state) + : WaitForConditionResult.continuePolling(2); + }, + condition); + if (armed.get()) { + try { + assertTrue(pollEntered.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException failure) { + throw new AssertionError(failure); + } + } + return String.valueOf(waiting.get()); + }; + try { + var first = caller.submit(() -> DurableExecutor.execute( + input(List.of()), null, TypeToken.get(String.class), handler, config)) + .get(3, TimeUnit.SECONDS); + assertEquals(ExecutionStatus.PENDING, first.status()); + var cachedPending = new ArrayList<>(client.getAllOperations()); + assertEquals( + OperationStatus.PENDING, + client.getOperationByName("condition").status()); + assertTrue(client.advanceTime()); + armed.set(true); + var second = caller.submit(() -> + DurableExecutor.execute(input(cachedPending), null, TypeToken.get(String.class), handler, config)); + assertTrue(pollEntered.await(3, TimeUnit.SECONDS)); + releaseReady.countDown(); + assertTrue(injected.await(3, TimeUnit.SECONDS)); + assertTrue(onCoordinator.get(), "Fault must run in the actual WFC coordinator continuation"); + DurableExecutionOutput output = null; + Throwable callerFailure = null; + try { + output = second.get(3, TimeUnit.SECONDS); + } catch (ExecutionException failure) { + callerFailure = failure.getCause(); + } + var ownerEscaped = escaped.await(300, TimeUnit.MILLISECONDS); + System.out.println("CONTINUATION_FATAL_PUBLIC owner=" + injectionOwner.get() + + " caller=" + + (output != null + ? output.status() + : callerFailure.getClass().getName()) + + " backend=" + client.getOperationByName("condition").status() + + " checks=" + checks.get() + " workerEscaped=" + ownerEscaped); + assertEquals(1, checks.get(), "A failed resumed state read must not rerun the condition body"); + assertTrue(ownerEscaped, "The actual coordinator worker must observe its JVM fatal"); + assertSame(fatal, ownerFailure.get()); + assertNull(output, "A failed coordinator must not return a durable terminal or PENDING response"); + var retry = assertInstanceOf(UnrecoverableDurableExecutionException.class, callerFailure); + assertTrue(retry.isRetryable()); + assertSame(fatal, retry.getCause()); + if (withPlugin) { + assertEquals( + List.of(InvocationStatus.PENDING, InvocationStatus.RETRYING), + ends.stream().map(InvocationEndInfo::invocationStatus).toList()); + assertSame(retry, ends.get(1).executionError()); + } else assertTrue(ends.isEmpty()); + } finally { + releaseReady.countDown(); + Thread.setDefaultUncaughtExceptionHandler(previousHandler); + caller.shutdownNow(); + workers.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + private static DurableExecutionInput input(List operations) { + var execution = Operation.builder() + .id("execution") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + var history = new ArrayList(); + history.add(execution); + history.addAll(operations); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/fatal/execution", + "token", + CheckpointUpdatedExecutionState.builder().operations(history).build()); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/UnrecoverableDurableExecutionException.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/UnrecoverableDurableExecutionException.java index acd2fd9fc..4315ed4ac 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/exception/UnrecoverableDurableExecutionException.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/exception/UnrecoverableDurableExecutionException.java @@ -10,7 +10,12 @@ public class UnrecoverableDurableExecutionException extends DurableExecutionExce private final boolean retryable; public UnrecoverableDurableExecutionException(ErrorObject errorObject, boolean retryable) { - super(errorObject.errorMessage()); + this(errorObject, retryable, null); + } + + /** Creates an invocation control failure while retaining its original in-process cause. */ + public UnrecoverableDurableExecutionException(ErrorObject errorObject, boolean retryable, Throwable cause) { + super(errorObject.errorMessage(), cause); this.errorObject = errorObject; this.retryable = retryable; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java index 9f260a688..aca7b654e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java @@ -22,6 +22,7 @@ import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.lambda.model.ErrorObject; import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationStatus; import software.amazon.awssdk.services.lambda.model.OperationType; @@ -579,28 +580,59 @@ CompletableFuture runCheckpointContinuation( } try { var completion = new CompletableFuture(); - coordinator.execute(() -> completeCheckpointContinuation(registration, continuation, completion)); + coordinator.execute(() -> completeCheckpointContinuation(owner, registration, continuation, completion)); return completion; } catch (RuntimeException | Error failure) { - finishCheckpointContinuation(registration); + try { + if ((owner != null && !isClosing()) + || failure instanceof VirtualMachineError + || failure instanceof ThreadDeath) signalContinuationFailure(failure); + } finally { + finishCheckpointContinuation(registration); + } throw failure; } } + @SuppressWarnings("removal") private void completeCheckpointContinuation( - Object registration, Runnable continuation, CompletableFuture completion) { + BaseDurableOperation owner, + Object registration, + Runnable continuation, + CompletableFuture completion) { try { try { if (!isClosing()) continuation.run(); + } catch (Throwable failure) { + // An operation polling chain can ignore observation. Select retry control before the final lease + // could select PENDING; unowned ordinary helper failures retain their observation-only contract. + if ((owner != null && !isClosing()) + || failure instanceof VirtualMachineError + || failure instanceof ThreadDeath) signalContinuationFailure(failure); + throw failure; } finally { finishCheckpointContinuation(registration); } completion.complete(null); } catch (Throwable failure) { completion.completeExceptionally(failure); + // The observation future may be ignored by a polling chain. Never hide a JVM fatal from its worker. + if (failure instanceof VirtualMachineError fatal) throw fatal; + if (failure instanceof ThreadDeath fatal) throw fatal; } } + private void signalContinuationFailure(Throwable failure) { + var control = new UnrecoverableDurableExecutionException( + ErrorObject.builder() + .errorType(failure.getClass().getName()) + .errorMessage("Error in SDK checkpoint continuation") + .build(), + true, + failure); + if (executionExceptionFuture.completeExceptionally(control)) stopAllOperations(control); + } + private Object registerCheckpointContinuation(BaseDurableOperation owner) { synchronized (activeThreads) { if (closing || executionExceptionFuture.isDone()) return null; @@ -613,6 +645,8 @@ private Object registerCheckpointContinuation(BaseDurableOperation owner) { private void finishCheckpointContinuation(Object registration) { synchronized (activeThreads) { + // An inline coordinator may rethrow after its task already released this registration. + if (!checkpointContinuations.containsKey(registration)) return; checkpointContinuations.remove(registration); try { finishCheckpointProcessing(); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java index a1514e6d5..bdb7fd11a 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java @@ -321,8 +321,21 @@ protected void runUserHandler(Runnable runnable, ThreadType threadType) { // registerActiveThread is idempotent (no-op if already registered). registerActiveThread(operationId); - runningUserHandler.set(CompletableFuture.runAsync( - wrapped, getContext().getDurableConfig().getExecutorService())); + try { + runningUserHandler.set(CompletableFuture.runAsync( + wrapped, getContext().getDurableConfig().getExecutorService())); + } catch (RuntimeException | Error dispatchFailure) { + // Admission failed, so no worker will execute its deregistration finally block. The submitting + // coordinator need not have a user ThreadContext; undo the registration directly on the manager. + if (operationId != null) { + try { + executionManager.deregisterActiveThread(operationId); + } catch (SuspendExecutionException ignored) { + // Preserve the dispatch failure; any suspension was already signaled by deregistration. + } + } + throw dispatchFailure; + } } /** diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java index 65d961f36..776459913 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java @@ -15,13 +15,16 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; import software.amazon.awssdk.services.lambda.model.GetDurableExecutionStateResponse; @@ -32,6 +35,7 @@ import software.amazon.lambda.durable.TestUtils; import software.amazon.lambda.durable.client.DurableExecutionClient; import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; import software.amazon.lambda.durable.model.DurableExecutionInput; import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.model.OperationSubType; @@ -55,6 +59,161 @@ private ExecutionManager createManager(List operations) { null); } + @SuppressWarnings("removal") + @ParameterizedTest + @CsvSource({"false,false", "false,true", "true,false", "true,true"}) + void continuationFatalSettlesOriginalBeforeEscapingWorker(boolean death, boolean rootActive) throws Exception { + Error fatal = death ? new ThreadDeath() : new InternalError("coordinator fatal"); + var escaped = new CountDownLatch(1); + var release = new CountDownLatch(1); + var ownerFailure = new AtomicReference(); + var observation = new AtomicReference>(); + var settledAtEscape = new AtomicBoolean(); + var executor = Executors.newSingleThreadExecutor(task -> { + var thread = new Thread(task, "continuation-fatal-probe"); + thread.setUncaughtExceptionHandler((owner, failure) -> { + ownerFailure.set(failure); + settledAtEscape.set(observation.get().isDone()); + escaped.countDown(); + }); + return thread; + }); + try (var manager = createManager(List.of(executionOp(), stepOp("pending", OperationStatus.PENDING)))) { + if (rootActive) manager.registerActiveThread("root-probe"); + var future = manager.runCheckpointContinuation( + () -> { + try { + assertTrue(release.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException interrupted) { + throw new AssertionError(interrupted); + } + throw fatal; + }, + executor); + observation.set(future); + var closedFromObserver = new AtomicBoolean(); + future.whenComplete((ignored, failure) -> { + manager.close(); // Must not wait for this callback's own continuation registration. + closedFromObserver.set(true); + }); + release.countDown(); + assertSame( + fatal, + assertThrows(ExecutionException.class, () -> future.get(3, TimeUnit.SECONDS)) + .getCause()); + assertTrue(escaped.await(2, TimeUnit.SECONDS), "Fatal must escape the actual coordinator worker"); + assertSame(fatal, ownerFailure.get()); + assertTrue(settledAtEscape.get(), "Observation must settle before worker escape"); + assertTrue(closedFromObserver.get(), "The activity lease must be released before observer callbacks"); + } finally { + release.countDown(); + executor.shutdownNow(); + assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @SuppressWarnings("removal") + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void inlineFatalContinuationReleasesItsRegistrationOnlyOnce(boolean death) { + Error fatal = death ? new ThreadDeath() : new InternalError("inline coordinator fatal"); + try (var manager = createManager(List.of(executionOp()))) { + manager.registerActiveThread("root"); + assertSame( + fatal, + assertThrows( + Error.class, + () -> manager.runCheckpointContinuation( + () -> { + throw fatal; + }, + Runnable::run))); + assertTrue(manager.isExecutionCompletedExceptionally()); + } + } + + @Test + void ordinaryContinuationFailureRetainsItsExistingObservationPolicy() throws Exception { + var original = new IllegalArgumentException("ordinary continuation"); + var worker = new AtomicReference(); + var executor = Executors.newSingleThreadExecutor(); + try (var manager = createManager(List.of(executionOp()))) { + manager.registerActiveThread("root"); + var result = manager.runCheckpointContinuation( + () -> { + worker.set(Thread.currentThread()); + throw original; + }, + executor); + assertSame( + original, + assertThrows(ExecutionException.class, () -> result.get(3, TimeUnit.SECONDS)) + .getCause()); + assertFalse(manager.isExecutionCompletedExceptionally()); + assertSame(worker.get(), executor.submit(Thread::currentThread).get(3, TimeUnit.SECONDS)); + } finally { + executor.shutdownNow(); + assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void ownedFailureReachesExecutionDespiteCancelledObservation(boolean cancelled) throws Exception { + var manager = createManager(List.of(executionOp(), stepOp("step", OperationStatus.PENDING))); + manager.registerActiveThread("root"); + var owner = mock(BaseDurableOperation.class); + when(owner.getCompletionFuture()).thenReturn(new CompletableFuture<>()); + var queued = new AtomicReference(); + var failure = new IllegalArgumentException("owned continuation failure"); + var selected = manager.runUntilCompleteOrSuspend(new CompletableFuture<>()); + var observation = manager.runCheckpointContinuation( + owner, + () -> { + throw failure; + }, + queued::set); + if (cancelled) assertTrue(observation.cancel(false)); + queued.get().run(); + var control = assertInstanceOf( + UnrecoverableDurableExecutionException.class, + assertThrows(ExecutionException.class, () -> selected.get(3, TimeUnit.SECONDS)) + .getCause()); + assertTrue(control.isRetryable()); + assertSame(failure, control.getCause()); + if (cancelled) assertTrue(observation.isCancelled()); + else + assertSame( + failure, + assertThrows(ExecutionException.class, () -> observation.get(3, TimeUnit.SECONDS)) + .getCause()); + CompletableFuture.runAsync(manager::close).get(3, TimeUnit.SECONDS); + } + + @Test + void ownedCoordinatorRejectionSettlesExecutionAndReleasesAdmission() throws Exception { + var manager = createManager(List.of(executionOp(), stepOp("step", OperationStatus.PENDING))); + manager.registerActiveThread("root"); + var owner = mock(BaseDurableOperation.class); + when(owner.getCompletionFuture()).thenReturn(new CompletableFuture<>()); + var selected = manager.runUntilCompleteOrSuspend(new CompletableFuture<>()); + var rejection = new RejectedExecutionException("coordinator admission rejected"); + assertSame( + rejection, + assertThrows( + RejectedExecutionException.class, + () -> manager.runCheckpointContinuation(owner, () -> fail("not admitted"), task -> { + throw rejection; + }))); + var control = assertInstanceOf( + UnrecoverableDurableExecutionException.class, + assertThrows(ExecutionException.class, () -> selected.get(3, TimeUnit.SECONDS)) + .getCause()); + assertTrue(control.isRetryable()); + assertSame(rejection, control.getCause()); + CompletableFuture.runAsync(manager::close).get(3, TimeUnit.SECONDS); + } + private Operation executionOp() { return Operation.builder() .id(EXECUTION_OP_ID) From 3b2cd214e832b63bca2b2aa9d110ca303d8b9c96 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 8 Oct 2026 22:46:16 +0000 Subject: [PATCH 31/40] fix: wake selected continuation waiters before legacy End callbacks --- otel-plugin/README.md | 4 + .../ContinuationEndCleanupOrderTest.java | 413 ++++++++++++++++++ .../durable/execution/DurableExecutor.java | 1 + .../durable/execution/ExecutionManager.java | 21 +- .../execution/ExecutionManagerTest.java | 158 +++++++ 5 files changed, 596 insertions(+), 1 deletion(-) create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationEndCleanupOrderTest.java diff --git a/otel-plugin/README.md b/otel-plugin/README.md index f2cec58fe..c1b13f084 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -21,6 +21,10 @@ retryable invocation control before releasing its activity lease. End reports `R for a later invocation. Rejected worker admission releases its activity registration. Direct `VirtualMachineError` and `ThreadDeath` also settle the continuation observation before escaping its coordinator worker. +Legacy outcome observers wake waiters for an already-selected continuation failure before invocation End can wait +for handler cleanup. The existing callback thread is retained, and concurrent publishers use the selected control +cause. Normal outcome handling and diagnostic inspection are unchanged. + Ordinary unowned helper failures retain their observation-only behavior. Normal manager closing does not replace the selected outcome or stop unrelated operations. This boundary does not change handler/predicate failure classification, hook threading, trace topology, or add general wrapped-fatal classification. diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationEndCleanupOrderTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationEndCleanupOrderTest.java new file mode 100644 index 000000000..0778afe54 --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationEndCleanupOrderTest.java @@ -0,0 +1,413 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.function.BiFunction; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.config.WaitForConditionConfig; +import software.amazon.lambda.durable.context.BaseContextImpl; +import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.execution.ExecutionManager; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.DurableExecutionOutput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.model.WaitForConditionResult; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; +import software.amazon.lambda.durable.util.ExceptionHelper; + +class ContinuationEndCleanupOrderTest { + static Stream cleanupCases() { + return Stream.of("async", "inline-root", "submit-root") + .flatMap(mode -> Stream.of("serde", "reject", "vm", "death").map(kind -> Arguments.of(mode, kind))); + } + + @SuppressWarnings("removal") + @ParameterizedTest + @MethodSource("cleanupCases") + void continuationFailureMustWakeBodyBeforeEndWaitsForCleanup(String mode, String kind) throws Exception { + boolean rejection = kind.equals("reject"); + boolean withPlugin = true; + Throwable failure = + switch (kind) { + case "reject" -> new RejectedExecutionException("resumed worker rejected"); + case "vm" -> new InternalError("resumed state fatal"); + case "death" -> new ThreadDeath(); + default -> new IllegalArgumentException("resumed state cannot deserialize"); + }; + var injectionThread = new AtomicReference(); + var callerThread = new AtomicReference(); + var escaped = new CountDownLatch(1); + var escapedFailure = new AtomicReference(); + var oldHandler = Thread.getDefaultUncaughtExceptionHandler(); + Thread.setDefaultUncaughtExceptionHandler((thread, error) -> { + if (error == failure) { + escapedFailure.set(error); + escaped.countDown(); + } else if (oldHandler != null) oldHandler.uncaughtException(thread, error); + }); + var managerForCleanup = new AtomicReference(); + var armed = new AtomicBoolean(); + var injected = new CountDownLatch(1); + var injectionOwner = new AtomicReference(); + var checks = new AtomicInteger(); + var bodyFinally = new CountDownLatch(1); + var cleanupOrder = new CopyOnWriteArrayList(); + var ends = new CopyOnWriteArrayList(); + var pollEntered = new CountDownLatch(1); + var releaseReady = new CountDownLatch(1); + var firstArmedPoll = new AtomicBoolean(true); + var onCoordinator = new AtomicBoolean(); + var pendingCall = new AtomicReference>(); + var serde = new SerDes() { + private final JacksonSerDes delegate = new JacksonSerDes(); + + public String serialize(Object value) { + return delegate.serialize(value); + } + + public T deserialize(String value, TypeToken type) { + if (!rejection + && armed.get() + && Thread.currentThread().getName().startsWith("durable-sdk-internal-")) { + injectionThread.set(Thread.currentThread()); + injectionOwner.set(Thread.currentThread().getName()); + onCoordinator.set(Arrays.stream(Thread.currentThread().getStackTrace()) + .anyMatch(frame -> frame.getMethodName().equals("completeCheckpointContinuation"))); + injected.countDown(); + ExceptionHelper.sneakyThrow(failure); + } + return delegate.deserialize(value, type); + } + }; + var client = new LocalMemoryExecutionClient() { + @Override + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List updates) { + if (armed.get() && updates.isEmpty() && firstArmedPoll.compareAndSet(true, false)) { + pollEntered.countDown(); + try { + assertTrue(releaseReady.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException interrupted) { + throw new AssertionError(interrupted); + } + } + return super.checkpoint(arn, token, updates); + } + }; + var workers = new RootExecutor(mode, () -> { + if (rejection && armed.get() && Thread.currentThread().getName().startsWith("durable-sdk-internal-")) { + injectionThread.set(Thread.currentThread()); + injectionOwner.set(Thread.currentThread().getName()); + onCoordinator.set(Arrays.stream(Thread.currentThread().getStackTrace()) + .anyMatch(frame -> frame.getMethodName().equals("completeCheckpointContinuation"))); + injected.countDown(); + ExceptionHelper.sneakyThrow(failure); + } + }); + var caller = Executors.newSingleThreadExecutor(task -> { + var thread = new Thread(task, "legacy-outcome-caller"); + callerThread.set(thread); + return thread; + }); + var builder = DurableConfig.builder() + .withExecutorService(workers) + .withDurableExecutionClient(client) + .withCheckpointDelay(Duration.ZERO) + .withPollingStrategy(attempt -> Duration.ZERO); + if (withPlugin) + builder.withPlugins(new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + ends.add(info); + if (info.invocationStatus() == InvocationStatus.RETRYING) { + assertSame( + mode.equals("async") ? injectionThread.get() : callerThread.get(), + Thread.currentThread(), + "The repair must preserve the legacy End completion thread"); + cleanupOrder.add("end-enter"); + try { + boolean finished = bodyFinally.await(2, TimeUnit.SECONDS); + cleanupOrder.add(finished ? "end-complete" : "end-timed-out"); + assertTrue(finished, "End must not prevent the manager from waking handler cleanup"); + } catch (InterruptedException interrupted) { + throw new AssertionError(interrupted); + } + } + } + }); + var config = builder.build(); + var condition = WaitForConditionConfig.builder() + .initialState(1) + .serDes(serde) + .waitStrategy((state, attempt) -> Duration.ofSeconds(1)) + .build(); + BiFunction handler = (input, context) -> { + managerForCleanup.set(((BaseContextImpl) context).getExecutionManager()); + var waiting = context.waitForConditionAsync( + "condition", + Integer.class, + (state, step) -> { + checks.incrementAndGet(); + return state == 2 + ? WaitForConditionResult.stopPolling(state) + : WaitForConditionResult.continuePolling(2); + }, + condition); + if (armed.get()) { + try { + assertTrue(pollEntered.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException interrupted) { + throw new AssertionError(interrupted); + } + } + try { + return String.valueOf(waiting.get()); + } finally { + if (armed.get()) { + cleanupOrder.add("body-finally"); + bodyFinally.countDown(); + } + } + }; + try { + var first = caller.submit(() -> DurableExecutor.execute( + input(List.of()), null, TypeToken.get(String.class), handler, config)) + .get(3, TimeUnit.SECONDS); + assertEquals(ExecutionStatus.PENDING, first.status()); + var cachedPending = new ArrayList<>(client.getAllOperations()); + assertEquals( + OperationStatus.PENDING, + client.getOperationByName("condition").status()); + assertTrue(client.advanceTime()); + armed.set(true); + workers.nextInvocation(); + var second = caller.submit(() -> + DurableExecutor.execute(input(cachedPending), null, TypeToken.get(String.class), handler, config)); + pendingCall.set(second); + assertTrue(pollEntered.await(3, TimeUnit.SECONDS)); + if (mode.equals("async")) awaitCallerOutcomeJoin(callerThread.get()); + releaseReady.countDown(); + assertTrue(injected.await(3, TimeUnit.SECONDS)); + assertTrue(onCoordinator.get(), "Fault must run in the actual WFC coordinator continuation"); + DurableExecutionOutput output = null; + Throwable callerFailure = null; + try { + output = second.get(5, TimeUnit.SECONDS); + } catch (ExecutionException observationFailure) { + callerFailure = observationFailure.getCause(); + } + System.out.println("CONTINUATION_ORDINARY_PUBLIC owner=" + injectionOwner.get() + + " caller=" + + (output != null + ? output.status() + : callerFailure.getClass().getName()) + + " backend=" + client.getOperationByName("condition").status() + + " checks=" + checks.get()); + assertEquals(1, checks.get(), "A failed resumed state read must not rerun the condition body"); + System.out.println("CONTINUATION_END_CLEANUP_ORDER " + cleanupOrder); + assertFalse(cleanupOrder.contains("end-timed-out")); + assertTrue(cleanupOrder.contains("end-complete")); + assertTrue(cleanupOrder.indexOf("body-finally") < cleanupOrder.indexOf("end-complete")); + if (failure instanceof Error) { + assertTrue(escaped.await(3, TimeUnit.SECONDS)); + assertSame(failure, escapedFailure.get()); + } + assertNull(output, "An SDK continuation failure must not produce a durable response"); + var retry = assertInstanceOf(UnrecoverableDurableExecutionException.class, callerFailure); + assertTrue(retry.isRetryable()); + assertSame(failure, retry.getCause()); + assertEquals( + OperationStatus.READY, + client.getOperationByName("condition").status()); + var activeField = ExecutionManager.class.getDeclaredField("activeThreads"); + activeField.setAccessible(true); + assertFalse( + ((Set) activeField.get(managerForCleanup.get())) + .contains(client.getOperationByName("condition").id()), + "A rejected operation worker must not leave a phantom activity registration"); + if (withPlugin) + assertEquals( + List.of(InvocationStatus.PENDING, InvocationStatus.RETRYING), + ends.stream().map(InvocationEndInfo::invocationStatus).toList()); + else assertTrue(ends.isEmpty()); + armed.set(false); + workers.nextInvocation(); + var third = caller.submit(() -> DurableExecutor.execute( + input(client.getAllOperations()), null, TypeToken.get(String.class), handler, config)); + pendingCall.set(third); + assertEquals( + ExecutionStatus.SUCCEEDED, third.get(3, TimeUnit.SECONDS).status()); + assertEquals(2, checks.get(), "Only the original and successful resumed predicates run"); + } finally { + releaseReady.countDown(); + // Cleanup-only escape hatch AFTER the real public invocation observation/timeout; never test behavior. + var manager = managerForCleanup.get(); + if (pendingCall.get() != null + && !pendingCall.get().isDone() + && manager != null + && !manager.isExecutionCompletedExceptionally()) { + try { + manager.terminateExecution(new UnrecoverableDurableExecutionException( + ErrorObject.builder().errorMessage("probe cleanup").build(), true)); + } catch (UnrecoverableDurableExecutionException expected) { + } + } + Thread.setDefaultUncaughtExceptionHandler(oldHandler); + caller.shutdownNow(); + workers.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @ParameterizedTest + @ValueSource(strings = {"async", "inline-root", "submit-root"}) + void ordinaryCompletionKeepsItsExistingEndThread(String mode) throws Exception { + var releaseBody = new CountDownLatch(1); + var entered = new CountDownLatch(1); + var callerThread = new AtomicReference(); + var bodyThread = new AtomicReference(); + var endThread = new AtomicReference(); + var workers = new RootExecutor(mode, () -> {}); + var caller = Executors.newSingleThreadExecutor(task -> { + var thread = new Thread(task, "normal-outcome-caller"); + callerThread.set(thread); + return thread; + }); + var config = DurableConfig.builder() + .withExecutorService(workers) + .withDurableExecutionClient(new LocalMemoryExecutionClient()) + .withPlugins(new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + endThread.set(Thread.currentThread()); + } + }) + .build(); + try { + var call = caller.submit(() -> DurableExecutor.execute( + input(List.of()), + null, + TypeToken.get(String.class), + (value, context) -> { + bodyThread.set(Thread.currentThread()); + entered.countDown(); + if (mode.equals("async")) { + try { + assertTrue(releaseBody.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException interrupted) { + throw new AssertionError(interrupted); + } + } + return "done"; + }, + config)); + assertTrue(entered.await(3, TimeUnit.SECONDS)); + if (mode.equals("async")) awaitCallerOutcomeJoin(callerThread.get()); + releaseBody.countDown(); + assertEquals( + ExecutionStatus.SUCCEEDED, call.get(3, TimeUnit.SECONDS).status()); + assertSame(mode.equals("async") ? bodyThread.get() : callerThread.get(), endThread.get()); + } finally { + releaseBody.countDown(); + caller.shutdownNow(); + workers.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + /** Wait for the actual public invocation caller to install its outcome observer and enter join, not a delay. */ + private static void awaitCallerOutcomeJoin(Thread caller) { + var deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (System.nanoTime() < deadline) { + if (Arrays.stream(caller.getStackTrace()) + .anyMatch(frame -> frame.getClassName().equals(CompletableFuture.class.getName()) + && frame.getMethodName().equals("join"))) return; + Thread.yield(); + } + fail("Invocation caller did not reach its installed outcome observer"); + } + + /** Root dispatch exercises inline/submit-and-wait; operation children remain asynchronous as supported. */ + private static final class RootExecutor extends ThreadPoolExecutor { + private final String mode; + private final Runnable beforeDispatch; + private final AtomicBoolean root = new AtomicBoolean(true); + + RootExecutor(String mode, Runnable beforeDispatch) { + super(0, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue<>()); + this.mode = mode; + this.beforeDispatch = beforeDispatch; + } + + void nextInvocation() { + root.set(true); + } + + @Override + public void execute(Runnable task) { + beforeDispatch.run(); + if (root.compareAndSet(true, false)) { + if (mode.equals("inline-root")) { + task.run(); + return; + } + if (mode.equals("submit-root")) { + var done = new CompletableFuture(); + super.execute(() -> { + try { + task.run(); + } finally { + done.complete(null); + } + }); + try { + done.get(5, TimeUnit.SECONDS); + } catch (Exception failure) { + throw new AssertionError(failure); + } + return; + } + } + super.execute(task); + } + } + + private static DurableExecutionInput input(List operations) { + var execution = Operation.builder() + .id("execution") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + var history = new ArrayList(); + history.add(execution); + history.addAll(operations); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/fatal/execution", + "token", + CheckpointUpdatedExecutionState.builder().operations(history).build()); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index d8db91326..168f579c0 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -124,6 +124,7 @@ public static DurableExecutionOutput execute( if (ex != null) { // an exception thrown from handlerFuture or suspension/termination occurred Throwable cause = ExceptionHelper.unwrapCompletableFuture(ex); + executionManager.wakePublishedContinuationWaiters(cause); // return PENDING if it's SuspendExecutionException if (cause instanceof SuspendExecutionException) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java index aca7b654e..2f1f109bf 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java @@ -9,6 +9,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -64,6 +65,9 @@ public class ExecutionManager implements SafeCloseable { // ===== Execution State ===== private final Map operationStorage; + // Only populated while a continuation failure is synchronously publishing its selected outcome. + private final Map continuationFailurePublishers = + Collections.synchronizedMap(new IdentityHashMap<>()); private final Operation executionOp; private final String durableExecutionArn; private final Context lambdaContext; @@ -630,7 +634,22 @@ private void signalContinuationFailure(Throwable failure) { .build(), true, failure); - if (executionExceptionFuture.completeExceptionally(control)) stopAllOperations(control); + continuationFailurePublishers.put(control, Thread.currentThread()); + try { + if (executionExceptionFuture.completeExceptionally(control)) stopAllOperations(control); + } finally { + continuationFailurePublishers.remove(control); + } + } + + /** + * A legacy outcome observer may run inside completeExceptionally, before its publisher returns to wake waiters. The + * outcome is already selected here. Wake those waiters before End can wait for their handler cleanup. + */ + void wakePublishedContinuationWaiters(Throwable selectedFailure) { + // A losing completeExceptionally call may drain the winner's callback on the losing publisher's thread. + // Match the already-selected control by identity, not the thread currently running the callback. + if (continuationFailurePublishers.containsKey(selectedFailure)) stopAllOperations(selectedFailure); } private Object registerCheckpointContinuation(BaseDurableOperation owner) { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java index 776459913..14b120659 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java @@ -12,6 +12,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -40,6 +41,7 @@ import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.operation.BaseDurableOperation; +import software.amazon.lambda.durable.util.ExceptionHelper; class ExecutionManagerTest { private static final String EXECUTION_OP_ID = "01234567-0123-0123-0123-012345678901"; @@ -214,6 +216,162 @@ void ownedCoordinatorRejectionSettlesExecutionAndReleasesAdmission() throws Exce CompletableFuture.runAsync(manager::close).get(3, TimeUnit.SECONDS); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void legacyOutcomeObserverWakesItsOwnerAndClearsPublisherAfterReentry(boolean observerThrows) throws Exception { + var manager = createManager(List.of(executionOp(), stepOp("step", OperationStatus.PENDING))); + manager.registerActiveThread("root"); + var owner = mock(BaseDurableOperation.class); + var ownerCompletion = new CompletableFuture(); + when(owner.getOperationId()).thenReturn("owner"); + when(owner.getCompletionFuture()).thenReturn(ownerCompletion); + manager.registerOperation(owner); + var failure = new IllegalArgumentException("continuation"); + var observerFailure = new AssertionError("observer failure"); + var selectedControl = new AtomicReference(); + var publisher = Thread.currentThread(); + var selected = manager.runUntilCompleteOrSuspend(new CompletableFuture<>()) + .handle((value, error) -> { + assertSame(publisher, Thread.currentThread(), "No finalizer thread change"); + manager.runCheckpointContinuation(() -> fail("No post-selection reentrant work"), Runnable::run) + .join(); + selectedControl.set(ExceptionHelper.unwrapCompletableFuture(error)); + manager.wakePublishedContinuationWaiters(selectedControl.get()); + var control = assertInstanceOf( + UnrecoverableDurableExecutionException.class, + assertThrows(CompletionException.class, ownerCompletion::join) + .getCause()); + assertSame(failure, control.getCause()); + if (observerThrows) throw observerFailure; + return "observed"; + }); + var observation = manager.runCheckpointContinuation( + owner, + () -> { + throw failure; + }, + Runnable::run); + assertSame( + failure, + assertThrows(CompletionException.class, observation::join).getCause()); + if (observerThrows) + assertSame( + observerFailure, + assertThrows(CompletionException.class, selected::join).getCause()); + else assertEquals("observed", selected.join()); + var unrelated = mock(BaseDurableOperation.class); + var unrelatedCompletion = new CompletableFuture(); + when(unrelated.getOperationId()).thenReturn("later"); + when(unrelated.getCompletionFuture()).thenReturn(unrelatedCompletion); + manager.registerOperation(unrelated); + manager.wakePublishedContinuationWaiters(selectedControl.get()); + assertFalse(unrelatedCompletion.isDone(), "A retained publisher marker would spuriously stop later work"); + manager.close(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void losingPublisherMustWakeTheSelectedWinnerCause(boolean winnerFatal) throws Exception { + var manager = createManager(List.of(executionOp(), stepOp("step", OperationStatus.PENDING))); + manager.registerActiveThread("root"); + var owner = mock(BaseDurableOperation.class); + var ownerCompletion = new CompletableFuture(); + when(owner.getOperationId()).thenReturn("owner"); + when(owner.getCompletionFuture()).thenReturn(ownerCompletion); + manager.registerOperation(owner); + Throwable first = winnerFatal ? new InternalError("winner") : new IllegalArgumentException("winner"); + Throwable second = winnerFatal ? new IllegalArgumentException("loser") : new InternalError("loser"); + var queued = new ArrayList(); + var winnerSelected = new CountDownLatch(1); + var releaseWinner = new CountDownLatch(1); + var endSeen = new CountDownLatch(1); + var observedCause = new AtomicReference(); + var awakenedCause = new AtomicReference(); + var observerThread = new AtomicReference(); + var loserThread = new AtomicReference(); + manager.runUntilCompleteOrSuspend(new CompletableFuture<>()).handle((value, error) -> { + observerThread.set(Thread.currentThread()); + observedCause.set(ExceptionHelper.unwrapCompletableFuture(error)); + manager.wakePublishedContinuationWaiters(observedCause.get()); + awakenedCause.set( + ownerCompletion.handle((ignored, failure) -> failure).join()); + endSeen.countDown(); + return null; + }); + var field = ExecutionManager.class.getDeclaredField("executionExceptionFuture"); + field.setAccessible(true); + var channel = (CompletableFuture) field.get(manager); + // Gate after the winner's CAS but before the older outcome callback is drained. A scheduler pause can + // create the same window with a single production observer; the gate makes it deterministic here. + channel.whenComplete((value, failure) -> { + winnerSelected.countDown(); + try { + assertTrue(releaseWinner.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException interrupted) { + throw new AssertionError(interrupted); + } + }); + var firstObservation = manager.runCheckpointContinuation( + owner, + () -> { + if (first instanceof Error fatal) throw fatal; + throw (RuntimeException) first; + }, + queued::add); + var secondObservation = manager.runCheckpointContinuation( + owner, + () -> { + if (second instanceof Error fatal) throw fatal; + throw (RuntimeException) second; + }, + queued::add); + var workers = Executors.newFixedThreadPool(2); + var winnerTask = workers.submit(queued.get(0)); + try { + assertTrue(winnerSelected.await(3, TimeUnit.SECONDS)); + var winner = assertThrows(CompletionException.class, channel::join).getCause(); + var loserTask = workers.submit(() -> { + loserThread.set(Thread.currentThread()); + queued.get(1).run(); + }); + assertTrue(endSeen.await(3, TimeUnit.SECONDS)); + if (second instanceof Error) + assertSame( + second, + assertThrows(ExecutionException.class, () -> loserTask.get(3, TimeUnit.SECONDS)) + .getCause()); + else loserTask.get(3, TimeUnit.SECONDS); + assertSame( + loserThread.get(), + observerThread.get(), + "The losing completeExceptionally drains the winner callback"); + assertSame(winner, observedCause.get()); + assertSame( + winner, + awakenedCause.get(), + "Waiters must receive the already-selected control, never the losing publisher"); + } finally { + releaseWinner.countDown(); + if (first instanceof Error) + assertSame( + first, + assertThrows(ExecutionException.class, () -> winnerTask.get(3, TimeUnit.SECONDS)) + .getCause()); + else winnerTask.get(3, TimeUnit.SECONDS); + assertSame( + first, + assertThrows(CompletionException.class, firstObservation::join) + .getCause()); + assertSame( + second, + assertThrows(CompletionException.class, secondObservation::join) + .getCause()); + workers.shutdownNow(); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + manager.close(); + } + } + private Operation executionOp() { return Operation.builder() .id(EXECUTION_OP_ID) From ec27233556db0df098b756e481d12cea63fd2f74 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 8 Oct 2026 23:32:13 +0000 Subject: [PATCH 32/40] fix: stop queued continuation admission before invocation End --- otel-plugin/README.md | 6 + .../otel/InvocationEndAdmissionTest.java | 257 ++++++++++++++++++ .../durable/execution/DurableExecutor.java | 1 + .../durable/execution/ExecutionManager.java | 7 +- .../plugin/DurableExecutionPlugin.java | 7 +- .../durable/plugin/InvocationEndInfo.java | 3 +- 6 files changed, 277 insertions(+), 4 deletions(-) create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndAdmissionTest.java diff --git a/otel-plugin/README.md b/otel-plugin/README.md index c1b13f084..73760582f 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -29,6 +29,12 @@ Ordinary unowned helper failures retain their observation-only behavior. Normal the selected outcome or stop unrelated operations. This boundary does not change handler/predicate failure classification, hook threading, trace topology, or add general wrapped-fatal classification. +The invocation End snapshot is taken after closing new checkpoint-continuation admission and signaling unfinished +owned continuations. Queued work may legitimately remain incomplete after early parallel success. This does not +join all accepted/running handlers before End: their remaining cleanup/checkpoint completion is still awaited by +normal manager close afterward, and late end hooks can occur. Await operations that must be included in the selected +outcome or snapshot. The admission cut does not provide a universal all-work-drained or final-state export guarantee. + ## Installation ```xml diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndAdmissionTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndAdmissionTest.java new file mode 100644 index 000000000..9d4df8f47 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndAdmissionTest.java @@ -0,0 +1,257 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.time.Duration; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.function.Function; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableFuture; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.CompletionConfig; +import software.amazon.lambda.durable.config.ParallelConfig; +import software.amazon.lambda.durable.config.WaitForConditionConfig; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.model.WaitForConditionResult; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +class InvocationEndAdmissionTest { + @ParameterizedTest + @ValueSource(strings = {"unawaited", "awaited", "early-parallel", "already-running"}) + void queuedResumptionsStopAtEndWhileAcceptedHandlersKeepTheirCleanupBoundary(String mode) throws Exception { + boolean awaitResult = mode.equals("awaited"); + boolean alreadyRunning = mode.equals("already-running"); + var runningEntered = new CountDownLatch(1); + var releaseRunning = new CountDownLatch(1); + var nextDispatch = new CountDownLatch(1); + var releaseDispatch = new CountDownLatch(1); + var endEntered = new CountDownLatch(1); + var releaseEnd = new CountDownLatch(1); + var terminal = new CountDownLatch(1); + var predicates = new AtomicInteger(); + var lateHooks = new AtomicInteger(); + var lateStarts = new AtomicInteger(); + var ending = new AtomicBoolean(); + var snapshotStatus = new AtomicReference(); + var firstResume = new AtomicBoolean(true); + var workers = + new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue()) { + @Override + public void execute(Runnable task) { + if (Thread.currentThread().getName().startsWith("durable-sdk-internal-") + && firstResume.compareAndSet(true, false)) { + nextDispatch.countDown(); + await(releaseDispatch); + } + super.execute(task); + } + }; + var client = new LocalMemoryExecutionClient() { + @Override + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List updates) { + var applied = super.checkpoint(arn, token, updates); + advanceTime(); + var ready = super.checkpoint(arn, applied.checkpointToken(), List.of()); + var states = new LinkedHashMap(); + applied.newExecutionState().operations().forEach(op -> states.put(op.id(), op)); + ready.newExecutionState().operations().forEach(op -> states.put(op.id(), op)); + if (getOperationByName("condition") != null + && getOperationByName("condition").status() == OperationStatus.SUCCEEDED) terminal.countDown(); + return ready.toBuilder() + .newExecutionState(CheckpointUpdatedExecutionState.builder() + .operations(states.values()) + .build()) + .build(); + } + }; + var exporter = InMemorySpanExporter.create(); + var otel = new InvocationOtelPlugin( + SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)), + OtelPluginConfig.builder() + .contextExtractor(() -> null) + .enableMdc(false) + .build()); + var gate = new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + snapshotStatus.set(info.operations().values().stream() + .filter(op -> "condition".equals(op.name())) + .findFirst() + .orElseThrow() + .status()); + ending.set(true); + endEntered.countDown(); + await(releaseEnd); + } + + public void onUserFunctionStart(UserFunctionStartInfo info) { + if (ending.get()) { + lateHooks.incrementAndGet(); + lateStarts.incrementAndGet(); + } + } + + public void onUserFunctionEnd(UserFunctionEndInfo info) { + if (ending.get()) lateHooks.incrementAndGet(); + } + + public void onOperationEnd(OperationEndInfo info) { + if (ending.get()) lateHooks.incrementAndGet(); + } + }; + // This legacy branch dispatches End in registration order, so the real OTel plugin finalizes before the gate. + var config = DurableConfig.builder() + .withExecutorService(workers) + .withDurableExecutionClient(client) + .withCheckpointDelay(Duration.ZERO) + .withPollingStrategy(attempt -> Duration.ZERO) + .withPlugins(otel, gate) + .build(); + var caller = Executors.newSingleThreadExecutor(); + try { + var call = caller.submit(() -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, context) -> { + Function> startCondition = + branch -> branch.waitForConditionAsync( + "condition", + Integer.class, + (state, step) -> { + predicates.incrementAndGet(); + if (alreadyRunning && state == 2) { + runningEntered.countDown(); + await(releaseRunning); + } + return state == 1 + ? WaitForConditionResult.continuePolling(2) + : WaitForConditionResult.stopPolling(state); + }, + WaitForConditionConfig.builder() + .initialState(1) + .waitStrategy((state, attempt) -> Duration.ofSeconds(1)) + .build()); + if (mode.equals("early-parallel")) { + var parallel = context.parallel( + "early", + ParallelConfig.builder() + .completionConfig(CompletionConfig.minSuccessful(1)) + .build()); + try (parallel) { + parallel.branch( + "waiting", + Integer.class, + branch -> startCondition.apply(branch).get()); + parallel.branch("winner", String.class, branch -> { + await(nextDispatch); + return "winner"; + }); + } + assertEquals(1, parallel.get().succeeded()); + } else { + var condition = startCondition.apply(context); + await(nextDispatch); + if (alreadyRunning) await(runningEntered); + if (awaitResult) condition.get(); + } + return "root-done"; + }, + config)); + assertTrue(nextDispatch.await(3, TimeUnit.SECONDS)); + if (alreadyRunning) { + releaseDispatch.countDown(); + assertTrue(runningEntered.await(3, TimeUnit.SECONDS)); + assertTrue(endEntered.await(3, TimeUnit.SECONDS), "End does not join already-running predicates"); + releaseEnd.countDown(); + assertThrows( + TimeoutException.class, + () -> call.get(100, TimeUnit.MILLISECONDS), + "Normal close still waits for the real handler"); + releaseRunning.countDown(); + } else if (!awaitResult) assertTrue(endEntered.await(3, TimeUnit.SECONDS)); + releaseDispatch.countDown(); + if (awaitResult || alreadyRunning) assertTrue(terminal.await(3, TimeUnit.SECONDS)); + assertTrue(endEntered.await(3, TimeUnit.SECONDS)); + releaseEnd.countDown(); + assertEquals( + ExecutionStatus.SUCCEEDED, call.get(3, TimeUnit.SECONDS).status()); + var spans = exporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals("condition")) + .map(span -> + span.getName() + ":" + span.getAttributes().get(SpanAttributes.DURABLE_OPERATION_STATUS)) + .toList(); + System.out.println("END_SNAPSHOT_PUBLIC mode=" + mode + " snapshot=" + snapshotStatus.get() + + " backend=" + client.getOperationByName("condition").status() + " predicates=" + predicates.get() + + " lateHooks=" + lateHooks.get() + " lateStarts=" + lateStarts.get() + " operationSpans=" + spans); + assertEquals( + 0, + lateStarts.get(), + "An unfinished queued resumption must not start a new predicate after the End cut"); + assertEquals(awaitResult || alreadyRunning ? 2 : 1, predicates.get()); + if (alreadyRunning) { + assertNotEquals( + OperationStatus.SUCCEEDED, snapshotStatus.get(), "The existing snapshot boundary is retained"); + assertEquals( + OperationStatus.SUCCEEDED, + client.getOperationByName("condition").status()); + } else { + assertEquals(awaitResult ? OperationStatus.SUCCEEDED : OperationStatus.READY, snapshotStatus.get()); + assertEquals( + snapshotStatus.get(), + client.getOperationByName("condition").status()); + } + } finally { + releaseRunning.countDown(); + releaseDispatch.countDown(); + releaseEnd.countDown(); + caller.shutdownNow(); + workers.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + exporter.close(); + } + } + + private static void await(CountDownLatch latch) { + try { + assertTrue(latch.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new AssertionError(interrupted); + } + } + + private static DurableExecutionInput input() { + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/end-boundary/execution", + "token", + CheckpointUpdatedExecutionState.builder() + .operations(Operation.builder() + .id("execution") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails(ExecutionDetails.builder() + .inputPayload("\"input\"") + .build()) + .build()) + .build()); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index 168f579c0..c1d1a7c97 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -210,6 +210,7 @@ private static void fireOnInvocationEnd( Throwable error, Object executionInput, Object executionResult) { + executionManager.beginInvocationEnd(); if (pluginRunner.isEmpty()) { return; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java index 2f1f109bf..8e303b1b5 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java @@ -745,13 +745,18 @@ public void close() { checkpointManager.shutdown(); } - private void stopCheckpointContinuations() { + /** Stops continuation admission and signals unfinished owners; explicit task/handler draining stays in close. */ + void beginInvocationEnd() { List owners; synchronized (activeThreads) { closing = true; owners = new ArrayList<>(checkpointContinuations.values()); } owners.forEach(ExecutionManager::stopContinuationOwner); + } + + private void stopCheckpointContinuations() { + beginInvocationEnd(); synchronized (activeThreads) { while (!checkpointContinuations.isEmpty()) { try { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPlugin.java b/sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPlugin.java index e2f8a46df..5f704ea08 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPlugin.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPlugin.java @@ -28,8 +28,11 @@ default void onInvocationStart(InvocationInfo info) {} /** * Called at the end of each Lambda invocation. Use to flush spans/metrics before Lambda freezes. * - *

    This hook is awaited — the SDK blocks until it returns. This is the only safe flush point before Lambda - * freezes the execution environment. + *

    This hook is awaited — the SDK blocks until it returns. Before constructing its operation snapshot, the SDK + * closes admission to checkpoint continuations and signals their unfinished owners. This is an admission cut, not a + * join of all already-accepted user handlers. Their cleanup and checkpoint shutdown remain in manager cleanup after + * this hook; later operation/user-function end hooks are therefore still possible. Await work that must be + * represented in the selected outcome or this snapshot; early-completion branches may remain incomplete. * *

    Check {@link InvocationEndInfo#invocationStatus()} to detect if the execution reached a terminal state in this * invocation (useful for writing summary records or flushing final data). diff --git a/sdk/src/main/java/software/amazon/lambda/durable/plugin/InvocationEndInfo.java b/sdk/src/main/java/software/amazon/lambda/durable/plugin/InvocationEndInfo.java index 145f48e65..190b7df8f 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/plugin/InvocationEndInfo.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/plugin/InvocationEndInfo.java @@ -16,7 +16,8 @@ * @param durableExecutionArn the durable execution ARN * @param isFirstInvocation true if this is the first invocation of the execution * @param executionStartTime the stable start timestamp of the durable execution - * @param operations a snapshot of operations known when the invocation ended, keyed by operation ID + * @param operations operations known at invocation End after continuation admission closes, keyed by operation ID; + * already-accepted handlers may finish during later manager cleanup * @param invocationStatus the invocation outcome * @param executionError non-null if the execution failed; this component is experimental * @param executionInput the deserialized execution input; this component is experimental From 2adc34c4d51c6f4dc566985cb74ad6c24bae61f1 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 9 Oct 2026 00:08:55 +0000 Subject: [PATCH 33/40] fix: coordinate step retries outside the checkpoint batcher --- .../durable/StepRetryContinuationTest.java | 483 ++++++++++++++++++ .../durable/operation/StepOperation.java | 24 +- 2 files changed, 500 insertions(+), 7 deletions(-) create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/StepRetryContinuationTest.java diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/StepRetryContinuationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/StepRetryContinuationTest.java new file mode 100644 index 000000000..ec723205c --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/StepRetryContinuationTest.java @@ -0,0 +1,483 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.management.ManagementFactory; +import java.lang.reflect.Method; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Deque; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.concurrent.locks.LockSupport; +import java.util.function.BiFunction; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.MDC; +import org.slf4j.spi.MDCAdapter; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.context.BaseContextImpl; +import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.execution.ExecutionManager; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.DurableExecutionOutput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.retry.RetryDecision; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +class StepRetryContinuationTest { + @ParameterizedTest + @ValueSource(strings = {"async", "reject", "direct", "submit-wait"}) + void realPendingStepRetryMustNotLoseDispatchFailureOrBlockItsCheckpointBatcher(String mode) throws Exception { + var armed = new AtomicBoolean(); + var calls = new AtomicInteger(); + var pollEntered = new CountDownLatch(1); + var releasePoll = new CountDownLatch(1); + var pollOnce = new AtomicBoolean(true); + var dispatchTimedOut = new AtomicBoolean(); + var dispatchThread = new AtomicReference(); + var dispatchFailure = new RejectedExecutionException("resumed step rejected"); + var managerRef = new AtomicReference(); + var pendingCall = new AtomicReference>(); + var client = new LocalMemoryExecutionClient() { + @Override + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List updates) { + if (armed.get() && updates.isEmpty() && pollOnce.compareAndSet(true, false)) { + pollEntered.countDown(); + await(releasePoll); + } + return super.checkpoint(arn, token, updates); + } + }; + var workers = + new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue()) { + @Override + public void execute(Runnable task) { + boolean resume = armed.get() + && Arrays.stream(Thread.currentThread().getStackTrace()) + .anyMatch(frame -> frame.getClassName().endsWith(".StepOperation") + && frame.getMethodName().equals("executeStepLogic")); + if (resume) { + dispatchThread.set(Thread.currentThread().getName()); + if (mode.equals("reject")) throw dispatchFailure; + if (mode.equals("direct")) { + task.run(); + return; + } + if (mode.equals("submit-wait")) { + var done = new CompletableFuture(); + super.execute(() -> { + try { + task.run(); + } finally { + done.complete(null); + } + }); + try { + done.get(2, TimeUnit.SECONDS); + } catch (TimeoutException expected) { + dispatchTimedOut.set(true); + } catch (Exception failure) { + throw new AssertionError(failure); + } + return; + } + } + super.execute(task); + } + }; + var config = DurableConfig.builder() + .withExecutorService(workers) + .withDurableExecutionClient(client) + .withCheckpointDelay(Duration.ZERO) + .withPollingStrategy(attempt -> Duration.ZERO) + .build(); + var stepConfig = StepConfig.builder() + .retryStrategy((error, attempt) -> + attempt < 2 ? RetryDecision.retry(Duration.ofSeconds(1)) : RetryDecision.fail()) + .build(); + var caller = Executors.newSingleThreadExecutor(); + BiFunction handler = (value, context) -> { + managerRef.set(((BaseContextImpl) context).getExecutionManager()); + var future = context.stepAsync( + "retry-step", + String.class, + step -> { + if (calls.incrementAndGet() == 1) throw new IllegalStateException("first attempt"); + return "stored"; + }, + stepConfig); + if (armed.get()) await(pollEntered); + return future.get(); + }; + try { + var first = caller.submit(() -> + DurableExecutor.execute(input(List.of()), null, TypeToken.get(String.class), handler, config)); + pendingCall.set(first); + assertEquals(ExecutionStatus.PENDING, first.get(5, TimeUnit.SECONDS).status()); + var pending = new ArrayList<>(client.getAllOperations()); + assertEquals( + OperationStatus.PENDING, + client.getOperationByName("retry-step").status()); + assertTrue(client.advanceTime()); + armed.set(true); + var second = caller.submit( + () -> DurableExecutor.execute(input(pending), null, TypeToken.get(String.class), handler, config)); + pendingCall.set(second); + assertTrue(pollEntered.await(5, TimeUnit.SECONDS)); + releasePoll.countDown(); + DurableExecutionOutput output = null; + Throwable failure = null; + try { + output = second.get(6, TimeUnit.SECONDS); + } catch (ExecutionException observed) { + failure = observed.getCause(); + } + System.out.println("STEP_RETRY_PUBLIC mode=" + mode + " thread=" + dispatchThread.get() + " output=" + + (output == null ? failure : output.status()) + " backend=" + + client.getOperationByName("retry-step").status() + + " calls=" + calls.get() + " dispatchTimedOut=" + dispatchTimedOut.get()); + assertFalse( + dispatchTimedOut.get(), "Submit-and-wait dispatch must not block the batcher needed by its step"); + if (mode.equals("reject")) { + var retry = assertInstanceOf(UnrecoverableDurableExecutionException.class, failure); + assertTrue(retry.isRetryable()); + assertSame(dispatchFailure, retry.getCause()); + assertNull(output); + assertEquals(1, calls.get()); + assertEquals( + OperationStatus.READY, + client.getOperationByName("retry-step").status()); + } else assertEquals(ExecutionStatus.SUCCEEDED, output.status()); + armed.set(false); + var resumed = caller.submit(() -> DurableExecutor.execute( + input(client.getAllOperations()), null, TypeToken.get(String.class), handler, config)); + pendingCall.set(resumed); + assertEquals( + ExecutionStatus.SUCCEEDED, resumed.get(5, TimeUnit.SECONDS).status()); + assertEquals(2, calls.get(), "Completed step bodies do not repeat on replay"); + } finally { + releasePoll.countDown(); + var manager = managerRef.get(); + if (pendingCall.get() != null + && !pendingCall.get().isDone() + && manager != null + && !manager.isExecutionCompletedExceptionally()) { + try { + manager.terminateExecution(new UnrecoverableDurableExecutionException( + ErrorObject.builder().errorMessage("probe cleanup").build(), true)); + } catch (UnrecoverableDurableExecutionException expected) { + } + } + caller.shutdownNow(); + workers.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + private static void await(CountDownLatch latch) { + try { + assertTrue(latch.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new AssertionError(interrupted); + } + } + + private static DurableExecutionInput input(List operations) { + var all = new ArrayList(); + all.add(Operation.builder() + .id("execution") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build()); + all.addAll(operations); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/step-retry/execution", + "token", + CheckpointUpdatedExecutionState.builder().operations(all).build()); + } + + @ParameterizedTest + @CsvSource({"true,false", "false,true", "true,true"}) + void liveRetryRetiresItsWorkerBeforeReplacementWithoutLosingActivity( + boolean holdWorkerExit, boolean holdExecutorReturn) throws Exception { + var gate = new HandoffGate(holdWorkerExit, holdExecutorReturn); + var client = new DelayedReadyClient(gate); + var calls = new AtomicInteger(); + var caller = Executors.newSingleThreadExecutor(); + var config = DurableConfig.builder() + .withExecutorService(gate) + .withDurableExecutionClient(client) + .withCheckpointDelay(Duration.ZERO) + .withPollingStrategy(attempt -> Duration.ZERO) + .build(); + var retry = StepConfig.builder() + .retryStrategy((error, attempt) -> + attempt == 1 ? RetryDecision.retry(Duration.ofSeconds(1)) : RetryDecision.fail()) + .build(); + BiFunction handler = (value, context) -> { + var step = context.stepAsync( + "retry-step", + String.class, + stepContext -> { + if (calls.incrementAndGet() == 1) { + gate.firstWorker.set(Thread.currentThread()); + throw new IllegalStateException("retry once"); + } + gate.nextCheckEntered.countDown(); + await(gate.releaseNextCheck); + return "stored"; + }, + retry); + // Keep the root active until the live retry has registered its backend poll. + await(gate.pollEntered); + return step.get(); + }; + try (var mdc = new MdcClearGate(gate)) { + try { + var result = caller.submit(() -> + DurableExecutor.execute(input(List.of()), null, TypeToken.get(String.class), handler, config)); + await(gate.pollEntered); + if (holdWorkerExit) await(gate.workerExitEntered); + else await(gate.executorReturnEntered); + assertTrue(client.advanceTime()); + gate.releaseReadyResponse.countDown(); + await(gate.readyResponseReturned); + awaitCheckpointHandoff(gate.pollThread.get()); + assertThrows(TimeoutException.class, () -> result.get(100, TimeUnit.MILLISECONDS)); + assertEquals(1, calls.get()); + gate.releaseWorkerExit.countDown(); + if (holdExecutorReturn) await(gate.executorReturnEntered); + gate.releaseExecutorReturn.countDown(); + await(gate.nextCheckEntered); + assertThrows( + TimeoutException.class, + () -> result.get(100, TimeUnit.MILLISECONDS), + "The replacement retains activity after its previous worker deregisters"); + gate.releaseNextCheck.countDown(); + var completed = result.get(5, TimeUnit.SECONDS); + assertEquals(ExecutionStatus.SUCCEEDED, completed.status()); + assertEquals("\"stored\"", completed.result()); + assertEquals( + OperationStatus.SUCCEEDED, + client.getOperationByName("retry-step").status()); + var replay = caller.submit(() -> DurableExecutor.execute( + input(client.getAllOperations()), null, TypeToken.get(String.class), handler, config)); + assertEquals(completed, replay.get(5, TimeUnit.SECONDS)); + assertEquals(2, calls.get(), "The completed retry body is skipped on replay"); + } finally { + gate.releaseAll(); + caller.shutdownNow(); + gate.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(gate.awaitTermination(5, TimeUnit.SECONDS)); + } + } + } + + private static void awaitCheckpointHandoff(Thread checkpoint) { + var deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (System.nanoTime() < deadline) { + for (var entry : Thread.getAllStackTraces().entrySet()) { + var continuation = entry.getKey(); + var stack = entry.getValue(); + if (continuation.getState() != Thread.State.WAITING + || !continuation.getName().startsWith("durable-sdk-internal-") + || Arrays.stream(stack) + .noneMatch(frame -> frame.getClassName().endsWith("StepOperation")) + || Arrays.stream(stack) + .noneMatch(frame -> frame.getClassName().equals(CompletableFuture.class.getName()) + && frame.getMethodName().equals("join"))) continue; + assertNotSame(checkpoint, continuation, "Worker publication must not block the checkpoint callback"); + assertTrue(Arrays.stream(stack) + .noneMatch(frame -> frame.getClassName().endsWith("ApiRequestDelayedBatcher"))); + if (Arrays.stream(checkpoint.getStackTrace()) + .anyMatch(frame -> frame.getClassName().endsWith("CheckpointManager") + && frame.getMethodName().equals("checkpointBatch"))) continue; + var bean = ManagementFactory.getThreadMXBean(); + if (bean.isObjectMonitorUsageSupported()) { + var info = bean.getThreadInfo(new long[] {continuation.getId()}, true, true)[0]; + assertEquals(0, info.getLockedMonitors().length); + System.out.println("READY_HANDOFF continuation=" + continuation.getName() + " checkpoint=" + + checkpoint.getName() + " continuationMonitors=[] checkpointBatchReturned=true"); + } + return; + } + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)); + } + fail("The independent READY continuation did not reach the controlled old-worker handoff window"); + } + + private static final class DelayedReadyClient extends LocalMemoryExecutionClient { + private final HandoffGate gate; + private final AtomicBoolean firstPoll = new AtomicBoolean(true); + + private DelayedReadyClient(HandoffGate gate) { + this.gate = gate; + } + + @Override + public CheckpointDurableExecutionResponse checkpoint(String arn, String token, List updates) { + if (updates.isEmpty() && firstPoll.compareAndSet(true, false)) { + gate.pollThread.set(Thread.currentThread()); + gate.pollEntered.countDown(); + await(gate.releaseReadyResponse); + var response = super.checkpoint(arn, token, updates); + gate.readyResponseReturned.countDown(); + return response; + } + return super.checkpoint(arn, token, updates); + } + } + + private static final class HandoffGate extends AbstractExecutorService { + private final ExecutorService delegate = Executors.newCachedThreadPool(); + private final AtomicInteger submitted = new AtomicInteger(); + private final AtomicBoolean exitHeld = new AtomicBoolean(); + private final AtomicReference firstWorker = new AtomicReference<>(); + private final AtomicReference pollThread = new AtomicReference<>(); + private final CountDownLatch pollEntered = new CountDownLatch(1); + private final CountDownLatch releaseReadyResponse = new CountDownLatch(1); + private final CountDownLatch readyResponseReturned = new CountDownLatch(1); + private final CountDownLatch workerExitEntered = new CountDownLatch(1); + private final CountDownLatch releaseWorkerExit = new CountDownLatch(1); + private final CountDownLatch executorReturnEntered = new CountDownLatch(1); + private final CountDownLatch releaseExecutorReturn = new CountDownLatch(1); + private final CountDownLatch nextCheckEntered = new CountDownLatch(1); + private final CountDownLatch releaseNextCheck = new CountDownLatch(1); + private final boolean holdWorkerExit; + private final boolean holdExecutorReturn; + + private HandoffGate(boolean holdWorkerExit, boolean holdExecutorReturn) { + this.holdWorkerExit = holdWorkerExit; + this.holdExecutorReturn = holdExecutorReturn; + } + + @Override + public void execute(Runnable task) { + var number = submitted.incrementAndGet(); + var finished = new CountDownLatch(1); + delegate.execute(() -> { + try { + task.run(); + } finally { + finished.countDown(); + } + }); + // Completing work before execute() returns is permitted by Executor's contract. + if (holdExecutorReturn && number == 2) { + await(finished); + executorReturnEntered.countDown(); + await(releaseExecutorReturn); + } + } + + private void releaseAll() { + releaseReadyResponse.countDown(); + releaseWorkerExit.countDown(); + releaseExecutorReturn.countDown(); + releaseNextCheck.countDown(); + } + + @Override + public void shutdown() { + delegate.shutdown(); + } + + @Override + public List shutdownNow() { + releaseAll(); + return delegate.shutdownNow(); + } + + @Override + public boolean isShutdown() { + return delegate.isShutdown(); + } + + @Override + public boolean isTerminated() { + return delegate.isTerminated(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return delegate.awaitTermination(timeout, unit); + } + } + + private static final class MdcClearGate implements AutoCloseable { + private final MDCAdapter previous = MDC.getMDCAdapter(); + private final Method setter = MDC.class.getDeclaredMethod("setMDCAdapter", MDCAdapter.class); + + private MdcClearGate(HandoffGate gate) throws Exception { + setter.setAccessible(true); + setter.invoke(null, new MDCAdapter() { + public void clear() { + if (gate.holdWorkerExit + && Thread.currentThread() == gate.firstWorker.get() + && gate.exitHeld.compareAndSet(false, true)) { + gate.workerExitEntered.countDown(); + await(gate.releaseWorkerExit); + } + previous.clear(); + } + + public void put(String key, String value) { + previous.put(key, value); + } + + public String get(String key) { + return previous.get(key); + } + + public void remove(String key) { + previous.remove(key); + } + + public Map getCopyOfContextMap() { + return previous.getCopyOfContextMap(); + } + + public void setContextMap(Map values) { + previous.setContextMap(values); + } + + public void pushByKey(String key, String value) { + previous.pushByKey(key, value); + } + + public String popByKey(String key) { + return previous.popByKey(key); + } + + public Deque getCopyOfDequeByKey(String key) { + return previous.getCopyOfDequeByKey(key); + } + + public void clearDequeByKey(String key) { + previous.clearDequeByKey(key); + } + }); + } + + public void close() throws Exception { + setter.invoke(null, previous); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java index 467a87b94..19fafbdc2 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java @@ -70,7 +70,7 @@ protected void replay(Operation existing) { case STARTED -> { if (isAtMostOnce()) { // AT_MOST_ONCE: treat as interrupted, go through retry logic - handleStepFailure(new StepInterruptedException(existing), attempt); + handleStepFailure(new StepInterruptedException(existing), attempt, null); } else { // AT_LEAST_ONCE: re-execute the step executeStepLogic(attempt); @@ -79,7 +79,7 @@ protected void replay(Operation existing) { // Step is pending retry - Start polling for PENDING -> READY transition case PENDING -> { if (existing.stepDetails() != null && existing.stepDetails().nextAttemptTimestamp() != null) { - pollReadyAndExecuteStepLogic(existing.stepDetails().nextAttemptTimestamp(), attempt); + pollReadyAndExecuteStepLogic(existing.stepDetails().nextAttemptTimestamp(), attempt, null); } else { throw terminateExecutionWithIllegalDurableOperationException( "Unexpected PENDING step without nextAttemptTimestamp: " + getOperationId()); @@ -93,16 +93,24 @@ protected void replay(Operation existing) { } } - private void pollReadyAndExecuteStepLogic(Instant nextAttemptInstant, int attempt) { + private void pollReadyAndExecuteStepLogic( + Instant nextAttemptInstant, int attempt, CompletableFuture> previousWorker) { pollForOperationUpdates(nextAttemptInstant) .thenCompose(op -> op.status() == OperationStatus.READY ? CompletableFuture.completedFuture(op) : pollForOperationUpdates(nextAttemptInstant)) - .thenRun(() -> executeStepLogic(attempt)); + .thenCompose(ignored -> executionManager.runCheckpointContinuation(this, () -> { + // Leave the serialized checkpoint callback before dispatching a configured executor. The lease + // spans retirement of the old attempt and registration of its replacement. + if (previousWorker != null) previousWorker.join().join(); + if (!isOperationCompleted()) executeStepLogic(attempt); + })); } private void executeStepLogic(int attempt) { + var publishedWorker = new CompletableFuture>(); Runnable userHandler = () -> { + if (isOperationCompleted()) return; // use a try-with-resources to // - add thread id/type to thread local when the step starts // - clear logger properties when the step finishes @@ -119,13 +127,14 @@ private void executeStepLogic(int attempt) { handleStepSucceeded(result); } catch (Throwable e) { - handleStepFailure(e, attempt); + handleStepFailure(e, attempt, publishedWorker); } } }; // Execute user provided step code in user-configured executor runUserHandler(userHandler, ThreadType.STEP); + publishedWorker.complete(getRunningUserHandler()); } private void checkpointStarted() { @@ -156,7 +165,8 @@ private void handleStepSucceeded(T result) { sendOperationUpdate(successUpdate); } - private void handleStepFailure(Throwable exception, int attempt) { + private void handleStepFailure( + Throwable exception, int attempt, CompletableFuture> publishedWorker) { exception = ExceptionHelper.unwrapCompletableFuture(exception); if (exception instanceof SuspendExecutionException suspendExecutionException) { throw suspendExecutionException; @@ -189,7 +199,7 @@ private void handleStepFailure(Throwable exception, int attempt) { sendOperationUpdate(retryUpdate); // Poll for READY status and then execute the step again - pollReadyAndExecuteStepLogic(Instant.now().plusSeconds(retryDelayInSeconds), attempt + 1); + pollReadyAndExecuteStepLogic(Instant.now().plusSeconds(retryDelayInSeconds), attempt + 1, publishedWorker); } else { // Send FAIL - retries exhausted var failUpdate = From 4091401e3243105cdc84faaad81b6dca5caa5fa9 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 9 Oct 2026 00:36:21 +0000 Subject: [PATCH 34/40] fix: finish and flush synthetic roots after output preparation failures --- otel-plugin/README.md | 8 + .../durable/otel/ExecutionOtelPlugin.java | 17 +- .../durable/otel/InvocationOtelPlugin.java | 17 +- .../durable/otel/OtelPluginSupport.java | 12 + .../ExecutionRootDeliveryFailureTest.java | 279 ++++++++++++++++++ .../otel/ExecutionRootLifecycleTest.java | 50 ++++ .../durable/execution/DurableExecutor.java | 35 ++- 7 files changed, 399 insertions(+), 19 deletions(-) create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootDeliveryFailureTest.java diff --git a/otel-plugin/README.md b/otel-plugin/README.md index 73760582f..4ee0dc66a 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -71,6 +71,14 @@ trace state. It ends at invocation end, before flushing, with the same fixed exe deferred sampler, the first actual fallback span is now `DurableExecutionRoot`; custom policies that depend on the first span name or attributes can observe that ordering. No arbitrary name/input invariance is promised. +If result serialization or oversized-result checkpointing fails before the SDK output is prepared, End reports +`RETRYING` on the existing completion thread before the original preparation failure propagates. This closes the +retained root without changing handler failure classification or claiming coverage of later response-stream writes +or runtime acknowledgment. A root processor `Exception` or `LinkageError` still permits flushing already completed +spans before the plugin boundary isolates that error. Unisolated `Error` failures keep their existing escape/skip-flush +behavior; a failing processor need not have exported the root itself. Cleanup preserves the original preparation +failure unless it introduces the first direct JVM fatal; an earlier JVM fatal remains primary. + The anchor is marked `durable.execution.synthetic_root=true`. Its trace and span IDs are deterministic and its start and end timestamps are the checkpointed execution start. It does not report execution status or duration; `Workflow` continues to report those at terminal completion. Complete remote parents remain externally owned and are never exported. diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index b9f042b66..be222812a 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -294,14 +294,15 @@ public void onInvocationEnd(InvocationEndInfo info) { var provider = sdkTracerProvider; var shouldFlush = tracingEnabled || root != null; executionRootSpan = null; // One-shot release even if an end processor throws or re-enters. - OtelPluginSupport.finishWithRootCleanup(() -> finishInvocationEnd(info), () -> { - if (root != null) root.end(rootTimestamp); - if (shouldFlush && provider != null) { - var flushResult = provider.forceFlush().join(5, TimeUnit.SECONDS); - if (!flushResult.isSuccess()) - logger.warn("OTel span flush failed or timed out — some spans may be lost"); - } - }); + OtelPluginSupport.finishWithRootCleanup( + () -> finishInvocationEnd(info), + () -> OtelPluginSupport.endRootAndFlush(root, rootTimestamp, () -> { + if (shouldFlush && provider != null) { + var flushResult = provider.forceFlush().join(5, TimeUnit.SECONDS); + if (!flushResult.isSuccess()) + logger.warn("OTel span flush failed or timed out — some spans may be lost"); + } + })); } private void finishInvocationEnd(InvocationEndInfo info) { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index eabccc4f8..1b24f5dfe 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -291,14 +291,15 @@ public void onInvocationEnd(InvocationEndInfo info) { var provider = sdkTracerProvider; var shouldFlush = tracingEnabled || root != null; executionRootSpan = null; // One-shot release even if an end processor throws or re-enters. - OtelPluginSupport.finishWithRootCleanup(() -> finishInvocationEnd(info), () -> { - if (root != null) root.end(rootTimestamp); - if (shouldFlush && provider != null) { - var flushResult = provider.forceFlush().join(5, TimeUnit.SECONDS); - if (!flushResult.isSuccess()) - logger.warn("OTel span flush failed or timed out — some spans may be lost"); - } - }); + OtelPluginSupport.finishWithRootCleanup( + () -> finishInvocationEnd(info), + () -> OtelPluginSupport.endRootAndFlush(root, rootTimestamp, () -> { + if (shouldFlush && provider != null) { + var flushResult = provider.forceFlush().join(5, TimeUnit.SECONDS); + if (!flushResult.isSuccess()) + logger.warn("OTel span flush failed or timed out — some spans may be lost"); + } + })); } private void finishInvocationEnd(InvocationEndInfo info) { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 0f81e1d18..2fe2cfd0c 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -21,6 +21,7 @@ import java.util.Collections; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.lambda.durable.util.ExceptionHelper; /** Shared utilities for OTel plugin default constructor support (ADOT Java agent SPI path). */ final class OtelPluginSupport { @@ -60,6 +61,17 @@ static Span startExecutionRoot( return root; } + /** Flushes completed spans after root end, including failures isolated by the plugin boundary. */ + static void endRootAndFlush(Span root, Instant timestamp, Runnable flush) { + try { + if (root != null) root.end(timestamp); + } catch (Exception | LinkageError failure) { + finishWithRootCleanup(() -> ExceptionHelper.sneakyThrow(failure), flush); + return; + } + flush.run(); + } + /** Always releases an invocation's new root resource without hiding an earlier failure or a later JVM fatal. */ static void finishWithRootCleanup(Runnable finish, Runnable cleanup) { Throwable primary = null; diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootDeliveryFailureTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootDeliveryFailureTest.java new file mode 100644 index 000000000..e4cc31f84 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootDeliveryFailureTest.java @@ -0,0 +1,279 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.ReadWriteSpan; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; +import software.amazon.lambda.durable.util.ExceptionHelper; + +class ExecutionRootDeliveryFailureTest { + @ParameterizedTest + @CsvSource({ + "false,serialize", + "true,serialize", + "false,checkpoint", + "true,checkpoint", + "false,root-end", + "true,root-end", + "false,success", + "true,success" + }) + void outputPreparationAndOrdinaryRootEndFailuresStillReleaseAndFlush(boolean executionView, String mode) + throws Exception { + var originalContext = Context.current(); + var failure = new IllegalStateException("controlled " + mode); + var failOnce = new AtomicBoolean(true); + var bodyCalls = new AtomicInteger(); + var rootEnds = new AtomicInteger(); + var flushes = new AtomicInteger(); + var roots = new CopyOnWriteArrayList(); + var ends = new CopyOnWriteArrayList(); + var observer = new SpanProcessor() { + public void onStart(Context parent, ReadWriteSpan span) { + if (span.getName().equals("DurableExecutionRoot")) roots.add(span); + } + + public boolean isStartRequired() { + return true; + } + + public void onEnd(ReadableSpan span) { + if (span.getName().equals("DurableExecutionRoot")) { + rootEnds.incrementAndGet(); + if (mode.equals("root-end") && failOnce.compareAndSet(true, false)) throw failure; + } + } + + public boolean isEndRequired() { + return true; + } + + public CompletableResultCode forceFlush() { + flushes.incrementAndGet(); + return CompletableResultCode.ofSuccess(); + } + }; + var client = new LocalMemoryExecutionClient() { + @Override + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List updates) { + if (mode.equals("checkpoint") + && updates.stream().anyMatch(u -> u.type() == OperationType.EXECUTION) + && failOnce.compareAndSet(true, false)) throw failure; + return super.checkpoint(arn, token, updates); + } + }; + var serDes = new SerDes() { + private final JacksonSerDes delegate = new JacksonSerDes(); + + public String serialize(Object value) { + if (mode.equals("serialize") && "result".equals(value) && failOnce.compareAndSet(true, false)) + throw failure; + return delegate.serialize(value); + } + + public T deserialize(String data, TypeToken type) { + return delegate.deserialize(data, type); + } + }; + var caller = Executors.newSingleThreadExecutor(); + var workers = Executors.newCachedThreadPool(); + try (var exporter = InMemorySpanExporter.create(); + var batch = BatchSpanProcessor.builder(exporter) + .setScheduleDelay(Duration.ofDays(1)) + .build()) { + var builder = SdkTracerProvider.builder().addSpanProcessor(observer).addSpanProcessor(batch); + var otelConfig = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(() -> null) + .build(); + DurableExecutionPlugin otel = executionView + ? new ExecutionOtelPlugin(builder, otelConfig) + : new InvocationOtelPlugin(builder, otelConfig); + var config = DurableConfig.builder() + .withDurableExecutionClient(client) + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withSerDes(serDes) + .withPlugins(otel, new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + ends.add(info); + } + }) + .build(); + try { + var first = caller.submit(() -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, context) -> bodyCalls.incrementAndGet() == 1 && mode.equals("checkpoint") + ? "x".repeat(6 * 1024 * 1024) + : "result", + config)); + if (mode.equals("serialize") || mode.equals("checkpoint")) { + var thrown = assertThrows(ExecutionException.class, () -> first.get(5, TimeUnit.SECONDS)); + assertSame(failure, thrown.getCause(), "Output preparation failure remains caller-visible"); + } else + assertEquals( + ExecutionStatus.SUCCEEDED, + first.get(5, TimeUnit.SECONDS).status()); + var exported = exporter.getFinishedSpanItems(); + System.out.println("ROOT_DELIVERY view=" + executionView + " mode=" + mode + " ends=" + ends.size() + + " rootEnds=" + rootEnds.get() + " rootRecording=" + + roots.get(0).isRecording() + + " flushes=" + flushes.get() + " exported=" + + exported.stream().map(s -> s.getName()).toList()); + assertEquals(1, ends.size(), "Output preparation failure still has exactly one End"); + assertEquals( + mode.equals("serialize") || mode.equals("checkpoint") + ? InvocationStatus.RETRYING + : InvocationStatus.SUCCEEDED, + ends.get(0).invocationStatus()); + assertEquals(1, rootEnds.get()); + assertFalse(roots.get(0).isRecording()); + assertEquals(1, flushes.get(), "Root onEnd failure does not skip flushing completed spans"); + assertTrue( + exported.stream().anyMatch(s -> s.getName().equals("Invocation")), + "The BatchSpanProcessor must be flushed before invocation return"); + var second = caller.submit(() -> DurableExecutor.execute( + input(), null, TypeToken.get(String.class), (value, context) -> "result", config)); + assertEquals( + ExecutionStatus.SUCCEEDED, + second.get(5, TimeUnit.SECONDS).status()); + assertEquals(2, ends.size()); + assertEquals(2, rootEnds.get()); + assertEquals(2, flushes.get()); + assertEquals(roots.get(0).getSpanContext(), roots.get(1).getSpanContext()); + assertTrue(roots.stream().noneMatch(ReadWriteSpan::isRecording)); + assertSame(originalContext, Context.current()); + } finally { + caller.shutdownNow(); + workers.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + } + + private static DurableExecutionInput input() { + var operation = Operation.builder() + .id("execution") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.ofEpochSecond(10)) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/root/execution", + "token", + CheckpointUpdatedExecutionState.builder().operations(operation).build()); + } + + static Stream preparationEndFailures() { + return Stream.of(false, true) + .flatMap(view -> Stream.of("ordinary", "vm", "death", "wrapped-vm") + .flatMap(preparation -> Stream.of("assertion", "vm", "death") + .map(end -> Arguments.of(view, preparation, end)))); + } + + @SuppressWarnings("removal") + @ParameterizedTest + @MethodSource("preparationEndFailures") + void preparationAndEndFailuresRetainCallerIdentityAndCleanupDiagnostics( + boolean executionView, String preparationKind, String endKind) { + Throwable original = + switch (preparationKind) { + case "vm", "wrapped-vm" -> new InternalError("preparation fatal"); + case "death" -> new ThreadDeath(); + default -> new IllegalStateException("preparation ordinary"); + }; + Throwable preparation = preparationKind.equals("wrapped-vm") ? new CompletionException(original) : original; + Error cleanup = + switch (endKind) { + case "vm" -> new InternalError("End fatal"); + case "death" -> new ThreadDeath(); + default -> new AssertionError("End error"); + }; + var ends = new AtomicInteger(); + var seen = new AtomicReference(); + var serDes = new SerDes() { + private final JacksonSerDes delegate = new JacksonSerDes(); + + public String serialize(Object value) { + if ("result".equals(value)) ExceptionHelper.sneakyThrow(preparation); + return delegate.serialize(value); + } + + public T deserialize(String value, TypeToken type) { + return delegate.deserialize(value, type); + } + }; + try (var exporter = InMemorySpanExporter.create()) { + var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var otelConfig = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(() -> null) + .build(); + DurableExecutionPlugin otel = executionView + ? new ExecutionOtelPlugin(builder, otelConfig) + : new InvocationOtelPlugin(builder, otelConfig); + var runner = LocalDurableTestRunner.create( + String.class, + (value, context) -> "result", + DurableConfig.builder() + .withSerDes(serDes) + .withPlugins(otel, new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + ends.incrementAndGet(); + seen.set(info); + throw cleanup; + } + }) + .build()); + Throwable expected = original instanceof VirtualMachineError || original instanceof ThreadDeath + ? original + : cleanup instanceof VirtualMachineError || cleanup instanceof ThreadDeath ? cleanup : original; + assertSame(expected, assertThrows(Throwable.class, () -> runner.run("input"))); + assertEquals(List.of(expected == original ? cleanup : original), List.of(expected.getSuppressed())); + assertEquals(1, ends.get()); + assertEquals(InvocationStatus.RETRYING, seen.get().invocationStatus()); + assertSame(original, seen.get().executionError()); + var roots = exporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals("DurableExecutionRoot")) + .toList(); + assertEquals(1, roots.size()); + assertEquals(roots.get(0).getStartEpochNanos(), roots.get(0).getEndEpochNanos()); + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java index 6092a4a0a..15cbbff63 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java @@ -3,7 +3,9 @@ package software.amazon.lambda.durable.otel; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; +import io.opentelemetry.api.trace.Span; import io.opentelemetry.context.Context; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; @@ -24,6 +26,7 @@ import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.util.ExceptionHelper; class ExecutionRootLifecycleTest { private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/root/id"; @@ -151,4 +154,51 @@ else if (mode.equals("fatal-root-cleanup")) { private static InvocationEndInfo end(boolean first) { return new InvocationEndInfo("request", ARN, first, InvocationStatus.PENDING, null); } + + @SuppressWarnings("removal") + @ParameterizedTest + @CsvSource({ + "runtime,ordinary", + "runtime,fatal", + "linkage,ordinary", + "linkage,fatal", + "assertion,unused", + "vm,unused", + "death,unused" + }) + void flushRunsOnlyAfterRootFailuresIsolatedByTheExistingPluginBoundary(String rootKind, String flushKind) { + Throwable rootFailure = + switch (rootKind) { + case "linkage" -> new LinkageError("root linkage"); + case "assertion" -> new AssertionError("root assertion"); + case "vm" -> new InternalError("root fatal"); + case "death" -> new ThreadDeath(); + default -> new IllegalStateException("root ordinary"); + }; + Throwable flushFailure = flushKind.equals("fatal") + ? new InternalError("flush fatal") + : new IllegalArgumentException("flush ordinary"); + var root = mock(Span.class); + doAnswer(call -> { + ExceptionHelper.sneakyThrow(rootFailure); + return null; + }) + .when(root) + .end(START); + var flushes = new AtomicInteger(); + var observed = assertThrows( + Throwable.class, + () -> OtelPluginSupport.endRootAndFlush(root, START, () -> { + flushes.incrementAndGet(); + ExceptionHelper.sneakyThrow(flushFailure); + })); + var isolated = rootKind.equals("runtime") || rootKind.equals("linkage"); + assertEquals(isolated ? 1 : 0, flushes.get()); + var expected = isolated && flushKind.equals("fatal") ? flushFailure : rootFailure; + assertSame(expected, observed); + assertEquals( + isolated ? List.of(expected == rootFailure ? flushFailure : rootFailure) : List.of(), + List.of(observed.getSuppressed())); + verify(root).end(START); + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index c1d1a7c97..bb74f51a6 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -176,9 +176,38 @@ public static DurableExecutionOutput execute( } // user handler complete successfully logger.debug("Execution completed"); - var outputPayload = config.getSerDes().serialize(result); - var output = - DurableExecutionOutput.success(handleLargePayload(executionManager, outputPayload)); + DurableExecutionOutput output; + try { + var outputPayload = config.getSerDes().serialize(result); + output = DurableExecutionOutput.success( + handleLargePayload(executionManager, outputPayload)); + } catch (Throwable preparationFailure) { + var cause = ExceptionHelper.unwrapCompletableFuture(preparationFailure); + try { + fireOnInvocationEnd( + pluginRunner, + executionManager, + requestId, + executionArn, + isFirstInvocation, + InvocationStatus.RETRYING, + cause, + pluginExecutionInput.get(), + null); + } catch (Error endFailure) { + if (endFailure != cause) { + if (!(cause instanceof VirtualMachineError || cause instanceof ThreadDeath) + && (endFailure instanceof VirtualMachineError + || endFailure instanceof ThreadDeath)) { + endFailure.addSuppressed(cause); + throw endFailure; + } + cause.addSuppressed(endFailure); + } + } + ExceptionHelper.sneakyThrow(cause); + return null; + } fireOnInvocationEnd( pluginRunner, executionManager, From eb449c4cada52414f687bb5b3ae9362a917448f2 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 9 Oct 2026 01:32:59 +0000 Subject: [PATCH 35/40] fix: select continuation failures atomically with close --- .../durable/execution/ExecutionManager.java | 14 +- .../ContinuationFailureCloseRaceTest.java | 133 ++++++++++++++++++ 2 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/execution/ContinuationFailureCloseRaceTest.java diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java index 8e303b1b5..a1c52be1e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java @@ -87,6 +87,8 @@ public class ExecutionManager implements SafeCloseable { // Guarded by activeThreads; an observation future can be cancelled independently of its actual task. private final Map checkpointContinuations = new HashMap<>(); private boolean closing; + // Guarded by activeThreads; selected against normal close before publishing callbacks outside the monitor. + private UnrecoverableDurableExecutionException selectedContinuationFailure; /** * Per-wait state used to coordinate the caller thread with the future completion callback. @@ -634,9 +636,17 @@ private void signalContinuationFailure(Throwable failure) { .build(), true, failure); - continuationFailurePublishers.put(control, Thread.currentThread()); + UnrecoverableDurableExecutionException selected; + synchronized (activeThreads) { + if (closing && !(failure instanceof VirtualMachineError || failure instanceof ThreadDeath)) return; + if (selectedContinuationFailure == null) selectedContinuationFailure = control; + selected = selectedContinuationFailure; + continuationFailurePublishers.put(control, Thread.currentThread()); + } + // A later admitted publisher may drain the winner's callbacks. Keep that CompletableFuture behavior, + // but publish/wake using the selected cause, outside the monitor needed by worker cleanup. try { - if (executionExceptionFuture.completeExceptionally(control)) stopAllOperations(control); + if (executionExceptionFuture.completeExceptionally(selected)) stopAllOperations(selected); } finally { continuationFailurePublishers.remove(control); } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/ContinuationFailureCloseRaceTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/ContinuationFailureCloseRaceTest.java new file mode 100644 index 000000000..2f80d921a --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/ContinuationFailureCloseRaceTest.java @@ -0,0 +1,133 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.execution; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import java.util.concurrent.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TestUtils; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.operation.BaseDurableOperation; + +class ContinuationFailureCloseRaceTest { + @ParameterizedTest + @CsvSource({"body,true", "reject,true", "body,false", "reject,false"}) + void failureSelectionIsOrderedWithCloseAndDoesNotRunCallbacksUnderCoordinationLock(String mode, boolean closeFirst) + throws Exception { + var input = new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/close-race/execution", + "token", + CheckpointUpdatedExecutionState.builder() + .operations(Operation.builder() + .id("execution") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .build()) + .build()); + var manager = new ExecutionManager( + input, + DurableConfig.builder() + .withDurableExecutionClient(TestUtils.createMockClient()) + .build(), + null); + manager.registerActiveThread("root"); + var callbacks = new CountDownLatch(1); + var callback = manager.runUntilCompleteOrSuspend(new CompletableFuture()) + .handle((value, error) -> { + try { + CompletableFuture.runAsync(() -> manager.registerActiveThread("callback-lock-proof")) + .get(3, TimeUnit.SECONDS); + } catch (Exception failure) { + throw new AssertionError("Failure callbacks must not hold the activeThreads monitor", failure); + } + callbacks.countDown(); + return error; + }); + var owner = mock(BaseDurableOperation.class); + var ownerCompletion = new CompletableFuture(); + when(owner.getCompletionFuture()).thenReturn(ownerCompletion); + var unrelated = mock(BaseDurableOperation.class); + var unrelatedCompletion = new CompletableFuture(); + when(unrelated.getOperationId()).thenReturn("unrelated"); + when(unrelated.getCompletionFuture()).thenReturn(unrelatedCompletion); + manager.registerOperation(unrelated); + var buildingFailure = new CountDownLatch(1); + var releaseFailure = new CountDownLatch(1); + RuntimeException failure = mode.equals("reject") + ? new RejectedExecutionException("coordinator rejected") + : new IllegalArgumentException("continuation failed"); + var publisher = CompletableFuture.runAsync(() -> { + // A scheduling gate inside ordinary control construction, after the old caller's isClosing precheck. + // It does not replace manager state, futures, close, or stopAllOperations. + try (var model = mockStatic(ErrorObject.class, CALLS_REAL_METHODS)) { + model.when(ErrorObject::builder).thenAnswer(call -> { + buildingFailure.countDown(); + assertTrue(releaseFailure.await(3, TimeUnit.SECONDS)); + return call.callRealMethod(); + }); + if (mode.equals("reject")) { + assertSame( + failure, + assertThrows( + RejectedExecutionException.class, + () -> manager.runCheckpointContinuation( + owner, () -> fail("Rejected work"), task -> { + throw failure; + }))); + } else { + var observed = manager.runCheckpointContinuation( + owner, + () -> { + throw failure; + }, + Runnable::run); + assertSame( + failure, + assertThrows(CompletionException.class, observed::join) + .getCause()); + } + } + }); + CompletableFuture closing = null; + try { + assertTrue(buildingFailure.await(3, TimeUnit.SECONDS)); + if (closeFirst) { + closing = CompletableFuture.runAsync(manager::close); + assertInstanceOf( + SuspendExecutionException.class, + assertThrows(ExecutionException.class, () -> ownerCompletion.get(3, TimeUnit.SECONDS)) + .getCause()); + assertFalse(unrelatedCompletion.isDone()); + var pendingClose = closing; + assertThrows(TimeoutException.class, () -> pendingClose.get(100, TimeUnit.MILLISECONDS)); + } + releaseFailure.countDown(); + publisher.get(3, TimeUnit.SECONDS); + if (closing == null) closing = CompletableFuture.runAsync(manager::close); + closing.get(3, TimeUnit.SECONDS); + System.out.println("CONTINUATION_CLOSE_RACE mode=" + mode + " closeFirst=" + closeFirst + + " unrelatedStopped=" + unrelatedCompletion.isDone() + + " managerFailure=" + manager.isExecutionCompletedExceptionally()); + assertEquals( + !closeFirst, + unrelatedCompletion.isDone(), + "Only failure selected before close may stop unrelated operations"); + assertEquals(!closeFirst, manager.isExecutionCompletedExceptionally()); + if (!closeFirst) { + assertNotNull(callback.get(3, TimeUnit.SECONDS)); + assertEquals(0, callbacks.getCount()); + } else assertEquals(1, callbacks.getCount()); + assertDoesNotThrow(() -> manager.deregisterActiveThread("root")); + } finally { + releaseFailure.countDown(); + publisher.get(3, TimeUnit.SECONDS); + if (closing == null) closing = CompletableFuture.runAsync(manager::close); + closing.get(3, TimeUnit.SECONDS); + } + } +} From 098c9616109c156ebae9788081bf1f04e9293176 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 9 Oct 2026 02:11:48 +0000 Subject: [PATCH 36/40] fix: prune completed polls before checkpoint batching --- .../durable/execution/CheckpointManager.java | 5 + .../execution/CompletedPollBatchTest.java | 99 +++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/execution/CompletedPollBatchTest.java diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java index 34f44134c..0430d6ab1 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java @@ -198,6 +198,11 @@ List fetchAllPages(CheckpointUpdatedExecutionState checkpointUpdatedE private void checkpointBatch(List updates) { synchronized (pollingFutures) { + // A READY recheck can complete a poll before its scheduled batch. Keep only consumers still waiting. + pollingFutures.entrySet().removeIf(entry -> { + entry.getValue().removeIf(CompletableFuture::isDone); + return entry.getValue().isEmpty(); + }); // filter the null values from pollers var request = updates.stream().filter(Objects::nonNull).toList(); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/CompletedPollBatchTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/CompletedPollBatchTest.java new file mode 100644 index 000000000..0646aaa46 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/CompletedPollBatchTest.java @@ -0,0 +1,99 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.execution; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.time.Duration; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.client.DurableExecutionClient; + +class CompletedPollBatchTest { + @ParameterizedTest + @ValueSource( + strings = {"ready", "terminal", "exceptional", "cancelled", "checkpoint", "same-id", "other-id", "pending"}) + void completedPollsDoNotCauseEmptyRpcButLivePollsAndUpdatesStillFlush(String mode) throws Exception { + var client = mock(DurableExecutionClient.class); + var starts = new AtomicInteger(); + var finishes = new AtomicInteger(); + var callbacks = new AtomicInteger(); + var config = DurableConfig.builder() + .withDurableExecutionClient(client) + .withCheckpointDelay(Duration.ofHours(1)) + .build(); + var manager = new CheckpointManager( + config, + "arn:test", + "token", + values -> callbacks.incrementAndGet(), + () -> { + starts.incrementAndGet(); + return true; + }, + finishes::incrementAndGet); + var ready = Operation.builder() + .id("watched") + .type(OperationType.STEP) + .status(OperationStatus.READY) + .build(); + var terminal = ready.toBuilder().status(OperationStatus.SUCCEEDED).build(); + var liveId = mode.equals("other-id") ? "other" : "watched"; + var response = ready.toBuilder().id(liveId).build(); + when(client.checkpoint(anyString(), anyString(), anyList())) + .thenReturn(CheckpointDurableExecutionResponse.builder() + .checkpointToken("next") + .newExecutionState(CheckpointUpdatedExecutionState.builder() + .operations(response) + .build()) + .build()); + var update = OperationUpdate.builder() + .id("write") + .type(OperationType.STEP) + .action(OperationAction.START) + .build(); + try { + var poll = manager.pollForUpdate("watched", attempt -> Duration.ofHours(1)); + CompletableFuture live = null; + if (mode.equals("same-id") || mode.equals("other-id")) + live = manager.pollForUpdate(liveId, attempt -> Duration.ofHours(1)); + switch (mode) { + case "exceptional" -> poll.completeExceptionally(new IllegalStateException("already failed")); + case "cancelled" -> poll.cancel(false); + case "terminal" -> poll.complete(terminal); + case "pending" -> {} + default -> poll.complete(ready); + } + var checkpoint = mode.equals("checkpoint") ? manager.checkpoint(update) : null; + // Force the real delayed batch now, without closing CheckpointManager (which would clear the poll map). + // This exercises the same pending batch as its timer, without a timing-based no-extra-call assertion. + var field = CheckpointManager.class.getDeclaredField("checkpointApiRequestDelayedBatcher"); + field.setAccessible(true); + var dispatcher = (ApiRequestDelayedBatcher) field.get(manager); + dispatcher.shutdown(); + boolean required = live != null || checkpoint != null || mode.equals("pending"); + System.out.println("COMPLETED_POLL mode=" + mode + " APIleases=" + starts.get() + " finished=" + + finishes.get() + " callbacks=" + callbacks.get()); + verify(client, times(required ? 1 : 0)) + .checkpoint(eq("arn:test"), eq("token"), eq(checkpoint != null ? List.of(update) : List.of())); + assertEquals(required ? 1 : 0, starts.get()); + assertEquals(starts.get(), finishes.get()); + assertEquals(required ? 1 : 0, callbacks.get()); + if (live != null) assertEquals(response, live.get(3, TimeUnit.SECONDS)); + if (checkpoint != null) checkpoint.get(3, TimeUnit.SECONDS); + if (mode.equals("pending")) assertEquals(response, poll.get(3, TimeUnit.SECONDS)); + else if (mode.equals("cancelled")) assertTrue(poll.isCancelled()); + else if (mode.equals("exceptional")) assertTrue(poll.isCompletedExceptionally()); + else assertSame(mode.equals("terminal") ? terminal : ready, poll.getNow(null)); + } finally { + manager.shutdown(); + } + } +} From 2e3edf1719367d93451687cdd3d1bcae87137823 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 9 Oct 2026 18:02:15 +0000 Subject: [PATCH 37/40] fix: close READY admission before output preparation --- otel-plugin/README.md | 15 +- .../otel/InvocationLateFlushProbeTest.java | 162 ++++++++++ ...vocationPreparationAdmissionProbeTest.java | 290 ++++++++++++++++++ .../durable/execution/DurableExecutor.java | 1 + .../durable/execution/ExecutionManager.java | 7 + 5 files changed, 470 insertions(+), 5 deletions(-) create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationLateFlushProbeTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationPreparationAdmissionProbeTest.java diff --git a/otel-plugin/README.md b/otel-plugin/README.md index 4ee0dc66a..d13673865 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -16,9 +16,9 @@ OpenTelemetry instrumentation plugin for the AWS Lambda Durable Execution SDK fo ## Checkpoint continuation failures An operation's SDK checkpoint continuation reports resumption/deserialization and dispatch failures through -retryable invocation control before releasing its activity lease. End reports `RETRYING`, the caller receives -`UnrecoverableDurableExecutionException` with the original cause, and persisted operation state remains available -for a later invocation. Rejected worker admission releases its activity registration. Direct `VirtualMachineError` +retryable invocation control before releasing its activity lease. When that control wins invocation outcome selection, +End reports `RETRYING`, the caller receives `UnrecoverableDurableExecutionException` with the original cause, and +persisted operation state remains available for a later invocation. Rejected worker admission releases its activity registration. Direct `VirtualMachineError` and `ThreadDeath` also settle the continuation observation before escaping its coordinator worker. Legacy outcome observers wake waiters for an already-selected continuation failure before invocation End can wait @@ -29,8 +29,13 @@ Ordinary unowned helper failures retain their observation-only behavior. Normal the selected outcome or stop unrelated operations. This boundary does not change handler/predicate failure classification, hook threading, trace topology, or add general wrapped-fatal classification. -The invocation End snapshot is taken after closing new checkpoint-continuation admission and signaling unfinished -owned continuations. Queued work may legitimately remain incomplete after early parallel success. This does not +After a successful root outcome is selected, new checkpoint-continuation admission closes before output serialization +or oversized-result checkpointing. This early guard does not iterate owners, run their completion callbacks, or join +cleanup. A subsequent rejected admission retains the existing stop-owner behavior. The selected root outcome is not +replaced by a later unawaited failure. + +The invocation End snapshot is taken after signaling unfinished owned continuations; PENDING and failure paths also +close admission at that boundary. Queued work may legitimately remain incomplete after early parallel success. This does not join all accepted/running handlers before End: their remaining cleanup/checkpoint completion is still awaited by normal manager close afterward, and late end hooks can occur. Await operations that must be included in the selected outcome or snapshot. The admission cut does not provide a universal all-work-drained or final-state export guarantee. diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationLateFlushProbeTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationLateFlushProbeTest.java new file mode 100644 index 000000000..108756b60 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationLateFlushProbeTest.java @@ -0,0 +1,162 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.CompletionConfig; +import software.amazon.lambda.durable.config.ParallelConfig; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +/** Diagnostic for the pre-existing End/close flush boundary; no proposed production policy in this class. */ +class InvocationLateFlushProbeTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void alreadyRunningStepCompletesAfterEnd(boolean earlyParallel) throws Exception { + var running = new CountDownLatch(1); + var release = new CountDownLatch(1); + var ended = new CountDownLatch(1); + var lateOperationEnds = new AtomicInteger(); + var bodyCalls = new AtomicInteger(); + var client = new LocalMemoryExecutionClient(); + var workers = Executors.newCachedThreadPool(); + var caller = Executors.newSingleThreadExecutor(); + try (var exporter = InMemorySpanExporter.create(); + var batch = BatchSpanProcessor.builder(exporter) + .setScheduleDelay(Duration.ofDays(1)) + .build()) { + var otel = new InvocationOtelPlugin( + SdkTracerProvider.builder().addSpanProcessor(batch), + OtelPluginConfig.builder() + .contextExtractor(() -> null) + .enableMdc(false) + .build()); + var config = DurableConfig.builder() + .withDurableExecutionClient(client) + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withPlugins(otel, new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + ended.countDown(); + } + + public void onOperationEnd(OperationEndInfo info) { + if (ended.getCount() == 0) lateOperationEnds.incrementAndGet(); + } + }) + .build(); + try { + var call = caller.submit(() -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, root) -> { + if (earlyParallel) { + var parallel = root.parallel( + "early", + ParallelConfig.builder() + .completionConfig(CompletionConfig.minSuccessful(1)) + .build()); + try (parallel) { + parallel.branch( + "waiting", + String.class, + child -> child.step("held", String.class, step -> { + bodyCalls.incrementAndGet(); + running.countDown(); + await(release); + return "completed"; + })); + parallel.branch("winner", String.class, child -> { + await(running); + return "winner"; + }); + } + assertEquals(1, parallel.get().succeeded()); + } else { + root.stepAsync("held", String.class, step -> { + bodyCalls.incrementAndGet(); + running.countDown(); + await(release); + return "completed"; + }); + await(running); + } + return "root-success"; + }, + config)); + assertTrue(ended.await(5, TimeUnit.SECONDS)); + assertThrows(TimeoutException.class, () -> call.get(100, TimeUnit.MILLISECONDS)); + release.countDown(); + assertEquals( + ExecutionStatus.SUCCEEDED, call.get(5, TimeUnit.SECONDS).status()); + assertEquals( + OperationStatus.SUCCEEDED, + client.getOperationByName("held").status()); + assertEquals(1, bodyCalls.get()); + var before = exporter.getFinishedSpanItems().stream() + .filter(x -> x.getName().equals("held")) + .map(x -> x.getAttributes().get(SpanAttributes.DURABLE_OPERATION_STATUS)) + .toList(); + batch.forceFlush().join(3, TimeUnit.SECONDS); + var after = exporter.getFinishedSpanItems().stream() + .filter(x -> x.getName().equals("held")) + .map(x -> x.getAttributes().get(SpanAttributes.DURABLE_OPERATION_STATUS)) + .toList(); + System.out.println("LATE_FLUSH_PUBLIC earlyParallel=" + earlyParallel + " backend=SUCCEEDED bodyCalls=" + + bodyCalls.get() + " lateOperationEnds=" + lateOperationEnds.get() + " beforeManualFlush=" + + before + " afterManualFlush=" + after); + assertTrue(lateOperationEnds.get() > 0, "Existing public operation-end occurs after invocation End"); + // This diagnostic records actual export behavior; it does not redefine which outcome wins. + } finally { + release.countDown(); + caller.shutdownNow(); + workers.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + } + + private static void await(CountDownLatch latch) { + try { + assertTrue(latch.await(8, TimeUnit.SECONDS)); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new AssertionError(failure); + } + } + + private static DurableExecutionInput input() { + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/late-flush/execution", + "token", + CheckpointUpdatedExecutionState.builder() + .operations(Operation.builder() + .id("execution") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails(ExecutionDetails.builder() + .inputPayload("\"input\"") + .build()) + .build()) + .build()); + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationPreparationAdmissionProbeTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationPreparationAdmissionProbeTest.java new file mode 100644 index 000000000..665bca59e --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationPreparationAdmissionProbeTest.java @@ -0,0 +1,290 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.*; +import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.model.WaitForConditionResult; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.serde.*; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +/** Diagnostic only: distinguishes new admission from rewriting an already-selected outcome. */ +class InvocationPreparationAdmissionProbeTest { + @ParameterizedTest + @CsvSource({ + "unawaited,false", + "unawaited,true", + "early-parallel,false", + "early-parallel,true", + "awaited,false", + "awaited,true" + }) + void preparationAdmissionWindow(String mode, boolean rejectResume) throws Exception { + var pending = new CountDownLatch(1); + var preparationEntered = new CountDownLatch(1); + var releasePreparation = new CountDownLatch(1); + var conditionFinished = new CountDownLatch(1); + var keeperEntered = new CountDownLatch(1); + var releaseKeeper = new CountDownLatch(1); + var backendReleased = new AtomicBoolean(); + var preparing = new AtomicBoolean(); + var predicates = new AtomicInteger(); + var preparationDispatches = new AtomicInteger(); + var childFailure = new AtomicReference(); + var ends = new CopyOnWriteArrayList(); + var rejection = new RejectedExecutionException("controlled READY admission rejection"); + var workers = + new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue()) { + @Override + public void execute(Runnable task) { + if (backendReleased.get()) { + if (preparing.get()) preparationDispatches.incrementAndGet(); + if (rejectResume) throw rejection; + } + super.execute(task); + } + }; + var client = new LocalMemoryExecutionClient() { + @Override + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List updates) { + var result = super.checkpoint(arn, token, updates); + var op = getOperationByName("condition"); + if (op != null && op.status() == OperationStatus.PENDING) pending.countDown(); + return result; + } + }; + var serializer = new SerDes() { + private final JacksonSerDes delegate = new JacksonSerDes(); + + public String serialize(Object value) { + if ("selected-root".equals(value)) { + preparing.set(true); + preparationEntered.countDown(); + await(releasePreparation); + } + return delegate.serialize(value); + } + + public T deserialize(String value, TypeToken type) { + return delegate.deserialize(value, type); + } + }; + var config = DurableConfig.builder() + .withExecutorService(workers) + .withDurableExecutionClient(client) + .withSerDes(serializer) + .withCheckpointDelay(Duration.ZERO) + .withPollingStrategy(attempt -> Duration.ofMillis(1)) + .withPlugins(new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + ends.add(info); + } + }) + .build(); + var caller = Executors.newSingleThreadExecutor(); + try { + var call = caller.submit(() -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, root) -> { + if (mode.equals("awaited")) { + root.runInChildContextAsync("keep-active", String.class, child -> { + keeperEntered.countDown(); + await(releaseKeeper); + return "keeper"; + }); + await(keeperEntered); + try { + condition(root, predicates, childFailure, conditionFinished); + } finally { + releaseKeeper.countDown(); + } + } else if (mode.equals("early-parallel")) { + var parallel = root.parallel( + "early", + ParallelConfig.builder() + .completionConfig(CompletionConfig.minSuccessful(1)) + .build()); + try (parallel) { + parallel.branch( + "waiting", + Integer.class, + child -> condition(child, predicates, childFailure, conditionFinished)); + parallel.branch("winner", String.class, child -> { + await(pending); + return "winner"; + }); + } + assertEquals(1, parallel.get().succeeded()); + } else { + root.runInChildContextAsync( + "unawaited", + Integer.class, + child -> condition(child, predicates, childFailure, conditionFinished)); + await(pending); + } + return "selected-root"; + }, + config)); + if (!pending.await(5, TimeUnit.SECONDS)) { + System.out.println("PROBE_SETUP mode=" + mode + " done=" + call.isDone()); + if (call.isDone()) System.out.println("PROBE_EARLY_RESULT " + call.get()); + fail("Real backend did not reach PENDING"); + } + if (!mode.equals("awaited")) + assertTrue( + preparationEntered.await(5, TimeUnit.SECONDS), + "Root success selected before real output serializer"); + backendReleased.set(true); + client.advanceTime(); + assertTrue(conditionFinished.await(5, TimeUnit.SECONDS), "Real durable waiter settled after backend READY"); + releasePreparation.countDown(); + releaseKeeper.countDown(); + if (mode.equals("awaited") && rejectResume) { + var thrown = assertThrows(ExecutionException.class, () -> call.get(5, TimeUnit.SECONDS)); + assertInstanceOf(UnrecoverableDurableExecutionException.class, thrown.getCause()); + assertSame(rejection, thrown.getCause().getCause()); + assertEquals(InvocationStatus.RETRYING, ends.get(0).invocationStatus()); + } else { + assertEquals( + ExecutionStatus.SUCCEEDED, call.get(5, TimeUnit.SECONDS).status()); + assertEquals(InvocationStatus.SUCCEEDED, ends.get(0).invocationStatus()); + } + assertEquals(1, ends.size()); + System.out.println("PREPARATION_ADMISSION mode=" + mode + " reject=" + rejectResume + + " dispatchesWhilePreparing=" + preparationDispatches.get() + + " predicates=" + predicates.get() + " childFailure=" + + (childFailure.get() == null + ? "none" + : childFailure.get().getClass().getName()) + + " originalRejectionCause=" + + (childFailure.get() != null && childFailure.get().getCause() == rejection) + + " backend=" + client.getOperationByName("condition").status() + + " End=" + ends.get(0).invocationStatus()); + if (!mode.equals("awaited")) { + // Proposed admission guarantee, distinct from replacing the selected SUCCEEDED outcome. + assertEquals(0, preparationDispatches.get(), "No new READY user-worker admission after root selection"); + assertEquals(1, predicates.get()); + } else assertEquals(rejectResume ? 1 : 2, predicates.get()); + } finally { + releasePreparation.countDown(); + releaseKeeper.countDown(); + caller.shutdownNow(); + workers.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + void successfulOversizedCheckpointStillRunsAfterAdmissionCloses() throws Exception { + var client = new LocalMemoryExecutionClient(); + var workers = Executors.newCachedThreadPool(); + var caller = Executors.newSingleThreadExecutor(); + var ends = new CopyOnWriteArrayList(); + var payload = "x".repeat(6 * 1024 * 1024); + try { + var config = DurableConfig.builder() + .withDurableExecutionClient(client) + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withPlugins(new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + ends.add(info); + } + }) + .build(); + var result = caller.submit(() -> DurableExecutor.execute( + input(), null, TypeToken.get(String.class), (value, context) -> payload, config)) + .get(5, TimeUnit.SECONDS); + assertEquals(ExecutionStatus.SUCCEEDED, result.status()); + assertEquals("", result.result()); + var updates = client.getOperationUpdates().stream() + .filter(x -> x.type() == OperationType.EXECUTION) + .toList(); + assertEquals(1, updates.size()); + assertEquals(OperationAction.SUCCEED, updates.get(0).action()); + assertEquals(new JacksonSerDes().serialize(payload), updates.get(0).payload()); + assertEquals(1, ends.size()); + assertEquals(InvocationStatus.SUCCEEDED, ends.get(0).invocationStatus()); + } finally { + caller.shutdownNow(); + workers.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + private static int condition( + DurableContext context, + AtomicInteger predicates, + AtomicReference failure, + CountDownLatch finished) { + try { + return context.waitForCondition( + "condition", + Integer.class, + (state, step) -> { + predicates.incrementAndGet(); + return state == 1 + ? WaitForConditionResult.continuePolling(2) + : WaitForConditionResult.stopPolling(state); + }, + WaitForConditionConfig.builder() + .initialState(1) + .waitStrategy((state, attempt) -> Duration.ofSeconds(1)) + .build()); + } catch (RuntimeException | Error thrown) { + failure.set(thrown); + throw thrown; + } finally { + finished.countDown(); + } + } + + private static void await(CountDownLatch latch) { + try { + assertTrue(latch.await(8, TimeUnit.SECONDS), "Probe gate released within bounded cleanup"); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new AssertionError(interrupted); + } + } + + private static DurableExecutionInput input() { + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/preparation/execution", + "token", + CheckpointUpdatedExecutionState.builder() + .operations(Operation.builder() + .id("execution") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails(ExecutionDetails.builder() + .inputPayload("\"input\"") + .build()) + .build()) + .build()); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index bb74f51a6..eb9a189f3 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -176,6 +176,7 @@ public static DurableExecutionOutput execute( } // user handler complete successfully logger.debug("Execution completed"); + executionManager.closeCheckpointContinuationAdmission(); DurableExecutionOutput output; try { var outputPayload = config.getSerDes().serialize(result); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java index a1c52be1e..f707fcd93 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java @@ -755,6 +755,13 @@ public void close() { checkpointManager.shutdown(); } + /** Closes new checkpoint-continuation admission without signaling owners or joining their cleanup. */ + void closeCheckpointContinuationAdmission() { + synchronized (activeThreads) { + closing = true; + } + } + /** Stops continuation admission and signals unfinished owners; explicit task/handler draining stays in close. */ void beginInvocationEnd() { List owners; From f8e1c09702750e15f9bc2c8537ada6adc4963be0 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 9 Oct 2026 18:32:08 +0000 Subject: [PATCH 38/40] fix: close OTel user scopes on their owning threads --- otel-plugin/README.md | 5 + .../durable/otel/ExecutionOtelPlugin.java | 23 +- .../durable/otel/InvocationOtelPlugin.java | 19 +- .../durable/otel/UserFunctionScopes.java | 33 ++ ...vocationWorkerContextCleanupProbeTest.java | 349 ++++++++++++++++++ 5 files changed, 399 insertions(+), 30 deletions(-) create mode 100644 otel-plugin/src/main/java/software/amazon/lambda/durable/otel/UserFunctionScopes.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationWorkerContextCleanupProbeTest.java diff --git a/otel-plugin/README.md b/otel-plugin/README.md index d13673865..ab7d028aa 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -40,6 +40,11 @@ join all accepted/running handlers before End: their remaining cleanup/checkpoin normal manager close afterward, and late end hooks can occur. Await operations that must be included in the selected outcome or snapshot. The admission cut does not provide a universal all-work-drained or final-state export guarantee. +OTel scopes remain with the user-function thread that opened them. Invocation End closes only scopes owned by its +current thread; a running worker retains its scope until its own UserFunctionEnd, including when tracing has already +been finalized. Late End performs that scope cleanup before checking tracing state. This restores the worker's +application context without extending recording-span lifetimes, changing MDC behavior, or moving the End/drain boundary. + ## Installation ```xml diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index be222812a..150a3fd31 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -15,7 +15,6 @@ import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; -import io.opentelemetry.context.Scope; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import java.time.Instant; @@ -128,7 +127,7 @@ private record ExecutionTrace(String traceId, TraceFlags flags) {} // Thread-safe storage for attempt spans/scopes (keyed by operationId + "-" + attempt) private final ConcurrentHashMap attemptSpans = new ConcurrentHashMap<>(); - private final ConcurrentHashMap attemptScopes = new ConcurrentHashMap<>(); + private final UserFunctionScopes attemptScopes = new UserFunctionScopes(); // Deterministic operation contexts (keyed by operationId), held between start and end so children and attempts can // parent onto an operation whose recording span is not created until onOperationEnd. @@ -320,12 +319,9 @@ private void finishInvocationEnd(InvocationEndInfo info) { operationContexts.clear(); operationStartTimes.clear(); - // Release OTel context on worker threads, then end any attempt spans still open so no recording span is - // abandoned. Attempt spans normally start and end within one user-function call, so this is a safeguard. - for (var scope : attemptScopes.values()) { - scope.close(); - } - attemptScopes.clear(); + // End open recording spans now, but leave a running worker's thread-bound scope for its own user-function End. + // Only scopes owned by this finalization thread can be closed here. + attemptScopes.closeCurrentThread(); for (var span : attemptSpans.values()) { span.end(); } @@ -511,15 +507,10 @@ public void onUserFunctionStart(UserFunctionStartInfo info) { @Override public void onUserFunctionEnd(UserFunctionEndInfo info) { - if (!tracingEnabled) return; - var key = attemptKey(info.id(), info.attempt()); - - // Close scope first (must happen on same thread as makeCurrent) - var scope = attemptScopes.remove(key); - if (scope != null) { - scope.close(); - } + // A running user function can finish after invocation End; its scope still belongs to this thread. + attemptScopes.close(key); + if (!tracingEnabled) return; // CONTEXT operations don't have attempt spans — scope cleanup is all we need if ("CONTEXT".equals(info.type())) { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index 1b24f5dfe..627d753b7 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -15,7 +15,6 @@ import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; -import io.opentelemetry.context.Scope; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import java.time.Instant; @@ -115,7 +114,7 @@ private record ExecutionTrace(String traceId, TraceFlags flags) {} // Thread-safe storage for attempt spans/scopes (keyed by operationId + "-" + attempt) private final ConcurrentHashMap attemptSpans = new ConcurrentHashMap<>(); - private final ConcurrentHashMap attemptScopes = new ConcurrentHashMap<>(); + private final UserFunctionScopes attemptScopes = new UserFunctionScopes(); // Store operation span contexts for parent resolution (keyed by operationId) private final ConcurrentHashMap operationContexts = new ConcurrentHashMap<>(); @@ -553,15 +552,10 @@ public void onUserFunctionStart(UserFunctionStartInfo info) { @Override public void onUserFunctionEnd(UserFunctionEndInfo info) { - if (!tracingEnabled) return; - var key = attemptKey(info.id(), info.attempt()); - - // Close scope first (must happen on same thread as makeCurrent) - var scope = attemptScopes.remove(key); - if (scope != null) { - scope.close(); - } + // A running user function can finish after invocation End; its scope still belongs to this thread. + attemptScopes.close(key); + if (!tracingEnabled) return; // Clear span-level MDC after user function completes (keep trace_id for handler-level logs between steps) if (enableMdc) { @@ -620,10 +614,7 @@ private boolean bindTracer() { private void endOpenSpansChildFirst() { // Attempt spans are children of operation spans. - for (var scope : attemptScopes.values()) { - scope.close(); - } - attemptScopes.clear(); + attemptScopes.closeCurrentThread(); for (var span : attemptSpans.values()) { span.end(); } diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/UserFunctionScopes.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/UserFunctionScopes.java new file mode 100644 index 000000000..f2bfb2647 --- /dev/null +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/UserFunctionScopes.java @@ -0,0 +1,33 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import io.opentelemetry.context.Scope; +import java.util.ArrayList; +import java.util.LinkedHashMap; + +/** Keeps thread-bound OTel scopes with their user-function owner, including after invocation End. */ +final class UserFunctionScopes { + private final ThreadLocal> scopes = ThreadLocal.withInitial(LinkedHashMap::new); + + void put(String key, Scope scope) { + scopes.get().put(key, scope); + } + + void close(String key) { + var owned = scopes.get(); + var scope = owned.remove(key); + if (owned.isEmpty()) scopes.remove(); + if (scope != null) scope.close(); + } + + /** Closes abandoned scopes only on this thread; running foreign owners retain their late End cleanup. */ + void closeCurrentThread() { + var owned = scopes.get(); + scopes.remove(); + var open = new ArrayList<>(owned.values()); + for (var index = open.size() - 1; index >= 0; index--) { + open.get(index).close(); + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationWorkerContextCleanupProbeTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationWorkerContextCleanupProbeTest.java new file mode 100644 index 000000000..baddca683 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationWorkerContextCleanupProbeTest.java @@ -0,0 +1,349 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextKey; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.CompletionConfig; +import software.amazon.lambda.durable.config.ParallelConfig; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +/** Diagnostic for the pre-existing End/close flush boundary; no proposed production policy in this class. */ +class InvocationWorkerContextCleanupProbeTest { + @ParameterizedTest + @CsvSource({ + "unawaited,false", + "early-parallel,false", + "awaited,false", + "unawaited,true", + "early-parallel,true", + "awaited,true" + }) + void alreadyRunningStepCompletesAfterEnd(String mode, boolean executionView) throws Exception { + boolean earlyParallel = mode.equals("early-parallel"); + boolean awaited = mode.equals("awaited"); + var running = new CountDownLatch(1); + var release = new CountDownLatch(1); + var ended = new CountDownLatch(1); + var lateOperationEnds = new AtomicInteger(); + var bodyCalls = new AtomicInteger(); + var client = new LocalMemoryExecutionClient(); + var owner = new AtomicReference(); + var initial = new ConcurrentHashMap(); + var workerContext = new AtomicReference(); + var captured = new AtomicBoolean(); + var retired = new CountDownLatch(1); + var workers = + new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue()) { + protected void beforeExecute(Thread thread, Runnable task) { + initial.put(thread, Context.current()); + } + + protected void afterExecute(Runnable task, Throwable failure) { + if (Thread.currentThread() == owner.get() && captured.compareAndSet(false, true)) { + workerContext.set(Context.current()); + retired.countDown(); + } + } + }; + var caller = Executors.newSingleThreadExecutor(); + try (var exporter = InMemorySpanExporter.create(); + var batch = BatchSpanProcessor.builder(exporter) + .setScheduleDelay(Duration.ofDays(1)) + .build()) { + var builder = SdkTracerProvider.builder().addSpanProcessor(batch); + var pluginConfig = OtelPluginConfig.builder() + .contextExtractor(() -> null) + .enableMdc(false) + .build(); + DurableExecutionPlugin otel = executionView + ? new ExecutionOtelPlugin(builder, pluginConfig) + : new InvocationOtelPlugin(builder, pluginConfig); + var config = DurableConfig.builder() + .withDurableExecutionClient(client) + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withPlugins(otel, new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + ended.countDown(); + } + + public void onOperationEnd(OperationEndInfo info) { + if (ended.getCount() == 0) lateOperationEnds.incrementAndGet(); + } + }) + .build(); + try { + var call = caller.submit(() -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, root) -> { + if (earlyParallel) { + var parallel = root.parallel( + "early", + ParallelConfig.builder() + .completionConfig(CompletionConfig.minSuccessful(1)) + .build()); + try (parallel) { + parallel.branch( + "waiting", + String.class, + child -> child.step("held", String.class, step -> { + owner.set(Thread.currentThread()); + bodyCalls.incrementAndGet(); + running.countDown(); + await(release); + return "completed"; + })); + parallel.branch("winner", String.class, child -> { + await(running); + return "winner"; + }); + } + assertEquals(1, parallel.get().succeeded()); + } else { + var future = root.stepAsync("held", String.class, step -> { + owner.set(Thread.currentThread()); + bodyCalls.incrementAndGet(); + running.countDown(); + await(release); + return "completed"; + }); + await(running); + if (awaited) future.get(); + } + return "root-success"; + }, + config)); + if (awaited) { + assertTrue(running.await(5, TimeUnit.SECONDS)); + release.countDown(); + } + assertTrue(ended.await(5, TimeUnit.SECONDS)); + if (!awaited) assertThrows(TimeoutException.class, () -> call.get(100, TimeUnit.MILLISECONDS)); + release.countDown(); + assertEquals( + ExecutionStatus.SUCCEEDED, call.get(5, TimeUnit.SECONDS).status()); + assertEquals( + OperationStatus.SUCCEEDED, + client.getOperationByName("held").status()); + assertEquals(1, bodyCalls.get()); + assertTrue(retired.await(5, TimeUnit.SECONDS)); + var before = exporter.getFinishedSpanItems().stream() + .filter(x -> x.getName().equals("held")) + .map(x -> x.getAttributes().get(SpanAttributes.DURABLE_OPERATION_STATUS)) + .toList(); + batch.forceFlush().join(3, TimeUnit.SECONDS); + var after = exporter.getFinishedSpanItems().stream() + .filter(x -> x.getName().equals("held")) + .map(x -> x.getAttributes().get(SpanAttributes.DURABLE_OPERATION_STATUS)) + .toList(); + System.out.println("LATE_FLUSH_PUBLIC earlyParallel=" + earlyParallel + " backend=SUCCEEDED bodyCalls=" + + bodyCalls.get() + " lateOperationEnds=" + lateOperationEnds.get() + " beforeManualFlush=" + + before + " afterManualFlush=" + after); + if (!awaited) + assertTrue( + lateOperationEnds.get() > 0, "Existing public operation-end occurs after invocation End"); + System.out.println("WORKER_CONTEXT_PUBLIC executionView=" + executionView + " mode=" + mode + + " restored=" + (initial.get(owner.get()) == workerContext.get()) + " remainingSpan=" + + Span.fromContext(workerContext.get()).getSpanContext()); + assertSame( + initial.get(owner.get()), + workerContext.get(), + "Real worker ambient Context restored after its task finishes"); + // This diagnostic records actual export behavior; it does not redefine which outcome wins. + } finally { + release.countDown(); + caller.shutdownNow(); + workers.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void retainedPluginAndWorkerRestoreApplicationAmbientOnReuse(boolean executionView) throws Exception { + var key = ContextKey.named("application-worker"); + var ambient = Context.root().with(key, "retained-application-context"); + var rootExecutor = Executors.newSingleThreadExecutor(); + var nextRoot = new AtomicBoolean(true); + var retired = new AtomicReference(); + var before = new AtomicReference(); + var after = new AtomicReference(); + var actualWorker = new AtomicReference(); + var workerExecutor = + new ThreadPoolExecutor( + 1, + 1, + 0, + TimeUnit.SECONDS, + new LinkedBlockingQueue(), + task -> new Thread( + () -> { + try (var ignored = ambient.makeCurrent()) { + task.run(); + } + }, + "owned-operation-worker")) { + protected void beforeExecute(Thread thread, Runnable task) { + actualWorker.set(thread); + before.set(Context.current()); + } + + protected void afterExecute(Runnable task, Throwable failure) { + after.set(Context.current()); + retired.get().countDown(); + } + }; + var dispatch = new AbstractExecutorService() { + public void execute(Runnable task) { + (nextRoot.compareAndSet(true, false) ? rootExecutor : workerExecutor).execute(task); + } + + public void shutdown() { + rootExecutor.shutdown(); + workerExecutor.shutdown(); + } + + public List shutdownNow() { + var pending = new ArrayList<>(rootExecutor.shutdownNow()); + pending.addAll(workerExecutor.shutdownNow()); + return pending; + } + + public boolean isShutdown() { + return rootExecutor.isShutdown() && workerExecutor.isShutdown(); + } + + public boolean isTerminated() { + return rootExecutor.isTerminated() && workerExecutor.isTerminated(); + } + + public boolean awaitTermination(long duration, TimeUnit unit) throws InterruptedException { + return rootExecutor.awaitTermination(duration, unit) && workerExecutor.awaitTermination(duration, unit); + } + }; + var caller = Executors.newSingleThreadExecutor(); + try (var exporter = InMemorySpanExporter.create(); + var batch = BatchSpanProcessor.builder(exporter) + .setScheduleDelay(Duration.ofDays(1)) + .build()) { + var builder = SdkTracerProvider.builder().addSpanProcessor(batch); + var pluginConfig = OtelPluginConfig.builder() + .contextExtractor(() -> null) + .enableMdc(false) + .build(); + DurableExecutionPlugin otel = executionView + ? new ExecutionOtelPlugin(builder, pluginConfig) + : new InvocationOtelPlugin(builder, pluginConfig); + Thread firstWorker = null; + for (var round = 0; round < 2; round++) { + var running = new CountDownLatch(1); + var release = new CountDownLatch(1); + var ended = new CountDownLatch(1); + retired.set(new CountDownLatch(1)); + nextRoot.set(true); + var config = DurableConfig.builder() + .withExecutorService(dispatch) + .withDurableExecutionClient(new LocalMemoryExecutionClient()) + .withCheckpointDelay(Duration.ZERO) + .withPlugins(otel, new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + ended.countDown(); + } + }) + .build(); + try { + var call = caller.submit(() -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, root) -> { + root.stepAsync("held", String.class, step -> { + assertEquals( + "retained-application-context", + Context.current().get(key)); + assertTrue(Span.current().getSpanContext().isValid()); + running.countDown(); + await(release); + return "done"; + }); + await(running); + return "root-success"; + }, + config)); + assertTrue(ended.await(5, TimeUnit.SECONDS)); + release.countDown(); + assertEquals( + ExecutionStatus.SUCCEEDED, + call.get(5, TimeUnit.SECONDS).status()); + assertTrue(retired.get().await(5, TimeUnit.SECONDS)); + assertSame(ambient, before.get()); + assertSame(ambient, after.get()); + if (firstWorker == null) firstWorker = actualWorker.get(); + else assertSame(firstWorker, actualWorker.get(), "The same actual worker was reused"); + System.out.println("WORKER_REUSE_PUBLIC executionView=" + executionView + " round=" + round + + " ambientRestored=true"); + } finally { + release.countDown(); + } + } + } finally { + caller.shutdownNow(); + dispatch.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(dispatch.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + private static void await(CountDownLatch latch) { + try { + assertTrue(latch.await(8, TimeUnit.SECONDS)); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new AssertionError(failure); + } + } + + private static DurableExecutionInput input() { + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/late-flush/execution", + "token", + CheckpointUpdatedExecutionState.builder() + .operations(Operation.builder() + .id("execution") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails(ExecutionDetails.builder() + .inputPayload("\"input\"") + .build()) + .build()) + .build()); + } +} From 3a6ad164f83dc14c2c223cbc13c3d0d0b1f8eb91 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 9 Oct 2026 18:32:08 +0000 Subject: [PATCH 39/40] test: stabilize percentage histories and retain concurrent replay Reuse the reviewed test-only pattern from 3b54d295c2000ef1ead2421564e045a4d7eaf12b and 87a83ea4436e717fd31b4a7114b6c439924df177. Preserve exact serialized-history assertions and independent default-concurrency, threshold, stored-result and side-effect replay coverage. --- .../lambda/durable/MapIntegrationTest.java | 95 ++++++++++++++++++- 1 file changed, 92 insertions(+), 3 deletions(-) diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/MapIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/MapIntegrationTest.java index 7d2b12982..65dbe2383 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/MapIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/MapIntegrationTest.java @@ -7,8 +7,11 @@ import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -844,8 +847,10 @@ void testNestedMap_replay(NestingType outerNestingType, NestingType innerNesting @CsvSource({"FLAT, 2", "NESTED, 16"}) void testMapWithToleratedFailurePercentage(NestingType nestingType, int events) { var runner = LocalDurableTestRunner.create(String.class, (input, context) -> { - var items = List.of("ok1", "FAIL1", "ok2", "FAIL2", "ok3", "FAIL3", "ok4"); + // Put the threshold-crossing failure last and serialize admission so exact history counts are stable. + var items = List.of("ok1", "ok2", "ok3", "ok4", "FAIL1", "FAIL2", "FAIL3"); var config = MapConfig.builder() + .maxConcurrency(1) .completionConfig(CompletionConfig.toleratedFailurePercentage(0.3)) .nestingType(nestingType) .build(); @@ -876,8 +881,10 @@ void testMapWithToleratedFailurePercentage_replay(NestingType nestingType, int e var executionCount = new AtomicInteger(0); var runner = LocalDurableTestRunner.create(String.class, (input, context) -> { - var items = List.of("ok1", "FAIL1", "ok2", "FAIL2", "ok3", "FAIL3", "ok4"); + // Put the threshold-crossing failure last and serialize admission so exact history counts are stable. + var items = List.of("ok1", "ok2", "ok3", "ok4", "FAIL1", "FAIL2", "FAIL3"); var config = MapConfig.builder() + .maxConcurrency(1) .completionConfig(CompletionConfig.toleratedFailurePercentage(0.3)) .nestingType(nestingType) .build(); @@ -901,9 +908,10 @@ void testMapWithToleratedFailurePercentage_replay(NestingType nestingType, int e var result1 = runner.runUntilComplete("test"); assertEquals(ExecutionStatus.SUCCEEDED, result1.getStatus()); var firstRunCount = executionCount.get(); + assertEquals(7, firstRunCount); assertEquals(events, result1.getHistoryEvents().size()); - // Replay — with unlimited concurrency, children replay simultaneously. + // Replay retains the checkpointed completion decision and every previously completed child. // Verify completionReason is consistent and no re-execution occurs. var result2 = runner.run("test"); assertEquals(ExecutionStatus.SUCCEEDED, result2.getStatus()); @@ -911,6 +919,87 @@ void testMapWithToleratedFailurePercentage_replay(NestingType nestingType, int e assertEquals(events, result2.getHistoryEvents().size()); } + @ParameterizedTest + @CsvSource({"FLAT", "NESTED"}) + void testDefaultConcurrencyPercentageFailureReplaysStoredResults(NestingType nestingType) { + var items = List.of("ok1", "FAIL1", "ok2", "FAIL2", "ok3", "FAIL3", "ok4"); + var executions = new AtomicIntegerArray(items.size()); + var runner = percentageReplayRunner(nestingType, items, executions); + var resultType = new TypeToken>() {}; + + var first = runner.runUntilComplete("test"); + assertEquals(ExecutionStatus.SUCCEEDED, first.getStatus()); + var stored = first.getResult(resultType); + assertPercentageFailureResults(items, stored); + for (var index = 0; index < items.size(); index++) assertEquals(1, executions.get(index)); + + var replay = runner.run("test"); + assertEquals(ExecutionStatus.SUCCEEDED, replay.getStatus()); + assertEquals(stored, replay.getResult(resultType), "Replay returns the exact checkpointed result and errors"); + assertEquals(first.getHistoryEvents().size(), replay.getHistoryEvents().size()); + for (var index = 0; index < items.size(); index++) { + assertEquals(1, executions.get(index), "Completed child bodies must not repeat on replay"); + } + } + + private static LocalDurableTestRunner> percentageReplayRunner( + NestingType nestingType, List items, AtomicIntegerArray executions) { + var allStarted = new CountDownLatch(items.size()); + return LocalDurableTestRunner.create( + String.class, + (input, context) -> context.map( + "default-percentage-replay", + items, + String.class, + (item, index, child) -> { + executions.incrementAndGet(index); + allStarted.countDown(); + try { + assertTrue( + allStarted.await(5, TimeUnit.SECONDS), + "Default concurrency must admit all children"); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new AssertionError(error); + } + if (item.startsWith("FAIL")) throw new IllegalArgumentException("failed: " + item); + return item.toUpperCase(); + }, + MapConfig.builder() + .completionConfig(CompletionConfig.toleratedFailurePercentage(0.3)) + .nestingType(nestingType) + .build())); + } + + private static void assertPercentageFailureResults(List inputs, MapResult result) { + assertEquals(ConcurrencyCompletionStatus.FAILURE_TOLERANCE_EXCEEDED, result.completionReason()); + assertEquals(inputs.size(), result.size()); + assertEquals(3, result.failed().size(), "The third failure crosses the 30 percent threshold"); + assertTrue(result.failed().size() > inputs.size() * 0.3); + for (var index = 0; index < inputs.size(); index++) { + var item = result.getItem(index); + switch (item.status()) { + case SUCCEEDED -> { + assertFalse(inputs.get(index).startsWith("FAIL")); + assertEquals(inputs.get(index).toUpperCase(), item.result()); + assertNull(item.error()); + } + case FAILED -> { + assertTrue(inputs.get(index).startsWith("FAIL")); + assertNull(item.result()); + assertNotNull(item.error()); + assertTrue(item.error().errorMessage().contains("failed: " + inputs.get(index))); + } + case SKIPPED -> { + // A success racing with the threshold may be absent from the terminal map snapshot. + assertFalse(inputs.get(index).startsWith("FAIL")); + assertNull(item.result()); + assertNull(item.error()); + } + } + } + } + @ParameterizedTest @CsvSource({"FLAT, 12", "NESTED, 16"}) void testMapAsyncWithWaitInsideBranches(NestingType nestingType, int events) { From 466fc1a2ad2291172916f80208275ed64b65b3c7 Mon Sep 17 00:00:00 2001 From: zhongkechen <65260095+zhongkechen@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:53:41 +0000 Subject: [PATCH 40/40] fix: stabilize fallback execution anchors --- otel-plugin/README.md | 46 ++-- .../durable/otel/DurableExecutionRoot.java | 67 ++++++ .../lambda/durable/otel/DurableSampler.java | 28 ++- .../durable/otel/DurableSamplingDecision.java | 14 -- .../durable/otel/ExecutionOtelPlugin.java | 28 ++- .../durable/otel/InvocationOtelPlugin.java | 28 ++- .../durable/otel/OtelPluginSupport.java | 56 +---- ...urableSamplingDecisionClassLoaderTest.java | 44 ++-- .../durable/otel/ExecutionOtelPluginTest.java | 18 +- .../otel/ExecutionRootLifecycleTest.java | 46 ++-- .../durable/otel/ExecutionRootParityTest.java | 205 ++++++++++++++++++ .../otel/InvocationOtelPluginTest.java | 8 +- .../otel/RootStartProcessorIsolationTest.java | 35 ++- 13 files changed, 434 insertions(+), 189 deletions(-) create mode 100644 otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableExecutionRoot.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootParityTest.java diff --git a/otel-plugin/README.md b/otel-plugin/README.md index ab7d028aa..9600a440f 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -74,24 +74,28 @@ If you configure your own `SdkTracerProviderBuilder`, add the OpenTelemetry SDK ## Fallback execution roots -When the backend supplies no complete remote parent, both views export a `DurableExecutionRoot` anchor before the first -invocation returns, including when it suspends with `PENDING` or fails with `RETRYING`. -The anchor is started before descendants so their parent context includes its provider-resolved trace flags and -trace state. It ends at invocation end, before flushing, with the same fixed execution-start timestamp. For a +When the backend supplies no complete remote parent and the installed SDK sampler selects `RECORD_AND_SAMPLE`, +both views export a `DurableExecutionRoot` anchor during the invocation Start hook, including invocations that later +suspend with `PENDING` or fail with `RETRYING`. The anchor starts before descendants and ends immediately with the +same checkpointed execution-start timestamp. It is zero-duration: descendants can end after it, and its ended span +context remains a valid parent. Trace viewers that enforce parent/child time containment may flag this shape. For a deferred sampler, the first actual fallback span is now `DurableExecutionRoot`; custom policies that depend on the first span name or attributes can observe that ordering. No arbitrary name/input invariance is promised. If result serialization or oversized-result checkpointing fails before the SDK output is prepared, End reports -`RETRYING` on the existing completion thread before the original preparation failure propagates. This closes the -retained root without changing handler failure classification or claiming coverage of later response-stream writes -or runtime acknowledgment. A root processor `Exception` or `LinkageError` still permits flushing already completed -spans before the plugin boundary isolates that error. Unisolated `Error` failures keep their existing escape/skip-flush -behavior; a failing processor need not have exported the root itself. Cleanup preserves the original preparation -failure unless it introduces the first direct JVM fatal; an earlier JVM fatal remains primary. +`RETRYING` on the existing completion thread before the original preparation failure propagates. End flushes completed +spans without changing handler failure classification or claiming coverage of later response-stream writes or runtime +acknowledgment. A root end processor's `Exception` or `LinkageError` is logged without suppressing descendant telemetry +or the End flush. Other `Error` failures still escape the Start hook; if invocation cleanup does not run, a batch +processor may not flush. A failing processor need not have exported the root itself. Existing End cleanup failure +priority remains unchanged. The anchor is marked `durable.execution.synthetic_root=true`. Its trace and span IDs are deterministic and its start and end timestamps are the checkpointed execution start. It does not report execution status or duration; `Workflow` -continues to report those at terminal completion. Complete remote parents remain externally owned and are never exported. +continues to report those at terminal completion. With the SDK sampler installed, the anchor carries only the two SDK +identity attributes and the sampled decision, excluding sampler-added attributes and updated trace state. `RECORD_ONLY` +and `DROP` do not export an anchor; descendants retain their full sampling result, including `RECORD_ONLY`, attributes, +and trace state. Complete remote parents remain externally owned and are never exported. Java's `InvocationInfo` requires a non-null execution start timestamp from the initial checkpointed execution operation. Missing timestamps are rejected before the plugin runs; anchor timestamps never fall back to the current wall clock. @@ -104,14 +108,15 @@ execution. With the SDK sampler retained in the provider, upstream sampling and to anchors and their descendants together. After the first invocation's export and flush succeed, its anchor remains available even if the execution is stopped -or times out while suspended and never invokes the plugin again. An invocation killed before its end hook or flush can -still lose its spans; a later invocation, including terminal completion, attempts to export the anchor again. Export +or times out while suspended and never invokes the plugin again. Ending the anchor during Start also lets batch +processors export it while customer code runs. An invocation killed before an export or flush can still lose spans; +a later sampled invocation, including terminal completion, attempts to export the anchor again. Export and flush failures do not provide a delivery guarantee. For a consistent hierarchy across invocations, preserve an explicit upstream sampling decision or use a deterministic sampling policy based on the stable trace ID. A non-deterministic sampler can export an anchor without a Workflow, or -a Workflow without its anchor. Any sampler-supplied attributes and trace state must also stay stable for identical -anchor re-exports. +a Workflow without its anchor. Sampler metadata on descendants remains provider-owned; it does not change the SDK +anchor's body. Provider resources and custom span processors can still change exported data. ## Choose one durable OTel view @@ -308,8 +313,9 @@ The SDK-wide sampling guarantees below require `DurableSampler` itself to be the [builder constructors](#configuration) and [ADOT extension setup](#1-adot-lambda-layer) install it automatically; configure its delegate through these documented paths and retain `DurableSampler` as the final sampler. An unrecognized outer wrapper is treated as a plain replacement. The SDK wrapper applies its sampling intent to every durable -span (DurableExecutionRoot, Workflow, Invocation, operation and attempt), preserving the full result, including `RECORD_ONLY`, sampler attributes -and trace state. +span. Workflow, Invocation, operation and attempt spans preserve the full result, including `RECORD_ONLY`, sampler +attributes and trace state. The re-exported `DurableExecutionRoot` is the narrow exception described above: it uses only +the sampled decision and stable SDK identity, and is omitted for `RECORD_ONLY` or `DROP`. The supported SDK sampler follows this precedence, highest first: @@ -336,7 +342,11 @@ active tracing. The SDK sampling wrapper preserves that decision. Parentless fallback-root exports also use the provider's sampler. If the SDK wrapper is replaced by a plain sampler, those anchors can follow its normal root policy independently of the flags used by their descendants; -full upstream sampling overrides require the installed SDK wrapper. +full upstream sampling overrides require the installed SDK wrapper. For example, a sampler that selects spans by name +can export only the anchor or only its Invocation child. This behavior is covered for both views by +`ExecutionRootParityTest.plainReplacementRetainsIndependentPerSpanSampling`; installing `DurableSampler` restores the +shared decision. Merely delaying anchor creation until terminal completion would not make an arbitrary replacement +sampler consistent across spans. ## Span Attributes diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableExecutionRoot.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableExecutionRoot.java new file mode 100644 index 000000000..4553360b4 --- /dev/null +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableExecutionRoot.java @@ -0,0 +1,67 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_ARN; +import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_SYNTHETIC_ROOT; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import java.time.Instant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Creates the SDK-owned zero-duration identity anchor for a fallback execution trace. */ +final class DurableExecutionRoot { + private static final Logger logger = LoggerFactory.getLogger(DurableExecutionRoot.class); + + private DurableExecutionRoot() {} + + /** Ends the anchor before customer code without letting an isolated processor failure suppress its descendants. */ + static void end(Span root, Instant timestamp) { + if (root == null) return; + try { + root.end(timestamp); + } catch (Exception | LinkageError failure) { + logger.warn( + "Failed to end the execution root; continuing invocation telemetry and flushing at End", failure); + } + } + + /** + * Starts the SDK-owned fallback ancestor before descendants so they inherit the provider's actual sampling + * metadata. The caller ends this identity anchor immediately at the checkpointed start timestamp. Recovery + * re-exports retain identity and timestamps; Workflow carries duration and outcome. Remote parents are never owned. + */ + static Span start( + Tracer tracer, + DeterministicIdGenerator idGenerator, + SpanContext ancestor, + String arn, + Instant start, + DurableSamplingDecision.Intent intent, + boolean useContextCarrier, + boolean useThreadCarrier) { + if (ancestor == null || ancestor.isRemote()) { + return null; + } + var builder = tracer.spanBuilder("DurableExecutionRoot") + .setSpanKind(SpanKind.INTERNAL) + .setParent(useContextCarrier ? DurableSamplingDecision.store(Context.root(), intent) : Context.root()) + .setAttribute(DURABLE_EXECUTION_ARN, arn) + .setAttribute(DURABLE_EXECUTION_SYNTHETIC_ROOT, true) + .setStartTimestamp(start); + Span root; + if (useThreadCarrier) { + try (var ignored = DurableSamplingDecision.openScope(intent)) { + root = idGenerator.startSpan(builder, ancestor.getTraceId(), ancestor.getSpanId()); + } + } else { + root = idGenerator.startSpan(builder, ancestor.getTraceId(), ancestor.getSpanId()); + } + return root; + } +} diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java index 78dc3d84e..2c9ae7f06 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java @@ -8,6 +8,7 @@ import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import io.opentelemetry.sdk.trace.data.LinkData; import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.sdk.trace.samplers.SamplingDecision; import io.opentelemetry.sdk.trace.samplers.SamplingResult; import java.util.Collections; import java.util.LinkedHashMap; @@ -21,7 +22,7 @@ *

    The durable plugins decide sampling for the whole execution exactly once per invocation and attach the resulting * {@link SamplingResult} to the {@link Context} used as the parent of each durable span (via * {@link DurableSamplingDecision}). When {@code shouldSample} sees that decision on the parent context it returns it - * verbatim, so: + * verbatim for descendants. Re-exported execution anchors use only the sampled decision and stable SDK identity, so: * *

      *
    • the wrapped (customer-configured or ADOT/community default) sampler is invoked at most once per invocation for @@ -120,17 +121,22 @@ public SamplingResult shouldSample( // Not a durable span: the customer's sampler governs it unchanged. return delegate.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); } - if (!intent.isDeferred()) { - // A resolved decision (explicit upstream, same-trace ambient, or a locally reproduced sampler): return it - // verbatim, preserving RECORD_ONLY and never re-invoking the delegate. - return intent.resolved(); + var result = intent.isDeferred() + ? deferredDecisions.computeIfAbsent( + new ExecutionKey( + intent.deferredTraceId(), attributes.get(SpanAttributes.DURABLE_EXECUTION_ARN)), + key -> delegate.shouldSample( + Context.root(), key.traceId(), name, spanKind, attributes, Collections.emptyList())) + : intent.resolved(); + if (name.equals("DurableExecutionRoot") + && Boolean.TRUE.equals(attributes.get(SpanAttributes.DURABLE_EXECUTION_SYNTHETIC_ROOT))) { + // Re-exported anchors carry only SDK identity and the sampled decision. Per-invocation sampler + // metadata remains on descendants; recording-only executions do not materialize an anchor. + return result.getDecision() == SamplingDecision.RECORD_AND_SAMPLE + ? SamplingResult.recordAndSample() + : SamplingResult.drop(); } - // Deferred (agent path, real sampler not reproducible here): evaluate the actual delegate once per execution - // and reuse it, so an installed drop/rate-limit policy is honored and consulted only once. - return deferredDecisions.computeIfAbsent( - new ExecutionKey(intent.deferredTraceId(), attributes.get(SpanAttributes.DURABLE_EXECUTION_ARN)), - key -> delegate.shouldSample( - Context.root(), key.traceId(), name, spanKind, attributes, Collections.emptyList())); + return result; } @Override diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java index 1d2de1217..28bbcfd24 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java @@ -100,20 +100,6 @@ static Scope openScope(Intent intent) { }; } - /** - * Inspects the unconsumed durable sampling intent for a span, or {@code null} when none is present. Prefers the - * full-fidelity context key (same class loader) and falls back to the thread-scoped system property (cross class - * loader). - */ - static Intent get(Context context) { - var holder = context.get(KEY); - var fromContext = holder != null ? holder.get() : null; - if (fromContext != null) { - return fromContext; - } - return fromScopedProperty(); - } - /** Consumes both carriers before callbacks can reuse the parent context or cross-loader bridge. */ static Intent consume(Context context) { try { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index 150a3fd31..41fb202ba 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -239,7 +239,7 @@ public void onInvocationStart(InvocationInfo info) { var sampled = OtelPluginSupport.isSampled(decision); var execCtx = ExecutionTraceContext.resolve(extracted, canonicalTraceId, arn(), idGenerator, () -> sampled); executionStartTime = info.executionStartTime(); - executionRootSpan = OtelPluginSupport.startExecutionRoot( + var root = DurableExecutionRoot.start( tracer, idGenerator, execCtx.executionAncestor(), @@ -248,6 +248,13 @@ public void onInvocationStart(InvocationInfo info) { samplingIntent, OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider), OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)); + try { + DurableExecutionRoot.end(root, executionStartTime); + } finally { + // Publish only after end callbacks, so a reentrant End cannot flush from inside root.end(). + // Retain the ended root as a flush marker even if an onEnd processor throws during Start. + executionRootSpan = root; + } // A visible plain replacement keeps its documented ancestor-flag fallback, without SDK-wide overrides. if (executionRootSpan != null && OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)) { execCtx = new ExecutionTraceContext(executionRootSpan.getSpanContext()); @@ -289,19 +296,16 @@ public void onInvocationStart(InvocationInfo info) { @Override public void onInvocationEnd(InvocationEndInfo info) { var root = executionRootSpan; - var rootTimestamp = executionStartTime; var provider = sdkTracerProvider; var shouldFlush = tracingEnabled || root != null; - executionRootSpan = null; // One-shot release even if an end processor throws or re-enters. - OtelPluginSupport.finishWithRootCleanup( - () -> finishInvocationEnd(info), - () -> OtelPluginSupport.endRootAndFlush(root, rootTimestamp, () -> { - if (shouldFlush && provider != null) { - var flushResult = provider.forceFlush().join(5, TimeUnit.SECONDS); - if (!flushResult.isSuccess()) - logger.warn("OTel span flush failed or timed out — some spans may be lost"); - } - })); + executionRootSpan = null; // The anchor already ended at Start; flush this invocation only once. + OtelPluginSupport.finishWithRootCleanup(() -> finishInvocationEnd(info), () -> { + if (shouldFlush && provider != null) { + var flushResult = provider.forceFlush().join(5, TimeUnit.SECONDS); + if (!flushResult.isSuccess()) + logger.warn("OTel span flush failed or timed out — some spans may be lost"); + } + }); } private void finishInvocationEnd(InvocationEndInfo info) { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index 627d753b7..b28beef3a 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -233,7 +233,7 @@ public void onInvocationStart(InvocationInfo info) { var execCtx = ExecutionTraceContext.resolve( extracted, canonicalTraceId, info.durableExecutionArn(), idGenerator, () -> sampled); executionStartTime = info.executionStartTime(); - executionRootSpan = OtelPluginSupport.startExecutionRoot( + var root = DurableExecutionRoot.start( tracer, idGenerator, execCtx.executionAncestor(), @@ -242,6 +242,13 @@ public void onInvocationStart(InvocationInfo info) { samplingIntent, OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider), OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)); + try { + DurableExecutionRoot.end(root, executionStartTime); + } finally { + // Publish only after end callbacks, so a reentrant End cannot flush from inside root.end(). + // Retain the ended root as a flush marker even if an onEnd processor throws during Start. + executionRootSpan = root; + } // A visible plain replacement keeps its documented ancestor-flag fallback, without SDK-wide overrides. if (executionRootSpan != null && OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)) { execCtx = new ExecutionTraceContext(executionRootSpan.getSpanContext()); @@ -286,19 +293,16 @@ public void onInvocationStart(InvocationInfo info) { @Override public void onInvocationEnd(InvocationEndInfo info) { var root = executionRootSpan; - var rootTimestamp = executionStartTime; var provider = sdkTracerProvider; var shouldFlush = tracingEnabled || root != null; - executionRootSpan = null; // One-shot release even if an end processor throws or re-enters. - OtelPluginSupport.finishWithRootCleanup( - () -> finishInvocationEnd(info), - () -> OtelPluginSupport.endRootAndFlush(root, rootTimestamp, () -> { - if (shouldFlush && provider != null) { - var flushResult = provider.forceFlush().join(5, TimeUnit.SECONDS); - if (!flushResult.isSuccess()) - logger.warn("OTel span flush failed or timed out — some spans may be lost"); - } - })); + executionRootSpan = null; // The anchor already ended at Start; flush this invocation only once. + OtelPluginSupport.finishWithRootCleanup(() -> finishInvocationEnd(info), () -> { + if (shouldFlush && provider != null) { + var flushResult = provider.forceFlush().join(5, TimeUnit.SECONDS); + if (!flushResult.isSuccess()) + logger.warn("OTel span flush failed or timed out — some spans may be lost"); + } + }); } private void finishInvocationEnd(InvocationEndInfo info) { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 2fe2cfd0c..f12f2cb83 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -2,13 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.otel; -import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_ARN; -import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_SYNTHETIC_ROOT; - import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.api.trace.TracerProvider; @@ -17,62 +13,14 @@ import io.opentelemetry.sdk.trace.samplers.SamplingResult; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Instant; import java.util.Collections; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import software.amazon.lambda.durable.util.ExceptionHelper; -/** Shared utilities for OTel plugin default constructor support (ADOT Java agent SPI path). */ +/** Shared provider setup, sampling resolution, and invocation cleanup support. */ final class OtelPluginSupport { - /** - * Starts the SDK-owned fallback ancestor before descendants so they inherit the provider's actual sampling - * metadata. The caller ends this identity anchor at the same checkpointed start timestamp before invocation return. - * Recovery re-exports retain identity and timestamps; Workflow carries duration and outcome. Remote parents are - * never owned. - */ - static Span startExecutionRoot( - Tracer tracer, - DeterministicIdGenerator idGenerator, - SpanContext ancestor, - String arn, - Instant start, - DurableSamplingDecision.Intent intent, - boolean useContextCarrier, - boolean useThreadCarrier) { - if (ancestor == null || ancestor.isRemote()) { - return null; - } - var builder = tracer.spanBuilder("DurableExecutionRoot") - .setSpanKind(SpanKind.INTERNAL) - .setParent(useContextCarrier ? DurableSamplingDecision.store(Context.root(), intent) : Context.root()) - .setAttribute(DURABLE_EXECUTION_ARN, arn) - .setAttribute(DURABLE_EXECUTION_SYNTHETIC_ROOT, true) - .setStartTimestamp(start); - Span root; - if (useThreadCarrier) { - try (var ignored = DurableSamplingDecision.openScope(intent)) { - root = idGenerator.startSpan(builder, ancestor.getTraceId(), ancestor.getSpanId()); - } - } else { - root = idGenerator.startSpan(builder, ancestor.getTraceId(), ancestor.getSpanId()); - } - return root; - } - - /** Flushes completed spans after root end, including failures isolated by the plugin boundary. */ - static void endRootAndFlush(Span root, Instant timestamp, Runnable flush) { - try { - if (root != null) root.end(timestamp); - } catch (Exception | LinkageError failure) { - finishWithRootCleanup(() -> ExceptionHelper.sneakyThrow(failure), flush); - return; - } - flush.run(); - } - - /** Always releases an invocation's new root resource without hiding an earlier failure or a later JVM fatal. */ + /** Always runs invocation cleanup without hiding an earlier failure or a later JVM fatal. */ static void finishWithRootCleanup(Runnable finish, Runnable cleanup) { Throwable primary = null; try { diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java index 563b39aab..613e8e61c 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; @@ -66,8 +67,8 @@ class DurableSamplingDecisionClassLoaderTest { "ExecutionOtelPlugin,foreign,false", "ExecutionOtelPlugin,foreign,true", "ExecutionOtelPlugin,opaque,false", "ExecutionOtelPlugin,opaque,true" }) - void materializedRootSuppliesParentSamplingMetadata(String pluginName, String topology, boolean recordOnly) - throws Exception { + void managedAnchorKeepsStableIdentityAndChildrenRetainSamplerMetadata( + String pluginName, String topology, boolean recordOnly) throws Exception { var previousHeader = System.getProperty("com.amazonaws.xray.traceHeader"); GlobalOpenTelemetry.resetForTest(); OtelPluginAutoConfigurationState.markInstalled(); @@ -158,16 +159,22 @@ public ContextPropagators getPropagators() { var roots = spans.stream() .filter(span -> span.getName().equals("DurableExecutionRoot")) .toList(); - assertEquals(2, roots.size()); - var rootContext = roots.get(0).getSpanContext(); + assertEquals(recordOnly ? 0 : 2, roots.size()); + var rootContext = spans.stream() + .filter(span -> span.getName().equals("Invocation")) + .findFirst() + .orElseThrow() + .getParentSpanContext(); + assertEquals(TraceState.getDefault(), rootContext.getTraceState()); for (var anchor : roots) { assertEquals( rootContext, anchor.getSpanContext(), "Re-export keeps the complete stable root context"); assertEquals(10_000_000_000L, anchor.getStartEpochNanos()); assertEquals(anchor.getStartEpochNanos(), anchor.getEndEpochNanos()); - assertEquals("1", anchor.getSpanContext().getTraceState().get("depth")); - assertEquals(!recordOnly, anchor.getSpanContext().isSampled()); - assertEquals("kept", anchor.getAttributes().get(key)); + assertEquals( + TraceState.getDefault(), anchor.getSpanContext().getTraceState()); + assertTrue(anchor.getSpanContext().isSampled()); + assertNull(anchor.getAttributes().get(key)); } var descendants = spans.stream() .filter(span -> span.getName().equals("Invocation") @@ -179,7 +186,7 @@ public ContextPropagators getPropagators() { rootContext, child.getParentSpanContext(), "A child must record the materialized parent's flags and trace state: " + child.getName()); - assertEquals("2", child.getSpanContext().getTraceState().get("depth")); + assertEquals("1", child.getSpanContext().getTraceState().get("depth")); assertEquals("kept", child.getAttributes().get(key)); } assertEquals(topology.equals("local") ? 2 : 1, evaluations.get()); @@ -291,10 +298,19 @@ public ContextPropagators getPropagators() { } } var spans = exporter.getFinishedSpanItems().stream() - .filter(s -> - s.getName().equals("Invocation") || s.getName().equals("DurableExecutionRoot")) + .filter(s -> s.getName().equals("Invocation")) + .toList(); + assertEquals(executionCount * 2, spans.size()); + var anchors = exporter.getFinishedSpanItems().stream() + .filter(s -> s.getName().equals("DurableExecutionRoot")) .toList(); - assertEquals(executionCount * 4, spans.size()); + assertEquals(2, anchors.size(), "Only sampled executions export anchors, twice across resume"); + for (var anchor : anchors) { + assertTrue(anchor.getSpanContext().isSampled()); + assertNull(anchor.getAttributes().get(key)); + assertEquals( + TraceState.getDefault(), anchor.getSpanContext().getTraceState()); + } for (var span : spans) { var second = span.getAttributes() .get(SpanAttributes.DURABLE_EXECUTION_ARN) @@ -341,7 +357,7 @@ void decisionCrossesClassLoaderBoundary_viaScopedProperty() throws Exception { var openScope = appDecision.getDeclaredMethod("openScope", appIntentClass); openScope.setAccessible(true); - var get = agentDecision.getDeclaredMethod("get", Context.class); + var get = agentDecision.getDeclaredMethod("consume", Context.class); get.setAccessible(true); var consume = agentDecision.getDeclaredMethod("consume", Context.class); consume.setAccessible(true); @@ -408,10 +424,10 @@ void agentProcessorForwardingParentCannotReuseApplicationSamplingIntent( var agentWrap = agentSamplerType.getDeclaredMethod("wrap", Sampler.class); agentWrap.setAccessible(true); var appDecision = Class.forName(DurableSamplingDecision.class.getName(), true, appLoader); - var appGet = appDecision.getDeclaredMethod("get", Context.class); + var appGet = appDecision.getDeclaredMethod("consume", Context.class); appGet.setAccessible(true); var agentDecision = Class.forName(DurableSamplingDecision.class.getName(), true, agentLoader); - var agentGet = agentDecision.getDeclaredMethod("get", Context.class); + var agentGet = agentDecision.getDeclaredMethod("consume", Context.class); agentGet.setAccessible(true); var agentIdType = Class.forName(DeterministicIdGenerator.class.getName(), true, agentLoader); var callbacks = new AtomicInteger(); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java index b8b37d43a..0a104534a 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java @@ -406,8 +406,11 @@ void nonTerminalInvocation_doesNotExportWorkflowSpan() { var spans = spanExporter.getFinishedSpanItems(); // Invocation and DurableExecutionRoot are exported; Workflow remains deferred until terminal status. assertEquals(2, spans.size()); - assertEquals("Invocation", spans.get(0).getName()); - assertEquals(StatusCode.OK, spans.get(0).getStatus().getStatusCode(), "PENDING invocation span maps to OK"); + assertEquals("Invocation", spanByName(spans, "Invocation").getName()); + assertEquals( + StatusCode.OK, + spanByName(spans, "Invocation").getStatus().getStatusCode(), + "PENDING invocation span maps to OK"); } @Test @@ -452,7 +455,7 @@ void retryingInvocation_invocationSpanUnset_workflowNotExported() { var spans = spanExporter.getFinishedSpanItems(); assertEquals(2, spans.size(), "RETRYING is non-terminal — Workflow span not exported"); - var invocationSpan = spans.get(0); + var invocationSpan = spanByName(spans, "Invocation"); assertEquals("Invocation", invocationSpan.getName()); assertEquals( StatusCode.UNSET, @@ -943,12 +946,11 @@ void operationNotCompleted_notEndedAtInvocationEnd() { plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null)); var spans = spanExporter.getFinishedSpanItems(); - // Invocation and DurableExecutionRoot are exported. A still-open operation has no recording span (creation is - // deferred to - // onOperationEnd), so there is nothing to abandon, and the Workflow span is not exported on a non-terminal - // invocation. + // Invocation and DurableExecutionRoot are exported. A still-open operation has no recording span + // (creation is deferred to onOperationEnd), so there is nothing to abandon, and the Workflow span is not + // exported on a non-terminal invocation. assertEquals(2, spans.size()); - assertEquals("Invocation", spans.get(0).getName()); + assertEquals("Invocation", spanByName(spans, "Invocation").getName()); assertTrue( spans.stream().noneMatch(s -> s.getName().equals("my-wait")), "An operation still open at invocation end must not be ended/exported in onInvocationEnd"); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java index 15cbbff63..02779a68c 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java @@ -3,9 +3,7 @@ package software.amazon.lambda.durable.otel; import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.*; -import io.opentelemetry.api.trace.Span; import io.opentelemetry.context.Context; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; @@ -69,9 +67,6 @@ public void onEnd(ReadableSpan span) { && mode.contains("failure") && !mode.equals("start-failure") && failOnce.compareAndSet(true, false)) throw primary; - if (span.getName().equals("Invocation") - && mode.equals("fatal-root-cleanup") - && failOnce.compareAndSet(true, false)) throw primary; if (!span.getName().equals("DurableExecutionRoot")) return; rootsEnded.incrementAndGet(); inRootCallback.set(true); @@ -111,25 +106,24 @@ public CompletableResultCode forceFlush() { assertSame( primary, assertThrows(AssertionError.class, () -> plugin.get().onInvocationStart(first))); + else if (mode.equals("fatal-root-cleanup")) + assertSame( + cleanupFatal, + assertThrows(InternalError.class, () -> plugin.get().onInvocationStart(first))); else plugin.get().onInvocationStart(first); assertEquals("DurableExecutionRoot", starts.get(0)); - assertTrue(roots.get(0).isRecording(), "The root is retained until invocation end"); + assertFalse(roots.get(0).isRecording(), "The anchor ended before the Start hook returned or failed"); if (mode.equals("end-failure")) assertSame( primary, assertThrows(AssertionError.class, () -> plugin.get().onInvocationEnd(end(true)))); - else if (mode.equals("fatal-root-cleanup")) { - assertSame( - cleanupFatal, - assertThrows(InternalError.class, () -> plugin.get().onInvocationEnd(end(true)))); - assertEquals(List.of(primary), List.of(cleanupFatal.getSuppressed())); - } else plugin.get().onInvocationEnd(end(true)); + else plugin.get().onInvocationEnd(end(true)); assertEquals(1, rootsEnded.get()); assertFalse(roots.get(0).isRecording()); assertEquals( - mode.equals("fatal-root-cleanup") ? 0 : 1, + 1, flushes.get(), - "Normal and partial-start cleanup flush only after the root ends; fatal root cleanup escapes"); + "End flushes completed spans, including cleanup after a partially completed Start"); var beforeDuplicateEnd = flushes.get(); plugin.get().onInvocationEnd(end(true)); assertEquals(beforeDuplicateEnd, flushes.get()); @@ -162,11 +156,11 @@ private static InvocationEndInfo end(boolean first) { "runtime,fatal", "linkage,ordinary", "linkage,fatal", - "assertion,unused", - "vm,unused", - "death,unused" + "assertion,ordinary", + "vm,ordinary", + "death,ordinary" }) - void flushRunsOnlyAfterRootFailuresIsolatedByTheExistingPluginBoundary(String rootKind, String flushKind) { + void flushPreservesTheExistingEndFailurePriority(String rootKind, String flushKind) { Throwable rootFailure = switch (rootKind) { case "linkage" -> new LinkageError("root linkage"); @@ -178,27 +172,17 @@ void flushRunsOnlyAfterRootFailuresIsolatedByTheExistingPluginBoundary(String ro Throwable flushFailure = flushKind.equals("fatal") ? new InternalError("flush fatal") : new IllegalArgumentException("flush ordinary"); - var root = mock(Span.class); - doAnswer(call -> { - ExceptionHelper.sneakyThrow(rootFailure); - return null; - }) - .when(root) - .end(START); var flushes = new AtomicInteger(); var observed = assertThrows( Throwable.class, - () -> OtelPluginSupport.endRootAndFlush(root, START, () -> { + () -> OtelPluginSupport.finishWithRootCleanup(() -> ExceptionHelper.sneakyThrow(rootFailure), () -> { flushes.incrementAndGet(); ExceptionHelper.sneakyThrow(flushFailure); })); var isolated = rootKind.equals("runtime") || rootKind.equals("linkage"); - assertEquals(isolated ? 1 : 0, flushes.get()); + assertEquals(1, flushes.get()); var expected = isolated && flushKind.equals("fatal") ? flushFailure : rootFailure; assertSame(expected, observed); - assertEquals( - isolated ? List.of(expected == rootFailure ? flushFailure : rootFailure) : List.of(), - List.of(observed.getSuppressed())); - verify(root).end(START); + assertEquals(List.of(expected == rootFailure ? flushFailure : rootFailure), List.of(observed.getSuppressed())); } } diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootParityTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootParityTest.java new file mode 100644 index 000000000..fd4cd4be8 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootParityTest.java @@ -0,0 +1,205 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; +import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_ARN; +import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_SYNTHETIC_ROOT; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.sdk.trace.samplers.SamplingDecision; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; + +class ExecutionRootParityTest { + private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/parity/id"; + private static final Instant START = Instant.ofEpochSecond(10); + private static final AttributeKey SAMPLER_VALUE = AttributeKey.stringKey("sampler.invocation"); + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void sampledAnchorEndsBeforeTheStartHookReturns(boolean executionView) { + try (var exporter = InMemorySpanExporter.create()) { + var plugin = plugin(executionView, exporter, Sampler.alwaysOn()); + plugin.onInvocationStart(start(true)); + var roots = roots(exporter); + assertEquals(1, roots.size(), "The anchor is available to processors before handler execution"); + assertEquals(START.getEpochSecond() * 1_000_000_000L, roots.get(0).getStartEpochNanos()); + assertEquals(roots.get(0).getStartEpochNanos(), roots.get(0).getEndEpochNanos()); + plugin.onInvocationEnd(end(true)); + assertEquals(1, roots(exporter).size(), "End does not export the same anchor twice"); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void anchorReexportsExcludeChangingSamplerMetadataButChildrenKeepIt(boolean executionView) { + var evaluations = new AtomicInteger(); + var sampler = sampler((name) -> { + var value = Integer.toString(evaluations.incrementAndGet()); + return new SamplingResult() { + public SamplingDecision getDecision() { + return SamplingDecision.RECORD_AND_SAMPLE; + } + + public Attributes getAttributes() { + return Attributes.of(SAMPLER_VALUE, value); + } + + public TraceState getUpdatedTraceState(TraceState parent) { + return TraceState.builder().put("vendor", value).build(); + } + }; + }); + try (var exporter = InMemorySpanExporter.create()) { + var plugin = plugin(executionView, exporter, sampler); + for (var first : new boolean[] {true, false}) { + plugin.onInvocationStart(start(first)); + plugin.onInvocationEnd(end(first)); + } + var roots = roots(exporter); + assertEquals(2, roots.size()); + assertEquals(2, evaluations.get(), "The anchor must reuse the invocation's sampling result"); + for (var root : roots) { + assertNull(root.getAttributes().get(SAMPLER_VALUE)); + assertEquals( + Attributes.of(DURABLE_EXECUTION_ARN, ARN, DURABLE_EXECUTION_SYNTHETIC_ROOT, true) + .asMap(), + root.getAttributes().asMap()); + assertEquals(TraceState.getDefault(), root.getSpanContext().getTraceState()); + assertEquals(roots.get(0).getSpanContext(), root.getSpanContext()); + } + var invocations = exporter.getFinishedSpanItems().stream() + .filter(s -> s.getName().equals("Invocation")) + .toList(); + assertEquals(2, invocations.size()); + for (var i = 0; i < invocations.size(); i++) { + var value = Integer.toString(i + 1); + assertEquals(value, invocations.get(i).getAttributes().get(SAMPLER_VALUE)); + assertEquals( + value, + invocations.get(i).getSpanContext().getTraceState().get("vendor")); + assertEquals(roots.get(i).getSpanContext(), invocations.get(i).getParentSpanContext()); + } + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void recordOnlyStillRecordsChildrenWithoutAnAnchor(boolean executionView) { + try (var exporter = InMemorySpanExporter.create()) { + var plugin = plugin(executionView, exporter, sampler(name -> SamplingResult.recordOnly())); + plugin.onInvocationStart(start(true)); + plugin.onInvocationEnd(end(false)); + assertTrue(roots(exporter).isEmpty()); + var spans = exporter.getFinishedSpanItems(); + assertEquals(2, spans.size()); + assertTrue(spans.stream().allMatch(s -> !s.getSpanContext().isSampled())); + assertTrue(spans.stream().anyMatch(s -> s.getName().equals("Invocation"))); + assertTrue(spans.stream().anyMatch(s -> s.getName().equals("Workflow"))); + } + } + + @ParameterizedTest + @CsvSource({"false,false", "false,true", "true,false", "true,true"}) + void plainReplacementRetainsIndependentPerSpanSampling(boolean executionView, boolean sampleAnchor) { + GlobalOpenTelemetry.resetForTest(); + OtelPluginAutoConfigurationState.markInstalled(); + try (var exporter = InMemorySpanExporter.create(); + var provider = SdkTracerProvider.builder() + .setIdGenerator(new DeterministicIdGenerator()) + .setSampler(sampler(name -> name.equals("DurableExecutionRoot") == sampleAnchor + ? SamplingResult.recordAndSample() + : SamplingResult.drop())) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build()) { + GlobalOpenTelemetry.set( + OpenTelemetrySdk.builder().setTracerProvider(provider).build()); + DurableExecutionPlugin plugin = + executionView ? new ExecutionOtelPlugin(config()) : new InvocationOtelPlugin(config()); + plugin.onInvocationStart(start(true)); + plugin.onInvocationEnd(end(false)); + assertEquals(sampleAnchor ? 1 : 0, roots(exporter).size()); + assertEquals( + sampleAnchor ? 0 : 1, + exporter.getFinishedSpanItems().stream() + .filter(s -> s.getName().equals("Invocation")) + .count()); + } finally { + GlobalOpenTelemetry.resetForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); + } + } + + private static DurableExecutionPlugin plugin( + boolean executionView, InMemorySpanExporter exporter, Sampler sampler) { + var builder = SdkTracerProvider.builder() + .setSampler(sampler) + .addSpanProcessor(SimpleSpanProcessor.builder(exporter) + .setExportUnsampledSpans(true) + .build()); + return executionView ? new ExecutionOtelPlugin(builder, config()) : new InvocationOtelPlugin(builder, config()); + } + + private static OtelPluginConfig config() { + return OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(() -> null) + .build(); + } + + private static InvocationInfo start(boolean first) { + return new InvocationInfo("request", ARN, first, START); + } + + private static InvocationEndInfo end(boolean first) { + return new InvocationEndInfo( + "request", ARN, first, first ? InvocationStatus.PENDING : InvocationStatus.SUCCEEDED, null); + } + + private static List roots(InMemorySpanExporter exporter) { + return exporter.getFinishedSpanItems().stream() + .filter(s -> s.getName().equals("DurableExecutionRoot")) + .toList(); + } + + private static Sampler sampler(Function decision) { + return new Sampler() { + public SamplingResult shouldSample( + Context parent, + String traceId, + String name, + SpanKind kind, + Attributes attributes, + List links) { + return decision.apply(name); + } + + public String getDescription() { + return "root-parity"; + } + }; + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java index 314363eb1..0b7d185af 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java @@ -332,7 +332,7 @@ void invocationStart_and_end_createsSpan() { var spans = spanExporter.getFinishedSpanItems(); assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot - var span = spans.get(0); + var span = spanByName("Invocation"); assertEquals("Invocation", span.getName()); assertEquals(StatusCode.OK, span.getStatus().getStatusCode()); } @@ -418,7 +418,7 @@ void invocationSpan_hasInternalKind() { plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); - var span = spanExporter.getFinishedSpanItems().get(0); + var span = spanByName("Invocation"); assertEquals(SpanKind.INTERNAL, span.getKind(), "Invocation span must be INTERNAL kind"); } @@ -583,7 +583,7 @@ void invocationEnd_withFailure_setsErrorStatus() { var spans = spanExporter.getFinishedSpanItems(); assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot - assertEquals(StatusCode.ERROR, spans.get(0).getStatus().getStatusCode()); + assertEquals(StatusCode.ERROR, spanByName("Invocation").getStatus().getStatusCode()); } @Test @@ -596,7 +596,7 @@ void invocationEnd_withRetrying_leavesStatusUnset() { assertEquals(2, spans.size()); assertEquals( StatusCode.UNSET, - spans.get(0).getStatus().getStatusCode(), + spanByName("Invocation").getStatus().getStatusCode(), "RETRYING invocation span is UNSET (interface cannot distinguish STOPPED/TIMED_OUT)"); } diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java index 5c26b8671..df7f52dd0 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java @@ -45,13 +45,13 @@ static Stream cases() { @ParameterizedTest @MethodSource("cases") - void rootStartCallbacksRetainTheirOwnSamplingAndRandomIds( + void durableStartCallbacksRetainTheirOwnSamplingAndRandomIds( boolean executionView, boolean sampled, boolean sharedGenerator, boolean rootRecordOnly, boolean forwardParent) { - var observer = new StartObserver(forwardParent); + var observer = new StartObserver(forwardParent, rootRecordOnly ? "Invocation" : "DurableExecutionRoot"); var policy = sampled ? Sampler.alwaysOn() : Sampler.alwaysOff(); var builder = SdkTracerProvider.builder() .setSampler(rootSampler(rootRecordOnly, policy)) @@ -80,7 +80,7 @@ void rootStartCallbacksRetainTheirOwnSamplingAndRandomIds( invoke(plugin, false); assertControl(observer.tracer, sampled); assertSame(original, Context.current()); - assertObservations(observer, sampled); + assertObservations(observer, sampled, rootRecordOnly); } } @@ -96,21 +96,29 @@ private static void assertControl(Tracer tracer, boolean sampled) { span.end(); } - private static void assertObservations(StartObserver observer, boolean sampled) { + private static void assertObservations(StartObserver observer, boolean sampled, boolean recordOnly) { assertEquals(2, observer.roots.size()); - assertEquals(observer.roots.get(0), observer.roots.get(1)); var root = observer.roots.get(0); - assertEquals(new DeterministicIdGenerator().generateExecutionRootSpanId(ARN), root.getSpanId()); + assertEquals(recordOnly ? 0 : 2, observer.anchorStarts, "RECORD_ONLY has no anchor processor callback"); + if (!recordOnly) { + assertEquals(observer.roots.get(0), observer.roots.get(1)); + assertEquals(new DeterministicIdGenerator().generateExecutionRootSpanId(ARN), root.getSpanId()); + } assertEquals(2, observer.callbacks.size()); assertEquals(List.of(sampled, sampled), observer.recording); for (var callback : observer.callbacks) { assertEquals(sampled, callback.isSampled()); assertTrue(callback.isValid()); - assertNotEquals(root.getTraceId(), callback.getTraceId()); + if (recordOnly && observer.forwardParent) { + // An Invocation has a real execution ancestor; explicitly forwarding it preserves that trace. + assertEquals(root.getTraceId(), callback.getTraceId()); + } else { + assertNotEquals(root.getTraceId(), callback.getTraceId()); + } assertNotEquals(root.getSpanId(), callback.getSpanId()); } assertEquals( - 2, + recordOnly && observer.forwardParent ? 1 : 2, observer.callbacks.stream() .map(SpanContext::getTraceId) .distinct() @@ -149,9 +157,12 @@ public String getDescription() { private static final class StartObserver implements SpanProcessor { private Tracer tracer; private final boolean forwardParent; + private final String observedName; + private int anchorStarts; - private StartObserver(boolean forwardParent) { + private StartObserver(boolean forwardParent, String observedName) { this.forwardParent = forwardParent; + this.observedName = observedName; } private final List roots = new ArrayList<>(); @@ -160,8 +171,10 @@ private StartObserver(boolean forwardParent) { @Override public void onStart(Context parent, ReadWriteSpan span) { - // The name guard permits exactly one unrelated span per root and prevents processor recursion. - if (!span.getName().equals("DurableExecutionRoot")) return; + if (span.getName().equals("DurableExecutionRoot")) anchorStarts++; + // Observe the recording Invocation for RECORD_ONLY, which intentionally has no recording anchor. + // The name guard permits one unrelated span per observation and prevents processor recursion. + if (!span.getName().equals(observedName)) return; roots.add(span.getSpanContext()); var builder = tracer.spanBuilder("unrelated-onStart"); var callback = (forwardParent ? builder.setParent(parent) : builder.setNoParent()).startSpan();