attemptScopes = new ConcurrentHashMap<>();
+ private final UserFunctionScopes attemptScopes = new UserFunctionScopes();
// Deterministic operation contexts (keyed by operationId), held between start and end so children and attempts can
// parent onto an operation whose recording span is not created until onOperationEnd.
@@ -242,9 +243,29 @@ public void onInvocationStart(InvocationInfo info) {
: DurableSamplingDecision.Intent.deferred(canonicalTraceId);
var sampled = OtelPluginSupport.isSampled(decision);
var execCtx = ExecutionTraceContext.resolve(extracted, canonicalTraceId, arn(), idGenerator, () -> sampled);
+ executionStartTime = info.executionStartTime();
+ var root = DurableExecutionRoot.start(
+ tracer,
+ idGenerator,
+ execCtx.executionAncestor(),
+ info.durableExecutionArn(),
+ executionStartTime,
+ samplingIntent,
+ OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider),
+ OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider));
+ try {
+ DurableExecutionRoot.end(root, executionStartTime);
+ } finally {
+ // Publish only after end callbacks, so a reentrant End cannot flush from inside root.end().
+ // Retain the ended root as a flush marker even if an onEnd processor throws during Start.
+ executionRootSpan = root;
+ }
+ // A visible plain replacement keeps its documented ancestor-flag fallback, without SDK-wide overrides.
+ if (executionRootSpan != null && OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)) {
+ execCtx = new ExecutionTraceContext(executionRootSpan.getSpanContext());
+ }
executionTrace = new ExecutionTrace(canonicalTraceId, execCtx.traceFlags());
executionAncestor = execCtx.executionAncestor();
- executionStartTime = info.executionStartTime();
// Invocation span — child of the ambient Lambda span when it is on the execution trace, otherwise a child of
// the execution ancestor so it stays within the same trace.
@@ -282,8 +303,6 @@ private Scope activateHandlerContext() {
var trace = executionTrace;
if (!tracingEnabled || trace == null) return null;
var ambient = Span.current().getSpanContext();
- // Preserve a compatible ambient Lambda span. An absent or unrelated ambient span must not leave
- // handler instrumentation outside the durable execution's canonical trace.
if (ambient.isValid() && trace.traceId().equals(ambient.getTraceId())) return Scope.noop();
return Span.wrap(workflowSpanContext).makeCurrent();
}
@@ -296,6 +315,20 @@ public void onInvocationEnd(InvocationEndInfo info) {
}
private void endInvocation(InvocationEndInfo info) {
+ var root = executionRootSpan;
+ var provider = sdkTracerProvider;
+ var shouldFlush = tracingEnabled || root != null;
+ executionRootSpan = null; // The anchor already ended at Start; flush this invocation only once.
+ OtelPluginSupport.finishWithRootCleanup(() -> finishInvocationEnd(info), () -> {
+ if (shouldFlush && provider != null) {
+ var flushResult = provider.forceFlush().join(5, TimeUnit.SECONDS);
+ if (!flushResult.isSuccess())
+ logger.warn("OTel span flush failed or timed out — some spans may be lost");
+ }
+ });
+ }
+
+ private void finishInvocationEnd(InvocationEndInfo info) {
if (!tracingEnabled) {
return;
}
@@ -310,12 +343,9 @@ private void endInvocation(InvocationEndInfo info) {
operationContexts.clear();
operationStartTimes.clear();
- // Release OTel context on worker threads, then end any attempt spans still open so no recording span is
- // abandoned. Attempt spans normally start and end within one user-function call, so this is a safeguard.
- for (var scope : attemptScopes.values()) {
- scope.close();
- }
- attemptScopes.clear();
+ // End open recording spans now, but leave a running worker's thread-bound scope for its own user-function End.
+ // Only scopes owned by this finalization thread can be closed here.
+ attemptScopes.closeCurrentThread();
for (var span : attemptSpans.values()) {
span.end();
}
@@ -359,14 +389,6 @@ private void endInvocation(InvocationEndInfo info) {
executionAncestor = null;
executionStartTime = null;
samplingIntent = null;
-
- // Flush spans before Lambda freezes
- if (sdkTracerProvider != null) {
- var flushResult = sdkTracerProvider.forceFlush().join(5, TimeUnit.SECONDS);
- if (!flushResult.isSuccess()) {
- logger.warn("OTel span flush failed or timed out — some spans may be lost");
- }
- }
}
// ─── Operation hooks ─────────────────────────────────────────────────
@@ -509,15 +531,10 @@ public void onUserFunctionStart(UserFunctionStartInfo info) {
@Override
public void onUserFunctionEnd(UserFunctionEndInfo info) {
- if (!tracingEnabled) return;
-
var key = attemptKey(info.id(), info.attempt());
-
- // Close scope first (must happen on same thread as makeCurrent)
- var scope = attemptScopes.remove(key);
- if (scope != null) {
- scope.close();
- }
+ // A running user function can finish after invocation End; its scope still belongs to this thread.
+ attemptScopes.close(key);
+ if (!tracingEnabled) return;
// CONTEXT operations don't have attempt spans — scope cleanup is all we need
if ("CONTEXT".equals(info.type())) {
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionTraceContext.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionTraceContext.java
index cf8a3dc4b..b3e7eaea5 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionTraceContext.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionTraceContext.java
@@ -39,8 +39,9 @@
* so a remote parent left unsampled would make a parent-based sampler drop every child span; deferring to the sampler
* avoids that.
*
- * The ancestor is a non-recording context: it is either the external backend server span or a synthetic root the SDK
- * does not export.
+ *
The ancestor is a non-recording context: it is either the external backend server span or an SDK-owned synthetic
+ * root. Both plugins export the synthetic root before each invocation returns, including suspension, with its stable
+ * execution-start timestamp and zero duration. They never export a remote parent.
*
* @param executionAncestor the common parent context for the Workflow and Invocation spans
*/
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java
index 4e2e3cd70..8be19218b 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java
@@ -92,7 +92,8 @@ public class InvocationOtelPlugin implements DurableExecutionPlugin {
// Per-invocation state
private volatile boolean tracingEnabled;
private volatile Span invocationSpan;
- // Opened and closed by the invocation hooks on the root handler thread.
+ private volatile Span executionRootSpan;
+ // The root handler opens and closes this context through the base invocation lifecycle.
private Scope handlerScope;
private volatile String durableExecutionArn;
// Trace ID and flags of the execution trace, published together as one snapshot so readers never pair a trace ID
@@ -116,7 +117,7 @@ private record ExecutionTrace(String traceId, TraceFlags flags) {}
// Thread-safe storage for attempt spans/scopes (keyed by operationId + "-" + attempt)
private final ConcurrentHashMap attemptSpans = new ConcurrentHashMap<>();
- private final ConcurrentHashMap attemptScopes = new ConcurrentHashMap<>();
+ private final UserFunctionScopes attemptScopes = new UserFunctionScopes();
// Store operation span contexts for parent resolution (keyed by operationId)
private final ConcurrentHashMap operationContexts = new ConcurrentHashMap<>();
@@ -236,9 +237,29 @@ public void onInvocationStart(InvocationInfo info) {
var sampled = OtelPluginSupport.isSampled(decision);
var execCtx = ExecutionTraceContext.resolve(
extracted, canonicalTraceId, info.durableExecutionArn(), idGenerator, () -> sampled);
+ executionStartTime = info.executionStartTime();
+ var root = DurableExecutionRoot.start(
+ tracer,
+ idGenerator,
+ execCtx.executionAncestor(),
+ info.durableExecutionArn(),
+ executionStartTime,
+ samplingIntent,
+ OtelPluginSupport.usesLocalDurableSampler(sdkTracerProvider),
+ OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider));
+ try {
+ DurableExecutionRoot.end(root, executionStartTime);
+ } finally {
+ // Publish only after end callbacks, so a reentrant End cannot flush from inside root.end().
+ // Retain the ended root as a flush marker even if an onEnd processor throws during Start.
+ executionRootSpan = root;
+ }
+ // A visible plain replacement keeps its documented ancestor-flag fallback, without SDK-wide overrides.
+ if (executionRootSpan != null && OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)) {
+ execCtx = new ExecutionTraceContext(executionRootSpan.getSpanContext());
+ }
executionTrace = new ExecutionTrace(canonicalTraceId, execCtx.traceFlags());
executionAncestor = execCtx.executionAncestor();
- executionStartTime = info.executionStartTime();
// Invocation span parent — the same-trace ambient span when available, then the execution ancestor, so the
// Invocation span stays on the execution trace.
@@ -279,8 +300,6 @@ private Scope activateHandlerContext() {
var trace = executionTrace;
if (!tracingEnabled || trace == null) return null;
var ambient = Span.current().getSpanContext();
- // Preserve a compatible ambient Lambda span. An absent or unrelated ambient span must not leave
- // handler instrumentation outside the durable execution's canonical trace.
if (ambient.isValid() && trace.traceId().equals(ambient.getTraceId())) return Scope.noop();
return invocationSpan.makeCurrent();
}
@@ -293,6 +312,20 @@ public void onInvocationEnd(InvocationEndInfo info) {
}
private void endInvocation(InvocationEndInfo info) {
+ var root = executionRootSpan;
+ var provider = sdkTracerProvider;
+ var shouldFlush = tracingEnabled || root != null;
+ executionRootSpan = null; // The anchor already ended at Start; flush this invocation only once.
+ OtelPluginSupport.finishWithRootCleanup(() -> finishInvocationEnd(info), () -> {
+ if (shouldFlush && provider != null) {
+ var flushResult = provider.forceFlush().join(5, TimeUnit.SECONDS);
+ if (!flushResult.isSuccess())
+ logger.warn("OTel span flush failed or timed out — some spans may be lost");
+ }
+ });
+ }
+
+ private void finishInvocationEnd(InvocationEndInfo info) {
if (!tracingEnabled) {
return;
}
@@ -365,14 +398,6 @@ private void endInvocation(InvocationEndInfo info) {
executionAncestor = null;
executionStartTime = null;
samplingIntent = null;
-
- if (sdkTracerProvider != null) {
- // Flush spans before Lambda freezes
- var flushResult = sdkTracerProvider.forceFlush().join(5, TimeUnit.SECONDS);
- if (!flushResult.isSuccess()) {
- logger.warn("OTel span flush failed or timed out — some spans may be lost");
- }
- }
}
// ─── Operation hooks ─────────────────────────────────────────────────
@@ -551,15 +576,10 @@ public void onUserFunctionStart(UserFunctionStartInfo info) {
@Override
public void onUserFunctionEnd(UserFunctionEndInfo info) {
- if (!tracingEnabled) return;
-
var key = attemptKey(info.id(), info.attempt());
-
- // Close scope first (must happen on same thread as makeCurrent)
- var scope = attemptScopes.remove(key);
- if (scope != null) {
- scope.close();
- }
+ // A running user function can finish after invocation End; its scope still belongs to this thread.
+ attemptScopes.close(key);
+ if (!tracingEnabled) return;
// Clear span-level MDC after user function completes (keep trace_id for handler-level logs between steps)
if (enableMdc) {
@@ -618,10 +638,7 @@ private boolean bindTracer() {
private void endOpenSpansChildFirst() {
// Attempt spans are children of operation spans.
- for (var scope : attemptScopes.values()) {
- scope.close();
- }
- attemptScopes.clear();
+ attemptScopes.closeCurrentThread();
for (var span : attemptSpans.values()) {
span.end();
}
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java
index e3af17bab..0c011218b 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java
@@ -19,43 +19,28 @@
import org.slf4j.LoggerFactory;
import software.amazon.lambda.durable.execution.DurableExecutor;
-/** Shared utilities for OTel plugin default constructor support (ADOT Java agent SPI path). */
+/** Shared provider setup, sampling resolution, and invocation cleanup support. */
final class OtelPluginSupport {
- private static final Logger logger = LoggerFactory.getLogger(OtelPluginSupport.class);
-
- private OtelPluginSupport() {}
-
- /** Only the same DurableSampler class copy can consume this loader's parent-context holder. */
- static boolean usesLocalDurableSampler(SdkTracerProvider provider) {
- return provider != null && provider.getSampler() instanceof DurableSampler;
- }
-
- /** A visible replacement sampler cannot consume our bridge; an opaque agent provider may still need it. */
- static boolean usesDurableSamplingBridge(SdkTracerProvider provider) {
- // Class-name matching is only for the cross-loader wire carrier, never for the context-key ownership check.
- return provider == null || provider.getSampler().getClass().getName().equals(DurableSampler.class.getName());
- }
-
- /** Closes the owning thread's scope without hiding a finalization error or a later JVM-fatal cleanup failure. */
- static void runInvocationEnd(Scope scope, Runnable end) {
+ /** Always runs invocation cleanup without hiding an earlier failure or a later JVM fatal. */
+ static void finishWithRootCleanup(Runnable finish, Runnable cleanup) {
Throwable primary = null;
try {
- end.run();
+ finish.run();
} catch (Throwable failure) {
primary = failure;
throw failure;
} finally {
try {
- if (scope != null) scope.close();
- } catch (Throwable cleanup) {
- if (primary == null) throw cleanup;
- if (primary != cleanup) {
- if (endFailurePriority(cleanup) > endFailurePriority(primary)) {
- cleanup.addSuppressed(primary);
- throw cleanup;
+ cleanup.run();
+ } catch (Throwable failure) {
+ if (primary == null) throw failure;
+ if (primary != failure) {
+ if (endFailurePriority(failure) > endFailurePriority(primary)) {
+ failure.addSuppressed(primary);
+ throw failure;
}
- primary.addSuppressed(cleanup);
+ primary.addSuppressed(failure);
}
}
}
@@ -68,7 +53,14 @@ private static int endFailurePriority(Throwable failure) {
return 1;
}
- /** Rejects an older core before a plugin builds a provider or activates any thread-local context. */
+ /** Applies the base lifecycle's scope restoration with the same cleanup failure precedence. */
+ static void runInvocationEnd(Scope scope, Runnable end) {
+ finishWithRootCleanup(end, () -> {
+ if (scope != null) scope.close();
+ });
+ }
+
+ /** Rejects an older core before a plugin builds a provider or activates thread-local context. */
static void requireSameThreadInvocationHooks() {
var message = "This OpenTelemetry plugin requires same-thread invocation hooks from the Durable Execution"
+ " core (2.2.2 lifecycle capability). Upgrade the core together with the plugin layer.";
@@ -79,6 +71,21 @@ static void requireSameThreadInvocationHooks() {
}
}
+ private static final Logger logger = LoggerFactory.getLogger(OtelPluginSupport.class);
+
+ private OtelPluginSupport() {}
+
+ /** A visible replacement sampler cannot consume our bridge; an opaque agent provider may still need it. */
+ static boolean usesDurableSamplingBridge(SdkTracerProvider provider) {
+ // This identifies only the cross-loader wire consumer; context-key ownership still requires instanceof.
+ return provider == null || provider.getSampler().getClass().getName().equals(DurableSampler.class.getName());
+ }
+
+ /** Only the same DurableSampler class copy can consume this loader's parent-context holder. */
+ static boolean usesLocalDurableSampler(SdkTracerProvider provider) {
+ return provider != null && provider.getSampler() instanceof DurableSampler;
+ }
+
/** Creates a new DeterministicIdGenerator for the application-side state bridge. */
static DeterministicIdGenerator createDefaultIdGenerator() {
return new DeterministicIdGenerator();
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java
index 62510c547..c669c86b9 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java
@@ -10,6 +10,8 @@ final class SpanAttributes {
private SpanAttributes() {}
static final AttributeKey DURABLE_EXECUTION_ARN = AttributeKey.stringKey("durable.execution.arn");
+ static final AttributeKey DURABLE_EXECUTION_SYNTHETIC_ROOT =
+ AttributeKey.booleanKey("durable.execution.synthetic_root");
static final AttributeKey DURABLE_EXECUTION_STATUS = AttributeKey.stringKey("durable.execution.status");
static final AttributeKey DURABLE_OPERATION_ID = AttributeKey.stringKey("durable.operation.id");
static final AttributeKey DURABLE_OPERATION_TYPE = AttributeKey.stringKey("durable.operation.type");
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/UserFunctionScopes.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/UserFunctionScopes.java
new file mode 100644
index 000000000..f2bfb2647
--- /dev/null
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/UserFunctionScopes.java
@@ -0,0 +1,33 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import io.opentelemetry.context.Scope;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+
+/** Keeps thread-bound OTel scopes with their user-function owner, including after invocation End. */
+final class UserFunctionScopes {
+ private final ThreadLocal> scopes = ThreadLocal.withInitial(LinkedHashMap::new);
+
+ void put(String key, Scope scope) {
+ scopes.get().put(key, scope);
+ }
+
+ void close(String key) {
+ var owned = scopes.get();
+ var scope = owned.remove(key);
+ if (owned.isEmpty()) scopes.remove();
+ if (scope != null) scope.close();
+ }
+
+ /** Closes abandoned scopes only on this thread; running foreign owners retain their late End cleanup. */
+ void closeCurrentThread() {
+ var owned = scopes.get();
+ scopes.remove();
+ var open = new ArrayList<>(owned.values());
+ for (var index = open.size() - 1; index >= 0; index--) {
+ open.get(index).close();
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java
index 59b5cb32d..613e8e61c 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java
@@ -3,14 +3,18 @@
package software.amazon.lambda.durable.otel;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import io.opentelemetry.api.GlobalOpenTelemetry;
import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.trace.Span;
import io.opentelemetry.api.trace.SpanKind;
import io.opentelemetry.api.trace.TraceState;
import io.opentelemetry.api.trace.Tracer;
@@ -19,7 +23,10 @@
import io.opentelemetry.context.propagation.ContextPropagators;
import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
import io.opentelemetry.sdk.trace.IdGenerator;
+import io.opentelemetry.sdk.trace.ReadWriteSpan;
+import io.opentelemetry.sdk.trace.ReadableSpan;
import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.SpanProcessor;
import io.opentelemetry.sdk.trace.data.LinkData;
import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
import io.opentelemetry.sdk.trace.samplers.Sampler;
@@ -29,6 +36,7 @@
import java.net.URLClassLoader;
import java.time.Instant;
import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
@@ -50,6 +58,147 @@
* then asserts the thread-scoped system-property bridge carries the decision from one loader to the other.
*/
class DurableSamplingDecisionClassLoaderTest {
+ @ParameterizedTest
+ @CsvSource({
+ "InvocationOtelPlugin,local,false", "InvocationOtelPlugin,local,true",
+ "InvocationOtelPlugin,foreign,false", "InvocationOtelPlugin,foreign,true",
+ "InvocationOtelPlugin,opaque,false", "InvocationOtelPlugin,opaque,true",
+ "ExecutionOtelPlugin,local,false", "ExecutionOtelPlugin,local,true",
+ "ExecutionOtelPlugin,foreign,false", "ExecutionOtelPlugin,foreign,true",
+ "ExecutionOtelPlugin,opaque,false", "ExecutionOtelPlugin,opaque,true"
+ })
+ void managedAnchorKeepsStableIdentityAndChildrenRetainSamplerMetadata(
+ String pluginName, String topology, boolean recordOnly) throws Exception {
+ var previousHeader = System.getProperty("com.amazonaws.xray.traceHeader");
+ GlobalOpenTelemetry.resetForTest();
+ OtelPluginAutoConfigurationState.markInstalled();
+ System.setProperty("com.amazonaws.xray.traceHeader", "Root=1-6955b900-123456789012345678901234");
+ var evaluations = new AtomicInteger();
+ var key = AttributeKey.stringKey("sampler.extra");
+ var result = new SamplingResult() {
+ public SamplingDecision getDecision() {
+ return recordOnly ? SamplingDecision.RECORD_ONLY : SamplingDecision.RECORD_AND_SAMPLE;
+ }
+
+ public Attributes getAttributes() {
+ return Attributes.of(key, "kept");
+ }
+
+ public TraceState getUpdatedTraceState(TraceState parent) {
+ var depth = parent.get("depth");
+ return parent.toBuilder()
+ .put("vendor", "kept")
+ .put("depth", Integer.toString(depth == null ? 1 : Integer.parseInt(depth) + 1))
+ .build();
+ }
+ };
+ var delegate = new Sampler() {
+ public SamplingResult shouldSample(
+ Context parent,
+ String traceId,
+ String name,
+ SpanKind kind,
+ Attributes attributes,
+ List links) {
+ evaluations.incrementAndGet();
+ return result;
+ }
+
+ public String getDescription() {
+ return "custom-metadata";
+ }
+ };
+ try (var appLoader = pluginClassLoader();
+ var agentLoader = pluginClassLoader();
+ var exporter = InMemorySpanExporter.create()) {
+ var loader = topology.equals("local") ? appLoader : agentLoader;
+ var samplerType = Class.forName(DurableSampler.class.getName(), true, loader);
+ var wrap = samplerType.getDeclaredMethod("wrap", Sampler.class);
+ wrap.setAccessible(true);
+ var idType = Class.forName(DeterministicIdGenerator.class.getName(), true, loader);
+ try (var provider = SdkTracerProvider.builder()
+ .setSampler((Sampler) wrap.invoke(null, delegate))
+ .setIdGenerator((IdGenerator) idType.getConstructor().newInstance())
+ .addSpanProcessor(SimpleSpanProcessor.builder(exporter)
+ .setExportUnsampledSpans(true)
+ .build())
+ .build()) {
+ var hidden = new TracerProvider() {
+ public Tracer get(String name) {
+ return provider.get(name);
+ }
+
+ public Tracer get(String name, String version) {
+ return provider.get(name, version);
+ }
+ };
+ GlobalOpenTelemetry.set(new OpenTelemetry() {
+ public TracerProvider getTracerProvider() {
+ return topology.equals("opaque") ? hidden : provider;
+ }
+
+ public ContextPropagators getPropagators() {
+ return ContextPropagators.noop();
+ }
+ });
+ var plugin = (DurableExecutionPlugin)
+ Class.forName("software.amazon.lambda.durable.otel." + pluginName, true, appLoader)
+ .getConstructor()
+ .newInstance();
+ var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/custom/id";
+ for (var first : new boolean[] {true, false}) {
+ plugin.onInvocationStart(new InvocationInfo("request", arn, first, Instant.ofEpochSecond(10)));
+ plugin.onInvocationEnd(new InvocationEndInfo(
+ "request",
+ arn,
+ first,
+ first ? InvocationStatus.PENDING : InvocationStatus.SUCCEEDED,
+ null));
+ }
+ var spans = exporter.getFinishedSpanItems();
+ var roots = spans.stream()
+ .filter(span -> span.getName().equals("DurableExecutionRoot"))
+ .toList();
+ assertEquals(recordOnly ? 0 : 2, roots.size());
+ var rootContext = spans.stream()
+ .filter(span -> span.getName().equals("Invocation"))
+ .findFirst()
+ .orElseThrow()
+ .getParentSpanContext();
+ assertEquals(TraceState.getDefault(), rootContext.getTraceState());
+ for (var anchor : roots) {
+ assertEquals(
+ rootContext, anchor.getSpanContext(), "Re-export keeps the complete stable root context");
+ assertEquals(10_000_000_000L, anchor.getStartEpochNanos());
+ assertEquals(anchor.getStartEpochNanos(), anchor.getEndEpochNanos());
+ assertEquals(
+ TraceState.getDefault(), anchor.getSpanContext().getTraceState());
+ assertTrue(anchor.getSpanContext().isSampled());
+ assertNull(anchor.getAttributes().get(key));
+ }
+ var descendants = spans.stream()
+ .filter(span -> span.getName().equals("Invocation")
+ || span.getName().equals("Workflow"))
+ .toList();
+ assertEquals(3, descendants.size());
+ for (var child : descendants) {
+ assertEquals(
+ rootContext,
+ child.getParentSpanContext(),
+ "A child must record the materialized parent's flags and trace state: " + child.getName());
+ assertEquals("1", child.getSpanContext().getTraceState().get("depth"));
+ assertEquals("kept", child.getAttributes().get(key));
+ }
+ assertEquals(topology.equals("local") ? 2 : 1, evaluations.get());
+ }
+ } finally {
+ GlobalOpenTelemetry.resetForTest();
+ OtelPluginAutoConfigurationState.resetInstalledForTest();
+ if (previousHeader == null) System.clearProperty("com.amazonaws.xray.traceHeader");
+ else System.setProperty("com.amazonaws.xray.traceHeader", previousHeader);
+ }
+ }
+
@ParameterizedTest
@CsvSource({
"InvocationOtelPlugin,local,false", "InvocationOtelPlugin,local,true",
@@ -152,6 +301,16 @@ public ContextPropagators getPropagators() {
.filter(s -> s.getName().equals("Invocation"))
.toList();
assertEquals(executionCount * 2, spans.size());
+ var anchors = exporter.getFinishedSpanItems().stream()
+ .filter(s -> s.getName().equals("DurableExecutionRoot"))
+ .toList();
+ assertEquals(2, anchors.size(), "Only sampled executions export anchors, twice across resume");
+ for (var anchor : anchors) {
+ assertTrue(anchor.getSpanContext().isSampled());
+ assertNull(anchor.getAttributes().get(key));
+ assertEquals(
+ TraceState.getDefault(), anchor.getSpanContext().getTraceState());
+ }
for (var span : spans) {
var second = span.getAttributes()
.get(SpanAttributes.DURABLE_EXECUTION_ARN)
@@ -198,15 +357,18 @@ void decisionCrossesClassLoaderBoundary_viaScopedProperty() throws Exception {
var openScope = appDecision.getDeclaredMethod("openScope", appIntentClass);
openScope.setAccessible(true);
- var get = agentDecision.getDeclaredMethod("get", Context.class);
+ var get = agentDecision.getDeclaredMethod("consume", Context.class);
get.setAccessible(true);
+ var consume = agentDecision.getDeclaredMethod("consume", Context.class);
+ consume.setAccessible(true);
// The application-side loader publishes the intent on this thread; the agent-side loader reads it back from
// a ROOT context (its context key would be a different instance and would miss), reconstructing its own
// Intent from the bridged value.
var scope = (AutoCloseable) openScope.invoke(null, appIntent);
try {
- var crossLoaderIntent = get.invoke(null, Context.root());
+ var crossLoaderIntent = consume.invoke(null, Context.root());
+ assertNull(get.invoke(null, Context.root()), "The bridge is consumed before onStart processors run");
assertNotNull(
crossLoaderIntent, "The agent-side loader must read the intent published by the app side");
// Its Intent type is the agent loader's copy; read the resolved SamplingResult reflectively.
@@ -226,6 +388,147 @@ void decisionCrossesClassLoaderBoundary_viaScopedProperty() throws Exception {
}
}
+ @ParameterizedTest
+ @CsvSource({
+ "InvocationOtelPlugin,true,false,DurableExecutionRoot",
+ "InvocationOtelPlugin,false,false,DurableExecutionRoot",
+ "InvocationOtelPlugin,false,true,DurableExecutionRoot",
+ "ExecutionOtelPlugin,true,false,DurableExecutionRoot",
+ "ExecutionOtelPlugin,false,false,DurableExecutionRoot",
+ "ExecutionOtelPlugin,false,true,DurableExecutionRoot",
+ "InvocationOtelPlugin,true,false,Invocation",
+ "InvocationOtelPlugin,false,false,Invocation",
+ "InvocationOtelPlugin,false,true,Invocation",
+ "ExecutionOtelPlugin,true,false,Invocation",
+ "ExecutionOtelPlugin,false,false,Invocation",
+ "ExecutionOtelPlugin,false,true,Invocation",
+ "InvocationOtelPlugin,false,false,PlainInvocation",
+ "ExecutionOtelPlugin,false,false,PlainInvocation",
+ "InvocationOtelPlugin,false,false,PlainDurableExecutionRoot",
+ "ExecutionOtelPlugin,false,false,PlainDurableExecutionRoot"
+ })
+ void agentProcessorForwardingParentCannotReuseApplicationSamplingIntent(
+ String pluginName, boolean hideProvider, boolean localSampler, String observedSpan) throws Exception {
+ var plainSampler = observedSpan.startsWith("Plain");
+ var spanName = plainSampler ? observedSpan.substring("Plain".length()) : observedSpan;
+ var previousHeader = System.getProperty("com.amazonaws.xray.traceHeader");
+ GlobalOpenTelemetry.resetForTest();
+ OtelPluginAutoConfigurationState.markInstalled();
+ System.setProperty("com.amazonaws.xray.traceHeader", "Root=1-6955b900-123456789012345678901234;Sampled=1");
+ try (var appLoader = pluginClassLoader();
+ var agentLoader = pluginClassLoader()) {
+ var appSamplerType = Class.forName(DurableSampler.class.getName(), true, appLoader);
+ var appWrap = appSamplerType.getDeclaredMethod("wrap", Sampler.class);
+ appWrap.setAccessible(true);
+ var agentSamplerType = Class.forName(DurableSampler.class.getName(), true, agentLoader);
+ var agentWrap = agentSamplerType.getDeclaredMethod("wrap", Sampler.class);
+ agentWrap.setAccessible(true);
+ var appDecision = Class.forName(DurableSamplingDecision.class.getName(), true, appLoader);
+ var appGet = appDecision.getDeclaredMethod("consume", Context.class);
+ appGet.setAccessible(true);
+ var agentDecision = Class.forName(DurableSamplingDecision.class.getName(), true, agentLoader);
+ var agentGet = agentDecision.getDeclaredMethod("consume", Context.class);
+ agentGet.setAccessible(true);
+ var agentIdType = Class.forName(DeterministicIdGenerator.class.getName(), true, agentLoader);
+ var callbacks = new AtomicInteger();
+ var leakedSampling = new AtomicBoolean();
+ try (var appProvider = SdkTracerProvider.builder()
+ .setSampler((Sampler) appWrap.invoke(null, Sampler.alwaysOff()))
+ .build();
+ var agentProvider = SdkTracerProvider.builder()
+ .setSampler(
+ plainSampler
+ ? Sampler.alwaysOn()
+ : (Sampler) (localSampler ? appWrap : agentWrap)
+ .invoke(null, Sampler.alwaysOff()))
+ .setIdGenerator(
+ (IdGenerator) agentIdType.getConstructor().newInstance())
+ .addSpanProcessor(new SpanProcessor() {
+ @Override
+ public void onStart(Context parent, ReadWriteSpan span) {
+ if (!span.getName().equals(spanName)) return;
+ var unrelated = appProvider
+ .get("processor")
+ .spanBuilder("unrelated-forwarded-parent")
+ .setParent(parent)
+ .startSpan();
+ callbacks.incrementAndGet();
+ leakedSampling.compareAndSet(false, unrelated.isRecording());
+ unrelated.end();
+ }
+
+ @Override
+ public boolean isStartRequired() {
+ return true;
+ }
+
+ @Override
+ public void onEnd(ReadableSpan span) {}
+
+ @Override
+ public boolean isEndRequired() {
+ return false;
+ }
+ })
+ .build()) {
+ assertEquals(localSampler, agentProvider.getSampler().getClass() == appSamplerType);
+ var hiddenAgentProvider = new TracerProvider() {
+ @Override
+ public Tracer get(String name) {
+ return agentProvider.get(name);
+ }
+
+ @Override
+ public Tracer get(String name, String version) {
+ return agentProvider.get(name, version);
+ }
+ };
+ GlobalOpenTelemetry.set(new OpenTelemetry() {
+ @Override
+ public TracerProvider getTracerProvider() {
+ return hideProvider ? hiddenAgentProvider : agentProvider;
+ }
+
+ @Override
+ public ContextPropagators getPropagators() {
+ return ContextPropagators.noop();
+ }
+ });
+ var plugin = (DurableExecutionPlugin)
+ Class.forName("software.amazon.lambda.durable.otel." + pluginName, true, appLoader)
+ .getConstructor()
+ .newInstance();
+ var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id";
+ var previousAmbient = Span.current();
+ var ambient = appProvider
+ .get("ambient")
+ .spanBuilder("ambient-control")
+ .startSpan();
+ try (var ambientScope = ambient.makeCurrent()) {
+ for (var first : new boolean[] {true, false}) {
+ plugin.onInvocationStart(new InvocationInfo("request", arn, first, Instant.EPOCH));
+ plugin.onInvocationEnd(
+ new InvocationEndInfo("request", arn, first, InvocationStatus.PENDING, null));
+ assertSame(ambient, Span.current(), "Plugin cleanup must preserve the caller's ambient span");
+ assertNull(appGet.invoke(null, Context.root()), "Application sampling intent must be cleared");
+ assertNull(agentGet.invoke(null, Context.root()), "Agent sampling intent must be cleared");
+ }
+ } finally {
+ ambient.end();
+ }
+ assertSame(previousAmbient, Span.current());
+ assertEquals(2, callbacks.get(), "The selected span in both invocations must reach the real processor");
+ assertFalse(
+ leakedSampling.get(), "The supplied parent must not override the app provider's DROP policy");
+ }
+ } finally {
+ GlobalOpenTelemetry.resetForTest();
+ OtelPluginAutoConfigurationState.resetInstalledForTest();
+ if (previousHeader == null) System.clearProperty("com.amazonaws.xray.traceHeader");
+ else System.setProperty("com.amazonaws.xray.traceHeader", previousHeader);
+ }
+ }
+
/**
* A child-first class loader that loads {@code software.amazon.lambda.durable.otel.*} itself (so each instance
* holds its own copies, mirroring the two plugin class loaders) while delegating OpenTelemetry and JDK classes to
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java
index 129a1b045..175e7ea9d 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java
@@ -144,17 +144,15 @@ void waitAcrossInvocations_sharesOneTrace_andExportsWorkflowOnceOnTerminal() {
}
@Test
- void wrappedAgentProvider_withRootDroppingSampler_keepsExecutionTreeConsistentlySampled() {
- // The agent provider is hidden behind a classloader wrapper, so the plugin cannot reach the sampler and the
- // bridge published nothing. With no reachable decision, the synthetic execution root is treated as sampled so
- // the execution root and everything parented onto it are sampled together — no orphan operation spans exported
- // under a dropped root.
+ void wrappedAgentProvider_withInstalledRootDroppingSampler_keepsExecutionTreeDropped() {
+ // The documented agent customizer retains DurableSampler as the final sampler. Its actual delegate's
+ // root DROP decision must apply to the fallback anchor and descendants, including behind an opaque provider.
OtelPluginAutoConfigurationState.markInstalled();
GlobalOpenTelemetry.resetForTest();
var globalExporter = InMemorySpanExporter.create();
var sdkTracerProvider = SdkTracerProvider.builder()
.setIdGenerator(new DeterministicIdGenerator())
- .setSampler(Sampler.parentBased(Sampler.alwaysOff()))
+ .setSampler(DurableSampler.wrap(Sampler.parentBased(Sampler.alwaysOff())))
.addSpanProcessor(SimpleSpanProcessor.create(globalExporter))
.build();
var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(sdkTracerProvider);
@@ -180,18 +178,10 @@ public ContextPropagators getPropagators() {
var result = runner.runUntilComplete("World");
assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
- var spans = globalExporter.getFinishedSpanItems();
- assertSpanExists(spans, "Workflow");
- assertSpanExists(spans, "wrapped-step");
- var workflowTraceId = spans.stream()
- .filter(s -> s.getName().equals("Workflow"))
- .findFirst()
- .orElseThrow()
- .getTraceId();
- // Every exported span is on the one execution trace: the root and its children are sampled consistently.
assertTrue(
- spans.stream().allMatch(s -> s.getTraceId().equals(workflowTraceId)),
- "All spans share the sampled execution trace");
+ globalExporter.getFinishedSpanItems().isEmpty(),
+ "The installed sampler's root DROP policy must not force-sample an execution tree");
+ sdkTracerProvider.close();
}
// Helpers
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java
index 875ae6c57..9a6bded9a 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java
@@ -141,7 +141,7 @@ void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() {
new InvocationEndInfo("req-enabled", "arn:enabled", true, InvocationStatus.SUCCEEDED, null));
var spans = globalExporter.getFinishedSpanItems();
- assertEquals(3, spans.size());
+ assertEquals(4, spans.size());
assertTrue(spans.stream().anyMatch(span -> span.getName().equals("enabled-step")));
assertFalse(spans.stream().anyMatch(span -> span.getName().equals("disabled-step")));
}
@@ -178,8 +178,8 @@ void defaultConstructor_usesGlobalSdkTracerProviderDirectly() {
new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
var spans = globalExporter.getFinishedSpanItems();
- // Workflow + Invocation + operation = 3
- assertEquals(3, spans.size());
+ // Workflow + Invocation + operation + DurableExecutionRoot = 4
+ assertEquals(4, spans.size());
assertTrue(spans.stream().anyMatch(span -> span.getName().equals("Workflow")));
assertTrue(spans.stream().anyMatch(span -> span.getName().equals("Invocation")));
assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step")));
@@ -206,7 +206,7 @@ void terminalInvocation_exportsWorkflowAndInvocationSpans() {
plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null));
var spans = spanExporter.getFinishedSpanItems();
- assertEquals(2, spans.size(), "Terminal invocation should export the Workflow span and the invocation span");
+ assertEquals(3, spans.size(), "Terminal invocation exports Workflow, Invocation, and DurableExecutionRoot");
var workflowSpan = spanByName(spans, "Workflow");
var invocationSpan = spanByName(spans, "Invocation");
@@ -404,10 +404,13 @@ void nonTerminalInvocation_doesNotExportWorkflowSpan() {
plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null));
var spans = spanExporter.getFinishedSpanItems();
- // Only the invocation span is exported; the Workflow span is not ended on non-terminal status.
- assertEquals(1, spans.size());
- assertEquals("Invocation", spans.get(0).getName());
- assertEquals(StatusCode.OK, spans.get(0).getStatus().getStatusCode(), "PENDING invocation span maps to OK");
+ // Invocation and DurableExecutionRoot are exported; Workflow remains deferred until terminal status.
+ assertEquals(2, spans.size());
+ assertEquals("Invocation", spanByName(spans, "Invocation").getName());
+ assertEquals(
+ StatusCode.OK,
+ spanByName(spans, "Invocation").getStatus().getStatusCode(),
+ "PENDING invocation span maps to OK");
}
@Test
@@ -451,8 +454,8 @@ void retryingInvocation_invocationSpanUnset_workflowNotExported() {
"req-1", ARN, true, InvocationStatus.RETRYING, new RuntimeException("transient")));
var spans = spanExporter.getFinishedSpanItems();
- assertEquals(1, spans.size(), "RETRYING is non-terminal — Workflow span not exported");
- var invocationSpan = spans.get(0);
+ assertEquals(2, spans.size(), "RETRYING is non-terminal — Workflow span not exported");
+ var invocationSpan = spanByName(spans, "Invocation");
assertEquals("Invocation", invocationSpan.getName());
assertEquals(
StatusCode.UNSET,
@@ -943,11 +946,11 @@ void operationNotCompleted_notEndedAtInvocationEnd() {
plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null));
var spans = spanExporter.getFinishedSpanItems();
- // Only the invocation span is exported. A still-open operation has no recording span (creation is deferred to
- // onOperationEnd), so there is nothing to abandon, and the Workflow span is not exported on a non-terminal
- // invocation.
- assertEquals(1, spans.size());
- assertEquals("Invocation", spans.get(0).getName());
+ // Invocation and DurableExecutionRoot are exported. A still-open operation has no recording span
+ // (creation is deferred to onOperationEnd), so there is nothing to abandon, and the Workflow span is not
+ // exported on a non-terminal invocation.
+ assertEquals(2, spans.size());
+ assertEquals("Invocation", spanByName(spans, "Invocation").getName());
assertTrue(
spans.stream().noneMatch(s -> s.getName().equals("my-wait")),
"An operation still open at invocation end must not be ended/exported in onInvocationEnd");
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootDeliveryFailureTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootDeliveryFailureTest.java
new file mode 100644
index 000000000..e4cc31f84
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootDeliveryFailureTest.java
@@ -0,0 +1,279 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.common.CompletableResultCode;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.ReadWriteSpan;
+import io.opentelemetry.sdk.trace.ReadableSpan;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.SpanProcessor;
+import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import java.util.stream.Stream;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.MethodSource;
+import software.amazon.awssdk.services.lambda.model.*;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.TypeToken;
+import software.amazon.lambda.durable.execution.DurableExecutor;
+import software.amazon.lambda.durable.model.DurableExecutionInput;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.*;
+import software.amazon.lambda.durable.serde.JacksonSerDes;
+import software.amazon.lambda.durable.serde.SerDes;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient;
+import software.amazon.lambda.durable.util.ExceptionHelper;
+
+class ExecutionRootDeliveryFailureTest {
+ @ParameterizedTest
+ @CsvSource({
+ "false,serialize",
+ "true,serialize",
+ "false,checkpoint",
+ "true,checkpoint",
+ "false,root-end",
+ "true,root-end",
+ "false,success",
+ "true,success"
+ })
+ void outputPreparationAndOrdinaryRootEndFailuresStillReleaseAndFlush(boolean executionView, String mode)
+ throws Exception {
+ var originalContext = Context.current();
+ var failure = new IllegalStateException("controlled " + mode);
+ var failOnce = new AtomicBoolean(true);
+ var bodyCalls = new AtomicInteger();
+ var rootEnds = new AtomicInteger();
+ var flushes = new AtomicInteger();
+ var roots = new CopyOnWriteArrayList();
+ var ends = new CopyOnWriteArrayList();
+ var observer = new SpanProcessor() {
+ public void onStart(Context parent, ReadWriteSpan span) {
+ if (span.getName().equals("DurableExecutionRoot")) roots.add(span);
+ }
+
+ public boolean isStartRequired() {
+ return true;
+ }
+
+ public void onEnd(ReadableSpan span) {
+ if (span.getName().equals("DurableExecutionRoot")) {
+ rootEnds.incrementAndGet();
+ if (mode.equals("root-end") && failOnce.compareAndSet(true, false)) throw failure;
+ }
+ }
+
+ public boolean isEndRequired() {
+ return true;
+ }
+
+ public CompletableResultCode forceFlush() {
+ flushes.incrementAndGet();
+ return CompletableResultCode.ofSuccess();
+ }
+ };
+ var client = new LocalMemoryExecutionClient() {
+ @Override
+ public CheckpointDurableExecutionResponse checkpoint(
+ String arn, String token, List updates) {
+ if (mode.equals("checkpoint")
+ && updates.stream().anyMatch(u -> u.type() == OperationType.EXECUTION)
+ && failOnce.compareAndSet(true, false)) throw failure;
+ return super.checkpoint(arn, token, updates);
+ }
+ };
+ var serDes = new SerDes() {
+ private final JacksonSerDes delegate = new JacksonSerDes();
+
+ public String serialize(Object value) {
+ if (mode.equals("serialize") && "result".equals(value) && failOnce.compareAndSet(true, false))
+ throw failure;
+ return delegate.serialize(value);
+ }
+
+ public T deserialize(String data, TypeToken type) {
+ return delegate.deserialize(data, type);
+ }
+ };
+ var caller = Executors.newSingleThreadExecutor();
+ var workers = Executors.newCachedThreadPool();
+ try (var exporter = InMemorySpanExporter.create();
+ var batch = BatchSpanProcessor.builder(exporter)
+ .setScheduleDelay(Duration.ofDays(1))
+ .build()) {
+ var builder = SdkTracerProvider.builder().addSpanProcessor(observer).addSpanProcessor(batch);
+ var otelConfig = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(() -> null)
+ .build();
+ DurableExecutionPlugin otel = executionView
+ ? new ExecutionOtelPlugin(builder, otelConfig)
+ : new InvocationOtelPlugin(builder, otelConfig);
+ var config = DurableConfig.builder()
+ .withDurableExecutionClient(client)
+ .withExecutorService(workers)
+ .withCheckpointDelay(Duration.ZERO)
+ .withSerDes(serDes)
+ .withPlugins(otel, new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo info) {
+ ends.add(info);
+ }
+ })
+ .build();
+ try {
+ var first = caller.submit(() -> DurableExecutor.execute(
+ input(),
+ null,
+ TypeToken.get(String.class),
+ (value, context) -> bodyCalls.incrementAndGet() == 1 && mode.equals("checkpoint")
+ ? "x".repeat(6 * 1024 * 1024)
+ : "result",
+ config));
+ if (mode.equals("serialize") || mode.equals("checkpoint")) {
+ var thrown = assertThrows(ExecutionException.class, () -> first.get(5, TimeUnit.SECONDS));
+ assertSame(failure, thrown.getCause(), "Output preparation failure remains caller-visible");
+ } else
+ assertEquals(
+ ExecutionStatus.SUCCEEDED,
+ first.get(5, TimeUnit.SECONDS).status());
+ var exported = exporter.getFinishedSpanItems();
+ System.out.println("ROOT_DELIVERY view=" + executionView + " mode=" + mode + " ends=" + ends.size()
+ + " rootEnds=" + rootEnds.get() + " rootRecording="
+ + roots.get(0).isRecording()
+ + " flushes=" + flushes.get() + " exported="
+ + exported.stream().map(s -> s.getName()).toList());
+ assertEquals(1, ends.size(), "Output preparation failure still has exactly one End");
+ assertEquals(
+ mode.equals("serialize") || mode.equals("checkpoint")
+ ? InvocationStatus.RETRYING
+ : InvocationStatus.SUCCEEDED,
+ ends.get(0).invocationStatus());
+ assertEquals(1, rootEnds.get());
+ assertFalse(roots.get(0).isRecording());
+ assertEquals(1, flushes.get(), "Root onEnd failure does not skip flushing completed spans");
+ assertTrue(
+ exported.stream().anyMatch(s -> s.getName().equals("Invocation")),
+ "The BatchSpanProcessor must be flushed before invocation return");
+ var second = caller.submit(() -> DurableExecutor.execute(
+ input(), null, TypeToken.get(String.class), (value, context) -> "result", config));
+ assertEquals(
+ ExecutionStatus.SUCCEEDED,
+ second.get(5, TimeUnit.SECONDS).status());
+ assertEquals(2, ends.size());
+ assertEquals(2, rootEnds.get());
+ assertEquals(2, flushes.get());
+ assertEquals(roots.get(0).getSpanContext(), roots.get(1).getSpanContext());
+ assertTrue(roots.stream().noneMatch(ReadWriteSpan::isRecording));
+ assertSame(originalContext, Context.current());
+ } finally {
+ caller.shutdownNow();
+ workers.shutdownNow();
+ assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+ }
+
+ private static DurableExecutionInput input() {
+ var operation = Operation.builder()
+ .id("execution")
+ .type(OperationType.EXECUTION)
+ .status(OperationStatus.STARTED)
+ .startTimestamp(Instant.ofEpochSecond(10))
+ .executionDetails(
+ ExecutionDetails.builder().inputPayload("\"input\"").build())
+ .build();
+ return new DurableExecutionInput(
+ "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/root/execution",
+ "token",
+ CheckpointUpdatedExecutionState.builder().operations(operation).build());
+ }
+
+ static Stream preparationEndFailures() {
+ return Stream.of(false, true)
+ .flatMap(view -> Stream.of("ordinary", "vm", "death", "wrapped-vm")
+ .flatMap(preparation -> Stream.of("assertion", "vm", "death")
+ .map(end -> Arguments.of(view, preparation, end))));
+ }
+
+ @SuppressWarnings("removal")
+ @ParameterizedTest
+ @MethodSource("preparationEndFailures")
+ void preparationAndEndFailuresRetainCallerIdentityAndCleanupDiagnostics(
+ boolean executionView, String preparationKind, String endKind) {
+ Throwable original =
+ switch (preparationKind) {
+ case "vm", "wrapped-vm" -> new InternalError("preparation fatal");
+ case "death" -> new ThreadDeath();
+ default -> new IllegalStateException("preparation ordinary");
+ };
+ Throwable preparation = preparationKind.equals("wrapped-vm") ? new CompletionException(original) : original;
+ Error cleanup =
+ switch (endKind) {
+ case "vm" -> new InternalError("End fatal");
+ case "death" -> new ThreadDeath();
+ default -> new AssertionError("End error");
+ };
+ var ends = new AtomicInteger();
+ var seen = new AtomicReference();
+ var serDes = new SerDes() {
+ private final JacksonSerDes delegate = new JacksonSerDes();
+
+ public String serialize(Object value) {
+ if ("result".equals(value)) ExceptionHelper.sneakyThrow(preparation);
+ return delegate.serialize(value);
+ }
+
+ public T deserialize(String value, TypeToken type) {
+ return delegate.deserialize(value, type);
+ }
+ };
+ try (var exporter = InMemorySpanExporter.create()) {
+ var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter));
+ var otelConfig = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(() -> null)
+ .build();
+ DurableExecutionPlugin otel = executionView
+ ? new ExecutionOtelPlugin(builder, otelConfig)
+ : new InvocationOtelPlugin(builder, otelConfig);
+ var runner = LocalDurableTestRunner.create(
+ String.class,
+ (value, context) -> "result",
+ DurableConfig.builder()
+ .withSerDes(serDes)
+ .withPlugins(otel, new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo info) {
+ ends.incrementAndGet();
+ seen.set(info);
+ throw cleanup;
+ }
+ })
+ .build());
+ Throwable expected = original instanceof VirtualMachineError || original instanceof ThreadDeath
+ ? original
+ : cleanup instanceof VirtualMachineError || cleanup instanceof ThreadDeath ? cleanup : original;
+ assertSame(expected, assertThrows(Throwable.class, () -> runner.run("input")));
+ assertEquals(List.of(expected == original ? cleanup : original), List.of(expected.getSuppressed()));
+ assertEquals(1, ends.get());
+ assertEquals(InvocationStatus.RETRYING, seen.get().invocationStatus());
+ assertSame(original, seen.get().executionError());
+ var roots = exporter.getFinishedSpanItems().stream()
+ .filter(span -> span.getName().equals("DurableExecutionRoot"))
+ .toList();
+ assertEquals(1, roots.size());
+ assertEquals(roots.get(0).getStartEpochNanos(), roots.get(0).getEndEpochNanos());
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java
new file mode 100644
index 000000000..02779a68c
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootLifecycleTest.java
@@ -0,0 +1,188 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.common.CompletableResultCode;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.ReadWriteSpan;
+import io.opentelemetry.sdk.trace.ReadableSpan;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.SpanProcessor;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+import software.amazon.lambda.durable.plugin.InvocationStatus;
+import software.amazon.lambda.durable.util.ExceptionHelper;
+
+class ExecutionRootLifecycleTest {
+ private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/root/id";
+ private static final Instant START = Instant.ofEpochSecond(10);
+
+ @ParameterizedTest
+ @CsvSource({
+ "false,start-failure", "true,start-failure",
+ "false,end-failure", "true,end-failure",
+ "false,fatal-root-cleanup", "true,fatal-root-cleanup",
+ "false,reentrant-end", "true,reentrant-end"
+ })
+ void rootIsClosedOnceAcrossPartialStartEndFailureReentryAndReuse(boolean executionView, String mode) {
+ var original = Context.current();
+ var plugin = new AtomicReference();
+ var failOnce = new AtomicBoolean(true);
+ var rootFatalOnce = new AtomicBoolean(true);
+ var inRootCallback = new AtomicBoolean();
+ var rootsEnded = new AtomicInteger();
+ var flushes = new AtomicInteger();
+ var starts = new ArrayList();
+ var roots = new ArrayList();
+ var primary = new AssertionError("invocation processor failure");
+ var cleanupFatal = new InternalError("root cleanup fatal");
+ var observer = new SpanProcessor() {
+ public void onStart(Context parent, ReadWriteSpan span) {
+ starts.add(span.getName());
+ if (span.getName().equals("DurableExecutionRoot")) roots.add(span);
+ if (span.getName().equals("Invocation")
+ && mode.equals("start-failure")
+ && failOnce.compareAndSet(true, false)) throw primary;
+ }
+
+ public boolean isStartRequired() {
+ return true;
+ }
+
+ public void onEnd(ReadableSpan span) {
+ if (span.getName().equals("Invocation")
+ && mode.contains("failure")
+ && !mode.equals("start-failure")
+ && failOnce.compareAndSet(true, false)) throw primary;
+ if (!span.getName().equals("DurableExecutionRoot")) return;
+ rootsEnded.incrementAndGet();
+ inRootCallback.set(true);
+ try {
+ if (mode.equals("reentrant-end")) plugin.get().onInvocationEnd(end(false));
+ if (mode.equals("fatal-root-cleanup") && rootFatalOnce.compareAndSet(true, false))
+ throw cleanupFatal;
+ } finally {
+ inRootCallback.set(false);
+ }
+ }
+
+ public boolean isEndRequired() {
+ return true;
+ }
+
+ public CompletableResultCode forceFlush() {
+ assertFalse(inRootCallback.get(), "Reentrant End must not flush recursively from root end callbacks");
+ flushes.incrementAndGet();
+ return CompletableResultCode.ofSuccess();
+ }
+ };
+ try (var exporter = InMemorySpanExporter.create()) {
+ var builder = SdkTracerProvider.builder()
+ .addSpanProcessor(SimpleSpanProcessor.create(exporter))
+ .addSpanProcessor(observer);
+ var config = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(() -> null)
+ .build();
+ plugin.set(
+ executionView
+ ? new ExecutionOtelPlugin(builder, config)
+ : new InvocationOtelPlugin(builder, config));
+ var first = new InvocationInfo("first", ARN, true, START);
+ if (mode.equals("start-failure"))
+ assertSame(
+ primary,
+ assertThrows(AssertionError.class, () -> plugin.get().onInvocationStart(first)));
+ else if (mode.equals("fatal-root-cleanup"))
+ assertSame(
+ cleanupFatal,
+ assertThrows(InternalError.class, () -> plugin.get().onInvocationStart(first)));
+ else plugin.get().onInvocationStart(first);
+ assertEquals("DurableExecutionRoot", starts.get(0));
+ assertFalse(roots.get(0).isRecording(), "The anchor ended before the Start hook returned or failed");
+ if (mode.equals("end-failure"))
+ assertSame(
+ primary,
+ assertThrows(AssertionError.class, () -> plugin.get().onInvocationEnd(end(true))));
+ else plugin.get().onInvocationEnd(end(true));
+ assertEquals(1, rootsEnded.get());
+ assertFalse(roots.get(0).isRecording());
+ assertEquals(
+ 1,
+ flushes.get(),
+ "End flushes completed spans, including cleanup after a partially completed Start");
+ var beforeDuplicateEnd = flushes.get();
+ plugin.get().onInvocationEnd(end(true));
+ assertEquals(beforeDuplicateEnd, flushes.get());
+ assertEquals(1, rootsEnded.get());
+
+ plugin.get().onInvocationStart(new InvocationInfo("second", ARN, false, START));
+ plugin.get().onInvocationEnd(end(false));
+ assertEquals(2, rootsEnded.get());
+ assertEquals(roots.get(0).getSpanContext(), roots.get(1).getSpanContext());
+ var exportedRoots = exporter.getFinishedSpanItems().stream()
+ .filter(span -> span.getName().equals("DurableExecutionRoot"))
+ .toList();
+ assertEquals(2, exportedRoots.size());
+ for (var root : exportedRoots) {
+ assertEquals(10_000_000_000L, root.getStartEpochNanos());
+ assertEquals(root.getStartEpochNanos(), root.getEndEpochNanos());
+ }
+ assertSame(original, Context.current());
+ }
+ }
+
+ private static InvocationEndInfo end(boolean first) {
+ return new InvocationEndInfo("request", ARN, first, InvocationStatus.PENDING, null);
+ }
+
+ @SuppressWarnings("removal")
+ @ParameterizedTest
+ @CsvSource({
+ "runtime,ordinary",
+ "runtime,fatal",
+ "linkage,ordinary",
+ "linkage,fatal",
+ "assertion,ordinary",
+ "vm,ordinary",
+ "death,ordinary"
+ })
+ void flushPreservesTheExistingEndFailurePriority(String rootKind, String flushKind) {
+ Throwable rootFailure =
+ switch (rootKind) {
+ case "linkage" -> new LinkageError("root linkage");
+ case "assertion" -> new AssertionError("root assertion");
+ case "vm" -> new InternalError("root fatal");
+ case "death" -> new ThreadDeath();
+ default -> new IllegalStateException("root ordinary");
+ };
+ Throwable flushFailure = flushKind.equals("fatal")
+ ? new InternalError("flush fatal")
+ : new IllegalArgumentException("flush ordinary");
+ var flushes = new AtomicInteger();
+ var observed = assertThrows(
+ Throwable.class,
+ () -> OtelPluginSupport.finishWithRootCleanup(() -> ExceptionHelper.sneakyThrow(rootFailure), () -> {
+ flushes.incrementAndGet();
+ ExceptionHelper.sneakyThrow(flushFailure);
+ }));
+ var isolated = rootKind.equals("runtime") || rootKind.equals("linkage");
+ assertEquals(1, flushes.get());
+ var expected = isolated && flushKind.equals("fatal") ? flushFailure : rootFailure;
+ assertSame(expected, observed);
+ assertEquals(List.of(expected == rootFailure ? flushFailure : rootFailure), List.of(observed.getSuppressed()));
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootParityTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootParityTest.java
new file mode 100644
index 000000000..fd4cd4be8
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootParityTest.java
@@ -0,0 +1,205 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_ARN;
+import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_SYNTHETIC_ROOT;
+
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.api.trace.TraceState;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.data.LinkData;
+import io.opentelemetry.sdk.trace.data.SpanData;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import io.opentelemetry.sdk.trace.samplers.Sampler;
+import io.opentelemetry.sdk.trace.samplers.SamplingDecision;
+import io.opentelemetry.sdk.trace.samplers.SamplingResult;
+import java.time.Instant;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+import software.amazon.lambda.durable.plugin.InvocationStatus;
+
+class ExecutionRootParityTest {
+ private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/parity/id";
+ private static final Instant START = Instant.ofEpochSecond(10);
+ private static final AttributeKey SAMPLER_VALUE = AttributeKey.stringKey("sampler.invocation");
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void sampledAnchorEndsBeforeTheStartHookReturns(boolean executionView) {
+ try (var exporter = InMemorySpanExporter.create()) {
+ var plugin = plugin(executionView, exporter, Sampler.alwaysOn());
+ plugin.onInvocationStart(start(true));
+ var roots = roots(exporter);
+ assertEquals(1, roots.size(), "The anchor is available to processors before handler execution");
+ assertEquals(START.getEpochSecond() * 1_000_000_000L, roots.get(0).getStartEpochNanos());
+ assertEquals(roots.get(0).getStartEpochNanos(), roots.get(0).getEndEpochNanos());
+ plugin.onInvocationEnd(end(true));
+ assertEquals(1, roots(exporter).size(), "End does not export the same anchor twice");
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void anchorReexportsExcludeChangingSamplerMetadataButChildrenKeepIt(boolean executionView) {
+ var evaluations = new AtomicInteger();
+ var sampler = sampler((name) -> {
+ var value = Integer.toString(evaluations.incrementAndGet());
+ return new SamplingResult() {
+ public SamplingDecision getDecision() {
+ return SamplingDecision.RECORD_AND_SAMPLE;
+ }
+
+ public Attributes getAttributes() {
+ return Attributes.of(SAMPLER_VALUE, value);
+ }
+
+ public TraceState getUpdatedTraceState(TraceState parent) {
+ return TraceState.builder().put("vendor", value).build();
+ }
+ };
+ });
+ try (var exporter = InMemorySpanExporter.create()) {
+ var plugin = plugin(executionView, exporter, sampler);
+ for (var first : new boolean[] {true, false}) {
+ plugin.onInvocationStart(start(first));
+ plugin.onInvocationEnd(end(first));
+ }
+ var roots = roots(exporter);
+ assertEquals(2, roots.size());
+ assertEquals(2, evaluations.get(), "The anchor must reuse the invocation's sampling result");
+ for (var root : roots) {
+ assertNull(root.getAttributes().get(SAMPLER_VALUE));
+ assertEquals(
+ Attributes.of(DURABLE_EXECUTION_ARN, ARN, DURABLE_EXECUTION_SYNTHETIC_ROOT, true)
+ .asMap(),
+ root.getAttributes().asMap());
+ assertEquals(TraceState.getDefault(), root.getSpanContext().getTraceState());
+ assertEquals(roots.get(0).getSpanContext(), root.getSpanContext());
+ }
+ var invocations = exporter.getFinishedSpanItems().stream()
+ .filter(s -> s.getName().equals("Invocation"))
+ .toList();
+ assertEquals(2, invocations.size());
+ for (var i = 0; i < invocations.size(); i++) {
+ var value = Integer.toString(i + 1);
+ assertEquals(value, invocations.get(i).getAttributes().get(SAMPLER_VALUE));
+ assertEquals(
+ value,
+ invocations.get(i).getSpanContext().getTraceState().get("vendor"));
+ assertEquals(roots.get(i).getSpanContext(), invocations.get(i).getParentSpanContext());
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void recordOnlyStillRecordsChildrenWithoutAnAnchor(boolean executionView) {
+ try (var exporter = InMemorySpanExporter.create()) {
+ var plugin = plugin(executionView, exporter, sampler(name -> SamplingResult.recordOnly()));
+ plugin.onInvocationStart(start(true));
+ plugin.onInvocationEnd(end(false));
+ assertTrue(roots(exporter).isEmpty());
+ var spans = exporter.getFinishedSpanItems();
+ assertEquals(2, spans.size());
+ assertTrue(spans.stream().allMatch(s -> !s.getSpanContext().isSampled()));
+ assertTrue(spans.stream().anyMatch(s -> s.getName().equals("Invocation")));
+ assertTrue(spans.stream().anyMatch(s -> s.getName().equals("Workflow")));
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({"false,false", "false,true", "true,false", "true,true"})
+ void plainReplacementRetainsIndependentPerSpanSampling(boolean executionView, boolean sampleAnchor) {
+ GlobalOpenTelemetry.resetForTest();
+ OtelPluginAutoConfigurationState.markInstalled();
+ try (var exporter = InMemorySpanExporter.create();
+ var provider = SdkTracerProvider.builder()
+ .setIdGenerator(new DeterministicIdGenerator())
+ .setSampler(sampler(name -> name.equals("DurableExecutionRoot") == sampleAnchor
+ ? SamplingResult.recordAndSample()
+ : SamplingResult.drop()))
+ .addSpanProcessor(SimpleSpanProcessor.create(exporter))
+ .build()) {
+ GlobalOpenTelemetry.set(
+ OpenTelemetrySdk.builder().setTracerProvider(provider).build());
+ DurableExecutionPlugin plugin =
+ executionView ? new ExecutionOtelPlugin(config()) : new InvocationOtelPlugin(config());
+ plugin.onInvocationStart(start(true));
+ plugin.onInvocationEnd(end(false));
+ assertEquals(sampleAnchor ? 1 : 0, roots(exporter).size());
+ assertEquals(
+ sampleAnchor ? 0 : 1,
+ exporter.getFinishedSpanItems().stream()
+ .filter(s -> s.getName().equals("Invocation"))
+ .count());
+ } finally {
+ GlobalOpenTelemetry.resetForTest();
+ OtelPluginAutoConfigurationState.resetInstalledForTest();
+ }
+ }
+
+ private static DurableExecutionPlugin plugin(
+ boolean executionView, InMemorySpanExporter exporter, Sampler sampler) {
+ var builder = SdkTracerProvider.builder()
+ .setSampler(sampler)
+ .addSpanProcessor(SimpleSpanProcessor.builder(exporter)
+ .setExportUnsampledSpans(true)
+ .build());
+ return executionView ? new ExecutionOtelPlugin(builder, config()) : new InvocationOtelPlugin(builder, config());
+ }
+
+ private static OtelPluginConfig config() {
+ return OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(() -> null)
+ .build();
+ }
+
+ private static InvocationInfo start(boolean first) {
+ return new InvocationInfo("request", ARN, first, START);
+ }
+
+ private static InvocationEndInfo end(boolean first) {
+ return new InvocationEndInfo(
+ "request", ARN, first, first ? InvocationStatus.PENDING : InvocationStatus.SUCCEEDED, null);
+ }
+
+ private static List roots(InMemorySpanExporter exporter) {
+ return exporter.getFinishedSpanItems().stream()
+ .filter(s -> s.getName().equals("DurableExecutionRoot"))
+ .toList();
+ }
+
+ private static Sampler sampler(Function decision) {
+ return new Sampler() {
+ public SamplingResult shouldSample(
+ Context parent,
+ String traceId,
+ String name,
+ SpanKind kind,
+ Attributes attributes,
+ List links) {
+ return decision.apply(name);
+ }
+
+ public String getDescription() {
+ return "root-parity";
+ }
+ };
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java
new file mode 100644
index 000000000..2a00910d5
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java
@@ -0,0 +1,359 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_ARN;
+import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_STATUS;
+import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_SYNTHETIC_ROOT;
+
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.api.trace.StatusCode;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.common.CompletableResultCode;
+import io.opentelemetry.sdk.resources.Resource;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.ReadWriteSpan;
+import io.opentelemetry.sdk.trace.ReadableSpan;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder;
+import io.opentelemetry.sdk.trace.SpanProcessor;
+import io.opentelemetry.sdk.trace.data.SpanData;
+import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import io.opentelemetry.sdk.trace.export.SpanExporter;
+import io.opentelemetry.sdk.trace.samplers.Sampler;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+import software.amazon.lambda.durable.plugin.InvocationStatus;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+
+class ExecutionRootTest {
+ private static final Instant START = Instant.parse("2026-10-02T00:00:00Z");
+ private static final String TRACE = "6955b900123456789012345678901234";
+ private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id";
+
+ @ParameterizedTest
+ @CsvSource({"true,true", "true,false", "false,true", "false,false"})
+ void anchorIsExportedOnFirstPendingInvocationThenConnectsTerminalWorkflow(boolean executionView, boolean success) {
+ var exporter = InMemorySpanExporter.create();
+ var plugin = plugin(executionView, exporter, null, Sampler.alwaysOn());
+ var effects = new AtomicInteger();
+ var runner = LocalDurableTestRunner.create(
+ String.class,
+ (input, ctx) -> {
+ ctx.step("once", Integer.class, stepCtx -> effects.incrementAndGet());
+ ctx.wait("pause", Duration.ofSeconds(10));
+ if (!success) throw new IllegalArgumentException("expected");
+ return input;
+ },
+ DurableConfig.builder().withPlugins(plugin).build());
+ assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus());
+ var first = exporter.getFinishedSpanItems();
+ var anchor = named(first, "DurableExecutionRoot");
+ assertEquals(true, anchor.getAttributes().get(DURABLE_EXECUTION_SYNTHETIC_ROOT));
+ assertFalse(anchor.getParentSpanContext().isValid());
+ assertEquals(anchor.getStartEpochNanos(), anchor.getEndEpochNanos());
+ assertEquals(StatusCode.UNSET, anchor.getStatus().getStatusCode());
+ assertNull(anchor.getAttributes().get(DURABLE_EXECUTION_STATUS));
+ assertTrue(first.stream().noneMatch(span -> span.getName().equals("Workflow")));
+ assertEquals(anchor.getSpanId(), named(first, "Invocation").getParentSpanId());
+ runner.advanceTime();
+ assertEquals(
+ success ? ExecutionStatus.SUCCEEDED : ExecutionStatus.FAILED,
+ runner.runUntilComplete("input").getStatus());
+ assertEquals(1, effects.get());
+ var all = exporter.getFinishedSpanItems();
+ assertEquals(anchor.getSpanId(), named(all, "Workflow").getParentSpanId());
+ var anchors = all.stream()
+ .filter(span -> span.getName().equals("DurableExecutionRoot"))
+ .toList();
+ assertEquals(2, anchors.size());
+ assertStable(anchor, anchors.get(1));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void fallbackRecoveryKeepsIdentityAndRemoteParentsRemainExternal(boolean executionView) {
+ var cases = new ExtractedContext[] {
+ null,
+ new ExtractedContext("invalid", "bad", ExtractedContext.Sampling.UNDECIDED),
+ new ExtractedContext(TRACE, null, ExtractedContext.Sampling.SAMPLED),
+ new ExtractedContext(TRACE, "0000000000000000", ExtractedContext.Sampling.SAMPLED)
+ };
+ for (var extracted : cases) {
+ var exporter = InMemorySpanExporter.create();
+ var plugin = plugin(executionView, exporter, extracted, Sampler.alwaysOn());
+ invoke(plugin, ARN, true, InvocationStatus.RETRYING);
+ var root = named(exporter.getFinishedSpanItems(), "DurableExecutionRoot");
+ invoke(plugin, ARN, true, InvocationStatus.PENDING); // redelivery after a failed invocation
+ invoke(plugin, ARN, false, InvocationStatus.SUCCEEDED);
+ var roots = exporter.getFinishedSpanItems().stream()
+ .filter(span -> span.getName().equals("DurableExecutionRoot"))
+ .toList();
+ assertEquals(3, roots.size());
+ roots.forEach(span -> assertStable(root, span));
+ assertEquals(new DeterministicIdGenerator().generateExecutionRootSpanId(ARN), root.getSpanId());
+ assertEquals(START.getEpochSecond() * 1_000_000_000L, root.getStartEpochNanos());
+ if (extracted != null && extracted.hasValidTraceId()) assertEquals(TRACE, root.getTraceId());
+ }
+ var exporter = InMemorySpanExporter.create();
+ var plugin = plugin(
+ executionView,
+ exporter,
+ new ExtractedContext(TRACE, "1234567890123456", ExtractedContext.Sampling.SAMPLED),
+ Sampler.alwaysOn());
+ invoke(plugin, ARN, true, InvocationStatus.SUCCEEDED);
+ assertEquals(2, exporter.getFinishedSpanItems().size());
+ assertTrue(exporter.getFinishedSpanItems().stream()
+ .allMatch(span -> span.getParentSpanId().equals("1234567890123456")));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void anchorsHonorSamplingAndAreOwnedByExecutionOnSharedTrace(boolean executionView) {
+ for (var extracted :
+ new ExtractedContext[] {null, new ExtractedContext(TRACE, null, ExtractedContext.Sampling.NOT_SAMPLED)
+ }) {
+ var exporter = InMemorySpanExporter.create();
+ var plugin = plugin(
+ executionView, exporter, extracted, extracted == null ? Sampler.alwaysOff() : Sampler.alwaysOn());
+ invoke(plugin, ARN, true, InvocationStatus.PENDING);
+ invoke(plugin, ARN, false, InvocationStatus.SUCCEEDED);
+ assertTrue(exporter.getFinishedSpanItems().isEmpty());
+ }
+ var exporter = InMemorySpanExporter.create();
+ var extracted = new ExtractedContext(TRACE, null, ExtractedContext.Sampling.SAMPLED);
+ for (var service : List.of("caller", "callee")) {
+ var builder = SdkTracerProvider.builder()
+ .setResource(Resource.create(Attributes.of(AttributeKey.stringKey("service.name"), service)))
+ .setSampler(Sampler.alwaysOff())
+ .addSpanProcessor(SimpleSpanProcessor.create(exporter));
+ var plugin = plugin(executionView, builder, extracted);
+ var start = service.equals("caller") ? START : START.plusSeconds(10);
+ invoke(plugin, ARN + "-" + service, true, InvocationStatus.SUCCEEDED, start);
+ }
+ var roots = exporter.getFinishedSpanItems().stream()
+ .filter(span -> span.getName().equals("DurableExecutionRoot"))
+ .toList();
+ assertEquals(2, roots.size());
+ assertTrue(roots.stream().allMatch(span -> span.getTraceId().equals(TRACE)));
+ assertNotEquals(roots.get(0).getSpanId(), roots.get(1).getSpanId());
+ assertNotEquals(
+ roots.get(0).getAttributes().get(DURABLE_EXECUTION_ARN),
+ roots.get(1).getAttributes().get(DURABLE_EXECUTION_ARN));
+ assertNotEquals(roots.get(0).getStartEpochNanos(), roots.get(1).getStartEpochNanos());
+ assertNotEquals(roots.get(0).getResource(), roots.get(1).getResource());
+ for (var root : roots) {
+ var workflow = exporter.getFinishedSpanItems().stream()
+ .filter(span -> span.getName().equals("Workflow"))
+ .filter(span -> span.getAttributes()
+ .get(DURABLE_EXECUTION_ARN)
+ .equals(root.getAttributes().get(DURABLE_EXECUTION_ARN)))
+ .findFirst()
+ .orElseThrow();
+ assertEquals(root.getSpanContext(), workflow.getParentSpanContext());
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({"true,true", "true,false", "false,true", "false,false"})
+ void failedFirstFlushIsRecoveredInAnotherEnvironment(boolean executionView, boolean success) {
+ var first = failedSuspendedAnchor(executionView);
+ var exporter = InMemorySpanExporter.create();
+ var builder = SdkTracerProvider.builder()
+ .setResource(environmentResource("resumed"))
+ .setSampler(Sampler.alwaysOn())
+ .addSpanProcessor(SimpleSpanProcessor.create(exporter));
+ invoke(
+ plugin(executionView, builder, null),
+ ARN,
+ false,
+ success ? InvocationStatus.SUCCEEDED : InvocationStatus.FAILED);
+ var recovered = named(exporter.getFinishedSpanItems(), "DurableExecutionRoot");
+ assertStableSpanFields(first, recovered);
+ assertNotEquals(first.getResource(), recovered.getResource());
+ assertEquals(
+ recovered.getSpanContext(),
+ named(exporter.getFinishedSpanItems(), "Workflow").getParentSpanContext());
+ }
+
+ private static SpanData failedSuspendedAnchor(boolean executionView) {
+ var attempted = new ArrayList();
+ try (var processor = BatchSpanProcessor.builder(failingExporter(attempted))
+ .setScheduleDelay(Duration.ofDays(1))
+ .build()) {
+ var builder = SdkTracerProvider.builder()
+ .setResource(environmentResource("first"))
+ .setSampler(Sampler.alwaysOn())
+ .addSpanProcessor(processor);
+ invoke(plugin(executionView, builder, null), ARN, true, InvocationStatus.PENDING);
+ // The end hook's forceFlush attempts the anchor export before returning, even without a terminal hook.
+ assertTrue(attempted.stream().noneMatch(span -> span.getName().equals("Workflow")));
+ assertEquals(
+ START.getEpochSecond() * 1_000_000_000L,
+ named(attempted, "DurableExecutionRoot").getStartEpochNanos());
+ }
+ return named(attempted, "DurableExecutionRoot");
+ }
+
+ private static SpanExporter failingExporter(List attempted) {
+ return new SpanExporter() {
+ @Override
+ public CompletableResultCode export(Collection spans) {
+ attempted.addAll(spans);
+ return CompletableResultCode.ofFailure();
+ }
+
+ @Override
+ public CompletableResultCode flush() {
+ return CompletableResultCode.ofFailure();
+ }
+
+ @Override
+ public CompletableResultCode shutdown() {
+ return CompletableResultCode.ofSuccess();
+ }
+ };
+ }
+
+ private static Resource environmentResource(String instance) {
+ return Resource.create(Attributes.of(AttributeKey.stringKey("faas.instance"), instance));
+ }
+
+ @ParameterizedTest
+ @CsvSource({"true,true", "true,false", "false,true", "false,false"})
+ void processorSpansDoNotInheritDurableSamplingOverride(boolean executionView, boolean observeStart) {
+ var exporter = InMemorySpanExporter.create();
+ var observed = new AtomicReference();
+ try (var unrelated = SdkTracerProvider.builder()
+ .setSampler(DurableSampler.wrap(Sampler.alwaysOff()))
+ .build()) {
+ var builder = SdkTracerProvider.builder()
+ .setSampler(Sampler.alwaysOff())
+ .addSpanProcessor(new SamplingObserver(unrelated, observed, observeStart))
+ .addSpanProcessor(SimpleSpanProcessor.create(exporter));
+ var config = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(() -> new ExtractedContext(TRACE, null, ExtractedContext.Sampling.SAMPLED))
+ .build();
+ DurableExecutionPlugin plugin = executionView
+ ? new ExecutionOtelPlugin(builder, config)
+ : new InvocationOtelPlugin(builder, config);
+ invoke(plugin, ARN, true, InvocationStatus.PENDING);
+ assertTrue(named(exporter.getFinishedSpanItems(), "DurableExecutionRoot")
+ .getSpanContext()
+ .isSampled());
+ assertNotNull(observed.get(), "Root processor callback must have run");
+ assertFalse(observed.get().isSampled(), "Processor's unrelated span must retain its always-off policy");
+ }
+ }
+
+ private record SamplingObserver(
+ SdkTracerProvider provider, AtomicReference observed, boolean observeStart)
+ implements SpanProcessor {
+ @Override
+ public void onStart(Context parent, ReadWriteSpan span) {
+ if (observeStart) {
+ observe(span);
+ }
+ }
+
+ @Override
+ public boolean isStartRequired() {
+ return observeStart;
+ }
+
+ @Override
+ public void onEnd(ReadableSpan span) {
+ if (!observeStart) {
+ observe(span);
+ }
+ }
+
+ private void observe(ReadableSpan span) {
+ if (span.getName().equals("DurableExecutionRoot")) {
+ var callback =
+ provider.get("processor").spanBuilder("unrelated").startSpan();
+ observed.set(callback.getSpanContext());
+ callback.end();
+ }
+ }
+
+ @Override
+ public boolean isEndRequired() {
+ return !observeStart;
+ }
+ }
+
+ private static void assertStable(SpanData first, SpanData replay) {
+ assertStableSpanFields(first, replay);
+ assertEquals(first.getResource(), replay.getResource());
+ }
+
+ private static void assertStableSpanFields(SpanData first, SpanData replay) {
+ assertEquals(first.getSpanContext(), replay.getSpanContext());
+ assertEquals(first.getParentSpanContext(), replay.getParentSpanContext());
+ assertEquals(first.getName(), replay.getName());
+ assertEquals(first.getKind(), replay.getKind());
+ assertEquals(first.getStartEpochNanos(), replay.getStartEpochNanos());
+ assertEquals(first.getEndEpochNanos(), replay.getEndEpochNanos());
+ assertEquals(first.getAttributes(), replay.getAttributes());
+ assertEquals(first.getStatus(), replay.getStatus());
+ assertEquals(first.getEvents(), replay.getEvents());
+ assertEquals(first.getLinks(), replay.getLinks());
+ assertEquals(first.hasEnded(), replay.hasEnded());
+ assertEquals(first.getTotalAttributeCount(), replay.getTotalAttributeCount());
+ assertEquals(first.getTotalRecordedEvents(), replay.getTotalRecordedEvents());
+ assertEquals(first.getTotalRecordedLinks(), replay.getTotalRecordedLinks());
+ assertEquals(first.getInstrumentationScopeInfo(), replay.getInstrumentationScopeInfo());
+ }
+
+ private static SpanData named(List spans, String name) {
+ return spans.stream()
+ .filter(span -> span.getName().equals(name))
+ .findFirst()
+ .orElseThrow();
+ }
+
+ private static void invoke(DurableExecutionPlugin plugin, String arn, boolean first, InvocationStatus status) {
+ invoke(plugin, arn, first, status, START);
+ }
+
+ private static void invoke(
+ DurableExecutionPlugin plugin, String arn, boolean first, InvocationStatus status, Instant start) {
+ plugin.onInvocationStart(new InvocationInfo("request", arn, first, start));
+ plugin.onInvocationEnd(new InvocationEndInfo("request", arn, first, status, null));
+ }
+
+ private static DurableExecutionPlugin plugin(
+ boolean executionView, InMemorySpanExporter exporter, ExtractedContext extracted, Sampler sampler) {
+ var builder =
+ SdkTracerProvider.builder().setSampler(sampler).addSpanProcessor(SimpleSpanProcessor.create(exporter));
+ return plugin(executionView, builder, extracted);
+ }
+
+ private static DurableExecutionPlugin plugin(
+ boolean executionView, SdkTracerProviderBuilder builder, ExtractedContext extracted) {
+ var config = OtelPluginConfig.builder()
+ .contextExtractor(() -> extracted)
+ .enableMdc(false)
+ .build();
+ return executionView ? new ExecutionOtelPlugin(builder, config) : new InvocationOtelPlugin(builder, config);
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndAdmissionTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndAdmissionTest.java
new file mode 100644
index 000000000..9781b334f
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndAdmissionTest.java
@@ -0,0 +1,262 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import java.util.function.Function;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.awssdk.services.lambda.model.*;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.DurableContext;
+import software.amazon.lambda.durable.DurableFuture;
+import software.amazon.lambda.durable.TypeToken;
+import software.amazon.lambda.durable.config.CompletionConfig;
+import software.amazon.lambda.durable.config.ParallelConfig;
+import software.amazon.lambda.durable.config.WaitForConditionConfig;
+import software.amazon.lambda.durable.execution.DurableExecutor;
+import software.amazon.lambda.durable.model.DurableExecutionInput;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.model.WaitForConditionResult;
+import software.amazon.lambda.durable.plugin.*;
+import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient;
+
+class InvocationEndAdmissionTest {
+ @ParameterizedTest
+ @ValueSource(strings = {"unawaited", "awaited", "early-parallel", "already-running"})
+ void alreadyAdmittedWorkersRetainTheBaseCleanupBoundary(String mode) throws Exception {
+ boolean awaitResult = mode.equals("awaited");
+ boolean alreadyRunning = mode.equals("already-running");
+ var runningEntered = new CountDownLatch(1);
+ var resumedPredicate = new CountDownLatch(1);
+ var releaseRunning = new CountDownLatch(1);
+ var nextDispatch = new CountDownLatch(1);
+ var releaseDispatch = new CountDownLatch(1);
+ var endEntered = new CountDownLatch(1);
+ var releaseEnd = new CountDownLatch(1);
+ var terminal = new CountDownLatch(1);
+ var predicates = new AtomicInteger();
+ var lateHooks = new AtomicInteger();
+ var lateStarts = new AtomicInteger();
+ var ending = new AtomicBoolean();
+ var snapshotStatus = new AtomicReference();
+ var firstResume = new AtomicBoolean(true);
+ var workers =
+ new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue()) {
+ @Override
+ public void execute(Runnable task) {
+ if (Thread.currentThread().getName().startsWith("durable-sdk-internal-")
+ && firstResume.compareAndSet(true, false)) {
+ nextDispatch.countDown();
+ await(releaseDispatch);
+ }
+ super.execute(task);
+ }
+ };
+ var client = new LocalMemoryExecutionClient() {
+ @Override
+ public CheckpointDurableExecutionResponse checkpoint(
+ String arn, String token, List updates) {
+ var applied = super.checkpoint(arn, token, updates);
+ advanceTime();
+ var ready = super.checkpoint(arn, applied.checkpointToken(), List.of());
+ var states = new LinkedHashMap();
+ applied.newExecutionState().operations().forEach(op -> states.put(op.id(), op));
+ ready.newExecutionState().operations().forEach(op -> states.put(op.id(), op));
+ if (getOperationByName("condition") != null
+ && getOperationByName("condition").status() == OperationStatus.SUCCEEDED) terminal.countDown();
+ return ready.toBuilder()
+ .newExecutionState(CheckpointUpdatedExecutionState.builder()
+ .operations(states.values())
+ .build())
+ .build();
+ }
+ };
+ var exporter = InMemorySpanExporter.create();
+ var otel = new InvocationOtelPlugin(
+ SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)),
+ OtelPluginConfig.builder()
+ .contextExtractor(() -> null)
+ .enableMdc(false)
+ .build());
+ var gate = new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo info) {
+ snapshotStatus.set(info.operations().values().stream()
+ .filter(op -> "condition".equals(op.name()))
+ .findFirst()
+ .orElseThrow()
+ .status());
+ ending.set(true);
+ endEntered.countDown();
+ await(releaseEnd);
+ }
+
+ public void onUserFunctionStart(UserFunctionStartInfo info) {
+ if (ending.get()) {
+ lateHooks.incrementAndGet();
+ lateStarts.incrementAndGet();
+ }
+ }
+
+ public void onUserFunctionEnd(UserFunctionEndInfo info) {
+ if (ending.get()) lateHooks.incrementAndGet();
+ }
+
+ public void onOperationEnd(OperationEndInfo info) {
+ if (ending.get()) lateHooks.incrementAndGet();
+ }
+ };
+ // The merged lifecycle unwinds End in reverse order, so OTel must be registered after the gate.
+ // This fixture parks inside Executor.execute AFTER SDK admission. The remaining worker-start boundary
+ // is tracked in #787; new registration during preparation is covered separately without weakening that gate.
+ var config = DurableConfig.builder()
+ .withExecutorService(workers)
+ .withDurableExecutionClient(client)
+ .withCheckpointDelay(Duration.ZERO)
+ .withPollingStrategy(attempt -> Duration.ZERO)
+ .withPlugins(gate, otel)
+ .build();
+ var caller = Executors.newSingleThreadExecutor();
+ try {
+ var call = caller.submit(() -> DurableExecutor.execute(
+ input(),
+ null,
+ TypeToken.get(String.class),
+ (value, context) -> {
+ Function> startCondition =
+ branch -> branch.waitForConditionAsync(
+ "condition",
+ Integer.class,
+ (state, step) -> {
+ predicates.incrementAndGet();
+ if (state == 2) resumedPredicate.countDown();
+ if (alreadyRunning && state == 2) {
+ runningEntered.countDown();
+ await(releaseRunning);
+ }
+ return state == 1
+ ? WaitForConditionResult.continuePolling(2)
+ : WaitForConditionResult.stopPolling(state);
+ },
+ WaitForConditionConfig.builder()
+ .initialState(1)
+ .waitStrategy((state, attempt) -> Duration.ofSeconds(1))
+ .build());
+ if (mode.equals("early-parallel")) {
+ var parallel = context.parallel(
+ "early",
+ ParallelConfig.builder()
+ .completionConfig(CompletionConfig.minSuccessful(1))
+ .build());
+ try (parallel) {
+ parallel.branch(
+ "waiting",
+ Integer.class,
+ branch -> startCondition.apply(branch).get());
+ parallel.branch("winner", String.class, branch -> {
+ await(nextDispatch);
+ return "winner";
+ });
+ }
+ assertEquals(1, parallel.get().succeeded());
+ } else {
+ var condition = startCondition.apply(context);
+ await(nextDispatch);
+ if (alreadyRunning) await(runningEntered);
+ if (awaitResult) condition.get();
+ }
+ return "root-done";
+ },
+ config));
+ assertTrue(nextDispatch.await(3, TimeUnit.SECONDS));
+ if (alreadyRunning) {
+ releaseDispatch.countDown();
+ assertTrue(runningEntered.await(3, TimeUnit.SECONDS));
+ assertTrue(endEntered.await(3, TimeUnit.SECONDS), "End does not join already-running predicates");
+ releaseEnd.countDown();
+ assertThrows(
+ TimeoutException.class,
+ () -> call.get(100, TimeUnit.MILLISECONDS),
+ "Normal close still waits for the real handler");
+ releaseRunning.countDown();
+ } else if (!awaitResult) assertTrue(endEntered.await(3, TimeUnit.SECONDS));
+ releaseDispatch.countDown();
+ if (!awaitResult && !alreadyRunning) assertTrue(resumedPredicate.await(3, TimeUnit.SECONDS));
+ assertTrue(terminal.await(3, TimeUnit.SECONDS));
+ assertTrue(endEntered.await(3, TimeUnit.SECONDS));
+ releaseEnd.countDown();
+ assertEquals(
+ ExecutionStatus.SUCCEEDED, call.get(3, TimeUnit.SECONDS).status());
+ var spans = exporter.getFinishedSpanItems().stream()
+ .filter(span -> span.getName().equals("condition"))
+ .map(span ->
+ span.getName() + ":" + span.getAttributes().get(SpanAttributes.DURABLE_OPERATION_STATUS))
+ .toList();
+ System.out.println("END_SNAPSHOT_PUBLIC mode=" + mode + " snapshot=" + snapshotStatus.get()
+ + " backend=" + client.getOperationByName("condition").status() + " predicates=" + predicates.get()
+ + " lateHooks=" + lateHooks.get() + " lateStarts=" + lateStarts.get() + " operationSpans=" + spans);
+ assertEquals(
+ awaitResult || alreadyRunning ? 0 : 1,
+ lateStarts.get(),
+ "Already-admitted workers may start after End; this is not a new-registration guarantee");
+ assertEquals(2, predicates.get());
+ if (alreadyRunning) {
+ assertNotEquals(
+ OperationStatus.SUCCEEDED, snapshotStatus.get(), "The existing snapshot boundary is retained");
+ assertEquals(
+ OperationStatus.SUCCEEDED,
+ client.getOperationByName("condition").status());
+ } else {
+ assertEquals(awaitResult ? OperationStatus.SUCCEEDED : OperationStatus.READY, snapshotStatus.get());
+ assertEquals(
+ OperationStatus.SUCCEEDED,
+ client.getOperationByName("condition").status());
+ }
+ } finally {
+ releaseRunning.countDown();
+ releaseDispatch.countDown();
+ releaseEnd.countDown();
+ caller.shutdownNow();
+ workers.shutdownNow();
+ assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ exporter.close();
+ }
+ }
+
+ private static void await(CountDownLatch latch) {
+ try {
+ assertTrue(latch.await(5, TimeUnit.SECONDS));
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(interrupted);
+ }
+ }
+
+ private static DurableExecutionInput input() {
+ return new DurableExecutionInput(
+ "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/end-boundary/execution",
+ "token",
+ CheckpointUpdatedExecutionState.builder()
+ .operations(Operation.builder()
+ .id("execution")
+ .type(OperationType.EXECUTION)
+ .status(OperationStatus.STARTED)
+ .startTimestamp(Instant.EPOCH)
+ .executionDetails(ExecutionDetails.builder()
+ .inputPayload("\"input\"")
+ .build())
+ .build())
+ .build());
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationLateFlushProbeTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationLateFlushProbeTest.java
new file mode 100644
index 000000000..108756b60
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationLateFlushProbeTest.java
@@ -0,0 +1,162 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.awssdk.services.lambda.model.*;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.TypeToken;
+import software.amazon.lambda.durable.config.CompletionConfig;
+import software.amazon.lambda.durable.config.ParallelConfig;
+import software.amazon.lambda.durable.execution.DurableExecutor;
+import software.amazon.lambda.durable.model.DurableExecutionInput;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.*;
+import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient;
+
+/** Diagnostic for the pre-existing End/close flush boundary; no proposed production policy in this class. */
+class InvocationLateFlushProbeTest {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void alreadyRunningStepCompletesAfterEnd(boolean earlyParallel) throws Exception {
+ var running = new CountDownLatch(1);
+ var release = new CountDownLatch(1);
+ var ended = new CountDownLatch(1);
+ var lateOperationEnds = new AtomicInteger();
+ var bodyCalls = new AtomicInteger();
+ var client = new LocalMemoryExecutionClient();
+ var workers = Executors.newCachedThreadPool();
+ var caller = Executors.newSingleThreadExecutor();
+ try (var exporter = InMemorySpanExporter.create();
+ var batch = BatchSpanProcessor.builder(exporter)
+ .setScheduleDelay(Duration.ofDays(1))
+ .build()) {
+ var otel = new InvocationOtelPlugin(
+ SdkTracerProvider.builder().addSpanProcessor(batch),
+ OtelPluginConfig.builder()
+ .contextExtractor(() -> null)
+ .enableMdc(false)
+ .build());
+ var config = DurableConfig.builder()
+ .withDurableExecutionClient(client)
+ .withExecutorService(workers)
+ .withCheckpointDelay(Duration.ZERO)
+ .withPlugins(otel, new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo info) {
+ ended.countDown();
+ }
+
+ public void onOperationEnd(OperationEndInfo info) {
+ if (ended.getCount() == 0) lateOperationEnds.incrementAndGet();
+ }
+ })
+ .build();
+ try {
+ var call = caller.submit(() -> DurableExecutor.execute(
+ input(),
+ null,
+ TypeToken.get(String.class),
+ (value, root) -> {
+ if (earlyParallel) {
+ var parallel = root.parallel(
+ "early",
+ ParallelConfig.builder()
+ .completionConfig(CompletionConfig.minSuccessful(1))
+ .build());
+ try (parallel) {
+ parallel.branch(
+ "waiting",
+ String.class,
+ child -> child.step("held", String.class, step -> {
+ bodyCalls.incrementAndGet();
+ running.countDown();
+ await(release);
+ return "completed";
+ }));
+ parallel.branch("winner", String.class, child -> {
+ await(running);
+ return "winner";
+ });
+ }
+ assertEquals(1, parallel.get().succeeded());
+ } else {
+ root.stepAsync("held", String.class, step -> {
+ bodyCalls.incrementAndGet();
+ running.countDown();
+ await(release);
+ return "completed";
+ });
+ await(running);
+ }
+ return "root-success";
+ },
+ config));
+ assertTrue(ended.await(5, TimeUnit.SECONDS));
+ assertThrows(TimeoutException.class, () -> call.get(100, TimeUnit.MILLISECONDS));
+ release.countDown();
+ assertEquals(
+ ExecutionStatus.SUCCEEDED, call.get(5, TimeUnit.SECONDS).status());
+ assertEquals(
+ OperationStatus.SUCCEEDED,
+ client.getOperationByName("held").status());
+ assertEquals(1, bodyCalls.get());
+ var before = exporter.getFinishedSpanItems().stream()
+ .filter(x -> x.getName().equals("held"))
+ .map(x -> x.getAttributes().get(SpanAttributes.DURABLE_OPERATION_STATUS))
+ .toList();
+ batch.forceFlush().join(3, TimeUnit.SECONDS);
+ var after = exporter.getFinishedSpanItems().stream()
+ .filter(x -> x.getName().equals("held"))
+ .map(x -> x.getAttributes().get(SpanAttributes.DURABLE_OPERATION_STATUS))
+ .toList();
+ System.out.println("LATE_FLUSH_PUBLIC earlyParallel=" + earlyParallel + " backend=SUCCEEDED bodyCalls="
+ + bodyCalls.get() + " lateOperationEnds=" + lateOperationEnds.get() + " beforeManualFlush="
+ + before + " afterManualFlush=" + after);
+ assertTrue(lateOperationEnds.get() > 0, "Existing public operation-end occurs after invocation End");
+ // This diagnostic records actual export behavior; it does not redefine which outcome wins.
+ } finally {
+ release.countDown();
+ caller.shutdownNow();
+ workers.shutdownNow();
+ assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+ }
+
+ private static void await(CountDownLatch latch) {
+ try {
+ assertTrue(latch.await(8, TimeUnit.SECONDS));
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(failure);
+ }
+ }
+
+ private static DurableExecutionInput input() {
+ return new DurableExecutionInput(
+ "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/late-flush/execution",
+ "token",
+ CheckpointUpdatedExecutionState.builder()
+ .operations(Operation.builder()
+ .id("execution")
+ .type(OperationType.EXECUTION)
+ .status(OperationStatus.STARTED)
+ .startTimestamp(Instant.EPOCH)
+ .executionDetails(ExecutionDetails.builder()
+ .inputPayload("\"input\"")
+ .build())
+ .build())
+ .build());
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java
index 79926ac69..aad4cbe74 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java
@@ -121,7 +121,7 @@ void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() {
new InvocationEndInfo("req-enabled", "arn:enabled", true, InvocationStatus.SUCCEEDED, null));
var spans = globalExporter.getFinishedSpanItems();
- assertEquals(3, spans.size());
+ assertEquals(4, spans.size());
assertTrue(spans.stream().anyMatch(span -> span.getName().equals("enabled-step")));
assertFalse(spans.stream().anyMatch(span -> span.getName().equals("disabled-step")));
}
@@ -159,7 +159,7 @@ void defaultConstructor_usesGlobalSdkTracerProviderDirectly() {
var spans = globalExporter.getFinishedSpanItems();
// Plugin creates Workflow + Invocation + operation spans
- assertEquals(3, spans.size());
+ assertEquals(4, spans.size());
assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step")));
}
@@ -208,7 +208,7 @@ public ContextPropagators getPropagators() {
var spans = globalExporter.getFinishedSpanItems();
// Plugin creates Workflow + Invocation + operation spans
- assertEquals(3, spans.size());
+ assertEquals(4, spans.size());
assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step")));
var expectedIds = new DeterministicIdGenerator();
expectedIds.setDurableExecutionArn("arn:exec1");
@@ -330,9 +330,9 @@ void invocationStart_and_end_createsSpan() {
null));
var spans = spanExporter.getFinishedSpanItems();
- assertEquals(2, spans.size()); // invocation + Workflow
+ assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot
- var span = spans.get(0);
+ var span = spanByName("Invocation");
assertEquals("Invocation", span.getName());
assertEquals(StatusCode.OK, span.getStatus().getStatusCode());
}
@@ -418,7 +418,7 @@ void invocationSpan_hasInternalKind() {
plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
- var span = spanExporter.getFinishedSpanItems().get(0);
+ var span = spanByName("Invocation");
assertEquals(SpanKind.INTERNAL, span.getKind(), "Invocation span must be INTERNAL kind");
}
@@ -582,8 +582,8 @@ void invocationEnd_withFailure_setsErrorStatus() {
"req-123", "arn:exec1", true, InvocationStatus.FAILED, new RuntimeException("boom")));
var spans = spanExporter.getFinishedSpanItems();
- assertEquals(2, spans.size()); // invocation + Workflow
- assertEquals(StatusCode.ERROR, spans.get(0).getStatus().getStatusCode());
+ assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot
+ assertEquals(StatusCode.ERROR, spanByName("Invocation").getStatus().getStatusCode());
}
@Test
@@ -593,10 +593,10 @@ void invocationEnd_withRetrying_leavesStatusUnset() {
"req-123", "arn:exec1", true, InvocationStatus.RETRYING, new RuntimeException("transient")));
var spans = spanExporter.getFinishedSpanItems();
- assertEquals(1, spans.size());
+ assertEquals(2, spans.size());
assertEquals(
StatusCode.UNSET,
- spans.get(0).getStatus().getStatusCode(),
+ spanByName("Invocation").getStatus().getStatusCode(),
"RETRYING invocation span is UNSET (interface cannot distinguish STOPPED/TIMED_OUT)");
}
@@ -618,7 +618,7 @@ void operationStart_createsSpan_operationEnd_endsIt() {
plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
var spans = spanExporter.getFinishedSpanItems();
- assertEquals(3, spans.size()); // operation + invocation + Workflow
+ assertEquals(4, spans.size()); // operation + invocation + Workflow + DurableExecutionRoot
var operationSpan = spans.stream()
.filter(s -> s.getName().contains("step"))
@@ -650,7 +650,7 @@ void userFunctionStart_and_end_createsAttemptSpan() {
plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
var spans = spanExporter.getFinishedSpanItems();
- assertEquals(3, spans.size()); // attempt + invocation + Workflow
+ assertEquals(4, spans.size()); // attempt + invocation + Workflow + DurableExecutionRoot
var attemptSpan = spans.stream()
.filter(s -> s.getName().contains("attempt"))
@@ -941,7 +941,7 @@ void fullLifecycle_producesCorrectSpanHierarchy() {
var spans = spanExporter.getFinishedSpanItems();
// 2 attempt spans + 2 operation spans + 1 invocation span + 1 Workflow span = 6
- assertEquals(6, spans.size());
+ assertEquals(7, spans.size());
var workflowTraceId = spanByName("Workflow").getTraceId();
var invocationTraceId = spanByName("Invocation").getTraceId();
@@ -988,7 +988,7 @@ void operationNotCompleted_spanEndedAtInvocationEnd() {
var spans = spanExporter.getFinishedSpanItems();
// Should have: operation span (ended at invocation end) + invocation span
- assertEquals(2, spans.size());
+ assertEquals(3, spans.size());
var operationSpan = spans.stream()
.filter(s -> s.getName().contains("wait"))
@@ -1110,7 +1110,7 @@ void xrayExtraction_withoutParentDoesNotForceTraceId() {
xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
var spans = spanExporter.getFinishedSpanItems();
- assertEquals(2, spans.size()); // invocation + Workflow
+ assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot
var invocationSpan = spanByName("Invocation");
var workflowSpan = spanByName("Workflow");
// Remote trace but no parent → a synthetic execution root on the remote trace ID anchors the execution. Both
@@ -1230,7 +1230,7 @@ void xrayExtraction_withoutParentSpanId_invocationSpanParentsOntoSyntheticRoot()
xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
var spans = spanExporter.getFinishedSpanItems();
- assertEquals(2, spans.size()); // invocation + Workflow
+ assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot
// Remote trace, no parent → a synthetic execution root on the remote trace ID anchors the execution. The
// Invocation span has a valid parent (that synthetic root) and joins the remote trace.
@@ -1318,7 +1318,7 @@ void xrayExtraction_nullExtractor_sharesArnDerivedExecutionTrace() {
noXrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", arn, true, InvocationStatus.SUCCEEDED, null));
var spans = spanExporter.getFinishedSpanItems();
- assertEquals(2, spans.size()); // invocation + Workflow
+ assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot
var invocationTraceId = spanByName("Invocation").getTraceId();
var workflowTraceId = spanByName("Workflow").getTraceId();
@@ -1385,7 +1385,7 @@ void operationEnd_withoutMatchingStart_createsContinuationSpanWithLink() {
plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null));
var spans = spanExporter.getFinishedSpanItems();
- assertEquals(3, spans.size()); // continuation + invocation + Workflow
+ assertEquals(4, spans.size()); // continuation + invocation + Workflow + DurableExecutionRoot
var continuationSpan = spans.stream()
.filter(s -> s.getName().contains("wait"))
@@ -1622,9 +1622,9 @@ void multiInvocation_stepWaitStep_producesCorrectSpans() {
new OperationInfo("op-2", "pause", "WAIT", "Wait", null, Instant.now(), null, null, false));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", arn, true, InvocationStatus.PENDING, null));
- // Invocation 1 should have: step op + step attempt + wait (PENDING) + invocation = 4
+ // Step op + step attempt + wait (PENDING) + Invocation + DurableExecutionRoot = 5
var inv1Spans = spanExporter.getFinishedSpanItems();
- assertEquals(4, inv1Spans.size());
+ assertEquals(5, inv1Spans.size());
var inv1TraceId = inv1Spans.stream()
.filter(span -> span.getName().equals("Invocation"))
.findFirst()
@@ -1685,8 +1685,8 @@ void multiInvocation_stepWaitStep_producesCorrectSpans() {
plugin.onInvocationEnd(new InvocationEndInfo("req-2", arn, false, InvocationStatus.SUCCEEDED, null));
var inv2Spans = spanExporter.getFinishedSpanItems();
- // wait continuation + step-B op + step-B attempt + invocation + Workflow = 5
- assertEquals(5, inv2Spans.size());
+ // Wait continuation + step-B op + step-B attempt + Invocation + Workflow + DurableExecutionRoot = 6
+ assertEquals(6, inv2Spans.size()); // Includes DurableExecutionRoot.
var inv2TraceId = inv2Spans.stream()
.filter(span -> span.getName().equals("Invocation"))
@@ -1761,8 +1761,8 @@ void crossInvocation_stepRetry_attemptsParentedToRespectiveInvocations() {
plugin.onInvocationEnd(new InvocationEndInfo("req-1", arn, true, InvocationStatus.PENDING, null));
var inv1Spans = spanExporter.getFinishedSpanItems();
- // operation span (PENDING) + attempt 1 span + invocation span = 3
- assertEquals(3, inv1Spans.size());
+ // Operation (PENDING) + attempt 1 + Invocation + DurableExecutionRoot = 4
+ assertEquals(4, inv1Spans.size());
var inv1OperationSpan = inv1Spans.stream()
.filter(s ->
@@ -1817,8 +1817,8 @@ void crossInvocation_stepRetry_attemptsParentedToRespectiveInvocations() {
plugin.onInvocationEnd(new InvocationEndInfo("req-2", arn, false, InvocationStatus.SUCCEEDED, null));
var inv2Spans = spanExporter.getFinishedSpanItems();
- // operation span + attempt 2 span + invocation span + Workflow span = 4
- assertEquals(4, inv2Spans.size());
+ // Operation + attempt 2 + Invocation + Workflow + DurableExecutionRoot = 5
+ assertEquals(5, inv2Spans.size()); // Includes DurableExecutionRoot.
var inv2OperationSpan = inv2Spans.stream()
.filter(s ->
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationPreparationAdmissionProbeTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationPreparationAdmissionProbeTest.java
new file mode 100644
index 000000000..665bca59e
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationPreparationAdmissionProbeTest.java
@@ -0,0 +1,290 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import software.amazon.awssdk.services.lambda.model.*;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.DurableContext;
+import software.amazon.lambda.durable.TypeToken;
+import software.amazon.lambda.durable.config.*;
+import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException;
+import software.amazon.lambda.durable.execution.DurableExecutor;
+import software.amazon.lambda.durable.model.DurableExecutionInput;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.model.WaitForConditionResult;
+import software.amazon.lambda.durable.plugin.*;
+import software.amazon.lambda.durable.serde.*;
+import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient;
+
+/** Diagnostic only: distinguishes new admission from rewriting an already-selected outcome. */
+class InvocationPreparationAdmissionProbeTest {
+ @ParameterizedTest
+ @CsvSource({
+ "unawaited,false",
+ "unawaited,true",
+ "early-parallel,false",
+ "early-parallel,true",
+ "awaited,false",
+ "awaited,true"
+ })
+ void preparationAdmissionWindow(String mode, boolean rejectResume) throws Exception {
+ var pending = new CountDownLatch(1);
+ var preparationEntered = new CountDownLatch(1);
+ var releasePreparation = new CountDownLatch(1);
+ var conditionFinished = new CountDownLatch(1);
+ var keeperEntered = new CountDownLatch(1);
+ var releaseKeeper = new CountDownLatch(1);
+ var backendReleased = new AtomicBoolean();
+ var preparing = new AtomicBoolean();
+ var predicates = new AtomicInteger();
+ var preparationDispatches = new AtomicInteger();
+ var childFailure = new AtomicReference();
+ var ends = new CopyOnWriteArrayList();
+ var rejection = new RejectedExecutionException("controlled READY admission rejection");
+ var workers =
+ new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue()) {
+ @Override
+ public void execute(Runnable task) {
+ if (backendReleased.get()) {
+ if (preparing.get()) preparationDispatches.incrementAndGet();
+ if (rejectResume) throw rejection;
+ }
+ super.execute(task);
+ }
+ };
+ var client = new LocalMemoryExecutionClient() {
+ @Override
+ public CheckpointDurableExecutionResponse checkpoint(
+ String arn, String token, List updates) {
+ var result = super.checkpoint(arn, token, updates);
+ var op = getOperationByName("condition");
+ if (op != null && op.status() == OperationStatus.PENDING) pending.countDown();
+ return result;
+ }
+ };
+ var serializer = new SerDes() {
+ private final JacksonSerDes delegate = new JacksonSerDes();
+
+ public String serialize(Object value) {
+ if ("selected-root".equals(value)) {
+ preparing.set(true);
+ preparationEntered.countDown();
+ await(releasePreparation);
+ }
+ return delegate.serialize(value);
+ }
+
+ public T deserialize(String value, TypeToken type) {
+ return delegate.deserialize(value, type);
+ }
+ };
+ var config = DurableConfig.builder()
+ .withExecutorService(workers)
+ .withDurableExecutionClient(client)
+ .withSerDes(serializer)
+ .withCheckpointDelay(Duration.ZERO)
+ .withPollingStrategy(attempt -> Duration.ofMillis(1))
+ .withPlugins(new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo info) {
+ ends.add(info);
+ }
+ })
+ .build();
+ var caller = Executors.newSingleThreadExecutor();
+ try {
+ var call = caller.submit(() -> DurableExecutor.execute(
+ input(),
+ null,
+ TypeToken.get(String.class),
+ (value, root) -> {
+ if (mode.equals("awaited")) {
+ root.runInChildContextAsync("keep-active", String.class, child -> {
+ keeperEntered.countDown();
+ await(releaseKeeper);
+ return "keeper";
+ });
+ await(keeperEntered);
+ try {
+ condition(root, predicates, childFailure, conditionFinished);
+ } finally {
+ releaseKeeper.countDown();
+ }
+ } else if (mode.equals("early-parallel")) {
+ var parallel = root.parallel(
+ "early",
+ ParallelConfig.builder()
+ .completionConfig(CompletionConfig.minSuccessful(1))
+ .build());
+ try (parallel) {
+ parallel.branch(
+ "waiting",
+ Integer.class,
+ child -> condition(child, predicates, childFailure, conditionFinished));
+ parallel.branch("winner", String.class, child -> {
+ await(pending);
+ return "winner";
+ });
+ }
+ assertEquals(1, parallel.get().succeeded());
+ } else {
+ root.runInChildContextAsync(
+ "unawaited",
+ Integer.class,
+ child -> condition(child, predicates, childFailure, conditionFinished));
+ await(pending);
+ }
+ return "selected-root";
+ },
+ config));
+ if (!pending.await(5, TimeUnit.SECONDS)) {
+ System.out.println("PROBE_SETUP mode=" + mode + " done=" + call.isDone());
+ if (call.isDone()) System.out.println("PROBE_EARLY_RESULT " + call.get());
+ fail("Real backend did not reach PENDING");
+ }
+ if (!mode.equals("awaited"))
+ assertTrue(
+ preparationEntered.await(5, TimeUnit.SECONDS),
+ "Root success selected before real output serializer");
+ backendReleased.set(true);
+ client.advanceTime();
+ assertTrue(conditionFinished.await(5, TimeUnit.SECONDS), "Real durable waiter settled after backend READY");
+ releasePreparation.countDown();
+ releaseKeeper.countDown();
+ if (mode.equals("awaited") && rejectResume) {
+ var thrown = assertThrows(ExecutionException.class, () -> call.get(5, TimeUnit.SECONDS));
+ assertInstanceOf(UnrecoverableDurableExecutionException.class, thrown.getCause());
+ assertSame(rejection, thrown.getCause().getCause());
+ assertEquals(InvocationStatus.RETRYING, ends.get(0).invocationStatus());
+ } else {
+ assertEquals(
+ ExecutionStatus.SUCCEEDED, call.get(5, TimeUnit.SECONDS).status());
+ assertEquals(InvocationStatus.SUCCEEDED, ends.get(0).invocationStatus());
+ }
+ assertEquals(1, ends.size());
+ System.out.println("PREPARATION_ADMISSION mode=" + mode + " reject=" + rejectResume
+ + " dispatchesWhilePreparing=" + preparationDispatches.get()
+ + " predicates=" + predicates.get() + " childFailure="
+ + (childFailure.get() == null
+ ? "none"
+ : childFailure.get().getClass().getName())
+ + " originalRejectionCause="
+ + (childFailure.get() != null && childFailure.get().getCause() == rejection)
+ + " backend=" + client.getOperationByName("condition").status()
+ + " End=" + ends.get(0).invocationStatus());
+ if (!mode.equals("awaited")) {
+ // Proposed admission guarantee, distinct from replacing the selected SUCCEEDED outcome.
+ assertEquals(0, preparationDispatches.get(), "No new READY user-worker admission after root selection");
+ assertEquals(1, predicates.get());
+ } else assertEquals(rejectResume ? 1 : 2, predicates.get());
+ } finally {
+ releasePreparation.countDown();
+ releaseKeeper.countDown();
+ caller.shutdownNow();
+ workers.shutdownNow();
+ assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ void successfulOversizedCheckpointStillRunsAfterAdmissionCloses() throws Exception {
+ var client = new LocalMemoryExecutionClient();
+ var workers = Executors.newCachedThreadPool();
+ var caller = Executors.newSingleThreadExecutor();
+ var ends = new CopyOnWriteArrayList();
+ var payload = "x".repeat(6 * 1024 * 1024);
+ try {
+ var config = DurableConfig.builder()
+ .withDurableExecutionClient(client)
+ .withExecutorService(workers)
+ .withCheckpointDelay(Duration.ZERO)
+ .withPlugins(new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo info) {
+ ends.add(info);
+ }
+ })
+ .build();
+ var result = caller.submit(() -> DurableExecutor.execute(
+ input(), null, TypeToken.get(String.class), (value, context) -> payload, config))
+ .get(5, TimeUnit.SECONDS);
+ assertEquals(ExecutionStatus.SUCCEEDED, result.status());
+ assertEquals("", result.result());
+ var updates = client.getOperationUpdates().stream()
+ .filter(x -> x.type() == OperationType.EXECUTION)
+ .toList();
+ assertEquals(1, updates.size());
+ assertEquals(OperationAction.SUCCEED, updates.get(0).action());
+ assertEquals(new JacksonSerDes().serialize(payload), updates.get(0).payload());
+ assertEquals(1, ends.size());
+ assertEquals(InvocationStatus.SUCCEEDED, ends.get(0).invocationStatus());
+ } finally {
+ caller.shutdownNow();
+ workers.shutdownNow();
+ assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ private static int condition(
+ DurableContext context,
+ AtomicInteger predicates,
+ AtomicReference failure,
+ CountDownLatch finished) {
+ try {
+ return context.waitForCondition(
+ "condition",
+ Integer.class,
+ (state, step) -> {
+ predicates.incrementAndGet();
+ return state == 1
+ ? WaitForConditionResult.continuePolling(2)
+ : WaitForConditionResult.stopPolling(state);
+ },
+ WaitForConditionConfig.builder()
+ .initialState(1)
+ .waitStrategy((state, attempt) -> Duration.ofSeconds(1))
+ .build());
+ } catch (RuntimeException | Error thrown) {
+ failure.set(thrown);
+ throw thrown;
+ } finally {
+ finished.countDown();
+ }
+ }
+
+ private static void await(CountDownLatch latch) {
+ try {
+ assertTrue(latch.await(8, TimeUnit.SECONDS), "Probe gate released within bounded cleanup");
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(interrupted);
+ }
+ }
+
+ private static DurableExecutionInput input() {
+ return new DurableExecutionInput(
+ "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/preparation/execution",
+ "token",
+ CheckpointUpdatedExecutionState.builder()
+ .operations(Operation.builder()
+ .id("execution")
+ .type(OperationType.EXECUTION)
+ .status(OperationStatus.STARTED)
+ .startTimestamp(Instant.EPOCH)
+ .executionDetails(ExecutionDetails.builder()
+ .inputPayload("\"input\"")
+ .build())
+ .build())
+ .build());
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationWorkerContextCleanupProbeTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationWorkerContextCleanupProbeTest.java
new file mode 100644
index 000000000..baddca683
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationWorkerContextCleanupProbeTest.java
@@ -0,0 +1,349 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.ContextKey;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.awssdk.services.lambda.model.*;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.TypeToken;
+import software.amazon.lambda.durable.config.CompletionConfig;
+import software.amazon.lambda.durable.config.ParallelConfig;
+import software.amazon.lambda.durable.execution.DurableExecutor;
+import software.amazon.lambda.durable.model.DurableExecutionInput;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.*;
+import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient;
+
+/** Diagnostic for the pre-existing End/close flush boundary; no proposed production policy in this class. */
+class InvocationWorkerContextCleanupProbeTest {
+ @ParameterizedTest
+ @CsvSource({
+ "unawaited,false",
+ "early-parallel,false",
+ "awaited,false",
+ "unawaited,true",
+ "early-parallel,true",
+ "awaited,true"
+ })
+ void alreadyRunningStepCompletesAfterEnd(String mode, boolean executionView) throws Exception {
+ boolean earlyParallel = mode.equals("early-parallel");
+ boolean awaited = mode.equals("awaited");
+ var running = new CountDownLatch(1);
+ var release = new CountDownLatch(1);
+ var ended = new CountDownLatch(1);
+ var lateOperationEnds = new AtomicInteger();
+ var bodyCalls = new AtomicInteger();
+ var client = new LocalMemoryExecutionClient();
+ var owner = new AtomicReference();
+ var initial = new ConcurrentHashMap();
+ var workerContext = new AtomicReference();
+ var captured = new AtomicBoolean();
+ var retired = new CountDownLatch(1);
+ var workers =
+ new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue()) {
+ protected void beforeExecute(Thread thread, Runnable task) {
+ initial.put(thread, Context.current());
+ }
+
+ protected void afterExecute(Runnable task, Throwable failure) {
+ if (Thread.currentThread() == owner.get() && captured.compareAndSet(false, true)) {
+ workerContext.set(Context.current());
+ retired.countDown();
+ }
+ }
+ };
+ var caller = Executors.newSingleThreadExecutor();
+ try (var exporter = InMemorySpanExporter.create();
+ var batch = BatchSpanProcessor.builder(exporter)
+ .setScheduleDelay(Duration.ofDays(1))
+ .build()) {
+ var builder = SdkTracerProvider.builder().addSpanProcessor(batch);
+ var pluginConfig = OtelPluginConfig.builder()
+ .contextExtractor(() -> null)
+ .enableMdc(false)
+ .build();
+ DurableExecutionPlugin otel = executionView
+ ? new ExecutionOtelPlugin(builder, pluginConfig)
+ : new InvocationOtelPlugin(builder, pluginConfig);
+ var config = DurableConfig.builder()
+ .withDurableExecutionClient(client)
+ .withExecutorService(workers)
+ .withCheckpointDelay(Duration.ZERO)
+ .withPlugins(otel, new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo info) {
+ ended.countDown();
+ }
+
+ public void onOperationEnd(OperationEndInfo info) {
+ if (ended.getCount() == 0) lateOperationEnds.incrementAndGet();
+ }
+ })
+ .build();
+ try {
+ var call = caller.submit(() -> DurableExecutor.execute(
+ input(),
+ null,
+ TypeToken.get(String.class),
+ (value, root) -> {
+ if (earlyParallel) {
+ var parallel = root.parallel(
+ "early",
+ ParallelConfig.builder()
+ .completionConfig(CompletionConfig.minSuccessful(1))
+ .build());
+ try (parallel) {
+ parallel.branch(
+ "waiting",
+ String.class,
+ child -> child.step("held", String.class, step -> {
+ owner.set(Thread.currentThread());
+ bodyCalls.incrementAndGet();
+ running.countDown();
+ await(release);
+ return "completed";
+ }));
+ parallel.branch("winner", String.class, child -> {
+ await(running);
+ return "winner";
+ });
+ }
+ assertEquals(1, parallel.get().succeeded());
+ } else {
+ var future = root.stepAsync("held", String.class, step -> {
+ owner.set(Thread.currentThread());
+ bodyCalls.incrementAndGet();
+ running.countDown();
+ await(release);
+ return "completed";
+ });
+ await(running);
+ if (awaited) future.get();
+ }
+ return "root-success";
+ },
+ config));
+ if (awaited) {
+ assertTrue(running.await(5, TimeUnit.SECONDS));
+ release.countDown();
+ }
+ assertTrue(ended.await(5, TimeUnit.SECONDS));
+ if (!awaited) assertThrows(TimeoutException.class, () -> call.get(100, TimeUnit.MILLISECONDS));
+ release.countDown();
+ assertEquals(
+ ExecutionStatus.SUCCEEDED, call.get(5, TimeUnit.SECONDS).status());
+ assertEquals(
+ OperationStatus.SUCCEEDED,
+ client.getOperationByName("held").status());
+ assertEquals(1, bodyCalls.get());
+ assertTrue(retired.await(5, TimeUnit.SECONDS));
+ var before = exporter.getFinishedSpanItems().stream()
+ .filter(x -> x.getName().equals("held"))
+ .map(x -> x.getAttributes().get(SpanAttributes.DURABLE_OPERATION_STATUS))
+ .toList();
+ batch.forceFlush().join(3, TimeUnit.SECONDS);
+ var after = exporter.getFinishedSpanItems().stream()
+ .filter(x -> x.getName().equals("held"))
+ .map(x -> x.getAttributes().get(SpanAttributes.DURABLE_OPERATION_STATUS))
+ .toList();
+ System.out.println("LATE_FLUSH_PUBLIC earlyParallel=" + earlyParallel + " backend=SUCCEEDED bodyCalls="
+ + bodyCalls.get() + " lateOperationEnds=" + lateOperationEnds.get() + " beforeManualFlush="
+ + before + " afterManualFlush=" + after);
+ if (!awaited)
+ assertTrue(
+ lateOperationEnds.get() > 0, "Existing public operation-end occurs after invocation End");
+ System.out.println("WORKER_CONTEXT_PUBLIC executionView=" + executionView + " mode=" + mode
+ + " restored=" + (initial.get(owner.get()) == workerContext.get()) + " remainingSpan="
+ + Span.fromContext(workerContext.get()).getSpanContext());
+ assertSame(
+ initial.get(owner.get()),
+ workerContext.get(),
+ "Real worker ambient Context restored after its task finishes");
+ // This diagnostic records actual export behavior; it does not redefine which outcome wins.
+ } finally {
+ release.countDown();
+ caller.shutdownNow();
+ workers.shutdownNow();
+ assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void retainedPluginAndWorkerRestoreApplicationAmbientOnReuse(boolean executionView) throws Exception {
+ var key = ContextKey.named("application-worker");
+ var ambient = Context.root().with(key, "retained-application-context");
+ var rootExecutor = Executors.newSingleThreadExecutor();
+ var nextRoot = new AtomicBoolean(true);
+ var retired = new AtomicReference();
+ var before = new AtomicReference();
+ var after = new AtomicReference();
+ var actualWorker = new AtomicReference();
+ var workerExecutor =
+ new ThreadPoolExecutor(
+ 1,
+ 1,
+ 0,
+ TimeUnit.SECONDS,
+ new LinkedBlockingQueue(),
+ task -> new Thread(
+ () -> {
+ try (var ignored = ambient.makeCurrent()) {
+ task.run();
+ }
+ },
+ "owned-operation-worker")) {
+ protected void beforeExecute(Thread thread, Runnable task) {
+ actualWorker.set(thread);
+ before.set(Context.current());
+ }
+
+ protected void afterExecute(Runnable task, Throwable failure) {
+ after.set(Context.current());
+ retired.get().countDown();
+ }
+ };
+ var dispatch = new AbstractExecutorService() {
+ public void execute(Runnable task) {
+ (nextRoot.compareAndSet(true, false) ? rootExecutor : workerExecutor).execute(task);
+ }
+
+ public void shutdown() {
+ rootExecutor.shutdown();
+ workerExecutor.shutdown();
+ }
+
+ public List shutdownNow() {
+ var pending = new ArrayList<>(rootExecutor.shutdownNow());
+ pending.addAll(workerExecutor.shutdownNow());
+ return pending;
+ }
+
+ public boolean isShutdown() {
+ return rootExecutor.isShutdown() && workerExecutor.isShutdown();
+ }
+
+ public boolean isTerminated() {
+ return rootExecutor.isTerminated() && workerExecutor.isTerminated();
+ }
+
+ public boolean awaitTermination(long duration, TimeUnit unit) throws InterruptedException {
+ return rootExecutor.awaitTermination(duration, unit) && workerExecutor.awaitTermination(duration, unit);
+ }
+ };
+ var caller = Executors.newSingleThreadExecutor();
+ try (var exporter = InMemorySpanExporter.create();
+ var batch = BatchSpanProcessor.builder(exporter)
+ .setScheduleDelay(Duration.ofDays(1))
+ .build()) {
+ var builder = SdkTracerProvider.builder().addSpanProcessor(batch);
+ var pluginConfig = OtelPluginConfig.builder()
+ .contextExtractor(() -> null)
+ .enableMdc(false)
+ .build();
+ DurableExecutionPlugin otel = executionView
+ ? new ExecutionOtelPlugin(builder, pluginConfig)
+ : new InvocationOtelPlugin(builder, pluginConfig);
+ Thread firstWorker = null;
+ for (var round = 0; round < 2; round++) {
+ var running = new CountDownLatch(1);
+ var release = new CountDownLatch(1);
+ var ended = new CountDownLatch(1);
+ retired.set(new CountDownLatch(1));
+ nextRoot.set(true);
+ var config = DurableConfig.builder()
+ .withExecutorService(dispatch)
+ .withDurableExecutionClient(new LocalMemoryExecutionClient())
+ .withCheckpointDelay(Duration.ZERO)
+ .withPlugins(otel, new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo info) {
+ ended.countDown();
+ }
+ })
+ .build();
+ try {
+ var call = caller.submit(() -> DurableExecutor.execute(
+ input(),
+ null,
+ TypeToken.get(String.class),
+ (value, root) -> {
+ root.stepAsync("held", String.class, step -> {
+ assertEquals(
+ "retained-application-context",
+ Context.current().get(key));
+ assertTrue(Span.current().getSpanContext().isValid());
+ running.countDown();
+ await(release);
+ return "done";
+ });
+ await(running);
+ return "root-success";
+ },
+ config));
+ assertTrue(ended.await(5, TimeUnit.SECONDS));
+ release.countDown();
+ assertEquals(
+ ExecutionStatus.SUCCEEDED,
+ call.get(5, TimeUnit.SECONDS).status());
+ assertTrue(retired.get().await(5, TimeUnit.SECONDS));
+ assertSame(ambient, before.get());
+ assertSame(ambient, after.get());
+ if (firstWorker == null) firstWorker = actualWorker.get();
+ else assertSame(firstWorker, actualWorker.get(), "The same actual worker was reused");
+ System.out.println("WORKER_REUSE_PUBLIC executionView=" + executionView + " round=" + round
+ + " ambientRestored=true");
+ } finally {
+ release.countDown();
+ }
+ }
+ } finally {
+ caller.shutdownNow();
+ dispatch.shutdownNow();
+ assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
+ assertTrue(dispatch.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ private static void await(CountDownLatch latch) {
+ try {
+ assertTrue(latch.await(8, TimeUnit.SECONDS));
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(failure);
+ }
+ }
+
+ private static DurableExecutionInput input() {
+ return new DurableExecutionInput(
+ "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/late-flush/execution",
+ "token",
+ CheckpointUpdatedExecutionState.builder()
+ .operations(Operation.builder()
+ .id("execution")
+ .type(OperationType.EXECUTION)
+ .status(OperationStatus.STARTED)
+ .startTimestamp(Instant.EPOCH)
+ .executionDetails(ExecutionDetails.builder()
+ .inputPayload("\"input\"")
+ .build())
+ .build())
+ .build());
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ReplacementSamplerPolicyTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ReplacementSamplerPolicyTest.java
index 2d3fbe8a5..b682a9ea3 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ReplacementSamplerPolicyTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ReplacementSamplerPolicyTest.java
@@ -113,7 +113,9 @@ public ContextPropagators getPropagators() {
assertTrue(decisions.size() >= 2, "Initial invocation and actual replay must both execute the handler");
assertTrue(decisions.stream().allMatch(decision -> decision == expectedSampled), decisions.toString());
assertEquals(
- upstream.equals("root") ? (topology.equals("opaque") ? 1 : decisions.size()) : 0,
+ topology.equals("plain")
+ ? decisions.size() * (upstream.equals("root") ? 2 : 1)
+ : upstream.equals("root") ? (topology.equals("opaque") ? 1 : decisions.size()) : 0,
evaluations.get());
assertEquals(1, sideEffects.get());
} finally {
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java
new file mode 100644
index 000000000..df7f52dd0
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java
@@ -0,0 +1,199 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.trace.ReadWriteSpan;
+import io.opentelemetry.sdk.trace.ReadableSpan;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.SpanProcessor;
+import io.opentelemetry.sdk.trace.data.LinkData;
+import io.opentelemetry.sdk.trace.samplers.Sampler;
+import io.opentelemetry.sdk.trace.samplers.SamplingResult;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+import software.amazon.lambda.durable.plugin.InvocationStatus;
+
+class RootStartProcessorIsolationTest {
+ private static final String TRACE = "6955b900123456789012345678901234";
+ private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id";
+ private static final Instant START = Instant.parse("2026-10-02T00:00:00Z");
+
+ static Stream cases() {
+ return Stream.of(true, false)
+ .flatMap(view -> Stream.of(true, false)
+ .flatMap(sampled -> Stream.of(true, false)
+ .flatMap(shared -> Stream.of(true, false)
+ .flatMap(recordOnly -> Stream.of(true, false)
+ .map(forwardParent -> Arguments.of(
+ view, sampled, shared, recordOnly, forwardParent))))));
+ }
+
+ @ParameterizedTest
+ @MethodSource("cases")
+ void durableStartCallbacksRetainTheirOwnSamplingAndRandomIds(
+ boolean executionView,
+ boolean sampled,
+ boolean sharedGenerator,
+ boolean rootRecordOnly,
+ boolean forwardParent) {
+ var observer = new StartObserver(forwardParent, rootRecordOnly ? "Invocation" : "DurableExecutionRoot");
+ var policy = sampled ? Sampler.alwaysOn() : Sampler.alwaysOff();
+ var builder = SdkTracerProvider.builder()
+ .setSampler(rootSampler(rootRecordOnly, policy))
+ .addSpanProcessor(observer);
+ var config = OtelPluginConfig.builder()
+ .enableMdc(false)
+ .contextExtractor(() -> new ExtractedContext(
+ TRACE,
+ null,
+ rootRecordOnly ? ExtractedContext.Sampling.UNDECIDED : ExtractedContext.Sampling.SAMPLED))
+ .build();
+ DurableExecutionPlugin plugin =
+ executionView ? new ExecutionOtelPlugin(builder, config) : new InvocationOtelPlugin(builder, config);
+ // Rebuilding this configured builder shares the plugin-installed generator; the other branch exercises the
+ // same thread-scoped bridge through an independent generator instance.
+ try (var unrelated = sharedGenerator
+ ? builder.build()
+ : SdkTracerProvider.builder()
+ .setSampler(DurableSampler.wrap(policy))
+ .setIdGenerator(new DeterministicIdGenerator())
+ .build()) {
+ observer.tracer = unrelated.get("processor");
+ var original = Context.current();
+ assertControl(observer.tracer, sampled);
+ invoke(plugin, true);
+ invoke(plugin, false);
+ assertControl(observer.tracer, sampled);
+ assertSame(original, Context.current());
+ assertObservations(observer, sampled, rootRecordOnly);
+ }
+ }
+
+ private static void invoke(DurableExecutionPlugin plugin, boolean first) {
+ plugin.onInvocationStart(new InvocationInfo("request", ARN, first, START));
+ plugin.onInvocationEnd(new InvocationEndInfo("request", ARN, first, InvocationStatus.PENDING, null));
+ }
+
+ private static void assertControl(Tracer tracer, boolean sampled) {
+ var span = tracer.spanBuilder("control").setNoParent().startSpan();
+ assertEquals(sampled, span.getSpanContext().isSampled());
+ assertEquals(sampled, span.isRecording());
+ span.end();
+ }
+
+ private static void assertObservations(StartObserver observer, boolean sampled, boolean recordOnly) {
+ assertEquals(2, observer.roots.size());
+ var root = observer.roots.get(0);
+ assertEquals(recordOnly ? 0 : 2, observer.anchorStarts, "RECORD_ONLY has no anchor processor callback");
+ if (!recordOnly) {
+ assertEquals(observer.roots.get(0), observer.roots.get(1));
+ assertEquals(new DeterministicIdGenerator().generateExecutionRootSpanId(ARN), root.getSpanId());
+ }
+ assertEquals(2, observer.callbacks.size());
+ assertEquals(List.of(sampled, sampled), observer.recording);
+ for (var callback : observer.callbacks) {
+ assertEquals(sampled, callback.isSampled());
+ assertTrue(callback.isValid());
+ if (recordOnly && observer.forwardParent) {
+ // An Invocation has a real execution ancestor; explicitly forwarding it preserves that trace.
+ assertEquals(root.getTraceId(), callback.getTraceId());
+ } else {
+ assertNotEquals(root.getTraceId(), callback.getTraceId());
+ }
+ assertNotEquals(root.getSpanId(), callback.getSpanId());
+ }
+ assertEquals(
+ recordOnly && observer.forwardParent ? 1 : 2,
+ observer.callbacks.stream()
+ .map(SpanContext::getTraceId)
+ .distinct()
+ .count());
+ assertEquals(
+ 2,
+ observer.callbacks.stream()
+ .map(SpanContext::getSpanId)
+ .distinct()
+ .count());
+ }
+
+ private static Sampler rootSampler(boolean recordOnly, Sampler policy) {
+ if (!recordOnly) return policy;
+ return new Sampler() {
+ @Override
+ public SamplingResult shouldSample(
+ Context parent,
+ String trace,
+ String name,
+ SpanKind kind,
+ Attributes attributes,
+ List links) {
+ return name.equals("Workflow")
+ ? SamplingResult.recordOnly()
+ : policy.shouldSample(parent, trace, name, kind, attributes, links);
+ }
+
+ @Override
+ public String getDescription() {
+ return "RecordOnlyWorkflow";
+ }
+ };
+ }
+
+ private static final class StartObserver implements SpanProcessor {
+ private Tracer tracer;
+ private final boolean forwardParent;
+ private final String observedName;
+ private int anchorStarts;
+
+ private StartObserver(boolean forwardParent, String observedName) {
+ this.forwardParent = forwardParent;
+ this.observedName = observedName;
+ }
+
+ private final List roots = new ArrayList<>();
+ private final List callbacks = new ArrayList<>();
+ private final List recording = new ArrayList<>();
+
+ @Override
+ public void onStart(Context parent, ReadWriteSpan span) {
+ if (span.getName().equals("DurableExecutionRoot")) anchorStarts++;
+ // Observe the recording Invocation for RECORD_ONLY, which intentionally has no recording anchor.
+ // The name guard permits one unrelated span per observation and prevents processor recursion.
+ if (!span.getName().equals(observedName)) return;
+ roots.add(span.getSpanContext());
+ var builder = tracer.spanBuilder("unrelated-onStart");
+ var callback = (forwardParent ? builder.setParent(parent) : builder.setNoParent()).startSpan();
+ callbacks.add(callback.getSpanContext());
+ recording.add(callback.isRecording());
+ callback.end();
+ }
+
+ @Override
+ public boolean isStartRequired() {
+ return true;
+ }
+
+ @Override
+ public void onEnd(ReadableSpan span) {}
+
+ @Override
+ public boolean isEndRequired() {
+ return false;
+ }
+ }
+}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/SamplingProcessorIsolationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/SamplingProcessorIsolationTest.java
index 79981a9b4..976711b72 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/SamplingProcessorIsolationTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/SamplingProcessorIsolationTest.java
@@ -78,10 +78,10 @@ void agentProcessorForwardingParentCannotReuseApplicationSamplingIntent(
var agentWrap = agentSamplerType.getDeclaredMethod("wrap", Sampler.class);
agentWrap.setAccessible(true);
var appDecision = Class.forName(DurableSamplingDecision.class.getName(), true, appLoader);
- var appGet = appDecision.getDeclaredMethod("get", Context.class);
+ var appGet = appDecision.getDeclaredMethod("consume", Context.class);
appGet.setAccessible(true);
var agentDecision = Class.forName(DurableSamplingDecision.class.getName(), true, agentLoader);
- var agentGet = agentDecision.getDeclaredMethod("get", Context.class);
+ var agentGet = agentDecision.getDeclaredMethod("consume", Context.class);
agentGet.setAccessible(true);
var agentIdType = Class.forName(DeterministicIdGenerator.class.getName(), true, agentLoader);
var callbacks = new AtomicInteger();
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/VisibleReplacementSdkSpanPolicyTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/VisibleReplacementSdkSpanPolicyTest.java
new file mode 100644
index 000000000..d22efaa82
--- /dev/null
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/VisibleReplacementSdkSpanPolicyTest.java
@@ -0,0 +1,129 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.otel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.api.trace.TracerProvider;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.propagation.ContextPropagators;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.data.LinkData;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import io.opentelemetry.sdk.trace.samplers.Sampler;
+import io.opentelemetry.sdk.trace.samplers.SamplingResult;
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
+
+class VisibleReplacementSdkSpanPolicyTest {
+ @ParameterizedTest
+ @CsvSource({
+ "invocation,false,root,false", "execution,false,root,false",
+ "invocation,true,root,true", "execution,true,root,true",
+ "invocation,false,sampled,true", "execution,false,sampled,true",
+ "invocation,true,unsampled,false", "execution,true,unsampled,false"
+ })
+ void actualSdkSpansHonorVisibleReplacementRootPolicy(
+ String view, boolean rootSamples, String upstream, boolean expectedSampled) {
+ var oldHeader = System.getProperty("com.amazonaws.xray.traceHeader");
+ var rootPolicySnapshots = new AtomicInteger();
+ var invocations = new AtomicInteger();
+ var sideEffects = new AtomicInteger();
+ var header = "Root=1-6955b900-123456789012345678901234";
+ if (upstream.equals("sampled")) header += ";Sampled=1";
+ else if (upstream.equals("unsampled")) header += ";Sampled=0";
+ var root = new Sampler() {
+ public SamplingResult shouldSample(
+ Context parent,
+ String traceId,
+ String name,
+ SpanKind kind,
+ Attributes attributes,
+ List links) {
+ assertFalse(Span.fromContext(parent).getSpanContext().isValid());
+ // Count the SDK's root-policy snapshot, excluding normal provider sampling for a real synthetic-root
+ // export on branches that implement it. No exactly-once promise is made for a plain provider.
+ if (!name.equals("DurableExecutionRoot")
+ && attributes.get(AttributeKey.stringKey("durable.execution.arn")) != null) {
+ rootPolicySnapshots.incrementAndGet();
+ }
+ return rootSamples ? SamplingResult.recordAndSample() : SamplingResult.drop();
+ }
+
+ public String getDescription() {
+ return "counted-visible-root-policy";
+ }
+ };
+ GlobalOpenTelemetry.resetForTest();
+ OtelPluginAutoConfigurationState.markInstalled();
+ System.setProperty("com.amazonaws.xray.traceHeader", header);
+ try (var exporter = InMemorySpanExporter.create();
+ var provider = SdkTracerProvider.builder()
+ .setSampler(Sampler.parentBased(root))
+ .setIdGenerator(new DeterministicIdGenerator())
+ .addSpanProcessor(SimpleSpanProcessor.create(exporter))
+ .build()) {
+ GlobalOpenTelemetry.set(new OpenTelemetry() {
+ public TracerProvider getTracerProvider() {
+ return provider;
+ }
+
+ public ContextPropagators getPropagators() {
+ return ContextPropagators.noop();
+ }
+ });
+ DurableExecutionPlugin plugin =
+ view.equals("invocation") ? new InvocationOtelPlugin() : new ExecutionOtelPlugin();
+ var runner = LocalDurableTestRunner.create(
+ String.class,
+ (input, context) -> {
+ invocations.incrementAndGet();
+ context.step("saved", String.class, step -> {
+ sideEffects.incrementAndGet();
+ return "saved";
+ });
+ context.wait("pause", Duration.ofSeconds(1));
+ return "done";
+ },
+ DurableConfig.builder().withPlugins(plugin).build());
+ assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus());
+ assertEquals(
+ ExecutionStatus.SUCCEEDED, runner.runUntilComplete("input").getStatus());
+ assertTrue(invocations.get() >= 2);
+ var sdkSpans = exporter.getFinishedSpanItems().stream()
+ .filter(span ->
+ span.getInstrumentationScopeInfo().getName().equals("aws-durable-execution-sdk-java"))
+ // A plain provider keeps its native policy for parentless synthetic-root exports. Full SDK
+ // sampling overrides require DurableSampler; this regression concerns regular SDK descendants.
+ .filter(span -> !span.getName().equals("DurableExecutionRoot"))
+ .toList();
+ if (expectedSampled) {
+ assertFalse(sdkSpans.isEmpty());
+ assertTrue(
+ sdkSpans.stream().allMatch(span -> span.getSpanContext().isSampled()));
+ } else assertTrue(sdkSpans.isEmpty(), sdkSpans.toString());
+ assertEquals(upstream.equals("root") ? invocations.get() : 0, rootPolicySnapshots.get());
+ assertEquals(1, sideEffects.get());
+ } finally {
+ GlobalOpenTelemetry.resetForTest();
+ OtelPluginAutoConfigurationState.resetInstalledForTest();
+ DurableSamplingDecision.clearSharedStateForTest();
+ if (oldHeader == null) System.clearProperty("com.amazonaws.xray.traceHeader");
+ else System.setProperty("com.amazonaws.xray.traceHeader", oldHeader);
+ }
+ }
+}
diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationEndCleanupOrderTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationEndCleanupOrderTest.java
new file mode 100644
index 000000000..23a4784eb
--- /dev/null
+++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/ContinuationEndCleanupOrderTest.java
@@ -0,0 +1,415 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.*;
+import java.util.function.BiFunction;
+import java.util.stream.Stream;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+import software.amazon.awssdk.services.lambda.model.*;
+import software.amazon.lambda.durable.config.WaitForConditionConfig;
+import software.amazon.lambda.durable.context.BaseContextImpl;
+import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException;
+import software.amazon.lambda.durable.execution.DurableExecutor;
+import software.amazon.lambda.durable.execution.ExecutionManager;
+import software.amazon.lambda.durable.model.DurableExecutionInput;
+import software.amazon.lambda.durable.model.DurableExecutionOutput;
+import software.amazon.lambda.durable.model.ExecutionStatus;
+import software.amazon.lambda.durable.model.WaitForConditionResult;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationStatus;
+import software.amazon.lambda.durable.serde.JacksonSerDes;
+import software.amazon.lambda.durable.serde.SerDes;
+import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient;
+import software.amazon.lambda.durable.util.ExceptionHelper;
+
+class ContinuationEndCleanupOrderTest {
+ static Stream cleanupCases() {
+ return Stream.of("async", "inline-root", "submit-root")
+ .flatMap(mode -> Stream.of("serde", "reject", "vm", "death").map(kind -> Arguments.of(mode, kind)));
+ }
+
+ @SuppressWarnings("removal")
+ @ParameterizedTest
+ @MethodSource("cleanupCases")
+ void continuationFailureMustWakeBodyBeforeEndWaitsForCleanup(String mode, String kind) throws Exception {
+ boolean rejection = kind.equals("reject");
+ boolean withPlugin = true;
+ Throwable failure =
+ switch (kind) {
+ case "reject" -> new RejectedExecutionException("resumed worker rejected");
+ case "vm" -> new InternalError("resumed state fatal");
+ case "death" -> new ThreadDeath();
+ default -> new IllegalArgumentException("resumed state cannot deserialize");
+ };
+ var injectionThread = new AtomicReference();
+ var handlerThread = new AtomicReference();
+ var callerThread = new AtomicReference();
+ var escaped = new CountDownLatch(1);
+ var escapedFailure = new AtomicReference();
+ var oldHandler = Thread.getDefaultUncaughtExceptionHandler();
+ Thread.setDefaultUncaughtExceptionHandler((thread, error) -> {
+ if (error == failure) {
+ escapedFailure.set(error);
+ escaped.countDown();
+ } else if (oldHandler != null) oldHandler.uncaughtException(thread, error);
+ });
+ var managerForCleanup = new AtomicReference();
+ var armed = new AtomicBoolean();
+ var injected = new CountDownLatch(1);
+ var injectionOwner = new AtomicReference();
+ var checks = new AtomicInteger();
+ var bodyFinally = new CountDownLatch(1);
+ var cleanupOrder = new CopyOnWriteArrayList();
+ var ends = new CopyOnWriteArrayList();
+ var pollEntered = new CountDownLatch(1);
+ var releaseReady = new CountDownLatch(1);
+ var firstArmedPoll = new AtomicBoolean(true);
+ var onCoordinator = new AtomicBoolean();
+ var pendingCall = new AtomicReference>();
+ var serde = new SerDes() {
+ private final JacksonSerDes delegate = new JacksonSerDes();
+
+ public String serialize(Object value) {
+ return delegate.serialize(value);
+ }
+
+ public T deserialize(String value, TypeToken type) {
+ if (!rejection
+ && armed.get()
+ && Thread.currentThread().getName().startsWith("durable-sdk-internal-")) {
+ injectionThread.set(Thread.currentThread());
+ injectionOwner.set(Thread.currentThread().getName());
+ onCoordinator.set(Arrays.stream(Thread.currentThread().getStackTrace())
+ .anyMatch(frame -> frame.getMethodName().equals("completeCheckpointContinuation")));
+ injected.countDown();
+ ExceptionHelper.sneakyThrow(failure);
+ }
+ return delegate.deserialize(value, type);
+ }
+ };
+ var client = new LocalMemoryExecutionClient() {
+ @Override
+ public CheckpointDurableExecutionResponse checkpoint(
+ String arn, String token, List updates) {
+ if (armed.get() && updates.isEmpty() && firstArmedPoll.compareAndSet(true, false)) {
+ pollEntered.countDown();
+ try {
+ assertTrue(releaseReady.await(3, TimeUnit.SECONDS));
+ } catch (InterruptedException interrupted) {
+ throw new AssertionError(interrupted);
+ }
+ }
+ return super.checkpoint(arn, token, updates);
+ }
+ };
+ var workers = new RootExecutor(mode, () -> {
+ if (rejection && armed.get() && Thread.currentThread().getName().startsWith("durable-sdk-internal-")) {
+ injectionThread.set(Thread.currentThread());
+ injectionOwner.set(Thread.currentThread().getName());
+ onCoordinator.set(Arrays.stream(Thread.currentThread().getStackTrace())
+ .anyMatch(frame -> frame.getMethodName().equals("completeCheckpointContinuation")));
+ injected.countDown();
+ ExceptionHelper.sneakyThrow(failure);
+ }
+ });
+ var caller = Executors.newSingleThreadExecutor(task -> {
+ var thread = new Thread(task, "legacy-outcome-caller");
+ callerThread.set(thread);
+ return thread;
+ });
+ var builder = DurableConfig.builder()
+ .withExecutorService(workers)
+ .withDurableExecutionClient(client)
+ .withCheckpointDelay(Duration.ZERO)
+ .withPollingStrategy(attempt -> Duration.ZERO);
+ if (withPlugin)
+ builder.withPlugins(new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo info) {
+ ends.add(info);
+ if (info.invocationStatus() == InvocationStatus.RETRYING) {
+ assertSame(
+ handlerThread.get(),
+ Thread.currentThread(),
+ "The merged lifecycle runs End on the handler owner after its finally block");
+ cleanupOrder.add("end-enter");
+ try {
+ boolean finished = bodyFinally.await(2, TimeUnit.SECONDS);
+ cleanupOrder.add(finished ? "end-complete" : "end-timed-out");
+ assertTrue(finished, "End must not prevent the manager from waking handler cleanup");
+ } catch (InterruptedException interrupted) {
+ throw new AssertionError(interrupted);
+ }
+ }
+ }
+ });
+ var config = builder.build();
+ var condition = WaitForConditionConfig.builder()
+ .initialState(1)
+ .serDes(serde)
+ .waitStrategy((state, attempt) -> Duration.ofSeconds(1))
+ .build();
+ BiFunction handler = (input, context) -> {
+ handlerThread.set(Thread.currentThread());
+ managerForCleanup.set(((BaseContextImpl) context).getExecutionManager());
+ var waiting = context.waitForConditionAsync(
+ "condition",
+ Integer.class,
+ (state, step) -> {
+ checks.incrementAndGet();
+ return state == 2
+ ? WaitForConditionResult.stopPolling(state)
+ : WaitForConditionResult.continuePolling(2);
+ },
+ condition);
+ if (armed.get()) {
+ try {
+ assertTrue(pollEntered.await(3, TimeUnit.SECONDS));
+ } catch (InterruptedException interrupted) {
+ throw new AssertionError(interrupted);
+ }
+ }
+ try {
+ return String.valueOf(waiting.get());
+ } finally {
+ if (armed.get()) {
+ cleanupOrder.add("body-finally");
+ bodyFinally.countDown();
+ }
+ }
+ };
+ try {
+ var first = caller.submit(() -> DurableExecutor.execute(
+ input(List.of()), null, TypeToken.get(String.class), handler, config))
+ .get(3, TimeUnit.SECONDS);
+ assertEquals(ExecutionStatus.PENDING, first.status());
+ var cachedPending = new ArrayList<>(client.getAllOperations());
+ assertEquals(
+ OperationStatus.PENDING,
+ client.getOperationByName("condition").status());
+ assertTrue(client.advanceTime());
+ armed.set(true);
+ workers.nextInvocation();
+ var second = caller.submit(() ->
+ DurableExecutor.execute(input(cachedPending), null, TypeToken.get(String.class), handler, config));
+ pendingCall.set(second);
+ assertTrue(pollEntered.await(3, TimeUnit.SECONDS));
+ if (mode.equals("async")) awaitCallerOutcomeJoin(callerThread.get());
+ releaseReady.countDown();
+ assertTrue(injected.await(3, TimeUnit.SECONDS));
+ assertTrue(onCoordinator.get(), "Fault must run in the actual WFC coordinator continuation");
+ DurableExecutionOutput output = null;
+ Throwable callerFailure = null;
+ try {
+ output = second.get(5, TimeUnit.SECONDS);
+ } catch (ExecutionException observationFailure) {
+ callerFailure = observationFailure.getCause();
+ }
+ System.out.println("CONTINUATION_ORDINARY_PUBLIC owner=" + injectionOwner.get()
+ + " caller="
+ + (output != null
+ ? output.status()
+ : callerFailure.getClass().getName())
+ + " backend=" + client.getOperationByName("condition").status()
+ + " checks=" + checks.get());
+ assertEquals(1, checks.get(), "A failed resumed state read must not rerun the condition body");
+ System.out.println("CONTINUATION_END_CLEANUP_ORDER " + cleanupOrder);
+ assertFalse(cleanupOrder.contains("end-timed-out"));
+ assertTrue(cleanupOrder.contains("end-complete"));
+ assertTrue(cleanupOrder.indexOf("body-finally") < cleanupOrder.indexOf("end-complete"));
+ if (failure instanceof Error) {
+ assertTrue(escaped.await(3, TimeUnit.SECONDS));
+ assertSame(failure, escapedFailure.get());
+ }
+ assertNull(output, "An SDK continuation failure must not produce a durable response");
+ var retry = assertInstanceOf(UnrecoverableDurableExecutionException.class, callerFailure);
+ assertTrue(retry.isRetryable());
+ assertSame(failure, retry.getCause());
+ assertEquals(
+ OperationStatus.READY,
+ client.getOperationByName("condition").status());
+ var activeField = ExecutionManager.class.getDeclaredField("activeThreads");
+ activeField.setAccessible(true);
+ assertFalse(
+ ((Set) activeField.get(managerForCleanup.get()))
+ .contains(client.getOperationByName("condition").id()),
+ "A rejected operation worker must not leave a phantom activity registration");
+ if (withPlugin)
+ assertEquals(
+ List.of(InvocationStatus.PENDING, InvocationStatus.RETRYING),
+ ends.stream().map(InvocationEndInfo::invocationStatus).toList());
+ else assertTrue(ends.isEmpty());
+ armed.set(false);
+ workers.nextInvocation();
+ var third = caller.submit(() -> DurableExecutor.execute(
+ input(client.getAllOperations()), null, TypeToken.get(String.class), handler, config));
+ pendingCall.set(third);
+ assertEquals(
+ ExecutionStatus.SUCCEEDED, third.get(3, TimeUnit.SECONDS).status());
+ assertEquals(2, checks.get(), "Only the original and successful resumed predicates run");
+ } finally {
+ releaseReady.countDown();
+ // Cleanup-only escape hatch AFTER the real public invocation observation/timeout; never test behavior.
+ var manager = managerForCleanup.get();
+ if (pendingCall.get() != null
+ && !pendingCall.get().isDone()
+ && manager != null
+ && !manager.isExecutionCompletedExceptionally()) {
+ try {
+ manager.terminateExecution(new UnrecoverableDurableExecutionException(
+ ErrorObject.builder().errorMessage("probe cleanup").build(), true));
+ } catch (UnrecoverableDurableExecutionException expected) {
+ }
+ }
+ Thread.setDefaultUncaughtExceptionHandler(oldHandler);
+ caller.shutdownNow();
+ workers.shutdownNow();
+ assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"async", "inline-root", "submit-root"})
+ void ordinaryCompletionEndsOnItsHandlerOwner(String mode) throws Exception {
+ var releaseBody = new CountDownLatch(1);
+ var entered = new CountDownLatch(1);
+ var callerThread = new AtomicReference();
+ var bodyThread = new AtomicReference();
+ var endThread = new AtomicReference();
+ var workers = new RootExecutor(mode, () -> {});
+ var caller = Executors.newSingleThreadExecutor(task -> {
+ var thread = new Thread(task, "normal-outcome-caller");
+ callerThread.set(thread);
+ return thread;
+ });
+ var config = DurableConfig.builder()
+ .withExecutorService(workers)
+ .withDurableExecutionClient(new LocalMemoryExecutionClient())
+ .withPlugins(new DurableExecutionPlugin() {
+ public void onInvocationEnd(InvocationEndInfo info) {
+ endThread.set(Thread.currentThread());
+ }
+ })
+ .build();
+ try {
+ var call = caller.submit(() -> DurableExecutor.execute(
+ input(List.of()),
+ null,
+ TypeToken.get(String.class),
+ (value, context) -> {
+ bodyThread.set(Thread.currentThread());
+ entered.countDown();
+ if (mode.equals("async")) {
+ try {
+ assertTrue(releaseBody.await(3, TimeUnit.SECONDS));
+ } catch (InterruptedException interrupted) {
+ throw new AssertionError(interrupted);
+ }
+ }
+ return "done";
+ },
+ config));
+ assertTrue(entered.await(3, TimeUnit.SECONDS));
+ if (mode.equals("async")) awaitCallerOutcomeJoin(callerThread.get());
+ releaseBody.countDown();
+ assertEquals(
+ ExecutionStatus.SUCCEEDED, call.get(3, TimeUnit.SECONDS).status());
+ assertSame(bodyThread.get(), endThread.get());
+ } finally {
+ releaseBody.countDown();
+ caller.shutdownNow();
+ workers.shutdownNow();
+ assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS));
+ assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ /** Wait for the actual public invocation caller to install its outcome observer and enter join, not a delay. */
+ private static void awaitCallerOutcomeJoin(Thread caller) {
+ var deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3);
+ while (System.nanoTime() < deadline) {
+ if (Arrays.stream(caller.getStackTrace())
+ .anyMatch(frame -> frame.getClassName().equals(CompletableFuture.class.getName())
+ && frame.getMethodName().equals("join"))) return;
+ Thread.yield();
+ }
+ fail("Invocation caller did not reach its installed outcome observer");
+ }
+
+ /** Root dispatch exercises inline/submit-and-wait; operation children remain asynchronous as supported. */
+ private static final class RootExecutor extends ThreadPoolExecutor {
+ private final String mode;
+ private final Runnable beforeDispatch;
+ private final AtomicBoolean root = new AtomicBoolean(true);
+
+ RootExecutor(String mode, Runnable beforeDispatch) {
+ super(0, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue<>());
+ this.mode = mode;
+ this.beforeDispatch = beforeDispatch;
+ }
+
+ void nextInvocation() {
+ root.set(true);
+ }
+
+ @Override
+ public void execute(Runnable task) {
+ beforeDispatch.run();
+ if (root.compareAndSet(true, false)) {
+ if (mode.equals("inline-root")) {
+ task.run();
+ return;
+ }
+ if (mode.equals("submit-root")) {
+ var done = new CompletableFuture();
+ super.execute(() -> {
+ try {
+ task.run();
+ } finally {
+ done.complete(null);
+ }
+ });
+ try {
+ done.get(5, TimeUnit.SECONDS);
+ } catch (Exception failure) {
+ throw new AssertionError(failure);
+ }
+ return;
+ }
+ }
+ super.execute(task);
+ }
+ }
+
+ private static DurableExecutionInput input(List operations) {
+ var execution = Operation.builder()
+ .id("execution")
+ .type(OperationType.EXECUTION)
+ .status(OperationStatus.STARTED)
+ .startTimestamp(Instant.EPOCH)
+ .executionDetails(
+ ExecutionDetails.builder().inputPayload("\"input\"").build())
+ .build();
+ var history = new ArrayList();
+ history.add(execution);
+ history.addAll(operations);
+ return new DurableExecutionInput(
+ "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/fatal/execution",
+ "token",
+ CheckpointUpdatedExecutionState.builder().operations(history).build());
+ }
+}
diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/MapIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/MapIntegrationTest.java
index 7d2b12982..65dbe2383 100644
--- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/MapIntegrationTest.java
+++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/MapIntegrationTest.java
@@ -7,8 +7,11 @@
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicIntegerArray;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
@@ -844,8 +847,10 @@ void testNestedMap_replay(NestingType outerNestingType, NestingType innerNesting
@CsvSource({"FLAT, 2", "NESTED, 16"})
void testMapWithToleratedFailurePercentage(NestingType nestingType, int events) {
var runner = LocalDurableTestRunner.create(String.class, (input, context) -> {
- var items = List.of("ok1", "FAIL1", "ok2", "FAIL2", "ok3", "FAIL3", "ok4");
+ // Put the threshold-crossing failure last and serialize admission so exact history counts are stable.
+ var items = List.of("ok1", "ok2", "ok3", "ok4", "FAIL1", "FAIL2", "FAIL3");
var config = MapConfig.builder()
+ .maxConcurrency(1)
.completionConfig(CompletionConfig.toleratedFailurePercentage(0.3))
.nestingType(nestingType)
.build();
@@ -876,8 +881,10 @@ void testMapWithToleratedFailurePercentage_replay(NestingType nestingType, int e
var executionCount = new AtomicInteger(0);
var runner = LocalDurableTestRunner.create(String.class, (input, context) -> {
- var items = List.of("ok1", "FAIL1", "ok2", "FAIL2", "ok3", "FAIL3", "ok4");
+ // Put the threshold-crossing failure last and serialize admission so exact history counts are stable.
+ var items = List.of("ok1", "ok2", "ok3", "ok4", "FAIL1", "FAIL2", "FAIL3");
var config = MapConfig.builder()
+ .maxConcurrency(1)
.completionConfig(CompletionConfig.toleratedFailurePercentage(0.3))
.nestingType(nestingType)
.build();
@@ -901,9 +908,10 @@ void testMapWithToleratedFailurePercentage_replay(NestingType nestingType, int e
var result1 = runner.runUntilComplete("test");
assertEquals(ExecutionStatus.SUCCEEDED, result1.getStatus());
var firstRunCount = executionCount.get();
+ assertEquals(7, firstRunCount);
assertEquals(events, result1.getHistoryEvents().size());
- // Replay — with unlimited concurrency, children replay simultaneously.
+ // Replay retains the checkpointed completion decision and every previously completed child.
// Verify completionReason is consistent and no re-execution occurs.
var result2 = runner.run("test");
assertEquals(ExecutionStatus.SUCCEEDED, result2.getStatus());
@@ -911,6 +919,87 @@ void testMapWithToleratedFailurePercentage_replay(NestingType nestingType, int e
assertEquals(events, result2.getHistoryEvents().size());
}
+ @ParameterizedTest
+ @CsvSource({"FLAT", "NESTED"})
+ void testDefaultConcurrencyPercentageFailureReplaysStoredResults(NestingType nestingType) {
+ var items = List.of("ok1", "FAIL1", "ok2", "FAIL2", "ok3", "FAIL3", "ok4");
+ var executions = new AtomicIntegerArray(items.size());
+ var runner = percentageReplayRunner(nestingType, items, executions);
+ var resultType = new TypeToken>() {};
+
+ var first = runner.runUntilComplete("test");
+ assertEquals(ExecutionStatus.SUCCEEDED, first.getStatus());
+ var stored = first.getResult(resultType);
+ assertPercentageFailureResults(items, stored);
+ for (var index = 0; index < items.size(); index++) assertEquals(1, executions.get(index));
+
+ var replay = runner.run("test");
+ assertEquals(ExecutionStatus.SUCCEEDED, replay.getStatus());
+ assertEquals(stored, replay.getResult(resultType), "Replay returns the exact checkpointed result and errors");
+ assertEquals(first.getHistoryEvents().size(), replay.getHistoryEvents().size());
+ for (var index = 0; index < items.size(); index++) {
+ assertEquals(1, executions.get(index), "Completed child bodies must not repeat on replay");
+ }
+ }
+
+ private static LocalDurableTestRunner> percentageReplayRunner(
+ NestingType nestingType, List items, AtomicIntegerArray executions) {
+ var allStarted = new CountDownLatch(items.size());
+ return LocalDurableTestRunner.create(
+ String.class,
+ (input, context) -> context.map(
+ "default-percentage-replay",
+ items,
+ String.class,
+ (item, index, child) -> {
+ executions.incrementAndGet(index);
+ allStarted.countDown();
+ try {
+ assertTrue(
+ allStarted.await(5, TimeUnit.SECONDS),
+ "Default concurrency must admit all children");
+ } catch (InterruptedException error) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(error);
+ }
+ if (item.startsWith("FAIL")) throw new IllegalArgumentException("failed: " + item);
+ return item.toUpperCase();
+ },
+ MapConfig.builder()
+ .completionConfig(CompletionConfig.toleratedFailurePercentage(0.3))
+ .nestingType(nestingType)
+ .build()));
+ }
+
+ private static void assertPercentageFailureResults(List inputs, MapResult result) {
+ assertEquals(ConcurrencyCompletionStatus.FAILURE_TOLERANCE_EXCEEDED, result.completionReason());
+ assertEquals(inputs.size(), result.size());
+ assertEquals(3, result.failed().size(), "The third failure crosses the 30 percent threshold");
+ assertTrue(result.failed().size() > inputs.size() * 0.3);
+ for (var index = 0; index < inputs.size(); index++) {
+ var item = result.getItem(index);
+ switch (item.status()) {
+ case SUCCEEDED -> {
+ assertFalse(inputs.get(index).startsWith("FAIL"));
+ assertEquals(inputs.get(index).toUpperCase(), item.result());
+ assertNull(item.error());
+ }
+ case FAILED -> {
+ assertTrue(inputs.get(index).startsWith("FAIL"));
+ assertNull(item.result());
+ assertNotNull(item.error());
+ assertTrue(item.error().errorMessage().contains("failed: " + inputs.get(index)));
+ }
+ case SKIPPED -> {
+ // A success racing with the threshold may be absent from the terminal map snapshot.
+ assertFalse(inputs.get(index).startsWith("FAIL"));
+ assertNull(item.result());
+ assertNull(item.error());
+ }
+ }
+ }
+ }
+
@ParameterizedTest
@CsvSource({"FLAT, 12", "NESTED, 16"})
void testMapAsyncWithWaitInsideBranches(NestingType nestingType, int events) {
diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java
index 8b97b9e27..5ccfa37d3 100644
--- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java
+++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java
@@ -169,6 +169,7 @@ private void fireOnInvocationStart() {
private DurableExecutionOutput finishInvocation(O value, Throwable failure) {
if (failure != null) return finishFailure(ExceptionHelper.unwrapCompletableFuture(failure));
+ manager.closeCheckpointContinuationAdmission();
DurableExecutionOutput output;
try {
var payload = config.getSerDes().serialize(value);
diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java
index 1aa148c2a..df687ee11 100644
--- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java
+++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java
@@ -736,6 +736,13 @@ public void close() {
checkpointManager.shutdown();
}
+ /** Closes new checkpoint-continuation admission without signaling owners or joining their cleanup. */
+ void closeCheckpointContinuationAdmission() {
+ synchronized (activeThreads) {
+ closing = true;
+ }
+ }
+
private void stopCheckpointContinuations() {
List owners;
synchronized (activeThreads) {