Skip to content

Commit 109960d

Browse files
committed
Fix: propagate per-run runtime failures
Read each completed a2a3 run's existing orchestration and scheduler status latch after its event fence. A nonzero status now enters the same runner recovery, validation logging, and force-reset path that stream sync previously triggered, without waiting for a queued successor.
1 parent 725e67f commit 109960d

4 files changed

Lines changed: 21 additions & 5 deletions

File tree

src/a2a3/platform/onboard/host/device_runner.cpp

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -632,11 +632,6 @@ int DeviceRunner::wait_run_completion_events(const PreparedExecution &prepared)
632632
return static_cast<int>(rc);
633633
}
634634
}
635-
// Event completion does not propagate a failed predecessor task. A stream
636-
// query preserves that sticky error while NOT_READY remains valid when a
637-
// healthy successor is queued after this run's event.
638-
const int stream_status = query_stream_pair_nonblocking(run_streams_.aicpu(), run_streams_.aicore());
639-
if (stream_status == SIMPLER_NATIVE_RUN_POLL_ERROR) return PTO_RUNTIME_ERR_INTERNAL;
640635
return 0;
641636
}
642637

src/a2a3/runtime/host_build_graph/host/runtime_maker.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,11 @@ static int32_t read_runtime_status(Runtime *runtime, const HostApi *api, SharedM
275275
return runtime_status_from_error_codes(orch_error_code, sched_error_code);
276276
}
277277

278+
extern "C" int completed_runtime_status_impl(Runtime *runtime, const HostApi *api) {
279+
SharedMemoryHeader host_header{};
280+
return read_runtime_status(runtime, api, &host_header);
281+
}
282+
278283
namespace {
279284

280285
// host_build_graph is host-orchestration-first: the HOST dlopens the

src/a2a3/runtime/tensormap_and_ringbuffer/host/runtime_maker.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,11 @@ static int32_t read_runtime_status(Runtime *runtime, const HostApi *api, SharedM
221221
return runtime_status_from_error_codes(orch_error_code, sched_error_code);
222222
}
223223

224+
extern "C" int completed_runtime_status_impl(Runtime *runtime, const HostApi *api) {
225+
SharedMemoryHeader host_header{};
226+
return read_runtime_status(runtime, api, &host_header);
227+
}
228+
224229
static void release_tensor_leases(Runtime *runtime, const HostApi *api) {
225230
int freed = 0;
226231
int buffer_noop = 0;

src/common/platform/onboard/host/c_api_shared.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ extern "C" {
6666
* =========================================================================== */
6767
int register_callable_impl(const ChipCallable *callable, const HostApi *api, CallableArtifacts *out);
6868
int validate_runtime_impl(Runtime *runtime, const HostApi *api, int execution_rc);
69+
__attribute__((weak)) int completed_runtime_status_impl(Runtime * /*runtime*/, const HostApi * /*api*/) { return 0; }
6970
__attribute__((weak)) int concurrent_native_prepare_supported_impl(void) { return 0; }
7071
__attribute__((weak)) int prepared_run_config_compatible_impl(
7172
const HostApi * /*api*/, const uint64_t * /*ring_task_window*/, const uint64_t * /*ring_heap*/,
@@ -643,6 +644,14 @@ static void emit_native_run_runner_wall(OnboardNativeRunContext *state) {
643644
state->runner_trace_start_ns = 0;
644645
}
645646

647+
static int completed_execution_rc(OnboardNativeRunContext *state, int drain_rc) {
648+
if (drain_rc != 0) return drain_rc;
649+
const int runtime_status = completed_runtime_status_impl(&state->runtime, &state->host_api);
650+
if (runtime_status == 0) return 0;
651+
state->runner->recover_device_or_mark_unusable(runtime_status);
652+
return runtime_status;
653+
}
654+
646655
int supports_concurrent_native_prepare_ctx(DeviceContextHandle ctx) {
647656
return ctx != nullptr && concurrent_native_prepare_supported_impl() != 0 ? 1 : 0;
648657
}
@@ -937,6 +946,7 @@ int simpler_wait_run(DeviceContextHandle ctx, RuntimeHandle runtime) {
937946
drain_rc = PTO_RUNTIME_ERR_INTERNAL;
938947
LOG_ERROR("simpler_wait_run: drain threw (%s)", state->trace_attrs);
939948
}
949+
drain_rc = completed_execution_rc(state, drain_rc);
940950
if (state->completion_rc == 0) state->completion_rc = drain_rc;
941951
state->phase.store(NativeRunPhase::Complete, std::memory_order_release);
942952
emit_native_run_runner_wall(state);
@@ -985,6 +995,7 @@ int simpler_finalize_run(DeviceContextHandle ctx, RuntimeHandle runtime) {
985995
LOG_ERROR("simpler_finalize_run: drain_execution threw (%s)", state->trace_attrs);
986996
}
987997
}
998+
drain_rc = completed_execution_rc(state, drain_rc);
988999
if (execution_rc == 0) execution_rc = drain_rc;
9891000
state->completion_rc = execution_rc;
9901001
state->phase.store(NativeRunPhase::Complete, std::memory_order_release);

0 commit comments

Comments
 (0)