From a8c1a775bfe646bbb37011a708f760aa8906d9e3 Mon Sep 17 00:00:00 2001 From: Laurent VOLFF Date: Sun, 18 Oct 2026 11:48:08 +0200 Subject: [PATCH 1/5] instrument: log per-segment capacity verdicts, device memory, and staging residency for #1976 OOM investigation (VERBOSE only, no behavior change) --- src/core/ggml_runner.cpp | 12 ++++++++++++ src/core/segment_weight_pipeline.cpp | 3 +++ src/model_manager.cpp | 12 ++++++++++++ 3 files changed, 27 insertions(+) diff --git a/src/core/ggml_runner.cpp b/src/core/ggml_runner.cpp index 4c156ed43d..5b131c1572 100644 --- a/src/core/ggml_runner.cpp +++ b/src/core/ggml_runner.cpp @@ -983,6 +983,18 @@ std::optional> GGMLRunner::execute_graph(ggml_cgraph* graph, int n } LOG_DEBUG("%s executing segment %zu/%zu: %s", get_desc().c_str(), index + 1, plan.segments.size(), segment.group_name.c_str()); + { + size_t dev_free_bytes = 0; + size_t dev_total_bytes = 0; + auto dev = ggml_backend_get_device(runtime_backend); + if (dev != nullptr) { + ggml_backend_dev_memory(dev, &dev_free_bytes, &dev_total_bytes); + } + LOG_VERBOSE("%s segment %zu/%zu start on %s: device reports %.2f MB free / %.2f MB total", + get_desc().c_str(), index + 1, plan.segments.size(), + ggml_backend_name(runtime_backend), + dev_free_bytes / (1024.0 * 1024.0), dev_total_bytes / (1024.0 * 1024.0)); + } if (!execute_segment(segment_graph, n_threads)) { return fail_segment("execution"); } diff --git a/src/core/segment_weight_pipeline.cpp b/src/core/segment_weight_pipeline.cpp index f769794df3..7aa6a950a3 100644 --- a/src/core/segment_weight_pipeline.cpp +++ b/src/core/segment_weight_pipeline.cpp @@ -3,6 +3,7 @@ #include #include "core/ggml_graph_cut.h" +#include "core/util.h" #include "device_residency_manager.h" namespace sd { @@ -175,6 +176,8 @@ namespace sd { params, preferred_eviction_order(), protected_params)) { + LOG_VERBOSE("segment weight pipeline: prefetch capacity check failed for segment %zu, disabling prefetch", + next_segment); disable(); return; } diff --git a/src/model_manager.cpp b/src/model_manager.cpp index 4562ebdfd4..45d603c7e7 100644 --- a/src/model_manager.cpp +++ b/src/model_manager.cpp @@ -1780,6 +1780,11 @@ ModelManager::CapacityCheck ModelManager::check_capacity( const size_t weights_resident = compute_backend_resident_bytes(backend); const size_t other_runtime = other_runtime_resident_bytes(request.owner_id, backend); const size_t resident = add(weights_resident, add(other_runtime, request.runtime_resident_bytes)); + LOG_VERBOSE("model manager memory on %s: reported free %.2f MB / total %.2f MB, tracked weights %.2f MB / other runtime %.2f MB / current runtime %.2f MB", + ggml_backend_name(backend), + free_bytes / (1024.0 * 1024.0), total_bytes / (1024.0 * 1024.0), + weights_resident / (1024.0 * 1024.0), other_runtime / (1024.0 * 1024.0), + request.runtime_resident_bytes / (1024.0 * 1024.0)); if (log_details) { LOG_WARN("model manager memory on %s: reported free %.2f MB / total %.2f MB, tracked weights %.2f MB / other runtime %.2f MB / current runtime %.2f MB", ggml_backend_name(backend), @@ -1838,6 +1843,13 @@ ModelManager::CapacityCheck ModelManager::check_capacity( result.available_budget_bytes = std::min(result.available_budget_bytes, available); } } + LOG_VERBOSE("capacity check verdict on %s: required device %.2f MB / available %.2f MB, required budget %.2f MB / available %.2f MB, fits %s", + ggml_backend_name(request.compute_backend), + result.required_device_bytes / (1024.0 * 1024.0), + result.available_device_bytes / (1024.0 * 1024.0), + result.required_budget_bytes / (1024.0 * 1024.0), + result.available_budget_bytes / (1024.0 * 1024.0), + result.fits() ? "yes" : "no"); return result; } From d4a2e9a57b0a5d8450067bd5949e8155cdf760be Mon Sep 17 00:00:00 2001 From: Laurent VOLFF Date: Sun, 18 Oct 2026 12:18:54 +0200 Subject: [PATCH 2/5] instrument: add host-side memory snapshot (avail phys/commit, process commit/ws, cut-cache bytes) per segment + crash-context catch --- src/core/ggml_runner.cpp | 54 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/src/core/ggml_runner.cpp b/src/core/ggml_runner.cpp index 5b131c1572..8c40540e47 100644 --- a/src/core/ggml_runner.cpp +++ b/src/core/ggml_runner.cpp @@ -3,6 +3,12 @@ #include #include +#ifdef _WIN32 + #define NOMINMAX + #include + #include +#endif + #include "core/ggml_extend.h" #include "core/ggml_extend_backend.h" #include "core/ggml_runner.h" @@ -990,13 +996,53 @@ std::optional> GGMLRunner::execute_graph(ggml_cgraph* graph, int n if (dev != nullptr) { ggml_backend_dev_memory(dev, &dev_free_bytes, &dev_total_bytes); } - LOG_VERBOSE("%s segment %zu/%zu start on %s: device reports %.2f MB free / %.2f MB total", + size_t cpu_free_bytes = 0; + size_t cpu_total_bytes = 0; + if (workspace_.cpu_backend() != nullptr) { + if (auto cpu_dev = ggml_backend_get_device(workspace_.cpu_backend()); cpu_dev != nullptr) { + ggml_backend_dev_memory(cpu_dev, &cpu_free_bytes, &cpu_total_bytes); + } + } + const double cut_mb = + (dev != nullptr ? cut_cache_.resident_bytes(dev) : 0) / (1024.0 * 1024.0); +#ifdef _WIN32 + MEMORYSTATUSEX mem_status{}; + PROCESS_MEMORY_COUNTERS_EX proc_counters{}; + mem_status.dwLength = sizeof(mem_status); + proc_counters.cb = sizeof(proc_counters); + const bool have_status = GlobalMemoryStatusEx(&mem_status) != 0; + const bool have_proc = GetProcessMemoryInfo(GetCurrentProcess(), + reinterpret_cast(&proc_counters), + sizeof(proc_counters)) != 0; + LOG_VERBOSE("%s segment %zu/%zu start on %s: device reports %.2f MB free / %.2f MB total" + " | host: avail phys %.2f MB, avail commit %.2f MB, proc commit %.2f MB, proc ws %.2f MB" + " | CPU device %.2f MB free / %.2f MB total | cut cache %.2f MB", get_desc().c_str(), index + 1, plan.segments.size(), ggml_backend_name(runtime_backend), - dev_free_bytes / (1024.0 * 1024.0), dev_total_bytes / (1024.0 * 1024.0)); + dev_free_bytes / (1024.0 * 1024.0), dev_total_bytes / (1024.0 * 1024.0), + have_status ? mem_status.ullAvailPhys / (1024.0 * 1024.0) : -1.0, + have_status ? mem_status.ullAvailPageFile / (1024.0 * 1024.0) : -1.0, + have_proc ? proc_counters.PrivateUsage / (1024.0 * 1024.0) : -1.0, + have_proc ? proc_counters.WorkingSetSize / (1024.0 * 1024.0) : -1.0, + cpu_free_bytes / (1024.0 * 1024.0), cpu_total_bytes / (1024.0 * 1024.0), cut_mb); +#else + LOG_VERBOSE("%s segment %zu/%zu start on %s: device reports %.2f MB free / %.2f MB total" + " | CPU device %.2f MB free / %.2f MB total | cut cache %.2f MB", + get_desc().c_str(), index + 1, plan.segments.size(), + ggml_backend_name(runtime_backend), + dev_free_bytes / (1024.0 * 1024.0), dev_total_bytes / (1024.0 * 1024.0), + cpu_free_bytes / (1024.0 * 1024.0), cpu_total_bytes / (1024.0 * 1024.0), cut_mb); +#endif } - if (!execute_segment(segment_graph, n_threads)) { - return fail_segment("execution"); + try { + if (!execute_segment(segment_graph, n_threads)) { + return fail_segment("execution"); + } + } catch (const std::exception& error) { + LOG_VERBOSE("%s segment %zu/%zu (%s) raised during execution: %s", + get_desc().c_str(), index + 1, plan.segments.size(), + segment.group_name.c_str(), error.what()); + throw; } auto cache_status = cache_.capture(segment_graph); if (cache_status == GGML_STATUS_SUCCESS) { From 16068f9ee09d0871385752e6f3f8bdf4a2cfa8ec Mon Sep 17 00:00:00 2001 From: Laurent VOLFF Date: Sun, 18 Oct 2026 19:43:06 +0200 Subject: [PATCH 3/5] fix: account host-pinned parameter memory by placement in capacity checks (#1976) Offloaded models (params backend != compute backend) load params lazily into the compute device's host-pinned buffer type (Vulkan_Host, pinned system memory). That pool is invisible to the compute device's heap report, so capacity checks stayed green while host availability collapsed and the Vulkan submit path ran out of system memory (#1976: submit OOM at segment 38/54 with the device heap 84% free and host availability at ~0.5 GB). The host dimension of CapacityCheck counts only bytes this request would allocate through host-visible buffer types: unloaded params whose params buffer type is host-pinned, plus staging bytes when the compute backend's default buffer type is host-visible. Device-local staging and prefetch remain on the device/budget dimensions, so device-local allocations are never rejected based on system RAM. A host failure feeds the existing eviction path, which reclaims params storage blocks and staged replicas. Windows-scoped via GlobalMemoryStatusEx; Linux needs the sysinfo equivalent. --- src/core/segment_weight_pipeline.cpp | 7 +++ src/model_manager.cpp | 79 +++++++++++++++++++++++++++- src/model_manager.h | 5 +- 3 files changed, 89 insertions(+), 2 deletions(-) diff --git a/src/core/segment_weight_pipeline.cpp b/src/core/segment_weight_pipeline.cpp index 7aa6a950a3..dfded03915 100644 --- a/src/core/segment_weight_pipeline.cpp +++ b/src/core/segment_weight_pipeline.cpp @@ -68,15 +68,22 @@ namespace sd { void SegmentWeightPipeline::activate(size_t segment_index) { if (!enabled_ || queued_segment_ == SIZE_MAX || queued_segment_ != segment_index) { + if (enabled_ && queued_segment_ != SIZE_MAX) { + LOG_VERBOSE("segment weight pipeline: segment %zu started without its prefetched block (queued for segment %zu)", + segment_index, queued_segment_); + } return; } auto manager = residency_manager_.lock(); if (manager == nullptr || !manager->activate_prefetched_params(owner_id_, queued_params_)) { + LOG_VERBOSE("segment weight pipeline: failed to activate the prefetched block for segment %zu, disabling prefetch", + segment_index); disable(); return; } + LOG_VERBOSE("segment weight pipeline: segment %zu activated its prefetched weight block", segment_index); queued_params_.clear(); queued_segment_ = SIZE_MAX; } diff --git a/src/model_manager.cpp b/src/model_manager.cpp index 45d603c7e7..1d0eb054aa 100644 --- a/src/model_manager.cpp +++ b/src/model_manager.cpp @@ -7,6 +7,11 @@ #include #include +#ifdef _WIN32 + #define NOMINMAX + #include +#endif + #include "core/ggml_extend_backend.h" #include "core/util.h" #include "model/adapter/lora.hpp" @@ -17,6 +22,20 @@ static size_t aligned_offset(const void* buffer, size_t offset, size_t alignment return offset + align; } +// True when allocations through this buffer type consume host-visible memory: +// plain host buffer types, or a compute device's host-pinned pool (e.g. the +// Vulkan "Vulkan_Host" buffer type backed by pinned system memory). +static bool model_manager_buffer_type_is_host_pinned(ggml_backend_buffer_type_t buft) { + if (buft == nullptr) { + return false; + } + if (ggml_backend_buft_is_host(buft)) { + return true; + } + ggml_backend_dev_t device = ggml_backend_buft_get_device(buft); + return device != nullptr && buft == ggml_backend_dev_host_buffer_type(device); +} + static bool lora_specs_equal(const std::vector& lhs, const std::vector& rhs) { if (lhs.size() != rhs.size()) { @@ -1843,12 +1862,70 @@ ModelManager::CapacityCheck ModelManager::check_capacity( result.available_budget_bytes = std::min(result.available_budget_bytes, available); } } - LOG_VERBOSE("capacity check verdict on %s: required device %.2f MB / available %.2f MB, required budget %.2f MB / available %.2f MB, fits %s", + // Host dimension (#1976): memory allocated through host-visible buffer + // types is invisible to the compute device's heap report. With an + // offloaded model (params backend != compute backend), params buffers + // take the compute device's host-pinned buffer type (e.g. Vulkan_Host) + // and are loaded lazily, so they accumulate on the host while every + // device-side gate stays green. Account for the actual placement: only + // bytes this request would allocate through host-visible buffer types + // count against host availability; device-local staging/prefetch stay on + // the device/budget dimensions. A host failure feeds the existing + // eviction path, which reclaims params storage blocks and staged replicas. + size_t host_missing_bytes = 0; + bool host_pool_in_use = false; + for (TensorState* state : states) { + if (state == nullptr) { + continue; + } + if (model_manager_buffer_type_is_host_pinned(params_buffer_type_for(*state))) { + host_pool_in_use = true; + if (!state->loaded_to_params_backend && state->tensor != nullptr) { + host_missing_bytes += ggml_nbytes(state->tensor); + } + } + } + if (missing > 0 && model_manager_buffer_type_is_host_pinned( + ggml_backend_get_default_buffer_type(request.compute_backend))) { + host_missing_bytes += missing; + host_pool_in_use = true; + } + if (host_pool_in_use) { + // The +1 byte enforces the safety floor itself — once the host-pinned + // pool is in use, a request that pins nothing new must still fail + // when availability has fallen to zero, so reclamation runs before + // the driver's allocations start failing. + result.required_host_bytes = host_missing_bytes + 1; +#ifdef _WIN32 + MEMORYSTATUSEX status{}; + status.dwLength = sizeof(status); + if (GlobalMemoryStatusEx(&status)) { + // Two thresholds, from measurements on the #1976 machine (32 GB + // RAM, both resident models loaded): the driver fails allocations + // once available physical memory drops below ~0.5-0.8 GB (512 MiB + // + request bytes = hard refusal region), while up to ~0.5 GB is + // consumed between a capacity check and the following submit + // (1 GiB = proactive reclamation watermark so eviction keeps the + // pool above the failing region instead of only reacting at it). + constexpr uint64_t host_safety_margin = 512ULL * 1024ULL * 1024ULL; + constexpr uint64_t host_reclaim_watermark = 1ULL * 1024ULL * 1024ULL * 1024ULL; + result.available_host_bytes = status.ullAvailPhys > host_safety_margin + ? static_cast(status.ullAvailPhys - host_safety_margin) + : 0; + if (status.ullAvailPhys < host_reclaim_watermark) { + result.requires_reclamation = true; + } + } +#endif + } + LOG_VERBOSE("capacity check verdict on %s: required device %.2f MB / available %.2f MB, required budget %.2f MB / available %.2f MB, required host %.2f MB / available %.2f MB, fits %s", ggml_backend_name(request.compute_backend), result.required_device_bytes / (1024.0 * 1024.0), result.available_device_bytes / (1024.0 * 1024.0), result.required_budget_bytes / (1024.0 * 1024.0), result.available_budget_bytes / (1024.0 * 1024.0), + result.required_host_bytes / (1024.0 * 1024.0), + result.available_host_bytes / (1024.0 * 1024.0), result.fits() ? "yes" : "no"); return result; } diff --git a/src/model_manager.h b/src/model_manager.h index fdb110825d..5798be72d6 100644 --- a/src/model_manager.h +++ b/src/model_manager.h @@ -150,12 +150,15 @@ class ModelManager : public DeviceResidencyManager { struct CapacityCheck { size_t required_device_bytes = 0; size_t required_budget_bytes = 0; + size_t required_host_bytes = 0; size_t available_device_bytes = SIZE_MAX; size_t available_budget_bytes = SIZE_MAX; + size_t available_host_bytes = SIZE_MAX; bool requires_reclamation = false; bool fits() const { return required_device_bytes <= available_device_bytes && - required_budget_bytes <= available_budget_bytes; + required_budget_bytes <= available_budget_bytes && + required_host_bytes <= available_host_bytes; } }; CapacityCheck check_capacity(const DeviceMemoryRequest& request, From ed5b5f842c41996b41e153e12da335dc0ca2ab6e Mon Sep 17 00:00:00 2001 From: Laurent VOLFF Date: Sun, 18 Oct 2026 23:34:03 +0200 Subject: [PATCH 4/5] review: host accounting hardening per CodeRabbit (#2166) - Skip ignored/optional-missing states in the host loop and probe placement only for unloaded tensors (the operator probe rejects loaded ones); detect the in-use host pool from existing host-pinned params storage blocks instead. - Demote host-pinned offloaded ParamBackend storage to Disk residency under host pressure so eviction can actually reclaim the host pool (previously only Disk-mode blocks were releasable and only device-resident params were demotable). - The final fallback now honors requires_reclamation, and the refusal warning prints the host dimension. --- src/model_manager.cpp | 78 ++++++++++++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 19 deletions(-) diff --git a/src/model_manager.cpp b/src/model_manager.cpp index 1d0eb054aa..2d23f792a2 100644 --- a/src/model_manager.cpp +++ b/src/model_manager.cpp @@ -1874,15 +1874,26 @@ ModelManager::CapacityCheck ModelManager::check_capacity( // eviction path, which reclaims params storage blocks and staged replicas. size_t host_missing_bytes = 0; bool host_pool_in_use = false; + // Existing host-pinned parameter storage keeps the floor active even when + // this request pins nothing new. + for (const auto& block : params_storage_blocks_) { + if (block != nullptr && block->buffer != nullptr && + model_manager_buffer_type_is_host_pinned(ggml_backend_buffer_get_type(block->buffer))) { + host_pool_in_use = true; + break; + } + } + // Placement probing is only valid for unloaded tensors (the operator + // probe rejects loaded ones); ignored and optional-missing states never + // load, so they must not be counted as missing host capacity. for (TensorState* state : states) { - if (state == nullptr) { + if (state == nullptr || should_ignore(*state) || is_optional_missing_tensor(state->name)) { continue; } - if (model_manager_buffer_type_is_host_pinned(params_buffer_type_for(*state))) { - host_pool_in_use = true; - if (!state->loaded_to_params_backend && state->tensor != nullptr) { - host_missing_bytes += ggml_nbytes(state->tensor); - } + if (!state->loaded_to_params_backend && state->tensor != nullptr && + model_manager_buffer_type_is_host_pinned(params_buffer_type_for(*state))) { + host_missing_bytes += ggml_nbytes(state->tensor); + host_pool_in_use = true; } } if (missing > 0 && model_manager_buffer_type_is_host_pinned( @@ -2063,17 +2074,37 @@ bool ModelManager::ensure_compute_backend_capacity( } } }; + // Offloaded parameter storage can also sit in host-pinned memory (the + // compute device's host buffer type); under host pressure those blocks + // must be reclaimable too, or only device staging gets evicted while the + // host pool cannot recover. + std::unordered_set host_pinned_states; + for (const auto& block : params_storage_blocks_) { + if (block != nullptr && block->buffer != nullptr && + model_manager_buffer_type_is_host_pinned(ggml_backend_buffer_get_type(block->buffer))) { + host_pinned_states.insert(block->states.begin(), block->states.end()); + } + } auto can_demote_to_disk = [&](TensorState* state) { - return state != nullptr && state->tensor != nullptr && state->tensor->view_src == nullptr && - state->residency_mode == ResidencyMode::ParamBackend && - state->params_backend != nullptr && state->params_backend == state->compute_backend && - state->compute_backend == compute_backend && !sd_backend_is_cpu(state->params_backend) && - state->pin_count == 0 && !state->staged_to_compute_backend && state->loaded_to_params_backend && - protected_states.find(state) == protected_states.end() && - required_set.find(state) == required_set.end() && - mapped_states.find(state) == mapped_states.end() && - !state->sources.empty() && state->applied_lora_epoch == UINT64_MAX && - state->split_buffer_type == nullptr; + if (state == nullptr || state->tensor == nullptr || state->tensor->view_src != nullptr || + state->residency_mode != ResidencyMode::ParamBackend || + state->pin_count != 0 || state->staged_to_compute_backend || !state->loaded_to_params_backend || + protected_states.find(state) != protected_states.end() || + required_set.find(state) != required_set.end() || + state->sources.empty() || state->applied_lora_epoch != UINT64_MAX || + state->split_buffer_type != nullptr) { + return false; + } + const bool device_resident = state->params_backend != nullptr && + state->params_backend == state->compute_backend && + state->compute_backend == compute_backend && + !sd_backend_is_cpu(state->params_backend) && + mapped_states.find(state) == mapped_states.end(); + // Host-pinned parameter storage of an offloaded model reloads from + // the model loader like Disk-resident states; mmap-backed blocks are + // still freed (the pinned copy is a separate allocation). + const bool host_pinned = host_pinned_states.find(state) != host_pinned_states.end(); + return device_resident || host_pinned; }; for (TensorState* state : global_candidates) { if (!can_demote_to_disk(state)) { @@ -2089,7 +2120,8 @@ bool ModelManager::ensure_compute_backend_capacity( } revert_demotions(); - if (check_capacity(request, required_states).fits()) { + const auto final_capacity = check_capacity(request, required_states); + if (!final_capacity.requires_reclamation && final_capacity.fits()) { return true; } const auto capacity = check_capacity(request, required_states, true); @@ -2099,11 +2131,19 @@ bool ModelManager::ensure_compute_backend_capacity( const std::string available_budget = capacity.available_budget_bytes == SIZE_MAX ? "unlimited" : sd_format("%.2f MB", capacity.available_budget_bytes / (1024.0 * 1024.0)); - LOG_WARN("model manager cannot make enough memory available on %s: need %.2f MB device / %.2f MB budget, available %s device / %s budget", + const std::string required_host = capacity.required_host_bytes == 0 + ? "none" + : sd_format("%.2f MB", capacity.required_host_bytes / (1024.0 * 1024.0)); + const std::string available_host = capacity.available_host_bytes == SIZE_MAX + ? "unknown" + : sd_format("%.2f MB", capacity.available_host_bytes / (1024.0 * 1024.0)); + LOG_WARN("model manager cannot make enough memory available on %s: need %.2f MB device / %.2f MB budget / %s host, available %s device / %s budget / %s host", ggml_backend_name(compute_backend), capacity.required_device_bytes / (1024.0 * 1024.0), capacity.required_budget_bytes / (1024.0 * 1024.0), + required_host.c_str(), available_device.c_str(), - available_budget.c_str()); + available_budget.c_str(), + available_host.c_str()); return false; } From 713e5bc5a6165d4d6e11c606bb4f3b3ebdc1cf2f Mon Sep 17 00:00:00 2001 From: Laurent VOLFF Date: Sun, 18 Oct 2026 23:52:55 +0200 Subject: [PATCH 5/5] review: admit host-pinned demoted states into eviction directly (#2166) add_evictable_state tests residency through staged_to_compute_backend, which host-pinned offloaded params never satisfy (they are resident via their parameter block). Insert demoted host-pinned states into eviction_states directly so the demoted block actually becomes releasable under host pressure. --- src/model_manager.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/model_manager.cpp b/src/model_manager.cpp index 2d23f792a2..70d812baa8 100644 --- a/src/model_manager.cpp +++ b/src/model_manager.cpp @@ -2113,6 +2113,14 @@ bool ModelManager::ensure_compute_backend_capacity( state->residency_mode = ResidencyMode::Disk; demoted_states.push_back(state); add_evictable_state(state); + if (host_pinned_states.find(state) != host_pinned_states.end()) { + // Host-pinned offloaded params are resident through their + // parameter block (loaded_to_params_backend), not through + // compute staging, so add_evictable_state's residency test + // cannot admit them — insert directly so the block becomes + // releasable. + eviction_states.insert(state); + } if (release_eviction_states()) { revert_demotions(); return true;