fix: honor --conv-direct for ESRGAN upscaler (#2163)

docs/backend.md

@@ -218,16 +218,20 @@ spatial tiling. Spatial retries use half-size tiles along each latent dimension.
 ## Direct convolution (`--conv-direct on|off|auto`)
 
 `--conv-direct` defaults to `auto` and accepts module assignments for
-`diffusion`, `vae` (including preview VAE), and `controlnet`, using the same
-aliases and precedence rules as `--flash-attn`:
+`diffusion`, `vae` (including preview VAE), `controlnet`, and the standalone
+`upscaler` (for ESRGAN), using the same aliases and precedence rules as
+`--flash-attn`:
 
 ```shell
 sd-cli -m model.safetensors -p "a cat" --conv-direct auto,vae=off
 sd-cli -m model.safetensors -p "a cat" --conv-direct off,diffusion=on
+sd-cli -M upscale --conv-direct auto,upscaler=on
 ```
 
 - `on` requests direct 2D and 3D convolution.
 - `off` uses the existing im2col-based convolution paths.
+- For the standalone upscaler, `auto` leaves direct convolution disabled; use
+  `upscaler=on` to enable it.
 - `auto` first checks support for a representative F16-weight/F32-input direct
   convolution graph. Vulkan then benchmarks direct convolution against im2col
   using two small shapes per dimension, with warmup, output validation, and five
@@ -247,12 +251,13 @@ These are backend-level defaults. GGML still chooses the actual kernel based on
 weight type, shape, and hardware; enabling direct convolution does not guarantee
 that every operation uses implicit GEMM. The CUDA-family implicit GEMM paths
 require F16 weights and can use ordinary direct kernels for other cases.
-This option does not change the separately created image upscaler context.
+`upscaler=on|off` controls direct convolution in the separately created ESRGAN
+image upscaler context.
 
 Repeating `--conv-direct` replaces the previous assignment string. Existing
 `--diffusion-conv-direct` (diffusion and ControlNet) and `--vae-conv-direct`
 flags are deprecated and emit a warning, but remain supported when
-`--conv-direct` is omitted. Use `--conv-direct auto,diffusion=on,controlnet=on`
+`--conv-direct` is omitted. Use `--conv-direct auto,diffusion=on,controlnet=on,upscaler=on`
 and `--conv-direct auto,vae=on`, respectively. An explicit
 `--conv-direct` takes precedence over both legacy flags regardless of order.
 Library callers can set the appended `sd_ctx_params_t::conv_direct_mode` field

examples/cli/main.cpp

@@ -957,7 +957,7 @@ int main(int argc, const char* argv[]) {
     int upscale_factor = 4;  // unused for RealESRGAN_x4plus_anime_6B.pth
     if (ctx_params.esrgan_path.size() > 0 && gen_params.upscale_repeats > 0) {
         UpscalerCtxPtr upscaler_ctx(new_upscaler_ctx(ctx_params.esrgan_path.c_str(),
-                                                     ctx_params.diffusion_conv_direct,
+                                                     ctx_params.upscaler_conv_direct(),
                                                      ctx_params.n_threads,
                                                      gen_params.upscale_tile_size,
                                                      sd_ctx_params.backend,

examples/common/common.cpp

@@ -723,9 +723,9 @@ ArgOptions SDContextParams::get_options() {
          }},
         {"",
          "--diffusion-conv-direct",
-         "deprecated; use --conv-direct auto,diffusion=on,controlnet=on",
+         "deprecated; use --conv-direct auto,diffusion=on,controlnet=on,upscaler=on",
          [this](int, const char**, int) {
-             LOG_WARN("--diffusion-conv-direct is deprecated; use --conv-direct auto,diffusion=on,controlnet=on instead.");
+             LOG_WARN("--diffusion-conv-direct is deprecated; use --conv-direct auto,diffusion=on,controlnet=on,upscaler=on instead.");
              diffusion_conv_direct = true;
              return 0;
          }},
@@ -739,7 +739,7 @@ ArgOptions SDContextParams::get_options() {
          }},
         {"",
          "--conv-direct",
-         "on|off|auto or diffusion/vae/controlnet assignments (default: auto), e.g. auto,vae=off. Overrides legacy convolution flags",
+         "on|off|auto or diffusion/vae/controlnet/upscaler assignments (default: auto), e.g. auto,vae=off. Overrides legacy convolution flags",
          [this](int argc, const char** argv, int index) {
              if (++index >= argc || argv[index][0] == '\0') {
                  LOG_ERROR("--conv-direct requires on, off, auto, or module assignments");
@@ -1017,6 +1017,41 @@ std::string SDContextParams::to_string() const {
     return oss.str();
 }
 
+bool SDContextParams::upscaler_conv_direct() const {
+    if (conv_direct_mode.empty()) {
+        return diffusion_conv_direct;
+    }
+
+    auto trim = [](std::string value) {
+        const size_t first = value.find_first_not_of(" \\t");
+        if (first == std::string::npos) {
+            return std::string();
+        }
+        const size_t last = value.find_last_not_of(" \\t");
+        return value.substr(first, last - first + 1);
+    };
+
+    std::string default_mode = "auto";
+    std::string upscaler_mode;
+    std::istringstream assignments(conv_direct_mode);
+    std::string part;
+    while (std::getline(assignments, part, ',')) {
+        part = trim(part);
+        if (part.empty()) {
+            continue;
+        }
+
+        const size_t equals = part.find('=');
+        if (equals == std::string::npos) {
+            default_mode = part;
+        } else if (trim(part.substr(0, equals)) == "upscaler") {
+            upscaler_mode = trim(part.substr(equals + 1));
+        }
+    }
+
+    return (upscaler_mode.empty() ? default_mode : upscaler_mode) == "on";
+}
+
 sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool taesd_preview) {
     prepare_backend_assignments();
     embedding_vec.clear();

examples/common/common.h

@@ -195,6 +195,7 @@ struct SDContextParams {
     bool validate(SDMode mode);
     bool resolve_and_validate(SDMode mode);
     std::string to_string() const;
+    bool upscaler_conv_direct() const;
     sd_ctx_params_t to_sd_ctx_params_t(bool taesd_preview);
 };
 

examples/server/routes_sdcpp.cpp

@@ -567,7 +567,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
             // One GPU: an upscale must not run while a generation is using it.
             std::lock_guard<std::mutex> ctx_lock(*runtime->sd_ctx_mutex);
             UpscalerCtxPtr upscaler_ctx(new_upscaler_ctx(model_path.c_str(),
-                                                         runtime->ctx_params->diffusion_conv_direct,
+                                                         runtime->ctx_params->upscaler_conv_direct(),
                                                          runtime->ctx_params->n_threads,
                                                          tile_size,
                                                          runtime->ctx_params->backend.c_str(),

src/core/ggml_extend_backend.cpp

@@ -835,6 +835,7 @@ static bool parse_operation_assignment(const std::string& spec, SDBackendAssignm
                 case SDBackendModule::DIFFUSION:
                 case SDBackendModule::VAE:
                 case SDBackendModule::CONTROL_NET:
+                case SDBackendModule::UPSCALER:
                     break;
                 case SDBackendModule::TE:
                 case SDBackendModule::CLIP_VISION: