diff --git a/docs/rpc.md b/docs/rpc.md index ea9edf1c3..cb28e256f 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -131,7 +131,7 @@ If you want to allow connections from other machines on the network: > **Drivers & Hardware:** Ensure the Server machine has the necessary drivers installed and functional (e.g., Nvidia Drivers for CUDA, Vulkan SDK, or Metal). If no devices are found, the server will simply fallback to CPU usage. - +``` + +The options can be given in either order; no model is required. The client exits +with an error if RPC support is unavailable or a server cannot provide devices. -### Step B: Run with RPC device +### Step C: Run with RPC device If everything is working correctly, you can now run the client while offloading some or all of the work to the RPC server. @@ -217,4 +217,4 @@ The client will map these servers to sequential device IDs (e.g., RPC0 from the ## 6. Performance Considerations -RPC performance is heavily dependent on network bandwidth, as large weights and activations must be transferred back and forth over the network, especially for large models, or when using high resolutions. For best results, ensure your network connection is stable and has sufficient bandwidth (>1Gbps recommended). This shoumd not be a concern if you are running the server and client on the same machine, as the data transfer will happen over the loopback interface. \ No newline at end of file +RPC performance is heavily dependent on network bandwidth, as large weights and activations must be transferred back and forth over the network, especially for large models, or when using high resolutions. For best results, ensure your network connection is stable and has sufficient bandwidth (>1Gbps recommended). This shoumd not be a concern if you are running the server and client on the same machine, as the data transfer will happen over the loopback interface. diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index ca8ce4e79..8ee51865f 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -301,6 +301,10 @@ void parse_args(int argc, const char** argv, SDCliParams& cli_params, SDContextP log_level = cli_params.log_level; log_color = cli_params.color; + if (ctx_params.list_devices) { + exit(ctx_params.print_devices() ? 0 : 1); + } + bool valid = cli_params.resolve_and_validate(); if (valid && cli_params.mode != METADATA) { valid = ctx_params.resolve_and_validate(cli_params.mode) && diff --git a/examples/common/common.cpp b/examples/common/common.cpp index 8a9067396..7f65edac2 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -596,6 +596,11 @@ ArgOptions SDContextParams::get_options() { }; options.bool_options = { + {"", + "--list-devices", + "list available ggml backend devices, including --rpc-servers (one 'namedescription' per line), and exit; " + "the names are the device names accepted by --backend and --params-backend", + true, &list_devices}, {"", "--disable-prefetch", "disable asynchronous next-segment weight prefetch (defaults to false)", @@ -819,23 +824,22 @@ ArgOptions SDContextParams::get_options() { "but it usually offers faster inference speed and, in some cases, lower memory usage. " "The at_runtime mode, on the other hand, is exactly the opposite.", on_lora_apply_mode_arg}, - {"", - "--list-devices", - "list available ggml backend devices (one 'namedescription' per line) and exit; " - "the names are the device names accepted by --backend and --params-backend", - [](int /*argc*/, const char** /*argv*/, int /*index*/) { - size_t device_list_size = sd_list_devices(nullptr, 0); - std::vector devices(device_list_size + 1); - sd_list_devices(devices.data(), devices.size()); - fputs(devices.data(), stdout); - std::exit(0); - return 0; - }}, }; return options; } +bool SDContextParams::print_devices() const { + if (!sd_add_rpc_servers(rpc_servers.c_str())) { + return false; + } + size_t device_list_size = sd_list_devices(nullptr, 0); + std::vector devices(device_list_size + 1); + sd_list_devices(devices.data(), devices.size()); + fputs(devices.data(), stdout); + return true; +} + void SDContextParams::build_embedding_map() { static const std::vector valid_ext = {".gguf", ".safetensors", ".pt", ".ckpt"}; diff --git a/examples/common/common.h b/examples/common/common.h index 91e75c685..992f74c17 100644 --- a/examples/common/common.h +++ b/examples/common/common.h @@ -168,6 +168,7 @@ struct SDContextParams { std::string model_args; bool auto_fit = true; std::string rpc_servers; + bool list_devices = false; std::string effective_backend; std::string effective_params_backend; bool enable_mmap = false; @@ -189,6 +190,7 @@ struct SDContextParams { float flow_shift = INFINITY; ArgOptions get_options(); + bool print_devices() const; void build_embedding_map(); void prepare_backend_assignments(); bool resolve(SDMode mode); diff --git a/examples/server/main.cpp b/examples/server/main.cpp index 170877686..d79c6e2f3 100644 --- a/examples/server/main.cpp +++ b/examples/server/main.cpp @@ -48,6 +48,10 @@ static void parse_args(int argc, log_level = svr_params.log_level; log_color = svr_params.color; + if (ctx_params.list_devices) { + exit(ctx_params.print_devices() ? 0 : 1); + } + const bool random_seed_requested = default_gen_params.seed < 0; if (!svr_params.resolve_and_validate() || diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index 404d8cbdd..85fe51d46 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -640,6 +640,11 @@ SD_API const char* sd_version(void); // terminator. Passing nullptr or buffer_size 0 only queries the required size. SD_API size_t sd_list_devices(char* buffer, size_t buffer_size); +// Register comma-separated RPC endpoints (host:port) before listing devices. +// Repeated endpoints are safe; nullptr or an empty string is a no-op. +// Returns false if RPC is unavailable or an endpoint has no usable devices. +SD_API bool sd_add_rpc_servers(const char* servers); + // for C API, caller needs to call free_sd_images to free the memory after use // This helps avoid CRT problems on Windows when memory is allocated in the library but freed in the caller, which may use a different CRT. SD_API void free_sd_images(sd_image_t* result_images, int num_images); diff --git a/src/core/ggml_extend_backend.cpp b/src/core/ggml_extend_backend.cpp index 57b1e8dea..9e57efaed 100644 --- a/src/core/ggml_extend_backend.cpp +++ b/src/core/ggml_extend_backend.cpp @@ -294,6 +294,10 @@ bool add_rpc_devices(const std::string& servers) { return false; } ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); + if (!rpc_reg) { + ggml_backend_load_all(); + rpc_reg = ggml_backend_reg_by_name("RPC"); + } if (!rpc_reg) { LOG_ERROR("RPC backend not found, cannot add RPC servers"); return false; @@ -307,12 +311,19 @@ bool add_rpc_devices(const std::string& servers) { for (const auto& server : rpc_servers) { LOG_INFO("Adding RPC server: %s", server.c_str()); auto reg = ggml_backend_rpc_add_server_fn(server.c_str()); - // no return value to check for success but should print errors from the RPC backend if it fails to add the server + if (!reg) { + LOG_ERROR("failed to add RPC server: %s", server.c_str()); + return false; + } ggml_backend_register(reg); } return true; } +bool sd_add_rpc_servers(const char* servers) { + return add_rpc_devices(servers ? servers : ""); +} + static void ggml_backend_load_all_once() { // If the registry already has devices and the CPU backend is present, // assume either static registration or explicit host-side preloading has