diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index d9ad7be734d1..2a734eb82e3e 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -3,6 +3,7 @@ #include "ggml-impl.h" #include "ggml.h" +#include #include #include #include @@ -15,6 +16,46 @@ ov::Core & ov_singleton_core() { return core; } +static bool has_prefix(const std::string & s, const std::string & prefix) { + return s.size() >= prefix.size() && std::equal(prefix.begin(), prefix.end(), s.begin()); +} + +static bool is_virtual_routing_device(const std::string & device_name) { + return has_prefix(device_name, "AUTO") || has_prefix(device_name, "MULTI") || has_prefix(device_name, "HETERO"); +} + +static std::vector ov_enumerate_devices() { + std::vector result; + + for (const auto & device : ov_singleton_core().get_available_devices()) { + if (!is_virtual_routing_device(device)) { + result.push_back(device); + } + } + + if (result.empty()) { + result.push_back("CPU"); + } + + std::sort(result.begin(), result.end()); + result.erase(std::unique(result.begin(), result.end()), result.end()); + return result; +} + +static std::string resolve_openvino_device_name(const std::vector & available_devices, + const std::string & requested) { + if (available_devices.empty()) { + return "CPU"; + } + + auto it = std::find(available_devices.begin(), available_devices.end(), requested); + if (it != available_devices.end()) { + return *it; + } + + return "CPU"; +} + // ===================================================== // Device Configuration Implementations // ===================================================== @@ -54,16 +95,17 @@ void ggml_openvino_device_config::init() { } } - device_name = ggml_openvino_getenv_str("GGML_OPENVINO_DEVICE", "CPU"); - auto available_devices = ov_singleton_core().get_available_devices(); - if (std::find(available_devices.begin(), available_devices.end(), device_name) == available_devices.end()) { - GGML_LOG_WARN("GGML OpenVINO Backend: device %s is not available, fallback to CPU\n", device_name.c_str()); - device_name = "CPU"; + const std::string requested_device = ggml_openvino_getenv_str("GGML_OPENVINO_DEVICE", "CPU"); + available_devices = ov_enumerate_devices(); + device_name = resolve_openvino_device_name(available_devices, requested_device); + if (device_name != requested_device) { + GGML_LOG_WARN("GGML OpenVINO Backend: device %s is not available, fallback to %s\n", requested_device.c_str(), + device_name.c_str()); } - is_npu = (device_name == "NPU"); + is_npu = has_prefix(device_name, "NPU"); const char * cache_dir = ggml_openvino_getenv_str("GGML_OPENVINO_CACHE_DIR"); - if (device_name == "NPU") { + if (has_prefix(device_name, "NPU")) { compile_config = { {"NPU_COMPILER_DYNAMIC_QUANTIZATION", "YES" }, {"NPU_USE_NPUW", "YES" }, @@ -85,7 +127,7 @@ void ggml_openvino_device_config::init() { } // Initialize remote context with queue sharing for GPU - if (device_name == "GPU") { + if (has_prefix(device_name, "GPU")) { // Create OpenCL context and queue cl_int err; cl_platform_id platform; @@ -120,7 +162,7 @@ void ggml_openvino_device_config::init() { // Release the context (queue keeps a reference) clReleaseContext(cl_ctx); - } else if (device_name == "NPU") { + } else if (has_prefix(device_name, "NPU")) { // remote tensor is not used for NPU yet // remote_context = ov_singleton_core().get_default_context(device_name); } @@ -151,6 +193,12 @@ const std::string & ggml_openvino_get_device_name() { return ggml_openvino_get_device_config().device_name; } +std::vector ggml_openvino_get_available_devices() { + auto & config = ggml_openvino_get_device_config(); + config.init(); + return config.available_devices; +} + // Get the value of a GGML_OPENVINO_* env var as a string. Returns // default_value when the var is unset or set to an empty string. const char * ggml_openvino_getenv_str(const char * var, const char * default_value) { diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.h b/ggml/src/ggml-openvino/ggml-openvino-extra.h index c2654fbfa1b8..524428bf8e5b 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.h +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.h @@ -60,6 +60,7 @@ clEnqueueMemcpyINTEL_fn ggml_openvino_get_clEnqueueMemcpyINTEL(); struct ggml_openvino_device_config { std::string device_name = "CPU"; + std::vector available_devices; bool is_npu = false; bool initialized = false; std::optional remote_context; @@ -80,6 +81,9 @@ void ggml_openvino_init_device_config(); // Get the device name const std::string & ggml_openvino_get_device_name(); +// Get all available physical OpenVINO devices +std::vector ggml_openvino_get_available_devices(); + // Environment variable accessors. All GGML_OPENVINO_* env vars are read once // during backend init and cached on the device config; consumers must go // through these helpers (never call ::getenv directly) so behavior stays diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 659dbd4b5acb..a80b35bf60ce 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -20,7 +20,10 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -656,7 +659,7 @@ static const ggml_backend_i ggml_backend_openvino_interface = { }; int ggml_backend_openvino_get_device_count() { - return 1; + return (int) ggml_openvino_get_available_devices().size(); } static ggml_guid_t ggml_backend_openvino_guid(void) { @@ -716,8 +719,90 @@ struct ggml_backend_openvino_device_context { int device; std::string name; std::string description; + size_t total_memory; }; +static bool ov_device_has_prefix(const std::string & s, const std::string & prefix) { + return s.size() >= prefix.size() && std::equal(prefix.begin(), prefix.end(), s.begin()); +} + +static std::string ov_device_description_from_name(const std::string & device_name) { + std::string description = device_name; + try { + description = ov_singleton_core().get_property(device_name, ov::device::full_name); + } catch (...) { + return device_name; + } + + if (ov_device_has_prefix(device_name, "NPU")) { + try { + const std::string arch = ov_singleton_core().get_property(device_name, "DEVICE_ARCHITECTURE").as(); + if (!arch.empty()) { + description += " (NPU " + arch + ")"; + } + } catch (...) { + } + } + + return description; +} + +static bool ov_try_get_size_t_property(const std::string & device, const std::string & property, size_t & out) { + try { + const ov::Any value = ov_singleton_core().get_property(device, property); + if (value.is()) { + out = value.as(); + return true; + } + if (value.is()) { + out = (size_t) value.as(); + return true; + } + if (value.is()) { + out = (size_t) value.as(); + return true; + } + if (value.is()) { + const int64_t v = value.as(); + if (v >= 0) { + out = (size_t) v; + return true; + } + } + } catch (...) { + } + return false; +} + +static bool ov_try_get_gpu_used_memory(const std::string & device, size_t & out) { + out = 0; + try { + const ov::Any stats_any = ov_singleton_core().get_property(device, "GPU_MEMORY_STATISTICS"); + if (stats_any.is>()) { + const auto stats = stats_any.as>(); + for (const auto & kv : stats) { + out += (size_t) kv.second; + } + return true; + } + if (stats_any.is()) { + const auto stats = stats_any.as(); + for (const auto & kv : stats) { + if (kv.second.is()) { + out += kv.second.as(); + } else if (kv.second.is()) { + out += (size_t) kv.second.as(); + } else if (kv.second.is()) { + out += (size_t) kv.second.as(); + } + } + return true; + } + } catch (...) { + } + return false; +} + static const char * ggml_backend_openvino_device_get_name(ggml_backend_dev_t dev) { ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context; return ctx->name.c_str(); @@ -729,6 +814,34 @@ static const char * ggml_backend_openvino_device_get_description(ggml_backend_de } static void ggml_backend_openvino_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { + ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context; + + if (ov_device_has_prefix(ctx->name, "GPU")) { + size_t used = 0; + if (ctx->total_memory == 0 || !ov_try_get_gpu_used_memory(ctx->name, used)) { + *total = 0; + *free = 0; + return; + } + + *total = ctx->total_memory; + *free = (used >= *total) ? 0 : (*total - used); + return; + } + + if (ov_device_has_prefix(ctx->name, "NPU")) { + size_t allocated = 0; + if (ctx->total_memory == 0 || !ov_try_get_size_t_property(ctx->name, "NPU_DEVICE_ALLOC_MEM_SIZE", allocated)) { + *total = 0; + *free = 0; + return; + } + + *total = ctx->total_memory; + *free = (allocated >= *total) ? 0 : (*total - allocated); + return; + } + #ifdef _WIN32 MEMORYSTATUSEX status; status.dwLength = sizeof(status); @@ -1174,6 +1287,11 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { GGML_ASSERT(dev->reg != nullptr); + ggml_backend_openvino_device_context * dev_ctx = (ggml_backend_openvino_device_context *) dev->context; + if (dev_ctx->name != ggml_openvino_get_device_name()) { + return false; + } + static std::unordered_set supported_types{ GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_I64, GGML_TYPE_I32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K}; @@ -1351,15 +1469,21 @@ GGML_BACKEND_API ggml_backend_reg_t ggml_backend_openvino_reg(void) { std::lock_guard lock(mutex); if (!initialized) { ggml_openvino_init(); + const std::vector openvino_devices = ggml_openvino_get_available_devices(); ggml_backend_openvino_reg_context * ctx = new ggml_backend_openvino_reg_context; for (int i = 0; i < ggml_backend_openvino_get_device_count(); i++) { ggml_backend_openvino_device_context * dev_ctx = new ggml_backend_openvino_device_context; dev_ctx->device = i; - dev_ctx->name = GGML_OPENVINO_NAME + std::to_string(i); - - dev_ctx->description = ov::get_openvino_version().description; + dev_ctx->name = openvino_devices[i]; + dev_ctx->description = ov_device_description_from_name(dev_ctx->name); + dev_ctx->total_memory = 0; + if (ov_device_has_prefix(dev_ctx->name, "GPU")) { + ov_try_get_size_t_property(dev_ctx->name, "GPU_DEVICE_TOTAL_MEM_SIZE", dev_ctx->total_memory); + } else if (ov_device_has_prefix(dev_ctx->name, "NPU")) { + ov_try_get_size_t_property(dev_ctx->name, "NPU_DEVICE_TOTAL_MEM_SIZE", dev_ctx->total_memory); + } ggml_backend_dev_t dev = new ggml_backend_device{/* .interface = */ ggml_backend_openvino_device_interface,