From 8e57990db8034f410c518e69786f7b8909f6c920 Mon Sep 17 00:00:00 2001 From: suryasidd Date: Fri, 17 Jul 2026 15:20:20 -0700 Subject: [PATCH] fix for op regression due to is_model_splitted --- ggml/src/ggml-openvino/utils.cpp | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 28a8b68c80ab..42c1cf116bbf 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -170,7 +170,22 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< const auto & stateful = r_ctx->stateful; static auto is_static = false; - bool model_is_splitted = is_model_splitted(cgraph); + static const bool cache_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); + + // is_model_splitted is O(n_nodes^2) plus a create_weight_nodes scan and takes ~20 ms + // on a Llama-1B decode graph. It is called once per graph_compute invocation but the + // graph shape is identical across all decode steps, so memoize by graph_key: compute + // graph_key first (a few hundred us), and if the same key is already in decoder_cache + // we know the graph is not splitted (only not-splitted graphs get inserted there). + graph_key key(cgraph); + bool key_seen = false; + if (!cache_disabled) { + std::lock_guard map_lock(r_ctx->ctx_mutex); + key_seen = r_ctx->decoder_cache.find(key) != r_ctx->decoder_cache.end(); + } + + bool model_is_splitted = key_seen ? false : is_model_splitted(cgraph); + if (is_naive(cgraph)) { if (!model_is_splitted) { return naive_compute(cgraph, core, device, config); @@ -185,8 +200,6 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< ComputeParams c_params; std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static); - graph_key key(cgraph); - static const bool cache_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); const bool cache_enabled = !model_is_splitted && !cache_disabled; bool cache_hit = false;