From 58724392b51af108293938affdb8b673e11591a0 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Wed, 15 Jul 2026 02:19:29 +0200 Subject: [PATCH 1/6] ggml-openvino: add Gemma-4 26B MoE support Adds support for the Gemma-4 26B-A4B hybrid-FFN MoE model on the OpenVINO backend (dense shared FFN + 128-expert top-8 MoE per layer, interleaved SWA/global attention, QK-norm, sandwich norms, logit soft-cap). Backend changes (all under ggml/src/ggml-openvino/): - 3D quantized expert weights: rank-2 GatherCompressed-matchable dequant (Q4_K gate/up, Q5_1 down), f16 zero-point (zp = -bias/scale) so the fusable Subtract form stays algebraically w*s + min without OOM; extracted in-place into the backend buffer (use_bias sizing). - Per-expert VIEW slicing on the non-static path + unique VIEW output names so the 8 same-named expert views no longer collide in the tensor_map. - MoE token dim kept dynamic through SUM_ROWS/DIV/CLAMP routing-norm ops and the per-expert scale, so all RoPE concats stay dynamic (fixes decode Broadcast mismatch and the GPU in-place-concat KV-cache corruption). - GET_ROWS batched-gather index broadcast tied to the data batch dim. - Full-MoE path: keep the whole MoE (routing gather/softmax/normalization + expert matmuls) on one OV submodel instead of fragmenting at every MoE node. Auto-enabled for MoE models on the dynamic-shape devices (CPU/GPU), latched on MUL_MAT_ID at placement; NPU keeps the fragmented static path. The un-fragmented graph is numerically correct on both CPU and GPU and avoids the cross-submodel index corruption the fragmented path hit. - op gating: exclude fused TOPK_MOE (uses ARGSORT routing), i64 CONCAT, borderline q4_1/q5_1 n=256 GET_ROWS, and oversized non-expert MUL_MAT_ID. - Guard is_kvcache() against a null view_src on SCALE ops (gemma4 inp_scaled), which otherwise segfaults compute_llm_params. Verified against ravi9/dev_backend_openvino: gemma4 26B MoE CPU greedy "Paris is the capital of France"; gemma4 E2B on CPU/GPU; dense Llama-3.2-1B on CPU+GPU byte-identical to baseline; test-backend-ops OPENVINO CPU 2622/2622 and GPU 2576/2576. --- ggml/src/ggml-openvino/ggml-decoder.cpp | 40 ++++- .../src/ggml-openvino/ggml-openvino-extra.cpp | 35 +++- ggml/src/ggml-openvino/ggml-openvino-extra.h | 18 ++ ggml/src/ggml-openvino/ggml-openvino.cpp | 168 ++++++++++++++++-- ggml/src/ggml-openvino/ggml-quants.cpp | 71 ++++++-- ggml/src/ggml-openvino/ggml-quants.h | 3 +- .../ggml-openvino/openvino/op/get_rows.cpp | 60 ++++++- .../ggml-openvino/openvino/op/mul_mat_id.cpp | 65 +++++-- ggml/src/ggml-openvino/openvino/op/view.cpp | 136 ++++++++++++-- ggml/src/ggml-openvino/openvino/utils.cpp | 33 +++- 10 files changed, 554 insertions(+), 75 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 16f059380759..7e1985845e3b 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -565,7 +565,7 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr if (node->op == GGML_OP_GATED_DELTA_NET) { model_params.state_size = node->src[0]->ne[0]; } - if (node->op == GGML_OP_SCALE && is_kvcache(node->view_src, nullptr)) { + if (node->op == GGML_OP_SCALE && node->view_src != nullptr && is_kvcache(node->view_src, nullptr)) { compute_params.cache_rs_reset_len = ggml_nelements(node) / node->view_src->ne[0]; compute_params.cache_rs_reset_idx = node->src[0]->view_offs / node->view_src->ne[0]; } @@ -958,6 +958,30 @@ std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor return weight_node; } + // 3D quantized MoE expert weights [k, m, n_expert]: flatten to a rank-2 + // [n_expert, m*k] tensor and build the dequant subgraph with use_bias=true (the + // exact f16 zero-point form). This is the path hit by test-backend-ops and the + // host-buffer load; the backend-buffer path builds the same node in set_tensor. + // translate_mul_mat_id gathers experts on axis 0 of this node and splits m*k. + if (ggml_is_quantized(tensor->type) && tensor->ne[2] > 1) { + GGML_ASSERT(tensor->ne[3] == 1 && "4D quantized expert weights are not supported"); + GGML_ASSERT(ggml_is_contiguous(tensor) && "expert weights must be contiguous to flatten"); + const int64_t n_expert = tensor->ne[2]; + const int64_t m = tensor->ne[1]; + const int64_t k = tensor->ne[0]; + ggml_tensor flat_tensor = *tensor; + flat_tensor.ne[0] = m * k; + flat_tensor.ne[1] = n_expert; + flat_tensor.ne[2] = 1; + flat_tensor.ne[3] = 1; + flat_tensor.nb[1] = ggml_row_size(tensor->type, m * k); + flat_tensor.nb[2] = ggml_nbytes(tensor); + flat_tensor.nb[3] = ggml_nbytes(tensor); + OvWeight flat_weight = process_weight_tensor(&flat_tensor, tensor->data, nullptr, /*use_bias=*/true); + flat_weight.weight_node->set_friendly_name(tensor->name); + return flat_weight.weight_node; + } + // There are three cases where we need to create a new weight node: // 1. weights are in openvino_host_buffer. Weight loading to host buffer will not trigger backend_buffer_set_tensor // 2. weights are in cpu/cpu_mapped buffer. On token_embd.weight goes to case 1 or 2, depending on whether mmap or direct_io is used @@ -1673,8 +1697,22 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { case GGML_OP_DIAG: case GGML_OP_TRI: case GGML_OP_REPEAT: + // Shape-preserving elementwise ops: the dynamic dim is unchanged from src[0]. + // DIV/CLAMP are used in the MoE routing-weight normalization + // (sum_rows -> clamp -> div). If they are left untracked here the dynamic + // (token) dim is lost there, the captured prefill token count gets baked into + // the downstream reshapes, and every decoder layer after layer 0 turns static + // (which then triggers the GPU in-place-concat KV-cache corruption). + case GGML_OP_DIV: + case GGML_OP_CLAMP: m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; break; + case GGML_OP_SUM_ROWS: + // SUM_ROWS reduces ggml axis 0 to size 1 and preserves all other axes, so the + // dynamic dim is preserved unless it was axis 0 (then it is summed away). + m_node_dynamic_dims[node] = + (m_node_dynamic_dims[node->src[0]] == 0) ? -1 : m_node_dynamic_dims[node->src[0]]; + break; case GGML_OP_MUL_MAT_ID: case GGML_OP_SOLVE_TRI: m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]]; diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 18df24c77e64..5b4dfe7874d2 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -173,6 +173,28 @@ bool ggml_openvino_is_npu() { return ggml_openvino_get_device_config().is_npu; } +// Latched true once a MUL_MAT_ID op is seen during op placement; see header. Plain +// non-atomic bool: placement runs single-threaded before the multi-threaded compute +// that reads it, and the flag only ever transitions false->true (idempotent). +static bool g_has_moe_expert_weights = false; + +void ggml_openvino_note_moe_expert_weight() { + g_has_moe_expert_weights = true; +} + +bool ggml_openvino_has_moe_expert_weights() { + return g_has_moe_expert_weights; +} + +bool ggml_openvino_full_moe_enabled() { + // Keep the whole MoE on one OV submodel instead of fragmenting the graph at every + // MoE routing node. Auto-detected: a MoE model is recognized when the expert-routed + // matmul (MUL_MAT_ID) has been seen (see ggml_openvino_note_moe_expert_weight). + // Enabled on the dynamic-shape devices (CPU and GPU); NPU uses the static path and + // keeps the fragmented behavior for now. + return !ggml_openvino_is_npu() && ggml_openvino_has_moe_expert_weights(); +} + // Get the remote context for the current device (returns empty optional for CPU) std::optional ggml_openvino_get_remote_context() { return ggml_openvino_get_device_config().remote_context; @@ -252,9 +274,12 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten return layout; } - // Most quantized weights use the existing 2D extraction path. MXFP4 also - // appears as 3D expert weights for MUL_MAT_ID, so allow that type through. - if (tensor->type != GGML_TYPE_MXFP4 && (tensor->ne[2] != 1 || tensor->ne[3] != 1)) { + // Handle 2D weight tensors, and 3D MoE expert weights [k, m, n_expert] which + // are treated as a flattened 2D [n_expert*m, k] tensor (each row is quantized + // independently along k, so the block layout is identical when flattened). This + // covers both our quantized gemma4 experts (Q4_K/Q5_1) and MXFP4 3D experts, + // which are handled by the dedicated block just below. + if (tensor->ne[3] != 1) { return layout; } @@ -390,6 +415,10 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten // For symmetric quantization, no zp needed (weights stored as signed) if (layout.is_symmetric) { layout.zp_size = 0; + } else if (use_bias) { + // use_bias stores the zero-point/bias as F16 (2 bytes/block), not a packed + // integer. Must size the buffer accordingly so the extracted data fits in-place. + layout.zp_size = n_blocks * sizeof(uint16_t); } else { layout.zp_size = layout.is_u4 ? ((n_blocks + 1) / 2) : n_blocks; } diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.h b/ggml/src/ggml-openvino/ggml-openvino-extra.h index c2654fbfa1b8..d3a0207e4e8c 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.h +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.h @@ -99,6 +99,24 @@ int ggml_openvino_getenv_int(const char * var, int default_value = 0); // Check if running on NPU bool ggml_openvino_is_npu(); +// MoE detection. ggml_openvino_note_moe_expert_weight() latches a process-global flag +// that ggml_openvino_has_moe_expert_weights() reports. It is called from supports_op() +// the first time a GGML_OP_MUL_MAT_ID (the expert-routed matmul) is seen, which is the +// defining op of a MoE model. The latch is set at op-placement time (not weight load): +// the scheduler queries op placement before the expert weights are streamed in, and it +// makes multiple placement passes, so the first pass that encounters MUL_MAT_ID sets the +// flag and subsequent passes converge on the full-MoE layout. This lets the backend +// recognize "this is a MoE model" without any architecture name. +void ggml_openvino_note_moe_expert_weight(); +bool ggml_openvino_has_moe_expert_weights(); + +// Whether to keep the whole MoE on one OV submodel instead of fragmenting at every +// MoE node (see the per-node "force to CPU" gates). Auto-detected: ON when the model +// has expert-routed matmuls (a MoE model) on the dynamic-shape devices (CPU/GPU), +// OFF on NPU (static path). Keeping the whole MoE on OV is numerically correct on both +// CPU and GPU and avoids the graph fragmentation that caused index corruption on GPU. +bool ggml_openvino_full_moe_enabled(); + // Get requantization type for a tensor type (returns nullopt if no requant needed) std::optional ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant = false); diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 0601afe0939b..db51855e94ed 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -235,13 +235,59 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer bool is_weight_buffer = (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS); // Full tensor set: offset=0, full size, not a view bool is_full_tensor_set = (offset == 0 && size == ggml_nbytes(tensor) && tensor->view_src == nullptr); - // 2D tensor (typical weight shape) + // 2D weight, or 3D MoE expert weight [k, m, n_expert] handled as flattened 2D. bool is_2d = (tensor->ne[2] == 1 && tensor->ne[3] == 1); - bool is_supported_weight_shape = is_2d || tensor->type == GGML_TYPE_MXFP4; + bool is_3d_expert = (tensor->ne[2] > 1 && tensor->ne[3] == 1 && ggml_is_quantized(tensor->type)); + bool is_supported_weight_shape = is_2d || is_3d_expert || tensor->type == GGML_TYPE_MXFP4; if (is_weight_buffer && is_full_tensor_set && is_supported_weight_shape) { try { - auto result = process_weight_tensor(tensor, data, tensor->data); + // Flatten 3D expert weights [k, m, n_expert] -> 2D [k, n_expert*m] so the + // extracted data is written in-place into this backend buffer (avoiding a + // large extra allocation), then reshape the dequant node back to 4D. + ggml_tensor proc_tensor = *tensor; + const int64_t n_expert = tensor->ne[2]; + const int64_t m = tensor->ne[1]; + const int64_t k = tensor->ne[0]; + if (is_3d_expert) { + GGML_ASSERT(ggml_is_contiguous(tensor) && "3D expert weights must be contiguous"); + // View the contiguous 3D expert tensor [k, m, n_expert] as a 2D tensor + // [m*k, n_expert] (ne[0]=m*k, ne[1]=n_expert): one quantized "row" of + // m*k weights per expert. This is bit-identical to the per-k-row + // quantization because k is a whole number of quant super-blocks for + // every expert type here (Q4_K: k%256==0, Q5_1: k%32==0), so regrouping + // the blocks does not change any block's contents. + // + // The 2D weight path then yields a rank-2 [n_expert, m*k] dequant + // subgraph: Constant(u4/u8) -> Convert -> [Subtract(zp)] -> Multiply + // -> Reshape(3D->2D) -> Convert(f32). + // translate_mul_mat_id gathers experts on axis 0 of this node DIRECTLY, + // which lets the CPU plugin's ConvertGatherToGatherCompressed pass fuse + // the gather + dequant into a single GatherCompressed op. That keeps the + // weights COMPRESSED through compile_model and decompresses only the + // selected experts at runtime. Reshaping the dequant output to a 4D + // [1,n_expert,m,k] (the previous approach) breaks the fusion, so the + // plugin const-folds the entire decompressed constant (~87GB f32 for 30 + // layers x 128 experts) and OOMs - disable_constant_folding does NOT + // help there (it just keeps both compressed and f32 copies). + proc_tensor.ne[0] = m * k; + proc_tensor.ne[1] = n_expert; + proc_tensor.ne[2] = 1; + proc_tensor.nb[1] = ggml_row_size(tensor->type, m * k); + proc_tensor.nb[2] = ggml_nbytes(tensor); + proc_tensor.nb[3] = ggml_nbytes(tensor); + } + + // For 3D MoE experts use the accurate dequant (use_bias=true). This routes + // through the f16 zero-point Subtract form in make_int*_weights, which is + // exact (no round(min/scale) error that corrupts Q4_K/Q5_1 experts) AND + // still folds to GatherCompressed (stays compressed, no OOM). + auto result = is_3d_expert ? process_weight_tensor(&proc_tensor, data, tensor->data, /*use_bias=*/true, + /*zp_buffer_is_f16=*/true) + : process_weight_tensor(&proc_tensor, data, tensor->data); + // For 3D experts, leave result.weight_node as the rank-2 [n_expert, m*k] + // dequant node - translate_mul_mat_id handles the expert gather and the + // m*k -> m,k split. Do NOT reshape to 4D or disable folding here. result.weight_node->set_friendly_name(tensor->name); // const auto & layout = result.layout; @@ -459,10 +505,14 @@ static size_t ggml_backend_openvino_buffer_type_get_alloc_size(ggml_backend_buff const ggml_tensor * tensor) { GGML_UNUSED(buft); - // For quantized weight tensors, we need extra space for extracted data. - if (ggml_is_quantized(tensor->type) && - ((tensor->ne[2] == 1 && tensor->ne[3] == 1) || tensor->type == GGML_TYPE_MXFP4)) { - ggml_openvino_extracted_layout layout = ggml_openvino_get_extracted_layout(tensor); + // For quantized 2D tensors (weights), 3D MoE expert weights, and MXFP4 experts, + // we need extra space for extracted data. + if (ggml_is_quantized(tensor->type) && (tensor->ne[3] == 1 || tensor->type == GGML_TYPE_MXFP4)) { + // 3D MoE experts (non-MXFP4) are extracted with use_bias=true (f16 zero-point), + // which needs a larger zp slot - size the buffer with the same use_bias so the + // in-place extracted data fits (must match set_tensor's process_weight_tensor call). + const bool expert_use_bias = (tensor->ne[2] > 1 && tensor->type != GGML_TYPE_MXFP4); + ggml_openvino_extracted_layout layout = ggml_openvino_get_extracted_layout(tensor, expert_use_bias); if (layout.total_size > 0) { // GGML_LOG_DEBUG("%s: tensor %s needs %zu bytes (original %zu, extracted: weights=%zu scales=%zu zp=%zu)\n", // __func__, tensor->name, layout.total_size, ggml_nbytes(tensor), layout.weights_size, @@ -883,6 +933,18 @@ static bool cpy_output_view_is_supported(const ggml_tensor * op) { return ggml_nbytes(op) == 0 || ggml_is_contiguous(op); } +// Keep the entire MoE — including the routing gather/softmax/argsort/normalization and the +// expert matmuls — on the OpenVINO device so the whole model compiles as ONE submodel instead +// of fragmenting at every MoE node. The per-node "force to CPU" gates below were added to work +// around GPU-plugin numerical issues, but they fragment the graph into dozens of submodels with +// cross-boundary tensor copies (which mis-handles e.g. the layer-5 argsort indices). With the +// dynamic-shape frontend fix in place the un-fragmented graph is numerically correct on both +// CPU and GPU, so this keeps the whole MoE on one OV submodel. Auto-enabled for MoE models on +// the dynamic-shape devices (CPU/GPU); see ggml_openvino_full_moe_enabled(). +static bool full_moe_enabled() { + return ggml_openvino_full_moe_enabled(); +} + static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) { const ggml_tensor * as = op->src[0]; const ggml_tensor * ids = op->src[2]; @@ -892,7 +954,33 @@ static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) { // The current OpenVINO translation materializes selected expert weights with // shape [n_tokens, n_used, rows, k]. Skip cases that would create a very - // large temporary on GPU and let the scheduler fall back instead. + // large temporary on GPU and let the scheduler fall back instead. The CPU + // device can handle the large intermediate, so only apply this cap on GPU. + if (ggml_openvino_get_device_name() != "GPU") { + return false; + } + // On the full-MoE GPU path the real gemma4 expert matmuls (ffn_moe_gate_up / + // ffn_moe_down) legitimately exceed this cap and are handled correctly, so + // exempt those named ops. The scheduler also queries these same expert matmuls + // during its reserve/measurement pass with an EMPTY name (and a worst-case + // full-batch ids->ne[1] that blows past the tmp cap); if we send them to CPU there + // the placement can stick for the real graph, and the expert matmul then runs on + // CPU reading OpenVINO-produced routing ids across the split boundary — which are + // garbage (crash in ggml-cpu mul_mat_id: "i02 >= 0 && i02 < n_as"). So for the + // unnamed reserve-pass query, fall back to a STRUCTURAL match: a genuine gemma4 + // expert matmul routes over a 3D quantized expert-weight tensor + // (as->ne[2] == n_expert > 1). Real graph ops always have a name, so the named + // MUL_MAT_ID_FUSION test cases (op named "out") are unaffected and still hit the + // cap / stay on CPU as before. + if (full_moe_enabled()) { + const bool name_match = strncmp(op->name, "ffn_moe_gate_up", sizeof("ffn_moe_gate_up") - 1) == 0 || + strncmp(op->name, "ffn_moe_down", sizeof("ffn_moe_down") - 1) == 0; + const bool unnamed_expert_matmul = op->name[0] == '\0' && as->ne[2] > 1 && ggml_is_quantized(as->type); + if (name_match || unnamed_expert_matmul) { + return false; + } + } + size_t tmp_elems = 1; if (!checked_mul_size(tmp_elems, static_cast(ids->ne[1]), tmp_elems) || !checked_mul_size(tmp_elems, static_cast(ids->ne[0]), tmp_elems) || @@ -947,16 +1035,21 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { op->src[0]->type == GGML_TYPE_BF16) { return true; } - if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K)) { + if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K || + op->src[0]->type == GGML_TYPE_Q5_1 || op->src[0]->type == GGML_TYPE_Q4_1)) { // ERR = 0.000000306 > 0.000000100 GET_ROWS(type=q4_K,n=256,m=5,r=4,be1=1,be2=1,v=0) // ERR = 0.000000197 > 0.000000100 GET_ROWS(type=q5_K,n=256,m=5,r=4,be1=1,be2=1,v=0) + // q5_1 and q4_1 dequant land right at the 1e-7 tolerance (ERR ~1.1-1.4e-7), so they + // flakily fail GET_ROWS(type=q5_1/q4_1,n=256,...); exclude them for the same reason. return true; } break; } case GGML_OP_RESHAPE: { - if (strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) { + if (!full_moe_enabled() && + (strncmp(op->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0 || + strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0)) { return true; } break; @@ -989,6 +1082,23 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { if (op->src[1]->ne[0] == 1 && op->src[1]->ne[1] == 1 && op->src[1]->ne[2] == 1 && op->src[1]->ne[3] == 384) { return true; } + + break; + } + case GGML_OP_SOFT_MAX: { + if (op->src[2] != nullptr) { + // GGML_LOG_WARN("OpenVINO backend does not support SOFT_MAX with sinks\n"); + return true; + } + + // GPU execution of the MoE routing weights softmax is numerically unstable + // when fused with the surrounding GET_ROWS/reshape path. Keep this softmax + // on CPU so the scheduler splits at the same boundary that restores parity. + if (!full_moe_enabled() && op->src[0] != nullptr && op->src[0]->op == GGML_OP_RESHAPE && + op->src[0]->src[0] != nullptr && + strncmp(op->src[0]->src[0]->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0) { + return true; + } break; } case GGML_OP_SUM_ROWS: { @@ -1059,9 +1169,25 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { break; } case GGML_OP_MUL_MAT: { + if (ggml_openvino_get_device_name() == "GPU" && op->src[1]->op == GGML_OP_SOFT_MAX && + op->src[0]->op == GGML_OP_CONT && op->src[0]->src[0] != nullptr && + op->src[0]->src[0]->op == GGML_OP_TRANSPOSE && op->src[0]->src[0]->src[0] != nullptr && + op->src[0]->src[0]->src[0]->op == GGML_OP_PERMUTE) { + return true; + } + if (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16) { + // Has accuracy issue, try enabling this and see `test-backend-ops -o "MUL_MAT"` + // GGML_LOG_WARN("OpenVINO backend does not support MUL_MAT with two F16 tensors\n"); + return true; + } if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) { return true; } + if (ggml_is_quantized(op->src[0]->type) && op->src[0]->ne[1] == 1) { + // MUL_MAT(type_a=q4_0,type_b=f32,m=1,n=2048,k=8192,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1) + // triggers a bug in ov matmul_shape_inference.hpp + return true; + } if (op->src[0]->op == GGML_OP_VIEW && op->src[1]->op == GGML_OP_VIEW) { return true; } @@ -1170,6 +1296,13 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { GGML_ASSERT(dev->reg != nullptr); + // A MUL_MAT_ID op is the expert-routed matmul: its presence means this is a MoE + // model. Latch it here (placement time) rather than at weight load, because the + // scheduler queries op placement before the expert weights are streamed in. + if (op->op == GGML_OP_MUL_MAT_ID) { + ggml_openvino_note_moe_expert_weight(); + } + static std::unordered_set supported_types{ GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_I64, GGML_TYPE_I32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K, @@ -1231,7 +1364,7 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con // GGML_LOG_WARN("OpenVINO backend does not support GLU op %s\n", ggml_glu_op_name(ggml_get_glu_op(op))); return false; } - if (has_view_op_input(op)) { + if (ggml_openvino_get_device_name() == "GPU" && !full_moe_enabled() && has_view_op_input(op)) { // GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n", // ggml_glu_op_name(ggml_get_glu_op(op))); return false; @@ -1269,11 +1402,14 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con // GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(src->type)); return false; } - const bool is_supported_3d_mxfp4_moe = op->op == GGML_OP_MUL_MAT_ID && i == 0 && - src->type == GGML_TYPE_MXFP4; - if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_mxfp4_moe) { - // GGML_LOG_WARN("OpenVINO backend does not support 3D quantized tensors\n"); - return false; + if (ggml_is_quantized(src->type) && src->ne[2] != 1) { + // 3D quantized tensors are only supported as MUL_MAT_ID expert weights + // (src[0]), which are dequantized per-expert in create_weight_node. This + // covers both the gemma4 Q4_K/Q5_1 experts and MXFP4 3D experts. + if (!(op->op == GGML_OP_MUL_MAT_ID && i == 0)) { + // GGML_LOG_WARN("OpenVINO backend does not support 3D quantized tensors\n"); + return false; + } } } diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp index d4e4d8f660b0..37cba74843a0 100644 --- a/ggml/src/ggml-openvino/ggml-quants.cpp +++ b/ggml/src/ggml-openvino/ggml-quants.cpp @@ -548,11 +548,28 @@ ov::Output make_int8_weights(ov::Tensor & weight, auto weights_f16 = std::make_shared(weights_node, ov::element::f16); if (use_bias && zp.get_size() > 0) { - // Bias path: w * s + b (zp tensor holds f16 bias values) - auto bias_f16 = std::make_shared(zp); - auto w_s = - std::make_shared(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY); - result = std::make_shared(w_s, bias_f16, ov::op::AutoBroadcastType::NUMPY); + // Accurate dequant in the FUSABLE zero-point form: (w - zp) * s, where the + // zero point is an exact f16 value zp = -bias/scale (bias held in the zp + // tensor). This is algebraically equal to w*s + bias but, unlike an Add(bias) + // graph, it matches OpenVINO's ConvertGatherToGatherCompressed pattern + // (Constant->Convert->Subtract->Multiply), so MoE expert weights stay + // compressed through compile_model (no f32 materialization / OOM). Using a + // real f16 zp instead of an integer one avoids the round(min/scale) error + // that corrupts Q4_K/Q5_1 experts. + // Convert bias -> zero-point IN PLACE in the (buffer-backed) zp tensor to + // avoid allocating a duplicate f16 array. + auto * bias_zp_data = zp.data(); + const auto * scale_data = scales.data(); + size_t n = zp.get_size(); + for (size_t i = 0; i < n; i++) { + float s = static_cast(scale_data[i]); + float b = static_cast(bias_zp_data[i]); + bias_zp_data[i] = ov::float16(s != 0.0f ? -b / s : 0.0f); + } + auto zero_point_f16 = std::make_shared(zp); + auto w_zp = + std::make_shared(weights_f16, zero_point_f16, ov::op::AutoBroadcastType::NUMPY); + result = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY); } else { // Zero point path: (w - zp) * s auto zero_point = std::make_shared(zp); @@ -622,11 +639,24 @@ ov::Output make_int4_weights(ov::Tensor & weight, auto weights_f16 = std::make_shared(weights_node, ov::element::f16); if (use_bias && zp.get_size() > 0) { - // Bias path: w * s + b (zp tensor holds f16 bias values) - auto bias_f16 = std::make_shared(zp); - auto w_s = - std::make_shared(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY); - result = std::make_shared(w_s, bias_f16, ov::op::AutoBroadcastType::NUMPY); + // Accurate dequant in the FUSABLE zero-point form: (w - zp) * s with an + // exact f16 zp = -bias/scale. Equivalent to w*s + bias but matches + // ConvertGatherToGatherCompressed so MoE experts stay compressed (no OOM), + // and avoids the round(min/scale) error of an integer zp. + // Convert bias -> zero-point IN PLACE in the zp tensor (which is backed by the + // backend buffer for experts) so we don't allocate a duplicate f16 array. + auto * bias_zp_data = zp.data(); + const auto * scale_data = scales.data(); + size_t n = zp.get_size(); + for (size_t i = 0; i < n; i++) { + float s = static_cast(scale_data[i]); + float b = static_cast(bias_zp_data[i]); + bias_zp_data[i] = ov::float16(s != 0.0f ? -b / s : 0.0f); + } + auto zero_point_f16 = std::make_shared(zp); + auto w_zp = + std::make_shared(weights_f16, zero_point_f16, ov::op::AutoBroadcastType::NUMPY); + result = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY); } else { // Zero point path: (w - zp) * s auto zero_points_node = std::make_shared(zp); @@ -816,7 +846,8 @@ std::shared_ptr requantize_to_buffers(const ggml_tensor * tensor, return result; } -OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, void * output_base_ptr, bool use_bias) { +OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, void * output_base_ptr, bool use_bias, + bool zp_buffer_is_f16) { GGML_ASSERT(tensor != nullptr); GGML_ASSERT(data != nullptr); @@ -887,10 +918,15 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo } if (use_bias) { - OPENVINO_ASSERT(!layout.is_requant, - "use_bias is only used for test-backend-ops, which should not have requantization"); - // bias node will be created on the fly and not use backend buffer - output_base_ptr = nullptr; + OPENVINO_ASSERT(!layout.is_requant, "use_bias cannot be combined with requantization"); + // The f16 bias/zero-point can be written into the backend buffer ONLY when that + // buffer was sized for an f16 zp (caller sets zp_buffer_is_f16 - true for the 3D + // MoE expert set_tensor path, whose get_alloc_size reserves f16 zp space). For any + // other use_bias caller (e.g. test-backend-ops 2D weights, buffer sized for an + // integer zp) writing f16 zp would overflow it, so self-allocate instead. + if (!zp_buffer_is_f16) { + output_base_ptr = nullptr; + } } // F16 requant path - no separate scales/zp needed in result @@ -936,7 +972,10 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16; result.scales = ov::Tensor(scale_type, scale_shape, buf_base + layout.scales_offset); if (!layout.is_symmetric) { - ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8; + // use_bias stores an f16 bias in the zp slot (layout reserved f16-sized + // space); otherwise a packed integer zero-point. + ov::element::Type zp_type = + use_bias ? ov::element::f16 : (layout.is_u4 ? ov::element::u4 : ov::element::u8); result.zp = ov::Tensor(zp_type, scale_shape, buf_base + layout.zp_offset); } // else: result.zp remains default-constructed (empty) for symmetric diff --git a/ggml/src/ggml-openvino/ggml-quants.h b/ggml/src/ggml-openvino/ggml-quants.h index 1b89fd887e16..b879dbee102c 100644 --- a/ggml/src/ggml-openvino/ggml-quants.h +++ b/ggml/src/ggml-openvino/ggml-quants.h @@ -133,7 +133,8 @@ OvWeight process_weight_tensor( const ggml_tensor * tensor, const void * data, // Source data pointer (may differ from tensor->data) void * output_base_ptr = nullptr, // Base pointer for output buffers (or nullptr for internal allocation) - bool use_bias = false); // Use fp bias instead of quantized zero_point, only used in test-backend-ops + bool use_bias = false, // Use fp bias instead of quantized zero_point (test-backend-ops + 3D experts) + bool zp_buffer_is_f16 = false); // output_base_ptr's zp slot is sized for f16 (3D-expert set_tensor path) void quantize_q4_0(const float * x, ov::Tensor & weights_arr, diff --git a/ggml/src/ggml-openvino/openvino/op/get_rows.cpp b/ggml/src/ggml-openvino/openvino/op/get_rows.cpp index 3d5c8cb413d0..2ac8ec0ba1df 100644 --- a/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +++ b/ggml/src/ggml-openvino/openvino/op/get_rows.cpp @@ -5,9 +5,12 @@ #include #include #include +#include +#include #include #include #include +#include #include #include #include @@ -59,7 +62,62 @@ OutputVector translate_get_rows(const NodeContext & context) { auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1}); data = std::make_shared(data, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); - res = std::make_shared(data, indices, axis, 1); + // data: [batch, rows, ...], indices: [batch, n] - this is a batched gather + // (batch_dims=1) along the rows axis. The data and indices batch dims are + // logically equal (both == n_tokens) but reach this node through independent + // reshapes, so the GPU plugin's gather shape inference cannot prove + // data.shape[0] == indices.shape[0] and rejects the node. We must tie both + // batch dims to the SAME value, and crucially that value must stay DYNAMIC. + const auto data_ps = data.get_partial_shape(); + const auto idx_ps = indices.get_partial_shape(); + const bool data_batch_static = data_ps.rank().is_static() && data_ps[0].is_static(); + const bool idx_batch_dynamic = idx_ps.rank().is_dynamic() || idx_ps[0].is_dynamic(); + + if (data_batch_static && idx_batch_dynamic) { + // MoE per-expert-scale path: `data` is a statically-tiled REPEAT + // (ggml_repeat_4d(scale, 1, n_expert, n_tokens, 1)) whose batch dim is a + // compile-time-constant n_tokens, and every batch slice is IDENTICAL (it was + // tiled from a single [1, n_expert, 1] scale). `indices` (selected_experts) + // carries the genuinely dynamic token dim. Broadcasting indices up to the + // static data batch (the naive fix) would freeze the token dim to the + // captured prefill length, and that static value then flows through the + // gather into the residual stream, making every following decoder layer + // static -> triggers the GPU in-place-concat KV-cache corruption (only + // layer 0 stays dynamic). A static->dynamic Broadcast cannot expand, so + // instead collapse the redundant data batch to 1 and broadcast 1->dynamic to + // match the indices batch. Mathematically identical (the slices are equal), + // and the whole graph stays dynamic. + auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto axis0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto data_b1 = std::make_shared(data, zero, one, one, axis0); // [1, rows, ...] + + auto idx_shape = std::make_shared(indices, ov::element::i64); + auto idx_batch = get_dimensions(idx_shape, {0}); // [batch] (dynamic) + auto data_b1_shape = std::make_shared(data_b1, ov::element::i64); + const auto rank = data_ps.rank().get_length(); + std::vector rest_axes; + for (int a = 1; a < rank; ++a) { + rest_axes.push_back(a); + } + auto data_rest = get_dimensions(data_b1_shape, rest_axes); // [rows, ...] + auto data_target = std::make_shared(ov::OutputVector{idx_batch, data_rest}, 0); + data = + std::make_shared(data_b1, data_target, ov::op::BroadcastType::BIDIRECTIONAL); + res = std::make_shared(data, indices, axis, 1); + } else { + // General case: tie the indices batch to the data batch (the data batch is + // already dynamic, e.g. the routing-weights gather whose data comes from the + // activations). Broadcast indices to [data_batch, indices_n]. + auto data_shape = std::make_shared(data, ov::element::i64); + auto data_batch = get_dimensions(data_shape, {0}); // [batch] + auto idx_shape = std::make_shared(indices, ov::element::i64); + auto idx_n = get_dimensions(idx_shape, {1}); // [n] + auto idx_target = std::make_shared(ov::OutputVector{data_batch, idx_n}, 0); + indices = std::make_shared(indices, idx_target, + ov::op::BroadcastType::BIDIRECTIONAL); + res = std::make_shared(data, indices, axis, 1); + } } } else if (context.is_stateful() && data.get_partial_shape().rank() == 3) { auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1}); diff --git a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp index 6df2784c2e45..a0f3403b4a3c 100644 --- a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +++ b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp @@ -145,21 +145,23 @@ OutputVector translate_mul_mat_id(const NodeContext & context) { } // OpenVINO sees GGML tensors in reversed dimension order: - // weights: [1, n_expert, m, k] // activations: [1, n_tokens, n_used_or_1, k] // ids: [1, 1, n_tokens, n_used] - // Rebuild the logical ranks explicitly from the 4D inputs instead of relying - // on fixed squeeze axes: real graphs can arrive through VIEW/RESHAPE chains - // where singleton axes are still represented differently at this point. - auto expert_weights_shape_4d = std::make_shared(expert_weights, ov::element::i64); + // The expert weights node is built specially in GgmlOvDecoder::create_weight_node + // as a rank-2 [n_expert, m*k] dequantization subgraph (Constant(u4)->Convert-> + // [Subtract]->Multiply->Reshape(3D->2D)->Convert). We MUST gather experts directly + // on this rank-2 node so the CPU plugin can fold the Gather + dequant into a single + // GatherCompressed op (keeping the weights compressed and decompressing only the + // selected experts at runtime). Reshaping the weights to [n_expert,m,k] before the + // Gather would break that fusion and cause the plugin to materialize all experts as + // f32 at compile time → OOM. So we gather on [n_expert, m*k] and split m*k -> m,k on + // the gathered result afterwards. auto activations_shape_4d = std::make_shared(activations, ov::element::i64); auto ids_shape_4d = std::make_shared(ids, ov::element::i64); - auto expert_weights_shape_3d = get_dimensions(expert_weights_shape_4d, {1, 2, 3}); auto activations_shape_3d = get_dimensions(activations_shape_4d, {1, 2, 3}); auto ids_shape_2d = get_dimensions(ids_shape_4d, {2, 3}); - expert_weights = std::make_shared(expert_weights, expert_weights_shape_3d, false); activations = std::make_shared(activations, activations_shape_3d, false); ids = std::make_shared(ids, ids_shape_2d, false); @@ -167,13 +169,49 @@ OutputVector translate_mul_mat_id(const NodeContext & context) { ids = std::make_shared(ids, ov::element::i32); } + // m (output row dim) is static; k = (m*k) / m. Gather experts on axis 0 of the + // rank-2 [n_expert, m*k] weight -> [n_tokens, n_used, m*k], then split to + // [n_tokens, n_used, m, k]. + const auto output_type = context.get_output_type(); + const auto mm_output_shape = context.get_output_shape(); + FRONT_END_OP_CONVERSION_CHECK(mm_output_shape.rank().is_static() && mm_output_shape.rank().get_length() == 4, + "Unexpected MUL_MAT_ID output rank"); + FRONT_END_OP_CONVERSION_CHECK(mm_output_shape[3].is_static(), + "Expected static row dimension (m) for MUL_MAT_ID output"); + const int64_t m_value = mm_output_shape[3].get_length(); + + // Normalize the weight to rank-2 [n_expert, m*k] so the expert Gather sits on a + // 2D node (required for the GatherCompressed fusion). The quantized expert path in + // GgmlOvDecoder::create_weight_node already produces [n_expert, m*k]. The + // non-quantized path (f32/f16 experts, e.g. test-backend-ops) produces a rank-4 + // [1, n_expert, m, k] constant; collapse it to [n_expert, m*k] here. + if (expert_weights.get_partial_shape().rank().is_static() && + expert_weights.get_partial_shape().rank().get_length() != 2) { + auto w_shape = std::make_shared(expert_weights, ov::element::i64); + auto n_expert_dim = get_dimensions(w_shape, {1}); + auto flat_w_dims = std::make_shared( + ov::OutputVector{n_expert_dim, ov::op::v0::Constant::create(ov::element::i64, {1}, {-1})}, 0); + expert_weights = std::make_shared(expert_weights, flat_w_dims, false); + } + auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); ov::Output selected_weights = std::make_shared(expert_weights, ids, gather_axis); - const auto output_type = context.get_output_type(); if (selected_weights.get_element_type() != ov::element::f32) { selected_weights = std::make_shared(selected_weights, ov::element::f32); } + + // Split the flattened m*k expert rows into [m, k]: reshape gathered + // [n_tokens, n_used, m*k] -> [n_tokens, n_used, m, -1]. + auto sel_ids_shape = std::make_shared(ids, ov::element::i64); + auto split_target_dims = std::make_shared( + ov::OutputVector{ + get_dimensions(sel_ids_shape, {0, 1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {m_value}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}), + }, + 0); + selected_weights = std::make_shared(selected_weights, split_target_dims, false); if (activations.get_element_type() != ov::element::f32) { activations = std::make_shared(activations, ov::element::f32); } @@ -187,19 +225,14 @@ OutputVector translate_mul_mat_id(const NodeContext & context) { get_dimensions(activations_shape, {2}), }, 0); - ov::Output acts_broadcasted = - std::make_shared(activations, acts_target_dims, ov::op::BroadcastType::BIDIRECTIONAL); + ov::Output acts_broadcasted = std::make_shared(activations, acts_target_dims, + ov::op::BroadcastType::BIDIRECTIONAL); auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); auto activations_expanded = std::make_shared(acts_broadcasted, unsqueeze_axes); auto batch_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto output_shape = context.get_output_shape(); - FRONT_END_OP_CONVERSION_CHECK(output_shape.rank().is_static() && output_shape.rank().get_length() == 4, - "Unexpected MUL_MAT_ID output rank"); - FRONT_END_OP_CONVERSION_CHECK(output_shape[3].is_static(), "Expected static row dimension for MUL_MAT_ID output"); - const auto row_dim_value = output_shape[3].get_length(); - auto row_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {row_dim_value}); + auto row_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {m_value}); ov::Output result = std::make_shared(activations_expanded, selected_weights, false, true); diff --git a/ggml/src/ggml-openvino/openvino/op/view.cpp b/ggml/src/ggml-openvino/openvino/op/view.cpp index 28004dcd2d8d..4b7f7a34e0a2 100644 --- a/ggml/src/ggml-openvino/openvino/op/view.cpp +++ b/ggml/src/ggml-openvino/openvino/op/view.cpp @@ -1,11 +1,12 @@ #include "../op_table.h" #include "../utils.h" - +#include #include +#include #include +#include #include #include - namespace ov { namespace frontend { namespace ggml { @@ -15,6 +16,114 @@ OutputVector translate_view(const NodeContext & context) { num_inputs_check(context, 1, 1); if (!context.is_static()) { + // On the stateless/non-static path VIEW is normally a no-op (consumers re-slice). + // EXCEPTION: the MoE expert aggregation slices each expert plane out of + // ffn_moe_weighted [n_embd, n_expert_used, n_tokens] with ggml_view_2d and then + // sums the planes with a chain of ADDs (llama-graph.cpp). Those ADDs read this + // VIEW node directly from the tensor map and do NOT re-slice, so a no-op here + // makes every plane the full tensor and the expert sum collapses. Materialize the + // single-expert slice here. Gated by name (ffn_moe_weighted...view) so it can't + // affect any other view. + const std::string & vname = context.get_name(); + if (vname.find("ffn_moe_weighted") != std::string::npos) { + auto src_ps = context.get_input_shape(0); + auto dst_ps = context.get_output_shape(); + if (src_ps.rank().is_static() && dst_ps.rank().is_static() && src_ps.rank() == dst_ps.rank() && + src_ps.is_static() && dst_ps.is_static()) { + auto sst = context.get_input_stride(0); + auto dst = context.get_output_stride(); + size_t voff = context.get_output_op_offset(); + auto ss = src_ps.to_shape(); + auto dd = dst_ps.to_shape(); + const size_t nd = ss.size(); + if (sst.size() == nd && dst.size() == nd) { + // Map each dst axis of size>1 to a src axis with equal (size,stride); + // the unmatched src axis of size>1 is the indexed expert axis. + // dst_to_src[d] records which src axis each dst axis came from, so we can + // later pull the dynamic (token) dim from the right source axis at runtime. + std::vector used(nd, false); + std::vector dst_to_src(nd, -1); + bool ok = true; + for (size_t d = 0; d < nd; ++d) { + if (dd[d] == 1) { + continue; + } + int found = -1; + for (size_t s = 0; s < nd; ++s) { + if (!used[s] && ss[s] == dd[d] && sst[s] == dst[d]) { found = (int) s; break; } + } + if (found < 0) { ok = false; break; } + used[found] = true; + dst_to_src[d] = found; + } + int dropped = -1; + if (ok) { + for (size_t s = 0; s < nd; ++s) { + if (!used[s] && ss[s] > 1) { + if (dropped >= 0) { ok = false; break; } + dropped = (int) s; + } + } + } + if (ok && dropped >= 0) { + const size_t dstr = sst[dropped]; + const int64_t dsz = (int64_t) ss[dropped]; + if (dstr > 0 && voff % dstr == 0) { + const int64_t sel = (int64_t) (voff / dstr); + if (sel >= 0 && sel < dsz) { + ov::Output sl = std::make_shared( + context.get_input(0), + ov::op::v0::Constant::create(ov::element::i64, {1}, {sel}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {sel + 1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {dropped})); + // Build the reshape target from the (concrete) dst shape, but + // keep the dynamic token axis dynamic instead of freezing it + // to the captured n_tokens. Without this the constant dst + // shape bakes in the prefill token count and the static value + // flows downstream, turning every later decoder layer static + // (the GPU in-place-concat KV-cache bug). The token axis is + // PERMUTED between the sliced input and the dst (e.g. input + // [1,tok,expert,emb] -> dst [1,1,tok,emb]), so special_zero + // (which copies the same-position dim) is not enough: pull the + // dynamic dim from the correct SOURCE axis via ShapeOf+Gather + // and place it at the dst token position. + const int32_t dyn = context.get_op_dynamic_dim(); // output ggml axis, -1 if none + int dst_ov_axis = (dyn != -1) ? (3 - (int) dyn) : -1; // get_shape() reverses ggml order + int src_ov_axis = (dst_ov_axis >= 0 && dst_ov_axis < (int) nd) + ? dst_to_src[dst_ov_axis] + : -1; + if (dst_ov_axis >= 0 && src_ov_axis >= 0) { + // target = concat of per-axis scalars; the token axis is a + // runtime Gather of the slice's shape, the rest are constants. + auto sl_shape = std::make_shared(sl, ov::element::i64); + auto tok_dim = std::make_shared( + sl_shape, + ov::op::v0::Constant::create(ov::element::i64, {1}, {src_ov_axis}), + ov::op::v0::Constant::create(ov::element::i64, {}, {0})); + ov::OutputVector parts; + for (int a = 0; a < (int) nd; ++a) { + if (a == dst_ov_axis) { + parts.push_back(tok_dim); + } else { + parts.push_back(ov::op::v0::Constant::create( + ov::element::i64, {1}, {(int64_t) dd[a]})); + } + } + auto dc = std::make_shared(parts, 0); + auto rs = std::make_shared(sl, dc, false); + return rename_outputs_with_suffix({rs}, context.get_name()); + } + auto dc = ov::op::v0::Constant::create( + ov::element::i64, {nd}, std::vector(dd.begin(), dd.end())); + auto rs = std::make_shared(sl, dc, false); + return rename_outputs_with_suffix({rs}, context.get_name()); + } + } + } + } + } + } return {context.get_input(0)}; } @@ -28,15 +137,11 @@ OutputVector translate_view(const NodeContext & context) { int64_t src_elems = 1, dst_elems = 1; for (int64_t i = 0; i < src_shape.rank().get_length(); ++i) { - if (src_shape[i].is_dynamic()) { - return {input}; - } + if (src_shape[i].is_dynamic()) return {input}; src_elems *= src_shape[i].get_length(); } for (int64_t i = 0; i < dst_shape.rank().get_length(); ++i) { - if (dst_shape[i].is_dynamic()) { - return {input}; - } + if (dst_shape[i].is_dynamic()) return {input}; dst_elems *= dst_shape[i].get_length(); } @@ -88,9 +193,7 @@ OutputVector translate_view(const NodeContext & context) { ov_stride_for_dim *= src_ov_shape[i]; } size_t elem_size = src_stride.back(); - if (elem_size == 0) { - elem_size = 1; - } + if (elem_size == 0) elem_size = 1; int64_t begin_val = 0; if (ov_stride_for_dim > 0 && elem_size > 0) { @@ -102,11 +205,12 @@ OutputVector translate_view(const NodeContext & context) { return {input}; } - auto sliced = - std::make_shared(input, ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim})); + auto sliced = std::make_shared( + input, + ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim})); sliced->set_friendly_name(context.get_output_name()); return {sliced->output(0)}; diff --git a/ggml/src/ggml-openvino/openvino/utils.cpp b/ggml/src/ggml-openvino/openvino/utils.cpp index e622d2305047..a99b9c839764 100644 --- a/ggml/src/ggml-openvino/openvino/utils.cpp +++ b/ggml/src/ggml-openvino/openvino/utils.cpp @@ -292,17 +292,40 @@ ov::Output process_view_input_new(const NodeContext & context, int inp // If translate_view already resolved this VIEW (produced a Slice), the input // will already have the expected shape — skip re-slicing. + // + // Two notions of "matches" are accepted per axis: + // - both dims static and equal, OR + // - both dims dynamic. + // The dynamic case matters for the MoE expert-plane views: translate_view now emits a + // DYNAMIC-token slice (so the token dim is not frozen). An all-static-only check would + // see the dynamic token dim, decide the shapes "don't match", and fall through to + // re-slice/flatten the already-resolved view (a Reshape to the full flattened + // n_expert_used*n_embd tail, which then conflicts with the single-plane input). Treat a + // dynamic-vs-dynamic axis as matching so the already-resolved view is reused as-is. + // + // A third case matters for split-model MoE fragments: translate_view resolves the + // expert-plane view against the fragment's INPUT parameter. When the graph is split + // the token axis of that parameter may already be concrete (static n_tokens) even + // though get_view_input_ov_shape() still reports it as dynamic (-1). The resolved + // view is then static [1,1,n_tokens,n_embd] while `expected` is [1,1,?,n_embd]. + // An "expected dynamic, actual static" axis is a valid concretization of the SAME + // resolved view, so treat it as matching too. Falling through to process_single_view + // here would re-slice/re-flatten the already-resolved single-plane view against the + // recorded (multi-plane) source strides and emit a constant-target Reshape whose baked + // dims no longer divide the concretized input -> "dimensions do not evenly divide". auto expected_ov_shape = context.get_view_input_ov_shape(input_index, 0); auto actual_shape = input.get_partial_shape(); if (expected_ov_shape.rank().is_static() && actual_shape.rank().is_static() && expected_ov_shape.rank() == actual_shape.rank()) { bool shapes_match = true; for (int64_t i = 0; i < expected_ov_shape.rank().get_length(); ++i) { - if (!expected_ov_shape[i].is_static() || !actual_shape[i].is_static()) { - shapes_match = false; - break; - } - if (expected_ov_shape[i] != actual_shape[i]) { + const bool both_dynamic = expected_ov_shape[i].is_dynamic() && actual_shape[i].is_dynamic(); + const bool both_static_equal = expected_ov_shape[i].is_static() && actual_shape[i].is_static() && + expected_ov_shape[i] == actual_shape[i]; + // expected dynamic, actual static: the resolved view already carries the + // concrete size for this fragment; reuse it rather than re-materializing. + const bool expected_dyn_actual_static = expected_ov_shape[i].is_dynamic() && actual_shape[i].is_static(); + if (!both_dynamic && !both_static_equal && !expected_dyn_actual_static) { shapes_match = false; break; } From cb8652c145c9e07d9db0da689c69e3c23c95c92e Mon Sep 17 00:00:00 2001 From: Ravi Panchumarthy Date: Wed, 15 Jul 2026 07:38:03 -0700 Subject: [PATCH 2/6] ci(ov): increase timeout as test-llama-archs included. --- .github/workflows/build-openvino.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml index 4fc30390c91b..019a221a575e 100644 --- a/.github/workflows/build-openvino.yml +++ b/.github/workflows/build-openvino.yml @@ -80,14 +80,14 @@ jobs: id: cmake_test_cpu run: | cd ${{ github.workspace }} - ctest --test-dir build/ReleaseOV -L main --verbose --timeout 2000 + ctest --test-dir build/ReleaseOV -L main --verbose --timeout 4000 - name: Test (GPU) id: cmake_test_gpu run: | cd ${{ github.workspace }} export GGML_OPENVINO_DEVICE=GPU - ctest --test-dir build/ReleaseOV -L main --verbose --timeout 3000 + ctest --test-dir build/ReleaseOV -L main --verbose --timeout 4000 openvino-windows-2022: runs-on: windows-2022 @@ -163,4 +163,4 @@ jobs: call "%OPENVINO_ROOT%\setupvars.bat" cd build - ctest --test-dir ReleaseOV -L main -C Release --verbose --timeout 3000 + ctest --test-dir ReleaseOV -L main -C Release --verbose --timeout 4000 From 4d74583de5d9cb90d9e35037d05b251174096562 Mon Sep 17 00:00:00 2001 From: Ravi Panchumarthy Date: Wed, 15 Jul 2026 09:47:32 -0700 Subject: [PATCH 3/6] ci(openvino): increase timeout for GPU --- .github/workflows/build-openvino.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml index 019a221a575e..206f596ffb38 100644 --- a/.github/workflows/build-openvino.yml +++ b/.github/workflows/build-openvino.yml @@ -87,7 +87,7 @@ jobs: run: | cd ${{ github.workspace }} export GGML_OPENVINO_DEVICE=GPU - ctest --test-dir build/ReleaseOV -L main --verbose --timeout 4000 + ctest --test-dir build/ReleaseOV -L main --verbose --output-on-failure --timeout 8000 openvino-windows-2022: runs-on: windows-2022 From 80fe58d8ce57bcf237bbb04b3315b7ab91225067 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Thu, 16 Jul 2026 00:08:25 +0200 Subject: [PATCH 4/6] ggml-openvino: fix 26B MoE GPU crash at ubatch>=32 (structural expert-matmul placement) The GPU full-MoE large-tmp exemption in mul_mat_id_requires_large_tmp used op names (ffn_moe_gate_up/ffn_moe_down or an empty reserve-pass name) to keep the expert matmuls on OpenVINO. At ubatch>=32 the ggml scheduler's reserve/measurement pass queries the expert down-proj MUL_MAT_ID under an auto-assigned name 'node_' (ggml_graph_add_node), which matched neither, so the worst-case ~2GB temporary exceeded the 1 GiB cap and the op was placed on ggml-CPU. It then read OpenVINO-produced routing ids across the OV<->CPU split boundary -> garbage indexing -> hard segfault / 'i02 >= 0 && i02 < n_as' assert during prefill (llama-bench -ub 32/64). Fix: exempt structurally instead of by name. A genuine MoE-model expert matmul routes over a 3D quantized expert-weight tensor (as->ne[2] > 1 && quantized) whose 'as' lives in a WEIGHTS buffer; the scheduler's earliest reserve query runs before weights are bound and reports an ANY buffer under an empty name, so treat that empty-name case as an expert matmul too. test-backend-ops MUL_MAT_ID/_FUSION cases use named, non-weights 'as' tensors, so they still hit the cap and stay on CPU as before. Verified: 26B MoE GPU llama-bench no longer crashes at -ub 32 (pp64 1.95) / -ub 64; test-backend-ops OPENVINO GPU MUL_MAT_ID 532/532 and MUL_MAT_ID_FUSION 9/9; default build compiles. --- ggml/src/ggml-openvino/ggml-openvino.cpp | 37 ++++++++++++------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index db51855e94ed..a0c22592fcf5 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -959,24 +959,25 @@ static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) { if (ggml_openvino_get_device_name() != "GPU") { return false; } - // On the full-MoE GPU path the real gemma4 expert matmuls (ffn_moe_gate_up / - // ffn_moe_down) legitimately exceed this cap and are handled correctly, so - // exempt those named ops. The scheduler also queries these same expert matmuls - // during its reserve/measurement pass with an EMPTY name (and a worst-case - // full-batch ids->ne[1] that blows past the tmp cap); if we send them to CPU there - // the placement can stick for the real graph, and the expert matmul then runs on - // CPU reading OpenVINO-produced routing ids across the split boundary — which are - // garbage (crash in ggml-cpu mul_mat_id: "i02 >= 0 && i02 < n_as"). So for the - // unnamed reserve-pass query, fall back to a STRUCTURAL match: a genuine gemma4 - // expert matmul routes over a 3D quantized expert-weight tensor - // (as->ne[2] == n_expert > 1). Real graph ops always have a name, so the named - // MUL_MAT_ID_FUSION test cases (op named "out") are unaffected and still hit the - // cap / stay on CPU as before. - if (full_moe_enabled()) { - const bool name_match = strncmp(op->name, "ffn_moe_gate_up", sizeof("ffn_moe_gate_up") - 1) == 0 || - strncmp(op->name, "ffn_moe_down", sizeof("ffn_moe_down") - 1) == 0; - const bool unnamed_expert_matmul = op->name[0] == '\0' && as->ne[2] > 1 && ggml_is_quantized(as->type); - if (name_match || unnamed_expert_matmul) { + // On the full-MoE GPU path a real model's expert matmuls legitimately exceed this + // cap and are handled correctly, so they must NOT be forced to CPU: if they are, the + // expert matmul runs on ggml-CPU reading OpenVINO-produced routing ids across the + // OV<->CPU split boundary — garbage (crash in ggml-cpu mul_mat_id: "i02 >= 0 && + // i02 < n_as", or a hard segfault at ubatch>=32). + // + // A genuine MoE-model expert matmul routes over a 3D quantized expert-weight tensor + // (as->ne[2] == n_expert > 1). That structural signal, plus the "as" tensor living in + // a WEIGHTS buffer, cleanly separates it from test-backend-ops MUL_MAT_ID/_FUSION + // cases (whose "as" is an ANY/COMPUTE buffer) without depending on op names — EXCEPT + // the scheduler's earliest reserve/measurement query, which runs before weights are + // bound and reports an ANY buffer under an empty op name. Treat that empty-name + // reserve query as an expert matmul too (test ops are explicitly named, never empty), + // so the exemption is consistent across both scheduler passes. + if (full_moe_enabled() && as->ne[2] > 1 && ggml_is_quantized(as->type)) { + const bool weights_buffer = + as->buffer != nullptr && ggml_backend_buffer_get_usage(as->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; + const bool reserve_measurement = op->name[0] == '\0'; + if (weights_buffer || reserve_measurement) { return false; } } From 530df6af14b862bec40616b650afa9eaf64a02a8 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Sat, 18 Jul 2026 01:42:09 +0200 Subject: [PATCH 5/6] ggml-openvino: fix stateful decode for Gemma-4 per-layer-type head sizes Gemma-4 uses different attention head dimensions per layer type (sliding_attention head_dim=256, full_attention global_head_dim=512), so the single scalar model_params.head_size cannot describe every layer. The stateful KV-cache reshape applied that one value to all layers, corrupting the SWA layers and failing SDPA shape inference. - ggml-decoder.cpp: derive the KV head size from each tensor's own combined dim (n_heads_kv * head_size) instead of the global scalar. - permute.cpp: the stateful path carries hidden-state tensors in a rank-3 layout; drop the leading batch axis from the rank-4 perm when the input is rank-3 (Gemma-4's per-layer-embedding permute), so the Transpose order matches the input rank. Stateful execution (GGML_OPENVINO_STATEFUL_EXECUTION) now runs and produces correct output for Gemma-4 on CPU and GPU. --- ggml/src/ggml-openvino/ggml-decoder.cpp | 14 ++++++++++---- ggml/src/ggml-openvino/openvino/op/permute.cpp | 16 ++++++++++++++-- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 7e1985845e3b..e5f6e0b2be2b 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -641,11 +641,17 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, if (is_stateful()) { // Convert stateless KV cache layout [1, 1, seq, n_heads_kv * head_size] // to stateful layout [1, seq, n_heads_kv, head_size]. + // NOTE: Gemma4 uses per-layer-type head sizes (sliding_attention layers + // head_dim=256, full_attention layers global_head_dim=512), so the single + // scalar m_model_params.head_size cannot describe every layer. Derive the + // head size from THIS tensor's own combined dim instead, so SWA and full + // layers each get their correct head_size. assert(input_shape.size() == 4 && input_shape[0] == 1 && input_shape[1] == 1 && - input_shape[2].is_dynamic() && - input_shape[3] == (m_model_params.n_heads_kv * m_model_params.head_size)); - input_shape = {input_shape[0], ov::Dimension::dynamic(), m_model_params.n_heads_kv, - m_model_params.head_size}; + input_shape[2].is_dynamic() && input_shape[3].is_static() && + input_shape[3].get_length() % m_model_params.n_heads_kv == 0); + const int64_t combined_dim = input_shape[3].get_length(); // n_heads_kv * head_size + const int64_t head_size = combined_dim / m_model_params.n_heads_kv; + input_shape = {input_shape[0], ov::Dimension::dynamic(), m_model_params.n_heads_kv, head_size}; } } else if (is_kv_idx(input, op)) { diff --git a/ggml/src/ggml-openvino/openvino/op/permute.cpp b/ggml/src/ggml-openvino/openvino/op/permute.cpp index 85550bff396b..f47c00b1965d 100644 --- a/ggml/src/ggml-openvino/openvino/op/permute.cpp +++ b/ggml/src/ggml-openvino/openvino/op/permute.cpp @@ -45,11 +45,22 @@ OutputVector translate_permute(const NodeContext & context) { static_cast(perm_values.size() - 1 - input_axis); } } - auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values); - if (op_case == 1 || context.is_stateful()) { + // The stateful path carries hidden-state tensors in a rank-3 layout (the + // leading batch dim is dropped, e.g. Gemma4's per-layer-embedding path). The + // perm above is rank-4; when the actual input is rank-3, drop the batch axis + // (perm[0], which is always the identity 0 here) and shift the rest down by 1 + // so the transpose order matches the input rank. + std::vector perm_used = perm_values; + const auto & src_ps = src.get_partial_shape(); + if (src_ps.rank().is_static() && src_ps.rank().get_length() == 3 && perm_values.size() == 4 && + perm_values[0] == 0) { + perm_used = {perm_values[1] - 1, perm_values[2] - 1, perm_values[3] - 1}; + } + auto perm = ov::op::v0::Constant::create(ov::element::i64, {(int64_t) perm_used.size()}, perm_used); res = std::make_shared(src, perm); } else if (op_case == 2) { + auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values); auto output_shape = context.get_output_shape().to_shape(); auto n_heads = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[1]}); auto head_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]}); @@ -68,6 +79,7 @@ OutputVector translate_permute(const NodeContext & context) { auto reshaped = std::make_shared(src, new_shape, true); res = std::make_shared(reshaped, perm); } else { + auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values); auto cache_shape = src.get_partial_shape(); auto output_shape = context.get_output_shape().to_shape(); int64_t head_size = output_shape[3]; From cbb82423e833cb5297a6e5213d7fad7321b1c499 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Sat, 18 Jul 2026 01:42:26 +0200 Subject: [PATCH 6/6] ggml-openvino: add GGML_OPENVINO_INT4_REQUANT decode-speed option Decode on the OpenVINO GPU backend is weight-bandwidth bound. By default the backend faithfully preserves the GGUF precision, which keeps some weights at 8-bit where the native OpenVINO export uses int4. This optional env flag re-quantizes those weights to symmetric int4 (off by default; small accuracy reduction): - MoE down-projection experts (Q5_1/Q8_0 -> u8) to int4 group-64. group-64 is required for the GatherMatmul fusion: it needs (m*k)/group divisible by the per-expert output rows N; the down expert has k=704 (704/64=11; 704/128 would break the fold). - dense attention/FFN weights (Q6_K/Q5_K -> per-channel int8) to int4 group-128. The int4 dequant chain still folds to GatherMatmulCompressed, so the experts stay compressed. Measured on Gemma-4 26B-A4B (Arc B390): prefill +13%, decode +18%. --- ggml/src/ggml-openvino/ggml-decoder.cpp | 11 +++++++- .../src/ggml-openvino/ggml-openvino-extra.cpp | 26 +++++++++++++++++++ ggml/src/ggml-openvino/ggml-openvino-extra.h | 2 +- ggml/src/ggml-openvino/ggml-openvino.cpp | 18 ++++++++++--- ggml/src/ggml-openvino/ggml-quants.cpp | 3 ++- 5 files changed, 54 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index e5f6e0b2be2b..0971fd7ed3c0 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -983,7 +983,16 @@ std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor flat_tensor.nb[1] = ggml_row_size(tensor->type, m * k); flat_tensor.nb[2] = ggml_nbytes(tensor); flat_tensor.nb[3] = ggml_nbytes(tensor); - OvWeight flat_weight = process_weight_tensor(&flat_tensor, tensor->data, nullptr, /*use_bias=*/true); + // GGML_OPENVINO_INT4_REQUANT: the down-projection experts are Q5_1/Q8_0 + // (kept as u8 = 8-bit). Requantize them to symmetric int4 (see + // ggml_openvino_get_requant_type). Requant is gated behind use_bias=false; the + // symmetric-int4 dequant chain (Constant(i4)->Convert->Multiply) still folds to + // GatherMatmulCompressed. gate_up (Q4_K) keeps use_bias=true (already int4). + static const bool int4_requant = ggml_openvino_getenv_int("GGML_OPENVINO_INT4_REQUANT") != 0; + const bool requant_this = + int4_requant && (tensor->type == GGML_TYPE_Q5_1 || tensor->type == GGML_TYPE_Q8_0); + OvWeight flat_weight = + process_weight_tensor(&flat_tensor, tensor->data, nullptr, /*use_bias=*/!requant_this); flat_weight.weight_node->set_friendly_name(tensor->name); return flat_weight.weight_node; } diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 5b4dfe7874d2..37a2cb76b4df 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -45,6 +45,7 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_DISABLE_CACHE", "GGML_OPENVINO_DISABLE_KV_SLICE", "GGML_OPENVINO_MANUAL_GQA_ATTN", + "GGML_OPENVINO_INT4_REQUANT", }; for (const char * const & env_var : env_var_names) { @@ -253,6 +254,26 @@ std::optional ggml_openvino_get_requant_type(const ggml_tensor * if (ggml_openvino_is_npu()) { return ExtraQuantType::Q4_0_128; } + // GGML_OPENVINO_INT4_REQUANT: re-quantize weights that ggml would otherwise keep at + // 8-bit down to symmetric int4, matching what the native OpenVINO export does. This + // halves those weights' + scales' bytes streamed per token (decode is weight-BW + // bound). Off by default: a small accuracy reduction the shipping backend avoids + // (it faithfully preserves the higher-precision GGUF weights). Applies to: + // - MoE down-projection experts stored as Q5_1/Q8_0 (default -> u8/8-bit) + // at group-64: the MoE GatherMatmul fusion needs (m*k)/group divisible by the + // per-expert output rows N; the down expert has k=704 (704/64=11 ok; + // 704/128=5.5 would break the fold). + // - dense attention/FFN weights stored as Q6_K/Q5_K (default -> per-channel int8) + // at group-128 (the verified dense config). + static const bool int4_requant = ggml_openvino_getenv_int("GGML_OPENVINO_INT4_REQUANT") != 0; + if (int4_requant) { + if (tensor->type == GGML_TYPE_Q5_1 || tensor->type == GGML_TYPE_Q8_0) { + return ExtraQuantType::Q4_0_64; + } + if (tensor->type == GGML_TYPE_Q6_K || tensor->type == GGML_TYPE_Q5_K) { + return ExtraQuantType::Q4_0_128; + } + } switch (tensor->type) { case GGML_TYPE_Q6_K: case GGML_TYPE_Q5_K: @@ -317,6 +338,11 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten layout.weights_per_block = 128; layout.is_symmetric = true; break; + case ExtraQuantType::Q4_0_64: + layout.is_u4 = true; + layout.weights_per_block = 64; + layout.is_symmetric = true; + break; case ExtraQuantType::Q4_0_C: layout.is_u4 = true; layout.weights_per_block = tensor->ne[0]; diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.h b/ggml/src/ggml-openvino/ggml-openvino-extra.h index d3a0207e4e8c..4fcf14caa84c 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.h +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.h @@ -15,7 +15,7 @@ #include // ExtraQuantType enum - defines requantization target formats -enum class ExtraQuantType { F16, Q4_0_C, Q8_1_C, Q4_0_128, Q8_0_C, Q8_0_32 }; +enum class ExtraQuantType { F16, Q4_0_C, Q8_1_C, Q4_0_128, Q4_0_64, Q8_0_C, Q8_0_32 }; ov::Core & ov_singleton_core(); diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index a0c22592fcf5..692d7e50051e 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -282,8 +282,15 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer // through the f16 zero-point Subtract form in make_int*_weights, which is // exact (no round(min/scale) error that corrupts Q4_K/Q5_1 experts) AND // still folds to GatherCompressed (stays compressed, no OOM). - auto result = is_3d_expert ? process_weight_tensor(&proc_tensor, data, tensor->data, /*use_bias=*/true, - /*zp_buffer_is_f16=*/true) + // GGML_OPENVINO_INT4_REQUANT: requantize the down experts (Q5_1/Q8_0) + // to symmetric int4 (use_bias=false enables the requant path). Must match the + // buffer sizing below (which uses the same expert_use_bias predicate). + static const bool int4_requant = ggml_openvino_getenv_int("GGML_OPENVINO_INT4_REQUANT") != 0; + const bool expert_requant = + int4_requant && is_3d_expert && (tensor->type == GGML_TYPE_Q5_1 || tensor->type == GGML_TYPE_Q8_0); + auto result = is_3d_expert ? process_weight_tensor(&proc_tensor, data, tensor->data, + /*use_bias=*/!expert_requant, + /*zp_buffer_is_f16=*/!expert_requant) : process_weight_tensor(&proc_tensor, data, tensor->data); // For 3D experts, leave result.weight_node as the rank-2 [n_expert, m*k] // dequant node - translate_mul_mat_id handles the expert gather and the @@ -511,7 +518,12 @@ static size_t ggml_backend_openvino_buffer_type_get_alloc_size(ggml_backend_buff // 3D MoE experts (non-MXFP4) are extracted with use_bias=true (f16 zero-point), // which needs a larger zp slot - size the buffer with the same use_bias so the // in-place extracted data fits (must match set_tensor's process_weight_tensor call). - const bool expert_use_bias = (tensor->ne[2] > 1 && tensor->type != GGML_TYPE_MXFP4); + // GGML_OPENVINO_INT4_REQUANT: down experts (Q5_1/Q8_0) are requantized to + // int4 with use_bias=false - size the buffer with the matching predicate. + static const bool int4_requant = ggml_openvino_getenv_int("GGML_OPENVINO_INT4_REQUANT") != 0; + const bool expert_requant = + int4_requant && (tensor->ne[2] > 1) && (tensor->type == GGML_TYPE_Q5_1 || tensor->type == GGML_TYPE_Q8_0); + const bool expert_use_bias = (tensor->ne[2] > 1 && tensor->type != GGML_TYPE_MXFP4 && !expert_requant); ggml_openvino_extracted_layout layout = ggml_openvino_get_extracted_layout(tensor, expert_use_bias); if (layout.total_size > 0) { // GGML_LOG_DEBUG("%s: tensor %s needs %zu bytes (original %zu, extracted: weights=%zu scales=%zu zp=%zu)\n", diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp index 37cba74843a0..86c7422a62ed 100644 --- a/ggml/src/ggml-openvino/ggml-quants.cpp +++ b/ggml/src/ggml-openvino/ggml-quants.cpp @@ -823,7 +823,8 @@ std::shared_ptr requantize_to_buffers(const ggml_tensor * tensor, } // Requantize to target quantized format - bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128); + bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128 || + requant_type == ExtraQuantType::Q4_0_64); if (is_u4) { quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size);