Gemma4 ov int4 and stateful fixes

.github/workflows/build-openvino.yml

@@ -80,14 +80,14 @@ jobs:
         id: cmake_test_cpu
         run: |
           cd ${{ github.workspace }}
-          ctest --test-dir build/ReleaseOV -L main --verbose --timeout 2000
+          ctest --test-dir build/ReleaseOV -L main --verbose --timeout 4000
 
       - name: Test (GPU)
         id: cmake_test_gpu
         run: |
           cd ${{ github.workspace }}
           export GGML_OPENVINO_DEVICE=GPU
-          ctest --test-dir build/ReleaseOV -L main --verbose --timeout 3000
+          ctest --test-dir build/ReleaseOV -L main --verbose --output-on-failure --timeout 8000
 
   openvino-windows-2022:
     runs-on: windows-2022
@@ -163,4 +163,4 @@ jobs:
           call "%OPENVINO_ROOT%\setupvars.bat"
 
           cd build
-          ctest --test-dir ReleaseOV -L main -C Release --verbose --timeout 3000
+          ctest --test-dir ReleaseOV -L main -C Release --verbose --timeout 4000

ggml/src/ggml-openvino/ggml-decoder.cpp

@@ -565,7 +565,7 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
         if (node->op == GGML_OP_GATED_DELTA_NET) {
             model_params.state_size = node->src[0]->ne[0];
         }
-        if (node->op == GGML_OP_SCALE && is_kvcache(node->view_src, nullptr)) {
+        if (node->op == GGML_OP_SCALE && node->view_src != nullptr && is_kvcache(node->view_src, nullptr)) {
             compute_params.cache_rs_reset_len = ggml_nelements(node) / node->view_src->ne[0];
             compute_params.cache_rs_reset_idx = node->src[0]->view_offs / node->view_src->ne[0];
         }
@@ -641,11 +641,17 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
         if (is_stateful()) {
             // Convert stateless KV cache layout [1, 1, seq, n_heads_kv * head_size]
             // to stateful layout [1, seq, n_heads_kv, head_size].
+            // NOTE: Gemma4 uses per-layer-type head sizes (sliding_attention layers
+            // head_dim=256, full_attention layers global_head_dim=512), so the single
+            // scalar m_model_params.head_size cannot describe every layer. Derive the
+            // head size from THIS tensor's own combined dim instead, so SWA and full
+            // layers each get their correct head_size.
             assert(input_shape.size() == 4 && input_shape[0] == 1 && input_shape[1] == 1 &&
-                   input_shape[2].is_dynamic() &&
-                   input_shape[3] == (m_model_params.n_heads_kv * m_model_params.head_size));
-            input_shape = {input_shape[0], ov::Dimension::dynamic(), m_model_params.n_heads_kv,
-                           m_model_params.head_size};
+                   input_shape[2].is_dynamic() && input_shape[3].is_static() &&
+                   input_shape[3].get_length() % m_model_params.n_heads_kv == 0);
+            const int64_t combined_dim = input_shape[3].get_length();  // n_heads_kv * head_size
+            const int64_t head_size = combined_dim / m_model_params.n_heads_kv;
+            input_shape = {input_shape[0], ov::Dimension::dynamic(), m_model_params.n_heads_kv, head_size};
         }
 
     } else if (is_kv_idx(input, op)) {
@@ -958,6 +964,39 @@ std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor
         return weight_node;
     }
 
+    // 3D quantized MoE expert weights [k, m, n_expert]: flatten to a rank-2
+    // [n_expert, m*k] tensor and build the dequant subgraph with use_bias=true (the
+    // exact f16 zero-point form). This is the path hit by test-backend-ops and the
+    // host-buffer load; the backend-buffer path builds the same node in set_tensor.
+    // translate_mul_mat_id gathers experts on axis 0 of this node and splits m*k.
+    if (ggml_is_quantized(tensor->type) && tensor->ne[2] > 1) {
+        GGML_ASSERT(tensor->ne[3] == 1 && "4D quantized expert weights are not supported");
+        GGML_ASSERT(ggml_is_contiguous(tensor) && "expert weights must be contiguous to flatten");
+        const int64_t n_expert = tensor->ne[2];
+        const int64_t m = tensor->ne[1];
+        const int64_t k = tensor->ne[0];
+        ggml_tensor flat_tensor = *tensor;
+        flat_tensor.ne[0] = m * k;
+        flat_tensor.ne[1] = n_expert;
+        flat_tensor.ne[2] = 1;
+        flat_tensor.ne[3] = 1;
+        flat_tensor.nb[1] = ggml_row_size(tensor->type, m * k);
+        flat_tensor.nb[2] = ggml_nbytes(tensor);
+        flat_tensor.nb[3] = ggml_nbytes(tensor);
+        // GGML_OPENVINO_INT4_REQUANT: the down-projection experts are Q5_1/Q8_0
+        // (kept as u8 = 8-bit). Requantize them to symmetric int4 (see
+        // ggml_openvino_get_requant_type). Requant is gated behind use_bias=false; the
+        // symmetric-int4 dequant chain (Constant(i4)->Convert->Multiply) still folds to
+        // GatherMatmulCompressed. gate_up (Q4_K) keeps use_bias=true (already int4).
+        static const bool int4_requant = ggml_openvino_getenv_int("GGML_OPENVINO_INT4_REQUANT") != 0;
+        const bool requant_this =
+            int4_requant && (tensor->type == GGML_TYPE_Q5_1 || tensor->type == GGML_TYPE_Q8_0);
+        OvWeight flat_weight =
+            process_weight_tensor(&flat_tensor, tensor->data, nullptr, /*use_bias=*/!requant_this);
+        flat_weight.weight_node->set_friendly_name(tensor->name);
+        return flat_weight.weight_node;
+    }
+
     // There are three cases where we need to create a new weight node:
     // 1. weights are in openvino_host_buffer. Weight loading to host buffer will not trigger backend_buffer_set_tensor
     // 2. weights are in cpu/cpu_mapped buffer. On token_embd.weight goes to case 1 or 2, depending on whether mmap or direct_io is used
@@ -1673,8 +1712,22 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
         case GGML_OP_DIAG:
         case GGML_OP_TRI:
         case GGML_OP_REPEAT:
+        // Shape-preserving elementwise ops: the dynamic dim is unchanged from src[0].
+        // DIV/CLAMP are used in the MoE routing-weight normalization
+        // (sum_rows -> clamp -> div). If they are left untracked here the dynamic
+        // (token) dim is lost there, the captured prefill token count gets baked into
+        // the downstream reshapes, and every decoder layer after layer 0 turns static
+        // (which then triggers the GPU in-place-concat KV-cache corruption).
+        case GGML_OP_DIV:
+        case GGML_OP_CLAMP:
             m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]];
             break;
+        case GGML_OP_SUM_ROWS:
+            // SUM_ROWS reduces ggml axis 0 to size 1 and preserves all other axes, so the
+            // dynamic dim is preserved unless it was axis 0 (then it is summed away).
+            m_node_dynamic_dims[node] =
+                (m_node_dynamic_dims[node->src[0]] == 0) ? -1 : m_node_dynamic_dims[node->src[0]];
+            break;
         case GGML_OP_MUL_MAT_ID:
         case GGML_OP_SOLVE_TRI:
             m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]];

ggml/src/ggml-openvino/ggml-openvino-extra.cpp

@@ -45,6 +45,7 @@ void ggml_openvino_device_config::init() {
         "GGML_OPENVINO_DISABLE_CACHE",
         "GGML_OPENVINO_DISABLE_KV_SLICE",
         "GGML_OPENVINO_MANUAL_GQA_ATTN",
+        "GGML_OPENVINO_INT4_REQUANT",
     };
 
     for (const char * const & env_var : env_var_names) {
@@ -173,6 +174,28 @@ bool ggml_openvino_is_npu() {
     return ggml_openvino_get_device_config().is_npu;
 }
 
+// Latched true once a MUL_MAT_ID op is seen during op placement; see header. Plain
+// non-atomic bool: placement runs single-threaded before the multi-threaded compute
+// that reads it, and the flag only ever transitions false->true (idempotent).
+static bool g_has_moe_expert_weights = false;
+
+void ggml_openvino_note_moe_expert_weight() {
+    g_has_moe_expert_weights = true;
+}
+
+bool ggml_openvino_has_moe_expert_weights() {
+    return g_has_moe_expert_weights;
+}
+
+bool ggml_openvino_full_moe_enabled() {
+    // Keep the whole MoE on one OV submodel instead of fragmenting the graph at every
+    // MoE routing node. Auto-detected: a MoE model is recognized when the expert-routed
+    // matmul (MUL_MAT_ID) has been seen (see ggml_openvino_note_moe_expert_weight).
+    // Enabled on the dynamic-shape devices (CPU and GPU); NPU uses the static path and
+    // keeps the fragmented behavior for now.
+    return !ggml_openvino_is_npu() && ggml_openvino_has_moe_expert_weights();
+}
+
 // Get the remote context for the current device (returns empty optional for CPU)
 std::optional<ov::RemoteContext> ggml_openvino_get_remote_context() {
     return ggml_openvino_get_device_config().remote_context;
@@ -231,6 +254,26 @@ std::optional<ExtraQuantType> ggml_openvino_get_requant_type(const ggml_tensor *
     if (ggml_openvino_is_npu()) {
         return ExtraQuantType::Q4_0_128;
     }
+    // GGML_OPENVINO_INT4_REQUANT: re-quantize weights that ggml would otherwise keep at
+    // 8-bit down to symmetric int4, matching what the native OpenVINO export does. This
+    // halves those weights' + scales' bytes streamed per token (decode is weight-BW
+    // bound). Off by default: a small accuracy reduction the shipping backend avoids
+    // (it faithfully preserves the higher-precision GGUF weights). Applies to:
+    //   - MoE down-projection experts stored as Q5_1/Q8_0 (default -> u8/8-bit)
+    //     at group-64: the MoE GatherMatmul fusion needs (m*k)/group divisible by the
+    //     per-expert output rows N; the down expert has k=704 (704/64=11 ok;
+    //     704/128=5.5 would break the fold).
+    //   - dense attention/FFN weights stored as Q6_K/Q5_K (default -> per-channel int8)
+    //     at group-128 (the verified dense config).
+    static const bool int4_requant = ggml_openvino_getenv_int("GGML_OPENVINO_INT4_REQUANT") != 0;
+    if (int4_requant) {
+        if (tensor->type == GGML_TYPE_Q5_1 || tensor->type == GGML_TYPE_Q8_0) {
+            return ExtraQuantType::Q4_0_64;
+        }
+        if (tensor->type == GGML_TYPE_Q6_K || tensor->type == GGML_TYPE_Q5_K) {
+            return ExtraQuantType::Q4_0_128;
+        }
+    }
     switch (tensor->type) {
     case GGML_TYPE_Q6_K:
     case GGML_TYPE_Q5_K:
@@ -252,9 +295,12 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
         return layout;
     }
 
-    // Most quantized weights use the existing 2D extraction path. MXFP4 also
-    // appears as 3D expert weights for MUL_MAT_ID, so allow that type through.
-    if (tensor->type != GGML_TYPE_MXFP4 && (tensor->ne[2] != 1 || tensor->ne[3] != 1)) {
+    // Handle 2D weight tensors, and 3D MoE expert weights [k, m, n_expert] which
+    // are treated as a flattened 2D [n_expert*m, k] tensor (each row is quantized
+    // independently along k, so the block layout is identical when flattened). This
+    // covers both our quantized gemma4 experts (Q4_K/Q5_1) and MXFP4 3D experts,
+    // which are handled by the dedicated block just below.
+    if (tensor->ne[3] != 1) {
         return layout;
     }
 
@@ -292,6 +338,11 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
             layout.weights_per_block = 128;
             layout.is_symmetric = true;
             break;
+        case ExtraQuantType::Q4_0_64:
+            layout.is_u4 = true;
+            layout.weights_per_block = 64;
+            layout.is_symmetric = true;
+            break;
         case ExtraQuantType::Q4_0_C:
             layout.is_u4 = true;
             layout.weights_per_block = tensor->ne[0];
@@ -390,6 +441,10 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
     // For symmetric quantization, no zp needed (weights stored as signed)
     if (layout.is_symmetric) {
         layout.zp_size = 0;
+    } else if (use_bias) {
+        // use_bias stores the zero-point/bias as F16 (2 bytes/block), not a packed
+        // integer. Must size the buffer accordingly so the extracted data fits in-place.
+        layout.zp_size = n_blocks * sizeof(uint16_t);
     } else {
         layout.zp_size = layout.is_u4 ? ((n_blocks + 1) / 2) : n_blocks;
     }

ggml/src/ggml-openvino/ggml-openvino-extra.h

@@ -15,7 +15,7 @@
 #include <string>
 
 // ExtraQuantType enum - defines requantization target formats
-enum class ExtraQuantType { F16, Q4_0_C, Q8_1_C, Q4_0_128, Q8_0_C, Q8_0_32 };
+enum class ExtraQuantType { F16, Q4_0_C, Q8_1_C, Q4_0_128, Q4_0_64, Q8_0_C, Q8_0_32 };
 
 ov::Core & ov_singleton_core();
 
@@ -99,6 +99,24 @@ int ggml_openvino_getenv_int(const char * var, int default_value = 0);
 // Check if running on NPU
 bool ggml_openvino_is_npu();
 
+// MoE detection. ggml_openvino_note_moe_expert_weight() latches a process-global flag
+// that ggml_openvino_has_moe_expert_weights() reports. It is called from supports_op()
+// the first time a GGML_OP_MUL_MAT_ID (the expert-routed matmul) is seen, which is the
+// defining op of a MoE model. The latch is set at op-placement time (not weight load):
+// the scheduler queries op placement before the expert weights are streamed in, and it
+// makes multiple placement passes, so the first pass that encounters MUL_MAT_ID sets the
+// flag and subsequent passes converge on the full-MoE layout. This lets the backend
+// recognize "this is a MoE model" without any architecture name.
+void ggml_openvino_note_moe_expert_weight();
+bool ggml_openvino_has_moe_expert_weights();
+
+// Whether to keep the whole MoE on one OV submodel instead of fragmenting at every
+// MoE node (see the per-node "force to CPU" gates). Auto-detected: ON when the model
+// has expert-routed matmuls (a MoE model) on the dynamic-shape devices (CPU/GPU),
+// OFF on NPU (static path). Keeping the whole MoE on OV is numerically correct on both
+// CPU and GPU and avoids the graph fragmentation that caused index corruption on GPU.
+bool ggml_openvino_full_moe_enabled();
+
 // Get requantization type for a tensor type (returns nullopt if no requant needed)
 std::optional<ExtraQuantType> ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant = false);