cp: feat(qwen3_5): port dense Qwen3.5 to a custom-model(2557) into r0.5.0

examples/vlm_finetune/qwen3_5/qwen3_5_27b_tp4pp4.yaml

@@ -37,6 +37,12 @@ rng:
 model:
   _target_: nemo_automodel.NeMoAutoModelForImageTextToText.from_pretrained
   pretrained_model_name_or_path: Qwen/Qwen3.5-27B
+  backend:
+    _target_: nemo_automodel.components.models.common.BackendConfig
+    attn: sdpa
+    linear: torch
+    rms_norm: torch
+    rope_fusion: false
   attn_implementation: sdpa
 
 processor:

examples/vlm_finetune/qwen3_5/qwen3_5_4b.yaml

@@ -38,6 +38,12 @@ rng:
 model:
   _target_: nemo_automodel.NeMoAutoModelForImageTextToText.from_pretrained
   pretrained_model_name_or_path: Qwen/Qwen3.5-4B
+  backend:
+    _target_: nemo_automodel.components.models.common.BackendConfig
+    attn: sdpa
+    linear: torch
+    rms_norm: torch
+    rope_fusion: false
   attn_implementation: sdpa
 
 

examples/vlm_finetune/qwen3_5/qwen3_5_4b_neat_packing.yaml

@@ -37,6 +37,12 @@ rng:
 model:
   _target_: nemo_automodel.NeMoAutoModelForImageTextToText.from_pretrained
   pretrained_model_name_or_path: Qwen/Qwen3.5-4B
+  backend:
+    _target_: nemo_automodel.components.models.common.BackendConfig
+    attn: sdpa
+    linear: torch
+    rms_norm: torch
+    rope_fusion: false
   attn_implementation: flash_attention_2
 
 processor:

examples/vlm_finetune/qwen3_5/qwen3_5_9b.yaml

@@ -37,6 +37,12 @@ rng:
 model:
   _target_: nemo_automodel.NeMoAutoModelForImageTextToText.from_pretrained
   pretrained_model_name_or_path: Qwen/Qwen3.5-9B
+  backend:
+    _target_: nemo_automodel.components.models.common.BackendConfig
+    attn: sdpa
+    linear: torch
+    rms_norm: torch
+    rope_fusion: false
   attn_implementation: sdpa
 
 

examples/vlm_finetune/qwen3_5/qwen3_6_27b.yaml

@@ -36,6 +36,12 @@ rng:
 model:
   _target_: nemo_automodel.NeMoAutoModelForImageTextToText.from_pretrained
   pretrained_model_name_or_path: Qwen/Qwen3.6-27B
+  backend:
+    _target_: nemo_automodel.components.models.common.BackendConfig
+    attn: sdpa
+    linear: torch
+    rms_norm: torch
+    rope_fusion: false
   attn_implementation: sdpa
   torch_dtype: bfloat16
   text_config:

examples/vlm_finetune/qwen3_5/qwen3_6_27b_lora.yaml

@@ -39,6 +39,12 @@ rng:
 model:
   _target_: nemo_automodel.NeMoAutoModelForImageTextToText.from_pretrained
   pretrained_model_name_or_path: Qwen/Qwen3.6-27B
+  backend:
+    _target_: nemo_automodel.components.models.common.BackendConfig
+    attn: sdpa
+    linear: torch
+    rms_norm: torch
+    rope_fusion: false
   attn_implementation: sdpa
   torch_dtype: bfloat16
 

examples/vlm_kd/qwen3_5/qwen3_5_vl_4b_kd.yaml

@@ -41,6 +41,12 @@ rng:
 model:
   _target_: nemo_automodel.NeMoAutoModelForImageTextToText.from_pretrained
   pretrained_model_name_or_path: Qwen/Qwen3.5-4B
+  backend:
+    _target_: nemo_automodel.components.models.common.BackendConfig
+    attn: sdpa
+    linear: torch
+    rms_norm: torch
+    rope_fusion: false
   attn_implementation: sdpa
 
 # Teacher

nemo_automodel/_transformers/capabilities.py

@@ -102,6 +102,10 @@ def _is_hybrid(model: "nn.Module") -> bool:
     inner = getattr(model, "language_model", None)
     if inner is not None:
         candidates.append(getattr(inner, "config", None))
+    # VLM configs nest the decoder config under ``text_config``.
+    for c in list(candidates):
+        if c is not None:
+            candidates.append(getattr(c, "text_config", None))
     for config in candidates:
         if config is None:
             continue
@@ -111,6 +115,11 @@ def _is_hybrid(model: "nn.Module") -> bool:
                 return True
         if getattr(config, "is_hybrid_model", False) is True:
             return True
+        # Qwen3.5 / Qwen3-Next style: per-layer ``layer_types`` mixing
+        # ``linear_attention`` (gated-delta / SSM) with ``full_attention``.
+        layer_types = getattr(config, "layer_types", None)
+        if layer_types and any(str(t) == "linear_attention" for t in layer_types):
+            return True
     return False
 
 

nemo_automodel/_transformers/kernel_patches.py

@@ -42,16 +42,10 @@
 
 logger = logging.getLogger(__name__)
 
-_MODEL_RUNTIME_PATCHES = {
-    "Qwen3_5ForCausalLM": (
-        "nemo_automodel.components.models.qwen3_5_moe.cp_linear_attn",
-        "apply_model_runtime_patches",
-    ),
-    "Qwen3_5ForConditionalGeneration": (
-        "nemo_automodel.components.models.qwen3_5_moe.cp_linear_attn",
-        "apply_model_runtime_patches",
-    ),
-}
+# Models build their CP/fp32-gate-aware modules at construction; no load-time
+# runtime patching is registered here. (Qwen3.5 dense/MoE build the native
+# CPAwareGatedDeltaNet + fp32 SSMGate in their model __init__.)
+_MODEL_RUNTIME_PATCHES = {}
 
 
 def _assert_same_signature(original, patched):

nemo_automodel/components/distributed/optimized_tp_plans.py

@@ -717,6 +717,10 @@ def _parallelize_falcon_h1(
     _get_class_qualname(Qwen3ForSequenceClassification): _parallelize_qwen_classification,
     # Hard-coded qualname to avoid eagerly importing transformers.models.qwen3_5.
     "transformers.models.qwen3_5.modeling_qwen3_5.Qwen3_5ForConditionalGeneration": _parallelize_qwen3_5_vlm,
+    # NeMo-native Qwen3.5 dense (custom-model port): same plan — shard self_attn +
+    # MLP, leave the GatedDeltaNet (linear_attn) replicated.
+    "nemo_automodel.components.models.qwen3_5.model.Qwen3_5ForConditionalGeneration": _parallelize_qwen3_5_vlm,
+    "nemo_automodel.components.models.qwen3_5.model.Qwen3_5ForCausalLM": _parallelize_qwen3_5_vlm,
     # Falcon-H1 (hybrid Transformer + Mamba2). HF ships only a minimal
     # _tp_plan ({"lm_head": "colwise_gather_output"}) and names its MLP
     # "feed_forward", so the generic fallback plan leaves feed_forward replicated

nemo_automodel/components/distributed/parallelizer.py

@@ -512,16 +512,13 @@ class Qwen3_5ParallelizationStrategy(DefaultParallelizationStrategy):
     """
 
     def parallelize(self, model, device_mesh, dp_shard_cp_mesh_name="dp_shard_cp", **kwargs):
-        # Patch HF GatedDeltaNet for FSDP mixed-dtype support (and CP if enabled)
-        from nemo_automodel.components.models.qwen3_5_moe.cp_linear_attn import patch_hf_model
-
         cp_mesh_name = dp_shard_cp_mesh_name.replace("dp_shard_", "")
         cp_enabled = cp_mesh_name in device_mesh.mesh_dim_names and device_mesh[cp_mesh_name].size() > 1
-        patch_hf_model(model, cp_enabled=cp_enabled)
 
-        # Submodules that must compute in fp32 even under fp32 master weights (bf16
-        # compute). patch_hf_model declares ``_fp32_params`` on the model.
-        fp32_compute_module_names = tuple(getattr(model, "_keep_in_fp32_modules_strict", None) or ())
+        # The Qwen3.5 model builds CPAwareGatedDeltaNet with a fp32 ``SSMGate``
+        # (``_fp32_params``) at construction — no runtime patch needed. Keep those
+        # params in their own dtype-uniform fp32 FSDP group (true master weights).
+        fp32_compute_module_names = ("_fp32_params",)
 
         # Delegate TP, AC, mixed precision to the default strategy, but
         # override the FSDP sharding to use fully_shard_by_dtype.

nemo_automodel/components/models/common/utils.py

@@ -451,7 +451,9 @@ def get_rope_config(config) -> tuple[float, dict, float]:
     return rope_theta, rope_parameters, partial_rotary_factor
 
 
-def cast_model_to_dtype(model: nn.Module, dtype: torch.dtype = torch.bfloat16) -> None:
+def cast_model_to_dtype(
+    model: nn.Module, dtype: torch.dtype = torch.bfloat16, skip_modules: tuple[str, ...] = ()
+) -> None:
     """Cast model parameters to the target dtype, keeping fp32 modules in full precision.
 
     Respects ``_keep_in_fp32_modules`` / ``_keep_in_fp32_modules_strict`` on
@@ -465,10 +467,34 @@ def cast_model_to_dtype(model: nn.Module, dtype: torch.dtype = torch.bfloat16) -
     Args:
         model: The model whose parameters should be cast.
         dtype: Target dtype (e.g. ``torch.bfloat16``).
+        skip_modules: Names of immediate submodules to leave entirely untouched
+            (kept at their current dtype). Unlike the ``_keep_in_fp32_modules``
+            restore path, these are *detached* during the cast so ``model.to()``
+            never visits them — the only reliable way to preserve an fp32
+            parameter once it is FSDP2-sharded (post-shard ``.data`` reassignment
+            does not stick). The caller must guarantee each skipped submodule is
+            its own dtype-uniform FSDP group (e.g. Qwen3.5's ``_fp32_params``
+            holder, sharded separately in fp32), so leaving it fp32 cannot break
+            FSDP's uniform-dtype rule.
     """
     fp32_keywords = _get_fp32_module_keywords(model)
 
-    model.to(dtype)
+    # Detach skip_modules so ``model.to(dtype)`` does not descend into them. This
+    # preserves their exact dtype (e.g. fp32 master weights) through the cast.
+    detached: list[tuple[nn.Module, str, nn.Module]] = []
+    if skip_modules:
+        for _, parent in model.named_modules():
+            for child_name, child in list(parent._modules.items()):
+                if child is not None and child_name in skip_modules:
+                    detached.append((parent, child_name, child))
+                    parent._modules[child_name] = None
+
+    try:
+        model.to(dtype)
+    finally:
+        for parent, child_name, child in detached:
+            parent._modules[child_name] = child
+
     if fp32_keywords:
         if _has_dtensor_params(model):
             logger.warning(