Upload folder using huggingface_hub

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 checkpoint-200000/trainer_state.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-250000/trainer_state.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-300000/trainer_state.json filter=lfs diff=lfs merge=lfs -text

checkpoint-300000/added_tokens.json

@@ -0,0 +1,45 @@
+{
+  "</box>": 151677,
+  "</img>": 151670,
+  "</quad>": 151673,
+  "</ref>": 151675,
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<IMG_CONTEXT>": 151671,
+  "<box>": 151676,
+  "<img>": 151669,
+  "<quad>": 151672,
+  "<ref>": 151674,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|action_end|>": 151680,
+  "<|action_pad|>": 151679,
+  "<|action_pass|>": 151681,
+  "<|action_start|>": 151678,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|state_end|>": 151684,
+  "<|state_pad|>": 151683,
+  "<|state_start|>": 151682,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652,
+  "<|vla|>": 151685
+}

checkpoint-300000/chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+'}}{% if message['content'] is string %}{{ message['content'] }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' %}{{ '<image>
+' }}{% elif content['type'] == 'video' %}{{ '<video>
+' }}{% elif content['type'] == 'text' %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}{{'<|im_end|>
+'}}{% endfor %}{% if add_generation_prompt %}{{'<|im_start|>assistant
+' }}{% endif %}

checkpoint-300000/config.json

@@ -0,0 +1,28 @@
+{
+  "action_chunk_size": 30,
+  "action_pass_id": 151681,
+  "action_token_id": 151679,
+  "architectures": [
+    "EO1InternVLPiFlowMatchingModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_eo1_internvl.EO1InternVLPiFlowMatchingConfig",
+    "AutoModel": "modeling_eo1_internvl.EO1InternVLPiFlowMatchingModel"
+  },
+  "backbone_name_or_path": "hugg_model/InternVL3_5-1B",
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "expert_hidden_size": 1024,
+  "expert_init_from_backbone": false,
+  "expert_intermediate_size": 3072,
+  "expert_layer_mapping": "last",
+  "expert_num_attention_heads": 16,
+  "expert_num_hidden_layers": 18,
+  "ignore_index": -100,
+  "img_context_token_id": 151671,
+  "max_action_dim": 32,
+  "model_type": "eo1_internvl_pi",
+  "num_denoise_steps": 10,
+  "pad_token_id": 151643,
+  "transformers_version": "4.56.0"
+}

checkpoint-300000/configuration_eo1_internvl.py

@@ -0,0 +1,79 @@
+# Copyright 2026 EO-Robotics Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from transformers.configuration_utils import PretrainedConfig
+
+
+class EO1InternVLPiFlowMatchingConfig(PretrainedConfig):
+    """
+    EO1 Flow-Matching wrapper for InternVL backbone + Pi05-style action expert.
+
+    Pi05 key properties (mirrors `openpi.models.pi0` with `pi05=True`):
+    - Prefix uses standard *causal* LM forward (flash-attn friendly) to build a per-layer KV cache.
+    - Action block is bidirectional within itself and can attend to the cached prefix KV.
+    - Flow-matching timestep is injected via AdaRMSNorm in the action expert (not concatenated into embeddings).
+    - Continuous state token in suffix is *disabled* (state should be encoded in text if needed).
+    """
+
+    model_type = "eo1_internvl_pi"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        backbone_name_or_path: str | None = None,
+        # Flow matching
+        action_chunk_size: int = 16,
+        max_action_dim: int = 32,
+        num_denoise_steps: int = 10,
+        rtc_delay: int = 5,
+        # Tokens
+        action_token_id: int | None = None,
+        action_pass_id: int | None = None,
+        img_context_token_id: int | None = None,
+        ignore_index: int = -100,
+        # Expert init
+        expert_init_from_backbone: bool = False,
+        # Expert architecture (Pi05-style: smaller action expert than VLM)
+        expert_num_hidden_layers: int | None = 18,
+        expert_hidden_size: int | None = 1024,
+        expert_intermediate_size: int | None = 3072,
+        expert_num_attention_heads: int | None = 16,
+        expert_layer_mapping: str = "last",
+        **kwargs,
+    ):
+        self.backbone_name_or_path = backbone_name_or_path
+
+        self.action_chunk_size = int(action_chunk_size)
+        self.max_action_dim = int(max_action_dim)
+        self.num_denoise_steps = int(num_denoise_steps)
+        self.rtc_delay = int(rtc_delay)
+
+        self.action_token_id = action_token_id
+        self.action_pass_id = action_pass_id
+        self.img_context_token_id = img_context_token_id
+        self.ignore_index = int(ignore_index)
+
+        self.expert_init_from_backbone = bool(expert_init_from_backbone)
+        self.expert_num_hidden_layers = None if expert_num_hidden_layers is None else int(expert_num_hidden_layers)
+        self.expert_hidden_size = None if expert_hidden_size is None else int(expert_hidden_size)
+        self.expert_intermediate_size = None if expert_intermediate_size is None else int(expert_intermediate_size)
+        self.expert_num_attention_heads = None if expert_num_attention_heads is None else int(expert_num_attention_heads)
+        self.expert_layer_mapping = str(expert_layer_mapping)
+
+        super().__init__(**kwargs)
+
+
+EO1InternVLPiFlowMatchingConfig.register_for_auto_class()

checkpoint-300000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:317388e105ef997dcb0f3ec00c76bc4a8a77767d21c93197206b8494ecbbbbd6
+size 3726487744

checkpoint-300000/modeling_eo1_internvl.py

@@ -0,0 +1,1284 @@
+# Copyright 2026 EO-Robotics Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import copy
+import math
+import os
+from dataclasses import dataclass
+from typing import Any
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F  # noqa: N812
+from torch import Tensor
+from transformers.modeling_outputs import ModelOutput
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import logging
+
+from .configuration_eo1_internvl import EO1InternVLPiFlowMatchingConfig
+
+
+logger = logging.get_logger(__name__)
+
+
+def create_sinusoidal_pos_embedding(
+    time: torch.tensor,
+    dimension: int,
+    min_period: float = 4e-3,
+    max_period: float = 4.0,
+    device: str | torch.device = "cpu",
+) -> Tensor:
+    """Sine-cosine embedding for scalar time in [0,1]. Matches openpi `posemb_sincos` sensitivity."""
+    if dimension % 2 != 0:
+        raise ValueError(f"dimension ({dimension}) must be divisible by 2")
+    if time.ndim != 1:
+        raise ValueError("The time tensor is expected to be of shape `(batch_size, )`.")
+
+    fraction = torch.linspace(0.0, 1.0, dimension // 2, device=device)
+    period = min_period * (max_period / min_period) ** fraction
+    scaling_factor = 1.0 / period * 2 * math.pi
+    sin_input = scaling_factor[None, :] * time[:, None]
+    return torch.cat([torch.sin(sin_input), torch.cos(sin_input)], dim=1)
+
+
+def _masked_fill_min(x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
+    """Fill with dtype-min where mask is False. `mask` is broadcastable to `x`."""
+    return x.masked_fill(~mask, torch.finfo(x.dtype).min)
+
+
+class AdaRMSNorm(nn.Module):
+    """
+    Pi05-style AdaRMSNorm (openpi `gemma.RMSNorm` with `cond!=None`):
+    - RMS normalize in float32
+    - per-layer modulation = Linear(cond -> 3*D) initialized to zeros
+    - output = normed * (1 + scale) + shift
+    - returns gate for gated residual.
+    """
+
+    def __init__(self, dim: int, *, eps: float = 1e-6):
+        super().__init__()
+        self.eps = float(eps)
+        self.modulation = nn.Linear(dim, dim * 3, bias=True)
+        nn.init.zeros_(self.modulation.weight)
+        nn.init.zeros_(self.modulation.bias)
+
+    def forward(self, x: torch.Tensor, cond: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+        if cond is None:
+            raise ValueError("AdaRMSNorm requires `cond` (Pi05 mode).")
+        if cond.ndim not in (2, 3):
+            raise ValueError(f"cond must be (B,D) or (B,T,D), got {tuple(cond.shape)}")
+        if x.ndim != 3:
+            raise ValueError(f"x must be (B,T,D), got {tuple(x.shape)}")
+        if x.shape[0] != cond.shape[0]:
+            raise ValueError(f"Batch mismatch: {x.shape[0]=} vs {cond.shape[0]=}")
+        if cond.shape[-1] != x.shape[-1]:
+            raise ValueError(f"Dim mismatch: {x.shape[-1]=} vs {cond.shape[-1]=}")
+        if cond.ndim == 3 and x.shape[:2] != cond.shape[:2]:
+            raise ValueError(f"Token mismatch: {x.shape[:2]=} vs {cond.shape[:2]=}")
+
+        x_dtype = x.dtype
+        x_f32 = x.float()
+        var = x_f32.pow(2).mean(dim=-1, keepdim=True)
+        normed = x_f32 * torch.rsqrt(var + self.eps)
+
+        mod = self.modulation(cond).to(dtype=x_f32.dtype)
+        scale, shift, gate = mod.chunk(3, dim=-1)
+        if cond.ndim == 2:
+            scale = scale[:, None, :]
+            shift = shift[:, None, :]
+            gate = gate[:, None, :]
+        out = normed * (1 + scale) + shift
+        return out.to(dtype=x_dtype), gate.to(dtype=x_dtype)
+
+
+class Qwen2PiSelfAttention(nn.Module):
+    """
+    Qwen2 attention variant for Pi05 action expert:
+    - queries from suffix tokens (action tokens)
+    - keys/values from concat(prefix_kv_cache, suffix_kv)
+    - uses full (non-causal) attention mask provided by caller.
+    """
+
+    def __init__(self, qwen_config: Any, layer_idx: int):
+        super().__init__()
+        try:
+            from transformers.models.qwen2.modeling_qwen2 import apply_rotary_pos_emb, repeat_kv
+        except Exception as e:  # pragma: no cover
+            raise ImportError("transformers qwen2 internals are required for eo_pi_internvl.") from e
+
+        self._apply_rotary_pos_emb = apply_rotary_pos_emb
+        self._repeat_kv = repeat_kv
+
+        self.layer_idx = int(layer_idx)
+        self.hidden_size = int(qwen_config.hidden_size)
+        self.num_heads = int(qwen_config.num_attention_heads)
+        self.num_kv_heads = int(qwen_config.num_key_value_heads)
+        self.num_kv_groups = self.num_heads // self.num_kv_heads
+        self.head_dim = int(getattr(qwen_config, "head_dim", self.hidden_size // self.num_heads))
+        self.scaling = self.head_dim**-0.5
+
+        self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=True)
+        self.k_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=True)
+        self.v_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=True)
+        self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        *,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        attention_mask: torch.Tensor | None,
+        prefix_k: torch.Tensor,
+        prefix_v: torch.Tensor,
+    ) -> torch.Tensor:
+        # hidden_states: (B, S, D)
+        bsz, seqlen, _ = hidden_states.shape
+        q = self.q_proj(hidden_states).view(bsz, seqlen, self.num_heads, self.head_dim).transpose(1, 2)
+        k = self.k_proj(hidden_states).view(bsz, seqlen, self.num_kv_heads, self.head_dim).transpose(1, 2)
+        v = self.v_proj(hidden_states).view(bsz, seqlen, self.num_kv_heads, self.head_dim).transpose(1, 2)
+
+        cos, sin = position_embeddings
+        q, k = self._apply_rotary_pos_emb(q, k, cos, sin)
+
+        if prefix_k.ndim != 4 or prefix_v.ndim != 4:
+            raise ValueError(f"prefix_k/v must be (B, n_kv, P, hd), got {tuple(prefix_k.shape)}, {tuple(prefix_v.shape)}")
+        if int(prefix_k.shape[0]) != bsz or int(prefix_v.shape[0]) != bsz:
+            raise ValueError("prefix_k/v batch mismatch.")
+        if int(prefix_k.shape[1]) != self.num_kv_heads or int(prefix_v.shape[1]) != self.num_kv_heads:
+            raise ValueError(
+                "prefix_k/v num_kv_heads mismatch: "
+                f"{int(prefix_k.shape[1])=} {int(prefix_v.shape[1])=} vs {self.num_kv_heads=}"
+            )
+        if int(prefix_k.shape[-1]) != self.head_dim or int(prefix_v.shape[-1]) != self.head_dim:
+            raise ValueError("prefix_k/v head_dim mismatch.")
+
+        k_all = torch.cat([prefix_k, k], dim=2)  # (B, n_kv, P+S, hd)
+        v_all = torch.cat([prefix_v, v], dim=2)
+
+        k_all = self._repeat_kv(k_all, self.num_kv_groups)  # (B, n_heads, P+S, hd)
+        v_all = self._repeat_kv(v_all, self.num_kv_groups)
+
+        # attention_mask: (B, 1, S, P+S) additive (0 or -inf), broadcast to heads
+        if attention_mask is not None:
+            if attention_mask.ndim != 4:
+                raise ValueError(f"attention_mask must be 4D (B,1,S,K), got {tuple(attention_mask.shape)}")
+            attn_mask = attention_mask.expand(bsz, self.num_heads, seqlen, k_all.shape[-2])
+        else:
+            attn_mask = None
+
+        attn_out = torch.nn.functional.scaled_dot_product_attention(
+            q, k_all, v_all, attn_mask=attn_mask, dropout_p=0.0, is_causal=False
+        )  # (B, n_heads, S, hd)
+        attn_out = attn_out.transpose(1, 2).contiguous().view(bsz, seqlen, self.num_heads * self.head_dim)
+        return self.o_proj(attn_out)
+
+
+class Qwen2PiMLP(nn.Module):
+    def __init__(self, qwen_config: Any):
+        super().__init__()
+        hidden = int(qwen_config.hidden_size)
+        inter = int(qwen_config.intermediate_size)
+        self.gate_proj = nn.Linear(hidden, inter, bias=False)
+        self.up_proj = nn.Linear(hidden, inter, bias=False)
+        self.down_proj = nn.Linear(inter, hidden, bias=False)
+        act_name = str(getattr(qwen_config, "hidden_act", "silu"))
+        if act_name != "silu":
+            logger.warning_once("EO Pi action expert: forcing SiLU hidden_act for MLP (got %s).", act_name)
+        self.act = nn.SiLU()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.down_proj(self.act(self.gate_proj(x)) * self.up_proj(x))
+
+
+class Qwen2PiDecoderLayer(nn.Module):
+    def __init__(self, qwen_config: Any, layer_idx: int):
+        super().__init__()
+        eps = float(getattr(qwen_config, "rms_norm_eps", 1e-6))
+        self.input_layernorm = AdaRMSNorm(int(qwen_config.hidden_size), eps=eps)
+        self.self_attn = Qwen2PiSelfAttention(qwen_config=qwen_config, layer_idx=layer_idx)
+        self.post_attention_layernorm = AdaRMSNorm(int(qwen_config.hidden_size), eps=eps)
+        self.mlp = Qwen2PiMLP(qwen_config=qwen_config)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        *,
+        adarms_cond: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        attention_mask: torch.Tensor | None,
+        prefix_k: torch.Tensor,
+        prefix_v: torch.Tensor,
+    ) -> torch.Tensor:
+        residual = hidden_states
+        x, gate = self.input_layernorm(hidden_states, adarms_cond)
+        x = self.self_attn(
+            x,
+            position_embeddings=position_embeddings,
+            attention_mask=attention_mask,
+            prefix_k=prefix_k,
+            prefix_v=prefix_v,
+        )
+        hidden_states = residual + x * gate
+
+        residual = hidden_states
+        x, gate = self.post_attention_layernorm(hidden_states, adarms_cond)
+        x = self.mlp(x)
+        hidden_states = residual + x * gate
+        return hidden_states
+
+
+class Qwen2PiActionExpert(nn.Module):
+    def __init__(self, qwen_config: Any, *, init_from_qwen2_lm: nn.Module | None = None):
+        super().__init__()
+        try:
+            from transformers.models.qwen2.modeling_qwen2 import Qwen2RotaryEmbedding
+        except Exception as e:  # pragma: no cover
+            raise ImportError("transformers qwen2 internals are required for eo_pi_internvl.") from e
+
+        self.config = qwen_config
+        self.num_layers = int(qwen_config.num_hidden_layers)
+        self.layers = nn.ModuleList([Qwen2PiDecoderLayer(qwen_config, i) for i in range(self.num_layers)])
+        self.rotary_emb = Qwen2RotaryEmbedding(qwen_config)
+        self.final_norm = AdaRMSNorm(int(qwen_config.hidden_size), eps=float(getattr(qwen_config, "rms_norm_eps", 1e-6)))
+
+        if init_from_qwen2_lm is not None:
+            self._init_from_qwen2_lm(init_from_qwen2_lm)
+
+    def _init_from_qwen2_lm(self, qwen2_lm: nn.Module):
+        """
+        Copy attention/MLP weights from a Qwen2ForCausalLM (or Qwen2Model) into this expert.
+        AdaRMSNorm modulation stays zero-init to match Pi05.
+        """
+        src_layers = None
+        if hasattr(qwen2_lm, "model") and hasattr(qwen2_lm.model, "layers"):
+            src_layers = qwen2_lm.model.layers
+        elif hasattr(qwen2_lm, "layers"):
+            src_layers = qwen2_lm.layers
+        if src_layers is None:
+            raise ValueError("Unsupported qwen2_lm: cannot locate `.model.layers`.")
+
+        if len(src_layers) != len(self.layers):
+            raise ValueError(f"Layer count mismatch: {len(src_layers)=} vs {len(self.layers)=}")
+
+        for dst, src in zip(self.layers, src_layers, strict=True):
+            # attention
+            dst.self_attn.q_proj.load_state_dict(src.self_attn.q_proj.state_dict())
+            dst.self_attn.k_proj.load_state_dict(src.self_attn.k_proj.state_dict())
+            dst.self_attn.v_proj.load_state_dict(src.self_attn.v_proj.state_dict())
+            dst.self_attn.o_proj.load_state_dict(src.self_attn.o_proj.state_dict())
+            # mlp
+            dst.mlp.gate_proj.load_state_dict(src.mlp.gate_proj.state_dict())
+            dst.mlp.up_proj.load_state_dict(src.mlp.up_proj.state_dict())
+            dst.mlp.down_proj.load_state_dict(src.mlp.down_proj.state_dict())
+
+    def forward(
+        self,
+        action_tokens: torch.Tensor,
+        *,
+        prefix_kv_cache: list[tuple[torch.Tensor, torch.Tensor]],
+        prefix_key_mask: torch.Tensor,
+        position_ids: torch.Tensor,
+        adarms_cond: torch.Tensor,
+        suffix_key_mask: torch.Tensor | None = None,
+    ) -> torch.Tensor:
+        """
+        Args:
+            action_tokens: (B, S, D)
+            prefix_kv_cache: list[(k,v)] each (B, n_kv, P, hd) from InternVL prefix expert.
+            prefix_key_mask: (B, P) bool, True = valid prefix token.
+            position_ids: (B, S) positions for action tokens (prefix_len + [0..S-1]).
+            adarms_cond: (B, D) time conditioning vector.
+            suffix_key_mask: (B, S) bool, True = valid suffix token (optional; for padding).
+        """
+        if action_tokens.ndim != 3:
+            raise ValueError(f"action_tokens must be (B,S,D), got {tuple(action_tokens.shape)}")
+        bsz, s_len, _ = action_tokens.shape
+        if prefix_key_mask.ndim != 2 or int(prefix_key_mask.shape[0]) != bsz:
+            raise ValueError(f"prefix_key_mask must be (B,P), got {tuple(prefix_key_mask.shape)}")
+        if position_ids.shape != (bsz, s_len):
+            raise ValueError(f"position_ids must be (B,S)={bsz,s_len}, got {tuple(position_ids.shape)}")
+        if len(prefix_kv_cache) == 0:
+            raise ValueError("prefix_kv_cache is empty.")
+
+        # (cos,sin) for suffix tokens only (RoPE positions already baked into prefix cache).
+        position_embeddings = self.rotary_emb(action_tokens, position_ids)
+
+        if suffix_key_mask is None:
+            suffix_key_mask = torch.ones((bsz, s_len), device=action_tokens.device, dtype=torch.bool)
+        if suffix_key_mask.shape != (bsz, s_len):
+            raise ValueError(f"suffix_key_mask must be (B,S), got {tuple(suffix_key_mask.shape)}")
+
+        # Build Pi05 action-block attention mask: suffix queries attend to (valid prefix keys) + (valid suffix keys) fully.
+        prefix_part = (suffix_key_mask[:, None, :, None] & prefix_key_mask[:, None, None, :])  # (B,1,S,P)
+        suffix_part = (suffix_key_mask[:, None, :, None] & suffix_key_mask[:, None, None, :])  # (B,1,S,S)
+        allow = torch.cat([prefix_part, suffix_part], dim=-1)  # (B,1,S,P+S)
+        attn_mask = torch.zeros(
+            (bsz, 1, s_len, int(prefix_key_mask.shape[1]) + s_len),
+            device=action_tokens.device,
+            dtype=action_tokens.dtype,
+        )
+        attn_mask = _masked_fill_min(attn_mask, allow)
+
+        x = action_tokens
+        for layer_idx, layer in enumerate(self.layers):
+            if layer_idx >= len(prefix_kv_cache):
+                raise ValueError(
+                    "prefix_kv_cache has fewer layers than action expert. "
+                    f"{len(prefix_kv_cache)=} < {len(self.layers)=}"
+                )
+            pk, pv = prefix_kv_cache[layer_idx]
+            x = layer(
+                x,
+                adarms_cond=adarms_cond,
+                position_embeddings=position_embeddings,
+                attention_mask=attn_mask,
+                prefix_k=pk,
+                prefix_v=pv,
+            )
+
+        x, _ = self.final_norm(x, adarms_cond)
+        return x
+
+
+class Qwen3HeadRMSNorm(nn.Module):
+    """Qwen3-style RMSNorm used for `q_norm`/`k_norm` on per-head dim."""
+
+    def __init__(self, dim: int, *, eps: float = 1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(dim))
+        self.eps = float(eps)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        dtype = x.dtype
+        x_f32 = x.float()
+        var = x_f32.pow(2).mean(dim=-1, keepdim=True)
+        x_norm = x_f32 * torch.rsqrt(var + self.eps)
+        return (self.weight * x_norm).to(dtype=dtype)
+
+
+class Qwen3PiSelfAttention(nn.Module):
+    """
+    Qwen3 attention variant for Pi05 action expert:
+    - queries from suffix tokens (action tokens)
+    - keys/values from concat(prefix_kv_cache, suffix_kv)
+    - uses full (non-causal) attention mask provided by caller.
+    """
+
+    def __init__(self, qwen_config: Any, layer_idx: int):
+        super().__init__()
+        try:
+            from transformers.models.qwen3.modeling_qwen3 import apply_rotary_pos_emb, repeat_kv
+        except Exception as e:  # pragma: no cover
+            raise ImportError("transformers qwen3 internals are required for eo_pi_internvl.") from e
+
+        self._apply_rotary_pos_emb = apply_rotary_pos_emb
+        self._repeat_kv = repeat_kv
+
+        self.layer_idx = int(layer_idx)
+        self.hidden_size = int(qwen_config.hidden_size)
+        self.num_heads = int(qwen_config.num_attention_heads)
+        self.num_kv_heads = int(qwen_config.num_key_value_heads)
+        self.num_kv_groups = self.num_heads // self.num_kv_heads
+        self.head_dim = int(getattr(qwen_config, "head_dim", self.hidden_size // self.num_heads))
+
+        attn_bias = bool(getattr(qwen_config, "attention_bias", False))
+        self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=attn_bias)
+        self.k_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=attn_bias)
+        self.v_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=attn_bias)
+        self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=attn_bias)
+
+        eps = float(getattr(qwen_config, "rms_norm_eps", 1e-6))
+        self.q_norm = Qwen3HeadRMSNorm(self.head_dim, eps=eps)
+        self.k_norm = Qwen3HeadRMSNorm(self.head_dim, eps=eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        *,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        attention_mask: torch.Tensor | None,
+        prefix_k: torch.Tensor,
+        prefix_v: torch.Tensor,
+    ) -> torch.Tensor:
+        bsz, seqlen, _ = hidden_states.shape
+        hidden_shape = (bsz, seqlen, -1, self.head_dim)
+
+        q = self.q_proj(hidden_states).view(hidden_shape)
+        k = self.k_proj(hidden_states).view(hidden_shape)
+        v = self.v_proj(hidden_states).view(hidden_shape)
+        q = self.q_norm(q).transpose(1, 2)  # (B,n_heads,S,hd)
+        k = self.k_norm(k).transpose(1, 2)  # (B,n_kv,S,hd)
+        v = v.transpose(1, 2)  # (B,n_kv,S,hd)
+
+        cos, sin = position_embeddings
+        q, k = self._apply_rotary_pos_emb(q, k, cos, sin)
+
+        if prefix_k.ndim != 4 or prefix_v.ndim != 4:
+            raise ValueError(
+                f"prefix_k/v must be (B, n_kv, P, hd), got {tuple(prefix_k.shape)}, {tuple(prefix_v.shape)}"
+            )
+        if int(prefix_k.shape[0]) != bsz or int(prefix_v.shape[0]) != bsz:
+            raise ValueError("prefix_k/v batch mismatch.")
+        if int(prefix_k.shape[1]) != self.num_kv_heads or int(prefix_v.shape[1]) != self.num_kv_heads:
+            raise ValueError(
+                "prefix_k/v num_kv_heads mismatch: "
+                f"{int(prefix_k.shape[1])=} {int(prefix_v.shape[1])=} vs {self.num_kv_heads=}"
+            )
+        if int(prefix_k.shape[-1]) != self.head_dim or int(prefix_v.shape[-1]) != self.head_dim:
+            raise ValueError("prefix_k/v head_dim mismatch.")
+
+        k_all = torch.cat([prefix_k, k], dim=2)  # (B, n_kv, P+S, hd)
+        v_all = torch.cat([prefix_v, v], dim=2)
+        k_all = self._repeat_kv(k_all, self.num_kv_groups)  # (B, n_heads, P+S, hd)
+        v_all = self._repeat_kv(v_all, self.num_kv_groups)
+
+        if attention_mask is not None:
+            if attention_mask.ndim != 4:
+                raise ValueError(f"attention_mask must be 4D (B,1,S,K), got {tuple(attention_mask.shape)}")
+            attn_mask = attention_mask.expand(bsz, self.num_heads, seqlen, k_all.shape[-2])
+        else:
+            attn_mask = None
+
+        attn_out = torch.nn.functional.scaled_dot_product_attention(
+            q, k_all, v_all, attn_mask=attn_mask, dropout_p=0.0, is_causal=False
+        )
+        attn_out = attn_out.transpose(1, 2).contiguous().view(bsz, seqlen, self.num_heads * self.head_dim)
+        return self.o_proj(attn_out)
+
+
+class Qwen3PiMLP(nn.Module):
+    def __init__(self, qwen_config: Any):
+        super().__init__()
+        hidden = int(qwen_config.hidden_size)
+        inter = int(qwen_config.intermediate_size)
+        self.gate_proj = nn.Linear(hidden, inter, bias=False)
+        self.up_proj = nn.Linear(hidden, inter, bias=False)
+        self.down_proj = nn.Linear(inter, hidden, bias=False)
+        act_name = str(getattr(qwen_config, "hidden_act", "silu"))
+        if act_name != "silu":
+            logger.warning_once("EO Pi action expert: forcing SiLU hidden_act for MLP (got %s).", act_name)
+        self.act = nn.SiLU()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.down_proj(self.act(self.gate_proj(x)) * self.up_proj(x))
+
+
+class Qwen3PiDecoderLayer(nn.Module):
+    def __init__(self, qwen_config: Any, layer_idx: int):
+        super().__init__()
+        eps = float(getattr(qwen_config, "rms_norm_eps", 1e-6))
+        self.input_layernorm = AdaRMSNorm(int(qwen_config.hidden_size), eps=eps)
+        self.self_attn = Qwen3PiSelfAttention(qwen_config=qwen_config, layer_idx=layer_idx)
+        self.post_attention_layernorm = AdaRMSNorm(int(qwen_config.hidden_size), eps=eps)
+        self.mlp = Qwen3PiMLP(qwen_config=qwen_config)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        *,
+        adarms_cond: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        attention_mask: torch.Tensor | None,
+        prefix_k: torch.Tensor,
+        prefix_v: torch.Tensor,
+    ) -> torch.Tensor:
+        residual = hidden_states
+        x, gate = self.input_layernorm(hidden_states, adarms_cond)
+        x = self.self_attn(
+            x,
+            position_embeddings=position_embeddings,
+            attention_mask=attention_mask,
+            prefix_k=prefix_k,
+            prefix_v=prefix_v,
+        )
+        hidden_states = residual + x * gate
+
+        residual = hidden_states
+        x, gate = self.post_attention_layernorm(hidden_states, adarms_cond)
+        x = self.mlp(x)
+        hidden_states = residual + x * gate
+        return hidden_states
+
+
+class Qwen3PiActionExpert(nn.Module):
+    def __init__(self, qwen_config: Any, *, init_from_qwen3_lm: nn.Module | None = None):
+        super().__init__()
+        try:
+            from transformers.models.qwen3.modeling_qwen3 import Qwen3RotaryEmbedding
+        except Exception as e:  # pragma: no cover
+            raise ImportError("transformers qwen3 internals are required for eo_pi_internvl.") from e
+
+        self.config = qwen_config
+        self.num_layers = int(qwen_config.num_hidden_layers)
+        self.layers = nn.ModuleList([Qwen3PiDecoderLayer(qwen_config, i) for i in range(self.num_layers)])
+        self.rotary_emb = Qwen3RotaryEmbedding(qwen_config)
+        self.final_norm = AdaRMSNorm(int(qwen_config.hidden_size), eps=float(getattr(qwen_config, "rms_norm_eps", 1e-6)))
+
+        if init_from_qwen3_lm is not None:
+            self._init_from_qwen3_lm(init_from_qwen3_lm)
+
+    def _init_from_qwen3_lm(self, qwen3_lm: nn.Module):
+        """
+        Copy attention/MLP weights from a Qwen3ForCausalLM (or Qwen3Model) into this expert.
+        AdaRMSNorm modulation stays zero-init to match Pi05.
+        """
+        src_layers = None
+        if hasattr(qwen3_lm, "model") and hasattr(qwen3_lm.model, "layers"):
+            src_layers = qwen3_lm.model.layers
+        elif hasattr(qwen3_lm, "layers"):
+            src_layers = qwen3_lm.layers
+        if src_layers is None:
+            raise ValueError("Unsupported qwen3_lm: cannot locate `.model.layers`.")
+
+        if len(src_layers) != len(self.layers):
+            raise ValueError(f"Layer count mismatch: {len(src_layers)=} vs {len(self.layers)=}")
+
+        for dst, src in zip(self.layers, src_layers, strict=True):
+            dst.self_attn.q_proj.load_state_dict(src.self_attn.q_proj.state_dict())
+            dst.self_attn.k_proj.load_state_dict(src.self_attn.k_proj.state_dict())
+            dst.self_attn.v_proj.load_state_dict(src.self_attn.v_proj.state_dict())
+            dst.self_attn.o_proj.load_state_dict(src.self_attn.o_proj.state_dict())
+            # head norms
+            if hasattr(src.self_attn, "q_norm") and hasattr(dst.self_attn, "q_norm"):
+                dst.self_attn.q_norm.weight.data.copy_(src.self_attn.q_norm.weight.data)
+            if hasattr(src.self_attn, "k_norm") and hasattr(dst.self_attn, "k_norm"):
+                dst.self_attn.k_norm.weight.data.copy_(src.self_attn.k_norm.weight.data)
+            # mlp
+            dst.mlp.gate_proj.load_state_dict(src.mlp.gate_proj.state_dict())
+            dst.mlp.up_proj.load_state_dict(src.mlp.up_proj.state_dict())
+            dst.mlp.down_proj.load_state_dict(src.mlp.down_proj.state_dict())
+
+    def forward(
+        self,
+        action_tokens: torch.Tensor,
+        *,
+        prefix_kv_cache: list[tuple[torch.Tensor, torch.Tensor]],
+        prefix_key_mask: torch.Tensor,
+        position_ids: torch.Tensor,
+        adarms_cond: torch.Tensor,
+        suffix_key_mask: torch.Tensor | None = None,
+    ) -> torch.Tensor:
+        if action_tokens.ndim != 3:
+            raise ValueError(f"action_tokens must be (B,S,D), got {tuple(action_tokens.shape)}")
+        bsz, s_len, _ = action_tokens.shape
+        if prefix_key_mask.ndim != 2 or int(prefix_key_mask.shape[0]) != bsz:
+            raise ValueError(f"prefix_key_mask must be (B,P), got {tuple(prefix_key_mask.shape)}")
+        if position_ids.shape != (bsz, s_len):
+            raise ValueError(f"position_ids must be (B,S)={bsz,s_len}, got {tuple(position_ids.shape)}")
+        if len(prefix_kv_cache) == 0:
+            raise ValueError("prefix_kv_cache is empty.")
+
+        position_embeddings = self.rotary_emb(action_tokens, position_ids)
+
+        if suffix_key_mask is None:
+            suffix_key_mask = torch.ones((bsz, s_len), device=action_tokens.device, dtype=torch.bool)
+        if suffix_key_mask.shape != (bsz, s_len):
+            raise ValueError(f"suffix_key_mask must be (B,S), got {tuple(suffix_key_mask.shape)}")
+
+        prefix_part = (suffix_key_mask[:, None, :, None] & prefix_key_mask[:, None, None, :])  # (B,1,S,P)
+        suffix_part = (suffix_key_mask[:, None, :, None] & suffix_key_mask[:, None, None, :])  # (B,1,S,S)
+        allow = torch.cat([prefix_part, suffix_part], dim=-1)  # (B,1,S,P+S)
+        attn_mask = torch.zeros(
+            (bsz, 1, s_len, int(prefix_key_mask.shape[1]) + s_len),
+            device=action_tokens.device,
+            dtype=action_tokens.dtype,
+        )
+        attn_mask = _masked_fill_min(attn_mask, allow)
+
+        x = action_tokens
+        for layer_idx, layer in enumerate(self.layers):
+            if layer_idx >= len(prefix_kv_cache):
+                raise ValueError(
+                    "prefix_kv_cache has fewer layers than action expert. "
+                    f"{len(prefix_kv_cache)=} < {len(self.layers)=}"
+                )
+            pk, pv = prefix_kv_cache[layer_idx]
+            x = layer(
+                x,
+                adarms_cond=adarms_cond,
+                position_embeddings=position_embeddings,
+                attention_mask=attn_mask,
+                prefix_k=pk,
+                prefix_v=pv,
+            )
+
+        x, _ = self.final_norm(x, adarms_cond)
+        return x
+
+
+@dataclass
+class EO1InternVLPiFlowMatchingOutput(ModelOutput):
+    loss: torch.FloatTensor | None = None
+    fm_loss: torch.FloatTensor | None = None
+    fm_loss_pos: torch.FloatTensor | None = None
+    fm_loss_rot: torch.FloatTensor | None = None
+    fm_loss_gripper: torch.FloatTensor | None = None
+    ar_loss: torch.FloatTensor | None = None
+    actions: torch.FloatTensor | None = None
+
+    logits: torch.FloatTensor | None = None
+    hidden_states: tuple[torch.FloatTensor] | None = None
+    attentions: tuple[torch.FloatTensor] | None = None
+
+def count_params(module, trainable_only=False):
+    ps = module.parameters()
+    if trainable_only:
+        ps = [p for p in ps if p.requires_grad]
+    return sum(p.numel() for p in ps)
+
+class EO1InternVLPiFlowMatchingModel(PreTrainedModel):
+    """EO1 action model with InternVL prefix expert + Pi05-style (Qwen2/Qwen3) action expert (AdaRMSNorm timestep)."""
+
+    config_class = EO1InternVLPiFlowMatchingConfig
+    supports_gradient_checkpointing = True
+
+    def __init__(
+        self,
+        config: EO1InternVLPiFlowMatchingConfig,
+        internvl_backbone: nn.Module,
+        action_expert: nn.Module | None = None,
+    ):
+        super().__init__(config)
+        self.internvl_backbone = internvl_backbone
+
+        # InternVL uses a HF causal LM as `.language_model` (e.g. Qwen2ForCausalLM).
+        if not hasattr(self.internvl_backbone, "language_model"):
+            raise ValueError("internvl_backbone must have `.language_model`.")
+        # Do NOT register an alias module (e.g. `self.prefix_lm = self.internvl_backbone.language_model`).
+        # Registering both creates shared tensors under different state_dict keys, which safetensors refuses
+        # to save unless they are declared as tied weights. Use the property `prefix_lm` instead.
+
+        # ------------------------- Build action expert config (Pi05-style: smaller expert) -------------------------
+        prefix_cfg = self.prefix_lm.config
+        cfg_name = prefix_cfg.__class__.__name__
+
+        expert_cfg = copy.deepcopy(prefix_cfg)
+        if getattr(config, "expert_hidden_size", None) is not None:
+            expert_cfg.hidden_size = int(config.expert_hidden_size)
+        if getattr(config, "expert_intermediate_size", None) is not None:
+            expert_cfg.intermediate_size = int(config.expert_intermediate_size)
+        if getattr(config, "expert_num_attention_heads", None) is not None:
+            expert_cfg.num_attention_heads = int(config.expert_num_attention_heads)
+        if getattr(config, "expert_num_hidden_layers", None) is not None:
+            expert_cfg.num_hidden_layers = int(config.expert_num_hidden_layers)
+        # Keep head geometry aligned with prefix kv-cache.
+        if int(getattr(expert_cfg, "num_key_value_heads", -1)) != int(getattr(prefix_cfg, "num_key_value_heads", -2)):
+            raise ValueError(
+                "To reuse prefix KV-cache, expert and prefix must share num_key_value_heads. "
+                f"{int(getattr(prefix_cfg, 'num_key_value_heads'))=} vs {int(getattr(expert_cfg, 'num_key_value_heads'))=}."
+            )
+        if int(getattr(expert_cfg, "head_dim", -1)) != int(getattr(prefix_cfg, "head_dim", -2)):
+            raise ValueError(
+                "To reuse prefix KV-cache, expert and prefix must share head_dim. "
+                f"{int(getattr(prefix_cfg, 'head_dim'))=} vs {int(getattr(expert_cfg, 'head_dim'))=}."
+            )
+        if int(expert_cfg.num_attention_heads) % int(expert_cfg.num_key_value_heads) != 0:
+            raise ValueError(
+                "expert_num_attention_heads must be divisible by num_key_value_heads. "
+                f"{int(expert_cfg.num_attention_heads)=} {int(expert_cfg.num_key_value_heads)=}."
+            )
+        # Keep layer_types length consistent (Qwen3Config defines it).
+        if hasattr(expert_cfg, "layer_types") and isinstance(getattr(expert_cfg, "layer_types"), list):
+            if len(expert_cfg.layer_types) != int(expert_cfg.num_hidden_layers):
+                expert_cfg.layer_types = ["full_attention"] * int(expert_cfg.num_hidden_layers)
+
+        self._expert_hidden_size = int(expert_cfg.hidden_size)
+        self._expert_num_layers = int(expert_cfg.num_hidden_layers)
+        self._prefix_num_layers = int(getattr(prefix_cfg, "num_hidden_layers", self._expert_num_layers))
+
+        if self._expert_num_layers > self._prefix_num_layers:
+            raise ValueError(
+                "expert_num_hidden_layers cannot exceed prefix LM layers when using prefix KV-cache. "
+                f"{self._expert_num_layers=} > {self._prefix_num_layers=}."
+            )
+
+        mapping = str(getattr(config, "expert_layer_mapping", "last")).strip().lower()
+        if mapping == "last":
+            start = self._prefix_num_layers - self._expert_num_layers
+            self._prefix_kv_layer_indices = list(range(start, self._prefix_num_layers))
+        elif mapping == "first":
+            self._prefix_kv_layer_indices = list(range(self._expert_num_layers))
+        else:
+            raise ValueError(f"Unsupported expert_layer_mapping={mapping!r} (expected 'last' or 'first').")
+
+        max_action_dim = int(config.max_action_dim)
+
+        # Pi05: action embeddings do NOT concatenate timestep.
+        self.action_in_proj = nn.Linear(max_action_dim, self._expert_hidden_size)
+        self.action_out_proj = nn.Linear(self._expert_hidden_size, max_action_dim)
+
+        # Pi05: timestep is injected via AdaRMSNorm in the action expert.
+        self.time_mlp_in = nn.Linear(self._expert_hidden_size, self._expert_hidden_size)
+        self.time_mlp_out = nn.Linear(self._expert_hidden_size, self._expert_hidden_size)
+
+        if action_expert is not None:
+            self.action_expert = action_expert
+        else:
+            # Default: build an action expert (Qwen2/Qwen3) with its own (possibly smaller) config.
+            init_from = self.prefix_lm if bool(getattr(self.config, "expert_init_from_backbone", False)) else None
+            try:
+                if cfg_name == "Qwen2Config":
+                    self.action_expert = Qwen2PiActionExpert(expert_cfg, init_from_qwen2_lm=init_from)
+                elif cfg_name == "Qwen3Config":
+                    self.action_expert = Qwen3PiActionExpert(expert_cfg, init_from_qwen3_lm=init_from)
+                else:
+                    raise NotImplementedError(
+                        "eo_pi_internvl currently supports only Qwen2/Qwen3 LMs for action expert. "
+                        f"Got: {cfg_name}"
+                    )
+            except Exception as e:
+                raise RuntimeError(
+                    "Failed to build/initialize action expert. If you set `expert_init_from_backbone=True`, "
+                    "make sure expert_* hyperparams exactly match the prefix LM shapes, or set it to False "
+                    "for Pi05-style random init."
+                ) from e
+                
+        n_all = count_params(self.action_expert)
+        n_train = count_params(self.action_expert, trainable_only=True)
+        print(f"action_expert params: {n_all/1e6:.2f}M (trainable {n_train/1e6:.2f}M)")
+        self.post_init()
+
+    @property
+    def prefix_lm(self) -> nn.Module:
+        # A convenience accessor for the InternVL backbone LM used as the prefix model.
+        return self.internvl_backbone.language_model
+
+    @staticmethod
+    def _action_group_indices(action_dim: int, *, action_dim_mask: torch.Tensor | None = None) -> dict[str, list[int]]:
+        """
+        Best-effort split of action dims into position/rotation/gripper groups.
+
+        Supports both common layouts:
+        1) Compact (single-arm): [xyz(3), rotvec(3), gripper(1)] -> 7 dims (or bimanual 14 dims).
+        2) EO unified action encoding (see `dataset/action_encoding.py`):
+           left:  0:3 pos, 3:6 rotvec (or 3:9 r6d), 9 gripper
+           right: 10:13 pos, 13:16 rotvec (or 13:19 r6d), 19 gripper
+
+        Rotation repr is controlled via env var `EO_ACTION_ROT_REPR` (default rotvec).
+        """
+        d = int(action_dim)
+        if d <= 0:
+            return {"pos": [], "rot": [], "gripper": [], "other": []}
+
+        rot_repr = os.environ.get("EO_ACTION_ROT_REPR", "rotvec").strip().lower()
+        rot_is_r6d = rot_repr in ("r6d", "rot6d", "6d")
+
+        m_any = None
+        if action_dim_mask is not None and torch.is_tensor(action_dim_mask):
+            m = action_dim_mask.detach()
+            if m.ndim == 1:
+                m_any = m.to(torch.bool)
+            elif m.ndim == 2:
+                m_any = m.to(torch.bool).any(dim=0)
+            elif m.ndim == 3 and int(m.shape[1]) == 1:
+                m_any = m[:, 0, :].to(torch.bool).any(dim=0)
+            else:
+                m_any = m.reshape(-1, m.shape[-1]).to(torch.bool).any(dim=0)
+            if int(m_any.numel()) != d:
+                if int(m_any.numel()) > d:
+                    m_any = m_any[:d]
+                else:
+                    pad = torch.zeros((d - int(m_any.numel()),), dtype=torch.bool, device=m_any.device)
+                    m_any = torch.cat([m_any, pad], dim=0)
+
+        # Infer effective dim span from mask if available (common when original action dim < max_action_dim).
+        eff = d
+        if m_any is not None and bool(m_any.any().item()):
+            eff = int(torch.nonzero(m_any, as_tuple=False).max().item()) + 1
+
+        pos: list[int] = []
+        rot: list[int] = []
+        gripper: list[int] = []
+
+        # Compact layout: 7D single-arm / 14D bimanual.
+        if eff in (7, 14):
+            arm_offsets = [0] if eff == 7 else [0, 7]
+            for off in arm_offsets:
+                pos.extend([off + i for i in range(0, 3) if off + i < d])
+                rot.extend([off + i for i in range(3, 6) if off + i < d])
+                g = off + 6
+                if g < d:
+                    gripper.append(g)
+            used = set(pos) | set(rot) | set(gripper)
+            other = [i for i in range(d) if i not in used]
+            return {"pos": pos, "rot": rot, "gripper": gripper, "other": other}
+
+        # EO unified layout (10 dims per arm slot, supports bimanual at offset 10).
+        right_active = (d >= 20)
+        if m_any is not None and int(m_any.numel()) >= 20:
+            right_active = bool(m_any[10:20].any().item())
+
+        # Left arm.
+        pos.extend([i for i in range(0, min(3, d))])
+        rot.extend([i for i in range(3, min(6, d))])
+        if rot_is_r6d:
+            rot.extend([i for i in range(6, min(9, d))])
+        if 9 < d:
+            gripper.append(9)
+
+        # Right arm (offset 10) when active.
+        if right_active and d >= 20:
+            pos.extend([i for i in range(10, min(13, d))])
+            rot.extend([i for i in range(13, min(16, d))])
+            if rot_is_r6d:
+                rot.extend([i for i in range(16, min(19, d))])
+            if 19 < d:
+                gripper.append(19)
+
+        used = set(pos) | set(rot) | set(gripper)
+        other = [i for i in range(d) if i not in used]
+        return {"pos": pos, "rot": rot, "gripper": gripper, "other": other}
+
+    # ------------------------- EO1 Flow Matching utils -------------------------
+    def sample_noise(self, shape, device):
+        return torch.normal(mean=0.0, std=1.0, size=shape, dtype=torch.float32, device=device)
+
+    def sample_time(self, bsize, device):
+        beta_dist = torch.distributions.Beta(concentration1=1.5, concentration0=1.0)
+        time_beta = beta_dist.sample((bsize,)).to(device=device, dtype=torch.float32)
+        return time_beta * 0.999 + 0.001
+
+    def _embed_time_cond(self, timestep: torch.Tensor, *, dtype: torch.dtype, device: torch.device) -> torch.Tensor:
+        hidden = int(getattr(self, "_expert_hidden_size", self.prefix_lm.config.hidden_size))
+        if timestep.ndim == 1:
+            t_emb = create_sinusoidal_pos_embedding(timestep, hidden, device=device).to(dtype=dtype)
+            t_emb = self.time_mlp_in(t_emb)
+            t_emb = F.silu(t_emb)
+            t_emb = self.time_mlp_out(t_emb)
+            t_emb = F.silu(t_emb)
+            return t_emb
+        if timestep.ndim == 2:
+            bsz, t_len = int(timestep.shape[0]), int(timestep.shape[1])
+            t_flat = timestep.reshape(-1)
+            t_emb = create_sinusoidal_pos_embedding(t_flat, hidden, device=device).to(dtype=dtype)
+            t_emb = self.time_mlp_in(t_emb)
+            t_emb = F.silu(t_emb)
+            t_emb = self.time_mlp_out(t_emb)
+            t_emb = F.silu(t_emb)
+            return t_emb.view(bsz, t_len, hidden)
+        raise ValueError(f"timestep must be (B,) or (B,T), got {tuple(timestep.shape)}")
+
+    def _select_prefix_kv_cache(self, prefix_kv_cache: list[tuple[torch.Tensor, torch.Tensor]]) -> list[tuple[torch.Tensor, torch.Tensor]]:
+        if not hasattr(self, "_prefix_kv_layer_indices"):
+            return prefix_kv_cache
+        idx = list(getattr(self, "_prefix_kv_layer_indices"))
+        if not idx:
+            return prefix_kv_cache
+        if max(idx) >= len(prefix_kv_cache):
+            raise ValueError(
+                "Prefix KV cache shorter than expected. "
+                f"{len(prefix_kv_cache)=} < {max(idx)+1=}."
+            )
+        return [prefix_kv_cache[i] for i in idx]
+
+    def _replace_img_context_embeddings(
+        self,
+        input_ids: torch.LongTensor,
+        inputs_embeds: torch.FloatTensor,
+        pixel_values: torch.FloatTensor,
+        image_flags: torch.LongTensor | None,
+    ) -> torch.FloatTensor:
+        img_context_token_id = self.config.img_context_token_id
+        if img_context_token_id is None:
+            raise ValueError("config.img_context_token_id is None (tokenizer/model not initialized).")
+
+        try:
+            vision_dtype = next(self.internvl_backbone.vision_model.parameters()).dtype
+        except Exception:
+            vision_dtype = inputs_embeds.dtype
+        pixel_values = pixel_values.to(device=inputs_embeds.device, dtype=vision_dtype)
+
+        vit_embeds = self.internvl_backbone.extract_feature(pixel_values)  # (n_img, n_token, hidden)
+        if image_flags is not None:
+            image_flags = image_flags.squeeze(-1)
+            vit_embeds = vit_embeds[image_flags == 1]
+
+        bsz, _, hidden = inputs_embeds.shape
+        selected = input_ids == int(img_context_token_id)  # (B,S)
+        n_ctx = int(selected.sum().item())
+        if n_ctx == 0:
+            return inputs_embeds
+
+        vit_flat = vit_embeds.reshape(-1, hidden)
+        if vit_flat.shape[0] < n_ctx:
+            raise ValueError(f"IMG_CONTEXT mismatch: need {n_ctx} embeddings, got {vit_flat.shape[0]}.")
+
+        mask3 = selected.unsqueeze(-1).expand_as(inputs_embeds)
+        src = vit_flat[:n_ctx].to(device=inputs_embeds.device, dtype=inputs_embeds.dtype).reshape(-1)
+        return inputs_embeds.masked_scatter(mask3, src)
+
+    @staticmethod
+    def _find_suffix_starts(action_mask_token: torch.Tensor, *, expected_horizon: int | None = None) -> torch.Tensor:
+        if action_mask_token.ndim != 2:
+            raise ValueError(f"action_mask_token must be (B,S), got {tuple(action_mask_token.shape)}")
+        bsz = int(action_mask_token.shape[0])
+        starts = torch.empty((bsz,), dtype=torch.long, device=action_mask_token.device)
+        for b in range(bsz):
+            pos = torch.nonzero(action_mask_token[b], as_tuple=False).squeeze(-1)
+            if int(pos.numel()) == 0:
+                raise ValueError(f"Expected at least 1 action token per sample, got 0 for batch {b}.")
+            if expected_horizon is not None and int(pos.numel()) not in (1, int(expected_horizon)):
+                raise ValueError(
+                    f"Expected 1 or {int(expected_horizon)} action tokens per sample, got {int(pos.numel())} for batch {b}."
+                )
+            starts[b] = pos.min()
+        return starts
+
+    # ------------------------- Forward (train) -------------------------
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,  # noqa: ARG002 - recomputed for pi05 mask
+        inputs_embeds: torch.FloatTensor | None = None,  # noqa: ARG002 - use InternVL embeddings
+        labels: torch.LongTensor | None = None,  # noqa: ARG002 - Pi05 does not train AR loss here
+        pixel_values: torch.FloatTensor | None = None,
+        image_flags: torch.LongTensor | None = None,
+        states: torch.Tensor | None = None,  # noqa: ARG002 - Pi05: state should be discrete in text
+        actions: torch.Tensor | None = None,
+        action_is_pad: torch.Tensor | None = None,
+        action_dim_mask: torch.Tensor | None = None,
+        use_cache: bool | None = None,  # noqa: ARG002
+        output_attentions: bool | None = None,  # noqa: ARG002
+        output_hidden_states: bool | None = None,  # noqa: ARG002
+        **kwargs,
+    ) -> EO1InternVLPiFlowMatchingOutput:
+        if input_ids is None:
+            raise ValueError("Pi model requires `input_ids`.")
+        if actions is None:
+            raise ValueError("Pi model forward requires `actions` (flow-matching).")
+        if attention_mask is None:
+            attention_mask = torch.ones_like(input_ids, dtype=torch.long, device=input_ids.device)
+
+        action_token_id = self.config.action_token_id
+        if action_token_id is None:
+            raise ValueError("config.action_token_id is None (tokenizer/model not initialized).")
+        action_pass_id = self.config.action_pass_id
+
+        noise_mask = input_ids == int(action_token_id)
+        pass_mask = (input_ids == int(action_pass_id)) if action_pass_id is not None else torch.zeros_like(noise_mask)
+        action_mask_token = noise_mask | pass_mask  # (B, S)
+
+        bsz, horizon, act_dim = actions.shape
+
+        suffix_starts = self._find_suffix_starts(action_mask_token, expected_horizon=int(horizon))  # (B,)
+        prefix_len = int(suffix_starts.max().item())
+
+        # ---------------- Prefix expert (InternVL LM) ----------------
+        prefix_ids = input_ids[:, :prefix_len]
+        prefix_am = attention_mask[:, :prefix_len].to(dtype=torch.bool, device=input_ids.device)
+        ar = torch.arange(prefix_len, device=input_ids.device)
+        prefix_valid = prefix_am & (ar[None, :] < suffix_starts[:, None])
+
+        prefix_embeds = self.prefix_lm.get_input_embeddings()(prefix_ids).clone()
+        if pixel_values is not None:
+            prefix_embeds = self._replace_img_context_embeddings(
+                input_ids=prefix_ids,
+                inputs_embeds=prefix_embeds,
+                pixel_values=pixel_values,
+                image_flags=image_flags,
+            )
+
+        prefix_attn = prefix_valid.to(dtype=torch.long)
+        prefix_out = self.prefix_lm.model(
+            inputs_embeds=prefix_embeds,
+            attention_mask=prefix_attn,
+            use_cache=True,
+            return_dict=True,
+        )
+        prefix_pkv = prefix_out.past_key_values
+        prefix_kv_cache = [prefix_pkv[i] for i in range(len(prefix_pkv))]
+        prefix_kv_cache = self._select_prefix_kv_cache(prefix_kv_cache)
+
+        # ---------------- Flow Matching ----------------
+        actions_f32 = actions.to(dtype=torch.float32, device=input_ids.device)
+        rtc_delay = int(getattr(self.config, "rtc_delay", 5))
+        delay = max(0, min(rtc_delay, int(horizon)))
+        prefix_mask = torch.zeros((int(bsz), int(horizon)), device=actions_f32.device, dtype=torch.bool)
+        if delay > 0:
+            prefix_mask = torch.arange(int(horizon), device=actions_f32.device)[None, :] < int(delay)
+            prefix_mask = prefix_mask.expand(int(bsz), -1)
+            if action_is_pad is not None:
+                prefix_mask = prefix_mask & (~action_is_pad.to(device=actions_f32.device, dtype=torch.bool))
+
+        time = self.sample_time(int(bsz), input_ids.device)  # (B,)
+        noise = self.sample_noise(actions_f32.shape, input_ids.device)
+        time_tokens = time[:, None].expand(int(bsz), int(horizon))
+        if delay > 0:
+            time_tokens = torch.where(prefix_mask, torch.zeros_like(time_tokens), time_tokens)
+        time_expanded = time_tokens[:, :, None]
+        x_t = time_expanded * noise + (1 - time_expanded) * actions_f32
+        u_t = noise - actions_f32
+
+        # Action tokens: no timestep concatenation in Pi05.
+        action_tokens = self.action_in_proj(x_t.to(dtype=self.action_in_proj.weight.dtype))  # (B,H,D)
+
+        # AdaRMSNorm conditioning vector (Pi05).
+        adarms_cond = self._embed_time_cond(time_tokens, dtype=action_tokens.dtype, device=action_tokens.device)
+
+        # Suffix RoPE positions follow the *per-sample* prefix length (suffix_starts).
+        pos_ids = suffix_starts[:, None] + torch.arange(horizon, device=input_ids.device)[None, :]
+
+        suffix_valid = torch.ones((int(bsz), int(horizon)), device=input_ids.device, dtype=torch.bool)
+        if action_is_pad is not None:
+            suffix_valid = suffix_valid & (~action_is_pad.to(device=input_ids.device, dtype=torch.bool))
+
+        expert_h = self.action_expert(
+            action_tokens,
+            prefix_kv_cache=prefix_kv_cache,
+            prefix_key_mask=prefix_valid,
+            position_ids=pos_ids,
+            adarms_cond=adarms_cond,
+            suffix_key_mask=suffix_valid,
+        )
+        v_t = self.action_out_proj(expert_h).to(dtype=torch.float32)  # (B,H,A)
+
+        # Loss: average over *valid elements* (step mask + action_dim_mask).
+        target = u_t.to(dtype=v_t.dtype)
+        per_elem = (v_t - target) ** 2  # (B,H,A)
+
+        valid = suffix_valid[:, :, None] if suffix_valid is not None else torch.ones((int(bsz), int(horizon), 1), device=per_elem.device, dtype=torch.bool)
+        if delay > 0:
+            valid = valid & (~prefix_mask[:, :, None])
+        adm_for_groups = None
+        if action_dim_mask is not None:
+            adm = action_dim_mask
+            if not torch.is_tensor(adm):
+                adm = torch.as_tensor(adm)
+            adm = adm.to(device=per_elem.device, dtype=torch.bool)
+            if adm.ndim == 1:
+                adm = adm.view(1, -1).expand(int(bsz), -1)
+            elif adm.ndim == 2:
+                pass
+            elif adm.ndim == 3 and int(adm.shape[1]) == 1:
+                adm = adm[:, 0, :]
+            else:
+                adm = adm.reshape(int(bsz), -1)
+
+            if int(adm.shape[-1]) == int(per_elem.shape[-1]):
+                valid = valid & adm[:, None, :]
+                adm_for_groups = adm
+            else:
+                logger.warning_once(
+                    "Ignoring action_dim_mask due to shape mismatch in PI FM loss: "
+                    f"{tuple(adm.shape)=} vs expected (B,{int(per_elem.shape[-1])})."
+                )
+
+        # Exclude padding/unused action dims ("other") from FM loss.
+        # We only train on {pos, rot, gripper} dims so `fm_loss` matches the meaningful action space.
+        pos_mask = rot_mask = grip_mask = None
+        try:
+            a_dim = int(per_elem.shape[-1])
+            pos_mask = torch.zeros((int(bsz), a_dim), device=per_elem.device, dtype=torch.bool)
+            rot_mask = torch.zeros_like(pos_mask)
+            grip_mask = torch.zeros_like(pos_mask)
+            for bi in range(int(bsz)):
+                g = self._action_group_indices(a_dim, action_dim_mask=(adm_for_groups[bi] if adm_for_groups is not None else None))
+                if g["pos"]:
+                    pos_mask[bi, g["pos"]] = True
+                if g["rot"]:
+                    rot_mask[bi, g["rot"]] = True
+                if g["gripper"]:
+                    grip_mask[bi, g["gripper"]] = True
+            group_mask = pos_mask | rot_mask | grip_mask  # (B,A)
+            empty = ~group_mask.any(dim=1)
+            if empty.any():
+                fallback = adm_for_groups if adm_for_groups is not None else torch.ones_like(group_mask)
+                group_mask[empty] = fallback[empty]
+            valid = valid & group_mask[:, None, :]
+        except Exception:
+            pos_mask = rot_mask = grip_mask = None
+
+        weight = valid.to(dtype=per_elem.dtype)
+        denom = weight.sum().clamp_min(1)
+        fm_loss = (per_elem * weight).sum() / denom
+
+        fm_loss_pos = None
+        fm_loss_rot = None
+        fm_loss_gripper = None
+        # Decompose FM loss by action groups (auxiliary logs only; never crash training for these).
+        try:
+            if pos_mask is not None:
+                pos_w = (weight * pos_mask[:, None, :].to(dtype=weight.dtype)).sum()
+                if bool((pos_w > 0).item()):
+                    fm_loss_pos = (per_elem * weight * pos_mask[:, None, :].to(dtype=weight.dtype)).sum() / pos_w.clamp_min(1)
+            if rot_mask is not None:
+                rot_w = (weight * rot_mask[:, None, :].to(dtype=weight.dtype)).sum()
+                if bool((rot_w > 0).item()):
+                    fm_loss_rot = (per_elem * weight * rot_mask[:, None, :].to(dtype=weight.dtype)).sum() / rot_w.clamp_min(1)
+            if grip_mask is not None:
+                grip_w = (weight * grip_mask[:, None, :].to(dtype=weight.dtype)).sum()
+                if bool((grip_w > 0).item()):
+                    fm_loss_gripper = (per_elem * weight * grip_mask[:, None, :].to(dtype=weight.dtype)).sum() / grip_w.clamp_min(1)
+        except Exception:
+            fm_loss_pos = fm_loss_rot = fm_loss_gripper = None
+
+        return EO1InternVLPiFlowMatchingOutput(
+            loss=fm_loss,
+            fm_loss=fm_loss,
+            fm_loss_pos=fm_loss_pos,
+            fm_loss_rot=fm_loss_rot,
+            fm_loss_gripper=fm_loss_gripper,
+            ar_loss=None,
+            actions=v_t,
+            logits=None,
+            hidden_states=None,
+            attentions=None,
+        )
+
+    @torch.no_grad()
+    def sample_actions(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,  # noqa: ARG002
+        pixel_values: torch.FloatTensor | None = None,
+        image_flags: torch.LongTensor | None = None,
+        num_steps: int | None = None,
+        noise: torch.Tensor | None = None,
+        action_prefix: torch.Tensor | None = None,
+        delay: int | None = None,
+        **kwargs,
+    ) -> Tensor:
+        if input_ids is None:
+            raise ValueError("sample_actions requires input_ids.")
+        if attention_mask is None:
+            attention_mask = torch.ones_like(input_ids, dtype=torch.long, device=input_ids.device)
+
+        chunk_size = int(self.config.action_chunk_size)
+        max_action_dim = int(self.config.max_action_dim)
+        steps = int(num_steps) if num_steps is not None else int(self.config.num_denoise_steps)
+        dt = torch.tensor(-1.0 / max(1, steps), device=input_ids.device, dtype=torch.float32)
+
+        action_token_id = self.config.action_token_id
+        if action_token_id is None:
+            raise ValueError("config.action_token_id is None (tokenizer/model not initialized).")
+        action_pass_id = self.config.action_pass_id
+
+        noise_mask = input_ids == int(action_token_id)
+        pass_mask = (input_ids == int(action_pass_id)) if action_pass_id is not None else torch.zeros_like(noise_mask)
+        action_mask_token = noise_mask | pass_mask
+
+        bsz = int(input_ids.shape[0])
+
+        suffix_starts = self._find_suffix_starts(action_mask_token, expected_horizon=chunk_size)
+        prefix_len = int(suffix_starts.max().item())
+
+        prefix_ids = input_ids[:, :prefix_len]
+        prefix_am = attention_mask[:, :prefix_len].to(dtype=torch.bool, device=input_ids.device)
+        ar = torch.arange(prefix_len, device=input_ids.device)
+        prefix_valid = prefix_am & (ar[None, :] < suffix_starts[:, None])
+
+        prefix_embeds = self.prefix_lm.get_input_embeddings()(prefix_ids).clone()
+        if pixel_values is not None:
+            prefix_embeds = self._replace_img_context_embeddings(
+                input_ids=prefix_ids,
+                inputs_embeds=prefix_embeds,
+                pixel_values=pixel_values,
+                image_flags=image_flags,
+            )
+
+        prefix_attn = prefix_valid.to(dtype=torch.long)
+        prefix_out = self.prefix_lm.model(
+            inputs_embeds=prefix_embeds,
+            attention_mask=prefix_attn,
+            use_cache=True,
+            return_dict=True,
+        )
+        prefix_pkv = prefix_out.past_key_values
+        prefix_kv_cache = [prefix_pkv[i] for i in range(len(prefix_pkv))]
+        prefix_kv_cache = self._select_prefix_kv_cache(prefix_kv_cache)
+
+        device = input_ids.device
+        if noise is None:
+            x_t = self.sample_noise((bsz, chunk_size, max_action_dim), device=device).to(dtype=torch.float32)
+        else:
+            x_t = noise.to(device=device, dtype=torch.float32)
+
+        suffix_valid = torch.ones((bsz, chunk_size), device=device, dtype=torch.bool)
+        pos_ids = suffix_starts[:, None] + torch.arange(chunk_size, device=device)[None, :]
+
+        use_prefix = action_prefix is not None
+        rtc_delay = int(delay) if delay is not None else int(getattr(self.config, "rtc_delay", 0))
+        if use_prefix and rtc_delay <= 0:
+            try:
+                rtc_delay = int(action_prefix.shape[1])
+            except Exception:
+                rtc_delay = 0
+        rtc_delay = max(0, min(int(rtc_delay), int(chunk_size)))
+
+        prefix_mask = None
+        if use_prefix and rtc_delay > 0:
+            prefix = action_prefix
+            if not torch.is_tensor(prefix):
+                prefix = torch.as_tensor(prefix)
+            prefix = prefix.to(device=device, dtype=torch.float32)
+            if prefix.ndim != 3 or int(prefix.shape[0]) != bsz:
+                raise ValueError(
+                    f"action_prefix must be (B, T, A) with B={bsz}, got {tuple(prefix.shape)}"
+                )
+
+            # Clamp delay to available prefix length.
+            rtc_delay = min(int(rtc_delay), int(prefix.shape[1]))
+
+            # Pad/trim time and action dims.
+            if int(prefix.shape[-1]) != int(max_action_dim):
+                if int(prefix.shape[-1]) > int(max_action_dim):
+                    prefix = prefix[..., : int(max_action_dim)]
+                else:
+                    pad_dim = int(max_action_dim) - int(prefix.shape[-1])
+                    pad = torch.zeros((bsz, int(prefix.shape[1]), pad_dim), device=device, dtype=prefix.dtype)
+                    prefix = torch.cat([prefix, pad], dim=-1)
+            if int(prefix.shape[1]) < int(chunk_size):
+                pad_t = int(chunk_size) - int(prefix.shape[1])
+                pad = torch.zeros((bsz, pad_t, int(max_action_dim)), device=device, dtype=prefix.dtype)
+                prefix = torch.cat([prefix, pad], dim=1)
+            elif int(prefix.shape[1]) > int(chunk_size):
+                prefix = prefix[:, : int(chunk_size), :]
+
+            action_prefix = prefix
+            prefix_mask = torch.arange(int(chunk_size), device=device)[None, :] < int(rtc_delay)
+            prefix_mask = prefix_mask.expand(bsz, -1)
+        else:
+            use_prefix = False
+
+        for s in range(steps):
+            t_scalar = 1.0 + float(s) * float(dt)
+            if use_prefix:
+                x_t = torch.where(prefix_mask[:, :, None], action_prefix, x_t)
+                time_tokens = torch.full((bsz, chunk_size), t_scalar, device=device, dtype=torch.float32)
+                time_tokens = torch.where(prefix_mask, torch.zeros_like(time_tokens), time_tokens)
+            else:
+                time_tokens = torch.full((bsz,), t_scalar, device=device, dtype=torch.float32)
+
+            action_tokens = self.action_in_proj(x_t.to(dtype=self.action_in_proj.weight.dtype))
+            adarms_cond = self._embed_time_cond(time_tokens, dtype=action_tokens.dtype, device=action_tokens.device)
+
+            expert_h = self.action_expert(
+                action_tokens,
+                prefix_kv_cache=prefix_kv_cache,
+                prefix_key_mask=prefix_valid,
+                position_ids=pos_ids,
+                adarms_cond=adarms_cond,
+                suffix_key_mask=suffix_valid,
+            )
+            v_t = self.action_out_proj(expert_h).to(dtype=torch.float32)
+            x_t = x_t + dt * v_t
+
+        return x_t
+
+
+EO1InternVLPiFlowMatchingModel.register_for_auto_class()

checkpoint-300000/preprocessor_config.json

@@ -0,0 +1,11 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_eo1_internvl.EO1VisionProcessor"
+  },
+  "image_processor_type": "_InternVLImageProcessor",
+  "max_pixels": null,
+  "merge_size": 1,
+  "min_pixels": null,
+  "processor_class": "EO1VisionProcessor",
+  "temporal_patch_size": 1
+}

checkpoint-300000/processing_eo1_internvl.py

@@ -0,0 +1,106 @@
+"""
+EO1Vision processor for `eo_pi_internvl`.
+
+This is the InternVL-backbone EO1 processor with a Pi05-style action prompt:
+- We keep a *single* `<|action_pad|>` as a placeholder suffix token in text prompts.
+- The action expert consumes *continuous* action tokens (length=`action_chunk_size`) internally, so we do not need to
+  repeat `<|action_pad|>` by chunk size in the text (this also keeps AR loss extensible).
+"""
+
+from __future__ import annotations
+
+import inspect
+
+import torch
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.image_utils import ImageInput
+from transformers.processing_utils import Unpack
+from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
+from transformers.video_utils import VideoInput
+
+from eo_internvl.model.processing_eo1_internvl import (
+    ACTION_END_TOKEN,
+    ACTION_START_TOKEN,
+    DEFAULT_ACTION_TOKEN,
+    EO1VisionProcessor as _BaseEO1VisionProcessor,
+    EO1VisionProcessorKwargs,
+    RobotInput,
+)
+
+
+class EO1VisionProcessor(_BaseEO1VisionProcessor):
+    def expand_action_prompt(self, chunk_size: int) -> str:
+        # Pi05-style: keep a single placeholder token in text; the model builds the full continuous action block.
+        return DEFAULT_ACTION_TOKEN
+
+    def __call__(
+        self,
+        images: ImageInput = None,
+        text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] = None,
+        videos: VideoInput = None,
+        states: RobotInput = None,
+        actions: RobotInput = None,
+        **kwargs: Unpack[EO1VisionProcessorKwargs],
+    ) -> BatchFeature:
+        # Force action-token expansion length to 1 (no-op), regardless of robot_config / caller.
+        text_kwargs = kwargs.get("text_kwargs") or {}
+        text_kwargs = dict(text_kwargs)
+        text_kwargs["noise_token_num"] = 1
+        kwargs["text_kwargs"] = text_kwargs
+        return super().__call__(images=images, text=text, videos=videos, states=states, actions=actions, **kwargs)
+
+    @torch.no_grad()
+    def select_action(self, model, batch: dict, return_raw_actions: bool = False, **kwargs):
+        if not hasattr(model, "sample_actions"):
+            raise NotImplementedError("InternVL EO1 model does not implement sample_actions yet.")
+
+        action_prefix = batch.pop("action_prefix", None)
+        rtc_delay = batch.pop("rtc_delay", None)
+
+        batch_messages, batch_states, repo_ids = self._prepare_robot_inputs(batch)
+        chunk_size = int(getattr(getattr(model, "config", None), "action_chunk_size", 0) or self.robot_config.get("action_chunk_size") or 0)
+        noise_prompt = self.expand_action_prompt(chunk_size) if chunk_size > 0 else f"{ACTION_START_TOKEN}{DEFAULT_ACTION_TOKEN}{ACTION_END_TOKEN}"
+
+        inputs = self.apply_chat_template(
+            batch_messages,
+            states=batch_states,
+            add_generation_prompt=True,
+            noise_prompt=noise_prompt,
+            tokenize=True,
+            padding=True,
+            truncation=True,
+            return_dict=True,
+            return_tensors="pt",
+        ).to(model.device)
+
+        sig = None
+        try:
+            sig = inspect.signature(model.sample_actions)
+        except Exception:
+            sig = None
+
+        if action_prefix is not None:
+            if isinstance(action_prefix, (list, tuple)):
+                elems = []
+                for v in action_prefix:
+                    if not torch.is_tensor(v):
+                        v = torch.as_tensor(v)
+                    elems.append(v)
+                action_prefix = torch.stack(elems, dim=0)
+            elif not torch.is_tensor(action_prefix):
+                action_prefix = torch.as_tensor(action_prefix)
+            action_prefix = action_prefix.to(device=model.device, dtype=torch.float32)
+
+        if sig is not None and "action_prefix" in sig.parameters:
+            actions = model.sample_actions(**inputs, action_prefix=action_prefix, delay=rtc_delay).cpu()
+        else:
+            actions = model.sample_actions(**inputs).cpu()
+
+        if return_raw_actions:
+            return BatchFeature({"action": actions})
+
+        output_actions = self._process_robot_outputs(repo_ids, actions)
+        return BatchFeature({"action": output_actions})
+
+
+EO1VisionProcessor.register_for_auto_class()

checkpoint-300000/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:388d0abae6e0753b56d1b79bd7c7b73fcfccb4957397a5ee57998394e88cece5
+size 16389

checkpoint-300000/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f0318f052e090c4c1091ee226138f0b0e38e16bf32f5460d93ae46cd2b62bf3
+size 16325

checkpoint-300000/rng_state_10.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ffefbc3679191f405a2eb0dde3e69c0f5edfde5f3946bff62ea54998861b001
+size 16340

checkpoint-300000/rng_state_11.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f2011371e60491f875266fc2da3784324fd2cdf1a4a155dfbbd3d402b44c4a4
+size 16340

checkpoint-300000/rng_state_12.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e46fe4a57d1ec96258b6f77a05ea7d6d30b382f3f4cdd376e27c8ddbcd720f91
+size 16340

checkpoint-300000/rng_state_13.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88a377276842f2f552fab96dc7f8a7d50ae8d60d11dc3baa2e31776d3e1605b6
+size 16340

checkpoint-300000/rng_state_14.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c56c295a6c7519f2c7035150ed0edf5a670584a8ec2ea0f2117f765fb08f1911
+size 16404

checkpoint-300000/rng_state_15.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea52e84fcdec4bb49342e67d480b02e28e7d5c4b4200b2b4edadbe3944d181d5
+size 16340

checkpoint-300000/rng_state_16.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc91185faa8105acfacdf6156b53a344486e0eb204062fc0c8e28a4383c601a5
+size 16404

checkpoint-300000/rng_state_17.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c28cbf4329acc1cfcefc2703b6d3fe7e479ac477825091ca4e8569a1210c79c5
+size 16340

checkpoint-300000/rng_state_18.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8d8808f51dd1e25529d6269a0c2ea3fa7f9efa104651e99901f0e393584aa2c
+size 16340

checkpoint-300000/rng_state_19.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:073af3a3597db492896416e5eb8b60c43124fffea5e242649aa54dd12f17239f
+size 16340

checkpoint-300000/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a147ca63f7fda57d6b3d773e040b21e76b5145c0945d7d06fbc98c38d8d7b6d
+size 16325

checkpoint-300000/rng_state_20.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:262b7889e695735cb78ade23e0cc5c6d599bc0e3e1f87f64ca0f9b4be4d32c7e
+size 16340

checkpoint-300000/rng_state_21.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb40e63e1866c584a56668679389b8a5d5ee59588037e789f70c862d71506132
+size 16340

checkpoint-300000/rng_state_22.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b38b632787005dd8e69799ae476cbfd6696b15c899a7fbe5e511495c493b199d
+size 16340

checkpoint-300000/rng_state_23.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a80b0699cc457e8aeaa78b0a842bab60c71bd73a384f23318337bd45a335b27
+size 16340

checkpoint-300000/rng_state_24.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87b1d6892e58225ce3ca8ec82f1afc68ed59bbba79aa6d7d894377a383a0df66
+size 16404

checkpoint-300000/rng_state_25.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7431ca30048ce99f9f7bd77aee0928144d4ca0ca1cdae12d4872b22861d2b049
+size 16340

checkpoint-300000/rng_state_26.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfe14873b9ee26f4451262c0f1fa43440abc293ca688d7b00ef1acb210a57cb4
+size 16404

checkpoint-300000/rng_state_27.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8a2b002e3b43745936ceebe0b5eba0f2e48da322e2e997a0e93655a594ee325
+size 16340

checkpoint-300000/rng_state_28.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12dd3c283e93897c36859ea8644b6e3207eaac15413764037c11ac541383bee6
+size 16340

checkpoint-300000/rng_state_29.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50955531e56f1c20e6020a44816df8a3c65a477213af8944daf6622c3731f3eb
+size 16340

checkpoint-300000/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60af6b5c5fe98e0c922980e556be3ea50cf5896d83e2b90d54b03072d2dcdaa9
+size 16325

checkpoint-300000/rng_state_30.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:852f8f8511c96525038c130713173d1a3ed5d4669cbb3da73f7f9f4f81cdf360
+size 16340

checkpoint-300000/rng_state_31.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ce14b79b193abd0d3a1f6c0d7931425808444e031eac78eeec415838809904e
+size 16340

checkpoint-300000/rng_state_32.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f34b482a8c9800fad7e6f42f2fd1ea479748130bcf9a14c924fce86c3410040
+size 16404

checkpoint-300000/rng_state_33.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:397467ff82ca5291b554f77c9f32c28b60224da61bd3b29c26ff4e0396586809
+size 16404

checkpoint-300000/rng_state_34.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a48d7eff57624f740c4501e1048eeafc4dff1f18126033f2ed9150221e813dc7
+size 16340

checkpoint-300000/rng_state_35.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e34fb6ea0b4d87f7421a15ade140e5daa7f3c7d8542872eb98eed57f4d4a49d4
+size 16340

checkpoint-300000/rng_state_36.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a972908767aa403094d4293403b051bde35700fc20cbbb17e9a0e6a9e705fed4
+size 16404

checkpoint-300000/rng_state_37.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:496ec0fdfb71f611489a7a705e1e5f94812b11630b798003ea8ff1de5d784453
+size 16340

checkpoint-300000/rng_state_38.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:260c419a4339617f2079ac5bf5c20722aacf6ea835aa411e55ad8d05b088e7cb
+size 16404

checkpoint-300000/rng_state_39.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48dea9a0a2f262d6321230a6028dc948f8807812567a75251ff461be069bd372
+size 16340

checkpoint-300000/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:757a7eb6dac311c44331320a83496d342f31659775ea5c09f6f15f88880edb1e
+size 16389

checkpoint-300000/rng_state_40.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd5bff9dc535da7868905861f72eabd55d4d77834cbbdfd7c06f103dfb11d9d0
+size 16340

checkpoint-300000/rng_state_41.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:849cfd652c91de8b4c688efb3bfab0adffd3f0bc9ece272e36cdb844ca48c5d2
+size 16340

checkpoint-300000/rng_state_42.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed11e4531c429edead3eb1107cf8c4b04e285edf9f4a11deae8da63c4a606dd9
+size 16340

checkpoint-300000/rng_state_43.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae7f1de5c77824717c3180390598da0e6d0e97d93ac401a422238d9891ff120b
+size 16340