diff --git a/python/sglang/srt/configs/__init__.py b/python/sglang/srt/configs/__init__.py index 70b1c2ebc..e16f2b4b1 100644 --- a/python/sglang/srt/configs/__init__.py +++ b/python/sglang/srt/configs/__init__.py @@ -1,6 +1,7 @@ from sglang.srt.configs.afmoe import AfmoeConfig from sglang.srt.configs.bailing_hybrid import BailingHybridConfig from sglang.srt.configs.chatglm import ChatGLMConfig +from sglang.srt.configs.cohere2_moe import Cohere2MoeConfig from sglang.srt.configs.dbrx import DbrxConfig from sglang.srt.configs.deepseekvl2 import DeepseekVL2Config from sglang.srt.configs.dots_ocr import DotsOCRConfig diff --git a/python/sglang/srt/configs/cohere2_moe.py b/python/sglang/srt/configs/cohere2_moe.py new file mode 100644 index 000000000..cd470bd69 --- /dev/null +++ b/python/sglang/srt/configs/cohere2_moe.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Cohere2Moe text config used by the Cohere Command-A Plus checkpoints.""" + +from transformers.configuration_utils import PreTrainedConfig +from transformers.models.auto.configuration_auto import CONFIG_MAPPING + +try: + from huggingface_hub.dataclasses import strict +except ImportError: # older huggingface_hub + + def strict(cls): # type: ignore[misc] + return cls + + +@strict +class Cohere2MoeConfig(PreTrainedConfig): + model_type = "cohere2_moe" + keys_to_ignore_at_inference = ["past_key_values"] + + vocab_size: int = 256000 + hidden_size: int = 8192 + intermediate_size: int = 22528 + logit_scale: float = 0.0625 + num_hidden_layers: int = 40 + num_attention_heads: int = 64 + num_key_value_heads: int | None = None + head_dim: int = 128 + hidden_act: str = "silu" + max_position_embeddings: int = 8192 + initializer_range: float = 0.02 + layer_norm_eps: float = 1e-5 + use_cache: bool = True + pad_token_id: int | None = 0 + bos_token_id: int | None = 5 + eos_token_id: int | list[int] | None = 255001 + tie_word_embeddings: bool = True + rope_theta: float | int = 10000.0 + rope_scaling: dict | None = None + attention_bias: bool = False + attention_dropout: float = 0.0 + sliding_window: int | None = 4096 + num_experts_per_tok: int = 2 + num_experts: int = 8 + num_shared_experts: int = 0 + shared_expert_combination_strategy: str = "average" + expert_selection_fn: str = "softmax" + layer_types: list[str] | None = None + first_k_dense_replace: int = 0 + prefix_dense_sliding_window_pattern: int = 1 + norm_topk_prob: bool = True + prefix_dense_intermediate_size: int | None = None + rms_norm_eps: float | None = None + sliding_window_pattern: int = 4 + + def __post_init__(self, **kwargs): + if self.num_key_value_heads is None: + self.num_key_value_heads = self.num_attention_heads + + if hasattr(self, "standardize_rope_params"): + try: + self.standardize_rope_params() + self.validate_rope() + except Exception: + pass + + if self.layer_types is None: + prefix_layers = [ + ( + "sliding_attention" + if ((i + 1) % self.prefix_dense_sliding_window_pattern) != 0 + else "full_attention" + ) + for i in range(self.first_k_dense_replace) + ] + rest_layers = [ + ( + "sliding_attention" + if ((i + 1) % self.sliding_window_pattern) != 0 + else "full_attention" + ) + for i in range(self.num_hidden_layers - self.first_k_dense_replace) + ] + self.layer_types = prefix_layers + rest_layers + + super().__post_init__(**kwargs) + + +try: + CONFIG_MAPPING.register("cohere2_moe", Cohere2MoeConfig) +except Exception: + CONFIG_MAPPING._extra_content["cohere2_moe"] = Cohere2MoeConfig diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index a3153bac1..929adc264 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -1509,6 +1509,7 @@ def is_generation_model(model_architectures: List[str], is_embedding: bool = Fal multimodal_model_archs = [ "CLIPModel", + "Cohere2VisionForConditionalGeneration", "DeepseekVL2ForCausalLM", "Ernie4_5_VLMoeForConditionalGeneration", "Gemma3ForConditionalGeneration", diff --git a/python/sglang/srt/function_call/cohere_command4_detector.py b/python/sglang/srt/function_call/cohere_command4_detector.py new file mode 100644 index 000000000..079ecabab --- /dev/null +++ b/python/sglang/srt/function_call/cohere_command4_detector.py @@ -0,0 +1,148 @@ +import json +import logging +from typing import List + +import orjson +from partial_json_parser.core.exceptions import MalformedJSON +from partial_json_parser.core.options import Allow + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + StructureInfo, + _GetInfoFunc, +) +from sglang.srt.function_call.utils import _partial_json_loads + +logger = logging.getLogger(__name__) + + +class CohereCommand4Detector(BaseFormatDetector): + """Detector for ``<|START_ACTION|>[...JSON array...]<|END_ACTION|>``.""" + + def __init__(self): + super().__init__() + self.bot_token = "<|START_ACTION|>" + self.eot_token = "<|END_ACTION|>" + # Per the chat template the array items are separated by ``,`` only -- + # the surrounding newlines/whitespace are also valid JSON whitespace. + self.tool_call_separator = "," + + def has_tool_call(self, text: str) -> bool: + return self.bot_token in text + + @staticmethod + def _normalize_calls(arr) -> List[dict]: + """Translate Cohere's per-item shape ``{tool_call_id, tool_name, + parameters}`` into the shape ``parse_base_json`` expects (``name`` / + ``parameters``). Drops ``tool_call_id`` since the OpenAI Chat + Completions schema assigns its own id.""" + if isinstance(arr, dict): + arr = [arr] + if not isinstance(arr, list): + return [] + out: List[dict] = [] + for act in arr: + if not isinstance(act, dict): + continue + normalized = dict(act) + if "name" not in normalized and "tool_name" in normalized: + normalized["name"] = normalized.pop("tool_name") + normalized.pop("tool_call_id", None) + out.append(normalized) + return out + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + """Non-streaming parse.""" + idx = text.find(self.bot_token) + if idx == -1: + return StreamingParseResult(normal_text=text) + normal_text = text[:idx] + body_start = idx + len(self.bot_token) + eot_idx = text.find(self.eot_token, body_start) + body = text[body_start:eot_idx] if eot_idx != -1 else text[body_start:] + + # body should be ``[ {...}, {...} ]`` (with arbitrary whitespace). + # Prefer the full-text JSON parser when the block is complete; fall + # back to ``_partial_json_loads`` to be forgiving when generation was + # truncated before ``<|END_ACTION|>``. + arr = None + try: + arr = orjson.loads(body) + except (orjson.JSONDecodeError, TypeError, ValueError): + try: + arr, _ = _partial_json_loads(body, Allow.ALL) + except (MalformedJSON, json.JSONDecodeError, ValueError) as e: + logger.warning( + f"Cohere tool-call body did not parse as JSON: {e}; " + "returning surrounding text as normal output." + ) + return StreamingParseResult(normal_text=normal_text) + + normalized = self._normalize_calls(arr) + return StreamingParseResult( + normal_text=normal_text, + calls=self.parse_base_json(normalized, tools), + ) + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + """Buffered streaming. Tool-call blocks are short (typically <2KB) so + we accumulate until the closing ``<|END_ACTION|>`` arrives and emit + the whole block at once. Anything before ``<|START_ACTION|>`` streams + through as normal text. + """ + self._buffer += new_text + current = self._buffer + + bot_pos = current.find(self.bot_token) + if bot_pos == -1: + # Defensive: keep any trailing characters that might be the start + # of a partial bot_token in the buffer for the next chunk. + partial = self._ends_with_partial_token(current, self.bot_token) + if partial: + head = current[:-partial] + self._buffer = current[-partial:] + return StreamingParseResult(normal_text=head) + self._buffer = "" + return StreamingParseResult(normal_text=current) + + # ``bot_token`` is somewhere in the buffer. Stream out anything before + # it as normal text exactly once. + if bot_pos > 0: + head = current[:bot_pos] + self._buffer = current[bot_pos:] + current = self._buffer + return StreamingParseResult(normal_text=head) + + # Buffer starts with bot_token. Wait for the closing token, then + # parse and emit the full call list. Anything past <|END_ACTION|> + # (typically <|END_OF_TURN_TOKEN|>) stays in the buffer for the next + # increment to handle. + eot_pos = current.find(self.eot_token, len(self.bot_token)) + if eot_pos == -1: + return StreamingParseResult() + + block_end = eot_pos + len(self.eot_token) + result = self.detect_and_parse(current[:block_end], tools) + self._buffer = current[block_end:] + return result + + def supports_structural_tag(self) -> bool: + return False + + def structure_info(self) -> _GetInfoFunc: + def _info(name: str) -> StructureInfo: + return StructureInfo( + begin=( + '<|START_ACTION|>[{"tool_call_id": "0", "tool_name": "' + + name + + '", "parameters": ' + ), + end="}]<|END_ACTION|>", + trigger="<|START_ACTION|>", + ) + + return _info diff --git a/python/sglang/srt/function_call/function_call_parser.py b/python/sglang/srt/function_call/function_call_parser.py index 432929d30..ab2e1c4a1 100644 --- a/python/sglang/srt/function_call/function_call_parser.py +++ b/python/sglang/srt/function_call/function_call_parser.py @@ -11,6 +11,7 @@ from sglang.srt.entrypoints.openai.protocol import ( ) from sglang.srt.environ import ToolStrictLevel, envs from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.cohere_command4_detector import CohereCommand4Detector from sglang.srt.function_call.core_types import ToolCallItem from sglang.srt.function_call.deepseekv3_detector import DeepSeekV3Detector from sglang.srt.function_call.deepseekv4_detector import DeepSeekV4Detector @@ -55,6 +56,7 @@ class FunctionCallParser: """ ToolCallParserEnum: Dict[str, Type[BaseFormatDetector]] = { + "cohere_command4": CohereCommand4Detector, "deepseekv3": DeepSeekV3Detector, "deepseekv31": DeepSeekV31Detector, "deepseekv32": DeepSeekV32Detector, diff --git a/python/sglang/srt/models/cohere2_moe.py b/python/sglang/srt/models/cohere2_moe.py new file mode 100644 index 000000000..a9924a338 --- /dev/null +++ b/python/sglang/srt/models/cohere2_moe.py @@ -0,0 +1,606 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# Adapted from: +# https://github.com/vllm-project/vllm/blob/v0.21.0/vllm/model_executor/models/cohere2_moe.py +"""Inference-only Cohere2Moe (Command A Plus) model compatible with HuggingFace weights.""" + +from typing import Iterable, Optional, Tuple + +import torch +from torch import nn +from transformers import PretrainedConfig + +from sglang.srt.distributed import ( + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce, +) +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.linear import ( + MergedColumnParallelLinear, + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear, +) +from sglang.srt.layers.logits_processor import LogitsProcessor +from sglang.srt.layers.moe.fused_moe_triton import FusedMoE +from sglang.srt.layers.moe.topk import TopK +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.layers.radix_attention import RadixAttention +from sglang.srt.layers.rotary_embedding import get_rope +from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding +from sglang.srt.model_executor.cuda_graph_runner import get_is_capture_mode +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_loader.weight_utils import default_weight_loader +from sglang.srt.utils import add_prefix, get_compiler_backend, is_cuda, make_layers + + +@torch.compile(backend=get_compiler_backend()) +def _cohere_layer_norm(hidden_states, weight, variance_epsilon): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + mean = hidden_states.mean(-1, keepdim=True) + variance = (hidden_states - mean).pow(2).mean(-1, keepdim=True) + hidden_states = (hidden_states - mean) * torch.rsqrt(variance + variance_epsilon) + hidden_states = weight.to(torch.float32) * hidden_states + return hidden_states.to(input_dtype) + + +class Cohere2MoeLayerNorm(nn.Module): + """Centered layer norm with learnable scale only (no bias).""" + + def __init__(self, hidden_size, eps=1e-5): + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + return _cohere_layer_norm(hidden_states, self.weight, self.variance_epsilon) + + +def cohere2_sigmoid_topk( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, +): + """Sigmoid -> top-k (-> renormalize) routing.""" + scores = gating_output.float().sigmoid() + topk_weights, topk_ids = torch.topk(scores, k=topk, dim=-1, sorted=False) + if renormalize: + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + return topk_weights.to(torch.float32), topk_ids.to(torch.int32) + + +class Cohere2MoeMLP(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + quant_config: Optional[QuantizationConfig] = None, + reduce_results: bool = True, + prefix: str = "", + ): + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, + [intermediate_size] * 2, + bias=False, + quant_config=quant_config, + prefix=add_prefix("gate_up_proj", prefix), + ) + self.down_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + reduce_results=reduce_results, + prefix=add_prefix("down_proj", prefix), + ) + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class Cohere2MoeAttention(nn.Module): + """Attention with optional RoPE on sliding-window layers only.""" + + def __init__( + self, + config: PretrainedConfig, + layer_id: int = 0, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + tp_size = get_tensor_model_parallel_world_size() + self.config = config + self.layer_id = layer_id + self.hidden_size = config.hidden_size + self.total_num_heads = config.num_attention_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.head_dim = getattr( + config, "head_dim", self.hidden_size // self.total_num_heads + ) + self.total_num_kv_heads = config.num_key_value_heads + if self.total_num_kv_heads >= tp_size: + assert self.total_num_kv_heads % tp_size == 0 + else: + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.max_position_embeddings = getattr( + config, "model_max_length", None + ) or getattr(config, "max_position_embeddings", 8192) + rope_parameters = getattr(config, "rope_parameters", None) + if rope_parameters is None: + rope_parameters = { + "rope_theta": getattr(config, "rope_theta", 10000.0), + "rope_type": "default", + } + self.rope_theta = rope_parameters.get( + "rope_theta", getattr(config, "rope_theta", 10000.0) + ) + self.rope_scaling = rope_parameters + + self.qkv_proj = QKVParallelLinear( + self.hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + prefix=add_prefix("qkv_proj", prefix), + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + self.hidden_size, + bias=False, + quant_config=quant_config, + prefix=add_prefix("o_proj", prefix), + reduce_results=False, + ) + + layer_types = getattr(config, "layer_types", None) + self.is_sliding = ( + layer_types is not None and layer_types[layer_id] == "sliding_attention" + ) + first_k_dense_replace = getattr(config, "first_k_dense_replace", 0) + prefix_dense_sliding_window_pattern = getattr( + config, "prefix_dense_sliding_window_pattern", 1 + ) + self.force_rope = bool( + first_k_dense_replace + and prefix_dense_sliding_window_pattern == 1 + and layer_id < first_k_dense_replace + ) + sliding_window = getattr(config, "sliding_window", None) + self.sliding_window_size = ( + sliding_window if (self.is_sliding and sliding_window is not None) else -1 + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=self.max_position_embeddings, + base=self.rope_theta, + rope_scaling=self.rope_scaling, + is_neox_style=False, + ) + + self.attn = RadixAttention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + layer_id=layer_id, + sliding_window_size=self.sliding_window_size, + quant_config=quant_config, + prefix=add_prefix("attn", prefix), + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + if self.is_sliding or self.force_rope: + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, forward_batch) + output, _ = self.o_proj(attn_output) + return output + + +class Cohere2MoeSparseMoeBlock(nn.Module): + """Sigmoid-routed MoE with optional shared experts (combined via 'sum' or 'average').""" + + def __init__( + self, + config: PretrainedConfig, + layer_id: int, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.tp_size = get_tensor_model_parallel_world_size() + self.hidden_size = config.hidden_size + self.num_experts = config.num_experts + self.top_k = config.num_experts_per_tok + self.layer_id = layer_id + + if self.tp_size > config.num_experts: + raise ValueError( + f"Tensor parallel size {self.tp_size} is greater than " + f"the number of experts {config.num_experts}." + ) + + self.expert_selection_fn = getattr(config, "expert_selection_fn", "softmax") + self.norm_topk_prob = getattr(config, "norm_topk_prob", True) + + if self.expert_selection_fn == "sigmoid": + custom_routing_function = cohere2_sigmoid_topk + scoring_func = "sigmoid" + else: + custom_routing_function = None + scoring_func = "softmax" + + self.gate = ReplicatedLinear( + config.hidden_size, + config.num_experts, + bias=False, + quant_config=None, + prefix=add_prefix("gate", prefix), + ) + + self.topk = TopK( + top_k=self.top_k, + renormalize=self.norm_topk_prob, + custom_routing_function=custom_routing_function, + scoring_func=scoring_func, + layer_id=layer_id, + ) + + self.experts = FusedMoE( + num_experts=config.num_experts, + top_k=self.top_k, + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + reduce_results=False, + quant_config=quant_config, + layer_id=layer_id, + prefix=add_prefix("experts", prefix), + ) + + num_shared_experts = getattr(config, "num_shared_experts", 0) + self.num_shared_experts = num_shared_experts + if num_shared_experts > 0: + self.shared_experts = Cohere2MoeMLP( + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size * num_shared_experts, + quant_config=quant_config, + reduce_results=False, + prefix=add_prefix("shared_experts", prefix), + ) + else: + self.shared_experts = None + + self.shared_expert_combination_strategy = getattr( + config, "shared_expert_combination_strategy", "sum" + ) + assert self.shared_expert_combination_strategy in ("average", "sum") + + # Auxiliary CUDA stream so shared_experts can overlap with the + # gate + routed-experts path inside a captured CUDA graph. Only used + # during capture/replay; outside capture the sync overhead outweighs it. + self.alt_stream = ( + torch.cuda.Stream() + if is_cuda() and self.shared_experts is not None + else None + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + orig_shape = hidden_states.shape + hidden_states = hidden_states.view(-1, self.hidden_size) + + if self.shared_experts is None: + router_logits, _ = self.gate(hidden_states) + topk_output = self.topk(hidden_states, router_logits) + final_hidden_states = self.experts(hidden_states, topk_output) + return final_hidden_states.view(orig_shape) + + # FusedMoE.experts can write back into its input buffer (observed for + # the unquantized triton BF16 path). Snapshot the post-norm input so + # the shared-expert branch sees the original layernorm output. + shared_input = hidden_states.clone() + + if self.alt_stream is not None and get_is_capture_mode(): + # Multi-stream overlap: shared_experts on alt stream, in parallel + # with gate + topk + routed experts on the main stream. + current_stream = torch.cuda.current_stream() + shared_input.record_stream(self.alt_stream) + self.alt_stream.wait_stream(current_stream) + with torch.cuda.stream(self.alt_stream): + shared_out = self.shared_experts(shared_input) + router_logits, _ = self.gate(hidden_states) + topk_output = self.topk(hidden_states, router_logits) + routed_out = self.experts(hidden_states, topk_output) + current_stream.wait_stream(self.alt_stream) + else: + router_logits, _ = self.gate(hidden_states) + topk_output = self.topk(hidden_states, router_logits) + routed_out = self.experts(hidden_states, topk_output) + shared_out = self.shared_experts(shared_input) + + final_hidden_states = routed_out + shared_out + if self.shared_expert_combination_strategy == "average": + final_hidden_states = final_hidden_states / 2 + # Returned un-reduced: the decoder layer folds attn + MoE TP-partials + # into a single all-reduce. + return final_hidden_states.view(orig_shape) + + +class Cohere2MoeDecoderLayer(nn.Module): + """Parallel attention + MLP: out = residual + attn(norm(x)) + mlp(norm(x)).""" + + def __init__( + self, + config: PretrainedConfig, + layer_id: int = 0, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.hidden_size = config.hidden_size + self.layer_id = layer_id + + self.self_attn = Cohere2MoeAttention( + config, + layer_id=layer_id, + quant_config=quant_config, + prefix=add_prefix("self_attn", prefix), + ) + + first_k_dense_replace = getattr(config, "first_k_dense_replace", 0) + if layer_id < first_k_dense_replace: + self.mlp = Cohere2MoeMLP( + hidden_size=config.hidden_size, + intermediate_size=getattr( + config, "prefix_dense_intermediate_size", config.intermediate_size + ), + quant_config=quant_config, + # Folded into the decoder layer's single all-reduce. + reduce_results=False, + prefix=add_prefix("mlp", prefix), + ) + else: + self.mlp = Cohere2MoeSparseMoeBlock( + config=config, + layer_id=layer_id, + quant_config=quant_config, + prefix=add_prefix("mlp", prefix), + ) + + norm_eps = getattr(config, "layer_norm_eps", 1e-5) + self.input_layernorm = Cohere2MoeLayerNorm(config.hidden_size, eps=norm_eps) + self.tp_size = get_tensor_model_parallel_world_size() + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + ) -> torch.Tensor: + # Parallel structure: y = x + attn(norm(x)) + mlp(norm(x)). The single + # residual lets the two TP all-reduces (attn.o_proj, mlp) fold into one + # sum-then-allreduce, halving per-layer all-reduces. + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + attn_out = self.self_attn( + positions=positions, + hidden_states=hidden_states, + forward_batch=forward_batch, + ) + mlp_out = self.mlp(hidden_states) + combined = attn_out + mlp_out + if self.tp_size > 1: + combined = tensor_model_parallel_all_reduce(combined) + return residual + combined + + +class Cohere2MoeModel(nn.Module): + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.config = config + self.vocab_size = config.vocab_size + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + prefix=add_prefix("embed_tokens", prefix), + ) + self.layers = make_layers( + config.num_hidden_layers, + lambda idx, prefix: Cohere2MoeDecoderLayer( + config=config, + layer_id=idx, + quant_config=quant_config, + prefix=prefix, + ), + prefix=add_prefix("layers", prefix), + ) + norm_eps = getattr(config, "layer_norm_eps", 1e-5) + self.norm = Cohere2MoeLayerNorm(config.hidden_size, eps=norm_eps) + + def get_input_embeddings(self, input_ids: Optional[torch.Tensor] = None): + """Return the embedding module, or the embedded tensor if ``input_ids`` + is given (SGLang's mm utils call this with no args).""" + if input_ids is None: + return self.embed_tokens + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if input_embeds is None: + hidden_states = self.embed_tokens(input_ids) + else: + hidden_states = input_embeds + for layer in self.layers: + hidden_states = layer(positions, hidden_states, forward_batch) + hidden_states = self.norm(hidden_states) + return hidden_states + + +class Cohere2MoeForCausalLM(nn.Module): + fall_back_to_pt_during_load = False + + packed_modules_mapping = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + "gate_up_proj": ["gate_proj", "up_proj"], + } + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.config = config + self.quant_config = quant_config + self.logit_scale = getattr(config, "logit_scale", None) + self.logits_processor = LogitsProcessor(config, logit_scale=self.logit_scale) + self.model = Cohere2MoeModel( + config, quant_config=quant_config, prefix=add_prefix("model", prefix) + ) + + def get_input_embeddings(self, input_ids: Optional[torch.Tensor] = None): + if input_ids is None: + return self.model.embed_tokens + return self.model.get_input_embeddings(input_ids) + + @torch.no_grad() + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: Optional[torch.Tensor] = None, + get_embedding: bool = False, + ) -> torch.Tensor: + hidden_states = self.model(input_ids, positions, forward_batch, input_embeds) + if get_embedding: + return hidden_states + return self.logits_processor( + input_ids, hidden_states, self.model.embed_tokens, forward_batch + ) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + + expert_params_mapping = FusedMoE.make_expert_params_mapping( + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=self.config.num_experts, + ) + + params_dict = dict(self.named_parameters()) + loaded_params = set() + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + + # Skip all-zero bias tensors that the checkpoint carries for + # bias-free Cohere layers (input_layernorm.bias, norm.bias, + # o_proj.bias, mlp.gate.bias, experts.*.[gate|up|down]_proj.bias). + if (name.endswith(".bias") or name.endswith("_bias")) and ( + name not in params_dict + and name.replace("q_proj", "qkv_proj") not in params_dict + and name.replace("gate_proj", "gate_up_proj") not in params_dict + ): + continue + + # Stacked attention / MLP weights. + matched = False + for param_name, shard_name, shard_id in stacked_params_mapping: + if shard_name not in name: + continue + if "mlp.experts" in name: + continue + new_name = name.replace(shard_name, param_name) + if new_name.endswith(".bias") and new_name not in params_dict: + matched = True + break + if new_name not in params_dict: + matched = True + break + param = params_dict[new_name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + loaded_params.add(new_name) + matched = True + break + if matched: + continue + + # Expert weights. + for mapping in expert_params_mapping: + param_name, weight_name, expert_id, shard_id = mapping + if weight_name not in name: + continue + new_name = name.replace(weight_name, param_name) + if new_name not in params_dict: + continue + param = params_dict[new_name] + weight_loader = param.weight_loader + weight_loader( + param, + loaded_weight, + new_name, + shard_id=shard_id, + expert_id=expert_id, + ) + loaded_params.add(new_name) + matched = True + break + if matched: + continue + + # lm_head is tied with embed_tokens; skip if missing. + if "lm_head.weight" in name: + continue + if name not in params_dict: + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + + return loaded_params + + +EntryClass = Cohere2MoeForCausalLM diff --git a/python/sglang/srt/models/cohere2_vision.py b/python/sglang/srt/models/cohere2_vision.py new file mode 100644 index 000000000..9d658404c --- /dev/null +++ b/python/sglang/srt/models/cohere2_vision.py @@ -0,0 +1,263 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +# Adapted from: +# https://github.com/vllm-project/vllm/blob/v0.21.0/vllm/model_executor/models/cohere2_vision.py +"""Inference-only Cohere2Vision (Command-A-Vision) multimodal model.""" + +import math +from typing import Iterable, List, Optional, Tuple + +import torch +import torch.nn.functional as F +from torch import nn +from transformers import PretrainedConfig +from transformers.modeling_outputs import BaseModelOutputWithPooling +from transformers.models.siglip import SiglipVisionModel + +from sglang.srt.layers.linear import ( + MergedColumnParallelLinear, + RowParallelLinear, +) +from sglang.srt.layers.logits_processor import LogitsProcessorOutput +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.managers.mm_utils import ( + MultiModalityDataPaddingPatternMultimodalTokens, + general_mm_embed_routine, +) +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalInputs, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_loader.weight_utils import default_weight_loader +from sglang.srt.models.cohere2_moe import Cohere2MoeForCausalLM +from sglang.srt.utils import add_prefix + + +class Cohere2VisionMultiModalProjector(nn.Module): + """Pixel-shuffle downsample -> SwiGLU MLP -> text hidden dim.""" + + def __init__(self, config: PretrainedConfig): + super().__init__() + self.downsample_factor = config.downsample_factor + input_dim = config.vision_config.hidden_size * (config.downsample_factor**2) + # HF stores a single ``linear_1`` split into SwiGLU gate/value halves; + # represent it as a 2-shard merged column-parallel linear. + self.intermediate_size = config.alignment_intermediate_size // 2 + self.linear_1 = MergedColumnParallelLinear( + input_dim, + [self.intermediate_size] * 2, + bias=True, + ) + self.linear_2 = RowParallelLinear( + self.intermediate_size, + config.text_config.hidden_size, + bias=True, + ) + + def pixel_shuffle(self, image_features: torch.Tensor) -> torch.Tensor: + batch_size, seq_len, _ = image_features.shape + height = width = int(math.isqrt(seq_len)) + image_features = image_features.reshape(batch_size, width, height, -1) + channels = image_features.shape[-1] + image_features = image_features.reshape( + batch_size, + width, + int(height / self.downsample_factor), + int(channels * self.downsample_factor), + ) + image_features = image_features.permute(0, 2, 1, 3) + image_features = image_features.reshape( + batch_size, + int(height / self.downsample_factor), + int(width / self.downsample_factor), + -1, + ) + image_features = image_features.permute(0, 2, 1, 3) + return image_features + + def forward(self, image_features: torch.Tensor) -> torch.Tensor: + image_features = self.pixel_shuffle(image_features) + # Flatten (B, H, W, D) -> (B, H*W, D) for the linear layers. + b, h, w, d = image_features.shape + image_features = image_features.reshape(b, h * w, d) + gate_up, _ = self.linear_1(image_features) + # HF Cohere2Vision SwiGLU: chunks (x, gate), output = x * silu(gate). + # SGLang's SiluAndMul swaps the halves, so we do the chunk inline. + x, gate = gate_up.chunk(2, dim=-1) + hidden_states = x * F.silu(gate) + hidden_states, _ = self.linear_2(hidden_states) + return hidden_states + + +def _remap_quant_config_for_sglang(quant_config): + """Rewrite the quant config ``ignore`` / target-scheme keys from HF module + names (``model.language_model.*``) to SGLang's layout + (``language_model.model.*``) so ``should_ignore_layer`` matches our prefixes.""" + if quant_config is None or not hasattr(quant_config, "ignore"): + return + + def _rewrite(name: str) -> str: + if name.startswith("model.language_model."): + return "language_model.model." + name[len("model.language_model.") :] + if name.startswith("model.vision_tower."): + return "vision_tower." + name[len("model.vision_tower.") :] + if name.startswith("model.multi_modal_projector."): + return ( + "multi_modal_projector." + name[len("model.multi_modal_projector.") :] + ) + return name + + quant_config.ignore = [_rewrite(n) for n in quant_config.ignore] + if hasattr(quant_config, "target_scheme_map") and isinstance( + quant_config.target_scheme_map, dict + ): + quant_config.target_scheme_map = { + _rewrite(k): v for k, v in quant_config.target_scheme_map.items() + } + + +class Cohere2VisionForConditionalGeneration(nn.Module): + packed_modules_mapping = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + "gate_up_proj": ["gate_proj", "up_proj"], + } + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.config = config + + # Must run before any Linear is instantiated. + _remap_quant_config_for_sglang(quant_config) + + # TODO: switch to sglang.srt.models.siglip.SiglipVisionModel once its + # SiglipMLP supports gelu_pytorch_tanh (it hardcodes QuickGELU) and + # qkv_proj weight loading is verified. The HF model below is correct. + self.vision_tower = SiglipVisionModel(config.vision_config) + self.multi_modal_projector = Cohere2VisionMultiModalProjector(config) + self.language_model = Cohere2MoeForCausalLM( + config=config.text_config, + quant_config=quant_config, + prefix=add_prefix("language_model", prefix), + ) + # Alias the text backbone as ``self.model`` so SGLang's piecewise + # CUDA-graph capture (checks ``hasattr(self.model, "model")`` then + # walks ``model.model.layers``) can locate the transformer layers. + self.model = self.language_model.model + + def pad_input_ids( + self, input_ids: List[int], mm_inputs: MultimodalInputs + ) -> List[int]: + pattern = MultiModalityDataPaddingPatternMultimodalTokens() + return pattern.pad_input_tokens(input_ids, mm_inputs) + + def get_image_feature(self, mm_input: List[MultimodalDataItem]) -> torch.Tensor: + pixel_values = torch.cat( + [ + torch.as_tensor(item.feature, device=self.vision_tower.device) + for item in mm_input + ], + dim=0, + ) + pixel_values = pixel_values.to(self.vision_tower.dtype) + + vision_outputs: BaseModelOutputWithPooling = self.vision_tower( + pixel_values=pixel_values, return_dict=True + ) + image_features = vision_outputs.last_hidden_state + image_features = self.multi_modal_projector(image_features) + + # Flatten patches: (np, tokens_per_patch, dim) -> (np*tokens, dim) + return image_features.reshape(-1, image_features.shape[-1]) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + get_embedding: bool = False, + **kwargs, + ) -> LogitsProcessorOutput: + return general_mm_embed_routine( + input_ids=input_ids, + forward_batch=forward_batch, + language_model=self.language_model, + data_embedding_funcs={ + Modality.IMAGE: self.get_image_feature, + }, + positions=positions, + get_embedding=get_embedding, + ) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + # The checkpoint stores tensors under ``model.language_model.``, + # ``model.vision_tower.``, and ``model.multi_modal_projector.`` + # prefixes; re-map them to our SGLang module names, then dispatch. + lm_weights: List[Tuple[str, torch.Tensor]] = [] + vision_weights: List[Tuple[str, torch.Tensor]] = [] + projector_weights: List[Tuple[str, torch.Tensor]] = [] + + for name, w in weights: + if name.startswith("model.language_model."): + # LM expects ``model.<...>`` names. + stripped = name[len("model.language_model.") :] + lm_weights.append((f"model.{stripped}", w)) + elif name.startswith("language_model."): + stripped = name[len("language_model.") :] + lm_weights.append((f"model.{stripped}", w)) + elif name.startswith("model.vision_tower."): + vision_weights.append((name[len("model.") :], w)) + elif name.startswith("vision_tower."): + vision_weights.append((name, w)) + elif name.startswith("model.multi_modal_projector."): + projector_weights.append((name[len("model.") :], w)) + elif name.startswith("multi_modal_projector."): + projector_weights.append((name, w)) + elif name.startswith("lm_head."): + # Tied with embed_tokens; ignore. + continue + else: + # Unknown top-level keys; pass through to LM as a fallback. + lm_weights.append((name, w)) + + self.language_model.load_weights(lm_weights) + + # transformers >=5 SiglipVisionModel exposes the encoder directly + # (params at ``embeddings.*`` / ``encoder.layers.*`` / ``post_layernorm.*``, + # no leading ``vision_model.``); the checkpoint keeps ``vision_model.``. + vt_params = dict(self.vision_tower.named_parameters()) + for name, w in vision_weights: + assert name.startswith("vision_tower.") + stripped = name[len("vision_tower.") :] + # Some HF versions still keep the ``vision_model.`` middle prefix. + if stripped not in vt_params and stripped.startswith("vision_model."): + stripped = stripped[len("vision_model.") :] + if stripped not in vt_params: + sample = sorted(vt_params.keys())[:3] + raise ValueError( + f"Unexpected vision tower weight: {name} (looked for " + f"{stripped!r}, sample params: {sample})" + ) + vt_params[stripped].data.copy_(w) + + # The HF checkpoint stores the merged ``linear_1`` as one [2*N, in] + # tensor matching MergedColumnParallelLinear, so the param's own + # weight_loader (or default_weight_loader) handles it. + proj_params = dict(self.multi_modal_projector.named_parameters()) + for name, w in projector_weights: + assert name.startswith("multi_modal_projector.") + stripped = name[len("multi_modal_projector.") :] + if stripped not in proj_params: + raise ValueError(f"Unexpected projector weight: {name}") + param = proj_params[stripped] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, w) + + +EntryClass = Cohere2VisionForConditionalGeneration diff --git a/python/sglang/srt/multimodal/processors/cohere2_vision.py b/python/sglang/srt/multimodal/processors/cohere2_vision.py new file mode 100644 index 000000000..e17156d88 --- /dev/null +++ b/python/sglang/srt/multimodal/processors/cohere2_vision.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SGLang Team +"""SGLang multimodal processor for Cohere2Vision (Command-A-Vision).""" + +from typing import Dict, List, Union + +from sglang.srt.managers.multimodal_processor import ( + BaseMultimodalProcessor as SGLangBaseProcessor, +) +from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput +from sglang.srt.models.cohere2_vision import Cohere2VisionForConditionalGeneration +from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens + + +class Cohere2VisionSGLangImageProcessor(SGLangBaseProcessor): + models = [Cohere2VisionForConditionalGeneration] + + def __init__(self, hf_config, server_args, _processor, *args, **kwargs): + super().__init__(hf_config, server_args, _processor, *args, **kwargs) + + # Cohere2Vision wraps each image as: + # <|START_OF_IMG|> [<|IMG_PATCH|> * P^2 + <|IMG_LINE_BREAK|>] * N <|END_OF_IMG|> + # (N = patch count, P = patch_size). The HF processor expands the single + # <|IMG_PATCH|> placeholder into that block. + proc = _processor + boi_token = proc.boi_token + eoi_token = proc.eoi_token + image_token = proc.image_token # "<|IMG_PATCH|>" + line_break_token = proc.img_line_break_token + + self.image_token_id = proc.image_token_id + self.boi_token_id = proc.tokenizer.convert_tokens_to_ids(boi_token) + self.eoi_token_id = proc.tokenizer.convert_tokens_to_ids(eoi_token) + self.img_line_break_token_id = proc.tokenizer.convert_tokens_to_ids( + line_break_token + ) + + # Match the unexpanded <|IMG_PATCH|> placeholder so SGLang pairs each + # one with its image_data entry before the HF processor expands it. + self.mm_tokens = MultimodalSpecialTokens( + image_token=image_token, + image_token_id=self.image_token_id, + ).build(_processor) + + async def process_mm_data_async( + self, + image_data: List[Union[str, bytes, Dict]], + input_text, + request_obj, + *args, + **kwargs, + ): + base_output = await self.load_mm_data( + prompt=input_text, + image_data=image_data, + multimodal_tokens=self.mm_tokens, + discard_alpha_channel=True, + ) + + mm_items, input_ids, _ = self.process_and_combine_mm_data( + base_output, self.mm_tokens + ) + return MultimodalProcessorOutput( + input_ids=input_ids.tolist(), + mm_items=mm_items, + im_token_id=self.image_token_id, + im_start_id=self.boi_token_id, + im_end_id=self.eoi_token_id, + ) diff --git a/python/sglang/srt/parser/reasoning_parser.py b/python/sglang/srt/parser/reasoning_parser.py index 6b63da114..035cb7a75 100644 --- a/python/sglang/srt/parser/reasoning_parser.py +++ b/python/sglang/srt/parser/reasoning_parser.py @@ -597,6 +597,245 @@ class _PoolsideV1Detector(Qwen3Detector): self.reasoning_default = "explicit_enable_thinking" +class CohereCommand4Detector(BaseReasoningFormatDetector): + """Detector for Cohere Command4 / Command-A family (incl. cohere2_moe and + cohere2_vision Command-A-Plus). + + Generated format (the assistant prefix in the chat template already emits + ``<|START_THINKING|>`` when ``reasoning=True``, so the *generated* text + typically begins inside the thinking block): + + thinking_content<|END_THINKING|><|START_TEXT|>final_answer<|END_TEXT|> + + When ``reasoning=False`` the chat template emits both START/END_THINKING + in the prefix and the generated text is just:: + + <|START_TEXT|>final_answer<|END_TEXT|> + + This detector returns: + - ``reasoning_text`` = the thinking block (between START_THINKING and + END_THINKING, with the START tag stripped if the model echoed it). + - ``normal_text`` = the content between ``<|START_TEXT|>`` and + ``<|END_TEXT|>``, with both markers stripped. If no ``<|START_TEXT|>`` + appears (the model exhausted max_new_tokens still inside thinking), + ``normal_text`` is the empty string. + + Matches the public token names from the model's + ``special_tokens_map.json`` (``<|START_THINKING|>`` etc.). + """ + + TEXT_START_TOKEN = "<|START_TEXT|>" + TEXT_END_TOKEN = "<|END_TEXT|>" + # When the model decides to call tools instead of producing a final text + # block, it emits an action block instead of a text block. The reasoning + # parser must leave that block intact so the downstream tool-call parser + # can pick it up. + ACTION_START_TOKEN = "<|START_ACTION|>" + + def __init__( + self, + stream_reasoning: bool = True, + force_reasoning: bool = True, + continue_final_message: bool = False, + previous_content: str = "", + ): + # The chat template puts <|START_THINKING|> in the assistant prefix + # when reasoning is enabled, so the *generated* text usually starts + # already inside thinking. ``force_reasoning=True`` makes the base + # detector treat the leading bytes as reasoning even though the + # generated stream typically does not echo <|START_THINKING|>. + super().__init__( + think_start_token="<|START_THINKING|>", + think_end_token="<|END_THINKING|>", + force_reasoning=force_reasoning, + stream_reasoning=stream_reasoning, + continue_final_message=continue_final_message, + previous_content=previous_content, + ) + # Streaming state machine. The model emits, in order: + # 1. reasoning (between START_THINKING [in prefix] and END_THINKING) + # 2. either ``<|START_TEXT|>...<|END_TEXT|>`` (final answer) or + # ``<|START_ACTION|>...<|END_ACTION|>`` (tool calls) -- never both. + # When ``reasoning=False`` the chat template emits both START/END + # thinking in the prefix and step 1 is empty; the generated stream + # then starts directly with the text or action block. + self._reasoning_done = False + self._saw_text_start = False + self._saw_text_end = False + self._in_action_mode = False + + @classmethod + def _strip_text_markers(cls, raw: str) -> str: + """Extract the substring between ``<|START_TEXT|>`` and + ``<|END_TEXT|>``. If ``<|START_TEXT|>`` is absent but a + ``<|START_ACTION|>`` block is present, the model produced a tool + call instead of a text answer -- return the raw text untouched so + the downstream tool-call parser can pick up the action block. If + neither marker is present (ran out of tokens still inside + thinking) return ``""``. If ``<|END_TEXT|>`` is absent (stop token + or max_new_tokens cut the stream off inside the text block) return + everything after ``<|START_TEXT|>``. + """ + if not raw: + return "" + s = raw.find(cls.TEXT_START_TOKEN) + if s == -1: + if cls.ACTION_START_TOKEN in raw: + return raw + return "" + s += len(cls.TEXT_START_TOKEN) + tail = raw[s:] + e = tail.find(cls.TEXT_END_TOKEN) + if e == -1: + return tail + return tail[:e] + + def detect_and_parse(self, text: str) -> StreamingParseResult: + # Direct parse: split on the (single) ``<|END_THINKING|>`` token if + # present. Anything before is reasoning, anything after is the + # final-text block. If no END_THINKING but a START_TEXT exists, + # we're in the reasoning=False case (chat template emitted both + # START/END thinking in the prefix; the model only generated the + # text block). Otherwise the model exhausted tokens still thinking + # and ``normal_text`` ends up empty -- matching the convention of + # the other detectors in this module (DeepSeekR1, Qwen3, ...). The + # empty content is propagated as ``message.content = None`` by + # serving_chat, and downstream code is expected to treat that as + # "no answer" rather than falling back to ``reasoning_content``. + end_think_idx = text.find(self.think_end_token) + text_start_idx = text.find(self.TEXT_START_TOKEN) + action_start_idx = text.find(self.ACTION_START_TOKEN) + if end_think_idx != -1: + reasoning = text[:end_think_idx] + rest = text[end_think_idx + len(self.think_end_token) :] + elif text_start_idx != -1: + reasoning = text[:text_start_idx] + rest = text[text_start_idx:] + elif action_start_idx != -1: + # reasoning=False + tool call: chat template emitted both + # START/END thinking in the prefix, the model only generated + # an action block. Treat the prefix before the action block as + # (probably empty) reasoning so the action block reaches the + # tool-call parser intact. + reasoning = text[:action_start_idx] + rest = text[action_start_idx:] + else: + reasoning = text + rest = "" + + # Some checkpoints echo the START_THINKING token even though the + # chat template put it in the prefix; drop it if so. + think_start_text = self.think_start_token + self.think_start_self_label + if reasoning.startswith(think_start_text): + reasoning = reasoning[len(think_start_text) :] + + return StreamingParseResult( + normal_text=self._strip_text_markers(rest), + reasoning_text=reasoning, + ) + + def parse_streaming_increment(self, new_text: str) -> StreamingParseResult: + """Streaming parse. Custom state machine -- we don't reuse the base + class because Cohere's "reasoning=False" path (the model emits no + ``<|END_THINKING|>``, just goes straight to a text or action block) + is fundamentally incompatible with the base detector's + ``force_reasoning`` semantics.""" + self._buffer += new_text + buf = self._buffer + + if not self._reasoning_done: + # Look for any marker that ends reasoning: an explicit + # END_THINKING, or an implicit transition via the start of the + # final-text or action block (reasoning=False case). + markers = ( + (self.think_end_token, "think_end"), + (self.TEXT_START_TOKEN, "text"), + (self.ACTION_START_TOKEN, "action"), + ) + first_pos = None + first_marker = None + first_kind = None + for marker_text, kind in markers: + p = buf.find(marker_text) + if p != -1 and (first_pos is None or p < first_pos): + first_pos, first_marker, first_kind = p, marker_text, kind + if first_pos is None: + # No marker seen yet. Stream the reasoning prefix, but keep + # enough tail in the buffer to recognise a marker split + # across chunk boundaries. + if not self.stream_reasoning: + return StreamingParseResult() + max_keep = max(len(m) for m, _ in markers) - 1 + if len(buf) > max_keep: + head = buf[:-max_keep] + self._buffer = buf[-max_keep:] + return StreamingParseResult(reasoning_text=head) + return StreamingParseResult() + + reasoning_chunk = buf[:first_pos] + if first_kind == "think_end": + self._buffer = buf[first_pos + len(first_marker) :] + else: + # Implicit reasoning-end: leave the start-of-block marker in + # the buffer for the post-thinking branch below to consume. + self._buffer = buf[first_pos:] + self._reasoning_done = True + if reasoning_chunk: + return StreamingParseResult(reasoning_text=reasoning_chunk) + buf = self._buffer + + # Reasoning is closed. Decide between text-stripping and + # action-passthrough on first sight of a marker. + if self._in_action_mode: + if not buf: + return StreamingParseResult() + self._buffer = "" + return StreamingParseResult(normal_text=buf) + + if not self._saw_text_start: + s_text = buf.find(self.TEXT_START_TOKEN) + s_action = buf.find(self.ACTION_START_TOKEN) + picks = [ + (p, k) for p, k in ((s_text, "text"), (s_action, "action")) if p != -1 + ] + if not picks: + max_keep = ( + max(len(self.TEXT_START_TOKEN), len(self.ACTION_START_TOKEN)) - 1 + ) + if len(buf) > max_keep: + self._buffer = buf[-max_keep:] + return StreamingParseResult() + picks.sort() + first_pos, first_kind = picks[0] + if first_kind == "action": + self._in_action_mode = True + out_normal = buf[first_pos:] + self._buffer = "" + return StreamingParseResult(normal_text=out_normal) + # Found <|START_TEXT|>. Drop everything up to and including the + # marker -- text content streams next. + self._buffer = buf[first_pos + len(self.TEXT_START_TOKEN) :] + self._saw_text_start = True + buf = self._buffer + + if self._saw_text_start and not self._saw_text_end: + e = buf.find(self.TEXT_END_TOKEN) + if e == -1: + # Emit everything except a possible partial END_TEXT tail. + keep = len(self.TEXT_END_TOKEN) - 1 + if len(buf) > keep: + out_normal = buf[:-keep] + self._buffer = buf[-keep:] + return StreamingParseResult(normal_text=out_normal) + return StreamingParseResult() + out_normal = buf[:e] + self._buffer = buf[e + len(self.TEXT_END_TOKEN) :] + self._saw_text_end = True + return StreamingParseResult(normal_text=out_normal) + + return StreamingParseResult() + + class ReasoningParser: """ Parser that handles both streaming and non-streaming scenarios for extracting @@ -629,6 +868,7 @@ class ReasoningParser: "nemotron_3": Nemotron3Detector, "interns1": Qwen3Detector, "gemma4": Gemma4Detector, + "cohere_command4": CohereCommand4Detector, } def __init__(