Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
60 lines
2.0 KiB
Python
60 lines
2.0 KiB
Python
import sys
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
from sglang.srt.managers.mm_utils import general_mm_embed_routine
|
|
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
|
from sglang.srt.runtime_context import get_context
|
|
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
|
from sglang.test.ci.ci_register import register_cpu_ci
|
|
|
|
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
|
|
|
|
|
|
class _MutatingLanguageModel:
|
|
def get_input_embeddings(self):
|
|
return lambda input_ids: torch.zeros((input_ids.shape[0], 2))
|
|
|
|
def __call__(self, *, input_ids, forward_batch, input_embeds):
|
|
input_embeds.add_(100)
|
|
return input_embeds
|
|
|
|
|
|
@pytest.mark.parametrize("algorithm", ["EAGLE", "EAGLE3", "NEXTN", "FROZEN_KV_MTP"])
|
|
def test_eagle_mm_embeddings_survive_in_place_target_updates(algorithm):
|
|
source_embeds = torch.arange(8, dtype=torch.float32).reshape(4, 2)
|
|
expected_embeds = source_embeds.clone()
|
|
forward_batch = SimpleNamespace(
|
|
forward_mode=ForwardMode.EXTEND,
|
|
contains_mm_inputs=lambda: True,
|
|
mm_inputs=[SimpleNamespace()],
|
|
extend_prefix_lens_cpu=[0],
|
|
extend_seq_lens_cpu=[4],
|
|
input_embeds=None,
|
|
spec_algorithm=SpeculativeAlgorithm.from_string(algorithm),
|
|
)
|
|
|
|
with (
|
|
get_context().override_server_args(),
|
|
patch(
|
|
"sglang.srt.managers.mm_utils.embed_mm_inputs",
|
|
return_value=(source_embeds, {}),
|
|
),
|
|
):
|
|
hidden_states = general_mm_embed_routine(
|
|
input_ids=torch.arange(4),
|
|
forward_batch=forward_batch,
|
|
language_model=_MutatingLanguageModel(),
|
|
)
|
|
|
|
torch.testing.assert_close(hidden_states, expected_embeds + 100)
|
|
torch.testing.assert_close(forward_batch.mm_input_embeds, expected_embeds)
|
|
assert forward_batch.mm_input_embeds.data_ptr() != hidden_states.data_ptr()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(pytest.main([__file__]))
|