[Disagg] Fix MegaMoE topk_ids dtype mismatch and FakeKVManager missing kv_args (#25380)
Co-authored-by: JoeLee314 <liqichao@baidu.com>
This commit is contained in:
@@ -28,6 +28,7 @@ class FakeKVManager(BaseKVManager):
|
||||
is_mla_backend: Optional[bool] = False,
|
||||
):
|
||||
super().__init__(args, disaggregation_mode, server_args, is_mla_backend)
|
||||
self.kv_args = args
|
||||
self.req_to_decode_prefix_len = {}
|
||||
|
||||
def register_to_bootstrap(self):
|
||||
|
||||
@@ -206,8 +206,8 @@ def _run_mega_routed(
|
||||
)
|
||||
|
||||
if num_tokens > 0:
|
||||
topk_ids_in = topk_ids
|
||||
topk_weights_in = topk_weights
|
||||
topk_ids_in = topk_ids.to(torch.int32)
|
||||
topk_weights_in = topk_weights.to(torch.float32)
|
||||
else:
|
||||
topk_ids_in = hidden_states.new_empty((0, top_k), dtype=torch.int32)
|
||||
topk_weights_in = hidden_states.new_empty((0, top_k), dtype=torch.float32)
|
||||
|
||||
Reference in New Issue
Block a user