minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

fix: Fix KTransformers hybrid inference with int8 quantization and format (#12536)

Browse Source
This commit is contained in:
Atream
2025-11-03 04:59:39 -08:00
committed by GitHub
parent ab8b83f71d
commit 14d8064803
2 changed files with 3 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+1
View File
@@ -751,6 +751,7 @@ class CompressedTensorsWNA16AMXMoEMethod(CompressedTensorsMoEMethod):
threadpool_count=self.threadpool_count, threadpool_count=self.threadpool_count,
amx_weight_path=self.amx_weight_path, amx_weight_path=self.amx_weight_path,
chunked_prefill_size=self.chunked_prefill_size, chunked_prefill_size=self.chunked_prefill_size,
amx_method=envs.SGLANG_KT_AMX_METHOD.value,
) )
def process_weights_after_loading(self, layer: torch.nn.Module) -> None: def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
python/sglang/srt/models/deepseek_v2.py
+2 -4
View File
@@ -751,12 +751,10 @@ class DeepseekV2MoE(nn.Module):
# router_logits: (num_tokens, n_experts) # router_logits: (num_tokens, n_experts)
router_logits = self.gate(hidden_states, gemm_output_zero_allocator) router_logits = self.gate(hidden_states, gemm_output_zero_allocator)
topk_output = self.topk(hidden_states, router_logits) topk_output = self.topk(hidden_states, router_logits)
if isinstance( final_hidden_states = self.experts(hidden_states, topk_output)
if not _is_cuda or isinstance(
self.experts.quant_method, CompressedTensorsWNA16AMXEPMoEMethod self.experts.quant_method, CompressedTensorsWNA16AMXEPMoEMethod
): ):
topk_output.topk_weights.mul_(self.routed_scaling_factor)
final_hidden_states = self.experts(hidden_states, topk_output)
if not _is_cuda:
final_hidden_states *= self.routed_scaling_factor final_hidden_states *= self.routed_scaling_factor
current_stream.wait_stream(self.alt_stream) current_stream.wait_stream(self.alt_stream)