[MLX] Size the attention KV pool at the compute dtype for quantized models (#30097)

Co-authored-by: siming-deng <deng_siming@apple.com>

python/sglang/srt/hardware_backend/mlx/model_runner.py

@@ -512,9 +512,17 @@ class MlxModelRunner:
         if hasattr(sample_attn, "k_proj") and hasattr(sample_attn.k_proj, "weight"):
             dtype = sample_attn.k_proj.weight.dtype
         if dtype not in _MLX_KV_FLOAT_DTYPES:
-            # QuantizedLinear stores packed weights as integers, while the KV
-            # cache stores dequantized projection outputs.
-            dtype = mx.float32
+            # QuantizedLinear packs weights as integers, but the KV cache
+            # stores dequantized projection outputs, which are produced in
+            # the compute dtype carried by the quantization scales.  Storing
+            # at that dtype instead of float32 halves pool bytes per slot
+            # and keeps prefix-hit forwards in the same dtype as the no-hit
+            # path (a float32 pool promoted every post-hit concat).
+            scales = getattr(sample_attn.k_proj, "scales", None)
+            if scales is not None and scales.dtype in _MLX_KV_FLOAT_DTYPES:
+                dtype = scales.dtype
+            else:
+                dtype = mx.float32
         return n_kv_heads, head_dim, dtype
 
     def _get_attn_config(self) -> tuple[int, int, mx.Dtype]: