diff --git a/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py b/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py index ffcd06131..4c9c8ea46 100644 --- a/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py +++ b/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py @@ -126,11 +126,7 @@ class DualChunkFlashAttentionBackend(AttentionBackend): self.req_to_token = model_runner.req_to_token_pool.req_to_token self.kv_cache_dtype = model_runner.kv_cache_dtype - self.kv_cache_dtype_str = getattr( - model_runner, - "kv_cache_dtype_str", - model_runner.server_args.kv_cache_dtype, - ) + self.kv_cache_dtype_str = model_runner.kv_cache_dtype_str self.page_size = model_runner.page_size assert self.num_heads % self.num_kv_heads == 0 diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index 973b2a2dd..ac440dbab 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -166,9 +166,7 @@ class FlashAttentionBackend(AttentionBackend): self.token_to_kv_pool = model_runner.token_to_kv_pool self.req_to_token = model_runner.req_to_token_pool.req_to_token self.kv_cache_dtype = model_runner.kv_cache_dtype - from sglang.srt.runtime_context import get_model - - self.kv_cache_dtype_str = get_model().kv_cache_dtype + self.kv_cache_dtype_str = model_runner.kv_cache_dtype_str self.kv_cache_is_mxfp8 = self.kv_cache_dtype_str == "mxfp8" self.page_size = model_runner.page_size # Static page-table width (upper bound). The device-side page-table build diff --git a/python/sglang/srt/layers/attention/linear/lightning_backend.py b/python/sglang/srt/layers/attention/linear/lightning_backend.py index 2a46ae1a1..6cd3c629e 100644 --- a/python/sglang/srt/layers/attention/linear/lightning_backend.py +++ b/python/sglang/srt/layers/attention/linear/lightning_backend.py @@ -65,11 +65,7 @@ class LightningAttentionBackend(MambaAttnBackendBase): self.decode_cuda_graph_metadata = {} self.kv_cache_dtype = model_runner.kv_cache_dtype - self.kv_cache_dtype_str = getattr( - model_runner, - "kv_cache_dtype_str", - model_runner.server_args.kv_cache_dtype, - ) + self.kv_cache_dtype_str = model_runner.kv_cache_dtype_str self.BLOCK = ( model_runner.model_config.block if hasattr(model_runner.model_config, "block") diff --git a/python/sglang/srt/layers/attention/xpu_backend.py b/python/sglang/srt/layers/attention/xpu_backend.py index 27ad57ce5..2b6b469ca 100644 --- a/python/sglang/srt/layers/attention/xpu_backend.py +++ b/python/sglang/srt/layers/attention/xpu_backend.py @@ -69,9 +69,7 @@ class XPUAttentionBackend(AttentionBackend): self.token_to_kv_pool = model_runner.token_to_kv_pool self.req_to_token = model_runner.req_to_token_pool.req_to_token self.kv_cache_dtype = model_runner.kv_cache_dtype - from sglang.srt.runtime_context import get_model - - self.kv_cache_dtype_str = get_model().kv_cache_dtype + self.kv_cache_dtype_str = model_runner.kv_cache_dtype_str self.page_size = model_runner.page_size self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA self.skip_prefill = skip_prefill diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py index 2c48ae75e..a69f07d06 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py @@ -318,6 +318,7 @@ class MockModelRunner(ModelRunner): self.device = device self.dtype = dtype self.kv_cache_dtype = dtype + self.kv_cache_dtype_str = "auto" self.gpu_id = 0 self.canary_manager = None self.page_size = case.page_size diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py index 62b6042a0..7fc5f7cb6 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py @@ -289,6 +289,7 @@ class DSAMockModelRunner(ModelRunner): # 656 bytes/token while the model still projects K/V in BF16; # `set_mla_kv_buffer` does the quantize on the way in. self.kv_cache_dtype = torch.float8_e4m3fn if fp8_kv_cache else dtype + self.kv_cache_dtype_str = "auto" # For TARGET_VERIFY / DRAFT_EXTEND, the DSA backend uses # `self.speculative_num_draft_tokens` to size `seqlens_expanded` # (`dsa_backend.py:482-486,510-515`). When zero, deep_gemm's diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py index 0d41fef66..626b6776c 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py @@ -328,6 +328,7 @@ class MockDSV4ModelRunner: self.device = device self.dtype = dtype self.kv_cache_dtype = dtype + self.kv_cache_dtype_str = "auto" self.gpu_id = 0 self.canary_manager = None self.page_size = case.page_size diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dual_chunk_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dual_chunk_attention.py index a0846f6c8..7802aef15 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dual_chunk_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dual_chunk_attention.py @@ -318,6 +318,7 @@ class DualChunkMockModelRunner(ModelRunner): self.device = device self.dtype = dtype self.kv_cache_dtype = dtype + self.kv_cache_dtype_str = "auto" self.gpu_id = 0 self.canary_manager = None self.page_size = case.page_size diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/gdn_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/gdn_attention.py index f42b26a83..409a92767 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/gdn_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/gdn_attention.py @@ -212,6 +212,7 @@ class MockGDNModelRunner(ModelRunner): self.device = device self.dtype = dtype self.kv_cache_dtype = dtype + self.kv_cache_dtype_str = "auto" self.gpu_id = 0 self.ps = ParallelState.trivial() self.canary_manager = None diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/kda_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/kda_attention.py index 4799286f0..481e724a6 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/kda_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/kda_attention.py @@ -218,6 +218,7 @@ class MockKDAModelRunner(ModelRunner): self.device = device self.dtype = dtype self.kv_cache_dtype = dtype + self.kv_cache_dtype_str = "auto" self.gpu_id = 0 self.ps = ParallelState.trivial() self.canary_manager = None diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/lightning_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/lightning_attention.py index 2f1de2d33..bbb0235e9 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/lightning_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/lightning_attention.py @@ -226,6 +226,7 @@ class MockLightningModelRunner(ModelRunner): self.device = device self.dtype = dtype self.kv_cache_dtype = dtype + self.kv_cache_dtype_str = "auto" self.gpu_id = 0 self.ps = ParallelState.trivial() self.canary_manager = None diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/mamba2_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/mamba2_attention.py index 6a98a1fcb..d1f91810d 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/mamba2_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/mamba2_attention.py @@ -311,6 +311,7 @@ class MockMamba2ModelRunner(ModelRunner): self.device = device self.dtype = dtype self.kv_cache_dtype = dtype + self.kv_cache_dtype_str = "auto" self.gpu_id = 0 self.ps = ParallelState.trivial() self.canary_manager = None diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py index cfe81c0b0..3fd845321 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py @@ -229,6 +229,7 @@ class MockMLAModelRunner(ModelRunner): # while the model still projects K/V in bf16; `set_mla_kv_buffer` # does the BF16->FP8 cast on the way in. self.kv_cache_dtype = torch.float8_e4m3fn if fp8_kv_cache else dtype + self.kv_cache_dtype_str = "fp8_e4m3" if fp8_kv_cache else "auto" self.gpu_id = 0 self.canary_manager = None self.page_size = case.page_size diff --git a/test/manual/attention/test_flashattn_backend.py b/test/manual/attention/test_flashattn_backend.py index 51c04968d..0122ef1f2 100644 --- a/test/manual/attention/test_flashattn_backend.py +++ b/test/manual/attention/test_flashattn_backend.py @@ -50,13 +50,13 @@ class MockModelRunner: self.kv_cache_dtype = ( self.dtype ) # torch dtype, required by FlashAttentionBackend + self.kv_cache_dtype_str = "auto" - # server_args is still needed for string-based config (kv_cache_dtype_str) self.server_args = type( "ServerArgs", (), { - "kv_cache_dtype": "auto", # string version for kv_cache_dtype_str + "kv_cache_dtype": "auto", "speculative_eagle_topk": None, "speculative_num_draft_tokens": 0, "enable_deterministic_inference": False, diff --git a/test/manual/attention/test_flashattn_mla_backend.py b/test/manual/attention/test_flashattn_mla_backend.py index d19f992b6..7bc8dd29e 100644 --- a/test/manual/attention/test_flashattn_mla_backend.py +++ b/test/manual/attention/test_flashattn_mla_backend.py @@ -49,6 +49,7 @@ class MockModelRunner: }, ) self.kv_cache_dtype = self.server_args.kv_cache_dtype + self.kv_cache_dtype_str = "auto" batch_size = 160 # Create a proper req_to_token_pool with the req_to_token attribute