clean up gemlite usage (#14444)

python/sglang/srt/layers/torchao_utils.py

@@ -3,8 +3,6 @@ Common utilities for torchao.
 """
 
 import logging
-import os
-import pwd
 from typing import Callable, Optional
 
 import torch
@@ -12,22 +10,6 @@ import torch
 logger = logging.getLogger(__name__)
 
 
-def get_gemlite_cache_path() -> str:
-    return f"/tmp/{pwd.getpwuid(os.getuid()).pw_gecos}_gemlite.json"
-
-
-def save_gemlite_cache(print_error: bool = False) -> bool:
-    try:
-        from gemlite.core import GemLiteLinearTriton
-
-        GemLiteLinearTriton.cache_config(get_gemlite_cache_path())
-    except Exception:
-        if print_error:
-            logger.error("Failed to save the GemLite cache.")
-        return False
-    return True
-
-
 def proj_filter(
     module: torch.nn.Module,
     fqn: str,
@@ -86,29 +68,6 @@ def apply_torchao_config_to_model(
             256,
         ], f"int4wo groupsize needs to be one of [32, 64, 128, 256] but got {group_size}"
         quantize_(model, int4_weight_only(group_size=group_size), filter_fn=filter_fn)
-    elif "gemlite" in torchao_config:
-        # gemlite-<packing_bitwidth>-<bit_width>-<group_size> or
-        # gemlite-<bit_width>-<group_size> (packing_bitwidth defaults to 32)
-        from gemlite.core import GemLiteLinearTriton
-        from torchao.quantization import gemlite_uintx_weight_only
-
-        _quant_args = torchao_config.split("-")
-        bit_width = int(_quant_args[-2])
-        group_size = None if _quant_args[-1] == "None" else int(_quant_args[-1])
-
-        try:
-            packing_bitwidth = int(_quant_args[-3])
-        except (ValueError, IndexError):
-            # if only 2 inputs found or conversion fails, use default value
-            packing_bitwidth = 32
-
-        quantize_(
-            model, gemlite_uintx_weight_only(group_size, bit_width, packing_bitwidth)
-        )
-
-        # try to load gemlite kernel config
-        GemLiteLinearTriton.load_config(get_gemlite_cache_path())
-
     elif "fp8wo" in torchao_config:
         # this requires newer hardware
         # [rank0]: AssertionError: fp8e4nv data type is not supported on CUDA arch < 89

python/sglang/srt/model_executor/cuda_graph_runner.py

@@ -51,7 +51,6 @@ from sglang.srt.layers.dp_attention import (
 from sglang.srt.layers.logits_processor import LogitsProcessorOutput
 from sglang.srt.layers.moe.token_dispatcher.deepep import DeepEPBuffer
 from sglang.srt.layers.moe.utils import get_deepep_mode, get_moe_a2a_backend
-from sglang.srt.layers.torchao_utils import save_gemlite_cache
 from sglang.srt.model_executor.forward_batch_info import (
     CaptureHiddenMode,
     ForwardBatch,
@@ -489,9 +488,6 @@ class CudaGraphRunner:
                     self.graphs[key] = graph
                     self.output_buffers[key] = output_buffers
 
-                    # Save gemlite cache after each capture
-                    save_gemlite_cache()
-
         # Trigger CUDA graph capture for specific shapes.
         # Capture the large shapes first so that the smaller shapes
         # can reuse the memory pool allocated for the large shapes.

python/sglang/srt/model_executor/piecewise_cuda_graph_runner.py

@@ -45,7 +45,6 @@ from sglang.srt.layers.dp_attention import (
 )
 from sglang.srt.layers.logits_processor import LogitsProcessorOutput
 from sglang.srt.layers.pooler import EmbeddingPoolerOutput
-from sglang.srt.layers.torchao_utils import save_gemlite_cache
 from sglang.srt.model_executor.forward_batch_info import (
     CaptureHiddenMode,
     ForwardBatch,
@@ -396,9 +395,6 @@ class PiecewiseCudaGraphRunner:
                 with set_compiled(True):
                     self.capture_one_batch_size(num_tokens)
 
-                # Save gemlite cache after each capture
-                save_gemlite_cache()
-
     def capture_one_batch_size(self, num_tokens: int):
         bs = 1