minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[Tiny Fix] Disable BCG when inner layer_model unresolved (#25021)

Browse Source 
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yuwei An
2026-05-12 14:51:27 +08:00
committed by GitHub
co-authored by Claude Opus 4.7
parent ca3bc05fea
commit 7582237f8e
1 changed files with 13 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/model_executor/breakable_cuda_graph_runner.py
+13 -6
View File
@@ -122,12 +122,17 @@ class BreakableCudaGraphRunner:
language_model = getattr( language_model = getattr(
model_runner.model, "language_model", model_runner.model model_runner.model, "language_model", model_runner.model
) )
self.layer_model = ( if hasattr(language_model, "model") and hasattr(language_model.model, "layers"):
language_model.model self.layer_model = language_model.model
if hasattr(language_model, "model") else:
and hasattr(language_model.model, "layers") # If we can't find the inner layer_model, disable BCG.
else language_model self.layer_model = None
) logger.warning(
"[BCG] Could not resolve inner layer_model on %s. BCG is "
"disabled for this model; prefill will fall back to eager.",
type(language_model).__name__,
)
return
# Memory pool # Memory pool
if get_global_graph_memory_pool() is None: if get_global_graph_memory_pool() is None:
@@ -330,6 +335,8 @@ class BreakableCudaGraphRunner:
self.output_buffers[num_tokens] = output self.output_buffers[num_tokens] = output
def can_run(self, forward_batch: "ForwardBatch"): def can_run(self, forward_batch: "ForwardBatch"):
if self.layer_model is None:
return False
if forward_batch.forward_mode.is_target_verify(): if forward_batch.forward_mode.is_target_verify():
return False return False
if forward_batch.input_embeds is not None: if forward_batch.input_embeds is not None: