Fix stale CUDA graph benchmark and docs refs (#28041)
This commit is contained in:
@@ -7,10 +7,10 @@ from torch.nn import functional as F
|
|||||||
from transformers import AutoConfig
|
from transformers import AutoConfig
|
||||||
|
|
||||||
from sglang.benchmark.bench_utils import run_bench
|
from sglang.benchmark.bench_utils import run_bench
|
||||||
|
from sglang.srt.compilation.torch_compile_decoration import set_torch_compile_config
|
||||||
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import (
|
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import (
|
||||||
fused_moe as fused_moe_triton,
|
fused_moe as fused_moe_triton,
|
||||||
)
|
)
|
||||||
from sglang.srt.model_executor.cuda_graph_runner import set_torch_compile_config
|
|
||||||
|
|
||||||
|
|
||||||
def get_model_config(model_name: str, tp_size: int):
|
def get_model_config(model_name: str, tp_size: int):
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ This mode is intended for debugging only — it eliminates the performance benef
|
|||||||
For production use, you can mark specific functions as "non-graphable" using the `@eager_on_graph` decorator. During CUDA graph capture, these functions run eagerly between captured graph segments. Outside of capture, they behave normally.
|
For production use, you can mark specific functions as "non-graphable" using the `@eager_on_graph` decorator. During CUDA graph capture, these functions run eagerly between captured graph segments. Outside of capture, they behave normally.
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from sglang.srt.model_executor.breakable_cuda_graph.breakable_cuda_graph import eager_on_graph
|
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import eager_on_graph
|
||||||
|
|
||||||
@eager_on_graph(enable=True)
|
@eager_on_graph(enable=True)
|
||||||
def my_dynamic_op(x):
|
def my_dynamic_op(x):
|
||||||
@@ -44,7 +44,7 @@ def my_dynamic_op(x):
|
|||||||
You can also insert a bare graph break (no computation) using the `break_graph()` helper:
|
You can also insert a bare graph break (no computation) using the `break_graph()` helper:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from sglang.srt.model_executor.breakable_cuda_graph.breakable_cuda_graph import break_graph
|
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import break_graph
|
||||||
|
|
||||||
def forward(self, x):
|
def forward(self, x):
|
||||||
x = self.layer1(x)
|
x = self.layer1(x)
|
||||||
@@ -169,16 +169,16 @@ For typical use cases with a small number of graph breaks, the overhead is negli
|
|||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>python/sglang/srt/model_executor/breakable_cuda_graph/breakable_cuda_graph.py</code></td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/breakable_cuda_graph.py</code></td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Core implementation: <code>eager_on_graph</code>, <code>BreakableCUDAGraph</code>, <code>BreakableCUDAGraphCapture</code></td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Core implementation: <code>eager_on_graph</code>, <code>BreakableCUDAGraph</code>, <code>BreakableCUDAGraphCapture</code></td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>python/sglang/srt/model_executor/breakable_cuda_graph/cuda_utils.py</code></td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/cuda_utils.py</code></td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>CUDA runtime binding utilities</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>CUDA runtime binding utilities</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>python/sglang/srt/model_executor/cuda_graph_runner.py</code></td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>python/sglang/srt/model_executor/runner_backend/breakable_cuda_graph_backend.py</code></td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Integration with the main CUDA graph runner</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Integration with CUDA graph runner backends</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>python/sglang/srt/server_args.py</code></td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>python/sglang/srt/server_args.py</code></td>
|
||||||
|
|||||||
@@ -264,8 +264,8 @@ Use `--enforce-piecewise-cuda-graph` to skip all auto-disable checks (for testin
|
|||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>python/sglang/srt/model_executor/piecewise_cuda_graph_runner.py</code></td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>python/sglang/srt/model_executor/runner_backend/tc_piecewise_cuda_graph_backend.py</code></td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Main runner: init, capture, replay</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Backend implementation: compile, capture, replay</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>python/sglang/srt/compilation/compile.py</code></td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>python/sglang/srt/compilation/compile.py</code></td>
|
||||||
|
|||||||
+2
-2
@@ -3,16 +3,16 @@
|
|||||||
Public API (also reachable via the deeper module paths):
|
Public API (also reachable via the deeper module paths):
|
||||||
- BreakableCUDAGraph, BreakableCUDAGraphCapture — capture/replay
|
- BreakableCUDAGraph, BreakableCUDAGraphCapture — capture/replay
|
||||||
- eager_on_graph — decorator that marks a callable as a graph break
|
- eager_on_graph — decorator that marks a callable as a graph break
|
||||||
|
- break_graph — helper that inserts a bare graph break
|
||||||
- enable_breakable_cuda_graph — context that flips the Breakable runtime flag
|
- enable_breakable_cuda_graph — context that flips the Breakable runtime flag
|
||||||
- is_in_breakable_cuda_graph — runtime flag getter
|
- is_in_breakable_cuda_graph — runtime flag getter
|
||||||
|
|
||||||
The legacy model_executor/breakable_cuda_graph/ package is a
|
|
||||||
backwards-compat shim that re-exports from here.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.breakable_cuda_graph import ( # noqa: F401
|
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.breakable_cuda_graph import ( # noqa: F401
|
||||||
BreakableCUDAGraph,
|
BreakableCUDAGraph,
|
||||||
BreakableCUDAGraphCapture,
|
BreakableCUDAGraphCapture,
|
||||||
|
break_graph,
|
||||||
eager_on_graph,
|
eager_on_graph,
|
||||||
)
|
)
|
||||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import ( # noqa: F401
|
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import ( # noqa: F401
|
||||||
|
|||||||
Reference in New Issue
Block a user