Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4e60127fb4 | ||
|
|
e9fdf3ad0a | ||
|
|
2087760ea8 | ||
|
|
ef1a5955f3 | ||
|
|
7e17dee8b7 | ||
|
|
0090df5d64 | ||
|
|
4938eea2dc | ||
|
|
8c6cbd8c40 | ||
|
|
c82aed9fef | ||
|
|
e6f6420275 | ||
|
|
566176d0cf | ||
|
|
4f52678224 | ||
|
|
068acff09d | ||
|
|
f9056f0fcd | ||
|
|
2aae0cb4ee | ||
|
|
ea5fc832c1 | ||
|
|
9894dc789c | ||
|
|
77e7555ae3 | ||
|
|
319ee3c056 | ||
|
|
e59014be87 | ||
|
|
fd86688428 | ||
|
|
da343ba5d1 | ||
|
|
9ea02a7201 | ||
|
|
6c1c54b514 | ||
|
|
51624199b4 | ||
|
|
31b3758c99 | ||
|
|
740372bef0 | ||
|
|
19fe20baec | ||
|
|
383df91fcc | ||
|
|
534842ac08 | ||
|
|
4dd95a8817 | ||
|
|
656e00dcd3 | ||
|
|
a4b1363bf4 | ||
|
|
ad2acaa2df | ||
|
|
ecaaa37cb0 | ||
|
|
1b70fc192c | ||
|
|
3ebbb2d536 | ||
|
|
5d52cccbee | ||
|
|
74d03143e2 | ||
|
|
2b192aade6 | ||
|
|
9247c97919 | ||
|
|
5e7190b87a | ||
|
|
e00930c548 | ||
|
|
d7ff74b984 | ||
|
|
bd78095030 | ||
|
|
dc1bd46802 | ||
|
|
c7be3e935b | ||
|
|
2ff52e3ceb | ||
|
|
9ae43b0eee | ||
|
|
5781206833 | ||
|
|
069523f999 | ||
|
|
6dad152490 | ||
|
|
49e7bd179d | ||
|
|
de9f850b36 | ||
|
|
701213c867 | ||
|
|
9b89b96230 | ||
|
|
49b5e76e45 | ||
|
|
acffb0d20d | ||
|
|
cb1fa3c399 | ||
|
|
42f58b4da7 | ||
|
|
6f72cc3b64 | ||
|
|
0a8ca85565 | ||
|
|
e49d7fb170 | ||
|
|
d9074f7943 | ||
|
|
6e1df69623 | ||
|
|
77091cea68 | ||
|
|
49d26ff0c2 | ||
|
|
79cafec013 | ||
|
|
b95329a74d | ||
|
|
7889fbab9a | ||
|
|
89e1316eae | ||
|
|
f124c5ceec | ||
|
|
9654e5c4bc | ||
|
|
096b066fb4 | ||
|
|
26d7539799 | ||
|
|
b07cb9e9e7 | ||
|
|
6fa3fe69e2 | ||
|
|
2a1c477648 | ||
|
|
f209e67767 | ||
|
|
6624999385 | ||
|
|
2e7e0802f4 | ||
|
|
8151378c1a | ||
|
|
0318a8d0af | ||
|
|
da3eb6db32 | ||
|
|
bddeb61d94 | ||
|
|
090439bf8f | ||
|
|
6aacca2d91 | ||
|
|
fa443412c5 | ||
|
|
2f88c65289 | ||
|
|
47ad1928a2 | ||
|
|
85be04978d | ||
|
|
6caf0ff4ae | ||
|
|
c02eaadeb2 | ||
|
|
6582425854 | ||
|
|
df378bca42 | ||
|
|
546bfa7221 | ||
|
|
e8eeff2956 | ||
|
|
7ee7bef79d | ||
|
|
6405cf2881 | ||
|
|
f4de6abee6 | ||
|
|
6c4f36e47a | ||
|
|
e75e3b8a98 | ||
|
|
6be9c78cf4 | ||
|
|
34f090053e | ||
|
|
25de80780a | ||
|
|
3d2827236d | ||
|
|
55cc90b533 | ||
|
|
cc012abd21 | ||
|
|
8b2ca8ecc2 | ||
|
|
6cbdfec453 | ||
|
|
0823651b0b | ||
|
|
81f27fb3a7 | ||
|
|
06d012eaca | ||
|
|
341d5273bd | ||
|
|
644014bc98 | ||
|
|
0e907b755d | ||
|
|
427c9e05c1 | ||
|
|
55d2ee5546 | ||
|
|
0971450c16 | ||
|
|
4fac02fe5d | ||
|
|
ecb8e9fc8f | ||
|
|
19eb56fa57 | ||
|
|
31329c222c | ||
|
|
f0ecf15c84 | ||
|
|
8c43c667cb | ||
|
|
6f370c3bcd | ||
|
|
2e7f0bb28c | ||
|
|
c289656434 | ||
|
|
deaa7f1463 | ||
|
|
6f2d2437b6 | ||
|
|
f5ae989a0e | ||
|
|
cad9eaf27d | ||
|
|
1c919e401d | ||
|
|
7a833ad0a2 | ||
|
|
380a4d0332 | ||
|
|
9f87d28db6 | ||
|
|
846181f1c4 | ||
|
|
446d181e23 | ||
|
|
583fada954 | ||
|
|
78eee88113 | ||
|
|
14ab74ef95 | ||
|
|
b252aceffe | ||
|
|
e581520c67 | ||
|
|
d10a09ecf3 | ||
|
|
b2800461e5 | ||
|
|
1ba84a4b59 | ||
|
|
c311bc9611 | ||
|
|
588049fe2e | ||
|
|
84622ce9d5 | ||
|
|
a0ba1968b1 | ||
|
|
f1e62e3a2e | ||
|
|
ae47bcd4da | ||
|
|
2a14f65fd7 | ||
|
|
37556c1b91 | ||
|
|
0bc074db83 | ||
|
|
aa7a976807 | ||
|
|
df1fc6ab64 | ||
|
|
acc15c193b | ||
|
|
38d865489a | ||
|
|
d27efca353 | ||
|
|
b4584ab838 | ||
|
|
effb752188 | ||
|
|
425a1f8f24 | ||
|
|
fc9e1c8d29 | ||
|
|
c10fa03fc4 | ||
|
|
c73f7077eb | ||
|
|
7bdd8fe6ec | ||
|
|
e1e97e8b49 | ||
|
|
cdbea5dccf | ||
|
|
710a41ba57 | ||
|
|
a947b56bc7 | ||
|
|
bd37e7f513 | ||
|
|
9cd6616c24 | ||
|
|
d3fcf8997c | ||
|
|
35b369ebba | ||
|
|
c081d4aacc | ||
|
|
8fc3ce48da | ||
|
|
7bc988446d | ||
|
|
a5c8afab48 | ||
|
|
3ed56a341b | ||
|
|
4bb645b8e2 | ||
|
|
8772916e06 | ||
|
|
7152aabdcb | ||
|
|
38ec649048 | ||
|
|
e538489e0e | ||
|
|
dfdbff931d | ||
|
|
3ecb6cd729 | ||
|
|
0e2aac500b | ||
|
|
f35ec18d8b | ||
|
|
4273e7de77 | ||
|
|
63d320c723 | ||
|
|
2ee82fe158 | ||
|
|
0967a013c2 | ||
|
|
85374532f2 | ||
|
|
c842d32b0a | ||
|
|
50e50e13ab | ||
|
|
1f6ce4b068 | ||
|
|
2afd343934 | ||
|
|
efd9a40bb8 | ||
|
|
95047d3464 | ||
|
|
516ab77619 | ||
|
|
27e883a20d | ||
|
|
c421d16563 | ||
|
|
cf5df82680 | ||
|
|
d3e3945e41 | ||
|
|
f88572a57c | ||
|
|
f337f01b80 | ||
|
|
ac62ddf164 | ||
|
|
24b6930076 | ||
|
|
6d35cee7fc | ||
|
|
671000aee6 | ||
|
|
31b931edc3 | ||
|
|
0154f72b48 | ||
|
|
67bb6a58d0 | ||
|
|
2f5c9ac43d | ||
|
|
a749d84303 | ||
|
|
62122c838a | ||
|
|
3450d68d4e | ||
|
|
26a3214cda | ||
|
|
8f5a636f8d | ||
|
|
b7f6d04a9a | ||
|
|
963e9fb42d | ||
|
|
9d7f44bdbb | ||
|
|
402df23188 | ||
|
|
1409f469be | ||
|
|
51c92e8df1 | ||
|
|
515f5be77e | ||
|
|
434c2e3adc | ||
|
|
cbe137727e | ||
|
|
0fb699ac03 | ||
|
|
9c8340a2e5 | ||
|
|
ec070ec8c8 | ||
|
|
3e7e652900 | ||
|
|
16d1c93b3d | ||
|
|
cec70a4708 | ||
|
|
3fc7a669bf | ||
|
|
e0b4d66733 | ||
|
|
7325b38412 | ||
|
|
37ebcac3c4 | ||
|
|
6103a8c743 | ||
|
|
9dc4c5d891 | ||
|
|
8ca82118e0 | ||
|
|
a2025b8c4d | ||
|
|
2f2f9d12f8 | ||
|
|
7c5bdb940c | ||
|
|
466e985f4b | ||
|
|
e047e50d4e | ||
|
|
3ec86307f2 | ||
|
|
6bbd689ab4 | ||
|
|
0c578d97fe | ||
|
|
a1eb691ab5 | ||
|
|
182f62d2de | ||
|
|
961404b010 | ||
|
|
36f59982fa | ||
|
|
0b53305f48 | ||
|
|
b129504f0e | ||
|
|
1446e24d13 | ||
|
|
5bb850d536 | ||
|
|
a80055d3c6 | ||
|
|
7c9f74c6e1 | ||
|
|
62ae032c46 | ||
|
|
5df1667077 | ||
|
|
07a3735e94 | ||
|
|
928e683f94 | ||
|
|
6a14b80141 | ||
|
|
1b03d31b31 | ||
|
|
f404db94d4 | ||
|
|
86b3558b41 | ||
|
|
78b382b1ad | ||
|
|
984994e3aa | ||
|
|
5bb24e399d | ||
|
|
8ec65e89c1 | ||
|
|
77173ffb08 | ||
|
|
752801e4d1 | ||
|
|
17a7484c06 | ||
|
|
cd11037009 | ||
|
|
4142235c2b | ||
|
|
ec7eb6bd79 | ||
|
|
ea5baf4022 | ||
|
|
28ec6700da | ||
|
|
3177d10ca6 | ||
|
|
ce06a14444 | ||
|
|
c6565c1a6d | ||
|
|
a69583f70f | ||
|
|
0a59830114 | ||
|
|
26c63335e2 | ||
|
|
d94d784441 | ||
|
|
bb1c98baa2 | ||
|
|
c85df5becc | ||
|
|
261cb826e6 | ||
|
|
174a5f37b9 | ||
|
|
41812afe43 | ||
|
|
81f2b43ab5 | ||
|
|
d65503fece | ||
|
|
1417345f5f | ||
|
|
cae4927fbf | ||
|
|
a164c6d9f1 | ||
|
|
43af9fcbb6 | ||
|
|
9f6fc55332 | ||
|
|
32290dda2c | ||
|
|
90663ccb41 | ||
|
|
efff836911 | ||
|
|
7a9feacf91 | ||
|
|
0dc8b29e15 | ||
|
|
c3685dff42 | ||
|
|
8eedf616f5 | ||
|
|
be495a6174 | ||
|
|
e2f4fedf04 | ||
|
|
77983865d8 | ||
|
|
5c44214d6a | ||
|
|
f4d9d2e670 | ||
|
|
f4b90382a2 | ||
|
|
cc6e8b0f25 | ||
|
|
e98b2f7538 | ||
|
|
8b5d77c268 | ||
|
|
82cdd72588 | ||
|
|
03fcbe147f | ||
|
|
81b81664a6 | ||
|
|
621136e094 | ||
|
|
81cb895b85 | ||
|
|
ec75d3d30f | ||
|
|
ffac53d779 | ||
|
|
542a043f1b | ||
|
|
208f6f7501 | ||
|
|
3f697899a3 | ||
|
|
0010f56a0d | ||
|
|
6fe4b66f5c | ||
|
|
79fec592a8 | ||
|
|
f2a1366584 | ||
|
|
4fa2c9c1de | ||
|
|
f766397f2e | ||
|
|
c39a1c0663 | ||
|
|
ab0c31bd94 | ||
|
|
e1048a215c | ||
|
|
7dd9640752 | ||
|
|
e31cbc01a2 | ||
|
|
701cf7e47e | ||
|
|
379e8f916b | ||
|
|
9ebe42238b | ||
|
|
0e80c73c92 | ||
|
|
954458567e | ||
|
|
2909f84e3b | ||
|
|
890a9605ce | ||
|
|
4bb5611dba | ||
|
|
f2f223e697 | ||
|
|
f2eebd5533 | ||
|
|
7fef014d51 | ||
|
|
5c154c214d | ||
|
|
c3e985203a | ||
|
|
8993f790be | ||
|
|
4e60d70ba4 | ||
|
|
172b1b4825 | ||
|
|
16e353d93e | ||
|
|
3fdd63a562 | ||
|
|
abef3efb64 | ||
|
|
2d25767759 | ||
|
|
58f622ebc4 | ||
|
|
4cd63da996 | ||
|
|
a89f849158 | ||
|
|
401d5aedf3 | ||
|
|
de123f38bb | ||
|
|
aa0b65c7c7 | ||
|
|
48c3854620 | ||
|
|
d4dcce12d4 | ||
|
|
0b0f947265 | ||
|
|
86cb4a0f97 | ||
|
|
abca3b2e52 | ||
|
|
eb5e8c84be | ||
|
|
aa0feca536 | ||
|
|
66ce8c55cc | ||
|
|
a3d11f9d95 | ||
|
|
28be39f72e | ||
|
|
58988bee2f | ||
|
|
5b8d8b2f88 | ||
|
|
ced0a2c98c | ||
|
|
9d267655e3 | ||
|
|
973fb44471 | ||
|
|
1d59ce7c90 | ||
|
|
525f14040d | ||
|
|
224a247d5b | ||
|
|
c2bc456b64 | ||
|
|
2bc435d32e | ||
|
|
2e3568567b | ||
|
|
40048f6e51 | ||
|
|
78980a3b0b | ||
|
|
0cd8be351d | ||
|
|
06008c170d | ||
|
|
4ce23542bf | ||
|
|
7b977ce5dc | ||
|
|
d34f7b2326 | ||
|
|
19ee4b566b | ||
|
|
b77833c504 | ||
|
|
c19dc43cc7 | ||
|
|
01275aab6b | ||
|
|
9a53f75124 | ||
|
|
d72629e4ba | ||
|
|
c08ffef709 | ||
|
|
542c817bfa | ||
|
|
3afdde5f05 | ||
|
|
077c319914 | ||
|
|
d6cc283da7 | ||
|
|
4cbf290fb9 | ||
|
|
720617bb5b | ||
|
|
5c18780c31 | ||
|
|
db73f35f4a | ||
|
|
6f4c2b9b91 | ||
|
|
91c329cc86 | ||
|
|
66454d704b | ||
|
|
b4701d672d | ||
|
|
8ab21c8a94 | ||
|
|
a7b96fdad7 | ||
|
|
6fd98c98b9 | ||
|
|
c79510cc2a | ||
|
|
4a1b69abc8 | ||
|
|
244db08d60 | ||
|
|
bc3a63ebdb | ||
|
|
ddebc52f23 | ||
|
|
2524b61f9b | ||
|
|
861b11f087 | ||
|
|
cb8dab06be | ||
|
|
8ef6d31cb9 | ||
|
|
9d58189c12 | ||
|
|
debbb5cde9 | ||
|
|
3f00fb7e2e | ||
|
|
70a2bb2a30 | ||
|
|
790551c382 | ||
|
|
c2f14bfbc4 | ||
|
|
367e3700cf | ||
|
|
d00adfd19c | ||
|
|
6412ad8c64 | ||
|
|
771c9d782d | ||
|
|
1815e490de | ||
|
|
c948114d58 |
@@ -0,0 +1,21 @@
|
||||
# Serving a model? Read its cookbook page first
|
||||
|
||||
Before running, deploying, benchmarking, or reproducing a specific model, read
|
||||
its page under `docs/cookbook/`:
|
||||
|
||||
- `docs/cookbook/autoregressive/` — LLM and VLM serving
|
||||
- `docs/cookbook/diffusion/` — image, video, 3D, robotics
|
||||
- `docs/cookbook/omni/`, `docs/cookbook/vla/`, `docs/cookbook/specbundle/`,
|
||||
`docs/cookbook/base/`
|
||||
|
||||
That page carries the deployment we recommend to users — GPU count, parallelism
|
||||
degrees, quantization, and the flags that matter for that model — so it is the
|
||||
answer to "how should this be served", and the baseline any tuning starts from.
|
||||
|
||||
Configs under `test/` are a different thing: pinned, reproducible CI setups for
|
||||
correctness and regression checks, not deployment advice. Reach for them when
|
||||
you need an exactly reproducible run, and say which cookbook flags you diverged
|
||||
from — a measurement taken on a configuration nobody deploys describes nothing.
|
||||
|
||||
Keep it true in both directions: when a change alters a model's recommended
|
||||
deployment, update that model's cookbook page in the same PR.
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
paths:
|
||||
- "**/bench_*.py"
|
||||
- "**/benchmark/**/*.py"
|
||||
- "benchmark/kernels/**"
|
||||
---
|
||||
|
||||
# Kernel benchmarks
|
||||
|
||||
How a benchmark is written is up to you. The requirement is only that the numbers are valid.
|
||||
|
||||
## Tools
|
||||
|
||||
In SGLang we recommend `marker.do_bench` (`python/sglang/kernels/jit/benchmark/marker.py`). It works for any callable, not only JIT kernels, and it measures with a cold L2 by default. It is not mandatory. CUPTI (e.g. CUDA 13 `cupti-python`), the torch profiler / `bench_kineto`, Triton's `do_bench` / proton, `flashinfer.testing`, Nsight Systems / Nsight Compute, or a hand-written loop are all fine. Each pitfall below must be handled, whichever tool you use.
|
||||
|
||||
## L2 cache reuse
|
||||
|
||||
Timing the same kernel on the same inputs in a loop measures an L2-hot kernel, not the one serving runs. `triton.testing.do_bench_cudagraph` with fixed arguments and bare `for` loops between two events both do this. Measure cold L2 by flushing between calls or by rotating enough input copies to exceed L2 several times over. Hot L2 is acceptable only when the working set far exceeds L2 or when hot L2 is the real serving pattern, and the reason must be stated.
|
||||
|
||||
Host-memory (zero-copy) loads do not go through device L2. Peer-GPU (NVLink) reads still see reuse (on remote GPU).
|
||||
@@ -1,10 +1,14 @@
|
||||
---
|
||||
name: add-jit-kernel
|
||||
description: Step-by-step tutorial for adding a new lightweight JIT CUDA kernel to sglang's jit_kernel module
|
||||
description: Step-by-step tutorial for adding a new lightweight JIT CUDA kernel to sglang.kernels JIT infrastructure and public operator groups
|
||||
---
|
||||
|
||||
# Tutorial: Adding a New JIT Kernel to SGLang
|
||||
|
||||
Apply [kernel-organization](../kernel-organization/SKILL.md) for the public
|
||||
operator namespace, logical grouping, lazy registry metadata, and test placement.
|
||||
The implementation tutorial below does not replace that API contract.
|
||||
|
||||
This tutorial walks through adding a simple element-wise scale operation as a JIT kernel. We'll implement `scale(x, factor) = x * factor` to demonstrate the complete workflow.
|
||||
|
||||
## Goal
|
||||
@@ -497,6 +501,32 @@ def scale(src: torch.Tensor, factor: float, out: torch.Tensor | None = None) ->
|
||||
|
||||
---
|
||||
|
||||
## Register the public entry point (required)
|
||||
|
||||
Add metadata in `python/sglang/kernels/ops/elementwise/__init__.py` so the
|
||||
operator appears in the registry without importing its implementation:
|
||||
|
||||
```python
|
||||
from sglang.kernels.registry import register_kernel
|
||||
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="elementwise.scale",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.elementwise.scale:scale",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
Record the devices the implementation actually supports; the CUDA-only scale
|
||||
example above does not imply that every JIT kernel is CUDA-only. Runtime and
|
||||
correctness tests import `scale` from `sglang.kernels.ops.elementwise.scale`.
|
||||
Do not import the implementation eagerly in the group initializer. When an
|
||||
existing `BaseFusedOp` owns the operation, add the implementation there instead
|
||||
of registering a second conflicting op/backend pair.
|
||||
|
||||
## Step 3 (optional): Tune JIT build flags
|
||||
|
||||
If your kernel uses some math functions like `expf` or `sinf`, consider enabling `--use_fast_math` for better performance (with a potential precision tradeoff):
|
||||
@@ -611,6 +641,8 @@ if __name__ == "__main__":
|
||||
|
||||
Benchmarks are `bench_*.py` files under `test/registered/kernels/benchmark/<group>/`. They are picked up by the same `run_suite.py` machinery as unit tests. Register them for **`base-b-kernel-benchmark-test-1-gpu-large`** (PR JIT benchmark job: `python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1-gpu-large`).
|
||||
|
||||
Every benchmark must account for L2 cache reuse — see [`rules/kernel-benchmark.md`](../../rules/kernel-benchmark.md).
|
||||
|
||||
Benchmarks use the project's own `marker` framework (in `python/sglang/kernels/jit/benchmark/marker.py`) — **do not** use `triton.testing.perf_report` / `triton.testing.do_bench` directly. The marker framework provides (public names: `benchmark`, `parametrize`, `do_bench`, `skip`, `BenchResult`, `BenchSkip`):
|
||||
|
||||
- **`@marker.benchmark(line_arg, line_vals, *, unit="us")`** — the **innermost** decorator (bottom of the stack, directly above `def benchmark`). Declares the column axis: each value in `line_vals` becomes a result column, and `line_arg` is the parameter name passed into the benchmark function. `unit` is one of `"us" | "ms" | "s"`.
|
||||
@@ -619,7 +651,7 @@ Benchmarks use the project's own `marker` framework (in `python/sglang/kernels/j
|
||||
- `memory_args`: defaults to `"all"` (footprint derived from all input args/kwargs). Pass an explicit tuple of tensors (e.g. `(k, v, indices)`) to count only the inputs the kernel actually touches.
|
||||
- `memory_output`: defaults to `"out"` — re-runs `fn` once to capture its **returned** tensor and counts it. For in-place kernels (which return `None`), pass the written tensors explicitly (e.g. `memory_output=(k, v)`); the re-run is then skipped. Set to `None` to count no output.
|
||||
- Together `memory_args` + `memory_output` give the GB/s column; with both defaults a function `out = f(src)` already reports `bytes(src) + bytes(out)`.
|
||||
- `graph_clone_args` / `graph_clone_kwargs`: which inputs to clone per CUDA-graph iteration to defeat L2 cache reuse. Defaults to `"all"` — pass an iterable of indices/keys to limit to the *read* args (writes don't need cloning).
|
||||
- `graph_clone_args` / `graph_clone_kwargs`: which inputs to clone per CUDA-graph iteration to defeat L2 cache reuse. Defaults to `"all"`; if you narrow it, keep every read, in-place, and output tensor.
|
||||
- `use_cuda_graph=False` for kernels that can't be captured.
|
||||
- `metrics=(0.5, "avg")` controls reported quantiles (the first metric becomes the table latency column).
|
||||
- `disable_log_bandwidth` (defaults from `SGLANG_JIT_BENCHMARK_DISABLE_LOG_BANDWIDTH=1`) skips the bandwidth column entirely.
|
||||
@@ -724,7 +756,7 @@ cd test && python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1
|
||||
- `python/sglang/kernels/jit/include/sgl_kernel/cta.cuh` — `cta::reduce_max`
|
||||
- `python/sglang/kernels/jit/include/sgl_kernel/atomic.cuh` — `atomic::max`
|
||||
- `python/sglang/kernels/jit/include/sgl_kernel/runtime.cuh` — occupancy / SM count helpers
|
||||
- `python/sglang/kernels/jit/csrc/add_constant.cuh` — minimal runnable reference
|
||||
- `python/sglang/kernels/jit/csrc/elementwise/add_constant.cuh` — minimal runnable reference
|
||||
- `python/sglang/kernels/jit/csrc/elementwise/rmsnorm.cuh` — real example using `TensorMatcher` + `LaunchKernel` + `tile::Memory`
|
||||
- `python/sglang/kernels/jit/csrc/elementwise/qknorm.cuh` — real example using `runtime::get_blocks_per_sm` + persistent kernel pattern
|
||||
- `python/sglang/kernels/jit/benchmark/marker.py` — `benchmark`, `parametrize`, `do_bench`, `BenchResult`
|
||||
|
||||
@@ -5,6 +5,10 @@ description: Step-by-step tutorial for adding a heavyweight AOT CUDA/C++ kernel
|
||||
|
||||
# Tutorial: Adding a New Kernel to `sgl-kernel` (AOT / Heavyweight)
|
||||
|
||||
Apply [kernel-organization](../kernel-organization/SKILL.md) for the public
|
||||
operator namespace, logical grouping, lazy registry metadata, and test placement.
|
||||
The implementation tutorial below does not replace that API contract.
|
||||
|
||||
This tutorial walks through adding a simple element-wise scale operation as an AOT kernel. We'll implement `scale(x, factor) = x * factor` to demonstrate the complete workflow.
|
||||
|
||||
## Goal
|
||||
@@ -25,7 +29,7 @@ Add a new operation that scales each element of a tensor by a scalar factor:
|
||||
In addition, every new kernel must ship with:
|
||||
|
||||
- **Tests** (pytest)
|
||||
- **A benchmark script** (triton.testing)
|
||||
- **A benchmark script** (`marker.do_bench` from `sglang.kernels.jit.benchmark` — it works for any callable, not only JIT kernels)
|
||||
|
||||
---
|
||||
|
||||
@@ -194,6 +198,21 @@ Then re-export it from `python/sglang/kernels/aot/python/sgl_kernel/__init__.py`
|
||||
|
||||
---
|
||||
|
||||
## SGLang integration (required for runtime use)
|
||||
|
||||
After exposing the AOT wheel symbol, add a lazy wrapper and a `KernelSpec` under
|
||||
`python/sglang/kernels/ops/<group>/`. Set `backend=KernelBackend.AOT`, record
|
||||
actual device/architecture support, and keep `sgl_kernel` imports inside the
|
||||
implementation path. SGLang runtime and integration tests import that wrapper.
|
||||
For this example, use `sglang.kernels.ops.elementwise.scale`.
|
||||
|
||||
The wheel-level tests below validate its standalone API/build. They do not
|
||||
replace CI-registered SGLang correctness tests in
|
||||
`test/registered/kernels/ops/elementwise/test_scale.py` and benchmarks in
|
||||
`test/registered/kernels/benchmark/elementwise/bench_scale.py`. Follow
|
||||
`write-sglang-test` for their registration and CI budget; do not register tests
|
||||
under the shipped `python/sglang/` package.
|
||||
|
||||
## Step 6: Write tests (required)
|
||||
|
||||
Create `python/sglang/kernels/aot/tests/test_scale.py`:
|
||||
@@ -241,62 +260,46 @@ if __name__ == "__main__":
|
||||
|
||||
## Step 7: Add a benchmark (required)
|
||||
|
||||
Every benchmark must account for L2 cache reuse — see [`rules/kernel-benchmark.md`](../../rules/kernel-benchmark.md).
|
||||
|
||||
Create `python/sglang/kernels/aot/benchmark/bench_scale.py`:
|
||||
|
||||
```python
|
||||
import itertools
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.testing
|
||||
|
||||
import sgl_kernel
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
IS_CI = is_in_ci()
|
||||
|
||||
dtypes = [torch.float16] if IS_CI else [torch.float16, torch.bfloat16, torch.float32]
|
||||
sizes = [4096] if IS_CI else [2**n for n in range(10, 20)] # 1K … 512K
|
||||
factors = [2.0]
|
||||
|
||||
configs = list(itertools.product(dtypes, sizes))
|
||||
from sglang.kernels.jit.benchmark import marker
|
||||
|
||||
|
||||
def torch_scale(input: torch.Tensor, factor: float) -> torch.Tensor:
|
||||
return input * factor
|
||||
def sglang_scale(input: torch.Tensor, factor: float, out: torch.Tensor) -> None:
|
||||
sgl_kernel.scale(input, factor, out=out)
|
||||
|
||||
|
||||
@triton.testing.perf_report(
|
||||
triton.testing.Benchmark(
|
||||
x_names=["dtype", "size"],
|
||||
x_vals=configs,
|
||||
line_arg="provider",
|
||||
line_vals=["sglang", "torch"],
|
||||
line_names=["SGL Kernel", "PyTorch"],
|
||||
styles=[("green", "-"), ("red", "--")],
|
||||
ylabel="µs (median)",
|
||||
plot_name="scale-performance",
|
||||
args={},
|
||||
def torch_scale(input: torch.Tensor, factor: float, out: torch.Tensor) -> None:
|
||||
torch.mul(input, factor, out=out)
|
||||
|
||||
|
||||
FN_MAP = {"sglang": sglang_scale, "torch": torch_scale}
|
||||
|
||||
|
||||
@marker.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32], [torch.float16])
|
||||
@marker.parametrize("size", [2**n for n in range(10, 20)], [4096]) # 1K .. 512K
|
||||
@marker.benchmark("provider", ["sglang", "torch"])
|
||||
def benchmark(dtype: torch.dtype, size: int, provider: str):
|
||||
input = torch.randn(size, dtype=dtype, device="cuda")
|
||||
out = torch.empty_like(input)
|
||||
return marker.do_bench(
|
||||
FN_MAP[provider],
|
||||
# Pass every tensor through input_args (not a closure) so marker rotates
|
||||
# them across CUDA-graph calls to defeat L2 reuse.
|
||||
input_args=(input, 2.0, out),
|
||||
# Bandwidth = bytes(input) + bytes(out), both already in input_args.
|
||||
memory_output=None,
|
||||
)
|
||||
)
|
||||
def benchmark(dtype, size, provider):
|
||||
input = torch.randn(size, dtype=dtype, device="cuda")
|
||||
out = torch.empty_like(input)
|
||||
factor = 2.0
|
||||
|
||||
if provider == "sglang":
|
||||
fn = lambda: sgl_kernel.scale(input, factor, out=out)
|
||||
else:
|
||||
fn = lambda: torch_scale(input, factor)
|
||||
|
||||
ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(
|
||||
fn, quantiles=[0.5, 0.2, 0.8]
|
||||
)
|
||||
return 1000 * ms, 1000 * max_ms, 1000 * min_ms
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
benchmark.run(print_data=True)
|
||||
benchmark.run()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
@@ -136,8 +136,8 @@ than restating.
|
||||
= `output tok/s/GPU × (isl+osl)/osl`, shown by the card as-is. Flag output-only values.
|
||||
- **Consistent accuracy harness across entries**: every value under one `accuracyLabels`
|
||||
column must be produced by the SAME harness — flag a page that, say, measures one
|
||||
platform's GSM8K with `few_shot_gsm8k --num-questions 200` and another's with
|
||||
`run_eval --eval-name gsm8k --num-examples 1319` and shows both as one "GSM8K %"
|
||||
platform's GSM8K with a historical raw completion harness and another's with
|
||||
`sgl-eval run gsm8k --num-examples 1319` and shows both as one "GSM8K %"
|
||||
(the scores aren't comparable). Either standardize on one harness (matching
|
||||
`benchmarkCommands.accuracy`) or require an explicit per-entry note. Common when folding
|
||||
a second contributor's measurements (e.g. an AMD/ROCm PR) into the page.
|
||||
|
||||
@@ -162,7 +162,7 @@ Picking between `ENABLE_FOO` and `DISABLE_FOO`: both verbs are valid. The only f
|
||||
For a rename where the old key must keep working with a warning:
|
||||
|
||||
```python
|
||||
SGLANG_DSA_FUSE_TOPK = EnvBoolWithAlias(True, deprecated_name="SGLANG_NSA_FUSE_TOPK")
|
||||
SGLANG_NEW_NAME = EnvBoolWithAlias(True, deprecated_name="SGLANG_OLD_NAME")
|
||||
```
|
||||
|
||||
Use `EnvBoolWithAlias` / `EnvIntWithAlias`. The fallback emits a `DeprecationWarning` and copies the old value over.
|
||||
|
||||
@@ -46,13 +46,11 @@ Typical startup time: 30-90 seconds depending on model size and whether CUDA gra
|
||||
### Step 3: Validate accuracy (sanity check)
|
||||
|
||||
```bash
|
||||
python3 -m sglang.test.run_eval --host 127.0.0.1 --port <port> --eval-name gsm8k --num-examples 20
|
||||
sgl-eval run gsm8k --base-url http://127.0.0.1:<port>/v1 --num-examples 20
|
||||
```
|
||||
|
||||
- Expected accuracy: **> 0.8** for capable models (Qwen3-8B, Llama-3.1-8B-Instruct, etc.)
|
||||
- This is a quick sanity check, not a rigorous benchmark.
|
||||
- `sglang.test.few_shot_gsm8k` is deprecated; use the unified `run_eval` entrypoint.
|
||||
- If you intentionally need the old completion-style GSM8K path, add `--api completion`.
|
||||
- If accuracy is unexpectedly low, something is wrong — do not proceed to profiling.
|
||||
|
||||
### Step 4: Generate the profile
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
---
|
||||
name: kernel-organization
|
||||
description: Apply the SGLang kernels RFC when adding, moving, splitting, or reviewing kernel APIs, registry metadata, kernel tests, benchmarks, and model-specific implementations. Use with add-jit-kernel, add-sgl-kernel, and write-sglang-test for placement and migration checks.
|
||||
---
|
||||
|
||||
# Kernel organization
|
||||
|
||||
Read `python/sglang/kernels/README.md` and `test/README.md` before choosing a
|
||||
location. [RFC #29630](https://github.com/sgl-project/sglang/issues/29630)
|
||||
establishes the namespace; subsequent migrations #32148 and #40922 clarify
|
||||
logical grouping and remove the old `_jit_` filename prefix.
|
||||
|
||||
## Choose the owner
|
||||
|
||||
- Put callable SGLang kernel APIs under `sglang.kernels.ops.<group>`. Runtime
|
||||
and integration tests import from that namespace, including its submodules.
|
||||
The `sgl_kernel` wheel retains its own public API and packaging tests.
|
||||
- Group by computation, not model or GPU: GEMM and GEMV in `gemm`, expert
|
||||
routing in `moe`, attention index selection in `attention`, sampling in
|
||||
`sampling`, normalization in `layernorm`. A quantized GEMM is still a GEMM;
|
||||
quantization alone belongs in `quantization`.
|
||||
- Model-specific files/subpackages and tuning data are allowed **inside** a
|
||||
logical group. Do not add a model bundle or an implementation file directly
|
||||
under `ops/`. Propose a new logical group only for a distinct responsibility,
|
||||
and update `ops._GROUPS`, documentation, and tests together.
|
||||
- Classify a fused operator by its complete contract. Do not split a fused
|
||||
kernel into separate launches just to separate norm, RoPE, or quantization.
|
||||
Split unrelated public entry points that happen to share a source file.
|
||||
- Keep shared CUDA build/runtime infrastructure in `kernels/jit`; operator
|
||||
wrappers call it from their logical group. Keep process groups, communicator
|
||||
state, model dispatch, and buffer ownership in `srt`. K3-specific adapters in
|
||||
`srt/layers/communication/` need not pretend to be generic interfaces.
|
||||
|
||||
## Register the public entry point
|
||||
|
||||
Add lazy `KernelSpec` metadata in the owning group's `__init__.py`, or use the
|
||||
existing `BaseFusedOp` registration when the operation has interchangeable
|
||||
backends. Group imports must not import GPU implementations or compile kernels.
|
||||
|
||||
Use `<group>.<name>` for the op id and `module:callable` for the target. Preserve
|
||||
existing backend dispatch and describe actual device/architecture restrictions
|
||||
with `CapabilityRequirement`; JIT/AOT are not device types. Input shape/dtype
|
||||
checks remain part of the entry point's contract. Registering a torch custom op
|
||||
does not register it in the SGLang kernel inventory. Predicates, private JIT
|
||||
factories, reference helpers, and runtime classes are not separate kernel APIs.
|
||||
|
||||
## Place and preserve tests
|
||||
|
||||
- Kernel numerical tests: `test/registered/kernels/ops/<group>/`.
|
||||
- CI microbenchmarks: `test/registered/kernels/benchmark/<group>/`.
|
||||
- Inventory/selector tests: `test/registered/unit/kernels/`.
|
||||
- Runtime unit tests: `test/registered/unit/<subsystem>/`, following the tested
|
||||
runtime module. A CUDA allocation does not make a runtime test a kernel test.
|
||||
- Non-CI smoke scripts and benchmarks: `test/manual/kernels/`; do not leave
|
||||
standalone test/benchmark entry points in production operator modules.
|
||||
- Shared test helpers: `sglang.test.kernels`. Preserve vendored upstream trees
|
||||
and AOT wheel packaging boundaries instead of reorganizing them incidentally.
|
||||
|
||||
For a move/split, preserve assertions, parametrization, fixtures, platform skips,
|
||||
execution entry points, and CI stages/runners. Apportion existing time estimates
|
||||
across split files; do not duplicate the original budget for every output file
|
||||
or silently drop a registration. Follow `write-sglang-test` for CI registration.
|
||||
|
||||
## Verify a migration
|
||||
|
||||
1. Search all runtime, test, benchmark, documentation, patch-string, and lazy
|
||||
registry references before deleting the old path. Check relative imports and
|
||||
package re-exports as well as direct imports. Do not leave forwarding shims.
|
||||
2. Keep tuning files with the GEMM that loads them and preserve relative lookup
|
||||
behavior. Check wheel/package inclusion as well as source-tree execution.
|
||||
3. Preserve module singleton state: every runtime caller must import the same
|
||||
new communication adapter, not a second copy of its buffer registry.
|
||||
4. Separate relocation commits from semantic changes. Use
|
||||
`mechanical-refactor-verify` to reproduce moves, and compare test inventories
|
||||
and CI registration coverage before/after. Only delete an experimental path
|
||||
after checking its call sites, flags, source, and dedicated tests together.
|
||||
5. Run the namespace/dispatch CPU tests, the registered-test validation hook,
|
||||
pre-commit, and relevant GPU tests when available. State exactly which GPU
|
||||
checks ran; import/AST checks do not prove numerical or performance parity.
|
||||
|
||||
Do not infer violations solely from a model name inside a group, a missing
|
||||
`_jit_` prefix, use of a runtime utility, or absence of `BaseFusedOp` inheritance.
|
||||
@@ -46,8 +46,8 @@ in-flight row as shipped.
|
||||
| Pattern | Trace keywords | Primary code | Existing path | Skill should conclude |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Fused residual add + RMSNorm | `fused_add_rmsnorm*`<br>`npu_add_rms_norm`<br>`add_rmsnorm_bias`<br>`gemma_fused_add_rmsnorm`<br>`gemma_rmsnorm_residual_scalar`<br>`_gemma_rmsnorm_residual_kernel`<br>residual add right before norm | `python/sglang/srt/layers/layernorm.py`<br>`python/sglang/srt/layers/gemma4_fused_ops.py`<br>`python/sglang/srt/layers/quantization/modelslim/modelslim.py` | Shared CUDA / ROCm / CPU / NPU fused add-RMSNorm implementations, including Gemma, Gemma4 scalar-residual, and NPU-bias variants | Treat split residual add + RMSNorm as an existing cross-backend fusion first, not a new idea. |
|
||||
| FlashInfer unified `allreduce_fusion` | `cross_device_reduce_1stage*`<br>`all_reduce`<br>`FusedAddRMSNormKernel`<br>`rmsnorm*` | `python/sglang/srt/layers/flashinfer_comm_fusion.py`<br>`python/sglang/srt/layers/layernorm.py::forward_with_allreduce_fusion`<br>`python/sglang/srt/layers/communicator.py::apply_flashinfer_allreduce_fusion` | FlashInfer workspace creation plus `allreduce_fusion(..., pattern=AllReduceFusionPattern.kARResidualRMSNorm, ...)` | First suspect missing / disabled / unsupported FlashInfer allreduce fusion, not a brand new TP fusion idea. |
|
||||
| AITER allreduce fusion | ROCm all-reduce plus RMSNorm still split | `python/sglang/srt/layers/layernorm.py::forward_with_allreduce_fusion`<br>`python/sglang/srt/distributed/communication_op.py::tensor_model_parallel_fused_allreduce_rmsnorm`<br>`python/sglang/srt/layers/communicator.py::apply_aiter_all_reduce_fusion` | ROCm-side fused TP all-reduce + RMSNorm with fallback to plain all-reduce plus norm | On AMD, rule out existing AITER fusion before proposing a new communication fusion. |
|
||||
| FlashInfer unified `allreduce_fusion` | `cross_device_reduce_1stage*`<br>`all_reduce`<br>`FusedAddRMSNormKernel`<br>`rmsnorm*` | `python/sglang/srt/layers/flashinfer_comm_fusion.py`<br>`python/sglang/srt/layers/layernorm.py::forward_with_allreduce_fusion`<br>`python/sglang/srt/layers/layer_boundary/residual/add_norm.py::apply_flashinfer_allreduce_fusion` | FlashInfer workspace creation plus `allreduce_fusion(..., pattern=AllReduceFusionPattern.kARResidualRMSNorm, ...)` | First suspect missing / disabled / unsupported FlashInfer allreduce fusion, not a brand new TP fusion idea. |
|
||||
| AITER allreduce fusion | ROCm all-reduce plus RMSNorm still split | `python/sglang/srt/layers/layernorm.py::forward_with_allreduce_fusion`<br>`python/sglang/srt/distributed/communication_op.py::tensor_model_parallel_fused_allreduce_rmsnorm`<br>`python/sglang/srt/layers/layer_boundary/residual/add_norm.py::apply_aiter_all_reduce_fusion` | ROCm-side fused TP all-reduce + RMSNorm with fallback to plain all-reduce plus norm | On AMD, rule out existing AITER fusion before proposing a new communication fusion. |
|
||||
| Fused activation-and-mul (`SwiGLU` / `GeGLU`) | `silu_and_mul`<br>`gelu_and_mul`<br>`npu_swiglu` | `python/sglang/srt/layers/activation.py` | Single op covers activation plus elementwise multiply across CUDA / CPU / NPU / XPU backends | Treat separate activation + mul on packed MLP outputs as missing existing fusion. |
|
||||
| Fused dual residual RMSNorm | residual add plus two RMSNorm-like kernels around Grok blocks | `python/sglang/srt/layers/elementwise.py::fused_dual_residual_rmsnorm`<br>`python/sglang/srt/models/grok.py` | One Triton kernel computes intermediate residual update and next RMSNorm output together | On Grok-like residual layouts, treat split residual + norm as missing existing fusion. |
|
||||
| In-place QK RMSNorm | split `q_norm` / `k_norm` kernels | `python/sglang/srt/models/utils.py::apply_qk_norm`<br>`python/sglang/kernels/ops/layernorm/norm.py::fused_inplace_qknorm` | In-place JIT QK norm plus optional `alt_stream` overlap for K | Check shape, dtype, deterministic mode, and in-place legality before proposing a new QK fuse. |
|
||||
@@ -70,8 +70,8 @@ in-flight row as shipped.
|
||||
| Fused MoE dispatch / permute / combine | token permutation<br>dispatch / combine<br>grouped top-k<br>many small MoE support kernels | `python/sglang/srt/layers/moe/fused_moe_triton/layer.py`<br>`python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py` | `FusedMoE` plus DeepEP / FlashInfer / FuseEP / standard dispatch backends and `permute_fusion=True` | First ask whether the model is missing an existing `FusedMoE`-style path or backend-specific dispatcher path. |
|
||||
| Fused MoE sum + all-reduce | routed MoE followed by explicit sum-reduce kernels | `python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py`<br>`python/sglang/srt/layers/moe/fused_moe_triton/fused_moe_triton_kernels.py` | `fuse_sum_all_reduce=True` path in the second MoE GEMM | Before inventing a new MoE reduction fuse, check whether `enable_fused_moe_sum_all_reduce` is simply off or the quant path is incompatible. |
|
||||
| Fused MoE activation + quant / re-quant | `silu_and_mul_*quant*`<br>`npu_dequant_swiglu_quant`<br>`swiglu_quant` | `python/sglang/srt/layers/moe/ep_moe/kernels.py`<br>`python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py`<br>`python/sglang/srt/layers/moe/cutlass_w4a8_moe.py`<br>`python/sglang/srt/hardware_backend/npu/quantization/moe_methods.py` | Quantized MoE backends fuse SwiGLU / SiLU-and-mul with FP8 / FP4 / NPU re-quant before the second expert GEMM | If MoE traces show standalone activation then quant kernels, first check whether the quantized fused path is missing. |
|
||||
| DeepSeek comm-prep fused RMSNorm + quant / flatten-quant | `fused_rms_fp8_group_quant`<br>`fused_rms_mxfp4_quant`<br>`fused_flatten_fp8_group_quant`<br>`fused_flatten_mxfp4_quant` | `python/sglang/srt/layers/communicator.py`<br>`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py`<br>`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py` | DeepSeek MLA / MHA ROCm paths fuse RMSNorm or flatten with FP8 / MXFP4 quantization for comm / attention prep | On DeepSeek quant traces, split norm + quant or flatten + quant is an existing family, not a new idea. |
|
||||
| NSA fused top-k transform / page-table build | `fast_topk_transform_fused`<br>`fast_topk_transform_ragged_fused` | `python/sglang/srt/layers/attention/nsa_backend.py` | NSA can fuse top-k selection with paged / ragged index transform instead of separate top-k plus metadata scatter | If NSA top-k metadata work is split, check `SGLANG_NSA_FUSE_TOPK` and backend support first. |
|
||||
| DeepSeek comm-prep fused RMSNorm + quant / flatten-quant | `fused_rms_fp8_group_quant`<br>`fused_rms_mxfp4_quant`<br>`fused_flatten_fp8_group_quant`<br>`fused_flatten_mxfp4_quant` | `python/sglang/srt/layers/layer_boundary/residual/add_norm.py`<br>`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py`<br>`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py` | DeepSeek MLA / MHA ROCm paths fuse RMSNorm or flatten with FP8 / MXFP4 quantization for comm / attention prep | On DeepSeek quant traces, split norm + quant or flatten + quant is an existing family, not a new idea. |
|
||||
| NSA fused top-k transform / page-table build | `fast_topk_transform_fused`<br>`fast_topk_transform_ragged_fused` | `python/sglang/srt/layers/attention/dsa_backend.py` | NSA can fuse top-k selection with paged / ragged index transform instead of separate top-k plus metadata scatter | If NSA top-k metadata work is split, check `SGLANG_DSA_FUSE_TOPK` and backend support first. |
|
||||
| NSA fused quantize + indexed K-cache store | `fused_store_index_k_cache`<br>`act_quant`<br>`index_k_with_scale_buffer` | `python/sglang/kernels/ops/attention/fused_store_index_cache.py`<br>`python/sglang/srt/layers/attention/nsa/nsa_indexer.py` | Single JIT kernel quantizes bf16 K to fp8 + scale and writes directly into NSA index cache | Treat split `act_quant` + buffer-store on CUDA as missing an existing fused store path. |
|
||||
| Fused sampling temperature + softmax | `fused_temperature_softmax*` | `python/sglang/srt/layers/fused_sampling.py`<br>`python/sglang/srt/layers/sampler.py` | Triton single-pass / multi-pass kernels fuse temperature scaling and softmax during decode | Separate temp-divide + softmax at decode batch sizes is often a missed existing fusion. |
|
||||
| Fused logit softcap | `fused_softcap`<br>`final_logit_softcapping` | `python/sglang/srt/layers/elementwise.py`<br>`python/sglang/srt/layers/logits_processor.py` | Triton kernels fuse cast-to-float and softcap / tanh math for logits or generic elementwise softcapping | Treat exposed cast + softcap ladders as an existing Triton fuse family. |
|
||||
@@ -148,8 +148,8 @@ Stable entries should be folded into the mainline family rows above.
|
||||
| PR `#22392` CUTLASS FP8 GEMM replacing nvjet | `cutlass_scaled_mm`<br>`fp8_scaled_mm`<br>`nvjet`<br>`cudaMemsetAsync` | `PR #22392`<br>`python/sglang/kernels/aot/python/sgl_kernel/gemm.py`<br>`python/sglang/srt/layers/quantization/fp8_utils.py` | Runtime replacement swaps nvjet FP8 GEMMs for CUTLASS kernels, removing per-launch memset bubbles and extra output-copy kernels | Treat nvjet GEMM + memset bubble ladders as an in-flight SGLang linear-kernel family before calling them novel. |
|
||||
| PR `#18612` NVFP4 CUTLASS MoE fused SiLU+Mul+quant | `silu_and_mul_scaled_nvfp4`<br>`nvfp4 expert quant`<br>`cutlass moe` | `PR #18612`<br>`python/sglang/srt/layers/moe/cutlass_w4a8_moe.py`<br>`python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py` | Fuses MoE activation epilogue and NVFP4 expert quantization before the CUTLASS MoE second GEMM | Treat split SiLU+Mul then NVFP4 expert quant in CUTLASS MoE traces as an in-flight upstream SGLang family. |
|
||||
| PR `#22918` FlashInfer per-token NVFP4 MoE | `per_token_nvfp4`<br>`trtllm_fp4_block_scale_moe`<br>`FlashInfer MoE` | `PR #22918`<br>`python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py` | Adds FlashInfer-backed per-token NVFP4 MoE execution so expert quant/dequant work can move into the fused MoE backend | Treat standalone per-token NVFP4 MoE support kernels as a candidate missing backend-selection path, not an automatically novel kernel idea. |
|
||||
| PR `#22851` NSA top-k backend and FlashInfer / PyTorch top-k split | `nsa topk`<br>`flashinfer_topk`<br>`pytorch_topk`<br>`fast_topk_transform` | `PR #22851`<br>`python/sglang/srt/layers/attention/nsa_backend.py` | Makes NSA top-k backend selection explicit and aligns fused top-k transform with FlashInfer / PyTorch fallbacks | When NSA top-k dominates decode, first classify it as backend selection or fused-transform eligibility work. |
|
||||
| PR `#24125` GLM5 NSA decode CatArrayBatchedCopy removal | `CatArrayBatchedCopy`<br>`GLM-5`<br>`NSA`<br>`TileLang decode` | `PR #24125`<br>`python/sglang/srt/layers/attention/nsa_backend.py` | Skips redundant cat/copy work in the GLM5 NSA TileLang decode path | Treat cat/copy bursts in GLM5 NSA decode as a concrete in-flight cleanup opportunity. |
|
||||
| PR `#22851` NSA top-k backend and FlashInfer / PyTorch top-k split | `nsa topk`<br>`flashinfer_topk`<br>`pytorch_topk`<br>`fast_topk_transform` | `PR #22851`<br>`python/sglang/srt/layers/attention/dsa_backend.py` | Makes NSA top-k backend selection explicit and aligns fused top-k transform with FlashInfer / PyTorch fallbacks | When NSA top-k dominates decode, first classify it as backend selection or fused-transform eligibility work. |
|
||||
| PR `#24125` GLM5 NSA decode CatArrayBatchedCopy removal | `CatArrayBatchedCopy`<br>`GLM-5`<br>`NSA`<br>`TileLang decode` | `PR #24125`<br>`python/sglang/srt/layers/attention/dsa_backend.py` | Skips redundant cat/copy work in the GLM5 NSA TileLang decode path | Treat cat/copy bursts in GLM5 NSA decode as a concrete in-flight cleanup opportunity. |
|
||||
| PR `#24007` MoE LoRA virtual experts for csgmv backend | `csgmv`<br>`virtual experts`<br>`MoE LoRA`<br>`fused_moe_lora` | `PR #24007`<br>`python/sglang/srt/layers/lora_backend.py`<br>`python/sglang/srt/layers/moe` | Routes MoE LoRA adapter work through virtual experts so csgmv-style kernels can batch it instead of launching fragmented adapter work | Treat MoE-LoRA tiny-kernel ladders as an in-flight batching/fusion family. |
|
||||
| PR `#24150` torch.compile local decode support | `enable_torch_compile`<br>`local compile`<br>`decode compile`<br>`torchinductor` | `PR #24150`<br>`python/sglang/srt` | Extends SGLang torch.compile coverage to local decode regions, so Inductor-generated fusion may replace hand-authored tiny kernels | When decode traces show compiler-generated kernels or missing named fused kernels, check this in-flight compile path before calling the shape unsupported. |
|
||||
|
||||
@@ -319,7 +319,7 @@ only vLLM references to the TokenSpeed package.
|
||||
| `enable_single_batch_overlap` | `python/sglang/srt/server_args.py` | Enables the SBO family. |
|
||||
| `enable_fused_moe_sum_all_reduce` | `python/sglang/srt/server_args.py` | Enables fused MoE sum-reduce in the down path. |
|
||||
| `SGLANG_BLACKWELL_OVERLAP_SHARED_EXPERTS_OUTSIDE_SBO` | `python/sglang/srt/environ.py` | Alters how DeepSeek-style shared-expert overlap behaves on Blackwell. |
|
||||
| `SGLANG_NSA_FUSE_TOPK` | `python/sglang/srt/environ.py` | Gates NSA fused top-k transform / page-table build. |
|
||||
| `SGLANG_DSA_FUSE_TOPK` | `python/sglang/srt/environ.py` | Gates NSA fused top-k transform / page-table build. |
|
||||
| `SGLANG_DISAGG_STAGING_BUFFER` | `python/sglang/srt/environ.py` | Enables the heterogeneous-TP staging-buffer family and its overlap windows. |
|
||||
| `SGLANG_STAGING_USE_TORCH` | `python/sglang/srt/disaggregation/common/staging_buffer.py` | Forces torch fallback for staging gather / scatter, so Triton staging kernels may disappear by design. |
|
||||
| `SGLANG_VIT_ENABLE_CUDA_GRAPH` | `python/sglang/srt/environ.py` | Can intentionally disable vision `aux_stream` overlap. |
|
||||
|
||||
@@ -391,7 +391,7 @@ FUSION_PATTERN_REGISTRY: Tuple[FusionPatternSpec, ...] = (
|
||||
candidate_path=(
|
||||
"python/sglang/srt/layers/flashinfer_comm_fusion.py"
|
||||
"<br>python/sglang/srt/layers/layernorm.py"
|
||||
"<br>python/sglang/srt/layers/communicator.py"
|
||||
"<br>python/sglang/srt/layers/layer_boundary/residual/add_norm.py"
|
||||
),
|
||||
active_keywords=(
|
||||
"allreduce_fusion",
|
||||
@@ -419,7 +419,7 @@ FUSION_PATTERN_REGISTRY: Tuple[FusionPatternSpec, ...] = (
|
||||
pattern="AITER allreduce fusion",
|
||||
candidate_path=(
|
||||
"python/sglang/srt/distributed/communication_op.py"
|
||||
"<br>python/sglang/srt/layers/communicator.py"
|
||||
"<br>python/sglang/srt/layers/layer_boundary/residual/add_norm.py"
|
||||
"<br>python/sglang/srt/layers/layernorm.py"
|
||||
),
|
||||
active_keywords=(
|
||||
@@ -721,7 +721,7 @@ FUSION_PATTERN_REGISTRY: Tuple[FusionPatternSpec, ...] = (
|
||||
FusionPatternSpec(
|
||||
pattern="DeepSeek comm-prep fused RMSNorm + quant / flatten-quant",
|
||||
candidate_path=(
|
||||
"python/sglang/srt/layers/communicator.py"
|
||||
"python/sglang/srt/layers/layer_boundary/residual/add_norm.py"
|
||||
"<br>python/sglang/srt/models/deepseek_common/attention_forward_methods/"
|
||||
"forward_mla.py"
|
||||
"<br>python/sglang/srt/models/deepseek_common/attention_forward_methods/"
|
||||
|
||||
@@ -5,6 +5,10 @@ description: Guide for writing SGLang CI/UT tests. Covers CustomTestCase, CI reg
|
||||
|
||||
# Writing SGLang CI / UT Tests
|
||||
|
||||
Apply [kernel-organization](../kernel-organization/SKILL.md) for the public
|
||||
operator namespace, logical grouping, lazy registry metadata, and test placement.
|
||||
The implementation tutorial below does not replace that API contract.
|
||||
|
||||
This skill covers **how to write and register tests**. For CI pipeline internals (stage ordering, fail-fast, gating, partitioning, debugging CI failures), see the [CI workflow guide](../ci-workflow-guide/SKILL.md). Whether a case is worth adding at all is decided by [`unit-test-admission`](../../rules/unit-test-admission.md) — read it before writing the case, not after.
|
||||
|
||||
## Core Rules
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
# Gitea Actions 自动构建 sglang 镜像(海外节点,原版源)
|
||||
# 基底 lmsysorg/sglang:dev-dsv41(docker.io),依赖走 pypi.org 默认源。
|
||||
# 触发:push 到 dsv41-pd 分支。
|
||||
#
|
||||
# 前置条件(一次性,在 Gitea 实例上配置):
|
||||
# 1. 实例已注册 act_runner(Gitea Actions runner,标签含 ubuntu-latest)
|
||||
# 2. 仓库 Settings → Secrets 添加:
|
||||
# REGISTRY_USERNAME / REGISTRY_PASSWORD(推送镜像的账号,如 Gitea 访问令牌)
|
||||
# 3. 可选:Settings → Actions → Variables 添加 REGISTRY(默认 git.agentwithu.com,
|
||||
# 即 Gitea 自带容器 registry;也可填 docker.io 等)
|
||||
name: build-sglang-image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dsv41-pd, dsv41-pd-visioncp]
|
||||
# compose/部署配置 与 workflow 自身的改动不触发镜像构建(省 runner 与推送带宽)。
|
||||
# 改了 workflow 想重建镜像时,需伴随任意代码改动或手动触发。
|
||||
paths-ignore:
|
||||
- 'deploy/**'
|
||||
- '.gitea/**'
|
||||
|
||||
env:
|
||||
# 直接写死 Gitea 自带 registry(vars context 在该实例上求值异常会导致回退 docker.io)
|
||||
REGISTRY: git.agentwithu.com
|
||||
IMAGE_NAME: minke.yu/sglang
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Resolve tag
|
||||
id: meta
|
||||
run: |
|
||||
SHA9=$(git rev-parse --short=9 HEAD)
|
||||
echo "tag=${{ github.ref_name }}-${SHA9}-ci-$(date -u +%Y%m%d-%H%M)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ secrets.REGISTRY_USERNAME }}
|
||||
password: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
|
||||
- uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile.gitea
|
||||
push: true
|
||||
# cache-from: type=registry
|
||||
# cache-to: type=registry,mode=max
|
||||
tags: |
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.tag }}
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}-latest
|
||||
|
||||
+2
-2
@@ -5,9 +5,9 @@
|
||||
/docs @wisclmy0611 @zijiexia @Richardczl98 @JustinTong0323 @sogalin
|
||||
/python/pyproject.toml @merrymercy @Fridge003 @ispobock
|
||||
/python/sglang/kernels @DarkSharpness @HaiShaw @BBuf @celve @HydraQYH @yuan-luo
|
||||
/python/sglang/kernels/ops/diffusion @yingluosanqian @BBuf @mickqian @triple-mu
|
||||
/python/sglang/kernels/ops/diffusion @yingluosanqian @BBuf @mickqian @triple-mu @niehen6174
|
||||
/python/sglang/kernels/ops/attention/fla @yizhang2077 @hebiao064 @yuan-luo
|
||||
/python/sglang/multimodal_gen @mickqian @ping1jing2 @HaiShaw @yichiche @AgainstEntropy @BBuf @triple-mu @kevin-mii
|
||||
/python/sglang/multimodal_gen @mickqian @ping1jing2 @HaiShaw @yichiche @AgainstEntropy @BBuf @triple-mu @kevin-mii @niehen6174
|
||||
/python/sglang/multimodal_gen/runtime/cache @DefTruth
|
||||
/python/sglang/multimodal_gen/runtime/layers/quantization/ @HaiShaw @BowenBao @OrangeRedeng @TallMessiWu
|
||||
/python/sglang/multimodal_gen/test/server/ascend @ping1jing2 @ssshinigami @Makcum888e @e-martirosian
|
||||
|
||||
@@ -114,7 +114,6 @@ unified-radix-cache:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- 'python/sglang/srt/mem_cache/unified_cache/**/*'
|
||||
- 'python/sglang/srt/mem_cache/cpp_radix_tree/**/*'
|
||||
- 'python/sglang/srt/mem_cache/**/*radix*'
|
||||
- 'python/sglang/srt/mem_cache/chunk_cache.py'
|
||||
- 'python/sglang/srt/mem_cache/evict_policy.py'
|
||||
@@ -125,7 +124,6 @@ hicache:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- '**/*hicache*'
|
||||
- '**/*hiradix*'
|
||||
- '**/HiCache/**/*'
|
||||
- 'benchmark/hicache/**/*'
|
||||
- 'test/registered/hicache/**/*'
|
||||
|
||||
@@ -19,7 +19,7 @@ on:
|
||||
type: string
|
||||
required: true
|
||||
npu_device_type:
|
||||
description: 'NPU device type, e.g. 910b, a3.'
|
||||
description: 'NPU device type, e.g. a3, 950.'
|
||||
type: string
|
||||
default: 'a3'
|
||||
run_timeout_minutes:
|
||||
@@ -39,7 +39,7 @@ on:
|
||||
type: string
|
||||
default: ''
|
||||
skip_pr_test_health_check:
|
||||
description: 'Set to true to skip the PR test health check (fail-fast gate).'
|
||||
description: 'Set to true to skip the PR test health check (fast-fail gate).'
|
||||
type: string
|
||||
default: 'false'
|
||||
is_nightly_pipeline_job:
|
||||
@@ -90,7 +90,112 @@ jobs:
|
||||
run: |
|
||||
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
||||
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
- name: Check PR test health
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
|
||||
with:
|
||||
github-token: ${{ inputs.github-token || github.token }}
|
||||
script: |
|
||||
core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
|
||||
|
||||
// Skip when explicitly requested via env var (e.g. release branch cut)
|
||||
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
|
||||
core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip for scheduled runs — they should collect all failures, not fast-fail
|
||||
if (context.eventName === 'schedule') {
|
||||
core.notice('[health-check] SKIP: scheduled run');
|
||||
return;
|
||||
}
|
||||
|
||||
// Check lint status from the separate Lint workflow (lint.yml).
|
||||
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
|
||||
// checks.listForRef which queries by commit SHA across ALL workflows.
|
||||
const ref = context.payload.pull_request?.head?.sha || context.sha;
|
||||
core.info(`[health-check] Checking lint for ref=${ref}`);
|
||||
const { data } = await github.rest.checks.listForRef({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
ref: ref,
|
||||
check_name: 'lint',
|
||||
});
|
||||
const lintRun = data.check_runs.find(
|
||||
cr => cr.app?.slug === 'github-actions'
|
||||
);
|
||||
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
|
||||
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
|
||||
core.setFailed('Fast-fail: lint check failed');
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
|
||||
// Lint check above still runs.
|
||||
let labels = [];
|
||||
if (context.payload.pull_request?.labels) {
|
||||
labels = context.payload.pull_request.labels.map(l => l.name);
|
||||
} else {
|
||||
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
commit_sha: ref,
|
||||
});
|
||||
if (prs.length > 0) {
|
||||
labels = prs[0].labels.map(l => l.name);
|
||||
}
|
||||
}
|
||||
core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
|
||||
if (labels.includes('bypass-fastfail')) {
|
||||
core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
|
||||
return;
|
||||
}
|
||||
|
||||
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: context.runId,
|
||||
per_page: 100,
|
||||
});
|
||||
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
|
||||
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
|
||||
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
|
||||
|
||||
// Find jobs that failed from a real error, not from fast-fail cascade
|
||||
const rootCauseFailures = jobs.filter(j => {
|
||||
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
|
||||
// h20 runners are flaky (dirty GPU state from prior runs); their failures
|
||||
// should not cascade fast-fail to other stages. j.name shape from
|
||||
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
|
||||
// + optional " (<matrix>)". Split off the base job key before exact
|
||||
// match so we cover both inline + reusable forms without confusing
|
||||
// 'h20' with the 'h200' prefix.
|
||||
const baseName = j.name.split(/[ /]/)[0];
|
||||
if (baseName === 'base-c-test-8-gpu-h20') {
|
||||
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
|
||||
return false;
|
||||
}
|
||||
// multimodal-gen NPU tests should not cascade fast-fail to perf/accuracy stages.
|
||||
if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') {
|
||||
core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`);
|
||||
return false;
|
||||
}
|
||||
// If the failing step is the health check, it's a cascade — skip it
|
||||
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
|
||||
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
|
||||
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
|
||||
|
||||
if (rootCauseFailures.length > 0) {
|
||||
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
|
||||
} else {
|
||||
core.notice('[health-check] PASS: no root cause failures detected');
|
||||
}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
|
||||
@@ -21,7 +21,7 @@ on:
|
||||
required: true
|
||||
type: string
|
||||
description: image for pods
|
||||
default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3"
|
||||
default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.1.0-a3"
|
||||
install_sglang_deps:
|
||||
required: false
|
||||
type: boolean
|
||||
@@ -31,7 +31,7 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'a3'
|
||||
description: device type for dependency installation (a3 or 910b)
|
||||
description: device type for dependency installation (a3 or 950)
|
||||
transformers_version:
|
||||
required: false
|
||||
type: string
|
||||
@@ -43,7 +43,7 @@ on:
|
||||
default: '{}'
|
||||
description: 'JSON run metadata {branch_label, workflow_name, create_time}, recorded once at workflow start'
|
||||
skip_pr_test_health_check:
|
||||
description: 'Set to true to skip the PR test health check (fail-fast gate).'
|
||||
description: 'Set to true to skip the PR test health check (fast-fail gate).'
|
||||
type: string
|
||||
default: 'false'
|
||||
is_nightly_pipeline_job:
|
||||
@@ -101,7 +101,112 @@ jobs:
|
||||
# checks out the same commit even if the branch advances mid-run.
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
- name: Check PR test health
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
|
||||
with:
|
||||
github-token: ${{ inputs.github-token || github.token }}
|
||||
script: |
|
||||
core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
|
||||
|
||||
// Skip when explicitly requested via env var (e.g. release branch cut)
|
||||
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
|
||||
core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip for scheduled runs — they should collect all failures, not fast-fail
|
||||
if (context.eventName === 'schedule') {
|
||||
core.notice('[health-check] SKIP: scheduled run');
|
||||
return;
|
||||
}
|
||||
|
||||
// Check lint status from the separate Lint workflow (lint.yml).
|
||||
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
|
||||
// checks.listForRef which queries by commit SHA across ALL workflows.
|
||||
const ref = context.payload.pull_request?.head?.sha || context.sha;
|
||||
core.info(`[health-check] Checking lint for ref=${ref}`);
|
||||
const { data } = await github.rest.checks.listForRef({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
ref: ref,
|
||||
check_name: 'lint',
|
||||
});
|
||||
const lintRun = data.check_runs.find(
|
||||
cr => cr.app?.slug === 'github-actions'
|
||||
);
|
||||
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
|
||||
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
|
||||
core.setFailed('Fast-fail: lint check failed');
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
|
||||
// Lint check above still runs.
|
||||
let labels = [];
|
||||
if (context.payload.pull_request?.labels) {
|
||||
labels = context.payload.pull_request.labels.map(l => l.name);
|
||||
} else {
|
||||
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
commit_sha: ref,
|
||||
});
|
||||
if (prs.length > 0) {
|
||||
labels = prs[0].labels.map(l => l.name);
|
||||
}
|
||||
}
|
||||
core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
|
||||
if (labels.includes('bypass-fastfail')) {
|
||||
core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
|
||||
return;
|
||||
}
|
||||
|
||||
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: context.runId,
|
||||
per_page: 100,
|
||||
});
|
||||
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
|
||||
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
|
||||
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
|
||||
|
||||
// Find jobs that failed from a real error, not from fast-fail cascade
|
||||
const rootCauseFailures = jobs.filter(j => {
|
||||
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
|
||||
// h20 runners are flaky (dirty GPU state from prior runs); their failures
|
||||
// should not cascade fast-fail to other stages. j.name shape from
|
||||
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
|
||||
// + optional " (<matrix>)". Split off the base job key before exact
|
||||
// match so we cover both inline + reusable forms without confusing
|
||||
// 'h20' with the 'h200' prefix.
|
||||
const baseName = j.name.split(/[ /]/)[0];
|
||||
if (baseName === 'base-c-test-8-gpu-h20') {
|
||||
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
|
||||
return false;
|
||||
}
|
||||
// multimodal-gen NPU tests should not cascade fast-fail to perf/accuracy stages.
|
||||
if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') {
|
||||
core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`);
|
||||
return false;
|
||||
}
|
||||
// If the failing step is the health check, it's a cascade — skip it
|
||||
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
|
||||
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
|
||||
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
|
||||
|
||||
if (rootCauseFailures.length > 0) {
|
||||
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
|
||||
} else {
|
||||
core.notice('[health-check] PASS: no root cause failures detected');
|
||||
}
|
||||
|
||||
- name: Check npu info
|
||||
run: |
|
||||
|
||||
@@ -27,6 +27,8 @@ on:
|
||||
value: ${{ jobs.run.outputs.jit_kernel }}
|
||||
multimodal_gen:
|
||||
value: ${{ jobs.run.outputs.multimodal_gen }}
|
||||
renderer:
|
||||
value: ${{ jobs.run.outputs.renderer }}
|
||||
rust_workspace:
|
||||
value: ${{ jobs.run.outputs.rust_workspace }}
|
||||
partitions:
|
||||
@@ -50,6 +52,7 @@ jobs:
|
||||
sgl_kernel: ${{ steps.filter.outputs.sgl_kernel }}
|
||||
jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }}
|
||||
multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }}
|
||||
renderer: ${{ steps.filter.outputs.renderer || steps.run-mode.outputs.run_all_tests }}
|
||||
rust_workspace: ${{ steps.filter.outputs.rust_workspace || steps.run-mode.outputs.run_all_tests }}
|
||||
partitions: ${{ steps.partitions.outputs.partitions }}
|
||||
partition_model_sha: ${{ steps.partition-model-sha.outputs.sha }}
|
||||
@@ -96,8 +99,11 @@ jobs:
|
||||
- "scripts/ci/cuda/*"
|
||||
- "scripts/ci/utils/*"
|
||||
- "test/**/!(*.md)"
|
||||
- "rust/**"
|
||||
- "rust/*"
|
||||
- "rust/!(sglang-renderer)/**"
|
||||
- "proto/sglang/runtime/v1/sglang.proto"
|
||||
renderer:
|
||||
- "rust/sglang-renderer/**"
|
||||
simulator:
|
||||
- ".github/workflows/pr-test-extra.yml"
|
||||
- ".github/workflows/_pr-test-simulator-cpu.yml"
|
||||
|
||||
@@ -13,7 +13,7 @@ on:
|
||||
type: string
|
||||
required: true
|
||||
check_changes:
|
||||
description: 'toJson(needs.check-changes.outputs). Read via fromJson(...).main_package / rust_workspace / continue_on_error / partition_model_sha.'
|
||||
description: 'toJson(needs.check-changes.outputs). Read via fromJson(...).main_package / renderer / rust_workspace / continue_on_error / partition_model_sha.'
|
||||
type: string
|
||||
required: true
|
||||
caller_inputs:
|
||||
@@ -49,7 +49,7 @@ jobs:
|
||||
# emits a skipped entry for wait-for-jobs to match. The schedule /
|
||||
# parallel-dispatch / !failure() half stays on the caller, where the
|
||||
# `needs` context it guards actually lives.
|
||||
if: fromJson(inputs.check_changes).main_package == 'true'
|
||||
if: fromJson(inputs.check_changes).main_package == 'true' || fromJson(inputs.check_changes).renderer == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# Delegates to pr-test-npu.yml with coverage_mode=true, which switches:
|
||||
# - Runners to the coverage pool (shared /root/.cache with setup-covstub)
|
||||
# - use_coverage_runner=true on all test jobs
|
||||
# - Skips pr-gate, base-a (910b), multimodal-gen, recommend, analyze
|
||||
# - Skips pr-gate, multimodal-gen, recommend, analyze
|
||||
# - Runs setup-covstub to assemble coverage data and publish it to the
|
||||
# shared disk at a fixed path (/root/.cache/tests/precise-test/coverage)
|
||||
# pr-test-npu-finish is the final gate in the called workflow.
|
||||
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
matrix: ${{ fromJson(needs.compute-diffusion-partitions.outputs.matrix-1npu) }}
|
||||
timeout-minutes: 150
|
||||
container:
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.1.0-a3-ubuntu22.04-py3.12
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -166,7 +166,7 @@ jobs:
|
||||
matrix: ${{ fromJson(needs.compute-diffusion-partitions.outputs.matrix-2npu) }}
|
||||
timeout-minutes: 150
|
||||
container:
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.1.0-a3-ubuntu22.04-py3.12
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
@@ -18,16 +18,11 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'all'
|
||||
image_a2:
|
||||
description: 'The a2 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b'
|
||||
image_a3:
|
||||
description: 'The a3 running docker image of the test task. Must be an image with sglang preinstalled (e.g. lmsysorg/sglang:*a3*).'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3'
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.1.0-a3'
|
||||
|
||||
concurrency:
|
||||
group: full-test-npu-${{ inputs.ref || github.ref }}
|
||||
@@ -39,7 +34,6 @@ jobs:
|
||||
outputs:
|
||||
ref: ${{ steps.set-vars.outputs.ref }}
|
||||
job_filter: ${{ steps.set-vars.outputs.job_filter }}
|
||||
image_a2: ${{ steps.set-vars.outputs.image_a2 }}
|
||||
image_a3: ${{ steps.set-vars.outputs.image_a3 }}
|
||||
steps:
|
||||
# When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
|
||||
@@ -58,14 +52,8 @@ jobs:
|
||||
echo "job_filter=${{ inputs.job_filter }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
if [ -z "${{ inputs.image_a2 }}" ]; then
|
||||
echo "image_a2=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "image_a2=${{ inputs.image_a2 }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
if [ -z "${{ inputs.image_a3 }}" ]; then
|
||||
echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3" >> $GITHUB_OUTPUT
|
||||
echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.1.0-a3" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
@@ -81,20 +69,6 @@ jobs:
|
||||
image_a3: ${{ needs.set-image-config.outputs.image_a3 }}
|
||||
secrets: inherit
|
||||
|
||||
full-4-npu-a2:
|
||||
name: full-4-npu-a2
|
||||
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
|
||||
needs: [set-image-config]
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: linux-aarch64-a2-4
|
||||
test_type: 'perf'
|
||||
test_suite: full-4-npu-a2
|
||||
is_nightly_pipeline_job: true
|
||||
skip_pr_test_health_check: 'true'
|
||||
image: ${{ needs.set-image-config.outputs.image_a2 }}
|
||||
device_type_for_deps: '910b'
|
||||
|
||||
full-1-npu-a3:
|
||||
name: full-1-npu-a3
|
||||
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
|
||||
@@ -317,7 +291,6 @@ jobs:
|
||||
if: github.repository == 'sgl-project/sglang' && always()
|
||||
needs:
|
||||
- nighly-test-npu
|
||||
- full-4-npu-a2
|
||||
- full-1-npu-a3
|
||||
- full-2-npu-a3
|
||||
- full-4-npu-a3
|
||||
|
||||
@@ -1,18 +1,16 @@
|
||||
name: Nightly Test (AMD MI355X 2N 1P1D Disagg)
|
||||
name: Nightly Test (AMD MI355X Disagg)
|
||||
|
||||
# Self-hosted runners on the amd-sglang login node salloc two or four MI355X
|
||||
# nodes and run the Docker harness (prefill + decode over MORI, a standalone
|
||||
# load balancer, then an sglang.bench_serving concurrency sweep) via
|
||||
# scripts/ci/slurm/launch_mi355x.sh.
|
||||
on:
|
||||
# Temporarily paused while the DI CI nodes are unavailable. Uncomment the
|
||||
# schedule below when the nodes return.
|
||||
# schedule:
|
||||
# - cron: '0 7 * * *'
|
||||
schedule:
|
||||
- cron: '0 7 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
image:
|
||||
description: 'Optional. ROCm SGLang Docker image to benchmark. Leave empty for the recipe default.'
|
||||
description: 'Optional. ROCm SGLang Docker image to benchmark. Leave empty for the latest MI35x ROCm 10 image.'
|
||||
required: false
|
||||
default: ''
|
||||
configs:
|
||||
@@ -44,29 +42,38 @@ jobs:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Resolve latest MI35x nightly image
|
||||
- name: Resolve latest MI35x ROCm 10 nightly image
|
||||
id: image
|
||||
# Track the newest published image so the nightly actually catches
|
||||
# regressions instead of re-testing one pinned build. We resolve on the
|
||||
# GitHub-hosted runner (clean Docker Hub access, no rate limits) and pass
|
||||
# the tag to the self-hosted runner. Scope is deliberately tight: only
|
||||
# the clean main MI35x ROCm 7.2 line
|
||||
# `lmsysorg/sglang-rocm:v*-rocm720-mi35x-YYYYMMDD` (excludes -test-,
|
||||
# miles-, mori-, vllm-, rocm7_14 variants). On any failure we emit an
|
||||
# empty string and the launcher falls back to the recipe's pinned
|
||||
# `runtime.image`.
|
||||
# the clean main MI35x ROCm 10 line
|
||||
# `lmsysorg/sglang-rocm:v*-rocm10-mi35x-YYYYMMDD` (excludes -test-,
|
||||
# miles-, mori-, vllm- variants). If resolution fails, use the pinned
|
||||
# ROCm 10 image from the last validated full workflow run.
|
||||
run: |
|
||||
set -o pipefail
|
||||
FALLBACK_IMAGE="lmsysorg/sglang-rocm:v0.5.20-rocm10-mi35x-20260922"
|
||||
RESOLVED=$(curl -fsSL "https://hub.docker.com/v2/repositories/lmsysorg/sglang-rocm/tags?page_size=100&ordering=last_updated" \
|
||||
| python3 -c "import json,re,sys;p=re.compile(r'^v[0-9][0-9A-Za-z.]*-rocm720-mi35x-([0-9]{8})\$');c=sorted((m.group(1),t['name']) for t in json.load(sys.stdin).get('results',[]) for m in [p.match(t.get('name',''))] if m);print(c[-1][1] if c else '')" || true)
|
||||
| python3 -c "import json,re,sys;p=re.compile(r'^v[0-9][0-9A-Za-z.]*-rocm10-mi35x-([0-9]{8})\$');c=sorted((m.group(1),t['name']) for t in json.load(sys.stdin).get('results',[]) for m in [p.match(t.get('name',''))] if m);print(c[-1][1] if c else '')" || true)
|
||||
if [ -n "$RESOLVED" ]; then
|
||||
echo "Resolved latest MI35x image: lmsysorg/sglang-rocm:$RESOLVED"
|
||||
echo "image=lmsysorg/sglang-rocm:$RESOLVED" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "Could not resolve a latest image; launcher will use recipe default."
|
||||
echo "image=" >> $GITHUB_OUTPUT
|
||||
echo "Could not resolve a latest image; using fallback: $FALLBACK_IMAGE"
|
||||
echo "image=$FALLBACK_IMAGE" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Record which image this run uses
|
||||
env:
|
||||
DISPATCH_IMAGE: ${{ inputs.image }}
|
||||
RESOLVED_IMAGE: ${{ steps.image.outputs.image }}
|
||||
run: |
|
||||
IMAGE="${DISPATCH_IMAGE:-$RESOLVED_IMAGE}"
|
||||
echo "### Image" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "\`$IMAGE\`" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Generate benchmark matrix
|
||||
id: generate
|
||||
env:
|
||||
@@ -133,7 +140,6 @@ jobs:
|
||||
CONFIG_FILE: ${{ matrix.config.config_file }}
|
||||
RESULT_FILENAME: mi355x-${{ matrix.config.name }}
|
||||
MATRIX_CONFIG_NAME: ${{ matrix.config.name }}
|
||||
SLURM_EXCLUDE: mia1-p01-g20,mia1-p02-g09
|
||||
# NOTE: RUNNER_NAME is a built-in default env var on every runner, so the
|
||||
# launch + cleanup steps read it directly. Do NOT set it here from
|
||||
# ${{ runner.name }} -- the `runner` context is not available in job-level
|
||||
@@ -261,8 +267,7 @@ jobs:
|
||||
- setup
|
||||
- nightly-mi355x-2n
|
||||
if: |
|
||||
false
|
||||
&& !cancelled() && github.repository == 'sgl-project/sglang'
|
||||
!cancelled() && github.repository == 'sgl-project/sglang'
|
||||
&& needs.setup.result == 'success'
|
||||
&& needs.setup.outputs.matrix-4n != '{"include":[]}'
|
||||
runs-on: linux-sglang-mi35x-di
|
||||
|
||||
@@ -93,8 +93,8 @@ on:
|
||||
- nightly-8-gpu-glm51
|
||||
# 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance)
|
||||
- nightly-8-gpu-mi35x-glm52-fp8
|
||||
# 8-GPU GLM-5-MXFP4 (MI35x only)
|
||||
- nightly-8-gpu-mi35x-glm5-mxfp4
|
||||
# 8-GPU GLM-5.3-Flash (MI35x)
|
||||
- nightly-8-gpu-mi35x-glm53-flash
|
||||
# 4-GPU MiniMax-M2.5 (MI35x)
|
||||
- nightly-4-gpu-mi35x-minimax-m25
|
||||
# 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance)
|
||||
@@ -2044,16 +2044,16 @@ jobs:
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# ==============================================================================
|
||||
# 8-GPU GLM-5-MXFP4 (MI35x only)
|
||||
# 8-GPU GLM-5.3-Flash (MI35x)
|
||||
# ==============================================================================
|
||||
|
||||
nightly-8-gpu-mi35x-glm5-mxfp4:
|
||||
name: ${{ format('nightly-8-gpu-mi35x-glm5-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
|
||||
nightly-8-gpu-mi35x-glm53-flash:
|
||||
name: ${{ format('nightly-8-gpu-mi35x-glm53-flash ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm5-mxfp4,'))
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm53-flash,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -2070,34 +2070,21 @@ jobs:
|
||||
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
ENABLE_CACHE_HOST: "1"
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
|
||||
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Accuracy Test MI35x ROCm (8-GPU GLM-5-MXFP4)
|
||||
- name: Accuracy Test MI35x ROCm (8-GPU GLM-5.3-Flash DSA+KDA)
|
||||
timeout-minutes: 180
|
||||
run: |
|
||||
> github_summary.md
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e SGLANG_USE_AITER=1 \
|
||||
-e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm5-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
- name: Performance Test MI35x ROCm (8-GPU GLM-5-MXFP4)
|
||||
timeout-minutes: 300
|
||||
continue-on-error: true
|
||||
run: |
|
||||
> github_summary.md
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e SGLANG_USE_AITER=1 \
|
||||
-e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 registered/amd/perf/mi35x/test_glm5_mxfp4_perf_mi35x.py || TEST_EXIT_CODE=$?
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm53-flash --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
@@ -2368,8 +2355,8 @@ jobs:
|
||||
- nightly-8-gpu-glm51
|
||||
# 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance)
|
||||
- nightly-8-gpu-mi35x-glm52-fp8
|
||||
# 8-GPU GLM-5-MXFP4 (MI35x only)
|
||||
- nightly-8-gpu-mi35x-glm5-mxfp4
|
||||
# 8-GPU GLM-5.3-Flash (MI35x)
|
||||
- nightly-8-gpu-mi35x-glm53-flash
|
||||
# 4-GPU MiniMax-M2.5 (MI35x)
|
||||
- nightly-4-gpu-mi35x-minimax-m25
|
||||
# 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance)
|
||||
|
||||
@@ -19,26 +19,16 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'all'
|
||||
image_a2:
|
||||
description: 'The a2 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b'
|
||||
image_a3:
|
||||
description: 'The a3 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3'
|
||||
image_a3_cann910:
|
||||
description: 'The a3 running docker image of the test task with cann 9.1.0.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-a3-nightly'
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.1.0-a3'
|
||||
image_a5:
|
||||
description: 'The a5 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-950-nightly'
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.1.0-950'
|
||||
skip_install_flag:
|
||||
description: 'Indicates whether to skip the installation of sglang, defaulting to false.'
|
||||
required: false
|
||||
@@ -56,21 +46,11 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'all'
|
||||
image_a2:
|
||||
description: 'The a2 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
image_a3:
|
||||
description: 'The a3 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
image_a3_cann910:
|
||||
description: 'The a3 running docker image of the test task with cann 9.1.0.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
image_a5:
|
||||
description: 'The a5 running docker image of the test task.'
|
||||
required: false
|
||||
@@ -90,9 +70,7 @@ jobs:
|
||||
outputs:
|
||||
ref: ${{ steps.set-vars.outputs.ref }}
|
||||
job_filter: ${{ steps.set-vars.outputs.job_filter }}
|
||||
image_a2: ${{ steps.set-vars.outputs.image_a2 }}
|
||||
image_a3: ${{ steps.set-vars.outputs.image_a3 }}
|
||||
image_a3_cann910: ${{ steps.set-vars.outputs.image_a3_cann910 }}
|
||||
image_a5: ${{ steps.set-vars.outputs.image_a5 }}
|
||||
skip_install_flag: ${{ steps.set-vars.outputs.skip_install_flag }}
|
||||
run_start_metadata: ${{ steps.set-vars.outputs.run_start_metadata }}
|
||||
@@ -115,23 +93,13 @@ jobs:
|
||||
else
|
||||
echo "job_filter=${{ inputs.job_filter }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
if [ -z "${{ inputs.image_a2 }}" ]; then
|
||||
echo "image_a2=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "image_a2=${{ inputs.image_a2 }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
if [ -z "${{ inputs.image_a3 }}" ]; then
|
||||
echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3" >> $GITHUB_OUTPUT
|
||||
echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.1.0-a3" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
if [ -z "${{ inputs.image_a3_cann910 }}" ]; then
|
||||
echo "image_a3_cann910=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-a3-nightly" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "image_a3_cann910=${{ inputs.image_a3_cann910 }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
if [ -z "${{ inputs.image_a5 }}" ]; then
|
||||
echo "image_a5=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-950-nightly" >> $GITHUB_OUTPUT
|
||||
echo "image_a5=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.1.0-950" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "image_a5=${{ inputs.image_a5 }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
@@ -329,41 +297,6 @@ jobs:
|
||||
install_sglang_deps: false
|
||||
device_type_for_deps: 'a5'
|
||||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||||
|
||||
# Suites suffixed with `-cann910` run against the CANN 9.1.0 image.
|
||||
nightly-perf-16-npu-a3-cann910:
|
||||
name: nightly-perf-16-npu-a3-cann910
|
||||
if: ${{ !cancelled() }}
|
||||
needs: [set-image-config]
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: linux-aarch64-a3-800t-16
|
||||
test_type: 'perf'
|
||||
test_suite: nightly-perf-16-npu-a3-cann910
|
||||
is_nightly_pipeline_job: true
|
||||
skip_pr_test_health_check: 'true'
|
||||
image: ${{ needs.set-image-config.outputs.image_a3_cann910 }}
|
||||
install_sglang_deps: false
|
||||
device_type_for_deps: 'a3'
|
||||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||||
test_timeout_minutes: '180'
|
||||
|
||||
nightly-acc-16-npu-a3-cann910:
|
||||
name: nightly-acc-16-npu-a3-cann910
|
||||
if: ${{ !cancelled() }}
|
||||
needs: [set-image-config]
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: linux-aarch64-a3-16-
|
||||
test_type: 'accuracy'
|
||||
test_suite: nightly-acc-16-npu-a3-cann910
|
||||
is_nightly_pipeline_job: true
|
||||
skip_pr_test_health_check: 'true'
|
||||
image: ${{ needs.set-image-config.outputs.image_a3_cann910 }}
|
||||
install_sglang_deps: false
|
||||
device_type_for_deps: 'a3'
|
||||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||||
|
||||
nightly-poc-multi-node-tests:
|
||||
name: multi-node-poc
|
||||
if: ${{ !cancelled() }}
|
||||
@@ -444,72 +377,6 @@ jobs:
|
||||
transformers_version: ''
|
||||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||||
|
||||
nightly-poc-multi-node-tests-cann910:
|
||||
name: multi-node-poc-cann910
|
||||
if: ${{ !cancelled() }}
|
||||
# Multi-node jobs run strictly one at a time to avoid contending for the
|
||||
# same A3 multi-node resources: 9.0.0 multi -> 9.0.0 mix -> cann910 multi -> cann910 mix.
|
||||
needs: [set-image-config, nightly-perf-16-npu-a3-cann910, nightly-acc-16-npu-a3-cann910, nightly-poc-multi-node-tests, nightly-poc-multi-node-mix-tests]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
test_config:
|
||||
# deepseek_v4_flash performance tests
|
||||
- name: deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms
|
||||
prefill_size: 1
|
||||
decode_size: 1
|
||||
router_size: 1
|
||||
test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py
|
||||
test_type: 'perf'
|
||||
prefill_decode_deployment: 'separation'
|
||||
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
|
||||
with:
|
||||
runner: linux-amd64-cpu-4
|
||||
test_type: ${{ matrix.test_config.test_type || 'perf' }}
|
||||
test_config_name: ${{ matrix.test_config.name }}
|
||||
prefill_size: ${{ matrix.test_config.prefill_size }}
|
||||
decode_size: ${{ matrix.test_config.decode_size }}
|
||||
router_size: ${{ matrix.test_config.router_size }}
|
||||
test_case: ${{ matrix.test_config.test_case }}
|
||||
image: ${{ needs.set-image-config.outputs.image_a3_cann910 }}
|
||||
install_sglang_from_source: false
|
||||
prefill_decode_deployment: ${{ matrix.test_config.prefill_decode_deployment }}
|
||||
transformers_version: ''
|
||||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||||
|
||||
nightly-poc-multi-node-mix-tests-cann910:
|
||||
name: multi-node-mix-poc-cann910
|
||||
if: false
|
||||
needs: [set-image-config, nightly-poc-multi-node-tests-cann910]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
test_config:
|
||||
# glm_5_2 accuracy tests
|
||||
- name: glm_5_2_w8a8_16p_gpqa
|
||||
node_size: 2
|
||||
test_case: test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w8a8_16p_gpqa.py
|
||||
test_type: 'accuracy'
|
||||
# kimi_k3 accuracy tests
|
||||
- name: kimi_k3_w4a8_32p_gpqa
|
||||
node_size: 4
|
||||
test_case: test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py
|
||||
test_type: 'accuracy'
|
||||
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
|
||||
with:
|
||||
runner: linux-amd64-cpu-4
|
||||
test_type: ${{ matrix.test_config.test_type || 'perf' }}
|
||||
test_config_name: ${{ matrix.test_config.name }}
|
||||
node_size: ${{ matrix.test_config.node_size }}
|
||||
test_case: ${{ matrix.test_config.test_case }}
|
||||
image: ${{ needs.set-image-config.outputs.image_a3_cann910 }}
|
||||
install_sglang_from_source: false
|
||||
prefill_decode_deployment: 'mix'
|
||||
transformers_version: ''
|
||||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||||
|
||||
check-all-jobs:
|
||||
if: ${{ !cancelled() }}
|
||||
needs:
|
||||
@@ -523,21 +390,15 @@ jobs:
|
||||
- nightly-perf-16-npu-a3
|
||||
- nightly-acc-2-npu-a3
|
||||
- nightly-acc-16-npu-a3
|
||||
- nightly-perf-16-npu-a3-cann910
|
||||
- nightly-acc-16-npu-a3-cann910
|
||||
- nightly-acc-4-npu-a5
|
||||
- nightly-poc-multi-node-tests
|
||||
- nightly-poc-multi-node-mix-tests
|
||||
- nightly-poc-multi-node-tests-cann910
|
||||
- nightly-poc-multi-node-mix-tests-cann910
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Generate results table
|
||||
run: |
|
||||
multi_result="${{ needs.nightly-poc-multi-node-tests.result }}"
|
||||
mix_result="${{ needs.nightly-poc-multi-node-mix-tests.result }}"
|
||||
multi_cann910_result="${{ needs.nightly-poc-multi-node-tests-cann910.result }}"
|
||||
mix_cann910_result="${{ needs.nightly-poc-multi-node-mix-tests-cann910.result }}"
|
||||
|
||||
# single-node suites are reported per suite; overall status is success if none failed
|
||||
single_result="success"
|
||||
@@ -552,8 +413,6 @@ jobs:
|
||||
"${{ needs.nightly-perf-16-npu-a3.result }}" \
|
||||
"${{ needs.nightly-acc-2-npu-a3.result }}" \
|
||||
"${{ needs.nightly-acc-16-npu-a3.result }}" \
|
||||
"${{ needs.nightly-perf-16-npu-a3-cann910.result }}" \
|
||||
"${{ needs.nightly-acc-16-npu-a3-cann910.result }}" \
|
||||
"${{ needs.nightly-acc-4-npu-a5.result }}"; do
|
||||
if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then
|
||||
single_result="failure"
|
||||
@@ -585,8 +444,6 @@ jobs:
|
||||
"nightly-perf-16-npu-a3:${{ needs.nightly-perf-16-npu-a3.result }}" \
|
||||
"nightly-acc-2-npu-a3:${{ needs.nightly-acc-2-npu-a3.result }}" \
|
||||
"nightly-acc-16-npu-a3:${{ needs.nightly-acc-16-npu-a3.result }}" \
|
||||
"nightly-perf-16-npu-a3-cann910:${{ needs.nightly-perf-16-npu-a3-cann910.result }}" \
|
||||
"nightly-acc-16-npu-a3-cann910:${{ needs.nightly-acc-16-npu-a3-cann910.result }}" \
|
||||
"nightly-acc-4-npu-a5:${{ needs.nightly-acc-4-npu-a5.result }}"; do
|
||||
suite="${entry%%:*}"
|
||||
r="${entry##*:}"
|
||||
@@ -594,12 +451,10 @@ jobs:
|
||||
done
|
||||
echo "| multi-node-poc | $(group_icon ${multi_result}) ${multi_result} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| multi-node-mix-poc | $(group_icon ${mix_result}) ${mix_result} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| multi-node-poc-cann910 | $(group_icon ${multi_cann910_result}) ${multi_cann910_result} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| multi-node-mix-poc-cann910 | $(group_icon ${mix_cann910_result}) ${mix_cann910_result} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
FAIL=0
|
||||
for r in "${single_result}" "${multi_result}" "${mix_result}" "${multi_cann910_result}" "${mix_cann910_result}"; do
|
||||
for r in "${single_result}" "${multi_result}" "${mix_result}"; do
|
||||
if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then FAIL=1; fi
|
||||
done
|
||||
exit $FAIL
|
||||
|
||||
@@ -9,6 +9,10 @@ on:
|
||||
description: "Whether the PR must also have the run-ci-extra label (in addition to run-ci)"
|
||||
type: boolean
|
||||
default: false
|
||||
require-label:
|
||||
description: "Additional label the PR must carry besides run-ci (empty = disabled)"
|
||||
type: string
|
||||
default: ''
|
||||
cool-down-minutes:
|
||||
description: "Cooldown period in minutes for low-permission users; 0 disables rate limiting"
|
||||
type: number
|
||||
@@ -18,10 +22,15 @@ jobs:
|
||||
pr-gate:
|
||||
# 1. for commits on main: no gating needed
|
||||
# 2. for workflow_dispatch: this can only be triggered by users with write access
|
||||
# 3. for pull_request and pull_request_target: apply the checks below.
|
||||
# This job only reads PR metadata through the API and never checks out
|
||||
# or executes PR code, so it is safe for pull_request_target callers.
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
IS_PR_EVENT: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}
|
||||
steps:
|
||||
- name: Fetch latest PR info
|
||||
if: github.event_name == 'pull_request'
|
||||
if: env.IS_PR_EVENT == 'true'
|
||||
id: pr
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
@@ -37,7 +46,7 @@ jobs:
|
||||
core.setOutput("user", pr.data.user.login);
|
||||
|
||||
- name: Log PR info
|
||||
if: github.event_name == 'pull_request'
|
||||
if: env.IS_PR_EVENT == 'true'
|
||||
run: |
|
||||
echo "===== PR Info ====="
|
||||
echo "PR Event: ${{ github.event_name }}"
|
||||
@@ -46,17 +55,18 @@ jobs:
|
||||
echo "PR User: ${{ steps.pr.outputs.user }}"
|
||||
echo "Require run-ci: ${{ inputs.require-run-ci }}"
|
||||
echo "Require run-ci-extra: ${{ inputs.require-run-ci-extra }}"
|
||||
echo "Require label: ${{ inputs.require-label }}"
|
||||
echo "Cool down minutes: ${{ inputs.cool-down-minutes }}"
|
||||
echo "==================="
|
||||
|
||||
- name: Block draft PR
|
||||
if: github.event_name == 'pull_request' && fromJson(steps.pr.outputs.draft)
|
||||
if: env.IS_PR_EVENT == 'true' && fromJson(steps.pr.outputs.draft)
|
||||
run: |
|
||||
echo "PR is draft. Blocking CI."
|
||||
exit 1
|
||||
|
||||
- name: Require run-ci label (optional)
|
||||
if: github.event_name == 'pull_request' && inputs.require-run-ci == true
|
||||
if: env.IS_PR_EVENT == 'true' && inputs.require-run-ci == true
|
||||
run: |
|
||||
labels='${{ steps.pr.outputs.labels }}'
|
||||
if [[ "${{ contains(fromJson(steps.pr.outputs.labels), 'run-ci') }}" == "false" ]]; then
|
||||
@@ -70,15 +80,28 @@ jobs:
|
||||
# then reruns) could never un-skip. A runtime step using gh-api-fetched
|
||||
# labels lets reruns pick up the new label set.
|
||||
- name: Require run-ci-extra label (optional)
|
||||
if: github.event_name == 'pull_request' && inputs.require-run-ci-extra == true
|
||||
if: env.IS_PR_EVENT == 'true' && inputs.require-run-ci-extra == true
|
||||
run: |
|
||||
if [[ "${{ contains(fromJson(steps.pr.outputs.labels), 'run-ci-extra') }}" == "false" ]]; then
|
||||
echo "Missing required label 'run-ci-extra'. Add the label (e.g. via /tag-and-rerun-ci extra) to opt this PR into the extra test workflow."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Generic opt-in label gate (same live-fetch semantics as run-ci-extra).
|
||||
# Lets a workflow require its own label on top of run-ci without adding
|
||||
# a dedicated input per label.
|
||||
- name: Require additional label (optional)
|
||||
if: env.IS_PR_EVENT == 'true' && inputs.require-label != ''
|
||||
env:
|
||||
REQUIRED_LABEL: ${{ inputs.require-label }}
|
||||
run: |
|
||||
if [[ "${{ contains(fromJson(steps.pr.outputs.labels), inputs.require-label) }}" == "false" ]]; then
|
||||
echo "Missing required label '${REQUIRED_LABEL}' (in addition to 'run-ci'). Add the label to opt this PR into this workflow."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Enforce rate limit for low-permission actors (optional)
|
||||
if: github.event_name == 'pull_request' && inputs.cool-down-minutes > 0
|
||||
if: env.IS_PR_EVENT == 'true' && inputs.cool-down-minutes > 0
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -87,6 +110,11 @@ jobs:
|
||||
const owner = context.repo.owner;
|
||||
const repo = context.repo.repo;
|
||||
const eventName = context.eventName;
|
||||
// pr-test.yml runs on pull_request, so pull_request_target callers must
|
||||
// query that event to share the same rate-limit history.
|
||||
const rateLimitEventName = eventName === "pull_request_target"
|
||||
? "pull_request"
|
||||
: eventName;
|
||||
const curRun = await github.rest.actions.getWorkflowRun({
|
||||
owner, repo, run_id: context.runId
|
||||
});
|
||||
@@ -161,14 +189,14 @@ jobs:
|
||||
|
||||
const cutoff = new Date(Date.now() - effectiveCooldownMinutes * 60 * 1000);
|
||||
core.info(
|
||||
`Checking for workflow runs since ${cutoff.toISOString()} (last ${effectiveCooldownMinutes} minutes) for event '${eventName}'.`
|
||||
`Checking for workflow runs since ${cutoff.toISOString()} (last ${effectiveCooldownMinutes} minutes) for event '${rateLimitEventName}'.`
|
||||
);
|
||||
|
||||
const { data } = await github.rest.actions.listWorkflowRuns({
|
||||
owner,
|
||||
repo,
|
||||
workflow_id: 'pr-test.yml',
|
||||
event: eventName,
|
||||
event: rateLimitEventName,
|
||||
per_page: 100,
|
||||
});
|
||||
|
||||
|
||||
@@ -106,29 +106,71 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
CANN_image_a3: ${{ steps.set-vars.outputs.CANN_image_a3 }}
|
||||
CANN_image_910b: ${{ steps.set-vars.outputs.CANN_image_910b }}
|
||||
steps:
|
||||
# When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
|
||||
- name: Set image config
|
||||
id: set-vars
|
||||
run: |
|
||||
echo "CANN_image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
|
||||
echo "CANN_image_910b=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-910b-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
|
||||
echo "CANN_image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.1.0-a3-ubuntu22.04-py3.12" >> $GITHUB_OUTPUT
|
||||
|
||||
base-a-test-1-npu-a2:
|
||||
needs: [check-changes, pr-gate, set-image-config]
|
||||
# Stage-A gate: CPU-only unit tests. The NPU attention-backend UTs stub out
|
||||
# torch_npu / sgl_kernel_npu before importing, so a hosted runner suffices.
|
||||
base-a-test-npu:
|
||||
needs: [check-changes, pr-gate]
|
||||
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
self_name: base-a-test-1-npu-a2
|
||||
runner_config: linux-aarch64-a2-1
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_910b }}
|
||||
npu_device_type: 910b
|
||||
run_timeout_minutes: '15'
|
||||
secrets: inherit
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Pin to the event SHA so this job tests the same commit as the rest of the run.
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
# Same gate the NPU stages run first: fail before the NPU jobs are scheduled.
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: 'python/pyproject_npu.toml'
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
# uv pip targets a venv by default; setup-python has no venv.
|
||||
UV_SYSTEM_PYTHON: "1"
|
||||
run: |
|
||||
# Resolve the same dependency set as the NPU stages, minus Ascend-only wheels.
|
||||
rm -f python/pyproject.toml
|
||||
mv python/pyproject_npu.toml python/pyproject.toml
|
||||
# Versions mirror npu_ci_install_dependency.sh's defaults; keep them in sync.
|
||||
# triton: sglang.srt.utils.common imports it unconditionally and the CPU
|
||||
# wheel does not pull it in (the NPU image uses triton-ascend instead).
|
||||
# unsafe-best-match: --index-url is the lowest-priority index, so the
|
||||
# default strategy would resolve PyPI's CUDA torch, not the +cpu build.
|
||||
uv pip install torch==2.10.0 torchvision==0.25.0 torchaudio==2.10.0 triton \
|
||||
--index-url https://download.pytorch.org/whl/cpu \
|
||||
--extra-index-url https://pypi.org/simple/ \
|
||||
--index-strategy unsafe-best-match
|
||||
uv pip install -e "python[dev_npu]" --index-strategy unsafe-best-match
|
||||
|
||||
- name: Run NPU unit tests
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
cd test
|
||||
python3 run_suite.py --hw npu --suite base-a-test-npu
|
||||
|
||||
base-b-test-1-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
@@ -142,7 +184,7 @@ jobs:
|
||||
secrets: inherit
|
||||
|
||||
base-b-test-2-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
@@ -156,7 +198,7 @@ jobs:
|
||||
secrets: inherit
|
||||
|
||||
base-b-test-4-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
@@ -171,7 +213,7 @@ jobs:
|
||||
secrets: inherit
|
||||
|
||||
base-b-test-8-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
@@ -185,7 +227,7 @@ jobs:
|
||||
secrets: inherit
|
||||
|
||||
base-b-test-16-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
@@ -199,7 +241,7 @@ jobs:
|
||||
secrets: inherit
|
||||
|
||||
multimodal-gen-test-1-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
|
||||
runs-on: linux-aarch64-a3-800t-2
|
||||
strategy:
|
||||
@@ -263,7 +305,7 @@ jobs:
|
||||
retention-days: 7
|
||||
|
||||
multimodal-gen-test-4-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
|
||||
runs-on: linux-aarch64-a3-800t-4
|
||||
strategy:
|
||||
@@ -483,7 +525,7 @@ jobs:
|
||||
name: Analyze failure report
|
||||
needs:
|
||||
[
|
||||
base-a-test-1-npu-a2,
|
||||
base-a-test-npu,
|
||||
base-b-test-1-npu-a3,
|
||||
base-b-test-2-npu-a3,
|
||||
base-b-test-4-npu-a3,
|
||||
@@ -646,7 +688,7 @@ jobs:
|
||||
[
|
||||
check-changes,
|
||||
|
||||
base-a-test-1-npu-a2,
|
||||
base-a-test-npu,
|
||||
base-b-test-1-npu-a3,
|
||||
base-b-test-2-npu-a3,
|
||||
base-b-test-4-npu-a3,
|
||||
|
||||
@@ -28,8 +28,8 @@ jobs:
|
||||
check-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
changes_exist: ${{ steps.filter.outputs.main_package == 'true' || steps.filter.outputs.multimodal_gen == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
||||
main_package: ${{ steps.filter.outputs.main_package == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
||||
changes_exist: ${{ steps.filter.outputs.main_package == 'true' || steps.filter.outputs.multimodal_gen == 'true' || steps.xpu-tests.outputs.changed == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
||||
main_package: ${{ steps.filter.outputs.main_package == 'true' || steps.xpu-tests.outputs.changed == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
||||
multimodal_gen: ${{ steps.filter.outputs.multimodal_gen == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -54,21 +54,38 @@ jobs:
|
||||
uses: dorny/paths-filter@v3
|
||||
if: steps.run-mode.outputs.run_all_tests != 'true'
|
||||
with:
|
||||
list-files: json
|
||||
filters: |
|
||||
# Baseline list below; any other changed test/registered/ file with a
|
||||
# per-commit register_xpu_ci is also picked up by the next step.
|
||||
registered_tests:
|
||||
- "test/registered/**/*.py"
|
||||
main_package:
|
||||
# Extend test/registered/ entries when adding a non-nightly register_xpu_ci.
|
||||
- "python/sglang/!(multimodal_gen|kernels|cli|test)/**/!(*.md)"
|
||||
- "python/sglang/test/*.py"
|
||||
- "python/sglang/test/!(ascend|observability|mock_model|manual|external_models|kernels)/**/!(*.md)"
|
||||
- "python/pyproject_xpu.toml"
|
||||
- "python/sglang/kernels/ops/speculative/@(reconstruct_tree|ngram_corpus).py"
|
||||
- "python/sglang/kernels/jit/csrc/ngram_corpus/**"
|
||||
- "test/registered/xpu/**/!(*.md)"
|
||||
- "test/registered/e2e/xpu/**/!(*.md)"
|
||||
- "test/registered/disaggregation/test_disaggregation_xpu.py"
|
||||
- "test/registered/attention/test_chunk_gated_delta_rule.py"
|
||||
- "test/registered/attention/test_deterministic.py"
|
||||
- "test/registered/kernels/ops/kvcache/test_hisparse.py"
|
||||
- "test/registered/kv_canary/test_self_unit_future_tensor.py"
|
||||
- "test/registered/kv_canary/test_self_unit_plan_input.py"
|
||||
- "test/registered/kv_canary/test_self_unit_req_to_expected_token_ids_manager.py"
|
||||
- "test/registered/kv_canary/test_self_unit_runner_swa_divergence.py"
|
||||
- "test/registered/layers/mamba/test_mamba_slot_fused.py"
|
||||
- "test/registered/lora/test_moe_lora_info.py"
|
||||
- "test/registered/lora/test_virtual_experts_kernels.py"
|
||||
- "test/registered/unit/layers/test_mamba_state_scatter_triton.py"
|
||||
- "test/registered/unit/managers/test_hisparse_unit.py"
|
||||
- "test/registered/unit/mem_cache/test_mamba_unittest.py"
|
||||
- "test/registered/unit/sampling/test_sampling_params.py"
|
||||
- "test/registered/unit/spec/test_adaptive_spec_params.py"
|
||||
- "test/registered/unit/spec/test_ngram_corpus.py"
|
||||
- "test/run_suite.py"
|
||||
- "test/pytest.ini"
|
||||
- ".github/workflows/pr-test-xpu.yml"
|
||||
@@ -94,6 +111,13 @@ jobs:
|
||||
- "docker/xpu.Dockerfile"
|
||||
- "scripts/ci/xpu/**"
|
||||
|
||||
- name: Detect changed per-commit XPU tests
|
||||
id: xpu-tests
|
||||
if: steps.filter.outputs.registered_tests == 'true'
|
||||
env:
|
||||
CHANGED_FILES: ${{ steps.filter.outputs.registered_tests_files }}
|
||||
run: echo "changed=$(python3 scripts/ci/xpu/detect_xpu_test_changes.py)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# ==================== PR Gate ==================== #
|
||||
pr-gate:
|
||||
needs: check-changes
|
||||
|
||||
@@ -161,7 +161,8 @@ jobs:
|
||||
needs.check-changes.outputs.main_package == 'true' ||
|
||||
needs.check-changes.outputs.sgl_kernel == 'true' ||
|
||||
needs.check-changes.outputs.jit_kernel == 'true' ||
|
||||
needs.check-changes.outputs.multimodal_gen == 'true'
|
||||
needs.check-changes.outputs.multimodal_gen == 'true' ||
|
||||
needs.check-changes.outputs.renderer == 'true'
|
||||
)
|
||||
uses: ./.github/workflows/pr-gate.yml
|
||||
secrets: inherit
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
name: Release Docker Images Nightly Miles ROCm7.2 (AMD)
|
||||
name: Release Docker Images Nightly Miles ROCm7.2.4 (AMD)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
variant:
|
||||
- rocm720-mi35x
|
||||
- rocm724-mi35x
|
||||
steps:
|
||||
- name: Checkout miles repository
|
||||
uses: actions/checkout@v4
|
||||
@@ -1,4 +1,5 @@
|
||||
name: Release Docker Images Nightly (NPU)
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
@@ -8,25 +9,41 @@ on:
|
||||
- 'docker/npu.Dockerfile'
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 14 * * *" # Execute at 10:00 p.m. Beijing Time every day
|
||||
|
||||
- cron: "0 14 * * *"
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-22.04-arm
|
||||
name: build ${{ matrix.platform }}-${{ matrix.device_type }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
cann_version: ["9.0.0"]
|
||||
device_type: ["910b", "a3"]
|
||||
include:
|
||||
- device_type: a3
|
||||
cann_version: "9.1.0"
|
||||
platform: amd64
|
||||
runner: ubuntu-latest
|
||||
- device_type: a3
|
||||
cann_version: "9.1.0"
|
||||
platform: arm64
|
||||
runner: ubuntu-22.04-arm
|
||||
- device_type: 950
|
||||
cann_version: "9.1.0"
|
||||
platform: amd64
|
||||
runner: ubuntu-latest
|
||||
- device_type: 950
|
||||
cann_version: "9.1.0"
|
||||
platform: arm64
|
||||
runner: ubuntu-22.04-arm
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Free up disk space
|
||||
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
|
||||
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
|
||||
with:
|
||||
tool-cache: true
|
||||
docker-images: false
|
||||
@@ -38,48 +55,64 @@ jobs:
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
lmsysorg/sglang
|
||||
# push with schedule event
|
||||
# push with workflow_dispatch event
|
||||
images: lmsysorg/sglang
|
||||
tags: |
|
||||
type=ref,event=pr
|
||||
type=ref,event=branch
|
||||
type=schedule,pattern=main
|
||||
flavor: |
|
||||
latest=false
|
||||
suffix=-cann${{ matrix.cann_version }}-${{ matrix.device_type }},onlatest=true
|
||||
# Login against a Docker registry except on PR
|
||||
# https://github.com/docker/login-action
|
||||
- name: Log into docker hub
|
||||
suffix=-cann${{ matrix.cann_version }}-${{ matrix.device_type }}-${{ matrix.platform }}
|
||||
|
||||
- name: Log into Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
if: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }}
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
# Enable Docker multi-architecture build environment
|
||||
# Emulate non-native architectures
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
# Required for building and pushing multi-arch Docker images
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
# Build and push Docker image with Buildx (don't push on PR)
|
||||
# https://github.com/docker/build-push-action
|
||||
- name: Build and push Docker image
|
||||
id: build-and-push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: docker
|
||||
file: docker/npu.Dockerfile
|
||||
platforms: linux/arm64,linux/amd64
|
||||
platforms: linux/${{ matrix.platform }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
push: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }}
|
||||
outputs: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' && 'type=registry,oci-mediatypes=false' || '' }}
|
||||
provenance: false
|
||||
sbom: false
|
||||
build-args: |
|
||||
SGLANG_KERNEL_NPU_TAG=2026.9.0.post5
|
||||
CANN_VERSION=${{ matrix.cann_version }}
|
||||
DEVICE_TYPE=${{ matrix.device_type }}
|
||||
|
||||
merge:
|
||||
name: merge multi-arch manifest
|
||||
needs: build
|
||||
if: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
cann_version: ["9.1.0"]
|
||||
device_type: ["a3", "950"]
|
||||
steps:
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log into Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Compute base image
|
||||
run: |
|
||||
echo "IMAGE=lmsysorg/sglang:${GITHUB_REF_NAME}-cann${{ matrix.cann_version }}-${{ matrix.device_type }}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Create multi-arch manifest
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t "${IMAGE}" \
|
||||
"${IMAGE}-amd64" \
|
||||
"${IMAGE}-arm64"
|
||||
|
||||
@@ -14,8 +14,8 @@ jobs:
|
||||
runs-on: ubuntu-22.04-arm
|
||||
strategy:
|
||||
matrix:
|
||||
cann_version: ["9.0.0"]
|
||||
device_type: ["910b", "a3"]
|
||||
cann_version: ["9.1.0"]
|
||||
device_type: ["a3", "950"]
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
name: Publish Renderer Docker Image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ["v[0-9]+.*"]
|
||||
paths:
|
||||
- "rust/sglang-renderer/**"
|
||||
- "rust/Cargo.toml"
|
||||
- "rust/Cargo.lock"
|
||||
- "rust/rust-toolchain.toml"
|
||||
- "docker/renderer.Dockerfile"
|
||||
- "LICENSE"
|
||||
- ".github/workflows/release-docker-renderer.yml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
if: github.repository == 'sgl-project/sglang'
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
publish: ${{ steps.image.outputs.publish }}
|
||||
repo: ${{ steps.image.outputs.repo }}
|
||||
tag: ${{ steps.image.outputs.tag }}
|
||||
steps:
|
||||
- name: Select image destination
|
||||
id: image
|
||||
run: |
|
||||
if [[ "$GITHUB_REF_TYPE" == "tag" ]]; then
|
||||
if [[ ! "$GITHUB_REF_NAME" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(\.post[0-9]+)?$ ]]; then
|
||||
echo "Skipping non-final release tag $GITHUB_REF_NAME"
|
||||
echo "publish=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "repo=lmsysorg/sglang" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=${GITHUB_REF_NAME}-renderer" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "repo=lmsysorg/sglang-staging" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=renderer-${GITHUB_SHA:0:8}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
echo "publish=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
build:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.publish == 'true'
|
||||
environment: ${{ github.ref_type == 'tag' && 'prod' || null }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-24.04
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
runs-on: ${{ matrix.runner }}
|
||||
env:
|
||||
IMAGE_REPO: ${{ needs.prepare.outputs.repo }}
|
||||
IMAGE_TAG: ${{ needs.prepare.outputs.tag }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Run renderer tests
|
||||
run: |
|
||||
docker buildx build . -f docker/renderer.Dockerfile \
|
||||
--platform ${{ matrix.platform }} --target test --progress=plain
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Build and push by digest
|
||||
run: |
|
||||
BUILD_COMMIT=$(git rev-parse HEAD)
|
||||
BUILD_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
docker buildx build . -f docker/renderer.Dockerfile \
|
||||
--platform ${{ matrix.platform }} --target runtime --progress=plain \
|
||||
--build-arg "SGLANG_BUILD_COMMIT=${BUILD_COMMIT}" \
|
||||
--build-arg "SGLANG_BUILD_URL=${BUILD_URL}" \
|
||||
--build-arg "SGLANG_IMAGE_TAG=${IMAGE_REPO}:${IMAGE_TAG}" \
|
||||
--output "type=image,name=${IMAGE_REPO},push-by-digest=true,name-canonical=true,push=true" \
|
||||
--metadata-file /tmp/renderer-metadata.json
|
||||
DIGEST=$(python3 -c "import json; print(json.load(open('/tmp/renderer-metadata.json'))['containerimage.digest'])")
|
||||
docker run --rm "${IMAGE_REPO}@${DIGEST}" --help
|
||||
mkdir -p /tmp/renderer-digests
|
||||
echo "$DIGEST" > "/tmp/renderer-digests/${{ matrix.arch }}"
|
||||
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: renderer-digest-${{ matrix.arch }}
|
||||
path: /tmp/renderer-digests/${{ matrix.arch }}
|
||||
retention-days: 7
|
||||
|
||||
publish:
|
||||
needs: [prepare, build]
|
||||
if: needs.prepare.outputs.publish == 'true'
|
||||
environment: ${{ github.ref_type == 'tag' && 'prod' || null }}
|
||||
runs-on: ubuntu-24.04
|
||||
env:
|
||||
IMAGE_REPO: ${{ needs.prepare.outputs.repo }}
|
||||
IMAGE_TAG: ${{ needs.prepare.outputs.tag }}
|
||||
steps:
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: /tmp/renderer-digests
|
||||
pattern: renderer-digest-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Publish multi-architecture tag
|
||||
run: |
|
||||
AMD64_DIGEST=$(cat /tmp/renderer-digests/amd64)
|
||||
ARM64_DIGEST=$(cat /tmp/renderer-digests/arm64)
|
||||
docker buildx imagetools create \
|
||||
-t "${IMAGE_REPO}:${IMAGE_TAG}" \
|
||||
"${IMAGE_REPO}@${AMD64_DIGEST}" \
|
||||
"${IMAGE_REPO}@${ARM64_DIGEST}"
|
||||
docker buildx imagetools inspect "${IMAGE_REPO}:${IMAGE_TAG}"
|
||||
@@ -86,8 +86,43 @@ permissions:
|
||||
issues: read
|
||||
|
||||
jobs:
|
||||
prepare-partitions:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
matrix: ${{ steps.partition.outputs.matrix }}
|
||||
steps:
|
||||
- name: Checkout partition script
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Checkout test registrations
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.pr_head_sha || github.sha }}
|
||||
path: partition-source
|
||||
persist-credentials: false
|
||||
|
||||
- name: Partition tests by estimated duration
|
||||
id: partition
|
||||
env:
|
||||
TEST_COMMAND: ${{ inputs.test_command }}
|
||||
TEST_ROOT: partition-source
|
||||
MODE: ${{ inputs.mode }}
|
||||
RUNS_ON: ${{ inputs.runs_on }}
|
||||
run: python3 scripts/ci/partition_rerun_tests.py
|
||||
|
||||
rerun-test-cuda:
|
||||
name: rerun-test-cuda (partition ${{ matrix.partition }})
|
||||
needs: prepare-partitions
|
||||
if: inputs.mode == 'cuda'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJson(needs.prepare-partitions.outputs.matrix) }}
|
||||
runs-on: ${{ inputs.runs_on }}
|
||||
timeout-minutes: 120
|
||||
permissions:
|
||||
@@ -150,7 +185,8 @@ jobs:
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
TEST_COMMAND: ${{ inputs.test_command }}
|
||||
TEST_COMMAND: ${{ matrix.test_command }}
|
||||
REQUESTED_TEST_COMMAND: ${{ inputs.test_command }}
|
||||
SGLANG_PRECISION_HF_TOKEN: ${{ inputs.refresh_precision_baseline && inputs.mode == 'cuda' && inputs.runs_on == '8-gpu-h200' && inputs.test_command == 'registered/debug_utils/test_nightly_precision_regression.py' && inputs.pr_head_sha == '' && secrets.HF_TOKEN_PRECISION_STORE || '' }}
|
||||
run: |
|
||||
if [[ "${{ inputs.refresh_precision_baseline }}" == "true" ]]; then
|
||||
@@ -158,7 +194,7 @@ jobs:
|
||||
if [[ "${{ inputs.mode }}" != "cuda" \
|
||||
|| "${{ inputs.runs_on }}" != "8-gpu-h200" \
|
||||
|| -n "${{ inputs.pr_head_sha }}" \
|
||||
|| "$TEST_COMMAND" != "$expected" ]]; then
|
||||
|| "$REQUESTED_TEST_COMMAND" != "$expected" ]]; then
|
||||
echo "::error::Precision baseline refresh only accepts $expected on 8-gpu-h200"
|
||||
exit 1
|
||||
fi
|
||||
@@ -172,6 +208,7 @@ jobs:
|
||||
cmds+=("$cmd")
|
||||
done <<< "$TEST_COMMAND"
|
||||
total=${#cmds[@]}
|
||||
failed=()
|
||||
suite_start=$SECONDS
|
||||
for idx in "${!cmds[@]}"; do
|
||||
i=$((idx + 1))
|
||||
@@ -189,7 +226,9 @@ jobs:
|
||||
if grep -qE 'register_cuda_ci\(.*stage[[:space:]]*=[[:space:]]*"base-a"' "test/$test_file" 2>/dev/null; then
|
||||
async_assert=false
|
||||
fi
|
||||
(cd test/ && SGLANG_ENABLE_ASYNC_ASSERT=$async_assert python3 $cmd -f) || exit 1
|
||||
if ! (cd test/ && SGLANG_ENABLE_ASYNC_ASSERT=$async_assert python3 $cmd -f); then
|
||||
failed+=("$cmd")
|
||||
fi
|
||||
elapsed=$(( SECONDS - file_start ))
|
||||
echo "."
|
||||
echo "End ($i/$total): elapsed=${elapsed}s"
|
||||
@@ -197,13 +236,29 @@ jobs:
|
||||
echo ""
|
||||
done
|
||||
total_elapsed=$(( SECONDS - suite_start ))
|
||||
if [ ${#failed[@]} -gt 0 ]; then
|
||||
echo "${#failed[@]}/$total test(s) failed in ${total_elapsed}s:"
|
||||
printf ' %s\n' "${failed[@]}"
|
||||
{
|
||||
echo "### Failed tests (partition ${{ matrix.partition }})"
|
||||
printf -- '- `%s`\n' "${failed[@]}"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
exit 1
|
||||
fi
|
||||
echo "All $total test(s) passed in ${total_elapsed}s"
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: failure()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.partition }}
|
||||
|
||||
rerun-test-multimodal-gen:
|
||||
name: rerun-test-multimodal-gen (partition ${{ matrix.partition }})
|
||||
needs: prepare-partitions
|
||||
if: inputs.mode == 'multimodal_gen'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJson(needs.prepare-partitions.outputs.matrix) }}
|
||||
runs-on: ${{ inputs.runs_on }}
|
||||
timeout-minutes: 120
|
||||
permissions:
|
||||
@@ -244,13 +299,16 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
TEST_COMMAND: ${{ matrix.test_command }}
|
||||
run: |
|
||||
cmds=()
|
||||
while IFS= read -r cmd; do
|
||||
[ -z "$cmd" ] && continue
|
||||
cmds+=("$cmd")
|
||||
done <<< "${{ inputs.test_command }}"
|
||||
done <<< "$TEST_COMMAND"
|
||||
total=${#cmds[@]}
|
||||
failed=()
|
||||
suite_start=$SECONDS
|
||||
for idx in "${!cmds[@]}"; do
|
||||
i=$((idx + 1))
|
||||
@@ -260,7 +318,9 @@ jobs:
|
||||
echo "Begin ($i/$total): python3 -m pytest $cmd -x"
|
||||
echo "."
|
||||
file_start=$SECONDS
|
||||
python3 -m pytest $cmd -x || exit 1
|
||||
if ! python3 -m pytest $cmd -x; then
|
||||
failed+=("$cmd")
|
||||
fi
|
||||
elapsed=$(( SECONDS - file_start ))
|
||||
echo "."
|
||||
echo "End ($i/$total): elapsed=${elapsed}s"
|
||||
@@ -268,13 +328,29 @@ jobs:
|
||||
echo ""
|
||||
done
|
||||
total_elapsed=$(( SECONDS - suite_start ))
|
||||
if [ ${#failed[@]} -gt 0 ]; then
|
||||
echo "${#failed[@]}/$total test(s) failed in ${total_elapsed}s:"
|
||||
printf ' %s\n' "${failed[@]}"
|
||||
{
|
||||
echo "### Failed tests (partition ${{ matrix.partition }})"
|
||||
printf -- '- `%s`\n' "${failed[@]}"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
exit 1
|
||||
fi
|
||||
echo "All $total test(s) passed in ${total_elapsed}s"
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: failure()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.partition }}
|
||||
|
||||
rerun-test-cpu:
|
||||
name: rerun-test-cpu (partition ${{ matrix.partition }})
|
||||
needs: prepare-partitions
|
||||
if: inputs.mode == 'cpu'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJson(needs.prepare-partitions.outputs.matrix) }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 120
|
||||
permissions:
|
||||
@@ -343,14 +419,17 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
TEST_COMMAND: ${{ matrix.test_command }}
|
||||
run: |
|
||||
cd test/
|
||||
cmds=()
|
||||
while IFS= read -r cmd; do
|
||||
[ -z "$cmd" ] && continue
|
||||
cmds+=("$cmd")
|
||||
done <<< "${{ inputs.test_command }}"
|
||||
done <<< "$TEST_COMMAND"
|
||||
total=${#cmds[@]}
|
||||
failed=()
|
||||
suite_start=$SECONDS
|
||||
for idx in "${!cmds[@]}"; do
|
||||
i=$((idx + 1))
|
||||
@@ -360,7 +439,9 @@ jobs:
|
||||
echo "Begin ($i/$total): python3 $cmd"
|
||||
echo "."
|
||||
file_start=$SECONDS
|
||||
python3 $cmd -f || exit 1
|
||||
if ! python3 $cmd -f; then
|
||||
failed+=("$cmd")
|
||||
fi
|
||||
elapsed=$(( SECONDS - file_start ))
|
||||
echo "."
|
||||
echo "End ($i/$total): elapsed=${elapsed}s"
|
||||
@@ -368,10 +449,19 @@ jobs:
|
||||
echo ""
|
||||
done
|
||||
total_elapsed=$(( SECONDS - suite_start ))
|
||||
if [ ${#failed[@]} -gt 0 ]; then
|
||||
echo "${#failed[@]}/$total test(s) failed in ${total_elapsed}s:"
|
||||
printf ' %s\n' "${failed[@]}"
|
||||
{
|
||||
echo "### Failed tests (partition ${{ matrix.partition }})"
|
||||
printf -- '- `%s`\n' "${failed[@]}"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
exit 1
|
||||
fi
|
||||
echo "All $total test(s) passed in ${total_elapsed}s"
|
||||
|
||||
write-back-result:
|
||||
needs: [rerun-test-cuda, rerun-test-multimodal-gen, rerun-test-cpu]
|
||||
needs: [prepare-partitions, rerun-test-cuda, rerun-test-multimodal-gen, rerun-test-cpu]
|
||||
if: always() && inputs.reply_comment_id != '' && inputs.reply_marker != ''
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
|
||||
@@ -166,8 +166,6 @@ cython_debug/
|
||||
*.swp
|
||||
|
||||
# SGL
|
||||
benchmark/mmlu/data
|
||||
benchmark/mmlu/data.tar
|
||||
benchmark/llava_bench/images
|
||||
benchmark/llava_bench/mme_pack
|
||||
*.jsonl
|
||||
|
||||
@@ -35,7 +35,7 @@ repos:
|
||||
hooks:
|
||||
- id: ruff
|
||||
args:
|
||||
- --select=F401,F821,UP037
|
||||
- --select=F401,F821,F823,UP037
|
||||
- --fix
|
||||
files: ^(benchmark/|docs/|examples/|python/sglang/|sgl-model-gateway/py_*|test/)
|
||||
exclude: |
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
# Gitea Actions 用(海外节点):原版 docker.io 基底 + 原版 pypi
|
||||
# 与 b300 离线版(/data/ymk/build/Dockerfile)的区别:
|
||||
# - 基底直用 docker.io 的 lmsysorg/sglang:dev-dsv41(不走 umirror/DaoCloud)
|
||||
# - 不设 PIP_INDEX_URL,用默认 pypi.org
|
||||
# - 源码由 CI checkout 后经 COPY 进镜像(不用 in-image clone)
|
||||
FROM lmsysorg/sglang:dev-dsv41
|
||||
|
||||
# checkout 含 .git,setuptools-scm 可打戳(main 系分支 version 显示 dev 属正常)
|
||||
COPY . /sgl-workspace/sglang
|
||||
|
||||
RUN pip install --no-cache-dir -e /sgl-workspace/sglang/python
|
||||
@@ -75,6 +75,7 @@ Its core features include:
|
||||
## Getting Started
|
||||
- [Install SGLang](https://docs.sglang.io/get_started/install.html)
|
||||
- [Quick Start](https://docs.sglang.io/basic_usage/send_request.html)
|
||||
- [Cookbook](https://docs.sglang.io/cookbook) — the deployment command we recommend for each supported model
|
||||
- [Backend Tutorial](https://docs.sglang.io/basic_usage/openai_api_completions.html)
|
||||
- [Frontend Tutorial](https://docs.sglang.io/references/frontend/frontend_tutorial.html)
|
||||
- [Contribution Guide](https://docs.sglang.io/developer_guide/contribution_guide.html)
|
||||
|
||||
@@ -271,7 +271,9 @@ Then we can benchmark the accuracy and latency by accessing the first node's exp
|
||||
|
||||
```bash
|
||||
# bench accuracy
|
||||
python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319 --host 10.0.0.1 --port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://10.0.0.1:30000/v1 \
|
||||
--num-examples 1319 --num-threads 64
|
||||
|
||||
# bench latency
|
||||
python3 -m sglang.bench_one_batch_server --model None --base-url http://10.0.0.1:30000 --batch-size 1 --input-len 128 --output-len 128
|
||||
@@ -322,13 +324,15 @@ Then on the **master node**, supposing the ShareGPT data is located at `/path/to
|
||||
|
||||
```bash
|
||||
# bench accuracy
|
||||
python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 1319 --num-threads 64
|
||||
|
||||
# bench serving
|
||||
python3 -m sglang.bench_serving --dataset-path /path/to/ShareGPT_V3_unfiltered_cleaned_split.json --dataset-name random --random-input 128 --random-output 128 --num-prompts 1000 --request-rate 128 --random-range-ratio 1.0
|
||||
```
|
||||
|
||||
> **Note: using `--parallel 200` can accelerate accuracy benchmarking**.
|
||||
> **Note: using `--num-threads 200` can accelerate accuracy benchmarking**.
|
||||
|
||||
### Example: Serving with 32 L40S with int8 Quantization
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
# DiffusionGemma serving measurements
|
||||
|
||||
`bench_diffusion_gemma.py` exercises a running server through its completions API using pretokenized chat prompts. For completed cases it records trial timings, returned text, prompt hashes, and usage counts. It rejects empty responses and unexpected token counts.
|
||||
|
||||
Start the server with the checkpoint and supported serving defaults:
|
||||
|
||||
```sh
|
||||
python -m sglang.launch_server --model-path "$MODEL" \
|
||||
--dllm-algorithm Gemma4Renoise
|
||||
```
|
||||
|
||||
Point the client at that server and the same tokenizer:
|
||||
|
||||
```sh
|
||||
python benchmark/dllm/bench_diffusion_gemma.py \
|
||||
--url http://127.0.0.1:30000 --tokenizer "$MODEL" \
|
||||
--input-lengths 128 2048 --concurrencies 1 4 8 --output-length 256 \
|
||||
--run-id serving-check --result result.json
|
||||
```
|
||||
|
||||
## Interpretation
|
||||
|
||||
The client disables EOS stopping to request a fixed output length. Canvas positions after EOS count toward the reported token rate, which is not useful-text throughput. `--denoising-steps` records metadata and does not configure or verify the server's convergence settings. Its default is unset. Set it only when the actual server work has been verified. If a run fails, keep both server and client logs, since the result file contains only completed cases.
|
||||
|
||||
For comparisons, freeze the protocol before collecting results. Match checkpoint identity, physical hardware, precision, prompts, resource limits, generation settings, and stopping policy. Use the same run ID to generate identical prompts. Give each runtime equivalent access to its normal compilation and graph features, preserve every trial and failed request, and disclose dependency and implementation differences. Validate normal text, images, mixed requests, and multi-block generation separately.
|
||||
|
||||
Performance for the merged revision remains unmeasured. Earlier measurements and test counts do not validate this revision. A fixed-work experiment does not establish adaptive-generation performance, model accuracy, or a serving performance lead.
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Compare fixed-work DiffusionGemma serving through the completions API.
|
||||
|
||||
Both servers must use the same checkpoint, canvas length, denoising limit and
|
||||
stopping settings. Set confidence_threshold=0.0 and stability_threshold=1 on
|
||||
both servers so early convergence cannot shorten work. This keeps the normal
|
||||
history size instead of adding history work to enforce the step count. Run them
|
||||
sequentially on the same physical GPUs, using the same run ID and arguments.
|
||||
Each request has a unique prompt prefix to avoid prefix-cache reuse.
|
||||
|
||||
This measures complete canvas computation with EOS stopping disabled. Canvas
|
||||
tokens per second include tokens after EOS and are not useful-text throughput.
|
||||
Validate output quality separately with the model's default generation settings.
|
||||
The denoising-steps argument records the server configuration; it does not change it.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import statistics
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import aiohttp
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
|
||||
async def benchmark(args):
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer)
|
||||
body = tokenizer.encode(
|
||||
"The city library has books about science, history, music and art. " * 256,
|
||||
add_special_tokens=False,
|
||||
)
|
||||
template = tokenizer.apply_chat_template(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Request __REQUEST_ID__.\n__BENCHMARK_BODY__\n"
|
||||
"Write a detailed explanation of why public libraries are useful."
|
||||
),
|
||||
}
|
||||
],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False,
|
||||
)
|
||||
prefix_template, suffix_text = template.split("__BENCHMARK_BODY__")
|
||||
suffix = tokenizer.encode(suffix_text, add_special_tokens=False)
|
||||
results = []
|
||||
request_number = 0
|
||||
timeout = aiohttp.ClientTimeout(total=600)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.get(args.url + "/v1/models") as response:
|
||||
response.raise_for_status()
|
||||
model = (await response.json())["data"][0]["id"]
|
||||
|
||||
async def request(prompt):
|
||||
start = time.perf_counter()
|
||||
async with session.post(
|
||||
args.url + "/v1/completions",
|
||||
json={
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"max_tokens": args.output_length,
|
||||
"ignore_eos": True,
|
||||
"stream": False,
|
||||
},
|
||||
) as response:
|
||||
payload = await response.json()
|
||||
if response.status != 200:
|
||||
raise RuntimeError(f"HTTP {response.status}: {payload}")
|
||||
elapsed = time.perf_counter() - start
|
||||
if not payload["choices"][0]["text"].strip():
|
||||
raise RuntimeError(
|
||||
"Empty completion; check the chat template and model"
|
||||
)
|
||||
usage = payload["usage"]
|
||||
if usage["completion_tokens"] != args.output_length:
|
||||
raise RuntimeError(f"Unexpected output length: {usage}")
|
||||
if usage["prompt_tokens"] != len(prompt):
|
||||
raise RuntimeError(f"Unexpected prompt length: {usage}")
|
||||
return {
|
||||
"latency_s": elapsed,
|
||||
"usage": usage,
|
||||
"text": payload["choices"][0]["text"],
|
||||
"prompt_sha256": hashlib.sha256(
|
||||
json.dumps(prompt).encode()
|
||||
).hexdigest(),
|
||||
}
|
||||
|
||||
for input_length in args.input_lengths:
|
||||
for concurrency in args.concurrencies:
|
||||
trials = []
|
||||
for trial in range(args.warmups + args.trials):
|
||||
prompts = []
|
||||
for _ in range(concurrency):
|
||||
# Vary the beginning of every prompt to avoid measuring
|
||||
# cached prompts. Keep run_id identical across runtimes.
|
||||
prefix = tokenizer.encode(
|
||||
prefix_template.replace(
|
||||
"__REQUEST_ID__", f"{args.run_id}-{request_number}"
|
||||
),
|
||||
add_special_tokens=False,
|
||||
)
|
||||
request_number += 1
|
||||
n = input_length - len(prefix) - len(suffix)
|
||||
if n < 0:
|
||||
raise ValueError("Input length is too short")
|
||||
prompts.append(prefix + body[:n] + suffix)
|
||||
start = time.perf_counter()
|
||||
requests = await asyncio.gather(*(request(p) for p in prompts))
|
||||
trials.append(
|
||||
{
|
||||
"warmup": trial < args.warmups,
|
||||
"wall_s": time.perf_counter() - start,
|
||||
"requests": requests,
|
||||
}
|
||||
)
|
||||
measured = trials[args.warmups :]
|
||||
wall_times = [t["wall_s"] for t in measured]
|
||||
latencies = [r["latency_s"] for t in measured for r in t["requests"]]
|
||||
result = {
|
||||
"input_length": input_length,
|
||||
"output_length": args.output_length,
|
||||
"concurrency": concurrency,
|
||||
"median_wall_s": statistics.median(wall_times),
|
||||
"median_request_s": statistics.median(latencies),
|
||||
"canvas_tokens_per_s": concurrency
|
||||
* args.output_length
|
||||
/ statistics.median(wall_times),
|
||||
"trials": trials,
|
||||
}
|
||||
results.append(result)
|
||||
print(
|
||||
json.dumps({k: v for k, v in result.items() if k != "trials"}),
|
||||
flush=True,
|
||||
)
|
||||
args.result.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"settings": vars(args) | {"result": str(args.result)},
|
||||
"results": results,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--url", required=True)
|
||||
parser.add_argument("--tokenizer", required=True)
|
||||
parser.add_argument("--input-lengths", type=int, nargs="+", default=[128, 512])
|
||||
parser.add_argument("--output-length", type=int, default=256)
|
||||
parser.add_argument(
|
||||
"--denoising-steps",
|
||||
type=int,
|
||||
help="Verified fixed denoising step count, recorded as metadata only",
|
||||
)
|
||||
parser.add_argument("--concurrencies", type=int, nargs="+", default=[1, 4, 8])
|
||||
parser.add_argument("--warmups", type=int, default=3)
|
||||
parser.add_argument("--trials", type=int, default=15)
|
||||
parser.add_argument("--run-id", required=True)
|
||||
parser.add_argument("--result", type=Path, required=True)
|
||||
asyncio.run(benchmark(parser.parse_args()))
|
||||
@@ -1,20 +0,0 @@
|
||||
## Run benchmark
|
||||
|
||||
### Using GSM8K Platinum
|
||||
|
||||
GSM8K Platinum is a revised version of the GSM8K test set with corrected labels and removed ambiguous questions. It can be more stable than the original GSM8K dataset. It's a drop-in replacement that can be used by adding the `--platinum` flag:
|
||||
|
||||
```
|
||||
python3 bench_sglang.py --num-shots 8 --num-questions 1209 --parallel 1209 --platinum
|
||||
```
|
||||
|
||||
For more information, see: https://huggingface.co/datasets/madrylab/gsm8k-platinum
|
||||
|
||||
### Benchmark sglang
|
||||
```
|
||||
python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000
|
||||
```
|
||||
|
||||
```
|
||||
python3 bench_sglang.py --num-questions 200
|
||||
```
|
||||
@@ -1,207 +0,0 @@
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
from datasets import load_dataset
|
||||
|
||||
from sglang.lang.api import set_default_backend
|
||||
from sglang.test.test_utils import (
|
||||
add_common_sglang_args_and_parse,
|
||||
dump_bench_raw_result,
|
||||
select_sglang_backend,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, dump_state_text, read_jsonl
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def main(args):
|
||||
# Select backend
|
||||
set_default_backend(select_sglang_backend(args))
|
||||
|
||||
# Load tokenizer if enable_thinking is set
|
||||
tokenizer = None
|
||||
if args.enable_thinking:
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
assert args.tokenizer_path is not None, (
|
||||
"--tokenizer-path is required when --enable-thinking is set"
|
||||
)
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
args.tokenizer_path, trust_remote_code=True
|
||||
)
|
||||
|
||||
# Read data
|
||||
if args.platinum:
|
||||
print("Loading GSM8K Platinum dataset from HuggingFace...")
|
||||
dataset = load_dataset("madrylab/gsm8k-platinum", "main", split="test")
|
||||
lines = [
|
||||
{"question": item["question"], "answer": item["answer"]} for item in dataset
|
||||
]
|
||||
else:
|
||||
data_path = args.data_path
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
if not os.path.isfile(data_path):
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
# Construct prompts
|
||||
num_questions = args.num_questions
|
||||
num_shots = args.num_shots
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
raw_question = few_shot_examples + get_one_example(lines, i, False)
|
||||
if tokenizer is not None:
|
||||
messages = [{"role": "user", "content": raw_question}]
|
||||
raw_question = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
**args.chat_template_kwargs,
|
||||
)
|
||||
questions.append(raw_question)
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
assert all(l != INVALID for l in labels)
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
#####################################
|
||||
######### SGL Program Begin #########
|
||||
#####################################
|
||||
|
||||
import sglang as sgl
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += question
|
||||
s += sgl.gen(
|
||||
"answer",
|
||||
max_tokens=args.max_new_tokens,
|
||||
stop=["Question", "Assistant:", "<|separator|>"],
|
||||
)
|
||||
|
||||
#####################################
|
||||
########## SGL Program End ##########
|
||||
#####################################
|
||||
|
||||
# Run requests
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments,
|
||||
temperature=args.temperature,
|
||||
top_p=args.top_p,
|
||||
num_threads=args.parallel,
|
||||
progress_bar=True,
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = []
|
||||
for i in range(len(states)):
|
||||
preds.append(get_answer_value(states[i]["answer"]))
|
||||
|
||||
# Compute accuracy
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid = np.mean(np.array(preds) == INVALID)
|
||||
|
||||
# Compute speed
|
||||
num_output_tokens = sum(
|
||||
s.get_meta_info("answer")["completion_tokens"] for s in states
|
||||
)
|
||||
output_throughput = num_output_tokens / latency
|
||||
|
||||
# Print results
|
||||
print(f"Accuracy: {acc:.3f}")
|
||||
print(f"Invalid: {invalid:.3f}")
|
||||
print(f"Latency: {latency:.3f} s")
|
||||
print(f"Output throughput: {output_throughput:.3f} token/s")
|
||||
|
||||
# Dump results
|
||||
dump_state_text(f"tmp_output_{args.backend}.txt", states)
|
||||
dump_bench_raw_result(
|
||||
path=args.raw_result_file,
|
||||
states=states,
|
||||
preds=preds,
|
||||
labels=labels,
|
||||
)
|
||||
|
||||
with open(args.result_file, "a") as fout:
|
||||
value = {
|
||||
"task": "gsm8k-platinum" if args.platinum else "gsm8k",
|
||||
"backend": args.backend,
|
||||
"num_gpus": 1,
|
||||
"latency": round(latency, 3),
|
||||
"accuracy": round(acc, 3),
|
||||
"num_requests": args.num_questions,
|
||||
"other": {
|
||||
"num_questions": args.num_questions,
|
||||
"parallel": args.parallel,
|
||||
},
|
||||
}
|
||||
fout.write(json.dumps(value) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--num-shots", type=int, default=5)
|
||||
parser.add_argument("--data-path", type=str, default="test.jsonl")
|
||||
parser.add_argument("--num-questions", type=int, default=200)
|
||||
parser.add_argument("--max-new-tokens", type=int, default=512)
|
||||
parser.add_argument("--temperature", type=float, default=0.0)
|
||||
parser.add_argument("--top-p", type=float, default=1.0)
|
||||
parser.add_argument(
|
||||
"--enable-thinking",
|
||||
action="store_true",
|
||||
help="Enable thinking mode by wrapping prompts with chat template",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chat-template-kwargs",
|
||||
type=json.loads,
|
||||
default='{"enable_thinking": true}',
|
||||
help="JSON dict passed through to tokenizer.apply_chat_template. "
|
||||
"The thinking-toggle kwarg name is model-specific, e.g. "
|
||||
"'{\"enable_thinking\": true}' (Qwen) or '{\"thinking\": true}' (Kimi).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tokenizer-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to tokenizer (required when --enable-thinking is set)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--platinum",
|
||||
action="store_true",
|
||||
help="Use GSM8K Platinum dataset (drop-in replacement with corrected labels)",
|
||||
)
|
||||
args = add_common_sglang_args_and_parse(parser)
|
||||
main(args)
|
||||
@@ -0,0 +1,198 @@
|
||||
"""CPU comparison of baseline and optimized offset-based scheduler padding.
|
||||
|
||||
Run from the repository root:
|
||||
PYTHONPATH=python python benchmark/io/bench_mm_token_padding.py --baseline-ref BASE_SHA
|
||||
|
||||
Inputs are synthetic. Both paths include the scheduler's final ownership copy.
|
||||
This measures padding CPU time, not model forward time or end-to-end TTFT.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import gc
|
||||
import json
|
||||
import platform
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
from array import array
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers import mm_utils
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
duplicate_pad_mm_input_ids,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
MultimodalInputs,
|
||||
)
|
||||
|
||||
|
||||
def load_baseline(ref):
|
||||
"""Load the exact baseline helper classes without replacing installed SGLang."""
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
commit = subprocess.check_output(
|
||||
["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], cwd=root, text=True
|
||||
).strip()
|
||||
path = "python/sglang/srt/managers/mm_utils.py"
|
||||
source = subprocess.check_output(
|
||||
["git", "show", f"{commit}:{path}"], cwd=root, text=True
|
||||
)
|
||||
names = {
|
||||
"MultiModalityDataPaddingPattern",
|
||||
"MultiModalityDataPaddingPatternMultimodalTokens",
|
||||
}
|
||||
classes = [
|
||||
node
|
||||
for node in ast.parse(source).body
|
||||
if isinstance(node, ast.ClassDef) and node.name in names
|
||||
]
|
||||
if {node.name for node in classes} != names:
|
||||
raise ValueError("Baseline does not contain the expected padding helpers")
|
||||
namespace = dict(vars(mm_utils))
|
||||
# Reuse dependency imports, but compile the original method bodies verbatim.
|
||||
exec(
|
||||
compile(ast.Module(body=classes, type_ignores=[]), f"{commit}:{path}", "exec"),
|
||||
namespace,
|
||||
)
|
||||
return commit, namespace["MultiModalityDataPaddingPatternMultimodalTokens"]()
|
||||
|
||||
|
||||
def make_case(num_tokens, num_items, span_tokens):
|
||||
if num_items and num_tokens < 3 * num_items:
|
||||
raise ValueError(
|
||||
"Each item needs three tokens for its span and surrounding text"
|
||||
)
|
||||
input_ids = array("q", range(257, 257 + num_tokens))
|
||||
items = []
|
||||
stride = num_tokens // max(num_items, 1)
|
||||
for index in range(num_items):
|
||||
start = index * stride + 1
|
||||
end = start + min(span_tokens, stride - 2) - 1
|
||||
items.append(
|
||||
MultimodalDataItem(
|
||||
modality=Modality.IMAGE,
|
||||
pad_value=2**40 + index,
|
||||
offsets=[(start, end)],
|
||||
)
|
||||
)
|
||||
return input_ids, MultimodalInputs(mm_items=items, im_token_id=1)
|
||||
|
||||
|
||||
def benchmark_case(num_tokens, num_items, args, baseline):
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
input_ids, mm_inputs = make_case(num_tokens, num_items, args.span_tokens)
|
||||
original = input_ids.tobytes()
|
||||
|
||||
def baseline_padding():
|
||||
return array("q", baseline.pad_input_tokens(input_ids, mm_inputs))
|
||||
|
||||
def optimized_padding():
|
||||
return duplicate_pad_mm_input_ids(
|
||||
input_ids, mm_inputs, pattern.pad_input_tokens
|
||||
)
|
||||
|
||||
expected = baseline_padding()
|
||||
actual = optimized_padding()
|
||||
assert actual == expected, "Token values differ"
|
||||
assert input_ids.tobytes() == original, "Original input was mutated"
|
||||
assert actual is not input_ids, "Scheduler result aliases the original input"
|
||||
|
||||
functions = {"baseline": baseline_padding, "optimized": optimized_padding}
|
||||
timings = {name: [] for name in functions}
|
||||
for _ in range(args.warmups):
|
||||
for function in functions.values():
|
||||
function()
|
||||
gc_was_enabled = gc.isenabled()
|
||||
gc.disable()
|
||||
try:
|
||||
for trial in range(args.trials):
|
||||
order = (
|
||||
("baseline", "optimized") if trial % 2 else ("optimized", "baseline")
|
||||
)
|
||||
for name in order:
|
||||
start = time.perf_counter_ns()
|
||||
result = functions[name]()
|
||||
elapsed = time.perf_counter_ns() - start
|
||||
timings[name].append(elapsed / 1e6)
|
||||
del result
|
||||
finally:
|
||||
if gc_was_enabled:
|
||||
gc.enable()
|
||||
assert input_ids.tobytes() == original
|
||||
medians = {name: statistics.median(samples) for name, samples in timings.items()}
|
||||
return {
|
||||
"tokens": num_tokens,
|
||||
"items": num_items,
|
||||
"baseline_ms": medians["baseline"],
|
||||
"optimized_ms": medians["optimized"],
|
||||
"speedup": medians["baseline"] / medians["optimized"],
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--baseline-ref", required=True, help="Local pre-change git ref"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tokens", nargs="+", type=int, default=[128, 4096, 16384, 131072]
|
||||
)
|
||||
parser.add_argument("--items", nargs="+", type=int, default=[0, 1, 10])
|
||||
parser.add_argument("--span-tokens", type=int, default=32)
|
||||
parser.add_argument("--trials", type=int, default=41)
|
||||
parser.add_argument("--warmups", type=int, default=5)
|
||||
parser.add_argument("--threads", type=int, default=1)
|
||||
parser.add_argument("--json", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if (
|
||||
min(args.tokens + args.items) < 0
|
||||
or min(args.span_tokens, args.trials, args.threads) < 1
|
||||
or args.warmups < 0
|
||||
):
|
||||
parser.error("Lengths/items/warmups must be nonnegative; other counts positive")
|
||||
if any(n < 3 * items for n in args.tokens for items in args.items):
|
||||
parser.error("Each token length must accommodate at least 3 tokens per item")
|
||||
torch.set_num_threads(args.threads)
|
||||
baseline_commit, baseline = load_baseline(args.baseline_ref)
|
||||
rows = [
|
||||
benchmark_case(tokens, items, args, baseline)
|
||||
for tokens in args.tokens
|
||||
for items in args.items
|
||||
]
|
||||
if args.json:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"baseline_commit": baseline_commit,
|
||||
"python": platform.python_version(),
|
||||
"torch": torch.__version__,
|
||||
"architecture": platform.machine(),
|
||||
"threads": args.threads,
|
||||
"trials": args.trials,
|
||||
"warmups": args.warmups,
|
||||
"span_tokens": args.span_tokens,
|
||||
"results": rows,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return
|
||||
print("Offset-based CPU padding; median milliseconds; synthetic inputs")
|
||||
print(
|
||||
f"{'Tokens':>8} {'Items':>6} {'Baseline (ms)':>14} {'Optimized (ms)':>14} {'Speedup':>9}"
|
||||
)
|
||||
for row in rows:
|
||||
print(
|
||||
f"{row['tokens']:>8} {row['items']:>6} "
|
||||
f"{row['baseline_ms']:>14.4f} {row['optimized_ms']:>14.4f} "
|
||||
f"{row['speedup']:>8.2f}x"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,284 @@
|
||||
"""Benchmark multi-node MSCCL++ AllGather against NCCL."""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from sglang.srt.distributed import init_distributed_environment
|
||||
from sglang.srt.distributed.parallel_state import (
|
||||
cleanup_dist_env_and_memory,
|
||||
get_tensor_model_parallel_group,
|
||||
initialize_model_parallel,
|
||||
set_mscclpp,
|
||||
)
|
||||
from sglang.test.test_utils import publish_build_topology
|
||||
|
||||
|
||||
def _max_across_ranks(value: float, group) -> float:
|
||||
tensor = torch.tensor([value], dtype=torch.float64)
|
||||
dist.all_reduce(tensor, op=dist.ReduceOp.MAX, group=group)
|
||||
return tensor.item()
|
||||
|
||||
|
||||
def _time_eager(func, group, warmup: int = 2, repeat: int = 10) -> float:
|
||||
for _ in range(warmup):
|
||||
func()
|
||||
torch.cuda.synchronize()
|
||||
dist.barrier(group=group)
|
||||
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
start.record()
|
||||
for _ in range(repeat):
|
||||
func()
|
||||
end.record()
|
||||
end.synchronize()
|
||||
return _max_across_ranks(start.elapsed_time(end) * 1000 / repeat, group)
|
||||
|
||||
|
||||
def _time_cuda_graph(
|
||||
func,
|
||||
group,
|
||||
warmup: int = 100,
|
||||
graph_loop: int = 10,
|
||||
repeat: int = 100,
|
||||
) -> float:
|
||||
current_stream = torch.cuda.current_stream()
|
||||
capture_stream = torch.cuda.Stream()
|
||||
capture_stream.wait_stream(current_stream)
|
||||
with torch.cuda.stream(capture_stream):
|
||||
func(capture_stream)
|
||||
capture_stream.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=capture_stream):
|
||||
for _ in range(graph_loop):
|
||||
func(capture_stream)
|
||||
|
||||
with torch.cuda.stream(capture_stream):
|
||||
for _ in range(warmup):
|
||||
graph.replay()
|
||||
capture_stream.synchronize()
|
||||
|
||||
dist.barrier(group=group)
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
start.record(capture_stream)
|
||||
with torch.cuda.stream(capture_stream):
|
||||
for _ in range(repeat):
|
||||
graph.replay()
|
||||
end.record(capture_stream)
|
||||
end.synchronize()
|
||||
torch.cuda.current_stream().wait_stream(capture_stream)
|
||||
return _max_across_ranks(
|
||||
start.elapsed_time(end) * 1000 / (repeat * graph_loop),
|
||||
group,
|
||||
)
|
||||
|
||||
|
||||
def _validate_cuda_graph_updates(
|
||||
func,
|
||||
reference_func,
|
||||
input_tensor: torch.Tensor,
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
group,
|
||||
) -> None:
|
||||
current_stream = torch.cuda.current_stream()
|
||||
capture_stream = torch.cuda.Stream()
|
||||
capture_stream.wait_stream(current_stream)
|
||||
with torch.cuda.stream(capture_stream):
|
||||
func(capture_stream)
|
||||
capture_stream.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=capture_stream):
|
||||
func(capture_stream)
|
||||
|
||||
input_tensor.neg_()
|
||||
reference_func()
|
||||
current_stream.synchronize()
|
||||
capture_stream.wait_stream(current_stream)
|
||||
with torch.cuda.stream(capture_stream):
|
||||
graph.replay()
|
||||
capture_stream.synchronize()
|
||||
|
||||
if not torch.equal(actual, expected):
|
||||
raise AssertionError("MSCCL++ CUDA graph replay used stale AllGather data")
|
||||
dist.barrier(group=group)
|
||||
|
||||
|
||||
def _message_sizes(min_bytes: int, max_bytes: int):
|
||||
size = 1 << (min_bytes - 1).bit_length()
|
||||
while size <= max_bytes:
|
||||
yield size
|
||||
size <<= 1
|
||||
|
||||
|
||||
def human_readable_size(size, decimal_places=1):
|
||||
for unit in ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]:
|
||||
if size < 1024.0 or unit == "PiB":
|
||||
break
|
||||
size /= 1024.0
|
||||
return f"{size:.{decimal_places}f} {unit}"
|
||||
|
||||
|
||||
try:
|
||||
from tabulate import tabulate
|
||||
except ImportError:
|
||||
print("tabulate not installed, skipping table printing")
|
||||
tabulate = None
|
||||
|
||||
|
||||
def print_markdown_table(data):
|
||||
if tabulate is not None:
|
||||
print(tabulate(data, headers="keys", tablefmt="github"))
|
||||
return
|
||||
headers = data[0].keys()
|
||||
header_row = "| " + " | ".join(headers) + " |"
|
||||
separator = "| " + " | ".join(["---"] * len(headers)) + " |"
|
||||
rows = []
|
||||
for item in data:
|
||||
row = "| " + " | ".join(str(item[key]) for key in headers) + " |"
|
||||
rows.append(row)
|
||||
markdown_table = "\n".join([header_row, separator] + rows)
|
||||
print(markdown_table)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--min-bytes", type=int, default=1 << 10)
|
||||
parser.add_argument("--max-bytes", type=int, default=8 << 20)
|
||||
args = parser.parse_args()
|
||||
if not (1 << 10) <= args.min_bytes <= args.max_bytes <= (8 << 20):
|
||||
parser.error("Require 1 KiB <= min-bytes <= max-bytes <= 8 MiB")
|
||||
|
||||
local_rank = int(os.environ.get("LOCAL_RANK", 0))
|
||||
torch.cuda.set_device(local_rank)
|
||||
if not dist.is_initialized():
|
||||
dist.init_process_group(
|
||||
backend="nccl",
|
||||
device_id=torch.device("cuda", local_rank),
|
||||
)
|
||||
world_size = dist.get_world_size()
|
||||
rank = dist.get_rank()
|
||||
device = torch.cuda.current_device()
|
||||
set_mscclpp(True)
|
||||
init_distributed_environment(
|
||||
world_size=world_size,
|
||||
rank=rank,
|
||||
local_rank=local_rank,
|
||||
)
|
||||
publish_build_topology(world_rank=rank, tp_size=world_size)
|
||||
initialize_model_parallel()
|
||||
tp_group = get_tensor_model_parallel_group()
|
||||
gpu_group = tp_group.device_group
|
||||
cpu_group = tp_group.cpu_group
|
||||
pynccl_comm = tp_group.pynccl_comm
|
||||
mscclpp_comm = tp_group.pymscclpp_comm
|
||||
assert pynccl_comm is not None
|
||||
assert mscclpp_comm is not None
|
||||
if args.min_bytes % world_size != 0 or args.max_bytes % world_size != 0:
|
||||
parser.error("Total message sizes must be divisible by WORLD_SIZE")
|
||||
|
||||
if not mscclpp_comm._best_configs["allgather"]:
|
||||
raise RuntimeError("No tuned MSCCL++ AllGather configuration is available")
|
||||
|
||||
result = []
|
||||
|
||||
for total_bytes in _message_sizes(args.min_bytes, args.max_bytes):
|
||||
input_bytes = total_bytes // world_size
|
||||
if input_bytes % torch.bfloat16.itemsize != 0:
|
||||
continue
|
||||
elements = input_bytes // torch.bfloat16.itemsize
|
||||
input_tensor = (
|
||||
torch.arange(
|
||||
elements,
|
||||
dtype=torch.long,
|
||||
device=device,
|
||||
)
|
||||
% 193
|
||||
+ rank
|
||||
).to(torch.bfloat16)
|
||||
nccl_output = torch.empty(
|
||||
elements * world_size,
|
||||
dtype=input_tensor.dtype,
|
||||
device=device,
|
||||
)
|
||||
mscclpp_output = torch.empty_like(nccl_output)
|
||||
|
||||
def torch_nccl():
|
||||
dist.all_gather_into_tensor(
|
||||
nccl_output,
|
||||
input_tensor,
|
||||
group=gpu_group,
|
||||
)
|
||||
|
||||
def py_nccl():
|
||||
pynccl_comm.all_gather(nccl_output, input_tensor)
|
||||
|
||||
def mscclpp():
|
||||
mscclpp_comm.all_gather(mscclpp_output, input_tensor)
|
||||
|
||||
def mscclpp_on_stream(stream):
|
||||
mscclpp_comm.all_gather(
|
||||
mscclpp_output,
|
||||
input_tensor,
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
def py_nccl_on_stream(_stream):
|
||||
with pynccl_comm.change_state(enable=True):
|
||||
pynccl_comm.all_gather(nccl_output, input_tensor)
|
||||
|
||||
torch_nccl()
|
||||
mscclpp()
|
||||
py_nccl()
|
||||
torch.cuda.synchronize()
|
||||
if not torch.equal(mscclpp_output, nccl_output):
|
||||
raise AssertionError(
|
||||
f"MSCCL++ AllGather mismatch at {input_bytes} input bytes"
|
||||
)
|
||||
_validate_cuda_graph_updates(
|
||||
mscclpp_on_stream,
|
||||
torch_nccl,
|
||||
input_tensor,
|
||||
mscclpp_output,
|
||||
nccl_output,
|
||||
cpu_group,
|
||||
)
|
||||
|
||||
nccl_eager_us = _time_eager(torch_nccl, cpu_group)
|
||||
mscclpp_eager_us = _time_eager(mscclpp, cpu_group)
|
||||
mscclpp_graph_us = _time_cuda_graph(mscclpp_on_stream, cpu_group)
|
||||
pynccl_graph_us = _time_cuda_graph(py_nccl_on_stream, cpu_group)
|
||||
|
||||
result.append(
|
||||
{
|
||||
"msg_size": human_readable_size(total_bytes),
|
||||
"torch eager time": nccl_eager_us,
|
||||
"msccl eager time": mscclpp_eager_us,
|
||||
"msccl graph time": mscclpp_graph_us,
|
||||
"pynccl graph time": pynccl_graph_us,
|
||||
}
|
||||
)
|
||||
if rank == 0:
|
||||
print(
|
||||
f"sz={total_bytes // input_tensor.element_size()}, "
|
||||
f"dtype={input_tensor.dtype}: correctness check PASS!"
|
||||
)
|
||||
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
if rank == 0:
|
||||
print_markdown_table(result)
|
||||
|
||||
mscclpp_comm.destroy()
|
||||
dist.barrier()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -12,9 +12,9 @@ torchrun --nproc_per_node gpu \
|
||||
--master_port $MASTER_PORT benchmark/kernels/all_reduce/benchmark_mscclpp.py
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from contextlib import nullcontext
|
||||
from typing import List
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
@@ -23,13 +23,16 @@ from torch.distributed import ProcessGroup
|
||||
from sglang.srt.distributed import init_distributed_environment
|
||||
from sglang.srt.distributed.device_communicators.pymscclpp import PyMscclppCommunicator
|
||||
from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
from sglang.srt.distributed.parallel_state import (
|
||||
cleanup_dist_env_and_memory,
|
||||
get_tensor_model_parallel_group,
|
||||
graph_capture,
|
||||
initialize_model_parallel,
|
||||
set_mscclpp_all_reduce,
|
||||
set_mscclpp,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.test_utils import publish_build_topology
|
||||
|
||||
|
||||
@@ -51,14 +54,21 @@ def pynccl_allreduce(
|
||||
return msccl_input
|
||||
|
||||
|
||||
def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=10):
|
||||
graph_input = inp_randn.clone()
|
||||
graph_input_snapshot = inp_randn.clone()
|
||||
def _bench_graph_time(
|
||||
func,
|
||||
inp_randn,
|
||||
tp_group,
|
||||
warmup_loop=2,
|
||||
graph_loop=10,
|
||||
test_loop=10,
|
||||
):
|
||||
with use_symmetric_memory(tp_group):
|
||||
graph_input = inp_randn.clone()
|
||||
with graph_capture() as graph_capture_context:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
|
||||
for _ in range(graph_loop):
|
||||
graph_input.copy_(graph_input_snapshot)
|
||||
graph_input.copy_(inp_randn)
|
||||
graph_out = func(graph_input)
|
||||
|
||||
graph.replay()
|
||||
@@ -71,7 +81,7 @@ def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=1
|
||||
start_event = torch.cuda.Event(enable_timing=True)
|
||||
end_event = torch.cuda.Event(enable_timing=True)
|
||||
|
||||
latencies: List[float] = []
|
||||
latencies = []
|
||||
for _ in range(test_loop):
|
||||
torch.cuda.synchronize()
|
||||
dist.barrier()
|
||||
@@ -85,8 +95,9 @@ def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=1
|
||||
return func_output, func_cost_us
|
||||
|
||||
|
||||
def _bench_eager_time(func, inp_randn, warmup_loop=2, test_loop=10):
|
||||
eager_input = inp_randn.clone()
|
||||
def _bench_eager_time(func, inp_randn, tp_group, warmup_loop=2, test_loop=10):
|
||||
with use_symmetric_memory(tp_group):
|
||||
eager_input = inp_randn.clone()
|
||||
eager_output = func(eager_input)
|
||||
func_output = eager_output.clone()
|
||||
|
||||
@@ -156,57 +167,77 @@ def print_markdown_table(data):
|
||||
if __name__ == "__main__":
|
||||
import logging
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--enable-symm-mem",
|
||||
action="store_true",
|
||||
help="Allocate collective inputs from NCCL symmetric memory.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
force=True,
|
||||
)
|
||||
local_rank = int(os.environ["LOCAL_RANK"])
|
||||
torch.cuda.set_device(local_rank)
|
||||
device = torch.cuda.current_device()
|
||||
if not dist.is_initialized():
|
||||
dist.init_process_group(backend="nccl")
|
||||
dist.init_process_group(
|
||||
backend="nccl", device_id=torch.device("cuda", local_rank)
|
||||
)
|
||||
world, world_size = dist.group.WORLD, dist.get_world_size()
|
||||
rank = dist.get_rank()
|
||||
torch.cuda.set_device(rank % 8)
|
||||
device = torch.cuda.current_device()
|
||||
set_mscclpp_all_reduce(True)
|
||||
set_mscclpp(True)
|
||||
init_distributed_environment(
|
||||
world_size=world_size,
|
||||
rank=rank,
|
||||
local_rank=rank % 8,
|
||||
local_rank=local_rank,
|
||||
)
|
||||
publish_build_topology(world_rank=rank, tp_size=world_size)
|
||||
initialize_model_parallel()
|
||||
group = get_tensor_model_parallel_group().device_group
|
||||
cpu_group = get_tensor_model_parallel_group().cpu_group
|
||||
pynccl_comm = get_tensor_model_parallel_group().pynccl_comm
|
||||
pymscclpp_comm = get_tensor_model_parallel_group().pymscclpp_comm
|
||||
publish_build_topology(
|
||||
world_rank=rank,
|
||||
tp_size=world_size,
|
||||
enable_symm_mem=args.enable_symm_mem,
|
||||
)
|
||||
initialize_model_parallel(enable_symm_mem=args.enable_symm_mem)
|
||||
tp_group = get_parallel().tp_group
|
||||
group = tp_group.device_group
|
||||
pynccl_comm = tp_group.pynccl_comm
|
||||
pymscclpp_comm = tp_group.pymscclpp_comm
|
||||
dist.barrier()
|
||||
profile = False
|
||||
dtype = torch.bfloat16
|
||||
ctx = get_torch_prof_ctx(profile)
|
||||
result = []
|
||||
max_message_bytes = 1 << 20
|
||||
|
||||
with ctx:
|
||||
for i in range(10, 20):
|
||||
sz = 2**i
|
||||
if sz * dtype.itemsize > 2**20:
|
||||
if sz * dtype.itemsize > max_message_bytes:
|
||||
break
|
||||
inp_randn = torch.randint(1, 16, (sz,), dtype=dtype, device=device)
|
||||
inp_randn = torch.randint(0, 2, (sz,), dtype=dtype, device=device)
|
||||
|
||||
memory = torch.empty_like(inp_randn)
|
||||
memory_out = torch.empty_like(memory)
|
||||
torch_eager_output, torch_eager_time = _bench_eager_time(
|
||||
lambda inp: torch_allreduce(inp, group), inp_randn
|
||||
lambda inp: torch_allreduce(inp, group), inp_randn, tp_group
|
||||
)
|
||||
msccl_eager_output, msccl_eager_time = _bench_eager_time(
|
||||
lambda inp: msccl_allreduce(inp, pymscclpp_comm), inp_randn
|
||||
lambda inp: msccl_allreduce(inp, pymscclpp_comm),
|
||||
inp_randn,
|
||||
tp_group,
|
||||
)
|
||||
msccl_graph_output, msccl_graph_time = _bench_graph_time(
|
||||
lambda inp: msccl_allreduce(inp, pymscclpp_comm), inp_randn
|
||||
lambda inp: msccl_allreduce(inp, pymscclpp_comm),
|
||||
inp_randn,
|
||||
tp_group,
|
||||
)
|
||||
# since pynccl is inplace op, this return result is not correct if graph loop > 1
|
||||
_, pynccl_graph_time = _bench_graph_time(
|
||||
lambda inp: pynccl_allreduce(inp, pynccl_comm), inp_randn
|
||||
lambda inp: pynccl_allreduce(inp, pynccl_comm),
|
||||
inp_randn,
|
||||
tp_group,
|
||||
)
|
||||
torch.testing.assert_close(torch_eager_output, msccl_graph_output)
|
||||
torch.testing.assert_close(torch_eager_output, msccl_eager_output)
|
||||
|
||||
@@ -39,11 +39,11 @@ from sglang.srt.distributed.device_communicators.torch_symm_mem import (
|
||||
TorchSymmMemCommunicator,
|
||||
)
|
||||
from sglang.srt.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_group,
|
||||
graph_capture,
|
||||
initialize_model_parallel,
|
||||
set_torch_symm_mem_all_reduce,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.test_utils import publish_build_topology
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
@@ -191,10 +191,10 @@ if __name__ == "__main__":
|
||||
)
|
||||
publish_build_topology(world_rank=rank, tp_size=world_size)
|
||||
initialize_model_parallel()
|
||||
group = get_tensor_model_parallel_group().device_group
|
||||
cpu_group = get_tensor_model_parallel_group().cpu_group
|
||||
pynccl_comm = get_tensor_model_parallel_group().pynccl_comm
|
||||
torch_symm_mem_comm = get_tensor_model_parallel_group().torch_symm_mem_comm
|
||||
group = get_parallel().tp_group.device_group
|
||||
cpu_group = get_parallel().tp_group.cpu_group
|
||||
pynccl_comm = get_parallel().tp_group.pynccl_comm
|
||||
torch_symm_mem_comm = get_parallel().tp_group.torch_symm_mem_comm
|
||||
dist.barrier()
|
||||
profile = False
|
||||
dtype = torch.bfloat16
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
# ViT attention with FA4 on SM103: row max from tcgen05.ld.red
|
||||
# (SGLANG_FA4_TMEM_LOAD_RED_MAX=1) vs FMNMX (=0). Shapes and call follow
|
||||
# VisionFlash4Attention (varlen, non-causal, no LSE). Each round CUDA-graph times both
|
||||
# back to back, in alternating order; the per-round ratio is the headline number.
|
||||
# Outputs must be bit-identical.
|
||||
#
|
||||
# Run on B300 / GB300 (elsewhere the flag is ignored):
|
||||
# python benchmark/kernels/attention/bench_flash_attention_tmem_red_max.py
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from typing import Iterable
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.flash_attn.cute.interface import (
|
||||
flash_attn_varlen_func,
|
||||
)
|
||||
|
||||
RED_MAX_ENV = "SGLANG_FA4_TMEM_LOAD_RED_MAX"
|
||||
|
||||
# images, tokens per image, heads, head dim
|
||||
SHAPES = [
|
||||
(8, 4096, 16, 80), # Qwen2.5-VL
|
||||
(8, 4096, 16, 72), # Qwen3-VL
|
||||
(8, 4096, 16, 64), # InternViT, GLM / DeepSeek-V4 ViT
|
||||
(64, 1024, 16, 80),
|
||||
(64, 1024, 16, 64),
|
||||
(1, 32768, 16, 80), # one large image or a video
|
||||
(1, 32768, 16, 72),
|
||||
(1, 32768, 16, 64),
|
||||
]
|
||||
|
||||
|
||||
def _bits(t: torch.Tensor) -> torch.Tensor:
|
||||
int_dtype = {1: torch.uint8, 2: torch.int16, 4: torch.int32}[t.element_size()]
|
||||
return t.contiguous().view(int_dtype)
|
||||
|
||||
|
||||
def time_graph(fn, iters: int, calls_per_graph: int = 5) -> float:
|
||||
"""Mean seconds per call, replaying a CUDA graph of `calls_per_graph` calls."""
|
||||
for _ in range(3):
|
||||
fn()
|
||||
torch.cuda.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
for _ in range(calls_per_graph):
|
||||
fn()
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
graph.replay()
|
||||
torch.cuda.synchronize()
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
start.record()
|
||||
for _ in range(iters):
|
||||
graph.replay()
|
||||
end.record()
|
||||
torch.cuda.synchronize()
|
||||
return start.elapsed_time(end) / 1e3 / (iters * calls_per_graph)
|
||||
|
||||
|
||||
def main(argv: Iterable[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--rounds",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Rounds per shape; each round times both variants back to back.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--iters", type=int, default=50, help="Graph replays per round."
|
||||
)
|
||||
parser.add_argument("--seed", type=int, default=0)
|
||||
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("CUDA is required")
|
||||
major, minor = torch.cuda.get_device_capability()
|
||||
if major != 10:
|
||||
raise RuntimeError(f"This benchmark is for SM10x. Got {major}.{minor}.")
|
||||
if minor < 3:
|
||||
print(
|
||||
f"Note: SM{major}{minor} has no tcgen05.ld.red; {RED_MAX_ENV} is ignored "
|
||||
"and both variants run the same kernel."
|
||||
)
|
||||
|
||||
torch.manual_seed(args.seed)
|
||||
device = "cuda"
|
||||
mismatches = []
|
||||
print(
|
||||
f"{'shape':28s} {'FMNMX us':>9s} {'ld.red us':>9s} {'ld.red TF':>9s} "
|
||||
f"{'best':>7s} {'paired':>7s} {'wins':>5s} bitwise"
|
||||
)
|
||||
for images, tokens, heads, head_dim in SHAPES:
|
||||
torch.cuda.empty_cache()
|
||||
q, k, v = [
|
||||
torch.randn(
|
||||
images * tokens, heads, head_dim, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
for _ in range(3)
|
||||
]
|
||||
cu_seqlens = torch.arange(images + 1, dtype=torch.int32, device=device) * tokens
|
||||
kwargs = dict(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
cu_seqlens_q=cu_seqlens,
|
||||
cu_seqlens_k=cu_seqlens,
|
||||
max_seqlen_q=tokens,
|
||||
max_seqlen_k=tokens,
|
||||
)
|
||||
|
||||
outputs, times = {}, {False: [], True: []}
|
||||
for red_max in (False, True):
|
||||
os.environ[RED_MAX_ENV] = "1" if red_max else "0"
|
||||
outputs[red_max] = flash_attn_varlen_func(**kwargs)[0]
|
||||
# Swap the order every round so clock and thermal drift hit both equally.
|
||||
for r in range(args.rounds):
|
||||
for red_max in (False, True) if r % 2 == 0 else (True, False):
|
||||
os.environ[RED_MAX_ENV] = "1" if red_max else "0"
|
||||
times[red_max].append(
|
||||
time_graph(lambda: flash_attn_varlen_func(**kwargs), args.iters)
|
||||
)
|
||||
same = torch.equal(_bits(outputs[True]), _bits(outputs[False]))
|
||||
name = f"{images} x {tokens} tok, h{heads} d{head_dim}"
|
||||
if not same:
|
||||
mismatches.append(name)
|
||||
t0, t1 = min(times[False]), min(times[True])
|
||||
# Ratio of the two variants within each round, then the median over rounds.
|
||||
paired = sorted(a / c for a, c in zip(times[False], times[True]))
|
||||
wins = f"{sum(p > 1 for p in paired)}/{len(paired)}"
|
||||
flops = 4.0 * images * heads * head_dim * tokens * tokens
|
||||
print(
|
||||
f"{name:28s} {t0 * 1e6:9.1f} {t1 * 1e6:9.1f} {flops / t1 / 1e12:9.1f} "
|
||||
f"{t0 / t1:6.3f}x {paired[len(paired) // 2]:6.3f}x {wins:>5s} "
|
||||
f"{'identical' if same else 'DIFFERENT'}",
|
||||
flush=True,
|
||||
)
|
||||
del q, k, v, outputs
|
||||
|
||||
os.environ.pop(RED_MAX_ENV, None)
|
||||
if mismatches:
|
||||
print(f"\nOutputs differ between the variants for: {', '.join(mismatches)}")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -7,7 +7,7 @@ import torch.utils.benchmark as benchmark
|
||||
from flashinfer import BatchDecodeWithPagedKVCacheWrapper
|
||||
|
||||
from sglang.kernels.ops.attention.decode_attention import decode_attention_fwd
|
||||
from sglang.kernels.ops.attention.flashinfer_backend import should_use_tensor_core
|
||||
from sglang.srt.layers.attention.flashinfer_backend import should_use_tensor_core
|
||||
|
||||
|
||||
def benchmark_forward(
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.benchmark.deepseek_utils import (
|
||||
per_block_cast_to_fp8,
|
||||
per_token_cast_to_fp8,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
from sglang.kernels.ops.gemm.fp8_kernel import (
|
||||
w8a8_block_fp8_matmul_deepgemm as w8a8_block_fp8_matmul,
|
||||
)
|
||||
|
||||
|
||||
@@ -9,10 +9,8 @@ from sglang.benchmark.deepseek_utils import (
|
||||
get_weight_shapes,
|
||||
per_block_cast_to_fp8,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
sglang_per_token_group_quant_fp8,
|
||||
w8a8_block_fp8_matmul_deepgemm,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.fp8_kernel import w8a8_block_fp8_matmul_deepgemm
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import sglang_per_token_group_quant_fp8
|
||||
from sglang.srt.layers.quantization.fp8_utils import requant_weight_ue8m0
|
||||
|
||||
BLOCK_SIZE = 128
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Compare default, H200, and H20-3e block-FP8 configs on an H20-3e.
|
||||
|
||||
Example:
|
||||
python benchmark/kernels/deepseek/benchmark_h20_block_fp8.py \
|
||||
--output h20_block_fp8.jsonl
|
||||
|
||||
This measures individual GEMMs, not end-to-end model throughput. Run on an
|
||||
otherwise idle GPU. Inputs are synthetic; no model weights are needed.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import pathlib
|
||||
import statistics
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
import triton
|
||||
|
||||
from sglang.kernels.ops.gemm import fp8_kernel
|
||||
|
||||
SHAPES = [
|
||||
(1792, 5120),
|
||||
(25600, 6144),
|
||||
(4096, 1280),
|
||||
(5120, 1024),
|
||||
(5120, 288),
|
||||
(576, 5120),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output", type=pathlib.Path, required=True)
|
||||
parser.add_argument(
|
||||
"--batch-sizes",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64],
|
||||
)
|
||||
parser.add_argument("--trials", type=int, default=5)
|
||||
parser.add_argument("--rep-ms", type=int, default=50)
|
||||
parser.add_argument("--seed", type=int, default=98765)
|
||||
args = parser.parse_args()
|
||||
assert torch.cuda.get_device_name() == "NVIDIA H20-3e"
|
||||
assert args.trials > 0 and args.rep_ms > 0 and all(m > 0 for m in args.batch_sizes)
|
||||
torch.backends.cuda.matmul.allow_tf32 = False
|
||||
torch.manual_seed(args.seed)
|
||||
config_dir = pathlib.Path(fp8_kernel.__file__).parent / "configs"
|
||||
with args.output.open("x") as output:
|
||||
|
||||
def emit(row):
|
||||
text = json.dumps(row)
|
||||
output.write(text + "\n")
|
||||
output.flush()
|
||||
print(text, flush=True)
|
||||
|
||||
emit(
|
||||
{
|
||||
"kind": "environment",
|
||||
"device": torch.cuda.get_device_name(),
|
||||
"torch": torch.__version__,
|
||||
"triton": triton.__version__,
|
||||
"seed": args.seed,
|
||||
"batch_sizes": args.batch_sizes,
|
||||
}
|
||||
)
|
||||
for n, k in SHAPES:
|
||||
b = (torch.randn(n, k, device="cuda") * 0.2).to(torch.float8_e4m3fn)
|
||||
bs = torch.rand(n // 32, k // 32, device="cuda") + 0.1
|
||||
b_dequant = b.float() * bs.repeat_interleave(32, 0).repeat_interleave(32, 1)
|
||||
tables = {"default": None}
|
||||
for device in ("NVIDIA_H200", "NVIDIA_H20-3e"):
|
||||
path = (
|
||||
config_dir
|
||||
/ f"N={n},K={k},device_name={device},dtype=fp8_w8a8,block_shape=[32, 32].json"
|
||||
)
|
||||
if path.exists():
|
||||
tables[device] = {
|
||||
int(m): cfg for m, cfg in json.loads(path.read_text()).items()
|
||||
}
|
||||
assert "NVIDIA_H20-3e" in tables, (
|
||||
"Install the H20-3e configuration files before benchmarking"
|
||||
)
|
||||
for m in args.batch_sizes:
|
||||
a = (torch.randn(m, k, device="cuda") * 0.2).to(torch.float8_e4m3fn)
|
||||
scales = (torch.rand(k // 32, m, device="cuda") + 0.1).T
|
||||
reference = (
|
||||
(a.float() * scales.repeat_interleave(32, 1)) @ b_dequant.T
|
||||
).to(torch.bfloat16)
|
||||
results = {}
|
||||
# Compile and validate every variant before timing any variant.
|
||||
for label, table in tables.items():
|
||||
with patch.object(
|
||||
fp8_kernel, "get_w8a8_block_fp8_configs", return_value=table
|
||||
):
|
||||
actual = fp8_kernel.w8a8_block_fp8_matmul_triton(
|
||||
a, b, scales, bs, [32, 32], torch.bfloat16
|
||||
)
|
||||
relative_l2 = (
|
||||
torch.linalg.vector_norm(actual.float() - reference.float())
|
||||
/ torch.linalg.vector_norm(reference.float())
|
||||
).item()
|
||||
assert torch.isfinite(actual).all() and relative_l2 < 0.01
|
||||
results[label] = {
|
||||
"config": None
|
||||
if table is None
|
||||
else table[min(table, key=lambda key: abs(key - m))],
|
||||
"relative_l2": relative_l2,
|
||||
"trials_us": [],
|
||||
}
|
||||
for trial in range(args.trials):
|
||||
labels = list(tables)
|
||||
# Rotate variant order to reduce systematic timing-order bias.
|
||||
labels = (
|
||||
labels[trial % len(labels) :] + labels[: trial % len(labels)]
|
||||
)
|
||||
for label in labels:
|
||||
with patch.object(
|
||||
fp8_kernel,
|
||||
"get_w8a8_block_fp8_configs",
|
||||
return_value=tables[label],
|
||||
):
|
||||
|
||||
def invoke():
|
||||
return fp8_kernel.w8a8_block_fp8_matmul_triton(
|
||||
a, b, scales, bs, [32, 32], torch.bfloat16
|
||||
)
|
||||
|
||||
elapsed = (
|
||||
triton.testing.do_bench_cudagraph(
|
||||
invoke, rep=args.rep_ms
|
||||
)
|
||||
* 1000
|
||||
)
|
||||
results[label]["trials_us"].append(elapsed)
|
||||
for result in results.values():
|
||||
result["median_us"] = statistics.median(result["trials_us"])
|
||||
emit(
|
||||
{"kind": "measurement", "m": m, "n": n, "k": k, "results": results}
|
||||
)
|
||||
emit({"kind": "complete"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -78,7 +78,7 @@ python benchmark/kernels/quantization/tuning_block_wise_kernel.py --N 5120 --K 1
|
||||
|
||||
## Output
|
||||
|
||||
Generates JSON config files saved to `python/sglang/srt/layers/quantization/configs/`:
|
||||
Generates JSON config files saved to `python/sglang/kernels/ops/gemm/configs/`:
|
||||
```
|
||||
N={N},K={K},device_name={DEVICE},dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
```
|
||||
|
||||
@@ -28,11 +28,11 @@ from tqdm import tqdm
|
||||
mp.set_start_method("spawn", force=True)
|
||||
|
||||
from sglang.benchmark.deepseek_utils import get_weight_shapes
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
from sglang.kernels.ops.gemm.fp8_kernel import (
|
||||
_w8a8_block_fp8_matmul,
|
||||
_w8a8_block_fp8_matmul_unrolledx4,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.int8_kernel import _w8a8_block_int8_matmul
|
||||
from sglang.kernels.ops.gemm.int8_kernel import _w8a8_block_int8_matmul
|
||||
from sglang.srt.utils import (
|
||||
get_device,
|
||||
get_device_core_count,
|
||||
@@ -491,7 +491,7 @@ if __name__ == "__main__":
|
||||
parser.add_argument(
|
||||
"--save-path",
|
||||
type=str,
|
||||
default="python/sglang/kernels/ops/quantization/configs",
|
||||
default="python/sglang/kernels/ops/gemm/configs",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Tune the gfx950 native MXFP8 dense route for one (N, K) and write its rows into
|
||||
python/sglang/kernels/ops/quantization/mxfp8_gemv_gfx95_configs.json.
|
||||
|
||||
Two tables: the scaled-MFMA GEMV config per M bucket (1 ... 32) and the dot_scaled tile per
|
||||
M bucket (64 ... 16384). Each candidate is timed by HIP-graph replay rotating over weight
|
||||
copies that together exceed the last-level cache, so the weight streams from HBM as it does
|
||||
in decode. The GEMV pick minimizes the fp8-input plus the bf16-input time: fused producers
|
||||
hand it fp8, other callers bf16.
|
||||
|
||||
python benchmark/kernels/quantization/tuning_mxfp8_native_gfx95.py --N 1792 --K 5120
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import itertools
|
||||
import json
|
||||
|
||||
import torch
|
||||
from triton.runtime.errors import OutOfResources
|
||||
|
||||
from sglang.kernels.jit.utils import empty_sentinel
|
||||
from sglang.kernels.ops.quantization import mxfp8_native_amd_gfx95 as native
|
||||
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import fp8_grid_quantize
|
||||
|
||||
GRAPH_CALLS = 20
|
||||
ROTATION_BYTES = 512 << 20
|
||||
|
||||
|
||||
def graph_time_us(calls, replays: int) -> float:
|
||||
"""Mean time of one call, replaying a graph of GRAPH_CALLS calls cycling through calls."""
|
||||
for call in calls:
|
||||
call()
|
||||
torch.cuda.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
for i in range(GRAPH_CALLS):
|
||||
calls[i % len(calls)]()
|
||||
graph.replay()
|
||||
torch.cuda.synchronize()
|
||||
start, end = (
|
||||
torch.cuda.Event(enable_timing=True),
|
||||
torch.cuda.Event(enable_timing=True),
|
||||
)
|
||||
start.record()
|
||||
for _ in range(replays):
|
||||
graph.replay()
|
||||
end.record()
|
||||
torch.cuda.synchronize()
|
||||
return start.elapsed_time(end) * 1000 / (GRAPH_CALLS * replays)
|
||||
|
||||
|
||||
def make_weights(n: int, k: int):
|
||||
copies = max(2, min(16, -(-ROTATION_BYTES // (n * k))))
|
||||
weights = []
|
||||
for _ in range(copies):
|
||||
weight = (torch.randn(n, k, device="cuda") * 0.05).to(torch.float8_e4m3fn)
|
||||
scale = torch.exp2(
|
||||
torch.randint(-3, 3, (n // 32, k // 32), device="cuda").float()
|
||||
)
|
||||
weights.append(native.prepare_mxfp8_native_weight(weight, scale, [32, 32]))
|
||||
return weights
|
||||
|
||||
|
||||
def tune_gemv(n: int, k: int, weights) -> dict:
|
||||
rows = {}
|
||||
for bucket in native._M_BUCKETS:
|
||||
x = torch.randn(bucket, k, device="cuda", dtype=torch.bfloat16)
|
||||
xq, xs = fp8_grid_quantize(x)
|
||||
xq = xq.view(torch.uint8)
|
||||
no_scale = empty_sentinel(x.device, torch.uint8)
|
||||
out = torch.empty(bucket, n, dtype=torch.bfloat16, device="cuda")
|
||||
times = {}
|
||||
for waves, steps, tile_rows, tokens in itertools.product(
|
||||
(4, 8, 16), (1, 2, 4), (16, 32), (16, 32)
|
||||
):
|
||||
config = native._GemvConfig(waves, steps, tile_rows, tokens)
|
||||
if not config.valid_for(bucket, n):
|
||||
continue
|
||||
fp8_kernel = native._jit_mxfp8_gemv_module(config, False)
|
||||
bf16_kernel = native._jit_mxfp8_gemv_module(config, True)
|
||||
fp8_us = graph_time_us(
|
||||
[lambda w=w: fp8_kernel.run(w[0], w[1], xq, xs, out) for w in weights],
|
||||
replays=10,
|
||||
)
|
||||
bf16_us = graph_time_us(
|
||||
[
|
||||
lambda w=w: bf16_kernel.run(w[0], w[1], x, no_scale, out)
|
||||
for w in weights
|
||||
],
|
||||
replays=10,
|
||||
)
|
||||
times[f"w{waves}s{steps}r{tile_rows}t{tokens}k"] = fp8_us + bf16_us
|
||||
best = min(times, key=times.get)
|
||||
rows[f"gfx950:{n}:{k}:{bucket}"] = best
|
||||
print(f"gemv M={bucket:5d}: {best} {times[best]:.2f} us (fp8 + bf16)")
|
||||
return rows
|
||||
|
||||
|
||||
def tune_large_m(n: int, k: int, weights) -> dict:
|
||||
k_pad = weights[0][0].shape[1] * 128
|
||||
rows = {}
|
||||
for bucket in native._LARGE_M_BUCKETS:
|
||||
xq, xs = fp8_grid_quantize(
|
||||
torch.randn(bucket, k, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
times = {}
|
||||
for bm, bn, bk, warps, split_k in itertools.product(
|
||||
(32, 64, 128, 256), (64, 128, 256), (128, 256), (4, 8), (1, 2, 4, 5, 8)
|
||||
):
|
||||
if k_pad % bk or (k_pad // bk) % split_k or bm > max(64, 2 * bucket):
|
||||
continue
|
||||
if split_k > 1 and bucket > 1024:
|
||||
continue
|
||||
tile = (bm, bn, bk, warps)
|
||||
try:
|
||||
times[f"{bm},{bn},{bk},{warps},{split_k}"] = graph_time_us(
|
||||
[
|
||||
lambda w=w: native._mxfp8_shuffled_gemm(
|
||||
xq, xs, w[0], w[1], tile, split_k
|
||||
)
|
||||
for w in weights
|
||||
],
|
||||
replays=5,
|
||||
)
|
||||
except OutOfResources:
|
||||
continue # the tile's LDS footprint exceeds the CU's
|
||||
best = min(times, key=times.get)
|
||||
rows[f"gfx950:{n}:{k}:{bucket}"] = best
|
||||
print(f"large_m M={bucket:5d}: {best} {times[best]:.2f} us")
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--N", type=int, required=True)
|
||||
parser.add_argument("--K", type=int, required=True)
|
||||
args = parser.parse_args()
|
||||
assert native.native_route_supports(args.N, args.K), (args.N, args.K)
|
||||
assert torch.cuda.get_device_properties(0).gcnArchName.startswith("gfx950")
|
||||
|
||||
torch.manual_seed(0)
|
||||
weights = make_weights(args.N, args.K)
|
||||
gemv_rows = tune_gemv(args.N, args.K, weights)
|
||||
large_m_rows = tune_large_m(args.N, args.K, weights)
|
||||
|
||||
with open(native._CONFIG_FILE) as f:
|
||||
table = json.load(f)
|
||||
for section, rows in (("configs", gemv_rows), ("large_m", large_m_rows)):
|
||||
table[section] = dict(sorted({**table[section], **rows}.items()))
|
||||
with open(native._CONFIG_FILE, "w") as f:
|
||||
json.dump(table, f, indent=1)
|
||||
f.write("\n")
|
||||
print(f"wrote {len(gemv_rows) + len(large_m_rows)} rows to {native._CONFIG_FILE}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,20 +0,0 @@
|
||||
## Download data
|
||||
```
|
||||
bash download_data.sh
|
||||
```
|
||||
|
||||
## Run benchmark
|
||||
|
||||
### Benchmark sglang
|
||||
```
|
||||
python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000
|
||||
```
|
||||
|
||||
```
|
||||
python3 bench_sglang.py --nsub 10
|
||||
```
|
||||
|
||||
```
|
||||
# OpenAI models
|
||||
python3 bench_sglang.py --backend gpt-3.5-turbo --parallel 8
|
||||
```
|
||||
@@ -1,151 +0,0 @@
|
||||
"""
|
||||
Usage:
|
||||
python3 bench_hf.py --model-path meta-llama/Llama-2-7b-hf --data-dir data --ntrain 5
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
from tqdm import tqdm
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
choices = ["A", "B", "C", "D"]
|
||||
|
||||
|
||||
def format_subject(subject):
|
||||
l = subject.split("_")
|
||||
s = ""
|
||||
for entry in l:
|
||||
s += " " + entry
|
||||
return s
|
||||
|
||||
|
||||
def format_example(df, idx, include_answer=True):
|
||||
prompt = df.iloc[idx, 0]
|
||||
k = df.shape[1] - 2
|
||||
for j in range(k):
|
||||
prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1])
|
||||
prompt += "\nAnswer:"
|
||||
if include_answer:
|
||||
prompt += " {}\n\n".format(df.iloc[idx, k + 1])
|
||||
return prompt
|
||||
|
||||
|
||||
def gen_prompt(train_df, subject, k=-1):
|
||||
prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format(
|
||||
format_subject(subject)
|
||||
)
|
||||
if k == -1:
|
||||
k = train_df.shape[0]
|
||||
for i in range(k):
|
||||
prompt += format_example(train_df, i)
|
||||
return prompt
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def main(args):
|
||||
print(f"Loading model: {args.model_path}")
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
args.model_path,
|
||||
torch_dtype=torch.bfloat16,
|
||||
trust_remote_code=True,
|
||||
device_map="auto",
|
||||
).eval()
|
||||
|
||||
subjects = sorted(
|
||||
[
|
||||
f.split("_test.csv")[0]
|
||||
for f in os.listdir(os.path.join(args.data_dir, "test"))
|
||||
if "_test.csv" in f
|
||||
]
|
||||
)
|
||||
|
||||
all_cors = []
|
||||
num_requests = 0
|
||||
total_latency = 0
|
||||
|
||||
for subject in tqdm(subjects[: args.nsub]):
|
||||
dev_df = pd.read_csv(
|
||||
os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None
|
||||
)[: args.ntrain]
|
||||
test_df = pd.read_csv(
|
||||
os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None
|
||||
)
|
||||
|
||||
k = args.ntrain
|
||||
few_shot_examples = gen_prompt(dev_df, subject, k)
|
||||
while len(tokenizer.encode(few_shot_examples)) > 1536:
|
||||
k -= 1
|
||||
if k < 0:
|
||||
break
|
||||
few_shot_examples = gen_prompt(dev_df, subject, k)
|
||||
|
||||
preds = []
|
||||
labels = []
|
||||
tic = time.perf_counter()
|
||||
|
||||
for i in range(test_df.shape[0]):
|
||||
prompt_end = format_example(test_df, i, include_answer=False)
|
||||
prompt = few_shot_examples + prompt_end
|
||||
|
||||
input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.device)
|
||||
output_ids = model.generate(
|
||||
input_ids,
|
||||
max_new_tokens=1,
|
||||
do_sample=False,
|
||||
pad_token_id=tokenizer.eos_token_id,
|
||||
)
|
||||
|
||||
output_str = tokenizer.decode(
|
||||
output_ids[0][input_ids.shape[-1] :], skip_special_tokens=True
|
||||
)
|
||||
preds.append(output_str.strip()[0] if len(output_str.strip()) > 0 else "")
|
||||
labels.append(test_df.iloc[i, test_df.shape[1] - 1])
|
||||
|
||||
latency = time.perf_counter() - tic
|
||||
total_latency += latency
|
||||
|
||||
cors = [pred == label for pred, label in zip(preds, labels)]
|
||||
all_cors.append(cors)
|
||||
num_requests += len(test_df)
|
||||
|
||||
print(
|
||||
f"Subject: {subject}, Accuracy: {np.mean(cors):.3f}, Latency: {latency:.3f}s"
|
||||
)
|
||||
|
||||
weighted_acc = np.mean(np.concatenate(all_cors))
|
||||
print(f"Total Latency: {total_latency:.3f}s")
|
||||
print(f"Average Accuracy: {weighted_acc:.3f}")
|
||||
|
||||
if args.output:
|
||||
with open(args.output, "a") as fout:
|
||||
value = {
|
||||
"task": "mmlu",
|
||||
"backend": "hf",
|
||||
"model": args.model_path,
|
||||
"latency": round(total_latency, 3),
|
||||
"accuracy": round(weighted_acc, 3),
|
||||
"num_requests": num_requests,
|
||||
"other": {
|
||||
"nsub": args.nsub,
|
||||
"ntrain": args.ntrain,
|
||||
},
|
||||
}
|
||||
fout.write(json.dumps(value) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model-path", type=str, required=True)
|
||||
parser.add_argument("--ntrain", type=int, default=5)
|
||||
parser.add_argument("--data-dir", type=str, default="data")
|
||||
parser.add_argument("--nsub", type=int, default=60)
|
||||
parser.add_argument("--output", type=str, help="Output file path")
|
||||
args = parser.parse_args()
|
||||
main(args)
|
||||
@@ -1,210 +0,0 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tarfile
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import tiktoken
|
||||
|
||||
from sglang.test.test_utils import (
|
||||
add_common_sglang_args_and_parse,
|
||||
dump_bench_raw_result,
|
||||
select_sglang_backend,
|
||||
)
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
choices = ["A", "B", "C", "D"]
|
||||
|
||||
tokenizer = tiktoken.encoding_for_model("gpt-3.5-turbo")
|
||||
|
||||
|
||||
def format_subject(subject):
|
||||
l = subject.split("_")
|
||||
s = ""
|
||||
for entry in l:
|
||||
s += " " + entry
|
||||
return s
|
||||
|
||||
|
||||
def format_example(df, idx, include_answer=True):
|
||||
prompt = df.iloc[idx, 0]
|
||||
k = df.shape[1] - 2
|
||||
for j in range(k):
|
||||
prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1])
|
||||
prompt += "\nAnswer:"
|
||||
if include_answer:
|
||||
prompt += " {}\n\n".format(df.iloc[idx, k + 1])
|
||||
return prompt
|
||||
|
||||
|
||||
def gen_prompt(train_df, subject, k=-1):
|
||||
prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format(
|
||||
format_subject(subject)
|
||||
)
|
||||
if k == -1:
|
||||
k = train_df.shape[0]
|
||||
for i in range(k):
|
||||
prompt += format_example(train_df, i)
|
||||
return prompt
|
||||
|
||||
|
||||
def download_data(data_dir):
|
||||
"""Download and extract MMLU data if it doesn't exist."""
|
||||
if os.path.isdir(os.path.join(data_dir, "test")):
|
||||
return
|
||||
print(f"Data not found at {data_dir}. Downloading...")
|
||||
os.makedirs(data_dir, exist_ok=True)
|
||||
tar_path = os.path.join(data_dir, "data.tar")
|
||||
subprocess.check_call(
|
||||
["wget", "-O", tar_path, "https://people.eecs.berkeley.edu/~hendrycks/data.tar"]
|
||||
)
|
||||
with tarfile.open(tar_path) as tar:
|
||||
tar.extractall(path=data_dir, filter="data")
|
||||
# The tarball extracts into a "data/" subdirectory; move contents up if needed
|
||||
nested = os.path.join(data_dir, "data")
|
||||
if os.path.isdir(nested):
|
||||
for item in os.listdir(nested):
|
||||
os.rename(os.path.join(nested, item), os.path.join(data_dir, item))
|
||||
os.rmdir(nested)
|
||||
os.remove(tar_path)
|
||||
print("Download complete.")
|
||||
|
||||
|
||||
def main(args):
|
||||
subjects = sorted(
|
||||
[
|
||||
f.split("_test.csv")[0]
|
||||
for f in os.listdir(os.path.join(args.data_dir, "test"))
|
||||
if "_test.csv" in f
|
||||
]
|
||||
)
|
||||
|
||||
# Build prompts
|
||||
arguments = []
|
||||
labels = []
|
||||
num_questions = []
|
||||
|
||||
for subject in subjects[: args.nsub]:
|
||||
dev_df = pd.read_csv(
|
||||
os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None
|
||||
)[: args.ntrain]
|
||||
test_df = pd.read_csv(
|
||||
os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None
|
||||
)
|
||||
num_questions.append(test_df.shape[0])
|
||||
|
||||
k = args.ntrain
|
||||
few_shot_examples = gen_prompt(dev_df, subject, k)
|
||||
while len(tokenizer.encode(few_shot_examples)) > 1536:
|
||||
k -= 1
|
||||
few_shot_examples = gen_prompt(dev_df, subject, k)
|
||||
|
||||
for i in range(test_df.shape[0]):
|
||||
prompt_end = format_example(test_df, i, include_answer=False)
|
||||
|
||||
arguments.append(
|
||||
{
|
||||
"examples": few_shot_examples,
|
||||
"question": prompt_end,
|
||||
}
|
||||
)
|
||||
|
||||
label = test_df.iloc[i, test_df.shape[1] - 1]
|
||||
labels.append(label)
|
||||
|
||||
#####################################
|
||||
######### SGL Program Begin #########
|
||||
#####################################
|
||||
|
||||
import sglang as sgl
|
||||
|
||||
if args.backend.startswith("gpt-"):
|
||||
|
||||
@sgl.function
|
||||
def few_shot_mmlu(s, examples, question):
|
||||
s += sgl.user(examples + question)
|
||||
s += sgl.assistant(sgl.gen("answer"))
|
||||
|
||||
else:
|
||||
|
||||
@sgl.function
|
||||
def few_shot_mmlu(s, examples, question):
|
||||
s += examples + question + sgl.gen("answer")
|
||||
|
||||
#####################################
|
||||
########## SGL Program End ##########
|
||||
#####################################
|
||||
|
||||
# Select backend
|
||||
backend = select_sglang_backend(args)
|
||||
|
||||
# Run
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_mmlu.run_batch(
|
||||
arguments,
|
||||
temperature=0,
|
||||
max_new_tokens=1,
|
||||
backend=backend,
|
||||
num_threads=args.parallel,
|
||||
progress_bar=True,
|
||||
)
|
||||
preds = [
|
||||
s["answer"].strip()[0] if len(s["answer"].strip()) > 0 else "" for s in states
|
||||
]
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
# Compute accuracy
|
||||
cors = [pred == label for pred, label in zip(preds, labels)]
|
||||
|
||||
pt = 0
|
||||
for subject, num_qs in zip(subjects[: args.nsub], num_questions):
|
||||
print(
|
||||
f"subject: {subject}, #q:{num_qs}, acc: {np.mean(cors[pt : pt + num_qs]):.3f}"
|
||||
)
|
||||
pt += num_qs
|
||||
assert pt == len(cors)
|
||||
weighted_acc = np.mean(cors)
|
||||
|
||||
dump_bench_raw_result(
|
||||
path=args.raw_result_file,
|
||||
states=states,
|
||||
preds=preds,
|
||||
labels=labels,
|
||||
)
|
||||
|
||||
# Print results
|
||||
print("Total latency: {:.3f}".format(latency))
|
||||
print("Average accuracy: {:.3f}".format(weighted_acc))
|
||||
|
||||
# Write results
|
||||
with open(args.result_file, "a") as fout:
|
||||
value = {
|
||||
"task": "mmlu",
|
||||
"backend": args.backend,
|
||||
"num_gpus": 1,
|
||||
"latency": round(latency, 3),
|
||||
"accuracy": round(weighted_acc, 3),
|
||||
"num_requests": len(arguments),
|
||||
"other": {
|
||||
"nsub": args.nsub,
|
||||
"parallel": args.parallel,
|
||||
},
|
||||
}
|
||||
fout.write(json.dumps(value) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--ntrain", "-k", type=int, default=5)
|
||||
parser.add_argument(
|
||||
"--data_dir", "-d", type=str, default=os.path.join(SCRIPT_DIR, "data")
|
||||
)
|
||||
parser.add_argument("--save_dir", "-s", type=str, default="results")
|
||||
parser.add_argument("--nsub", type=int, default=60)
|
||||
args = add_common_sglang_args_and_parse(parser)
|
||||
download_data(args.data_dir)
|
||||
main(args)
|
||||
@@ -1,2 +0,0 @@
|
||||
wget https://people.eecs.berkeley.edu/~hendrycks/data.tar
|
||||
tar xf data.tar
|
||||
@@ -0,0 +1,271 @@
|
||||
"""Engine benchmark for setwise (multi-position) scoring.
|
||||
|
||||
Benchmarks `Engine.async_score(..., score_extraction_token_id=...)` — the setwise
|
||||
path that pools the classification head at each candidate's anchor token and
|
||||
returns an ``[N, num_labels]`` matrix in a single forward pass.
|
||||
|
||||
Unlike the general ``bench_score.py`` (label-token / MIS scoring), this is a
|
||||
focused, self-contained harness: it sweeps the **set size** (number of
|
||||
candidates packed into one item) and reports latency percentiles and throughput
|
||||
(requests/s and candidates/s) so you can see how a single-forward setwise pass
|
||||
scales with N.
|
||||
|
||||
It talks to the Engine directly (no HTTP) and requires the setwise deployment
|
||||
constraints (``--disable-radix-cache``, ``--chunked-prefill-size -1``), which it
|
||||
sets automatically.
|
||||
|
||||
Usage:
|
||||
# Default sweep over set sizes on the target model
|
||||
python bench_setwise_score.py --model-path /path/to/seqcls_model
|
||||
|
||||
# Custom sweep + concurrency + per-candidate length
|
||||
python bench_setwise_score.py --model-path /path/to/model \
|
||||
--set-sizes 8 16 32 64 --concurrency 16 --num-requests 100 \
|
||||
--item-tokens 180 --query-tokens 64
|
||||
|
||||
# Include pooled hidden states in the response (per-anchor pre-head vectors)
|
||||
python bench_setwise_score.py --model-path /path/to/model --return-pooled-hidden-states
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import csv
|
||||
import random
|
||||
import statistics
|
||||
import time
|
||||
from typing import List, Optional
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Benchmark setwise scoring via Engine.async_score",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model-path",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Path to a SequenceClassification model (e.g. Qwen3ForSequenceClassification).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--score-extraction-token",
|
||||
type=str,
|
||||
default="<|object_ref_start|>",
|
||||
help="Anchor token pooled at each candidate (default: <|object_ref_start|>).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--set-sizes",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=[8, 16, 32, 64],
|
||||
help="Candidates per set (each is one anchor in one item). Swept in order.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--item-tokens",
|
||||
type=int,
|
||||
default=180,
|
||||
help="Content tokens per candidate before its anchor (default: 180).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--query-tokens",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Shared query-prefix tokens (default: 0).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-requests",
|
||||
type=int,
|
||||
default=50,
|
||||
help="Timed requests per set size (default: 50).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--concurrency",
|
||||
type=int,
|
||||
default=8,
|
||||
help="Max in-flight requests (default: 8).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--warmup",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Warmup requests per set size, not timed (default: 3).",
|
||||
)
|
||||
parser.add_argument("--apply-softmax", action="store_true")
|
||||
parser.add_argument("--return-pooled-hidden-states", action="store_true")
|
||||
|
||||
# Engine configuration
|
||||
parser.add_argument("--dtype", type=str, default="float16")
|
||||
parser.add_argument("--mem-fraction-static", type=float, default=0.5)
|
||||
parser.add_argument("--max-prefill-tokens", type=int, default=60000)
|
||||
parser.add_argument("--attention-backend", type=str, default="flashinfer")
|
||||
parser.add_argument("--seed", type=int, default=0)
|
||||
parser.add_argument(
|
||||
"--output", "-o", type=str, default=None, help="Optional CSV output path."
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _random_content(num_tokens: int, anchor_id: int, lo: int = 10, hi: int = 30000):
|
||||
"""Random content token ids that never collide with the anchor id."""
|
||||
ids = [random.randint(lo, hi) for _ in range(num_tokens)]
|
||||
return [t + 1 if t == anchor_id else t for t in ids]
|
||||
|
||||
|
||||
def _build_setwise_request(args, anchor_id: int, n_candidates: int):
|
||||
"""One setwise request: shared query prefix + ONE item with N anchors."""
|
||||
query_ids = (
|
||||
_random_content(args.query_tokens, anchor_id) if args.query_tokens else []
|
||||
)
|
||||
item_ids: List[int] = []
|
||||
for _ in range(n_candidates):
|
||||
item_ids.extend(_random_content(args.item_tokens, anchor_id))
|
||||
item_ids.append(anchor_id)
|
||||
return query_ids, [item_ids]
|
||||
|
||||
|
||||
def _percentile(sorted_vals: List[float], pct: float) -> float:
|
||||
if not sorted_vals:
|
||||
return 0.0
|
||||
k = max(0, min(len(sorted_vals) - 1, round((pct / 100.0) * (len(sorted_vals) - 1))))
|
||||
return sorted_vals[k]
|
||||
|
||||
|
||||
async def _run_set_size(engine, args, anchor_id: int, n: int) -> Optional[dict]:
|
||||
# Pre-build requests so generation cost is outside the timed section.
|
||||
requests = [
|
||||
_build_setwise_request(args, anchor_id, n)
|
||||
for _ in range(args.num_requests + args.warmup)
|
||||
]
|
||||
prompt_tokens = len(requests[0][0]) + len(requests[0][1][0])
|
||||
|
||||
async def _one(query_ids, items):
|
||||
t0 = time.perf_counter()
|
||||
res = await engine.async_score(
|
||||
query=query_ids,
|
||||
items=items,
|
||||
score_extraction_token_id=anchor_id,
|
||||
apply_softmax=args.apply_softmax,
|
||||
return_pooled_hidden_states=args.return_pooled_hidden_states,
|
||||
)
|
||||
# Setwise scores are nested (one [Ni x num_labels] matrix per item) and
|
||||
# this harness sends exactly one item, so the per-candidate row count is
|
||||
# len(res.scores[0]), not len(res.scores) (which is always 1 = num items).
|
||||
return time.perf_counter() - t0, len(res.scores[0]) if res.scores else 0
|
||||
|
||||
# Warmup (also validates the row count == N).
|
||||
for q, it in requests[: args.warmup]:
|
||||
_, rows = await _one(q, it)
|
||||
if rows != n:
|
||||
print(f"[warn] set_size={n}: expected {n} score rows, got {rows}")
|
||||
|
||||
sem = asyncio.Semaphore(args.concurrency)
|
||||
|
||||
async def _guarded(q, it):
|
||||
async with sem:
|
||||
return await _one(q, it)
|
||||
|
||||
timed = requests[args.warmup :]
|
||||
wall_start = time.perf_counter()
|
||||
results = await asyncio.gather(*(_guarded(q, it) for q, it in timed))
|
||||
wall = time.perf_counter() - wall_start
|
||||
|
||||
latencies_ms = sorted(1000.0 * lat for lat, _ in results)
|
||||
n_req = len(results)
|
||||
return {
|
||||
"set_size": n,
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"requests": n_req,
|
||||
"concurrency": args.concurrency,
|
||||
"avg_ms": statistics.mean(latencies_ms),
|
||||
"p50_ms": _percentile(latencies_ms, 50),
|
||||
"p95_ms": _percentile(latencies_ms, 95),
|
||||
"p99_ms": _percentile(latencies_ms, 99),
|
||||
"req_per_s": n_req / wall if wall > 0 else 0.0,
|
||||
"cand_per_s": (n_req * n) / wall if wall > 0 else 0.0,
|
||||
}
|
||||
|
||||
|
||||
def _print_table(rows: List[dict]):
|
||||
headers = [
|
||||
"set_size",
|
||||
"prompt_tokens",
|
||||
"requests",
|
||||
"avg_ms",
|
||||
"p50_ms",
|
||||
"p95_ms",
|
||||
"p99_ms",
|
||||
"req_per_s",
|
||||
"cand_per_s",
|
||||
]
|
||||
print("\n" + " | ".join(f"{h:>13}" for h in headers))
|
||||
print("-" * (16 * len(headers)))
|
||||
for r in rows:
|
||||
cells = []
|
||||
for h in headers:
|
||||
v = r[h]
|
||||
cells.append(f"{v:>13.2f}" if isinstance(v, float) else f"{v:>13}")
|
||||
print(" | ".join(cells))
|
||||
|
||||
|
||||
async def main():
|
||||
args = parse_args()
|
||||
random.seed(args.seed)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True)
|
||||
anchor_id = tokenizer.convert_tokens_to_ids(args.score_extraction_token)
|
||||
unk_id = tokenizer.unk_token_id
|
||||
if anchor_id is None or (unk_id is not None and anchor_id == unk_id):
|
||||
raise ValueError(
|
||||
f"score_extraction_token {args.score_extraction_token!r} did not resolve "
|
||||
f"to a dedicated token id."
|
||||
)
|
||||
|
||||
from sglang.srt.entrypoints.engine import Engine
|
||||
|
||||
# Setwise deployment constraints: positions are full-prompt coordinates, so
|
||||
# radix reuse / chunked prefill must be off (the engine also validates this).
|
||||
engine = Engine(
|
||||
model_path=args.model_path,
|
||||
dtype=args.dtype,
|
||||
mem_fraction_static=args.mem_fraction_static,
|
||||
max_prefill_tokens=args.max_prefill_tokens,
|
||||
attention_backend=args.attention_backend,
|
||||
disable_radix_cache=True,
|
||||
chunked_prefill_size=-1,
|
||||
)
|
||||
|
||||
print(
|
||||
f"Setwise scoring benchmark | model={args.model_path}\n"
|
||||
f"anchor={args.score_extraction_token!r} (id={anchor_id}) | "
|
||||
f"item_tokens={args.item_tokens} query_tokens={args.query_tokens} | "
|
||||
f"concurrency={args.concurrency} num_requests={args.num_requests}"
|
||||
)
|
||||
|
||||
rows = []
|
||||
try:
|
||||
for n in args.set_sizes:
|
||||
row = await _run_set_size(engine, args, anchor_id, n)
|
||||
if row is not None:
|
||||
rows.append(row)
|
||||
print(
|
||||
f"set_size={n:>4}: avg={row['avg_ms']:.1f}ms "
|
||||
f"p95={row['p95_ms']:.1f}ms "
|
||||
f"{row['req_per_s']:.1f} req/s {row['cand_per_s']:.0f} cand/s"
|
||||
)
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
_print_table(rows)
|
||||
|
||||
if args.output and rows:
|
||||
with open(args.output, "w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
print(f"\nWrote {len(rows)} rows to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Finite cold-cache decode benchmark using synthetic token IDs.
|
||||
|
||||
Run against an otherwise idle SGLang server. This clears its prefix cache.
|
||||
Example:
|
||||
python benchmark/serving/benchmark_h20_decode.py \
|
||||
--model-path /path/to/DeepSeek-V4.1-Flash --output before.jsonl
|
||||
|
||||
Repeat after installing the H20-3e configurations with identical server flags.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
import statistics
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
from tokenizers import Tokenizer
|
||||
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("--label", default="measurement")
|
||||
p.add_argument("--url", default="http://127.0.0.1:8000")
|
||||
p.add_argument("--model-path", type=pathlib.Path, required=True)
|
||||
p.add_argument("--output", type=pathlib.Path, required=True)
|
||||
args = p.parse_args()
|
||||
out = args.output
|
||||
assert not out.exists()
|
||||
tokenizer = Tokenizer.from_file(str(args.model_path / "tokenizer.json"))
|
||||
filler = tokenizer.encode(
|
||||
(
|
||||
"The research server processes independent records. Each record contains an identifier, a timestamp, and a measured value. We compare latency using the same data and the same output budget.\n"
|
||||
)
|
||||
* 16000,
|
||||
add_special_tokens=False,
|
||||
).ids
|
||||
|
||||
|
||||
def emit(row):
|
||||
row = {
|
||||
"time": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"label": args.label,
|
||||
**row,
|
||||
}
|
||||
with out.open("a") as f:
|
||||
f.write(json.dumps(row) + "\n")
|
||||
print(json.dumps(row), flush=True)
|
||||
|
||||
|
||||
def request(path, body=None):
|
||||
return urllib.request.urlopen(
|
||||
urllib.request.Request(
|
||||
args.url.rstrip("/") + path,
|
||||
data=None if body is None else json.dumps(body).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
),
|
||||
timeout=None,
|
||||
)
|
||||
|
||||
|
||||
def flush():
|
||||
with request("/flush_cache?timeout=30", {}) as r:
|
||||
r.read()
|
||||
|
||||
|
||||
def native(case, isl, osl, index):
|
||||
ids = (
|
||||
tokenizer.encode(
|
||||
f"Independent record {index}: {case}.\n", add_special_tokens=False
|
||||
).ids
|
||||
+ filler
|
||||
)[:isl]
|
||||
start = time.perf_counter()
|
||||
first = last = None
|
||||
final = None
|
||||
events = 0
|
||||
body = {
|
||||
"input_ids": ids,
|
||||
"sampling_params": {
|
||||
"temperature": 0.0,
|
||||
"max_new_tokens": osl,
|
||||
"ignore_eos": True,
|
||||
},
|
||||
"stream": True,
|
||||
}
|
||||
with request("/generate", body) as response:
|
||||
for line in response:
|
||||
if not line.startswith(b"data:"):
|
||||
continue
|
||||
value = line[5:].strip()
|
||||
if value == b"[DONE]":
|
||||
break
|
||||
obj = json.loads(value)
|
||||
now = time.perf_counter()
|
||||
if obj.get("error"):
|
||||
raise RuntimeError(str(obj["error"]))
|
||||
if obj.get("meta_info", {}).get("completion_tokens", 0) > 0:
|
||||
first = first or now
|
||||
last = now
|
||||
final = obj
|
||||
events += 1
|
||||
elapsed = time.perf_counter() - start
|
||||
assert final is not None, (case, index)
|
||||
meta = final["meta_info"]
|
||||
assert meta["completion_tokens"] == osl and meta["prompt_tokens"] == isl, meta
|
||||
assert meta.get("cached_tokens") == 0, meta
|
||||
return {
|
||||
"kind": "request",
|
||||
"case": case,
|
||||
"index": index,
|
||||
"input_hash": hashlib.sha256(json.dumps(ids).encode()).hexdigest(),
|
||||
"prompt_tokens": isl,
|
||||
"completion_tokens": osl,
|
||||
"cached_tokens": meta["cached_tokens"],
|
||||
"ttft_s": first - start,
|
||||
"elapsed_s": elapsed,
|
||||
"decode_tokens_per_s": (osl - 1) / (last - first) if osl > 1 else None,
|
||||
"tpot_ms": 1000 * (last - first) / (osl - 1) if osl > 1 else None,
|
||||
"events": events,
|
||||
}
|
||||
|
||||
|
||||
cases = [
|
||||
("8k_1000_c1", 8192, 1000, 1),
|
||||
("1k_128_c1", 1024, 128, 1),
|
||||
("1k_128_c8", 1024, 128, 8),
|
||||
("1k_128_c32", 1024, 128, 32),
|
||||
("8k_128_c8", 8192, 128, 8),
|
||||
("32k_128_c1", 32768, 128, 1),
|
||||
]
|
||||
emit(
|
||||
{
|
||||
"kind": "start",
|
||||
"cases": cases,
|
||||
"repetitions": 3,
|
||||
}
|
||||
)
|
||||
try:
|
||||
for case, isl, osl, concurrency in cases:
|
||||
flush()
|
||||
emit({"kind": "warmup_start", "case": case})
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
list(
|
||||
pool.map(
|
||||
lambda i: native("warm_" + case, isl, 16, i), range(concurrency)
|
||||
)
|
||||
)
|
||||
for repeat in range(3):
|
||||
flush()
|
||||
start = time.perf_counter()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
rows = list(
|
||||
pool.map(
|
||||
lambda i: native(case, isl, osl, i + repeat * 100),
|
||||
range(concurrency),
|
||||
)
|
||||
)
|
||||
wall = time.perf_counter() - start
|
||||
for row in rows:
|
||||
emit({**row, "repeat": repeat})
|
||||
emit(
|
||||
{
|
||||
"kind": "batch",
|
||||
"case": case,
|
||||
"repeat": repeat,
|
||||
"concurrency": concurrency,
|
||||
"wall_s": wall,
|
||||
"output_tokens_per_s": concurrency * osl / wall,
|
||||
"median_ttft_s": statistics.median(x["ttft_s"] for x in rows),
|
||||
"median_decode_tokens_per_s": statistics.median(
|
||||
x["decode_tokens_per_s"] for x in rows
|
||||
),
|
||||
"median_tpot_ms": statistics.median(x["tpot_ms"] for x in rows),
|
||||
}
|
||||
)
|
||||
emit({"kind": "complete"})
|
||||
except Exception as exc:
|
||||
emit({"kind": "failed", "error": repr(exc)})
|
||||
raise
|
||||
@@ -1,21 +0,0 @@
|
||||
# 优化版(2026-09-24):解决清华源限流下每次构建重下 torch 的问题。
|
||||
# 要点:
|
||||
# 1. pyproject build-system.requires 含 torch==2.13.0,pip 隔离构建环境每次都重下 ~6GB。
|
||||
# 基底镜像已装 torch 2.13 → 用 --no-build-isolation 复用,构建环境零下载。
|
||||
# 2. 基底缺 setuptools-scm(版本打戳要用),单层预装 + pip cache mount,只下载一次。
|
||||
# 3. RUN 的 pip cache mount 持久化在宿主机,依赖解析命中的包不再重复下载。
|
||||
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
|
||||
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV PIP_INDEX_URL=${PIP_INDEX}
|
||||
|
||||
# 构建后端(pyproject build-system.requires,torch 除外——基底已有)
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
|
||||
|
||||
# 整个源码树(含 .git,用于 setuptools-scm 版本打戳)
|
||||
COPY sglang/ /sgl-workspace/sglang/
|
||||
|
||||
# editable 安装,全量依赖解析(pyproject.toml 变化时走这个)
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
@@ -1,15 +0,0 @@
|
||||
# fast 路径:python/pyproject.toml 未变化时使用(build.sh 自动判断)。
|
||||
# --no-deps 跳过全部依赖解析(连 index 元数据请求都省掉),纯源码迭代秒级完成。
|
||||
# 依赖有变化时必须走 Dockerfile 全量路径(build.sh 按 pyproject sha256 自动切换)。
|
||||
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
|
||||
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV PIP_INDEX_URL=${PIP_INDEX}
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
|
||||
|
||||
COPY sglang/ /sgl-workspace/sglang/
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-deps --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
@@ -1,33 +0,0 @@
|
||||
#!/bin/bash
|
||||
# 用法: /data/ymk/build/build.sh [tag]
|
||||
# 默认 tag: ymkymx/sglang:<分支>-<sha9>-local-<UTC构建日期时间>(命名规则见 AGENTS.md)
|
||||
# 另打 <分支>-latest-local 别名。
|
||||
#
|
||||
# 依赖路径自动选择:
|
||||
# python/pyproject.toml 的 sha256 与上次成功构建一致 → Dockerfile.fast(--no-deps,秒级)
|
||||
# 不一致(或 FULL=1)→ Dockerfile 全量依赖解析,成功后记录新 hash
|
||||
set -euo pipefail
|
||||
cd /data/ymk
|
||||
SHA=$(git -C sglang rev-parse --short=9 HEAD)
|
||||
BR=$(git -C sglang rev-parse --abbrev-ref HEAD)
|
||||
DT=$(date -u +%Y%m%d-%H%M)
|
||||
TAG=${1:-ymkymx/sglang:$BR-$SHA-local-$DT}
|
||||
|
||||
HASH_FILE=build/.last_pyproject_sha256
|
||||
CUR=$(sha256sum sglang/python/pyproject.toml | cut -d' ' -f1)
|
||||
PREV=$(cat "$HASH_FILE" 2>/dev/null || echo none)
|
||||
|
||||
DOCKERFILE=build/Dockerfile
|
||||
if [ "$CUR" = "$PREV" ] && [ "${FULL:-0}" != "1" ]; then
|
||||
DOCKERFILE=build/Dockerfile.fast
|
||||
echo "== pyproject 未变,走 fast 路径(--no-deps)"
|
||||
else
|
||||
echo "== pyproject 有变化或 FULL=1,走全量依赖解析"
|
||||
fi
|
||||
|
||||
echo "== building $TAG from sglang@$(git -C sglang log --oneline -1) [$DOCKERFILE]"
|
||||
docker build -f "$DOCKERFILE" -t "$TAG" .
|
||||
# 构建成功才记录 hash / 打别名
|
||||
echo "$CUR" > "$HASH_FILE"
|
||||
docker tag "$TAG" "ymkymx/sglang:$BR-latest-local"
|
||||
echo "== done: $TAG (别名 $BR-latest-local)"
|
||||
@@ -1,33 +0,0 @@
|
||||
# B300 ds41 部署 compose 档案
|
||||
|
||||
b300-01 上 DeepSeek-V4.1-Flash 各部署方案的 docker compose 存档。
|
||||
**本目录是 source of truth**;b300 上的运行目录 `/data/ymk/ds41/` 是工作副本。
|
||||
|
||||
方案说明、参数矩阵、特殊 env、已知坑见飞书部署手册(组内共享):
|
||||
《B300 sglang 部署手册》 https://u04wb5irxz.feishu.cn/docx/FOi8dbnybodHr0xIfkPcQ6gtnRx
|
||||
|
||||
## 同步流程
|
||||
|
||||
```bash
|
||||
# 本机(D:\B300\sglang)
|
||||
git add deploy/ && git commit -m "deploy: ..."
|
||||
git push origin dsv41-pd-visioncp # Gitea(paths-ignore,不触发镜像构建)
|
||||
git push b300 dsv41-pd-visioncp # b300 裸仓库
|
||||
|
||||
# b300-01
|
||||
git -C /data/ymk/sglang pull # deploy/ 出现在 /data/ymk/sglang/deploy/b300-ds41/
|
||||
```
|
||||
|
||||
## CI 说明
|
||||
|
||||
`.gitea/workflows/build-image.yaml` 的 push 触发器带 `paths-ignore: ['deploy/**', '.gitea/**']`,
|
||||
改本目录不会触发镜像构建。注意:workflow 自身的改动也不再自动触发,需要重建镜像时
|
||||
伴随代码改动 push 或手动触发。
|
||||
|
||||
## 目录规则
|
||||
|
||||
- 命名:`dockerserve-<拓扑>-<特性>-<卡位>.yml`(如 `-b4` = 后 4 卡)
|
||||
- 退役文件不要留在主目录:移入 `archive-YYYYMMDD/`(b300 侧同样执行)
|
||||
- 禁止提交 `.bak` 备份文件
|
||||
- `mooncake-store.json` 是 hicache L3 mooncake store 配置,被
|
||||
`dockerserve-pd2-p-r1/r2.yml` 通过 `SGLANG_HICACHE_MOONCAKE_CONFIG_PATH` 引用
|
||||
@@ -1,171 +0,0 @@
|
||||
# 3x cp2-P + 1x dp2-D PD 分离 + L3 mooncake 互联缓存 + engram host table
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-d
|
||||
environment:
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -1,195 +0,0 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -1,195 +0,0 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -1,173 +0,0 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram(2026-09-25,mr256,nsys on p1/d1)
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
nsys launch --session-new=p1prof --cuda-graph-trace=node sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
nsys launch --session-new=d1prof --cuda-graph-trace=node sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -1,173 +0,0 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram(2026-09-25,mr256,nsys on p1/d1)
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -1,43 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.60
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,43 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.60
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
|
||||
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,42 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,160 +0,0 @@
|
||||
# 4×cp2 独立实例(非 PD)+ hicache L3(mooncake) + dspark + engram host table 卸载(2026-09-23 晚)
|
||||
# 用户指定组合:4 cp2 hicache l3 dspark roundrobin loadbalance engram offload
|
||||
# 每实例:tp2 ep2 + interleave CP2 + dspark b5 + hicache L3 write_through(size 0) + engram host table
|
||||
# (tp2 权重必须靠 engram 卸载才放得下,见 goals-track G1.9 反转)
|
||||
# GPU: a=0,1 b=2,3 c=4,5 d=6,7;端口: 30000/30010/30020/30030;rr router=30002
|
||||
# 参考:dockerserve-cpdspark-l3.yml(hicache 参数)、dockerserve-3cp2-pd.yml(cp2/engram 参数)
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
# 热补丁(2026-09-23):CP 对称 idle 检查,修 health-check × hicache drain 死锁
|
||||
# 源文件:/data/ymk/sglang(dsv41-pd-visioncp 分支工作区已改);补丁脚本见
|
||||
# D:\B300\experiments\cp2x4\patch_healthcheck_cp_idle.py
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
environment: &env
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_MS_AUTO_DISC: "0"
|
||||
MOONCAKE_MASTER: 127.0.0.1:50051
|
||||
MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE
|
||||
MOONCAKE_PROTOCOL: tcp
|
||||
MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
a:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-a
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
b:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-b
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
c:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-c
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
d:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-d2
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30030
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -1,44 +0,0 @@
|
||||
# cp8 单机非 PD 测试(2026-09-23 晚)
|
||||
# 背景:dp8 单机 dspark 健康但长上下文 prefill 结构性慢(单请求只落 1 rank),
|
||||
# 换 cp8(tp8 + interleave prefill CP8,单请求 prefill 切到 8 卡)对照。
|
||||
# 关 engram、无 hicache L3、无 PD。cp+dspark 无 replay 是已知健康组合
|
||||
# (cp4+dspark+replay 三元组必崩,本配置不开 replay)。
|
||||
# GPU 0-7,端口 30000,容器 ds41-cp8,project cp8
|
||||
services:
|
||||
cp8:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-cp8
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
SGLANG_RAGGED_VERIFY_MODE: "static"
|
||||
CUDA_VISIBLE_DEVICES: "0,1,2,3,4,5,6,7"
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 8 --ep-size 8
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -1,50 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-b
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,51 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-b
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,50 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-a
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,51 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-a
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,44 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-92632a60b-20260922-2100
|
||||
container_name: ds41-cpdspark-nr-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30001"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,44 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-92632a60b-20260922-2100
|
||||
container_name: ds41-cpdspark-nr
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,44 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-cpdspark
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,62 +0,0 @@
|
||||
# dp2 对照(2026-09-24 上午):验证 dp2(dp-attention)下 dspark 是否正常。
|
||||
# 与 dockerserve-tp2.yml 逐参数对齐,唯一差异 = 加 --dp-size 2 --enable-dp-attention --enable-dp-lm-head。
|
||||
# GPU 6-7,端口 30030 直连。镜像/补丁挂载/L3/engram/tok8 全部相同。
|
||||
# 注:dp+dspark 不能开 --enable-decoder-swa-bounded-replay(本配置本来就没开)。
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
environment: &env
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_MS_AUTO_DISC: "0"
|
||||
MOONCAKE_MASTER: 127.0.0.1:50051
|
||||
MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE
|
||||
MOONCAKE_PROTOCOL: tcp
|
||||
MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
dp2:
|
||||
<<: *common
|
||||
container_name: ds41-dp2
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--dp-size 2 --enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30030
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -1,42 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-dp4test
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,44 +0,0 @@
|
||||
# dp8+tp8 单机非 PD 对照测试(2026-09-23 晚)
|
||||
# 背景:3cp2+dp2 PD 部署 dspark accept 回归(32k ctx accept len 3.14→1.31,
|
||||
# 见 experiments/3cp2-pd/bench-20260923.md「未决问题」)。本配置退回单机验证
|
||||
# dspark 健康度:engram host table 关闭、无 hicache L3、无 PD、无 mooncake。
|
||||
# 参数基准:dockerserve-3cp2-pd.yml 的 d 服务(dp 路径),去掉 PD/L3/engram 相关。
|
||||
# GPU 0-7,端口 30000,容器 ds41-dp8,project dp8
|
||||
services:
|
||||
dp8:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-dp8
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
SGLANG_RAGGED_VERIFY_MODE: "static"
|
||||
CUDA_VISIBLE_DEVICES: "0,1,2,3,4,5,6,7"
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 8 --ep-size 8 --dp-size 8
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -1,47 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--disaggregation-mode prefill
|
||||
--optimistic-prefill-attempts 4
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,51 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd-decode
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30001
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -1,50 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-pd-decode
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30001
|
||||
--enable-cache-report
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,55 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-c74a4037f-20260921-1300
|
||||
container_name: ds41-pd-prefill-dp
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--load-balance-method total_tokens
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30010
|
||||
--enable-cache-report
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-bootstrap-port 8918
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,53 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -1,52 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-pd-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,57 +0,0 @@
|
||||
# PD2 二分 Round 2 D 侧(2026-09-24):干净 D + 嫌疑项②
|
||||
# SGLANG_DISAGGREGATION_QUEUE_SIZE=16 + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-decode
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=6,7
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
- SGLANG_DISAGGREGATION_QUEUE_SIZE=16
|
||||
- SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--dp-size 2
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30030
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -1,54 +0,0 @@
|
||||
# PD 干净对照实验 D 节点(2026-09-24 上午):dp2 decode,配 dockerserve-pd2-p.yml 使用。
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-decode
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=6,7
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--dp-size 2
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30030
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -1,67 +0,0 @@
|
||||
# PD2 二分 Round 1(2026-09-24):干净基线 + 嫌疑项①「P 侧 L3 hicache 块 + mooncake-store.json」
|
||||
# 相对 dockerserve-pd2-p.yml 的差异:
|
||||
# env 加 SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
|
||||
# hicache 参数块换成坏部署同款:page_first_direct + direct + write_through + mooncake +
|
||||
# wait_complete + size 0(替代 write_back L2)
|
||||
# D 侧不变(坏部署 D 本无 L3)。探针:bs16 low_entropy decode 看 accept len。
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
- SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30020
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -1,65 +0,0 @@
|
||||
# PD2 二分 Round 2 P 侧(2026-09-24):R1(L3 块)之上再加嫌疑项②
|
||||
# SGLANG_DISAGGREGATION_QUEUE_SIZE=16 + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
- SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
|
||||
- SGLANG_DISAGGREGATION_QUEUE_SIZE=16
|
||||
- SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30020
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -1,59 +0,0 @@
|
||||
# PD 干净对照实验(2026-09-24 上午):验证「cp2-P → dp2-D」传输路径下 dspark 是否正常。
|
||||
# 用户疑问:单机六形态健康不等于 PD 链路健康,可能两边传输没对齐。
|
||||
# 以已验证可用的 dockerserve-pd-{p,d}-vision.yml 为底,缩到 2+2 卡;P 加 engram host table
|
||||
# (2 卡权重放不下,见 G1.9)。P=GPU4-5 端口 30020,D=GPU6-7 端口 30030,mini_lb=30004。
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30020
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -1,42 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-replay-off
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,42 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-replay-on
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,92 +0,0 @@
|
||||
# 纯 tp2 单实例对照(2026-09-24 早):与 dockerserve-cp2x4.yml 逐参数对齐,唯一差异 = 去掉
|
||||
# --enable-prefill-cp --cp-strategy interleave(即无 CP),用于测「tp2 vs cp2」的 prefill 效率差。
|
||||
# GPU 4-5(a)/6-7(b),端口 30020/30030;2×tp2 聚合经 rr router 30003。
|
||||
# 镜像/补丁挂载/L3/engram/tok8 全部与 cp2x4 相同。
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
# 与 cp2x4 保持同一份 scheduler.py(含未 commit 的 CP idle 补丁),保证唯一变量是 CP 开关
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
environment: &env
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_MS_AUTO_DISC: "0"
|
||||
MOONCAKE_MASTER: 127.0.0.1:50051
|
||||
MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE
|
||||
MOONCAKE_PROTOCOL: tcp
|
||||
MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
tp2:
|
||||
<<: *common
|
||||
container_name: ds41-tp2
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
b:
|
||||
<<: *common
|
||||
container_name: ds41-tp2-b
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30030
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -1,43 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
|
||||
ports:
|
||||
- "30000:30000"
|
||||
|
||||
volumes:
|
||||
- /data:/data
|
||||
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
|
||||
@@ -1,451 +0,0 @@
|
||||
# ds41-3p1d: DeepSeek-V4.1-Flash 3P1D PD 分离(cs32k + OTLP trace),RBG 版
|
||||
# 拓扑: 3×prefill(tp2+ep2+cp2, cs32k, dspark, hicache L3) + 1×decode(tp2+ep2+dp2, cg256 mr320, dspark,
|
||||
# load-balance-method=total_requests 按请求数分流) + mooncake-master(hicache L3 store) + router(mini-lb PD)
|
||||
# 全部钉 b300-01(单机 8 卡,hostNetwork + hostPID——mooncake NVLink-intra 跨 Pod CUDA IPC 必需,
|
||||
# 等同 docker compose 的 pid: host;参考 AGENTS.md「部署四要素」)
|
||||
# 镜像已侧载 containerd: docker.io/ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix(IfNotPresent)
|
||||
# 参数来源: deploy/b300-ds41/dockerserve-3p1d-otlp-cs32k.yml(docker compose 版)
|
||||
# 对外: 仅 ClusterIP Service ds41-3p1d:50002(router 用普通 Pod 网络,不占宿主机端口,
|
||||
# 天然不对宿主机/外网暴露);new-api-gw 用集群 DNS http://ds41-3p1d:50002/v1 访问
|
||||
# trace: OTLP → 127.0.0.1:4317(同节点 jaeger-trace cw,badger 6h 轮窗)
|
||||
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||
kind: RoleBasedGroup
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
app: ds41-3p1d
|
||||
sglang-model: ds-v41-flash
|
||||
spec:
|
||||
roleTemplates:
|
||||
- name: prefill-base
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
sglang-model: ds-v41-flash
|
||||
sglang-role: prefill
|
||||
spec:
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: sglang
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash \
|
||||
--tp 2 --ep-size 2 \
|
||||
--enable-prefill-cp --cp-strategy interleave \
|
||||
--chunked-prefill-size 32768 --max-prefill-tokens 32768 \
|
||||
--mem-fraction-static 0.75 \
|
||||
--attention-backend dsv4 \
|
||||
--moe-runner-backend flashinfer_mxfp4 \
|
||||
--cuda-graph-max-bs-decode 32 \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--max-running-requests 256 \
|
||||
--tokenizer-worker-num 8 \
|
||||
--host 0.0.0.0 --port "$SGLANG_PORT" \
|
||||
--enable-cache-report --enable-metrics \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
|
||||
--enable-hierarchical-cache \
|
||||
--hicache-ratio 2 \
|
||||
--hicache-mem-layout page_first_direct \
|
||||
--hicache-io-backend direct \
|
||||
--hicache-write-policy write_through \
|
||||
--hicache-storage-backend mooncake \
|
||||
--hicache-storage-prefetch-policy wait_complete \
|
||||
--hicache-size 0 \
|
||||
--disaggregation-mode prefill \
|
||||
--disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port "$SGLANG_BOOTSTRAP_PORT" \
|
||||
--enable-trace \
|
||||
--trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 \
|
||||
--otlp-service-name "$OTLP_NAME"
|
||||
env:
|
||||
- name: SGLANG_TORCH_PROFILER_DIR
|
||||
value: /data/ymk/prof/torch
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "1"
|
||||
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||
value: /data/ymk/ds41/mooncake-store.json
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 90
|
||||
resources:
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 220Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 950Gi
|
||||
nvidia.com/gpu: "2"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: sglang-cache
|
||||
mountPath: /root/.cache/sglang
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
- name: sglang-cache
|
||||
hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 32Gi
|
||||
- name: decode-base
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
sglang-model: ds-v41-flash
|
||||
sglang-role: decode
|
||||
spec:
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: sglang
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash \
|
||||
--tp 2 --ep-size 2 --dp-size 2 \
|
||||
--enable-dp-attention --enable-dp-lm-head \
|
||||
--mem-fraction-static 0.83 \
|
||||
--attention-backend dsv4 \
|
||||
--moe-runner-backend flashinfer_mxfp4 \
|
||||
--cuda-graph-max-bs-decode 256 \
|
||||
--load-balance-method total_requests \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--max-running-requests 320 \
|
||||
--tokenizer-worker-num 8 \
|
||||
--host 0.0.0.0 --port 30030 \
|
||||
--enable-cache-report --enable-metrics \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
|
||||
--disaggregation-mode decode \
|
||||
--disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port 9001 \
|
||||
--enable-trace \
|
||||
--trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 \
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
env:
|
||||
- name: SGLANG_TORCH_PROFILER_DIR
|
||||
value: /data/ymk/prof/torch
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "1"
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30030
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 90
|
||||
resources:
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 100Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 600Gi
|
||||
nvidia.com/gpu: "2"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: sglang-cache
|
||||
mountPath: /root/.cache/sglang
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
- name: sglang-cache
|
||||
hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 32Gi
|
||||
roles:
|
||||
- name: mooncake-master
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-mooncake-master
|
||||
spec:
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: mooncake-master
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec /opt/sglang/bin/mooncake_master \
|
||||
--port 50051 \
|
||||
--enable_http_metadata_server=true \
|
||||
--http_metadata_server_port=18080 \
|
||||
--eviction_high_watermark_ratio=0.90
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: 50051
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
failureThreshold: 60
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 8Gi
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: 64Gi
|
||||
- name: p1
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p1
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "0,1"
|
||||
- name: SGLANG_PORT
|
||||
value: "30000"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "8998"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p1
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30000
|
||||
- name: p2
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p2
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "2,3"
|
||||
- name: SGLANG_PORT
|
||||
value: "30010"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "8999"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p2
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30010
|
||||
- name: p3
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p3
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "4,5"
|
||||
- name: SGLANG_PORT
|
||||
value: "30020"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "9000"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p3
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30020
|
||||
- name: d1
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: decode-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-d1
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "6,7"
|
||||
- name: router
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- p1
|
||||
- p2
|
||||
- p3
|
||||
- d1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-router
|
||||
spec:
|
||||
# 入口不走 hostNetwork:普通 Pod 网络 + ClusterIP,天然不对宿主机/外网暴露。
|
||||
# P/D 地址用节点内网 IP(hostNetwork 监听在 192.168.1.243 上,Calico 可达)。
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: router
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang-router launch \
|
||||
--mini-lb --pd-disaggregation \
|
||||
--host 0.0.0.0 --port 50002 \
|
||||
--prefill http://192.168.1.243:30000 8998 \
|
||||
--prefill http://192.168.1.243:30010 8999 \
|
||||
--prefill http://192.168.1.243:30020 9000 \
|
||||
--decode http://192.168.1.243:30030
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 50002
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 30
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 4Gi
|
||||
limits:
|
||||
cpu: "8"
|
||||
memory: 16Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
app: ds41-3p1d
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
rbg.workloads.x-k8s.io/group-name: ds41-3p1d
|
||||
rbg.workloads.x-k8s.io/role-name: router
|
||||
ports:
|
||||
- name: http
|
||||
port: 50002
|
||||
targetPort: 50002
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PodMonitor
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
sglang-model: ds-v41-flash
|
||||
podMetricsEndpoints:
|
||||
- interval: 15s
|
||||
path: /metrics
|
||||
port: metrics
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user