Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4e60127fb4 | ||
|
|
e9fdf3ad0a | ||
|
|
2087760ea8 | ||
|
|
ef1a5955f3 | ||
|
|
7e17dee8b7 | ||
|
|
0090df5d64 | ||
|
|
4938eea2dc | ||
|
|
8c6cbd8c40 | ||
|
|
c82aed9fef | ||
|
|
e6f6420275 | ||
|
|
566176d0cf | ||
|
|
4f52678224 | ||
|
|
068acff09d | ||
|
|
f9056f0fcd | ||
|
|
2aae0cb4ee | ||
|
|
ea5fc832c1 | ||
|
|
9894dc789c | ||
|
|
77e7555ae3 | ||
|
|
319ee3c056 | ||
|
|
e59014be87 | ||
|
|
fd86688428 | ||
|
|
da343ba5d1 | ||
|
|
9ea02a7201 | ||
|
|
6c1c54b514 | ||
|
|
51624199b4 | ||
|
|
31b3758c99 | ||
|
|
740372bef0 | ||
|
|
19fe20baec | ||
|
|
383df91fcc | ||
|
|
534842ac08 | ||
|
|
4dd95a8817 | ||
|
|
656e00dcd3 | ||
|
|
a4b1363bf4 | ||
|
|
ad2acaa2df | ||
|
|
ecaaa37cb0 | ||
|
|
1b70fc192c | ||
|
|
3ebbb2d536 | ||
|
|
5d52cccbee | ||
|
|
74d03143e2 | ||
|
|
2b192aade6 | ||
|
|
9247c97919 | ||
|
|
5e7190b87a | ||
|
|
e00930c548 | ||
|
|
d7ff74b984 | ||
|
|
bd78095030 | ||
|
|
dc1bd46802 | ||
|
|
c7be3e935b | ||
|
|
2ff52e3ceb | ||
|
|
9ae43b0eee | ||
|
|
5781206833 | ||
|
|
069523f999 | ||
|
|
6dad152490 | ||
|
|
49e7bd179d | ||
|
|
de9f850b36 | ||
|
|
701213c867 | ||
|
|
9b89b96230 | ||
|
|
49b5e76e45 | ||
|
|
acffb0d20d | ||
|
|
cb1fa3c399 | ||
|
|
42f58b4da7 | ||
|
|
6f72cc3b64 | ||
|
|
0a8ca85565 | ||
|
|
e49d7fb170 | ||
|
|
d9074f7943 | ||
|
|
6e1df69623 | ||
|
|
77091cea68 | ||
|
|
49d26ff0c2 | ||
|
|
79cafec013 | ||
|
|
b95329a74d | ||
|
|
7889fbab9a | ||
|
|
89e1316eae | ||
|
|
f124c5ceec | ||
|
|
9654e5c4bc | ||
|
|
096b066fb4 | ||
|
|
26d7539799 | ||
|
|
b07cb9e9e7 | ||
|
|
6fa3fe69e2 | ||
|
|
2a1c477648 | ||
|
|
f209e67767 | ||
|
|
6624999385 | ||
|
|
2e7e0802f4 | ||
|
|
8151378c1a | ||
|
|
0318a8d0af | ||
|
|
da3eb6db32 | ||
|
|
bddeb61d94 | ||
|
|
090439bf8f | ||
|
|
6aacca2d91 | ||
|
|
fa443412c5 | ||
|
|
2f88c65289 | ||
|
|
47ad1928a2 | ||
|
|
85be04978d | ||
|
|
6caf0ff4ae | ||
|
|
c02eaadeb2 | ||
|
|
6582425854 | ||
|
|
df378bca42 | ||
|
|
546bfa7221 | ||
|
|
e8eeff2956 | ||
|
|
7ee7bef79d | ||
|
|
6405cf2881 | ||
|
|
f4de6abee6 | ||
|
|
6c4f36e47a | ||
|
|
e75e3b8a98 | ||
|
|
6be9c78cf4 | ||
|
|
34f090053e | ||
|
|
25de80780a | ||
|
|
3d2827236d | ||
|
|
55cc90b533 | ||
|
|
cc012abd21 | ||
|
|
8b2ca8ecc2 | ||
|
|
6cbdfec453 | ||
|
|
0823651b0b | ||
|
|
81f27fb3a7 | ||
|
|
06d012eaca | ||
|
|
341d5273bd | ||
|
|
644014bc98 | ||
|
|
0e907b755d | ||
|
|
427c9e05c1 | ||
|
|
55d2ee5546 | ||
|
|
0971450c16 | ||
|
|
4fac02fe5d | ||
|
|
ecb8e9fc8f | ||
|
|
19eb56fa57 | ||
|
|
31329c222c | ||
|
|
f0ecf15c84 | ||
|
|
8c43c667cb | ||
|
|
6f370c3bcd | ||
|
|
2e7f0bb28c | ||
|
|
c289656434 | ||
|
|
deaa7f1463 | ||
|
|
6f2d2437b6 | ||
|
|
f5ae989a0e | ||
|
|
cad9eaf27d | ||
|
|
1c919e401d | ||
|
|
7a833ad0a2 | ||
|
|
380a4d0332 | ||
|
|
9f87d28db6 | ||
|
|
846181f1c4 | ||
|
|
446d181e23 | ||
|
|
583fada954 | ||
|
|
78eee88113 | ||
|
|
14ab74ef95 | ||
|
|
b252aceffe | ||
|
|
e581520c67 | ||
|
|
d10a09ecf3 | ||
|
|
b2800461e5 | ||
|
|
1ba84a4b59 | ||
|
|
c311bc9611 | ||
|
|
588049fe2e | ||
|
|
84622ce9d5 | ||
|
|
a0ba1968b1 | ||
|
|
f1e62e3a2e | ||
|
|
ae47bcd4da | ||
|
|
2a14f65fd7 | ||
|
|
37556c1b91 | ||
|
|
0bc074db83 | ||
|
|
aa7a976807 | ||
|
|
df1fc6ab64 | ||
|
|
acc15c193b | ||
|
|
38d865489a | ||
|
|
d27efca353 | ||
|
|
b4584ab838 | ||
|
|
effb752188 | ||
|
|
425a1f8f24 | ||
|
|
fc9e1c8d29 | ||
|
|
c10fa03fc4 | ||
|
|
c73f7077eb | ||
|
|
7bdd8fe6ec | ||
|
|
e1e97e8b49 | ||
|
|
cdbea5dccf | ||
|
|
710a41ba57 | ||
|
|
a947b56bc7 | ||
|
|
bd37e7f513 | ||
|
|
9cd6616c24 | ||
|
|
d3fcf8997c | ||
|
|
35b369ebba | ||
|
|
c081d4aacc | ||
|
|
8fc3ce48da | ||
|
|
7bc988446d | ||
|
|
a5c8afab48 | ||
|
|
3ed56a341b | ||
|
|
4bb645b8e2 | ||
|
|
8772916e06 | ||
|
|
7152aabdcb | ||
|
|
38ec649048 | ||
|
|
e538489e0e | ||
|
|
dfdbff931d | ||
|
|
3ecb6cd729 | ||
|
|
0e2aac500b | ||
|
|
f35ec18d8b | ||
|
|
4273e7de77 | ||
|
|
63d320c723 | ||
|
|
2ee82fe158 | ||
|
|
0967a013c2 | ||
|
|
85374532f2 | ||
|
|
c842d32b0a | ||
|
|
50e50e13ab | ||
|
|
1f6ce4b068 | ||
|
|
2afd343934 | ||
|
|
efd9a40bb8 | ||
|
|
95047d3464 | ||
|
|
516ab77619 | ||
|
|
27e883a20d | ||
|
|
c421d16563 | ||
|
|
cf5df82680 | ||
|
|
d3e3945e41 | ||
|
|
f88572a57c | ||
|
|
f337f01b80 | ||
|
|
ac62ddf164 | ||
|
|
24b6930076 | ||
|
|
6d35cee7fc | ||
|
|
671000aee6 | ||
|
|
31b931edc3 | ||
|
|
0154f72b48 | ||
|
|
67bb6a58d0 | ||
|
|
2f5c9ac43d | ||
|
|
a749d84303 | ||
|
|
62122c838a | ||
|
|
3450d68d4e | ||
|
|
26a3214cda | ||
|
|
8f5a636f8d | ||
|
|
b7f6d04a9a | ||
|
|
963e9fb42d | ||
|
|
9d7f44bdbb | ||
|
|
402df23188 | ||
|
|
1409f469be | ||
|
|
51c92e8df1 | ||
|
|
515f5be77e | ||
|
|
434c2e3adc | ||
|
|
cbe137727e | ||
|
|
0fb699ac03 | ||
|
|
9c8340a2e5 | ||
|
|
ec070ec8c8 | ||
|
|
3e7e652900 | ||
|
|
16d1c93b3d | ||
|
|
cec70a4708 | ||
|
|
3fc7a669bf | ||
|
|
e0b4d66733 | ||
|
|
7325b38412 | ||
|
|
37ebcac3c4 | ||
|
|
6103a8c743 | ||
|
|
9dc4c5d891 | ||
|
|
8ca82118e0 | ||
|
|
a2025b8c4d | ||
|
|
2f2f9d12f8 | ||
|
|
7c5bdb940c | ||
|
|
466e985f4b | ||
|
|
e047e50d4e | ||
|
|
3ec86307f2 | ||
|
|
6bbd689ab4 | ||
|
|
0c578d97fe | ||
|
|
a1eb691ab5 | ||
|
|
182f62d2de | ||
|
|
961404b010 | ||
|
|
36f59982fa | ||
|
|
0b53305f48 | ||
|
|
b129504f0e | ||
|
|
1446e24d13 | ||
|
|
5bb850d536 | ||
|
|
a80055d3c6 | ||
|
|
7c9f74c6e1 | ||
|
|
62ae032c46 | ||
|
|
5df1667077 | ||
|
|
07a3735e94 | ||
|
|
928e683f94 | ||
|
|
6a14b80141 | ||
|
|
1b03d31b31 | ||
|
|
f404db94d4 | ||
|
|
86b3558b41 | ||
|
|
78b382b1ad | ||
|
|
984994e3aa | ||
|
|
5bb24e399d | ||
|
|
8ec65e89c1 | ||
|
|
77173ffb08 | ||
|
|
752801e4d1 | ||
|
|
17a7484c06 | ||
|
|
cd11037009 | ||
|
|
4142235c2b | ||
|
|
ec7eb6bd79 | ||
|
|
ea5baf4022 | ||
|
|
28ec6700da | ||
|
|
3177d10ca6 | ||
|
|
ce06a14444 | ||
|
|
c6565c1a6d | ||
|
|
a69583f70f | ||
|
|
0a59830114 | ||
|
|
26c63335e2 | ||
|
|
d94d784441 | ||
|
|
bb1c98baa2 | ||
|
|
c85df5becc | ||
|
|
261cb826e6 | ||
|
|
174a5f37b9 | ||
|
|
41812afe43 | ||
|
|
81f2b43ab5 | ||
|
|
d65503fece | ||
|
|
1417345f5f | ||
|
|
cae4927fbf | ||
|
|
a164c6d9f1 | ||
|
|
43af9fcbb6 | ||
|
|
9f6fc55332 | ||
|
|
32290dda2c | ||
|
|
90663ccb41 | ||
|
|
efff836911 | ||
|
|
7a9feacf91 | ||
|
|
0dc8b29e15 | ||
|
|
c3685dff42 | ||
|
|
8eedf616f5 | ||
|
|
be495a6174 | ||
|
|
e2f4fedf04 | ||
|
|
77983865d8 | ||
|
|
5c44214d6a | ||
|
|
f4d9d2e670 | ||
|
|
f4b90382a2 | ||
|
|
cc6e8b0f25 | ||
|
|
e98b2f7538 | ||
|
|
8b5d77c268 | ||
|
|
82cdd72588 | ||
|
|
03fcbe147f | ||
|
|
81b81664a6 | ||
|
|
621136e094 | ||
|
|
81cb895b85 | ||
|
|
ec75d3d30f | ||
|
|
ffac53d779 | ||
|
|
542a043f1b | ||
|
|
208f6f7501 | ||
|
|
3f697899a3 | ||
|
|
0010f56a0d | ||
|
|
6fe4b66f5c | ||
|
|
79fec592a8 | ||
|
|
f2a1366584 | ||
|
|
4fa2c9c1de | ||
|
|
f766397f2e | ||
|
|
c39a1c0663 | ||
|
|
ab0c31bd94 | ||
|
|
e1048a215c | ||
|
|
7dd9640752 | ||
|
|
e31cbc01a2 | ||
|
|
701cf7e47e | ||
|
|
379e8f916b | ||
|
|
9ebe42238b | ||
|
|
0e80c73c92 | ||
|
|
954458567e | ||
|
|
2909f84e3b | ||
|
|
890a9605ce | ||
|
|
4bb5611dba | ||
|
|
f2f223e697 | ||
|
|
f2eebd5533 | ||
|
|
7fef014d51 | ||
|
|
5c154c214d | ||
|
|
c3e985203a | ||
|
|
8993f790be | ||
|
|
4e60d70ba4 | ||
|
|
172b1b4825 | ||
|
|
16e353d93e | ||
|
|
3fdd63a562 | ||
|
|
abef3efb64 | ||
|
|
2d25767759 | ||
|
|
58f622ebc4 | ||
|
|
4cd63da996 | ||
|
|
a89f849158 | ||
|
|
401d5aedf3 | ||
|
|
de123f38bb | ||
|
|
aa0b65c7c7 | ||
|
|
48c3854620 | ||
|
|
d4dcce12d4 | ||
|
|
0b0f947265 | ||
|
|
86cb4a0f97 | ||
|
|
abca3b2e52 | ||
|
|
eb5e8c84be | ||
|
|
aa0feca536 | ||
|
|
66ce8c55cc | ||
|
|
a3d11f9d95 | ||
|
|
28be39f72e | ||
|
|
58988bee2f | ||
|
|
5b8d8b2f88 | ||
|
|
ced0a2c98c | ||
|
|
9d267655e3 | ||
|
|
973fb44471 | ||
|
|
1d59ce7c90 | ||
|
|
525f14040d | ||
|
|
224a247d5b | ||
|
|
c2bc456b64 | ||
|
|
2bc435d32e | ||
|
|
2e3568567b | ||
|
|
40048f6e51 | ||
|
|
78980a3b0b | ||
|
|
0cd8be351d | ||
|
|
06008c170d | ||
|
|
4ce23542bf | ||
|
|
7b977ce5dc | ||
|
|
d34f7b2326 | ||
|
|
19ee4b566b | ||
|
|
b77833c504 | ||
|
|
c19dc43cc7 | ||
|
|
01275aab6b | ||
|
|
9a53f75124 | ||
|
|
d72629e4ba | ||
|
|
c08ffef709 | ||
|
|
542c817bfa | ||
|
|
3afdde5f05 | ||
|
|
077c319914 | ||
|
|
d6cc283da7 | ||
|
|
4cbf290fb9 | ||
|
|
720617bb5b | ||
|
|
5c18780c31 | ||
|
|
db73f35f4a | ||
|
|
6f4c2b9b91 | ||
|
|
91c329cc86 | ||
|
|
66454d704b | ||
|
|
b4701d672d | ||
|
|
8ab21c8a94 | ||
|
|
a7b96fdad7 | ||
|
|
6fd98c98b9 | ||
|
|
c79510cc2a | ||
|
|
4a1b69abc8 | ||
|
|
244db08d60 | ||
|
|
bc3a63ebdb | ||
|
|
ddebc52f23 | ||
|
|
2524b61f9b | ||
|
|
861b11f087 | ||
|
|
cb8dab06be | ||
|
|
8ef6d31cb9 | ||
|
|
9d58189c12 | ||
|
|
debbb5cde9 | ||
|
|
3f00fb7e2e | ||
|
|
70a2bb2a30 | ||
|
|
790551c382 | ||
|
|
c2f14bfbc4 | ||
|
|
367e3700cf | ||
|
|
d00adfd19c | ||
|
|
6412ad8c64 | ||
|
|
771c9d782d | ||
|
|
1815e490de | ||
|
|
c948114d58 |
@@ -0,0 +1,21 @@
|
||||
# Serving a model? Read its cookbook page first
|
||||
|
||||
Before running, deploying, benchmarking, or reproducing a specific model, read
|
||||
its page under `docs/cookbook/`:
|
||||
|
||||
- `docs/cookbook/autoregressive/` — LLM and VLM serving
|
||||
- `docs/cookbook/diffusion/` — image, video, 3D, robotics
|
||||
- `docs/cookbook/omni/`, `docs/cookbook/vla/`, `docs/cookbook/specbundle/`,
|
||||
`docs/cookbook/base/`
|
||||
|
||||
That page carries the deployment we recommend to users — GPU count, parallelism
|
||||
degrees, quantization, and the flags that matter for that model — so it is the
|
||||
answer to "how should this be served", and the baseline any tuning starts from.
|
||||
|
||||
Configs under `test/` are a different thing: pinned, reproducible CI setups for
|
||||
correctness and regression checks, not deployment advice. Reach for them when
|
||||
you need an exactly reproducible run, and say which cookbook flags you diverged
|
||||
from — a measurement taken on a configuration nobody deploys describes nothing.
|
||||
|
||||
Keep it true in both directions: when a change alters a model's recommended
|
||||
deployment, update that model's cookbook page in the same PR.
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
paths:
|
||||
- "**/bench_*.py"
|
||||
- "**/benchmark/**/*.py"
|
||||
- "benchmark/kernels/**"
|
||||
---
|
||||
|
||||
# Kernel benchmarks
|
||||
|
||||
How a benchmark is written is up to you. The requirement is only that the numbers are valid.
|
||||
|
||||
## Tools
|
||||
|
||||
In SGLang we recommend `marker.do_bench` (`python/sglang/kernels/jit/benchmark/marker.py`). It works for any callable, not only JIT kernels, and it measures with a cold L2 by default. It is not mandatory. CUPTI (e.g. CUDA 13 `cupti-python`), the torch profiler / `bench_kineto`, Triton's `do_bench` / proton, `flashinfer.testing`, Nsight Systems / Nsight Compute, or a hand-written loop are all fine. Each pitfall below must be handled, whichever tool you use.
|
||||
|
||||
## L2 cache reuse
|
||||
|
||||
Timing the same kernel on the same inputs in a loop measures an L2-hot kernel, not the one serving runs. `triton.testing.do_bench_cudagraph` with fixed arguments and bare `for` loops between two events both do this. Measure cold L2 by flushing between calls or by rotating enough input copies to exceed L2 several times over. Hot L2 is acceptable only when the working set far exceeds L2 or when hot L2 is the real serving pattern, and the reason must be stated.
|
||||
|
||||
Host-memory (zero-copy) loads do not go through device L2. Peer-GPU (NVLink) reads still see reuse (on remote GPU).
|
||||
@@ -1,10 +1,14 @@
|
||||
---
|
||||
name: add-jit-kernel
|
||||
description: Step-by-step tutorial for adding a new lightweight JIT CUDA kernel to sglang's jit_kernel module
|
||||
description: Step-by-step tutorial for adding a new lightweight JIT CUDA kernel to sglang.kernels JIT infrastructure and public operator groups
|
||||
---
|
||||
|
||||
# Tutorial: Adding a New JIT Kernel to SGLang
|
||||
|
||||
Apply [kernel-organization](../kernel-organization/SKILL.md) for the public
|
||||
operator namespace, logical grouping, lazy registry metadata, and test placement.
|
||||
The implementation tutorial below does not replace that API contract.
|
||||
|
||||
This tutorial walks through adding a simple element-wise scale operation as a JIT kernel. We'll implement `scale(x, factor) = x * factor` to demonstrate the complete workflow.
|
||||
|
||||
## Goal
|
||||
@@ -497,6 +501,32 @@ def scale(src: torch.Tensor, factor: float, out: torch.Tensor | None = None) ->
|
||||
|
||||
---
|
||||
|
||||
## Register the public entry point (required)
|
||||
|
||||
Add metadata in `python/sglang/kernels/ops/elementwise/__init__.py` so the
|
||||
operator appears in the registry without importing its implementation:
|
||||
|
||||
```python
|
||||
from sglang.kernels.registry import register_kernel
|
||||
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="elementwise.scale",
|
||||
backend=KernelBackend.JIT,
|
||||
target="sglang.kernels.ops.elementwise.scale:scale",
|
||||
capabilities=frozenset({CapabilityRequirement.CUDA}),
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
Record the devices the implementation actually supports; the CUDA-only scale
|
||||
example above does not imply that every JIT kernel is CUDA-only. Runtime and
|
||||
correctness tests import `scale` from `sglang.kernels.ops.elementwise.scale`.
|
||||
Do not import the implementation eagerly in the group initializer. When an
|
||||
existing `BaseFusedOp` owns the operation, add the implementation there instead
|
||||
of registering a second conflicting op/backend pair.
|
||||
|
||||
## Step 3 (optional): Tune JIT build flags
|
||||
|
||||
If your kernel uses some math functions like `expf` or `sinf`, consider enabling `--use_fast_math` for better performance (with a potential precision tradeoff):
|
||||
@@ -611,6 +641,8 @@ if __name__ == "__main__":
|
||||
|
||||
Benchmarks are `bench_*.py` files under `test/registered/kernels/benchmark/<group>/`. They are picked up by the same `run_suite.py` machinery as unit tests. Register them for **`base-b-kernel-benchmark-test-1-gpu-large`** (PR JIT benchmark job: `python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1-gpu-large`).
|
||||
|
||||
Every benchmark must account for L2 cache reuse — see [`rules/kernel-benchmark.md`](../../rules/kernel-benchmark.md).
|
||||
|
||||
Benchmarks use the project's own `marker` framework (in `python/sglang/kernels/jit/benchmark/marker.py`) — **do not** use `triton.testing.perf_report` / `triton.testing.do_bench` directly. The marker framework provides (public names: `benchmark`, `parametrize`, `do_bench`, `skip`, `BenchResult`, `BenchSkip`):
|
||||
|
||||
- **`@marker.benchmark(line_arg, line_vals, *, unit="us")`** — the **innermost** decorator (bottom of the stack, directly above `def benchmark`). Declares the column axis: each value in `line_vals` becomes a result column, and `line_arg` is the parameter name passed into the benchmark function. `unit` is one of `"us" | "ms" | "s"`.
|
||||
@@ -619,7 +651,7 @@ Benchmarks use the project's own `marker` framework (in `python/sglang/kernels/j
|
||||
- `memory_args`: defaults to `"all"` (footprint derived from all input args/kwargs). Pass an explicit tuple of tensors (e.g. `(k, v, indices)`) to count only the inputs the kernel actually touches.
|
||||
- `memory_output`: defaults to `"out"` — re-runs `fn` once to capture its **returned** tensor and counts it. For in-place kernels (which return `None`), pass the written tensors explicitly (e.g. `memory_output=(k, v)`); the re-run is then skipped. Set to `None` to count no output.
|
||||
- Together `memory_args` + `memory_output` give the GB/s column; with both defaults a function `out = f(src)` already reports `bytes(src) + bytes(out)`.
|
||||
- `graph_clone_args` / `graph_clone_kwargs`: which inputs to clone per CUDA-graph iteration to defeat L2 cache reuse. Defaults to `"all"` — pass an iterable of indices/keys to limit to the *read* args (writes don't need cloning).
|
||||
- `graph_clone_args` / `graph_clone_kwargs`: which inputs to clone per CUDA-graph iteration to defeat L2 cache reuse. Defaults to `"all"`; if you narrow it, keep every read, in-place, and output tensor.
|
||||
- `use_cuda_graph=False` for kernels that can't be captured.
|
||||
- `metrics=(0.5, "avg")` controls reported quantiles (the first metric becomes the table latency column).
|
||||
- `disable_log_bandwidth` (defaults from `SGLANG_JIT_BENCHMARK_DISABLE_LOG_BANDWIDTH=1`) skips the bandwidth column entirely.
|
||||
@@ -724,7 +756,7 @@ cd test && python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1
|
||||
- `python/sglang/kernels/jit/include/sgl_kernel/cta.cuh` — `cta::reduce_max`
|
||||
- `python/sglang/kernels/jit/include/sgl_kernel/atomic.cuh` — `atomic::max`
|
||||
- `python/sglang/kernels/jit/include/sgl_kernel/runtime.cuh` — occupancy / SM count helpers
|
||||
- `python/sglang/kernels/jit/csrc/add_constant.cuh` — minimal runnable reference
|
||||
- `python/sglang/kernels/jit/csrc/elementwise/add_constant.cuh` — minimal runnable reference
|
||||
- `python/sglang/kernels/jit/csrc/elementwise/rmsnorm.cuh` — real example using `TensorMatcher` + `LaunchKernel` + `tile::Memory`
|
||||
- `python/sglang/kernels/jit/csrc/elementwise/qknorm.cuh` — real example using `runtime::get_blocks_per_sm` + persistent kernel pattern
|
||||
- `python/sglang/kernels/jit/benchmark/marker.py` — `benchmark`, `parametrize`, `do_bench`, `BenchResult`
|
||||
|
||||
@@ -5,6 +5,10 @@ description: Step-by-step tutorial for adding a heavyweight AOT CUDA/C++ kernel
|
||||
|
||||
# Tutorial: Adding a New Kernel to `sgl-kernel` (AOT / Heavyweight)
|
||||
|
||||
Apply [kernel-organization](../kernel-organization/SKILL.md) for the public
|
||||
operator namespace, logical grouping, lazy registry metadata, and test placement.
|
||||
The implementation tutorial below does not replace that API contract.
|
||||
|
||||
This tutorial walks through adding a simple element-wise scale operation as an AOT kernel. We'll implement `scale(x, factor) = x * factor` to demonstrate the complete workflow.
|
||||
|
||||
## Goal
|
||||
@@ -25,7 +29,7 @@ Add a new operation that scales each element of a tensor by a scalar factor:
|
||||
In addition, every new kernel must ship with:
|
||||
|
||||
- **Tests** (pytest)
|
||||
- **A benchmark script** (triton.testing)
|
||||
- **A benchmark script** (`marker.do_bench` from `sglang.kernels.jit.benchmark` — it works for any callable, not only JIT kernels)
|
||||
|
||||
---
|
||||
|
||||
@@ -194,6 +198,21 @@ Then re-export it from `python/sglang/kernels/aot/python/sgl_kernel/__init__.py`
|
||||
|
||||
---
|
||||
|
||||
## SGLang integration (required for runtime use)
|
||||
|
||||
After exposing the AOT wheel symbol, add a lazy wrapper and a `KernelSpec` under
|
||||
`python/sglang/kernels/ops/<group>/`. Set `backend=KernelBackend.AOT`, record
|
||||
actual device/architecture support, and keep `sgl_kernel` imports inside the
|
||||
implementation path. SGLang runtime and integration tests import that wrapper.
|
||||
For this example, use `sglang.kernels.ops.elementwise.scale`.
|
||||
|
||||
The wheel-level tests below validate its standalone API/build. They do not
|
||||
replace CI-registered SGLang correctness tests in
|
||||
`test/registered/kernels/ops/elementwise/test_scale.py` and benchmarks in
|
||||
`test/registered/kernels/benchmark/elementwise/bench_scale.py`. Follow
|
||||
`write-sglang-test` for their registration and CI budget; do not register tests
|
||||
under the shipped `python/sglang/` package.
|
||||
|
||||
## Step 6: Write tests (required)
|
||||
|
||||
Create `python/sglang/kernels/aot/tests/test_scale.py`:
|
||||
@@ -241,62 +260,46 @@ if __name__ == "__main__":
|
||||
|
||||
## Step 7: Add a benchmark (required)
|
||||
|
||||
Every benchmark must account for L2 cache reuse — see [`rules/kernel-benchmark.md`](../../rules/kernel-benchmark.md).
|
||||
|
||||
Create `python/sglang/kernels/aot/benchmark/bench_scale.py`:
|
||||
|
||||
```python
|
||||
import itertools
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.testing
|
||||
|
||||
import sgl_kernel
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
IS_CI = is_in_ci()
|
||||
|
||||
dtypes = [torch.float16] if IS_CI else [torch.float16, torch.bfloat16, torch.float32]
|
||||
sizes = [4096] if IS_CI else [2**n for n in range(10, 20)] # 1K … 512K
|
||||
factors = [2.0]
|
||||
|
||||
configs = list(itertools.product(dtypes, sizes))
|
||||
from sglang.kernels.jit.benchmark import marker
|
||||
|
||||
|
||||
def torch_scale(input: torch.Tensor, factor: float) -> torch.Tensor:
|
||||
return input * factor
|
||||
def sglang_scale(input: torch.Tensor, factor: float, out: torch.Tensor) -> None:
|
||||
sgl_kernel.scale(input, factor, out=out)
|
||||
|
||||
|
||||
@triton.testing.perf_report(
|
||||
triton.testing.Benchmark(
|
||||
x_names=["dtype", "size"],
|
||||
x_vals=configs,
|
||||
line_arg="provider",
|
||||
line_vals=["sglang", "torch"],
|
||||
line_names=["SGL Kernel", "PyTorch"],
|
||||
styles=[("green", "-"), ("red", "--")],
|
||||
ylabel="µs (median)",
|
||||
plot_name="scale-performance",
|
||||
args={},
|
||||
def torch_scale(input: torch.Tensor, factor: float, out: torch.Tensor) -> None:
|
||||
torch.mul(input, factor, out=out)
|
||||
|
||||
|
||||
FN_MAP = {"sglang": sglang_scale, "torch": torch_scale}
|
||||
|
||||
|
||||
@marker.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32], [torch.float16])
|
||||
@marker.parametrize("size", [2**n for n in range(10, 20)], [4096]) # 1K .. 512K
|
||||
@marker.benchmark("provider", ["sglang", "torch"])
|
||||
def benchmark(dtype: torch.dtype, size: int, provider: str):
|
||||
input = torch.randn(size, dtype=dtype, device="cuda")
|
||||
out = torch.empty_like(input)
|
||||
return marker.do_bench(
|
||||
FN_MAP[provider],
|
||||
# Pass every tensor through input_args (not a closure) so marker rotates
|
||||
# them across CUDA-graph calls to defeat L2 reuse.
|
||||
input_args=(input, 2.0, out),
|
||||
# Bandwidth = bytes(input) + bytes(out), both already in input_args.
|
||||
memory_output=None,
|
||||
)
|
||||
)
|
||||
def benchmark(dtype, size, provider):
|
||||
input = torch.randn(size, dtype=dtype, device="cuda")
|
||||
out = torch.empty_like(input)
|
||||
factor = 2.0
|
||||
|
||||
if provider == "sglang":
|
||||
fn = lambda: sgl_kernel.scale(input, factor, out=out)
|
||||
else:
|
||||
fn = lambda: torch_scale(input, factor)
|
||||
|
||||
ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(
|
||||
fn, quantiles=[0.5, 0.2, 0.8]
|
||||
)
|
||||
return 1000 * ms, 1000 * max_ms, 1000 * min_ms
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
benchmark.run(print_data=True)
|
||||
benchmark.run()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
@@ -136,8 +136,8 @@ than restating.
|
||||
= `output tok/s/GPU × (isl+osl)/osl`, shown by the card as-is. Flag output-only values.
|
||||
- **Consistent accuracy harness across entries**: every value under one `accuracyLabels`
|
||||
column must be produced by the SAME harness — flag a page that, say, measures one
|
||||
platform's GSM8K with `few_shot_gsm8k --num-questions 200` and another's with
|
||||
`run_eval --eval-name gsm8k --num-examples 1319` and shows both as one "GSM8K %"
|
||||
platform's GSM8K with a historical raw completion harness and another's with
|
||||
`sgl-eval run gsm8k --num-examples 1319` and shows both as one "GSM8K %"
|
||||
(the scores aren't comparable). Either standardize on one harness (matching
|
||||
`benchmarkCommands.accuracy`) or require an explicit per-entry note. Common when folding
|
||||
a second contributor's measurements (e.g. an AMD/ROCm PR) into the page.
|
||||
|
||||
@@ -162,7 +162,7 @@ Picking between `ENABLE_FOO` and `DISABLE_FOO`: both verbs are valid. The only f
|
||||
For a rename where the old key must keep working with a warning:
|
||||
|
||||
```python
|
||||
SGLANG_DSA_FUSE_TOPK = EnvBoolWithAlias(True, deprecated_name="SGLANG_NSA_FUSE_TOPK")
|
||||
SGLANG_NEW_NAME = EnvBoolWithAlias(True, deprecated_name="SGLANG_OLD_NAME")
|
||||
```
|
||||
|
||||
Use `EnvBoolWithAlias` / `EnvIntWithAlias`. The fallback emits a `DeprecationWarning` and copies the old value over.
|
||||
|
||||
@@ -46,13 +46,11 @@ Typical startup time: 30-90 seconds depending on model size and whether CUDA gra
|
||||
### Step 3: Validate accuracy (sanity check)
|
||||
|
||||
```bash
|
||||
python3 -m sglang.test.run_eval --host 127.0.0.1 --port <port> --eval-name gsm8k --num-examples 20
|
||||
sgl-eval run gsm8k --base-url http://127.0.0.1:<port>/v1 --num-examples 20
|
||||
```
|
||||
|
||||
- Expected accuracy: **> 0.8** for capable models (Qwen3-8B, Llama-3.1-8B-Instruct, etc.)
|
||||
- This is a quick sanity check, not a rigorous benchmark.
|
||||
- `sglang.test.few_shot_gsm8k` is deprecated; use the unified `run_eval` entrypoint.
|
||||
- If you intentionally need the old completion-style GSM8K path, add `--api completion`.
|
||||
- If accuracy is unexpectedly low, something is wrong — do not proceed to profiling.
|
||||
|
||||
### Step 4: Generate the profile
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
---
|
||||
name: kernel-organization
|
||||
description: Apply the SGLang kernels RFC when adding, moving, splitting, or reviewing kernel APIs, registry metadata, kernel tests, benchmarks, and model-specific implementations. Use with add-jit-kernel, add-sgl-kernel, and write-sglang-test for placement and migration checks.
|
||||
---
|
||||
|
||||
# Kernel organization
|
||||
|
||||
Read `python/sglang/kernels/README.md` and `test/README.md` before choosing a
|
||||
location. [RFC #29630](https://github.com/sgl-project/sglang/issues/29630)
|
||||
establishes the namespace; subsequent migrations #32148 and #40922 clarify
|
||||
logical grouping and remove the old `_jit_` filename prefix.
|
||||
|
||||
## Choose the owner
|
||||
|
||||
- Put callable SGLang kernel APIs under `sglang.kernels.ops.<group>`. Runtime
|
||||
and integration tests import from that namespace, including its submodules.
|
||||
The `sgl_kernel` wheel retains its own public API and packaging tests.
|
||||
- Group by computation, not model or GPU: GEMM and GEMV in `gemm`, expert
|
||||
routing in `moe`, attention index selection in `attention`, sampling in
|
||||
`sampling`, normalization in `layernorm`. A quantized GEMM is still a GEMM;
|
||||
quantization alone belongs in `quantization`.
|
||||
- Model-specific files/subpackages and tuning data are allowed **inside** a
|
||||
logical group. Do not add a model bundle or an implementation file directly
|
||||
under `ops/`. Propose a new logical group only for a distinct responsibility,
|
||||
and update `ops._GROUPS`, documentation, and tests together.
|
||||
- Classify a fused operator by its complete contract. Do not split a fused
|
||||
kernel into separate launches just to separate norm, RoPE, or quantization.
|
||||
Split unrelated public entry points that happen to share a source file.
|
||||
- Keep shared CUDA build/runtime infrastructure in `kernels/jit`; operator
|
||||
wrappers call it from their logical group. Keep process groups, communicator
|
||||
state, model dispatch, and buffer ownership in `srt`. K3-specific adapters in
|
||||
`srt/layers/communication/` need not pretend to be generic interfaces.
|
||||
|
||||
## Register the public entry point
|
||||
|
||||
Add lazy `KernelSpec` metadata in the owning group's `__init__.py`, or use the
|
||||
existing `BaseFusedOp` registration when the operation has interchangeable
|
||||
backends. Group imports must not import GPU implementations or compile kernels.
|
||||
|
||||
Use `<group>.<name>` for the op id and `module:callable` for the target. Preserve
|
||||
existing backend dispatch and describe actual device/architecture restrictions
|
||||
with `CapabilityRequirement`; JIT/AOT are not device types. Input shape/dtype
|
||||
checks remain part of the entry point's contract. Registering a torch custom op
|
||||
does not register it in the SGLang kernel inventory. Predicates, private JIT
|
||||
factories, reference helpers, and runtime classes are not separate kernel APIs.
|
||||
|
||||
## Place and preserve tests
|
||||
|
||||
- Kernel numerical tests: `test/registered/kernels/ops/<group>/`.
|
||||
- CI microbenchmarks: `test/registered/kernels/benchmark/<group>/`.
|
||||
- Inventory/selector tests: `test/registered/unit/kernels/`.
|
||||
- Runtime unit tests: `test/registered/unit/<subsystem>/`, following the tested
|
||||
runtime module. A CUDA allocation does not make a runtime test a kernel test.
|
||||
- Non-CI smoke scripts and benchmarks: `test/manual/kernels/`; do not leave
|
||||
standalone test/benchmark entry points in production operator modules.
|
||||
- Shared test helpers: `sglang.test.kernels`. Preserve vendored upstream trees
|
||||
and AOT wheel packaging boundaries instead of reorganizing them incidentally.
|
||||
|
||||
For a move/split, preserve assertions, parametrization, fixtures, platform skips,
|
||||
execution entry points, and CI stages/runners. Apportion existing time estimates
|
||||
across split files; do not duplicate the original budget for every output file
|
||||
or silently drop a registration. Follow `write-sglang-test` for CI registration.
|
||||
|
||||
## Verify a migration
|
||||
|
||||
1. Search all runtime, test, benchmark, documentation, patch-string, and lazy
|
||||
registry references before deleting the old path. Check relative imports and
|
||||
package re-exports as well as direct imports. Do not leave forwarding shims.
|
||||
2. Keep tuning files with the GEMM that loads them and preserve relative lookup
|
||||
behavior. Check wheel/package inclusion as well as source-tree execution.
|
||||
3. Preserve module singleton state: every runtime caller must import the same
|
||||
new communication adapter, not a second copy of its buffer registry.
|
||||
4. Separate relocation commits from semantic changes. Use
|
||||
`mechanical-refactor-verify` to reproduce moves, and compare test inventories
|
||||
and CI registration coverage before/after. Only delete an experimental path
|
||||
after checking its call sites, flags, source, and dedicated tests together.
|
||||
5. Run the namespace/dispatch CPU tests, the registered-test validation hook,
|
||||
pre-commit, and relevant GPU tests when available. State exactly which GPU
|
||||
checks ran; import/AST checks do not prove numerical or performance parity.
|
||||
|
||||
Do not infer violations solely from a model name inside a group, a missing
|
||||
`_jit_` prefix, use of a runtime utility, or absence of `BaseFusedOp` inheritance.
|
||||
@@ -46,8 +46,8 @@ in-flight row as shipped.
|
||||
| Pattern | Trace keywords | Primary code | Existing path | Skill should conclude |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Fused residual add + RMSNorm | `fused_add_rmsnorm*`<br>`npu_add_rms_norm`<br>`add_rmsnorm_bias`<br>`gemma_fused_add_rmsnorm`<br>`gemma_rmsnorm_residual_scalar`<br>`_gemma_rmsnorm_residual_kernel`<br>residual add right before norm | `python/sglang/srt/layers/layernorm.py`<br>`python/sglang/srt/layers/gemma4_fused_ops.py`<br>`python/sglang/srt/layers/quantization/modelslim/modelslim.py` | Shared CUDA / ROCm / CPU / NPU fused add-RMSNorm implementations, including Gemma, Gemma4 scalar-residual, and NPU-bias variants | Treat split residual add + RMSNorm as an existing cross-backend fusion first, not a new idea. |
|
||||
| FlashInfer unified `allreduce_fusion` | `cross_device_reduce_1stage*`<br>`all_reduce`<br>`FusedAddRMSNormKernel`<br>`rmsnorm*` | `python/sglang/srt/layers/flashinfer_comm_fusion.py`<br>`python/sglang/srt/layers/layernorm.py::forward_with_allreduce_fusion`<br>`python/sglang/srt/layers/communicator.py::apply_flashinfer_allreduce_fusion` | FlashInfer workspace creation plus `allreduce_fusion(..., pattern=AllReduceFusionPattern.kARResidualRMSNorm, ...)` | First suspect missing / disabled / unsupported FlashInfer allreduce fusion, not a brand new TP fusion idea. |
|
||||
| AITER allreduce fusion | ROCm all-reduce plus RMSNorm still split | `python/sglang/srt/layers/layernorm.py::forward_with_allreduce_fusion`<br>`python/sglang/srt/distributed/communication_op.py::tensor_model_parallel_fused_allreduce_rmsnorm`<br>`python/sglang/srt/layers/communicator.py::apply_aiter_all_reduce_fusion` | ROCm-side fused TP all-reduce + RMSNorm with fallback to plain all-reduce plus norm | On AMD, rule out existing AITER fusion before proposing a new communication fusion. |
|
||||
| FlashInfer unified `allreduce_fusion` | `cross_device_reduce_1stage*`<br>`all_reduce`<br>`FusedAddRMSNormKernel`<br>`rmsnorm*` | `python/sglang/srt/layers/flashinfer_comm_fusion.py`<br>`python/sglang/srt/layers/layernorm.py::forward_with_allreduce_fusion`<br>`python/sglang/srt/layers/layer_boundary/residual/add_norm.py::apply_flashinfer_allreduce_fusion` | FlashInfer workspace creation plus `allreduce_fusion(..., pattern=AllReduceFusionPattern.kARResidualRMSNorm, ...)` | First suspect missing / disabled / unsupported FlashInfer allreduce fusion, not a brand new TP fusion idea. |
|
||||
| AITER allreduce fusion | ROCm all-reduce plus RMSNorm still split | `python/sglang/srt/layers/layernorm.py::forward_with_allreduce_fusion`<br>`python/sglang/srt/distributed/communication_op.py::tensor_model_parallel_fused_allreduce_rmsnorm`<br>`python/sglang/srt/layers/layer_boundary/residual/add_norm.py::apply_aiter_all_reduce_fusion` | ROCm-side fused TP all-reduce + RMSNorm with fallback to plain all-reduce plus norm | On AMD, rule out existing AITER fusion before proposing a new communication fusion. |
|
||||
| Fused activation-and-mul (`SwiGLU` / `GeGLU`) | `silu_and_mul`<br>`gelu_and_mul`<br>`npu_swiglu` | `python/sglang/srt/layers/activation.py` | Single op covers activation plus elementwise multiply across CUDA / CPU / NPU / XPU backends | Treat separate activation + mul on packed MLP outputs as missing existing fusion. |
|
||||
| Fused dual residual RMSNorm | residual add plus two RMSNorm-like kernels around Grok blocks | `python/sglang/srt/layers/elementwise.py::fused_dual_residual_rmsnorm`<br>`python/sglang/srt/models/grok.py` | One Triton kernel computes intermediate residual update and next RMSNorm output together | On Grok-like residual layouts, treat split residual + norm as missing existing fusion. |
|
||||
| In-place QK RMSNorm | split `q_norm` / `k_norm` kernels | `python/sglang/srt/models/utils.py::apply_qk_norm`<br>`python/sglang/kernels/ops/layernorm/norm.py::fused_inplace_qknorm` | In-place JIT QK norm plus optional `alt_stream` overlap for K | Check shape, dtype, deterministic mode, and in-place legality before proposing a new QK fuse. |
|
||||
@@ -70,8 +70,8 @@ in-flight row as shipped.
|
||||
| Fused MoE dispatch / permute / combine | token permutation<br>dispatch / combine<br>grouped top-k<br>many small MoE support kernels | `python/sglang/srt/layers/moe/fused_moe_triton/layer.py`<br>`python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py` | `FusedMoE` plus DeepEP / FlashInfer / FuseEP / standard dispatch backends and `permute_fusion=True` | First ask whether the model is missing an existing `FusedMoE`-style path or backend-specific dispatcher path. |
|
||||
| Fused MoE sum + all-reduce | routed MoE followed by explicit sum-reduce kernels | `python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py`<br>`python/sglang/srt/layers/moe/fused_moe_triton/fused_moe_triton_kernels.py` | `fuse_sum_all_reduce=True` path in the second MoE GEMM | Before inventing a new MoE reduction fuse, check whether `enable_fused_moe_sum_all_reduce` is simply off or the quant path is incompatible. |
|
||||
| Fused MoE activation + quant / re-quant | `silu_and_mul_*quant*`<br>`npu_dequant_swiglu_quant`<br>`swiglu_quant` | `python/sglang/srt/layers/moe/ep_moe/kernels.py`<br>`python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py`<br>`python/sglang/srt/layers/moe/cutlass_w4a8_moe.py`<br>`python/sglang/srt/hardware_backend/npu/quantization/moe_methods.py` | Quantized MoE backends fuse SwiGLU / SiLU-and-mul with FP8 / FP4 / NPU re-quant before the second expert GEMM | If MoE traces show standalone activation then quant kernels, first check whether the quantized fused path is missing. |
|
||||
| DeepSeek comm-prep fused RMSNorm + quant / flatten-quant | `fused_rms_fp8_group_quant`<br>`fused_rms_mxfp4_quant`<br>`fused_flatten_fp8_group_quant`<br>`fused_flatten_mxfp4_quant` | `python/sglang/srt/layers/communicator.py`<br>`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py`<br>`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py` | DeepSeek MLA / MHA ROCm paths fuse RMSNorm or flatten with FP8 / MXFP4 quantization for comm / attention prep | On DeepSeek quant traces, split norm + quant or flatten + quant is an existing family, not a new idea. |
|
||||
| NSA fused top-k transform / page-table build | `fast_topk_transform_fused`<br>`fast_topk_transform_ragged_fused` | `python/sglang/srt/layers/attention/nsa_backend.py` | NSA can fuse top-k selection with paged / ragged index transform instead of separate top-k plus metadata scatter | If NSA top-k metadata work is split, check `SGLANG_NSA_FUSE_TOPK` and backend support first. |
|
||||
| DeepSeek comm-prep fused RMSNorm + quant / flatten-quant | `fused_rms_fp8_group_quant`<br>`fused_rms_mxfp4_quant`<br>`fused_flatten_fp8_group_quant`<br>`fused_flatten_mxfp4_quant` | `python/sglang/srt/layers/layer_boundary/residual/add_norm.py`<br>`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py`<br>`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py` | DeepSeek MLA / MHA ROCm paths fuse RMSNorm or flatten with FP8 / MXFP4 quantization for comm / attention prep | On DeepSeek quant traces, split norm + quant or flatten + quant is an existing family, not a new idea. |
|
||||
| NSA fused top-k transform / page-table build | `fast_topk_transform_fused`<br>`fast_topk_transform_ragged_fused` | `python/sglang/srt/layers/attention/dsa_backend.py` | NSA can fuse top-k selection with paged / ragged index transform instead of separate top-k plus metadata scatter | If NSA top-k metadata work is split, check `SGLANG_DSA_FUSE_TOPK` and backend support first. |
|
||||
| NSA fused quantize + indexed K-cache store | `fused_store_index_k_cache`<br>`act_quant`<br>`index_k_with_scale_buffer` | `python/sglang/kernels/ops/attention/fused_store_index_cache.py`<br>`python/sglang/srt/layers/attention/nsa/nsa_indexer.py` | Single JIT kernel quantizes bf16 K to fp8 + scale and writes directly into NSA index cache | Treat split `act_quant` + buffer-store on CUDA as missing an existing fused store path. |
|
||||
| Fused sampling temperature + softmax | `fused_temperature_softmax*` | `python/sglang/srt/layers/fused_sampling.py`<br>`python/sglang/srt/layers/sampler.py` | Triton single-pass / multi-pass kernels fuse temperature scaling and softmax during decode | Separate temp-divide + softmax at decode batch sizes is often a missed existing fusion. |
|
||||
| Fused logit softcap | `fused_softcap`<br>`final_logit_softcapping` | `python/sglang/srt/layers/elementwise.py`<br>`python/sglang/srt/layers/logits_processor.py` | Triton kernels fuse cast-to-float and softcap / tanh math for logits or generic elementwise softcapping | Treat exposed cast + softcap ladders as an existing Triton fuse family. |
|
||||
@@ -148,8 +148,8 @@ Stable entries should be folded into the mainline family rows above.
|
||||
| PR `#22392` CUTLASS FP8 GEMM replacing nvjet | `cutlass_scaled_mm`<br>`fp8_scaled_mm`<br>`nvjet`<br>`cudaMemsetAsync` | `PR #22392`<br>`python/sglang/kernels/aot/python/sgl_kernel/gemm.py`<br>`python/sglang/srt/layers/quantization/fp8_utils.py` | Runtime replacement swaps nvjet FP8 GEMMs for CUTLASS kernels, removing per-launch memset bubbles and extra output-copy kernels | Treat nvjet GEMM + memset bubble ladders as an in-flight SGLang linear-kernel family before calling them novel. |
|
||||
| PR `#18612` NVFP4 CUTLASS MoE fused SiLU+Mul+quant | `silu_and_mul_scaled_nvfp4`<br>`nvfp4 expert quant`<br>`cutlass moe` | `PR #18612`<br>`python/sglang/srt/layers/moe/cutlass_w4a8_moe.py`<br>`python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py` | Fuses MoE activation epilogue and NVFP4 expert quantization before the CUTLASS MoE second GEMM | Treat split SiLU+Mul then NVFP4 expert quant in CUTLASS MoE traces as an in-flight upstream SGLang family. |
|
||||
| PR `#22918` FlashInfer per-token NVFP4 MoE | `per_token_nvfp4`<br>`trtllm_fp4_block_scale_moe`<br>`FlashInfer MoE` | `PR #22918`<br>`python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py` | Adds FlashInfer-backed per-token NVFP4 MoE execution so expert quant/dequant work can move into the fused MoE backend | Treat standalone per-token NVFP4 MoE support kernels as a candidate missing backend-selection path, not an automatically novel kernel idea. |
|
||||
| PR `#22851` NSA top-k backend and FlashInfer / PyTorch top-k split | `nsa topk`<br>`flashinfer_topk`<br>`pytorch_topk`<br>`fast_topk_transform` | `PR #22851`<br>`python/sglang/srt/layers/attention/nsa_backend.py` | Makes NSA top-k backend selection explicit and aligns fused top-k transform with FlashInfer / PyTorch fallbacks | When NSA top-k dominates decode, first classify it as backend selection or fused-transform eligibility work. |
|
||||
| PR `#24125` GLM5 NSA decode CatArrayBatchedCopy removal | `CatArrayBatchedCopy`<br>`GLM-5`<br>`NSA`<br>`TileLang decode` | `PR #24125`<br>`python/sglang/srt/layers/attention/nsa_backend.py` | Skips redundant cat/copy work in the GLM5 NSA TileLang decode path | Treat cat/copy bursts in GLM5 NSA decode as a concrete in-flight cleanup opportunity. |
|
||||
| PR `#22851` NSA top-k backend and FlashInfer / PyTorch top-k split | `nsa topk`<br>`flashinfer_topk`<br>`pytorch_topk`<br>`fast_topk_transform` | `PR #22851`<br>`python/sglang/srt/layers/attention/dsa_backend.py` | Makes NSA top-k backend selection explicit and aligns fused top-k transform with FlashInfer / PyTorch fallbacks | When NSA top-k dominates decode, first classify it as backend selection or fused-transform eligibility work. |
|
||||
| PR `#24125` GLM5 NSA decode CatArrayBatchedCopy removal | `CatArrayBatchedCopy`<br>`GLM-5`<br>`NSA`<br>`TileLang decode` | `PR #24125`<br>`python/sglang/srt/layers/attention/dsa_backend.py` | Skips redundant cat/copy work in the GLM5 NSA TileLang decode path | Treat cat/copy bursts in GLM5 NSA decode as a concrete in-flight cleanup opportunity. |
|
||||
| PR `#24007` MoE LoRA virtual experts for csgmv backend | `csgmv`<br>`virtual experts`<br>`MoE LoRA`<br>`fused_moe_lora` | `PR #24007`<br>`python/sglang/srt/layers/lora_backend.py`<br>`python/sglang/srt/layers/moe` | Routes MoE LoRA adapter work through virtual experts so csgmv-style kernels can batch it instead of launching fragmented adapter work | Treat MoE-LoRA tiny-kernel ladders as an in-flight batching/fusion family. |
|
||||
| PR `#24150` torch.compile local decode support | `enable_torch_compile`<br>`local compile`<br>`decode compile`<br>`torchinductor` | `PR #24150`<br>`python/sglang/srt` | Extends SGLang torch.compile coverage to local decode regions, so Inductor-generated fusion may replace hand-authored tiny kernels | When decode traces show compiler-generated kernels or missing named fused kernels, check this in-flight compile path before calling the shape unsupported. |
|
||||
|
||||
@@ -319,7 +319,7 @@ only vLLM references to the TokenSpeed package.
|
||||
| `enable_single_batch_overlap` | `python/sglang/srt/server_args.py` | Enables the SBO family. |
|
||||
| `enable_fused_moe_sum_all_reduce` | `python/sglang/srt/server_args.py` | Enables fused MoE sum-reduce in the down path. |
|
||||
| `SGLANG_BLACKWELL_OVERLAP_SHARED_EXPERTS_OUTSIDE_SBO` | `python/sglang/srt/environ.py` | Alters how DeepSeek-style shared-expert overlap behaves on Blackwell. |
|
||||
| `SGLANG_NSA_FUSE_TOPK` | `python/sglang/srt/environ.py` | Gates NSA fused top-k transform / page-table build. |
|
||||
| `SGLANG_DSA_FUSE_TOPK` | `python/sglang/srt/environ.py` | Gates NSA fused top-k transform / page-table build. |
|
||||
| `SGLANG_DISAGG_STAGING_BUFFER` | `python/sglang/srt/environ.py` | Enables the heterogeneous-TP staging-buffer family and its overlap windows. |
|
||||
| `SGLANG_STAGING_USE_TORCH` | `python/sglang/srt/disaggregation/common/staging_buffer.py` | Forces torch fallback for staging gather / scatter, so Triton staging kernels may disappear by design. |
|
||||
| `SGLANG_VIT_ENABLE_CUDA_GRAPH` | `python/sglang/srt/environ.py` | Can intentionally disable vision `aux_stream` overlap. |
|
||||
|
||||
@@ -391,7 +391,7 @@ FUSION_PATTERN_REGISTRY: Tuple[FusionPatternSpec, ...] = (
|
||||
candidate_path=(
|
||||
"python/sglang/srt/layers/flashinfer_comm_fusion.py"
|
||||
"<br>python/sglang/srt/layers/layernorm.py"
|
||||
"<br>python/sglang/srt/layers/communicator.py"
|
||||
"<br>python/sglang/srt/layers/layer_boundary/residual/add_norm.py"
|
||||
),
|
||||
active_keywords=(
|
||||
"allreduce_fusion",
|
||||
@@ -419,7 +419,7 @@ FUSION_PATTERN_REGISTRY: Tuple[FusionPatternSpec, ...] = (
|
||||
pattern="AITER allreduce fusion",
|
||||
candidate_path=(
|
||||
"python/sglang/srt/distributed/communication_op.py"
|
||||
"<br>python/sglang/srt/layers/communicator.py"
|
||||
"<br>python/sglang/srt/layers/layer_boundary/residual/add_norm.py"
|
||||
"<br>python/sglang/srt/layers/layernorm.py"
|
||||
),
|
||||
active_keywords=(
|
||||
@@ -721,7 +721,7 @@ FUSION_PATTERN_REGISTRY: Tuple[FusionPatternSpec, ...] = (
|
||||
FusionPatternSpec(
|
||||
pattern="DeepSeek comm-prep fused RMSNorm + quant / flatten-quant",
|
||||
candidate_path=(
|
||||
"python/sglang/srt/layers/communicator.py"
|
||||
"python/sglang/srt/layers/layer_boundary/residual/add_norm.py"
|
||||
"<br>python/sglang/srt/models/deepseek_common/attention_forward_methods/"
|
||||
"forward_mla.py"
|
||||
"<br>python/sglang/srt/models/deepseek_common/attention_forward_methods/"
|
||||
|
||||
@@ -5,6 +5,10 @@ description: Guide for writing SGLang CI/UT tests. Covers CustomTestCase, CI reg
|
||||
|
||||
# Writing SGLang CI / UT Tests
|
||||
|
||||
Apply [kernel-organization](../kernel-organization/SKILL.md) for the public
|
||||
operator namespace, logical grouping, lazy registry metadata, and test placement.
|
||||
The implementation tutorial below does not replace that API contract.
|
||||
|
||||
This skill covers **how to write and register tests**. For CI pipeline internals (stage ordering, fail-fast, gating, partitioning, debugging CI failures), see the [CI workflow guide](../ci-workflow-guide/SKILL.md). Whether a case is worth adding at all is decided by [`unit-test-admission`](../../rules/unit-test-admission.md) — read it before writing the case, not after.
|
||||
|
||||
## Core Rules
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
# Gitea Actions 自动构建 sglang 镜像(海外节点,原版源)
|
||||
# 基底 lmsysorg/sglang:dev-dsv41(docker.io),依赖走 pypi.org 默认源。
|
||||
# 触发:push 到 dsv41-pd 分支。
|
||||
#
|
||||
# 前置条件(一次性,在 Gitea 实例上配置):
|
||||
# 1. 实例已注册 act_runner(Gitea Actions runner,标签含 ubuntu-latest)
|
||||
# 2. 仓库 Settings → Secrets 添加:
|
||||
# REGISTRY_USERNAME / REGISTRY_PASSWORD(推送镜像的账号,如 Gitea 访问令牌)
|
||||
# 3. 可选:Settings → Actions → Variables 添加 REGISTRY(默认 git.agentwithu.com,
|
||||
# 即 Gitea 自带容器 registry;也可填 docker.io 等)
|
||||
name: build-sglang-image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dsv41-pd, dsv41-pd-visioncp]
|
||||
# compose/部署配置 与 workflow 自身的改动不触发镜像构建(省 runner 与推送带宽)。
|
||||
# 改了 workflow 想重建镜像时,需伴随任意代码改动或手动触发。
|
||||
paths-ignore:
|
||||
- 'deploy/**'
|
||||
- '.gitea/**'
|
||||
|
||||
env:
|
||||
# 直接写死 Gitea 自带 registry(vars context 在该实例上求值异常会导致回退 docker.io)
|
||||
REGISTRY: git.agentwithu.com
|
||||
IMAGE_NAME: minke.yu/sglang
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Resolve tag
|
||||
id: meta
|
||||
run: |
|
||||
SHA9=$(git rev-parse --short=9 HEAD)
|
||||
echo "tag=${{ github.ref_name }}-${SHA9}-ci-$(date -u +%Y%m%d-%H%M)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ secrets.REGISTRY_USERNAME }}
|
||||
password: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
|
||||
- uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile.gitea
|
||||
push: true
|
||||
# cache-from: type=registry
|
||||
# cache-to: type=registry,mode=max
|
||||
tags: |
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.tag }}
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}-latest
|
||||
|
||||
+2
-2
@@ -5,9 +5,9 @@
|
||||
/docs @wisclmy0611 @zijiexia @Richardczl98 @JustinTong0323 @sogalin
|
||||
/python/pyproject.toml @merrymercy @Fridge003 @ispobock
|
||||
/python/sglang/kernels @DarkSharpness @HaiShaw @BBuf @celve @HydraQYH @yuan-luo
|
||||
/python/sglang/kernels/ops/diffusion @yingluosanqian @BBuf @mickqian @triple-mu
|
||||
/python/sglang/kernels/ops/diffusion @yingluosanqian @BBuf @mickqian @triple-mu @niehen6174
|
||||
/python/sglang/kernels/ops/attention/fla @yizhang2077 @hebiao064 @yuan-luo
|
||||
/python/sglang/multimodal_gen @mickqian @ping1jing2 @HaiShaw @yichiche @AgainstEntropy @BBuf @triple-mu @kevin-mii
|
||||
/python/sglang/multimodal_gen @mickqian @ping1jing2 @HaiShaw @yichiche @AgainstEntropy @BBuf @triple-mu @kevin-mii @niehen6174
|
||||
/python/sglang/multimodal_gen/runtime/cache @DefTruth
|
||||
/python/sglang/multimodal_gen/runtime/layers/quantization/ @HaiShaw @BowenBao @OrangeRedeng @TallMessiWu
|
||||
/python/sglang/multimodal_gen/test/server/ascend @ping1jing2 @ssshinigami @Makcum888e @e-martirosian
|
||||
|
||||
@@ -114,7 +114,6 @@ unified-radix-cache:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- 'python/sglang/srt/mem_cache/unified_cache/**/*'
|
||||
- 'python/sglang/srt/mem_cache/cpp_radix_tree/**/*'
|
||||
- 'python/sglang/srt/mem_cache/**/*radix*'
|
||||
- 'python/sglang/srt/mem_cache/chunk_cache.py'
|
||||
- 'python/sglang/srt/mem_cache/evict_policy.py'
|
||||
@@ -125,7 +124,6 @@ hicache:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- '**/*hicache*'
|
||||
- '**/*hiradix*'
|
||||
- '**/HiCache/**/*'
|
||||
- 'benchmark/hicache/**/*'
|
||||
- 'test/registered/hicache/**/*'
|
||||
|
||||
@@ -19,7 +19,7 @@ on:
|
||||
type: string
|
||||
required: true
|
||||
npu_device_type:
|
||||
description: 'NPU device type, e.g. 910b, a3.'
|
||||
description: 'NPU device type, e.g. a3, 950.'
|
||||
type: string
|
||||
default: 'a3'
|
||||
run_timeout_minutes:
|
||||
@@ -39,7 +39,7 @@ on:
|
||||
type: string
|
||||
default: ''
|
||||
skip_pr_test_health_check:
|
||||
description: 'Set to true to skip the PR test health check (fail-fast gate).'
|
||||
description: 'Set to true to skip the PR test health check (fast-fail gate).'
|
||||
type: string
|
||||
default: 'false'
|
||||
is_nightly_pipeline_job:
|
||||
@@ -90,7 +90,112 @@ jobs:
|
||||
run: |
|
||||
git config --system --add safe.directory ${GITHUB_WORKSPACE}
|
||||
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
- name: Check PR test health
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
|
||||
with:
|
||||
github-token: ${{ inputs.github-token || github.token }}
|
||||
script: |
|
||||
core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
|
||||
|
||||
// Skip when explicitly requested via env var (e.g. release branch cut)
|
||||
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
|
||||
core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip for scheduled runs — they should collect all failures, not fast-fail
|
||||
if (context.eventName === 'schedule') {
|
||||
core.notice('[health-check] SKIP: scheduled run');
|
||||
return;
|
||||
}
|
||||
|
||||
// Check lint status from the separate Lint workflow (lint.yml).
|
||||
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
|
||||
// checks.listForRef which queries by commit SHA across ALL workflows.
|
||||
const ref = context.payload.pull_request?.head?.sha || context.sha;
|
||||
core.info(`[health-check] Checking lint for ref=${ref}`);
|
||||
const { data } = await github.rest.checks.listForRef({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
ref: ref,
|
||||
check_name: 'lint',
|
||||
});
|
||||
const lintRun = data.check_runs.find(
|
||||
cr => cr.app?.slug === 'github-actions'
|
||||
);
|
||||
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
|
||||
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
|
||||
core.setFailed('Fast-fail: lint check failed');
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
|
||||
// Lint check above still runs.
|
||||
let labels = [];
|
||||
if (context.payload.pull_request?.labels) {
|
||||
labels = context.payload.pull_request.labels.map(l => l.name);
|
||||
} else {
|
||||
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
commit_sha: ref,
|
||||
});
|
||||
if (prs.length > 0) {
|
||||
labels = prs[0].labels.map(l => l.name);
|
||||
}
|
||||
}
|
||||
core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
|
||||
if (labels.includes('bypass-fastfail')) {
|
||||
core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
|
||||
return;
|
||||
}
|
||||
|
||||
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: context.runId,
|
||||
per_page: 100,
|
||||
});
|
||||
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
|
||||
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
|
||||
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
|
||||
|
||||
// Find jobs that failed from a real error, not from fast-fail cascade
|
||||
const rootCauseFailures = jobs.filter(j => {
|
||||
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
|
||||
// h20 runners are flaky (dirty GPU state from prior runs); their failures
|
||||
// should not cascade fast-fail to other stages. j.name shape from
|
||||
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
|
||||
// + optional " (<matrix>)". Split off the base job key before exact
|
||||
// match so we cover both inline + reusable forms without confusing
|
||||
// 'h20' with the 'h200' prefix.
|
||||
const baseName = j.name.split(/[ /]/)[0];
|
||||
if (baseName === 'base-c-test-8-gpu-h20') {
|
||||
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
|
||||
return false;
|
||||
}
|
||||
// multimodal-gen NPU tests should not cascade fast-fail to perf/accuracy stages.
|
||||
if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') {
|
||||
core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`);
|
||||
return false;
|
||||
}
|
||||
// If the failing step is the health check, it's a cascade — skip it
|
||||
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
|
||||
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
|
||||
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
|
||||
|
||||
if (rootCauseFailures.length > 0) {
|
||||
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
|
||||
} else {
|
||||
core.notice('[health-check] PASS: no root cause failures detected');
|
||||
}
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
|
||||
@@ -21,7 +21,7 @@ on:
|
||||
required: true
|
||||
type: string
|
||||
description: image for pods
|
||||
default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3"
|
||||
default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.1.0-a3"
|
||||
install_sglang_deps:
|
||||
required: false
|
||||
type: boolean
|
||||
@@ -31,7 +31,7 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'a3'
|
||||
description: device type for dependency installation (a3 or 910b)
|
||||
description: device type for dependency installation (a3 or 950)
|
||||
transformers_version:
|
||||
required: false
|
||||
type: string
|
||||
@@ -43,7 +43,7 @@ on:
|
||||
default: '{}'
|
||||
description: 'JSON run metadata {branch_label, workflow_name, create_time}, recorded once at workflow start'
|
||||
skip_pr_test_health_check:
|
||||
description: 'Set to true to skip the PR test health check (fail-fast gate).'
|
||||
description: 'Set to true to skip the PR test health check (fast-fail gate).'
|
||||
type: string
|
||||
default: 'false'
|
||||
is_nightly_pipeline_job:
|
||||
@@ -101,7 +101,112 @@ jobs:
|
||||
# checks out the same commit even if the branch advances mid-run.
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
- name: Check PR test health
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
|
||||
with:
|
||||
github-token: ${{ inputs.github-token || github.token }}
|
||||
script: |
|
||||
core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
|
||||
|
||||
// Skip when explicitly requested via env var (e.g. release branch cut)
|
||||
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
|
||||
core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip for scheduled runs — they should collect all failures, not fast-fail
|
||||
if (context.eventName === 'schedule') {
|
||||
core.notice('[health-check] SKIP: scheduled run');
|
||||
return;
|
||||
}
|
||||
|
||||
// Check lint status from the separate Lint workflow (lint.yml).
|
||||
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
|
||||
// checks.listForRef which queries by commit SHA across ALL workflows.
|
||||
const ref = context.payload.pull_request?.head?.sha || context.sha;
|
||||
core.info(`[health-check] Checking lint for ref=${ref}`);
|
||||
const { data } = await github.rest.checks.listForRef({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
ref: ref,
|
||||
check_name: 'lint',
|
||||
});
|
||||
const lintRun = data.check_runs.find(
|
||||
cr => cr.app?.slug === 'github-actions'
|
||||
);
|
||||
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
|
||||
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
|
||||
core.setFailed('Fast-fail: lint check failed');
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
|
||||
// Lint check above still runs.
|
||||
let labels = [];
|
||||
if (context.payload.pull_request?.labels) {
|
||||
labels = context.payload.pull_request.labels.map(l => l.name);
|
||||
} else {
|
||||
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
commit_sha: ref,
|
||||
});
|
||||
if (prs.length > 0) {
|
||||
labels = prs[0].labels.map(l => l.name);
|
||||
}
|
||||
}
|
||||
core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
|
||||
if (labels.includes('bypass-fastfail')) {
|
||||
core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
|
||||
return;
|
||||
}
|
||||
|
||||
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: context.runId,
|
||||
per_page: 100,
|
||||
});
|
||||
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
|
||||
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
|
||||
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
|
||||
|
||||
// Find jobs that failed from a real error, not from fast-fail cascade
|
||||
const rootCauseFailures = jobs.filter(j => {
|
||||
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
|
||||
// h20 runners are flaky (dirty GPU state from prior runs); their failures
|
||||
// should not cascade fast-fail to other stages. j.name shape from
|
||||
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
|
||||
// + optional " (<matrix>)". Split off the base job key before exact
|
||||
// match so we cover both inline + reusable forms without confusing
|
||||
// 'h20' with the 'h200' prefix.
|
||||
const baseName = j.name.split(/[ /]/)[0];
|
||||
if (baseName === 'base-c-test-8-gpu-h20') {
|
||||
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
|
||||
return false;
|
||||
}
|
||||
// multimodal-gen NPU tests should not cascade fast-fail to perf/accuracy stages.
|
||||
if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') {
|
||||
core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`);
|
||||
return false;
|
||||
}
|
||||
// If the failing step is the health check, it's a cascade — skip it
|
||||
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
|
||||
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
|
||||
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
|
||||
|
||||
if (rootCauseFailures.length > 0) {
|
||||
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
|
||||
} else {
|
||||
core.notice('[health-check] PASS: no root cause failures detected');
|
||||
}
|
||||
|
||||
- name: Check npu info
|
||||
run: |
|
||||
|
||||
@@ -27,6 +27,8 @@ on:
|
||||
value: ${{ jobs.run.outputs.jit_kernel }}
|
||||
multimodal_gen:
|
||||
value: ${{ jobs.run.outputs.multimodal_gen }}
|
||||
renderer:
|
||||
value: ${{ jobs.run.outputs.renderer }}
|
||||
rust_workspace:
|
||||
value: ${{ jobs.run.outputs.rust_workspace }}
|
||||
partitions:
|
||||
@@ -50,6 +52,7 @@ jobs:
|
||||
sgl_kernel: ${{ steps.filter.outputs.sgl_kernel }}
|
||||
jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }}
|
||||
multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }}
|
||||
renderer: ${{ steps.filter.outputs.renderer || steps.run-mode.outputs.run_all_tests }}
|
||||
rust_workspace: ${{ steps.filter.outputs.rust_workspace || steps.run-mode.outputs.run_all_tests }}
|
||||
partitions: ${{ steps.partitions.outputs.partitions }}
|
||||
partition_model_sha: ${{ steps.partition-model-sha.outputs.sha }}
|
||||
@@ -96,8 +99,11 @@ jobs:
|
||||
- "scripts/ci/cuda/*"
|
||||
- "scripts/ci/utils/*"
|
||||
- "test/**/!(*.md)"
|
||||
- "rust/**"
|
||||
- "rust/*"
|
||||
- "rust/!(sglang-renderer)/**"
|
||||
- "proto/sglang/runtime/v1/sglang.proto"
|
||||
renderer:
|
||||
- "rust/sglang-renderer/**"
|
||||
simulator:
|
||||
- ".github/workflows/pr-test-extra.yml"
|
||||
- ".github/workflows/_pr-test-simulator-cpu.yml"
|
||||
|
||||
@@ -13,7 +13,7 @@ on:
|
||||
type: string
|
||||
required: true
|
||||
check_changes:
|
||||
description: 'toJson(needs.check-changes.outputs). Read via fromJson(...).main_package / rust_workspace / continue_on_error / partition_model_sha.'
|
||||
description: 'toJson(needs.check-changes.outputs). Read via fromJson(...).main_package / renderer / rust_workspace / continue_on_error / partition_model_sha.'
|
||||
type: string
|
||||
required: true
|
||||
caller_inputs:
|
||||
@@ -49,7 +49,7 @@ jobs:
|
||||
# emits a skipped entry for wait-for-jobs to match. The schedule /
|
||||
# parallel-dispatch / !failure() half stays on the caller, where the
|
||||
# `needs` context it guards actually lives.
|
||||
if: fromJson(inputs.check_changes).main_package == 'true'
|
||||
if: fromJson(inputs.check_changes).main_package == 'true' || fromJson(inputs.check_changes).renderer == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# Delegates to pr-test-npu.yml with coverage_mode=true, which switches:
|
||||
# - Runners to the coverage pool (shared /root/.cache with setup-covstub)
|
||||
# - use_coverage_runner=true on all test jobs
|
||||
# - Skips pr-gate, base-a (910b), multimodal-gen, recommend, analyze
|
||||
# - Skips pr-gate, multimodal-gen, recommend, analyze
|
||||
# - Runs setup-covstub to assemble coverage data and publish it to the
|
||||
# shared disk at a fixed path (/root/.cache/tests/precise-test/coverage)
|
||||
# pr-test-npu-finish is the final gate in the called workflow.
|
||||
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
matrix: ${{ fromJson(needs.compute-diffusion-partitions.outputs.matrix-1npu) }}
|
||||
timeout-minutes: 150
|
||||
container:
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.1.0-a3-ubuntu22.04-py3.12
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -166,7 +166,7 @@ jobs:
|
||||
matrix: ${{ fromJson(needs.compute-diffusion-partitions.outputs.matrix-2npu) }}
|
||||
timeout-minutes: 150
|
||||
container:
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11
|
||||
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.1.0-a3-ubuntu22.04-py3.12
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
@@ -18,16 +18,11 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'all'
|
||||
image_a2:
|
||||
description: 'The a2 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b'
|
||||
image_a3:
|
||||
description: 'The a3 running docker image of the test task. Must be an image with sglang preinstalled (e.g. lmsysorg/sglang:*a3*).'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3'
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.1.0-a3'
|
||||
|
||||
concurrency:
|
||||
group: full-test-npu-${{ inputs.ref || github.ref }}
|
||||
@@ -39,7 +34,6 @@ jobs:
|
||||
outputs:
|
||||
ref: ${{ steps.set-vars.outputs.ref }}
|
||||
job_filter: ${{ steps.set-vars.outputs.job_filter }}
|
||||
image_a2: ${{ steps.set-vars.outputs.image_a2 }}
|
||||
image_a3: ${{ steps.set-vars.outputs.image_a3 }}
|
||||
steps:
|
||||
# When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
|
||||
@@ -58,14 +52,8 @@ jobs:
|
||||
echo "job_filter=${{ inputs.job_filter }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
if [ -z "${{ inputs.image_a2 }}" ]; then
|
||||
echo "image_a2=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "image_a2=${{ inputs.image_a2 }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
if [ -z "${{ inputs.image_a3 }}" ]; then
|
||||
echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3" >> $GITHUB_OUTPUT
|
||||
echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.1.0-a3" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
@@ -81,20 +69,6 @@ jobs:
|
||||
image_a3: ${{ needs.set-image-config.outputs.image_a3 }}
|
||||
secrets: inherit
|
||||
|
||||
full-4-npu-a2:
|
||||
name: full-4-npu-a2
|
||||
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
|
||||
needs: [set-image-config]
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: linux-aarch64-a2-4
|
||||
test_type: 'perf'
|
||||
test_suite: full-4-npu-a2
|
||||
is_nightly_pipeline_job: true
|
||||
skip_pr_test_health_check: 'true'
|
||||
image: ${{ needs.set-image-config.outputs.image_a2 }}
|
||||
device_type_for_deps: '910b'
|
||||
|
||||
full-1-npu-a3:
|
||||
name: full-1-npu-a3
|
||||
if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
|
||||
@@ -317,7 +291,6 @@ jobs:
|
||||
if: github.repository == 'sgl-project/sglang' && always()
|
||||
needs:
|
||||
- nighly-test-npu
|
||||
- full-4-npu-a2
|
||||
- full-1-npu-a3
|
||||
- full-2-npu-a3
|
||||
- full-4-npu-a3
|
||||
|
||||
@@ -1,18 +1,16 @@
|
||||
name: Nightly Test (AMD MI355X 2N 1P1D Disagg)
|
||||
name: Nightly Test (AMD MI355X Disagg)
|
||||
|
||||
# Self-hosted runners on the amd-sglang login node salloc two or four MI355X
|
||||
# nodes and run the Docker harness (prefill + decode over MORI, a standalone
|
||||
# load balancer, then an sglang.bench_serving concurrency sweep) via
|
||||
# scripts/ci/slurm/launch_mi355x.sh.
|
||||
on:
|
||||
# Temporarily paused while the DI CI nodes are unavailable. Uncomment the
|
||||
# schedule below when the nodes return.
|
||||
# schedule:
|
||||
# - cron: '0 7 * * *'
|
||||
schedule:
|
||||
- cron: '0 7 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
image:
|
||||
description: 'Optional. ROCm SGLang Docker image to benchmark. Leave empty for the recipe default.'
|
||||
description: 'Optional. ROCm SGLang Docker image to benchmark. Leave empty for the latest MI35x ROCm 10 image.'
|
||||
required: false
|
||||
default: ''
|
||||
configs:
|
||||
@@ -44,29 +42,38 @@ jobs:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Resolve latest MI35x nightly image
|
||||
- name: Resolve latest MI35x ROCm 10 nightly image
|
||||
id: image
|
||||
# Track the newest published image so the nightly actually catches
|
||||
# regressions instead of re-testing one pinned build. We resolve on the
|
||||
# GitHub-hosted runner (clean Docker Hub access, no rate limits) and pass
|
||||
# the tag to the self-hosted runner. Scope is deliberately tight: only
|
||||
# the clean main MI35x ROCm 7.2 line
|
||||
# `lmsysorg/sglang-rocm:v*-rocm720-mi35x-YYYYMMDD` (excludes -test-,
|
||||
# miles-, mori-, vllm-, rocm7_14 variants). On any failure we emit an
|
||||
# empty string and the launcher falls back to the recipe's pinned
|
||||
# `runtime.image`.
|
||||
# the clean main MI35x ROCm 10 line
|
||||
# `lmsysorg/sglang-rocm:v*-rocm10-mi35x-YYYYMMDD` (excludes -test-,
|
||||
# miles-, mori-, vllm- variants). If resolution fails, use the pinned
|
||||
# ROCm 10 image from the last validated full workflow run.
|
||||
run: |
|
||||
set -o pipefail
|
||||
FALLBACK_IMAGE="lmsysorg/sglang-rocm:v0.5.20-rocm10-mi35x-20260922"
|
||||
RESOLVED=$(curl -fsSL "https://hub.docker.com/v2/repositories/lmsysorg/sglang-rocm/tags?page_size=100&ordering=last_updated" \
|
||||
| python3 -c "import json,re,sys;p=re.compile(r'^v[0-9][0-9A-Za-z.]*-rocm720-mi35x-([0-9]{8})\$');c=sorted((m.group(1),t['name']) for t in json.load(sys.stdin).get('results',[]) for m in [p.match(t.get('name',''))] if m);print(c[-1][1] if c else '')" || true)
|
||||
| python3 -c "import json,re,sys;p=re.compile(r'^v[0-9][0-9A-Za-z.]*-rocm10-mi35x-([0-9]{8})\$');c=sorted((m.group(1),t['name']) for t in json.load(sys.stdin).get('results',[]) for m in [p.match(t.get('name',''))] if m);print(c[-1][1] if c else '')" || true)
|
||||
if [ -n "$RESOLVED" ]; then
|
||||
echo "Resolved latest MI35x image: lmsysorg/sglang-rocm:$RESOLVED"
|
||||
echo "image=lmsysorg/sglang-rocm:$RESOLVED" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "Could not resolve a latest image; launcher will use recipe default."
|
||||
echo "image=" >> $GITHUB_OUTPUT
|
||||
echo "Could not resolve a latest image; using fallback: $FALLBACK_IMAGE"
|
||||
echo "image=$FALLBACK_IMAGE" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Record which image this run uses
|
||||
env:
|
||||
DISPATCH_IMAGE: ${{ inputs.image }}
|
||||
RESOLVED_IMAGE: ${{ steps.image.outputs.image }}
|
||||
run: |
|
||||
IMAGE="${DISPATCH_IMAGE:-$RESOLVED_IMAGE}"
|
||||
echo "### Image" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "\`$IMAGE\`" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Generate benchmark matrix
|
||||
id: generate
|
||||
env:
|
||||
@@ -133,7 +140,6 @@ jobs:
|
||||
CONFIG_FILE: ${{ matrix.config.config_file }}
|
||||
RESULT_FILENAME: mi355x-${{ matrix.config.name }}
|
||||
MATRIX_CONFIG_NAME: ${{ matrix.config.name }}
|
||||
SLURM_EXCLUDE: mia1-p01-g20,mia1-p02-g09
|
||||
# NOTE: RUNNER_NAME is a built-in default env var on every runner, so the
|
||||
# launch + cleanup steps read it directly. Do NOT set it here from
|
||||
# ${{ runner.name }} -- the `runner` context is not available in job-level
|
||||
@@ -261,8 +267,7 @@ jobs:
|
||||
- setup
|
||||
- nightly-mi355x-2n
|
||||
if: |
|
||||
false
|
||||
&& !cancelled() && github.repository == 'sgl-project/sglang'
|
||||
!cancelled() && github.repository == 'sgl-project/sglang'
|
||||
&& needs.setup.result == 'success'
|
||||
&& needs.setup.outputs.matrix-4n != '{"include":[]}'
|
||||
runs-on: linux-sglang-mi35x-di
|
||||
|
||||
@@ -93,8 +93,8 @@ on:
|
||||
- nightly-8-gpu-glm51
|
||||
# 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance)
|
||||
- nightly-8-gpu-mi35x-glm52-fp8
|
||||
# 8-GPU GLM-5-MXFP4 (MI35x only)
|
||||
- nightly-8-gpu-mi35x-glm5-mxfp4
|
||||
# 8-GPU GLM-5.3-Flash (MI35x)
|
||||
- nightly-8-gpu-mi35x-glm53-flash
|
||||
# 4-GPU MiniMax-M2.5 (MI35x)
|
||||
- nightly-4-gpu-mi35x-minimax-m25
|
||||
# 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance)
|
||||
@@ -2044,16 +2044,16 @@ jobs:
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
# ==============================================================================
|
||||
# 8-GPU GLM-5-MXFP4 (MI35x only)
|
||||
# 8-GPU GLM-5.3-Flash (MI35x)
|
||||
# ==============================================================================
|
||||
|
||||
nightly-8-gpu-mi35x-glm5-mxfp4:
|
||||
name: ${{ format('nightly-8-gpu-mi35x-glm5-mxfp4 ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
|
||||
nightly-8-gpu-mi35x-glm53-flash:
|
||||
name: ${{ format('nightly-8-gpu-mi35x-glm53-flash ({0}, linux-mi35x-gpu-8)', matrix.rocm_version) }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm10", "rocm724", "rocm720"]') }}
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm5-mxfp4,'))
|
||||
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm53-flash,'))
|
||||
runs-on: linux-mi35x-gpu-8
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -2070,34 +2070,21 @@ jobs:
|
||||
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
ENABLE_CACHE_HOST: "1"
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
|
||||
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
|
||||
|
||||
- name: Accuracy Test MI35x ROCm (8-GPU GLM-5-MXFP4)
|
||||
- name: Accuracy Test MI35x ROCm (8-GPU GLM-5.3-Flash DSA+KDA)
|
||||
timeout-minutes: 180
|
||||
run: |
|
||||
> github_summary.md
|
||||
> github_summary.md # Clear summary file
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e SGLANG_USE_AITER=1 \
|
||||
-e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm5-mxfp4 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
- name: Performance Test MI35x ROCm (8-GPU GLM-5-MXFP4)
|
||||
timeout-minutes: 300
|
||||
continue-on-error: true
|
||||
run: |
|
||||
> github_summary.md
|
||||
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
|
||||
-e SGLANG_USE_AITER=1 \
|
||||
-e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
|
||||
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
|
||||
python3 registered/amd/perf/mi35x/test_glm5_mxfp4_perf_mi35x.py || TEST_EXIT_CODE=$?
|
||||
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm53-flash --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
||||
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
|
||||
exit ${TEST_EXIT_CODE:-0}
|
||||
|
||||
@@ -2368,8 +2355,8 @@ jobs:
|
||||
- nightly-8-gpu-glm51
|
||||
# 8-GPU GLM-5.2-FP8 (MI35x accuracy + performance)
|
||||
- nightly-8-gpu-mi35x-glm52-fp8
|
||||
# 8-GPU GLM-5-MXFP4 (MI35x only)
|
||||
- nightly-8-gpu-mi35x-glm5-mxfp4
|
||||
# 8-GPU GLM-5.3-Flash (MI35x)
|
||||
- nightly-8-gpu-mi35x-glm53-flash
|
||||
# 4-GPU MiniMax-M2.5 (MI35x)
|
||||
- nightly-4-gpu-mi35x-minimax-m25
|
||||
# 4-GPU MiniMax-M3 MXFP8 (MI35x accuracy + performance)
|
||||
|
||||
@@ -19,26 +19,16 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'all'
|
||||
image_a2:
|
||||
description: 'The a2 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b'
|
||||
image_a3:
|
||||
description: 'The a3 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3'
|
||||
image_a3_cann910:
|
||||
description: 'The a3 running docker image of the test task with cann 9.1.0.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-a3-nightly'
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.1.0-a3'
|
||||
image_a5:
|
||||
description: 'The a5 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-950-nightly'
|
||||
default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.1.0-950'
|
||||
skip_install_flag:
|
||||
description: 'Indicates whether to skip the installation of sglang, defaulting to false.'
|
||||
required: false
|
||||
@@ -56,21 +46,11 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'all'
|
||||
image_a2:
|
||||
description: 'The a2 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
image_a3:
|
||||
description: 'The a3 running docker image of the test task.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
image_a3_cann910:
|
||||
description: 'The a3 running docker image of the test task with cann 9.1.0.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
image_a5:
|
||||
description: 'The a5 running docker image of the test task.'
|
||||
required: false
|
||||
@@ -90,9 +70,7 @@ jobs:
|
||||
outputs:
|
||||
ref: ${{ steps.set-vars.outputs.ref }}
|
||||
job_filter: ${{ steps.set-vars.outputs.job_filter }}
|
||||
image_a2: ${{ steps.set-vars.outputs.image_a2 }}
|
||||
image_a3: ${{ steps.set-vars.outputs.image_a3 }}
|
||||
image_a3_cann910: ${{ steps.set-vars.outputs.image_a3_cann910 }}
|
||||
image_a5: ${{ steps.set-vars.outputs.image_a5 }}
|
||||
skip_install_flag: ${{ steps.set-vars.outputs.skip_install_flag }}
|
||||
run_start_metadata: ${{ steps.set-vars.outputs.run_start_metadata }}
|
||||
@@ -115,23 +93,13 @@ jobs:
|
||||
else
|
||||
echo "job_filter=${{ inputs.job_filter }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
if [ -z "${{ inputs.image_a2 }}" ]; then
|
||||
echo "image_a2=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "image_a2=${{ inputs.image_a2 }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
if [ -z "${{ inputs.image_a3 }}" ]; then
|
||||
echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3" >> $GITHUB_OUTPUT
|
||||
echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.1.0-a3" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
if [ -z "${{ inputs.image_a3_cann910 }}" ]; then
|
||||
echo "image_a3_cann910=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-a3-nightly" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "image_a3_cann910=${{ inputs.image_a3_cann910 }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
if [ -z "${{ inputs.image_a5 }}" ]; then
|
||||
echo "image_a5=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-950-nightly" >> $GITHUB_OUTPUT
|
||||
echo "image_a5=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.1.0-950" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "image_a5=${{ inputs.image_a5 }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
@@ -329,41 +297,6 @@ jobs:
|
||||
install_sglang_deps: false
|
||||
device_type_for_deps: 'a5'
|
||||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||||
|
||||
# Suites suffixed with `-cann910` run against the CANN 9.1.0 image.
|
||||
nightly-perf-16-npu-a3-cann910:
|
||||
name: nightly-perf-16-npu-a3-cann910
|
||||
if: ${{ !cancelled() }}
|
||||
needs: [set-image-config]
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: linux-aarch64-a3-800t-16
|
||||
test_type: 'perf'
|
||||
test_suite: nightly-perf-16-npu-a3-cann910
|
||||
is_nightly_pipeline_job: true
|
||||
skip_pr_test_health_check: 'true'
|
||||
image: ${{ needs.set-image-config.outputs.image_a3_cann910 }}
|
||||
install_sglang_deps: false
|
||||
device_type_for_deps: 'a3'
|
||||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||||
test_timeout_minutes: '180'
|
||||
|
||||
nightly-acc-16-npu-a3-cann910:
|
||||
name: nightly-acc-16-npu-a3-cann910
|
||||
if: ${{ !cancelled() }}
|
||||
needs: [set-image-config]
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: linux-aarch64-a3-16-
|
||||
test_type: 'accuracy'
|
||||
test_suite: nightly-acc-16-npu-a3-cann910
|
||||
is_nightly_pipeline_job: true
|
||||
skip_pr_test_health_check: 'true'
|
||||
image: ${{ needs.set-image-config.outputs.image_a3_cann910 }}
|
||||
install_sglang_deps: false
|
||||
device_type_for_deps: 'a3'
|
||||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||||
|
||||
nightly-poc-multi-node-tests:
|
||||
name: multi-node-poc
|
||||
if: ${{ !cancelled() }}
|
||||
@@ -444,72 +377,6 @@ jobs:
|
||||
transformers_version: ''
|
||||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||||
|
||||
nightly-poc-multi-node-tests-cann910:
|
||||
name: multi-node-poc-cann910
|
||||
if: ${{ !cancelled() }}
|
||||
# Multi-node jobs run strictly one at a time to avoid contending for the
|
||||
# same A3 multi-node resources: 9.0.0 multi -> 9.0.0 mix -> cann910 multi -> cann910 mix.
|
||||
needs: [set-image-config, nightly-perf-16-npu-a3-cann910, nightly-acc-16-npu-a3-cann910, nightly-poc-multi-node-tests, nightly-poc-multi-node-mix-tests]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
test_config:
|
||||
# deepseek_v4_flash performance tests
|
||||
- name: deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms
|
||||
prefill_size: 1
|
||||
decode_size: 1
|
||||
router_size: 1
|
||||
test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py
|
||||
test_type: 'perf'
|
||||
prefill_decode_deployment: 'separation'
|
||||
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
|
||||
with:
|
||||
runner: linux-amd64-cpu-4
|
||||
test_type: ${{ matrix.test_config.test_type || 'perf' }}
|
||||
test_config_name: ${{ matrix.test_config.name }}
|
||||
prefill_size: ${{ matrix.test_config.prefill_size }}
|
||||
decode_size: ${{ matrix.test_config.decode_size }}
|
||||
router_size: ${{ matrix.test_config.router_size }}
|
||||
test_case: ${{ matrix.test_config.test_case }}
|
||||
image: ${{ needs.set-image-config.outputs.image_a3_cann910 }}
|
||||
install_sglang_from_source: false
|
||||
prefill_decode_deployment: ${{ matrix.test_config.prefill_decode_deployment }}
|
||||
transformers_version: ''
|
||||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||||
|
||||
nightly-poc-multi-node-mix-tests-cann910:
|
||||
name: multi-node-mix-poc-cann910
|
||||
if: false
|
||||
needs: [set-image-config, nightly-poc-multi-node-tests-cann910]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
test_config:
|
||||
# glm_5_2 accuracy tests
|
||||
- name: glm_5_2_w8a8_16p_gpqa
|
||||
node_size: 2
|
||||
test_case: test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w8a8_16p_gpqa.py
|
||||
test_type: 'accuracy'
|
||||
# kimi_k3 accuracy tests
|
||||
- name: kimi_k3_w4a8_32p_gpqa
|
||||
node_size: 4
|
||||
test_case: test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py
|
||||
test_type: 'accuracy'
|
||||
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
|
||||
with:
|
||||
runner: linux-amd64-cpu-4
|
||||
test_type: ${{ matrix.test_config.test_type || 'perf' }}
|
||||
test_config_name: ${{ matrix.test_config.name }}
|
||||
node_size: ${{ matrix.test_config.node_size }}
|
||||
test_case: ${{ matrix.test_config.test_case }}
|
||||
image: ${{ needs.set-image-config.outputs.image_a3_cann910 }}
|
||||
install_sglang_from_source: false
|
||||
prefill_decode_deployment: 'mix'
|
||||
transformers_version: ''
|
||||
run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
|
||||
|
||||
check-all-jobs:
|
||||
if: ${{ !cancelled() }}
|
||||
needs:
|
||||
@@ -523,21 +390,15 @@ jobs:
|
||||
- nightly-perf-16-npu-a3
|
||||
- nightly-acc-2-npu-a3
|
||||
- nightly-acc-16-npu-a3
|
||||
- nightly-perf-16-npu-a3-cann910
|
||||
- nightly-acc-16-npu-a3-cann910
|
||||
- nightly-acc-4-npu-a5
|
||||
- nightly-poc-multi-node-tests
|
||||
- nightly-poc-multi-node-mix-tests
|
||||
- nightly-poc-multi-node-tests-cann910
|
||||
- nightly-poc-multi-node-mix-tests-cann910
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Generate results table
|
||||
run: |
|
||||
multi_result="${{ needs.nightly-poc-multi-node-tests.result }}"
|
||||
mix_result="${{ needs.nightly-poc-multi-node-mix-tests.result }}"
|
||||
multi_cann910_result="${{ needs.nightly-poc-multi-node-tests-cann910.result }}"
|
||||
mix_cann910_result="${{ needs.nightly-poc-multi-node-mix-tests-cann910.result }}"
|
||||
|
||||
# single-node suites are reported per suite; overall status is success if none failed
|
||||
single_result="success"
|
||||
@@ -552,8 +413,6 @@ jobs:
|
||||
"${{ needs.nightly-perf-16-npu-a3.result }}" \
|
||||
"${{ needs.nightly-acc-2-npu-a3.result }}" \
|
||||
"${{ needs.nightly-acc-16-npu-a3.result }}" \
|
||||
"${{ needs.nightly-perf-16-npu-a3-cann910.result }}" \
|
||||
"${{ needs.nightly-acc-16-npu-a3-cann910.result }}" \
|
||||
"${{ needs.nightly-acc-4-npu-a5.result }}"; do
|
||||
if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then
|
||||
single_result="failure"
|
||||
@@ -585,8 +444,6 @@ jobs:
|
||||
"nightly-perf-16-npu-a3:${{ needs.nightly-perf-16-npu-a3.result }}" \
|
||||
"nightly-acc-2-npu-a3:${{ needs.nightly-acc-2-npu-a3.result }}" \
|
||||
"nightly-acc-16-npu-a3:${{ needs.nightly-acc-16-npu-a3.result }}" \
|
||||
"nightly-perf-16-npu-a3-cann910:${{ needs.nightly-perf-16-npu-a3-cann910.result }}" \
|
||||
"nightly-acc-16-npu-a3-cann910:${{ needs.nightly-acc-16-npu-a3-cann910.result }}" \
|
||||
"nightly-acc-4-npu-a5:${{ needs.nightly-acc-4-npu-a5.result }}"; do
|
||||
suite="${entry%%:*}"
|
||||
r="${entry##*:}"
|
||||
@@ -594,12 +451,10 @@ jobs:
|
||||
done
|
||||
echo "| multi-node-poc | $(group_icon ${multi_result}) ${multi_result} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| multi-node-mix-poc | $(group_icon ${mix_result}) ${mix_result} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| multi-node-poc-cann910 | $(group_icon ${multi_cann910_result}) ${multi_cann910_result} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| multi-node-mix-poc-cann910 | $(group_icon ${mix_cann910_result}) ${mix_cann910_result} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
FAIL=0
|
||||
for r in "${single_result}" "${multi_result}" "${mix_result}" "${multi_cann910_result}" "${mix_cann910_result}"; do
|
||||
for r in "${single_result}" "${multi_result}" "${mix_result}"; do
|
||||
if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then FAIL=1; fi
|
||||
done
|
||||
exit $FAIL
|
||||
|
||||
@@ -9,6 +9,10 @@ on:
|
||||
description: "Whether the PR must also have the run-ci-extra label (in addition to run-ci)"
|
||||
type: boolean
|
||||
default: false
|
||||
require-label:
|
||||
description: "Additional label the PR must carry besides run-ci (empty = disabled)"
|
||||
type: string
|
||||
default: ''
|
||||
cool-down-minutes:
|
||||
description: "Cooldown period in minutes for low-permission users; 0 disables rate limiting"
|
||||
type: number
|
||||
@@ -18,10 +22,15 @@ jobs:
|
||||
pr-gate:
|
||||
# 1. for commits on main: no gating needed
|
||||
# 2. for workflow_dispatch: this can only be triggered by users with write access
|
||||
# 3. for pull_request and pull_request_target: apply the checks below.
|
||||
# This job only reads PR metadata through the API and never checks out
|
||||
# or executes PR code, so it is safe for pull_request_target callers.
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
IS_PR_EVENT: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}
|
||||
steps:
|
||||
- name: Fetch latest PR info
|
||||
if: github.event_name == 'pull_request'
|
||||
if: env.IS_PR_EVENT == 'true'
|
||||
id: pr
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
@@ -37,7 +46,7 @@ jobs:
|
||||
core.setOutput("user", pr.data.user.login);
|
||||
|
||||
- name: Log PR info
|
||||
if: github.event_name == 'pull_request'
|
||||
if: env.IS_PR_EVENT == 'true'
|
||||
run: |
|
||||
echo "===== PR Info ====="
|
||||
echo "PR Event: ${{ github.event_name }}"
|
||||
@@ -46,17 +55,18 @@ jobs:
|
||||
echo "PR User: ${{ steps.pr.outputs.user }}"
|
||||
echo "Require run-ci: ${{ inputs.require-run-ci }}"
|
||||
echo "Require run-ci-extra: ${{ inputs.require-run-ci-extra }}"
|
||||
echo "Require label: ${{ inputs.require-label }}"
|
||||
echo "Cool down minutes: ${{ inputs.cool-down-minutes }}"
|
||||
echo "==================="
|
||||
|
||||
- name: Block draft PR
|
||||
if: github.event_name == 'pull_request' && fromJson(steps.pr.outputs.draft)
|
||||
if: env.IS_PR_EVENT == 'true' && fromJson(steps.pr.outputs.draft)
|
||||
run: |
|
||||
echo "PR is draft. Blocking CI."
|
||||
exit 1
|
||||
|
||||
- name: Require run-ci label (optional)
|
||||
if: github.event_name == 'pull_request' && inputs.require-run-ci == true
|
||||
if: env.IS_PR_EVENT == 'true' && inputs.require-run-ci == true
|
||||
run: |
|
||||
labels='${{ steps.pr.outputs.labels }}'
|
||||
if [[ "${{ contains(fromJson(steps.pr.outputs.labels), 'run-ci') }}" == "false" ]]; then
|
||||
@@ -70,15 +80,28 @@ jobs:
|
||||
# then reruns) could never un-skip. A runtime step using gh-api-fetched
|
||||
# labels lets reruns pick up the new label set.
|
||||
- name: Require run-ci-extra label (optional)
|
||||
if: github.event_name == 'pull_request' && inputs.require-run-ci-extra == true
|
||||
if: env.IS_PR_EVENT == 'true' && inputs.require-run-ci-extra == true
|
||||
run: |
|
||||
if [[ "${{ contains(fromJson(steps.pr.outputs.labels), 'run-ci-extra') }}" == "false" ]]; then
|
||||
echo "Missing required label 'run-ci-extra'. Add the label (e.g. via /tag-and-rerun-ci extra) to opt this PR into the extra test workflow."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Generic opt-in label gate (same live-fetch semantics as run-ci-extra).
|
||||
# Lets a workflow require its own label on top of run-ci without adding
|
||||
# a dedicated input per label.
|
||||
- name: Require additional label (optional)
|
||||
if: env.IS_PR_EVENT == 'true' && inputs.require-label != ''
|
||||
env:
|
||||
REQUIRED_LABEL: ${{ inputs.require-label }}
|
||||
run: |
|
||||
if [[ "${{ contains(fromJson(steps.pr.outputs.labels), inputs.require-label) }}" == "false" ]]; then
|
||||
echo "Missing required label '${REQUIRED_LABEL}' (in addition to 'run-ci'). Add the label to opt this PR into this workflow."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Enforce rate limit for low-permission actors (optional)
|
||||
if: github.event_name == 'pull_request' && inputs.cool-down-minutes > 0
|
||||
if: env.IS_PR_EVENT == 'true' && inputs.cool-down-minutes > 0
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -87,6 +110,11 @@ jobs:
|
||||
const owner = context.repo.owner;
|
||||
const repo = context.repo.repo;
|
||||
const eventName = context.eventName;
|
||||
// pr-test.yml runs on pull_request, so pull_request_target callers must
|
||||
// query that event to share the same rate-limit history.
|
||||
const rateLimitEventName = eventName === "pull_request_target"
|
||||
? "pull_request"
|
||||
: eventName;
|
||||
const curRun = await github.rest.actions.getWorkflowRun({
|
||||
owner, repo, run_id: context.runId
|
||||
});
|
||||
@@ -161,14 +189,14 @@ jobs:
|
||||
|
||||
const cutoff = new Date(Date.now() - effectiveCooldownMinutes * 60 * 1000);
|
||||
core.info(
|
||||
`Checking for workflow runs since ${cutoff.toISOString()} (last ${effectiveCooldownMinutes} minutes) for event '${eventName}'.`
|
||||
`Checking for workflow runs since ${cutoff.toISOString()} (last ${effectiveCooldownMinutes} minutes) for event '${rateLimitEventName}'.`
|
||||
);
|
||||
|
||||
const { data } = await github.rest.actions.listWorkflowRuns({
|
||||
owner,
|
||||
repo,
|
||||
workflow_id: 'pr-test.yml',
|
||||
event: eventName,
|
||||
event: rateLimitEventName,
|
||||
per_page: 100,
|
||||
});
|
||||
|
||||
|
||||
@@ -106,29 +106,71 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
CANN_image_a3: ${{ steps.set-vars.outputs.CANN_image_a3 }}
|
||||
CANN_image_910b: ${{ steps.set-vars.outputs.CANN_image_910b }}
|
||||
steps:
|
||||
# When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
|
||||
- name: Set image config
|
||||
id: set-vars
|
||||
run: |
|
||||
echo "CANN_image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
|
||||
echo "CANN_image_910b=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-910b-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
|
||||
echo "CANN_image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.1.0-a3-ubuntu22.04-py3.12" >> $GITHUB_OUTPUT
|
||||
|
||||
base-a-test-1-npu-a2:
|
||||
needs: [check-changes, pr-gate, set-image-config]
|
||||
# Stage-A gate: CPU-only unit tests. The NPU attention-backend UTs stub out
|
||||
# torch_npu / sgl_kernel_npu before importing, so a hosted runner suffices.
|
||||
base-a-test-npu:
|
||||
needs: [check-changes, pr-gate]
|
||||
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.main_package == 'true' }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
self_name: base-a-test-1-npu-a2
|
||||
runner_config: linux-aarch64-a2-1
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_910b }}
|
||||
npu_device_type: 910b
|
||||
run_timeout_minutes: '15'
|
||||
secrets: inherit
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Pin to the event SHA so this job tests the same commit as the rest of the run.
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
# Same gate the NPU stages run first: fail before the NPU jobs are scheduled.
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: 'python/pyproject_npu.toml'
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
# uv pip targets a venv by default; setup-python has no venv.
|
||||
UV_SYSTEM_PYTHON: "1"
|
||||
run: |
|
||||
# Resolve the same dependency set as the NPU stages, minus Ascend-only wheels.
|
||||
rm -f python/pyproject.toml
|
||||
mv python/pyproject_npu.toml python/pyproject.toml
|
||||
# Versions mirror npu_ci_install_dependency.sh's defaults; keep them in sync.
|
||||
# triton: sglang.srt.utils.common imports it unconditionally and the CPU
|
||||
# wheel does not pull it in (the NPU image uses triton-ascend instead).
|
||||
# unsafe-best-match: --index-url is the lowest-priority index, so the
|
||||
# default strategy would resolve PyPI's CUDA torch, not the +cpu build.
|
||||
uv pip install torch==2.10.0 torchvision==0.25.0 torchaudio==2.10.0 triton \
|
||||
--index-url https://download.pytorch.org/whl/cpu \
|
||||
--extra-index-url https://pypi.org/simple/ \
|
||||
--index-strategy unsafe-best-match
|
||||
uv pip install -e "python[dev_npu]" --index-strategy unsafe-best-match
|
||||
|
||||
- name: Run NPU unit tests
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
cd test
|
||||
python3 run_suite.py --hw npu --suite base-a-test-npu
|
||||
|
||||
base-b-test-1-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
@@ -142,7 +184,7 @@ jobs:
|
||||
secrets: inherit
|
||||
|
||||
base-b-test-2-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
@@ -156,7 +198,7 @@ jobs:
|
||||
secrets: inherit
|
||||
|
||||
base-b-test-4-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
@@ -171,7 +213,7 @@ jobs:
|
||||
secrets: inherit
|
||||
|
||||
base-b-test-8-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
@@ -185,7 +227,7 @@ jobs:
|
||||
secrets: inherit
|
||||
|
||||
base-b-test-16-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
@@ -199,7 +241,7 @@ jobs:
|
||||
secrets: inherit
|
||||
|
||||
multimodal-gen-test-1-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
|
||||
runs-on: linux-aarch64-a3-800t-2
|
||||
strategy:
|
||||
@@ -263,7 +305,7 @@ jobs:
|
||||
retention-days: 7
|
||||
|
||||
multimodal-gen-test-4-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-1-npu-a2]
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
|
||||
runs-on: linux-aarch64-a3-800t-4
|
||||
strategy:
|
||||
@@ -483,7 +525,7 @@ jobs:
|
||||
name: Analyze failure report
|
||||
needs:
|
||||
[
|
||||
base-a-test-1-npu-a2,
|
||||
base-a-test-npu,
|
||||
base-b-test-1-npu-a3,
|
||||
base-b-test-2-npu-a3,
|
||||
base-b-test-4-npu-a3,
|
||||
@@ -646,7 +688,7 @@ jobs:
|
||||
[
|
||||
check-changes,
|
||||
|
||||
base-a-test-1-npu-a2,
|
||||
base-a-test-npu,
|
||||
base-b-test-1-npu-a3,
|
||||
base-b-test-2-npu-a3,
|
||||
base-b-test-4-npu-a3,
|
||||
|
||||
@@ -28,8 +28,8 @@ jobs:
|
||||
check-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
changes_exist: ${{ steps.filter.outputs.main_package == 'true' || steps.filter.outputs.multimodal_gen == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
||||
main_package: ${{ steps.filter.outputs.main_package == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
||||
changes_exist: ${{ steps.filter.outputs.main_package == 'true' || steps.filter.outputs.multimodal_gen == 'true' || steps.xpu-tests.outputs.changed == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
||||
main_package: ${{ steps.filter.outputs.main_package == 'true' || steps.xpu-tests.outputs.changed == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
||||
multimodal_gen: ${{ steps.filter.outputs.multimodal_gen == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -54,21 +54,38 @@ jobs:
|
||||
uses: dorny/paths-filter@v3
|
||||
if: steps.run-mode.outputs.run_all_tests != 'true'
|
||||
with:
|
||||
list-files: json
|
||||
filters: |
|
||||
# Baseline list below; any other changed test/registered/ file with a
|
||||
# per-commit register_xpu_ci is also picked up by the next step.
|
||||
registered_tests:
|
||||
- "test/registered/**/*.py"
|
||||
main_package:
|
||||
# Extend test/registered/ entries when adding a non-nightly register_xpu_ci.
|
||||
- "python/sglang/!(multimodal_gen|kernels|cli|test)/**/!(*.md)"
|
||||
- "python/sglang/test/*.py"
|
||||
- "python/sglang/test/!(ascend|observability|mock_model|manual|external_models|kernels)/**/!(*.md)"
|
||||
- "python/pyproject_xpu.toml"
|
||||
- "python/sglang/kernels/ops/speculative/@(reconstruct_tree|ngram_corpus).py"
|
||||
- "python/sglang/kernels/jit/csrc/ngram_corpus/**"
|
||||
- "test/registered/xpu/**/!(*.md)"
|
||||
- "test/registered/e2e/xpu/**/!(*.md)"
|
||||
- "test/registered/disaggregation/test_disaggregation_xpu.py"
|
||||
- "test/registered/attention/test_chunk_gated_delta_rule.py"
|
||||
- "test/registered/attention/test_deterministic.py"
|
||||
- "test/registered/kernels/ops/kvcache/test_hisparse.py"
|
||||
- "test/registered/kv_canary/test_self_unit_future_tensor.py"
|
||||
- "test/registered/kv_canary/test_self_unit_plan_input.py"
|
||||
- "test/registered/kv_canary/test_self_unit_req_to_expected_token_ids_manager.py"
|
||||
- "test/registered/kv_canary/test_self_unit_runner_swa_divergence.py"
|
||||
- "test/registered/layers/mamba/test_mamba_slot_fused.py"
|
||||
- "test/registered/lora/test_moe_lora_info.py"
|
||||
- "test/registered/lora/test_virtual_experts_kernels.py"
|
||||
- "test/registered/unit/layers/test_mamba_state_scatter_triton.py"
|
||||
- "test/registered/unit/managers/test_hisparse_unit.py"
|
||||
- "test/registered/unit/mem_cache/test_mamba_unittest.py"
|
||||
- "test/registered/unit/sampling/test_sampling_params.py"
|
||||
- "test/registered/unit/spec/test_adaptive_spec_params.py"
|
||||
- "test/registered/unit/spec/test_ngram_corpus.py"
|
||||
- "test/run_suite.py"
|
||||
- "test/pytest.ini"
|
||||
- ".github/workflows/pr-test-xpu.yml"
|
||||
@@ -94,6 +111,13 @@ jobs:
|
||||
- "docker/xpu.Dockerfile"
|
||||
- "scripts/ci/xpu/**"
|
||||
|
||||
- name: Detect changed per-commit XPU tests
|
||||
id: xpu-tests
|
||||
if: steps.filter.outputs.registered_tests == 'true'
|
||||
env:
|
||||
CHANGED_FILES: ${{ steps.filter.outputs.registered_tests_files }}
|
||||
run: echo "changed=$(python3 scripts/ci/xpu/detect_xpu_test_changes.py)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# ==================== PR Gate ==================== #
|
||||
pr-gate:
|
||||
needs: check-changes
|
||||
|
||||
@@ -161,7 +161,8 @@ jobs:
|
||||
needs.check-changes.outputs.main_package == 'true' ||
|
||||
needs.check-changes.outputs.sgl_kernel == 'true' ||
|
||||
needs.check-changes.outputs.jit_kernel == 'true' ||
|
||||
needs.check-changes.outputs.multimodal_gen == 'true'
|
||||
needs.check-changes.outputs.multimodal_gen == 'true' ||
|
||||
needs.check-changes.outputs.renderer == 'true'
|
||||
)
|
||||
uses: ./.github/workflows/pr-gate.yml
|
||||
secrets: inherit
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
name: Release Docker Images Nightly Miles ROCm7.2 (AMD)
|
||||
name: Release Docker Images Nightly Miles ROCm7.2.4 (AMD)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
variant:
|
||||
- rocm720-mi35x
|
||||
- rocm724-mi35x
|
||||
steps:
|
||||
- name: Checkout miles repository
|
||||
uses: actions/checkout@v4
|
||||
@@ -1,4 +1,5 @@
|
||||
name: Release Docker Images Nightly (NPU)
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
@@ -8,25 +9,41 @@ on:
|
||||
- 'docker/npu.Dockerfile'
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 14 * * *" # Execute at 10:00 p.m. Beijing Time every day
|
||||
|
||||
- cron: "0 14 * * *"
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-22.04-arm
|
||||
name: build ${{ matrix.platform }}-${{ matrix.device_type }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
cann_version: ["9.0.0"]
|
||||
device_type: ["910b", "a3"]
|
||||
include:
|
||||
- device_type: a3
|
||||
cann_version: "9.1.0"
|
||||
platform: amd64
|
||||
runner: ubuntu-latest
|
||||
- device_type: a3
|
||||
cann_version: "9.1.0"
|
||||
platform: arm64
|
||||
runner: ubuntu-22.04-arm
|
||||
- device_type: 950
|
||||
cann_version: "9.1.0"
|
||||
platform: amd64
|
||||
runner: ubuntu-latest
|
||||
- device_type: 950
|
||||
cann_version: "9.1.0"
|
||||
platform: arm64
|
||||
runner: ubuntu-22.04-arm
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Free up disk space
|
||||
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
|
||||
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
|
||||
with:
|
||||
tool-cache: true
|
||||
docker-images: false
|
||||
@@ -38,48 +55,64 @@ jobs:
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
lmsysorg/sglang
|
||||
# push with schedule event
|
||||
# push with workflow_dispatch event
|
||||
images: lmsysorg/sglang
|
||||
tags: |
|
||||
type=ref,event=pr
|
||||
type=ref,event=branch
|
||||
type=schedule,pattern=main
|
||||
flavor: |
|
||||
latest=false
|
||||
suffix=-cann${{ matrix.cann_version }}-${{ matrix.device_type }},onlatest=true
|
||||
# Login against a Docker registry except on PR
|
||||
# https://github.com/docker/login-action
|
||||
- name: Log into docker hub
|
||||
suffix=-cann${{ matrix.cann_version }}-${{ matrix.device_type }}-${{ matrix.platform }}
|
||||
|
||||
- name: Log into Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
if: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }}
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
# Enable Docker multi-architecture build environment
|
||||
# Emulate non-native architectures
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
# Required for building and pushing multi-arch Docker images
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
# Build and push Docker image with Buildx (don't push on PR)
|
||||
# https://github.com/docker/build-push-action
|
||||
- name: Build and push Docker image
|
||||
id: build-and-push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: docker
|
||||
file: docker/npu.Dockerfile
|
||||
platforms: linux/arm64,linux/amd64
|
||||
platforms: linux/${{ matrix.platform }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
push: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }}
|
||||
outputs: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' && 'type=registry,oci-mediatypes=false' || '' }}
|
||||
provenance: false
|
||||
sbom: false
|
||||
build-args: |
|
||||
SGLANG_KERNEL_NPU_TAG=2026.9.0.post5
|
||||
CANN_VERSION=${{ matrix.cann_version }}
|
||||
DEVICE_TYPE=${{ matrix.device_type }}
|
||||
|
||||
merge:
|
||||
name: merge multi-arch manifest
|
||||
needs: build
|
||||
if: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
cann_version: ["9.1.0"]
|
||||
device_type: ["a3", "950"]
|
||||
steps:
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log into Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Compute base image
|
||||
run: |
|
||||
echo "IMAGE=lmsysorg/sglang:${GITHUB_REF_NAME}-cann${{ matrix.cann_version }}-${{ matrix.device_type }}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Create multi-arch manifest
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t "${IMAGE}" \
|
||||
"${IMAGE}-amd64" \
|
||||
"${IMAGE}-arm64"
|
||||
|
||||
@@ -14,8 +14,8 @@ jobs:
|
||||
runs-on: ubuntu-22.04-arm
|
||||
strategy:
|
||||
matrix:
|
||||
cann_version: ["9.0.0"]
|
||||
device_type: ["910b", "a3"]
|
||||
cann_version: ["9.1.0"]
|
||||
device_type: ["a3", "950"]
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
name: Publish Renderer Docker Image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ["v[0-9]+.*"]
|
||||
paths:
|
||||
- "rust/sglang-renderer/**"
|
||||
- "rust/Cargo.toml"
|
||||
- "rust/Cargo.lock"
|
||||
- "rust/rust-toolchain.toml"
|
||||
- "docker/renderer.Dockerfile"
|
||||
- "LICENSE"
|
||||
- ".github/workflows/release-docker-renderer.yml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
if: github.repository == 'sgl-project/sglang'
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
publish: ${{ steps.image.outputs.publish }}
|
||||
repo: ${{ steps.image.outputs.repo }}
|
||||
tag: ${{ steps.image.outputs.tag }}
|
||||
steps:
|
||||
- name: Select image destination
|
||||
id: image
|
||||
run: |
|
||||
if [[ "$GITHUB_REF_TYPE" == "tag" ]]; then
|
||||
if [[ ! "$GITHUB_REF_NAME" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(\.post[0-9]+)?$ ]]; then
|
||||
echo "Skipping non-final release tag $GITHUB_REF_NAME"
|
||||
echo "publish=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "repo=lmsysorg/sglang" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=${GITHUB_REF_NAME}-renderer" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "repo=lmsysorg/sglang-staging" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=renderer-${GITHUB_SHA:0:8}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
echo "publish=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
build:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.publish == 'true'
|
||||
environment: ${{ github.ref_type == 'tag' && 'prod' || null }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-24.04
|
||||
arch: amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
runs-on: ${{ matrix.runner }}
|
||||
env:
|
||||
IMAGE_REPO: ${{ needs.prepare.outputs.repo }}
|
||||
IMAGE_TAG: ${{ needs.prepare.outputs.tag }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Run renderer tests
|
||||
run: |
|
||||
docker buildx build . -f docker/renderer.Dockerfile \
|
||||
--platform ${{ matrix.platform }} --target test --progress=plain
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Build and push by digest
|
||||
run: |
|
||||
BUILD_COMMIT=$(git rev-parse HEAD)
|
||||
BUILD_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
docker buildx build . -f docker/renderer.Dockerfile \
|
||||
--platform ${{ matrix.platform }} --target runtime --progress=plain \
|
||||
--build-arg "SGLANG_BUILD_COMMIT=${BUILD_COMMIT}" \
|
||||
--build-arg "SGLANG_BUILD_URL=${BUILD_URL}" \
|
||||
--build-arg "SGLANG_IMAGE_TAG=${IMAGE_REPO}:${IMAGE_TAG}" \
|
||||
--output "type=image,name=${IMAGE_REPO},push-by-digest=true,name-canonical=true,push=true" \
|
||||
--metadata-file /tmp/renderer-metadata.json
|
||||
DIGEST=$(python3 -c "import json; print(json.load(open('/tmp/renderer-metadata.json'))['containerimage.digest'])")
|
||||
docker run --rm "${IMAGE_REPO}@${DIGEST}" --help
|
||||
mkdir -p /tmp/renderer-digests
|
||||
echo "$DIGEST" > "/tmp/renderer-digests/${{ matrix.arch }}"
|
||||
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: renderer-digest-${{ matrix.arch }}
|
||||
path: /tmp/renderer-digests/${{ matrix.arch }}
|
||||
retention-days: 7
|
||||
|
||||
publish:
|
||||
needs: [prepare, build]
|
||||
if: needs.prepare.outputs.publish == 'true'
|
||||
environment: ${{ github.ref_type == 'tag' && 'prod' || null }}
|
||||
runs-on: ubuntu-24.04
|
||||
env:
|
||||
IMAGE_REPO: ${{ needs.prepare.outputs.repo }}
|
||||
IMAGE_TAG: ${{ needs.prepare.outputs.tag }}
|
||||
steps:
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: /tmp/renderer-digests
|
||||
pattern: renderer-digest-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Publish multi-architecture tag
|
||||
run: |
|
||||
AMD64_DIGEST=$(cat /tmp/renderer-digests/amd64)
|
||||
ARM64_DIGEST=$(cat /tmp/renderer-digests/arm64)
|
||||
docker buildx imagetools create \
|
||||
-t "${IMAGE_REPO}:${IMAGE_TAG}" \
|
||||
"${IMAGE_REPO}@${AMD64_DIGEST}" \
|
||||
"${IMAGE_REPO}@${ARM64_DIGEST}"
|
||||
docker buildx imagetools inspect "${IMAGE_REPO}:${IMAGE_TAG}"
|
||||
@@ -86,8 +86,43 @@ permissions:
|
||||
issues: read
|
||||
|
||||
jobs:
|
||||
prepare-partitions:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
matrix: ${{ steps.partition.outputs.matrix }}
|
||||
steps:
|
||||
- name: Checkout partition script
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Checkout test registrations
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.pr_head_sha || github.sha }}
|
||||
path: partition-source
|
||||
persist-credentials: false
|
||||
|
||||
- name: Partition tests by estimated duration
|
||||
id: partition
|
||||
env:
|
||||
TEST_COMMAND: ${{ inputs.test_command }}
|
||||
TEST_ROOT: partition-source
|
||||
MODE: ${{ inputs.mode }}
|
||||
RUNS_ON: ${{ inputs.runs_on }}
|
||||
run: python3 scripts/ci/partition_rerun_tests.py
|
||||
|
||||
rerun-test-cuda:
|
||||
name: rerun-test-cuda (partition ${{ matrix.partition }})
|
||||
needs: prepare-partitions
|
||||
if: inputs.mode == 'cuda'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJson(needs.prepare-partitions.outputs.matrix) }}
|
||||
runs-on: ${{ inputs.runs_on }}
|
||||
timeout-minutes: 120
|
||||
permissions:
|
||||
@@ -150,7 +185,8 @@ jobs:
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
TEST_COMMAND: ${{ inputs.test_command }}
|
||||
TEST_COMMAND: ${{ matrix.test_command }}
|
||||
REQUESTED_TEST_COMMAND: ${{ inputs.test_command }}
|
||||
SGLANG_PRECISION_HF_TOKEN: ${{ inputs.refresh_precision_baseline && inputs.mode == 'cuda' && inputs.runs_on == '8-gpu-h200' && inputs.test_command == 'registered/debug_utils/test_nightly_precision_regression.py' && inputs.pr_head_sha == '' && secrets.HF_TOKEN_PRECISION_STORE || '' }}
|
||||
run: |
|
||||
if [[ "${{ inputs.refresh_precision_baseline }}" == "true" ]]; then
|
||||
@@ -158,7 +194,7 @@ jobs:
|
||||
if [[ "${{ inputs.mode }}" != "cuda" \
|
||||
|| "${{ inputs.runs_on }}" != "8-gpu-h200" \
|
||||
|| -n "${{ inputs.pr_head_sha }}" \
|
||||
|| "$TEST_COMMAND" != "$expected" ]]; then
|
||||
|| "$REQUESTED_TEST_COMMAND" != "$expected" ]]; then
|
||||
echo "::error::Precision baseline refresh only accepts $expected on 8-gpu-h200"
|
||||
exit 1
|
||||
fi
|
||||
@@ -172,6 +208,7 @@ jobs:
|
||||
cmds+=("$cmd")
|
||||
done <<< "$TEST_COMMAND"
|
||||
total=${#cmds[@]}
|
||||
failed=()
|
||||
suite_start=$SECONDS
|
||||
for idx in "${!cmds[@]}"; do
|
||||
i=$((idx + 1))
|
||||
@@ -189,7 +226,9 @@ jobs:
|
||||
if grep -qE 'register_cuda_ci\(.*stage[[:space:]]*=[[:space:]]*"base-a"' "test/$test_file" 2>/dev/null; then
|
||||
async_assert=false
|
||||
fi
|
||||
(cd test/ && SGLANG_ENABLE_ASYNC_ASSERT=$async_assert python3 $cmd -f) || exit 1
|
||||
if ! (cd test/ && SGLANG_ENABLE_ASYNC_ASSERT=$async_assert python3 $cmd -f); then
|
||||
failed+=("$cmd")
|
||||
fi
|
||||
elapsed=$(( SECONDS - file_start ))
|
||||
echo "."
|
||||
echo "End ($i/$total): elapsed=${elapsed}s"
|
||||
@@ -197,13 +236,29 @@ jobs:
|
||||
echo ""
|
||||
done
|
||||
total_elapsed=$(( SECONDS - suite_start ))
|
||||
if [ ${#failed[@]} -gt 0 ]; then
|
||||
echo "${#failed[@]}/$total test(s) failed in ${total_elapsed}s:"
|
||||
printf ' %s\n' "${failed[@]}"
|
||||
{
|
||||
echo "### Failed tests (partition ${{ matrix.partition }})"
|
||||
printf -- '- `%s`\n' "${failed[@]}"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
exit 1
|
||||
fi
|
||||
echo "All $total test(s) passed in ${total_elapsed}s"
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: failure()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.partition }}
|
||||
|
||||
rerun-test-multimodal-gen:
|
||||
name: rerun-test-multimodal-gen (partition ${{ matrix.partition }})
|
||||
needs: prepare-partitions
|
||||
if: inputs.mode == 'multimodal_gen'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJson(needs.prepare-partitions.outputs.matrix) }}
|
||||
runs-on: ${{ inputs.runs_on }}
|
||||
timeout-minutes: 120
|
||||
permissions:
|
||||
@@ -244,13 +299,16 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
TEST_COMMAND: ${{ matrix.test_command }}
|
||||
run: |
|
||||
cmds=()
|
||||
while IFS= read -r cmd; do
|
||||
[ -z "$cmd" ] && continue
|
||||
cmds+=("$cmd")
|
||||
done <<< "${{ inputs.test_command }}"
|
||||
done <<< "$TEST_COMMAND"
|
||||
total=${#cmds[@]}
|
||||
failed=()
|
||||
suite_start=$SECONDS
|
||||
for idx in "${!cmds[@]}"; do
|
||||
i=$((idx + 1))
|
||||
@@ -260,7 +318,9 @@ jobs:
|
||||
echo "Begin ($i/$total): python3 -m pytest $cmd -x"
|
||||
echo "."
|
||||
file_start=$SECONDS
|
||||
python3 -m pytest $cmd -x || exit 1
|
||||
if ! python3 -m pytest $cmd -x; then
|
||||
failed+=("$cmd")
|
||||
fi
|
||||
elapsed=$(( SECONDS - file_start ))
|
||||
echo "."
|
||||
echo "End ($i/$total): elapsed=${elapsed}s"
|
||||
@@ -268,13 +328,29 @@ jobs:
|
||||
echo ""
|
||||
done
|
||||
total_elapsed=$(( SECONDS - suite_start ))
|
||||
if [ ${#failed[@]} -gt 0 ]; then
|
||||
echo "${#failed[@]}/$total test(s) failed in ${total_elapsed}s:"
|
||||
printf ' %s\n' "${failed[@]}"
|
||||
{
|
||||
echo "### Failed tests (partition ${{ matrix.partition }})"
|
||||
printf -- '- `%s`\n' "${failed[@]}"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
exit 1
|
||||
fi
|
||||
echo "All $total test(s) passed in ${total_elapsed}s"
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: failure()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.partition }}
|
||||
|
||||
rerun-test-cpu:
|
||||
name: rerun-test-cpu (partition ${{ matrix.partition }})
|
||||
needs: prepare-partitions
|
||||
if: inputs.mode == 'cpu'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJson(needs.prepare-partitions.outputs.matrix) }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 120
|
||||
permissions:
|
||||
@@ -343,14 +419,17 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
TEST_COMMAND: ${{ matrix.test_command }}
|
||||
run: |
|
||||
cd test/
|
||||
cmds=()
|
||||
while IFS= read -r cmd; do
|
||||
[ -z "$cmd" ] && continue
|
||||
cmds+=("$cmd")
|
||||
done <<< "${{ inputs.test_command }}"
|
||||
done <<< "$TEST_COMMAND"
|
||||
total=${#cmds[@]}
|
||||
failed=()
|
||||
suite_start=$SECONDS
|
||||
for idx in "${!cmds[@]}"; do
|
||||
i=$((idx + 1))
|
||||
@@ -360,7 +439,9 @@ jobs:
|
||||
echo "Begin ($i/$total): python3 $cmd"
|
||||
echo "."
|
||||
file_start=$SECONDS
|
||||
python3 $cmd -f || exit 1
|
||||
if ! python3 $cmd -f; then
|
||||
failed+=("$cmd")
|
||||
fi
|
||||
elapsed=$(( SECONDS - file_start ))
|
||||
echo "."
|
||||
echo "End ($i/$total): elapsed=${elapsed}s"
|
||||
@@ -368,10 +449,19 @@ jobs:
|
||||
echo ""
|
||||
done
|
||||
total_elapsed=$(( SECONDS - suite_start ))
|
||||
if [ ${#failed[@]} -gt 0 ]; then
|
||||
echo "${#failed[@]}/$total test(s) failed in ${total_elapsed}s:"
|
||||
printf ' %s\n' "${failed[@]}"
|
||||
{
|
||||
echo "### Failed tests (partition ${{ matrix.partition }})"
|
||||
printf -- '- `%s`\n' "${failed[@]}"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
exit 1
|
||||
fi
|
||||
echo "All $total test(s) passed in ${total_elapsed}s"
|
||||
|
||||
write-back-result:
|
||||
needs: [rerun-test-cuda, rerun-test-multimodal-gen, rerun-test-cpu]
|
||||
needs: [prepare-partitions, rerun-test-cuda, rerun-test-multimodal-gen, rerun-test-cpu]
|
||||
if: always() && inputs.reply_comment_id != '' && inputs.reply_marker != ''
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
|
||||
@@ -166,8 +166,6 @@ cython_debug/
|
||||
*.swp
|
||||
|
||||
# SGL
|
||||
benchmark/mmlu/data
|
||||
benchmark/mmlu/data.tar
|
||||
benchmark/llava_bench/images
|
||||
benchmark/llava_bench/mme_pack
|
||||
*.jsonl
|
||||
|
||||
@@ -35,7 +35,7 @@ repos:
|
||||
hooks:
|
||||
- id: ruff
|
||||
args:
|
||||
- --select=F401,F821,UP037
|
||||
- --select=F401,F821,F823,UP037
|
||||
- --fix
|
||||
files: ^(benchmark/|docs/|examples/|python/sglang/|sgl-model-gateway/py_*|test/)
|
||||
exclude: |
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
# Gitea Actions 用(海外节点):原版 docker.io 基底 + 原版 pypi
|
||||
# 与 b300 离线版(/data/ymk/build/Dockerfile)的区别:
|
||||
# - 基底直用 docker.io 的 lmsysorg/sglang:dev-dsv41(不走 umirror/DaoCloud)
|
||||
# - 不设 PIP_INDEX_URL,用默认 pypi.org
|
||||
# - 源码由 CI checkout 后经 COPY 进镜像(不用 in-image clone)
|
||||
FROM lmsysorg/sglang:dev-dsv41
|
||||
|
||||
# checkout 含 .git,setuptools-scm 可打戳(main 系分支 version 显示 dev 属正常)
|
||||
COPY . /sgl-workspace/sglang
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译,与 b300 离线版 deploy/b300-build/Dockerfile 同构。
|
||||
# 基底自带 rustup stable,RUSTUP_TOOLCHAIN=stable 绕过 rust-toolchain.toml 的 1.90 钉版;
|
||||
# python 扩展必须显式 -p,产物重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件。
|
||||
# 海外 runner 直连 crates.io 无网络问题;cache mount 海外环境同样无害。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
RUN pip install --no-cache-dir -e /sgl-workspace/sglang/python
|
||||
@@ -75,6 +75,7 @@ Its core features include:
|
||||
## Getting Started
|
||||
- [Install SGLang](https://docs.sglang.io/get_started/install.html)
|
||||
- [Quick Start](https://docs.sglang.io/basic_usage/send_request.html)
|
||||
- [Cookbook](https://docs.sglang.io/cookbook) — the deployment command we recommend for each supported model
|
||||
- [Backend Tutorial](https://docs.sglang.io/basic_usage/openai_api_completions.html)
|
||||
- [Frontend Tutorial](https://docs.sglang.io/references/frontend/frontend_tutorial.html)
|
||||
- [Contribution Guide](https://docs.sglang.io/developer_guide/contribution_guide.html)
|
||||
|
||||
@@ -271,7 +271,9 @@ Then we can benchmark the accuracy and latency by accessing the first node's exp
|
||||
|
||||
```bash
|
||||
# bench accuracy
|
||||
python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319 --host 10.0.0.1 --port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://10.0.0.1:30000/v1 \
|
||||
--num-examples 1319 --num-threads 64
|
||||
|
||||
# bench latency
|
||||
python3 -m sglang.bench_one_batch_server --model None --base-url http://10.0.0.1:30000 --batch-size 1 --input-len 128 --output-len 128
|
||||
@@ -322,13 +324,15 @@ Then on the **master node**, supposing the ShareGPT data is located at `/path/to
|
||||
|
||||
```bash
|
||||
# bench accuracy
|
||||
python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 1319 --num-threads 64
|
||||
|
||||
# bench serving
|
||||
python3 -m sglang.bench_serving --dataset-path /path/to/ShareGPT_V3_unfiltered_cleaned_split.json --dataset-name random --random-input 128 --random-output 128 --num-prompts 1000 --request-rate 128 --random-range-ratio 1.0
|
||||
```
|
||||
|
||||
> **Note: using `--parallel 200` can accelerate accuracy benchmarking**.
|
||||
> **Note: using `--num-threads 200` can accelerate accuracy benchmarking**.
|
||||
|
||||
### Example: Serving with 32 L40S with int8 Quantization
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
# DiffusionGemma serving measurements
|
||||
|
||||
`bench_diffusion_gemma.py` exercises a running server through its completions API using pretokenized chat prompts. For completed cases it records trial timings, returned text, prompt hashes, and usage counts. It rejects empty responses and unexpected token counts.
|
||||
|
||||
Start the server with the checkpoint and supported serving defaults:
|
||||
|
||||
```sh
|
||||
python -m sglang.launch_server --model-path "$MODEL" \
|
||||
--dllm-algorithm Gemma4Renoise
|
||||
```
|
||||
|
||||
Point the client at that server and the same tokenizer:
|
||||
|
||||
```sh
|
||||
python benchmark/dllm/bench_diffusion_gemma.py \
|
||||
--url http://127.0.0.1:30000 --tokenizer "$MODEL" \
|
||||
--input-lengths 128 2048 --concurrencies 1 4 8 --output-length 256 \
|
||||
--run-id serving-check --result result.json
|
||||
```
|
||||
|
||||
## Interpretation
|
||||
|
||||
The client disables EOS stopping to request a fixed output length. Canvas positions after EOS count toward the reported token rate, which is not useful-text throughput. `--denoising-steps` records metadata and does not configure or verify the server's convergence settings. Its default is unset. Set it only when the actual server work has been verified. If a run fails, keep both server and client logs, since the result file contains only completed cases.
|
||||
|
||||
For comparisons, freeze the protocol before collecting results. Match checkpoint identity, physical hardware, precision, prompts, resource limits, generation settings, and stopping policy. Use the same run ID to generate identical prompts. Give each runtime equivalent access to its normal compilation and graph features, preserve every trial and failed request, and disclose dependency and implementation differences. Validate normal text, images, mixed requests, and multi-block generation separately.
|
||||
|
||||
Performance for the merged revision remains unmeasured. Earlier measurements and test counts do not validate this revision. A fixed-work experiment does not establish adaptive-generation performance, model accuracy, or a serving performance lead.
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Compare fixed-work DiffusionGemma serving through the completions API.
|
||||
|
||||
Both servers must use the same checkpoint, canvas length, denoising limit and
|
||||
stopping settings. Set confidence_threshold=0.0 and stability_threshold=1 on
|
||||
both servers so early convergence cannot shorten work. This keeps the normal
|
||||
history size instead of adding history work to enforce the step count. Run them
|
||||
sequentially on the same physical GPUs, using the same run ID and arguments.
|
||||
Each request has a unique prompt prefix to avoid prefix-cache reuse.
|
||||
|
||||
This measures complete canvas computation with EOS stopping disabled. Canvas
|
||||
tokens per second include tokens after EOS and are not useful-text throughput.
|
||||
Validate output quality separately with the model's default generation settings.
|
||||
The denoising-steps argument records the server configuration; it does not change it.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import statistics
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import aiohttp
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
|
||||
async def benchmark(args):
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer)
|
||||
body = tokenizer.encode(
|
||||
"The city library has books about science, history, music and art. " * 256,
|
||||
add_special_tokens=False,
|
||||
)
|
||||
template = tokenizer.apply_chat_template(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Request __REQUEST_ID__.\n__BENCHMARK_BODY__\n"
|
||||
"Write a detailed explanation of why public libraries are useful."
|
||||
),
|
||||
}
|
||||
],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False,
|
||||
)
|
||||
prefix_template, suffix_text = template.split("__BENCHMARK_BODY__")
|
||||
suffix = tokenizer.encode(suffix_text, add_special_tokens=False)
|
||||
results = []
|
||||
request_number = 0
|
||||
timeout = aiohttp.ClientTimeout(total=600)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.get(args.url + "/v1/models") as response:
|
||||
response.raise_for_status()
|
||||
model = (await response.json())["data"][0]["id"]
|
||||
|
||||
async def request(prompt):
|
||||
start = time.perf_counter()
|
||||
async with session.post(
|
||||
args.url + "/v1/completions",
|
||||
json={
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"max_tokens": args.output_length,
|
||||
"ignore_eos": True,
|
||||
"stream": False,
|
||||
},
|
||||
) as response:
|
||||
payload = await response.json()
|
||||
if response.status != 200:
|
||||
raise RuntimeError(f"HTTP {response.status}: {payload}")
|
||||
elapsed = time.perf_counter() - start
|
||||
if not payload["choices"][0]["text"].strip():
|
||||
raise RuntimeError(
|
||||
"Empty completion; check the chat template and model"
|
||||
)
|
||||
usage = payload["usage"]
|
||||
if usage["completion_tokens"] != args.output_length:
|
||||
raise RuntimeError(f"Unexpected output length: {usage}")
|
||||
if usage["prompt_tokens"] != len(prompt):
|
||||
raise RuntimeError(f"Unexpected prompt length: {usage}")
|
||||
return {
|
||||
"latency_s": elapsed,
|
||||
"usage": usage,
|
||||
"text": payload["choices"][0]["text"],
|
||||
"prompt_sha256": hashlib.sha256(
|
||||
json.dumps(prompt).encode()
|
||||
).hexdigest(),
|
||||
}
|
||||
|
||||
for input_length in args.input_lengths:
|
||||
for concurrency in args.concurrencies:
|
||||
trials = []
|
||||
for trial in range(args.warmups + args.trials):
|
||||
prompts = []
|
||||
for _ in range(concurrency):
|
||||
# Vary the beginning of every prompt to avoid measuring
|
||||
# cached prompts. Keep run_id identical across runtimes.
|
||||
prefix = tokenizer.encode(
|
||||
prefix_template.replace(
|
||||
"__REQUEST_ID__", f"{args.run_id}-{request_number}"
|
||||
),
|
||||
add_special_tokens=False,
|
||||
)
|
||||
request_number += 1
|
||||
n = input_length - len(prefix) - len(suffix)
|
||||
if n < 0:
|
||||
raise ValueError("Input length is too short")
|
||||
prompts.append(prefix + body[:n] + suffix)
|
||||
start = time.perf_counter()
|
||||
requests = await asyncio.gather(*(request(p) for p in prompts))
|
||||
trials.append(
|
||||
{
|
||||
"warmup": trial < args.warmups,
|
||||
"wall_s": time.perf_counter() - start,
|
||||
"requests": requests,
|
||||
}
|
||||
)
|
||||
measured = trials[args.warmups :]
|
||||
wall_times = [t["wall_s"] for t in measured]
|
||||
latencies = [r["latency_s"] for t in measured for r in t["requests"]]
|
||||
result = {
|
||||
"input_length": input_length,
|
||||
"output_length": args.output_length,
|
||||
"concurrency": concurrency,
|
||||
"median_wall_s": statistics.median(wall_times),
|
||||
"median_request_s": statistics.median(latencies),
|
||||
"canvas_tokens_per_s": concurrency
|
||||
* args.output_length
|
||||
/ statistics.median(wall_times),
|
||||
"trials": trials,
|
||||
}
|
||||
results.append(result)
|
||||
print(
|
||||
json.dumps({k: v for k, v in result.items() if k != "trials"}),
|
||||
flush=True,
|
||||
)
|
||||
args.result.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"settings": vars(args) | {"result": str(args.result)},
|
||||
"results": results,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--url", required=True)
|
||||
parser.add_argument("--tokenizer", required=True)
|
||||
parser.add_argument("--input-lengths", type=int, nargs="+", default=[128, 512])
|
||||
parser.add_argument("--output-length", type=int, default=256)
|
||||
parser.add_argument(
|
||||
"--denoising-steps",
|
||||
type=int,
|
||||
help="Verified fixed denoising step count, recorded as metadata only",
|
||||
)
|
||||
parser.add_argument("--concurrencies", type=int, nargs="+", default=[1, 4, 8])
|
||||
parser.add_argument("--warmups", type=int, default=3)
|
||||
parser.add_argument("--trials", type=int, default=15)
|
||||
parser.add_argument("--run-id", required=True)
|
||||
parser.add_argument("--result", type=Path, required=True)
|
||||
asyncio.run(benchmark(parser.parse_args()))
|
||||
@@ -1,20 +0,0 @@
|
||||
## Run benchmark
|
||||
|
||||
### Using GSM8K Platinum
|
||||
|
||||
GSM8K Platinum is a revised version of the GSM8K test set with corrected labels and removed ambiguous questions. It can be more stable than the original GSM8K dataset. It's a drop-in replacement that can be used by adding the `--platinum` flag:
|
||||
|
||||
```
|
||||
python3 bench_sglang.py --num-shots 8 --num-questions 1209 --parallel 1209 --platinum
|
||||
```
|
||||
|
||||
For more information, see: https://huggingface.co/datasets/madrylab/gsm8k-platinum
|
||||
|
||||
### Benchmark sglang
|
||||
```
|
||||
python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000
|
||||
```
|
||||
|
||||
```
|
||||
python3 bench_sglang.py --num-questions 200
|
||||
```
|
||||
@@ -1,207 +0,0 @@
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
from datasets import load_dataset
|
||||
|
||||
from sglang.lang.api import set_default_backend
|
||||
from sglang.test.test_utils import (
|
||||
add_common_sglang_args_and_parse,
|
||||
dump_bench_raw_result,
|
||||
select_sglang_backend,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, dump_state_text, read_jsonl
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def main(args):
|
||||
# Select backend
|
||||
set_default_backend(select_sglang_backend(args))
|
||||
|
||||
# Load tokenizer if enable_thinking is set
|
||||
tokenizer = None
|
||||
if args.enable_thinking:
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
assert args.tokenizer_path is not None, (
|
||||
"--tokenizer-path is required when --enable-thinking is set"
|
||||
)
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
args.tokenizer_path, trust_remote_code=True
|
||||
)
|
||||
|
||||
# Read data
|
||||
if args.platinum:
|
||||
print("Loading GSM8K Platinum dataset from HuggingFace...")
|
||||
dataset = load_dataset("madrylab/gsm8k-platinum", "main", split="test")
|
||||
lines = [
|
||||
{"question": item["question"], "answer": item["answer"]} for item in dataset
|
||||
]
|
||||
else:
|
||||
data_path = args.data_path
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
if not os.path.isfile(data_path):
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
# Construct prompts
|
||||
num_questions = args.num_questions
|
||||
num_shots = args.num_shots
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
raw_question = few_shot_examples + get_one_example(lines, i, False)
|
||||
if tokenizer is not None:
|
||||
messages = [{"role": "user", "content": raw_question}]
|
||||
raw_question = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
**args.chat_template_kwargs,
|
||||
)
|
||||
questions.append(raw_question)
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
assert all(l != INVALID for l in labels)
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
#####################################
|
||||
######### SGL Program Begin #########
|
||||
#####################################
|
||||
|
||||
import sglang as sgl
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += question
|
||||
s += sgl.gen(
|
||||
"answer",
|
||||
max_tokens=args.max_new_tokens,
|
||||
stop=["Question", "Assistant:", "<|separator|>"],
|
||||
)
|
||||
|
||||
#####################################
|
||||
########## SGL Program End ##########
|
||||
#####################################
|
||||
|
||||
# Run requests
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments,
|
||||
temperature=args.temperature,
|
||||
top_p=args.top_p,
|
||||
num_threads=args.parallel,
|
||||
progress_bar=True,
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = []
|
||||
for i in range(len(states)):
|
||||
preds.append(get_answer_value(states[i]["answer"]))
|
||||
|
||||
# Compute accuracy
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid = np.mean(np.array(preds) == INVALID)
|
||||
|
||||
# Compute speed
|
||||
num_output_tokens = sum(
|
||||
s.get_meta_info("answer")["completion_tokens"] for s in states
|
||||
)
|
||||
output_throughput = num_output_tokens / latency
|
||||
|
||||
# Print results
|
||||
print(f"Accuracy: {acc:.3f}")
|
||||
print(f"Invalid: {invalid:.3f}")
|
||||
print(f"Latency: {latency:.3f} s")
|
||||
print(f"Output throughput: {output_throughput:.3f} token/s")
|
||||
|
||||
# Dump results
|
||||
dump_state_text(f"tmp_output_{args.backend}.txt", states)
|
||||
dump_bench_raw_result(
|
||||
path=args.raw_result_file,
|
||||
states=states,
|
||||
preds=preds,
|
||||
labels=labels,
|
||||
)
|
||||
|
||||
with open(args.result_file, "a") as fout:
|
||||
value = {
|
||||
"task": "gsm8k-platinum" if args.platinum else "gsm8k",
|
||||
"backend": args.backend,
|
||||
"num_gpus": 1,
|
||||
"latency": round(latency, 3),
|
||||
"accuracy": round(acc, 3),
|
||||
"num_requests": args.num_questions,
|
||||
"other": {
|
||||
"num_questions": args.num_questions,
|
||||
"parallel": args.parallel,
|
||||
},
|
||||
}
|
||||
fout.write(json.dumps(value) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--num-shots", type=int, default=5)
|
||||
parser.add_argument("--data-path", type=str, default="test.jsonl")
|
||||
parser.add_argument("--num-questions", type=int, default=200)
|
||||
parser.add_argument("--max-new-tokens", type=int, default=512)
|
||||
parser.add_argument("--temperature", type=float, default=0.0)
|
||||
parser.add_argument("--top-p", type=float, default=1.0)
|
||||
parser.add_argument(
|
||||
"--enable-thinking",
|
||||
action="store_true",
|
||||
help="Enable thinking mode by wrapping prompts with chat template",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chat-template-kwargs",
|
||||
type=json.loads,
|
||||
default='{"enable_thinking": true}',
|
||||
help="JSON dict passed through to tokenizer.apply_chat_template. "
|
||||
"The thinking-toggle kwarg name is model-specific, e.g. "
|
||||
"'{\"enable_thinking\": true}' (Qwen) or '{\"thinking\": true}' (Kimi).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tokenizer-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to tokenizer (required when --enable-thinking is set)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--platinum",
|
||||
action="store_true",
|
||||
help="Use GSM8K Platinum dataset (drop-in replacement with corrected labels)",
|
||||
)
|
||||
args = add_common_sglang_args_and_parse(parser)
|
||||
main(args)
|
||||
@@ -0,0 +1,198 @@
|
||||
"""CPU comparison of baseline and optimized offset-based scheduler padding.
|
||||
|
||||
Run from the repository root:
|
||||
PYTHONPATH=python python benchmark/io/bench_mm_token_padding.py --baseline-ref BASE_SHA
|
||||
|
||||
Inputs are synthetic. Both paths include the scheduler's final ownership copy.
|
||||
This measures padding CPU time, not model forward time or end-to-end TTFT.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import gc
|
||||
import json
|
||||
import platform
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
from array import array
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers import mm_utils
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
duplicate_pad_mm_input_ids,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
MultimodalInputs,
|
||||
)
|
||||
|
||||
|
||||
def load_baseline(ref):
|
||||
"""Load the exact baseline helper classes without replacing installed SGLang."""
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
commit = subprocess.check_output(
|
||||
["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], cwd=root, text=True
|
||||
).strip()
|
||||
path = "python/sglang/srt/managers/mm_utils.py"
|
||||
source = subprocess.check_output(
|
||||
["git", "show", f"{commit}:{path}"], cwd=root, text=True
|
||||
)
|
||||
names = {
|
||||
"MultiModalityDataPaddingPattern",
|
||||
"MultiModalityDataPaddingPatternMultimodalTokens",
|
||||
}
|
||||
classes = [
|
||||
node
|
||||
for node in ast.parse(source).body
|
||||
if isinstance(node, ast.ClassDef) and node.name in names
|
||||
]
|
||||
if {node.name for node in classes} != names:
|
||||
raise ValueError("Baseline does not contain the expected padding helpers")
|
||||
namespace = dict(vars(mm_utils))
|
||||
# Reuse dependency imports, but compile the original method bodies verbatim.
|
||||
exec(
|
||||
compile(ast.Module(body=classes, type_ignores=[]), f"{commit}:{path}", "exec"),
|
||||
namespace,
|
||||
)
|
||||
return commit, namespace["MultiModalityDataPaddingPatternMultimodalTokens"]()
|
||||
|
||||
|
||||
def make_case(num_tokens, num_items, span_tokens):
|
||||
if num_items and num_tokens < 3 * num_items:
|
||||
raise ValueError(
|
||||
"Each item needs three tokens for its span and surrounding text"
|
||||
)
|
||||
input_ids = array("q", range(257, 257 + num_tokens))
|
||||
items = []
|
||||
stride = num_tokens // max(num_items, 1)
|
||||
for index in range(num_items):
|
||||
start = index * stride + 1
|
||||
end = start + min(span_tokens, stride - 2) - 1
|
||||
items.append(
|
||||
MultimodalDataItem(
|
||||
modality=Modality.IMAGE,
|
||||
pad_value=2**40 + index,
|
||||
offsets=[(start, end)],
|
||||
)
|
||||
)
|
||||
return input_ids, MultimodalInputs(mm_items=items, im_token_id=1)
|
||||
|
||||
|
||||
def benchmark_case(num_tokens, num_items, args, baseline):
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
input_ids, mm_inputs = make_case(num_tokens, num_items, args.span_tokens)
|
||||
original = input_ids.tobytes()
|
||||
|
||||
def baseline_padding():
|
||||
return array("q", baseline.pad_input_tokens(input_ids, mm_inputs))
|
||||
|
||||
def optimized_padding():
|
||||
return duplicate_pad_mm_input_ids(
|
||||
input_ids, mm_inputs, pattern.pad_input_tokens
|
||||
)
|
||||
|
||||
expected = baseline_padding()
|
||||
actual = optimized_padding()
|
||||
assert actual == expected, "Token values differ"
|
||||
assert input_ids.tobytes() == original, "Original input was mutated"
|
||||
assert actual is not input_ids, "Scheduler result aliases the original input"
|
||||
|
||||
functions = {"baseline": baseline_padding, "optimized": optimized_padding}
|
||||
timings = {name: [] for name in functions}
|
||||
for _ in range(args.warmups):
|
||||
for function in functions.values():
|
||||
function()
|
||||
gc_was_enabled = gc.isenabled()
|
||||
gc.disable()
|
||||
try:
|
||||
for trial in range(args.trials):
|
||||
order = (
|
||||
("baseline", "optimized") if trial % 2 else ("optimized", "baseline")
|
||||
)
|
||||
for name in order:
|
||||
start = time.perf_counter_ns()
|
||||
result = functions[name]()
|
||||
elapsed = time.perf_counter_ns() - start
|
||||
timings[name].append(elapsed / 1e6)
|
||||
del result
|
||||
finally:
|
||||
if gc_was_enabled:
|
||||
gc.enable()
|
||||
assert input_ids.tobytes() == original
|
||||
medians = {name: statistics.median(samples) for name, samples in timings.items()}
|
||||
return {
|
||||
"tokens": num_tokens,
|
||||
"items": num_items,
|
||||
"baseline_ms": medians["baseline"],
|
||||
"optimized_ms": medians["optimized"],
|
||||
"speedup": medians["baseline"] / medians["optimized"],
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--baseline-ref", required=True, help="Local pre-change git ref"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tokens", nargs="+", type=int, default=[128, 4096, 16384, 131072]
|
||||
)
|
||||
parser.add_argument("--items", nargs="+", type=int, default=[0, 1, 10])
|
||||
parser.add_argument("--span-tokens", type=int, default=32)
|
||||
parser.add_argument("--trials", type=int, default=41)
|
||||
parser.add_argument("--warmups", type=int, default=5)
|
||||
parser.add_argument("--threads", type=int, default=1)
|
||||
parser.add_argument("--json", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if (
|
||||
min(args.tokens + args.items) < 0
|
||||
or min(args.span_tokens, args.trials, args.threads) < 1
|
||||
or args.warmups < 0
|
||||
):
|
||||
parser.error("Lengths/items/warmups must be nonnegative; other counts positive")
|
||||
if any(n < 3 * items for n in args.tokens for items in args.items):
|
||||
parser.error("Each token length must accommodate at least 3 tokens per item")
|
||||
torch.set_num_threads(args.threads)
|
||||
baseline_commit, baseline = load_baseline(args.baseline_ref)
|
||||
rows = [
|
||||
benchmark_case(tokens, items, args, baseline)
|
||||
for tokens in args.tokens
|
||||
for items in args.items
|
||||
]
|
||||
if args.json:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"baseline_commit": baseline_commit,
|
||||
"python": platform.python_version(),
|
||||
"torch": torch.__version__,
|
||||
"architecture": platform.machine(),
|
||||
"threads": args.threads,
|
||||
"trials": args.trials,
|
||||
"warmups": args.warmups,
|
||||
"span_tokens": args.span_tokens,
|
||||
"results": rows,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return
|
||||
print("Offset-based CPU padding; median milliseconds; synthetic inputs")
|
||||
print(
|
||||
f"{'Tokens':>8} {'Items':>6} {'Baseline (ms)':>14} {'Optimized (ms)':>14} {'Speedup':>9}"
|
||||
)
|
||||
for row in rows:
|
||||
print(
|
||||
f"{row['tokens']:>8} {row['items']:>6} "
|
||||
f"{row['baseline_ms']:>14.4f} {row['optimized_ms']:>14.4f} "
|
||||
f"{row['speedup']:>8.2f}x"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,284 @@
|
||||
"""Benchmark multi-node MSCCL++ AllGather against NCCL."""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from sglang.srt.distributed import init_distributed_environment
|
||||
from sglang.srt.distributed.parallel_state import (
|
||||
cleanup_dist_env_and_memory,
|
||||
get_tensor_model_parallel_group,
|
||||
initialize_model_parallel,
|
||||
set_mscclpp,
|
||||
)
|
||||
from sglang.test.test_utils import publish_build_topology
|
||||
|
||||
|
||||
def _max_across_ranks(value: float, group) -> float:
|
||||
tensor = torch.tensor([value], dtype=torch.float64)
|
||||
dist.all_reduce(tensor, op=dist.ReduceOp.MAX, group=group)
|
||||
return tensor.item()
|
||||
|
||||
|
||||
def _time_eager(func, group, warmup: int = 2, repeat: int = 10) -> float:
|
||||
for _ in range(warmup):
|
||||
func()
|
||||
torch.cuda.synchronize()
|
||||
dist.barrier(group=group)
|
||||
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
start.record()
|
||||
for _ in range(repeat):
|
||||
func()
|
||||
end.record()
|
||||
end.synchronize()
|
||||
return _max_across_ranks(start.elapsed_time(end) * 1000 / repeat, group)
|
||||
|
||||
|
||||
def _time_cuda_graph(
|
||||
func,
|
||||
group,
|
||||
warmup: int = 100,
|
||||
graph_loop: int = 10,
|
||||
repeat: int = 100,
|
||||
) -> float:
|
||||
current_stream = torch.cuda.current_stream()
|
||||
capture_stream = torch.cuda.Stream()
|
||||
capture_stream.wait_stream(current_stream)
|
||||
with torch.cuda.stream(capture_stream):
|
||||
func(capture_stream)
|
||||
capture_stream.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=capture_stream):
|
||||
for _ in range(graph_loop):
|
||||
func(capture_stream)
|
||||
|
||||
with torch.cuda.stream(capture_stream):
|
||||
for _ in range(warmup):
|
||||
graph.replay()
|
||||
capture_stream.synchronize()
|
||||
|
||||
dist.barrier(group=group)
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
start.record(capture_stream)
|
||||
with torch.cuda.stream(capture_stream):
|
||||
for _ in range(repeat):
|
||||
graph.replay()
|
||||
end.record(capture_stream)
|
||||
end.synchronize()
|
||||
torch.cuda.current_stream().wait_stream(capture_stream)
|
||||
return _max_across_ranks(
|
||||
start.elapsed_time(end) * 1000 / (repeat * graph_loop),
|
||||
group,
|
||||
)
|
||||
|
||||
|
||||
def _validate_cuda_graph_updates(
|
||||
func,
|
||||
reference_func,
|
||||
input_tensor: torch.Tensor,
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
group,
|
||||
) -> None:
|
||||
current_stream = torch.cuda.current_stream()
|
||||
capture_stream = torch.cuda.Stream()
|
||||
capture_stream.wait_stream(current_stream)
|
||||
with torch.cuda.stream(capture_stream):
|
||||
func(capture_stream)
|
||||
capture_stream.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=capture_stream):
|
||||
func(capture_stream)
|
||||
|
||||
input_tensor.neg_()
|
||||
reference_func()
|
||||
current_stream.synchronize()
|
||||
capture_stream.wait_stream(current_stream)
|
||||
with torch.cuda.stream(capture_stream):
|
||||
graph.replay()
|
||||
capture_stream.synchronize()
|
||||
|
||||
if not torch.equal(actual, expected):
|
||||
raise AssertionError("MSCCL++ CUDA graph replay used stale AllGather data")
|
||||
dist.barrier(group=group)
|
||||
|
||||
|
||||
def _message_sizes(min_bytes: int, max_bytes: int):
|
||||
size = 1 << (min_bytes - 1).bit_length()
|
||||
while size <= max_bytes:
|
||||
yield size
|
||||
size <<= 1
|
||||
|
||||
|
||||
def human_readable_size(size, decimal_places=1):
|
||||
for unit in ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]:
|
||||
if size < 1024.0 or unit == "PiB":
|
||||
break
|
||||
size /= 1024.0
|
||||
return f"{size:.{decimal_places}f} {unit}"
|
||||
|
||||
|
||||
try:
|
||||
from tabulate import tabulate
|
||||
except ImportError:
|
||||
print("tabulate not installed, skipping table printing")
|
||||
tabulate = None
|
||||
|
||||
|
||||
def print_markdown_table(data):
|
||||
if tabulate is not None:
|
||||
print(tabulate(data, headers="keys", tablefmt="github"))
|
||||
return
|
||||
headers = data[0].keys()
|
||||
header_row = "| " + " | ".join(headers) + " |"
|
||||
separator = "| " + " | ".join(["---"] * len(headers)) + " |"
|
||||
rows = []
|
||||
for item in data:
|
||||
row = "| " + " | ".join(str(item[key]) for key in headers) + " |"
|
||||
rows.append(row)
|
||||
markdown_table = "\n".join([header_row, separator] + rows)
|
||||
print(markdown_table)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--min-bytes", type=int, default=1 << 10)
|
||||
parser.add_argument("--max-bytes", type=int, default=8 << 20)
|
||||
args = parser.parse_args()
|
||||
if not (1 << 10) <= args.min_bytes <= args.max_bytes <= (8 << 20):
|
||||
parser.error("Require 1 KiB <= min-bytes <= max-bytes <= 8 MiB")
|
||||
|
||||
local_rank = int(os.environ.get("LOCAL_RANK", 0))
|
||||
torch.cuda.set_device(local_rank)
|
||||
if not dist.is_initialized():
|
||||
dist.init_process_group(
|
||||
backend="nccl",
|
||||
device_id=torch.device("cuda", local_rank),
|
||||
)
|
||||
world_size = dist.get_world_size()
|
||||
rank = dist.get_rank()
|
||||
device = torch.cuda.current_device()
|
||||
set_mscclpp(True)
|
||||
init_distributed_environment(
|
||||
world_size=world_size,
|
||||
rank=rank,
|
||||
local_rank=local_rank,
|
||||
)
|
||||
publish_build_topology(world_rank=rank, tp_size=world_size)
|
||||
initialize_model_parallel()
|
||||
tp_group = get_tensor_model_parallel_group()
|
||||
gpu_group = tp_group.device_group
|
||||
cpu_group = tp_group.cpu_group
|
||||
pynccl_comm = tp_group.pynccl_comm
|
||||
mscclpp_comm = tp_group.pymscclpp_comm
|
||||
assert pynccl_comm is not None
|
||||
assert mscclpp_comm is not None
|
||||
if args.min_bytes % world_size != 0 or args.max_bytes % world_size != 0:
|
||||
parser.error("Total message sizes must be divisible by WORLD_SIZE")
|
||||
|
||||
if not mscclpp_comm._best_configs["allgather"]:
|
||||
raise RuntimeError("No tuned MSCCL++ AllGather configuration is available")
|
||||
|
||||
result = []
|
||||
|
||||
for total_bytes in _message_sizes(args.min_bytes, args.max_bytes):
|
||||
input_bytes = total_bytes // world_size
|
||||
if input_bytes % torch.bfloat16.itemsize != 0:
|
||||
continue
|
||||
elements = input_bytes // torch.bfloat16.itemsize
|
||||
input_tensor = (
|
||||
torch.arange(
|
||||
elements,
|
||||
dtype=torch.long,
|
||||
device=device,
|
||||
)
|
||||
% 193
|
||||
+ rank
|
||||
).to(torch.bfloat16)
|
||||
nccl_output = torch.empty(
|
||||
elements * world_size,
|
||||
dtype=input_tensor.dtype,
|
||||
device=device,
|
||||
)
|
||||
mscclpp_output = torch.empty_like(nccl_output)
|
||||
|
||||
def torch_nccl():
|
||||
dist.all_gather_into_tensor(
|
||||
nccl_output,
|
||||
input_tensor,
|
||||
group=gpu_group,
|
||||
)
|
||||
|
||||
def py_nccl():
|
||||
pynccl_comm.all_gather(nccl_output, input_tensor)
|
||||
|
||||
def mscclpp():
|
||||
mscclpp_comm.all_gather(mscclpp_output, input_tensor)
|
||||
|
||||
def mscclpp_on_stream(stream):
|
||||
mscclpp_comm.all_gather(
|
||||
mscclpp_output,
|
||||
input_tensor,
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
def py_nccl_on_stream(_stream):
|
||||
with pynccl_comm.change_state(enable=True):
|
||||
pynccl_comm.all_gather(nccl_output, input_tensor)
|
||||
|
||||
torch_nccl()
|
||||
mscclpp()
|
||||
py_nccl()
|
||||
torch.cuda.synchronize()
|
||||
if not torch.equal(mscclpp_output, nccl_output):
|
||||
raise AssertionError(
|
||||
f"MSCCL++ AllGather mismatch at {input_bytes} input bytes"
|
||||
)
|
||||
_validate_cuda_graph_updates(
|
||||
mscclpp_on_stream,
|
||||
torch_nccl,
|
||||
input_tensor,
|
||||
mscclpp_output,
|
||||
nccl_output,
|
||||
cpu_group,
|
||||
)
|
||||
|
||||
nccl_eager_us = _time_eager(torch_nccl, cpu_group)
|
||||
mscclpp_eager_us = _time_eager(mscclpp, cpu_group)
|
||||
mscclpp_graph_us = _time_cuda_graph(mscclpp_on_stream, cpu_group)
|
||||
pynccl_graph_us = _time_cuda_graph(py_nccl_on_stream, cpu_group)
|
||||
|
||||
result.append(
|
||||
{
|
||||
"msg_size": human_readable_size(total_bytes),
|
||||
"torch eager time": nccl_eager_us,
|
||||
"msccl eager time": mscclpp_eager_us,
|
||||
"msccl graph time": mscclpp_graph_us,
|
||||
"pynccl graph time": pynccl_graph_us,
|
||||
}
|
||||
)
|
||||
if rank == 0:
|
||||
print(
|
||||
f"sz={total_bytes // input_tensor.element_size()}, "
|
||||
f"dtype={input_tensor.dtype}: correctness check PASS!"
|
||||
)
|
||||
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
if rank == 0:
|
||||
print_markdown_table(result)
|
||||
|
||||
mscclpp_comm.destroy()
|
||||
dist.barrier()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -12,9 +12,9 @@ torchrun --nproc_per_node gpu \
|
||||
--master_port $MASTER_PORT benchmark/kernels/all_reduce/benchmark_mscclpp.py
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from contextlib import nullcontext
|
||||
from typing import List
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
@@ -23,13 +23,16 @@ from torch.distributed import ProcessGroup
|
||||
from sglang.srt.distributed import init_distributed_environment
|
||||
from sglang.srt.distributed.device_communicators.pymscclpp import PyMscclppCommunicator
|
||||
from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
from sglang.srt.distributed.parallel_state import (
|
||||
cleanup_dist_env_and_memory,
|
||||
get_tensor_model_parallel_group,
|
||||
graph_capture,
|
||||
initialize_model_parallel,
|
||||
set_mscclpp_all_reduce,
|
||||
set_mscclpp,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.test_utils import publish_build_topology
|
||||
|
||||
|
||||
@@ -51,14 +54,21 @@ def pynccl_allreduce(
|
||||
return msccl_input
|
||||
|
||||
|
||||
def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=10):
|
||||
graph_input = inp_randn.clone()
|
||||
graph_input_snapshot = inp_randn.clone()
|
||||
def _bench_graph_time(
|
||||
func,
|
||||
inp_randn,
|
||||
tp_group,
|
||||
warmup_loop=2,
|
||||
graph_loop=10,
|
||||
test_loop=10,
|
||||
):
|
||||
with use_symmetric_memory(tp_group):
|
||||
graph_input = inp_randn.clone()
|
||||
with graph_capture() as graph_capture_context:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
|
||||
for _ in range(graph_loop):
|
||||
graph_input.copy_(graph_input_snapshot)
|
||||
graph_input.copy_(inp_randn)
|
||||
graph_out = func(graph_input)
|
||||
|
||||
graph.replay()
|
||||
@@ -71,7 +81,7 @@ def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=1
|
||||
start_event = torch.cuda.Event(enable_timing=True)
|
||||
end_event = torch.cuda.Event(enable_timing=True)
|
||||
|
||||
latencies: List[float] = []
|
||||
latencies = []
|
||||
for _ in range(test_loop):
|
||||
torch.cuda.synchronize()
|
||||
dist.barrier()
|
||||
@@ -85,8 +95,9 @@ def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=1
|
||||
return func_output, func_cost_us
|
||||
|
||||
|
||||
def _bench_eager_time(func, inp_randn, warmup_loop=2, test_loop=10):
|
||||
eager_input = inp_randn.clone()
|
||||
def _bench_eager_time(func, inp_randn, tp_group, warmup_loop=2, test_loop=10):
|
||||
with use_symmetric_memory(tp_group):
|
||||
eager_input = inp_randn.clone()
|
||||
eager_output = func(eager_input)
|
||||
func_output = eager_output.clone()
|
||||
|
||||
@@ -156,57 +167,77 @@ def print_markdown_table(data):
|
||||
if __name__ == "__main__":
|
||||
import logging
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--enable-symm-mem",
|
||||
action="store_true",
|
||||
help="Allocate collective inputs from NCCL symmetric memory.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
force=True,
|
||||
)
|
||||
local_rank = int(os.environ["LOCAL_RANK"])
|
||||
torch.cuda.set_device(local_rank)
|
||||
device = torch.cuda.current_device()
|
||||
if not dist.is_initialized():
|
||||
dist.init_process_group(backend="nccl")
|
||||
dist.init_process_group(
|
||||
backend="nccl", device_id=torch.device("cuda", local_rank)
|
||||
)
|
||||
world, world_size = dist.group.WORLD, dist.get_world_size()
|
||||
rank = dist.get_rank()
|
||||
torch.cuda.set_device(rank % 8)
|
||||
device = torch.cuda.current_device()
|
||||
set_mscclpp_all_reduce(True)
|
||||
set_mscclpp(True)
|
||||
init_distributed_environment(
|
||||
world_size=world_size,
|
||||
rank=rank,
|
||||
local_rank=rank % 8,
|
||||
local_rank=local_rank,
|
||||
)
|
||||
publish_build_topology(world_rank=rank, tp_size=world_size)
|
||||
initialize_model_parallel()
|
||||
group = get_tensor_model_parallel_group().device_group
|
||||
cpu_group = get_tensor_model_parallel_group().cpu_group
|
||||
pynccl_comm = get_tensor_model_parallel_group().pynccl_comm
|
||||
pymscclpp_comm = get_tensor_model_parallel_group().pymscclpp_comm
|
||||
publish_build_topology(
|
||||
world_rank=rank,
|
||||
tp_size=world_size,
|
||||
enable_symm_mem=args.enable_symm_mem,
|
||||
)
|
||||
initialize_model_parallel(enable_symm_mem=args.enable_symm_mem)
|
||||
tp_group = get_parallel().tp_group
|
||||
group = tp_group.device_group
|
||||
pynccl_comm = tp_group.pynccl_comm
|
||||
pymscclpp_comm = tp_group.pymscclpp_comm
|
||||
dist.barrier()
|
||||
profile = False
|
||||
dtype = torch.bfloat16
|
||||
ctx = get_torch_prof_ctx(profile)
|
||||
result = []
|
||||
max_message_bytes = 1 << 20
|
||||
|
||||
with ctx:
|
||||
for i in range(10, 20):
|
||||
sz = 2**i
|
||||
if sz * dtype.itemsize > 2**20:
|
||||
if sz * dtype.itemsize > max_message_bytes:
|
||||
break
|
||||
inp_randn = torch.randint(1, 16, (sz,), dtype=dtype, device=device)
|
||||
inp_randn = torch.randint(0, 2, (sz,), dtype=dtype, device=device)
|
||||
|
||||
memory = torch.empty_like(inp_randn)
|
||||
memory_out = torch.empty_like(memory)
|
||||
torch_eager_output, torch_eager_time = _bench_eager_time(
|
||||
lambda inp: torch_allreduce(inp, group), inp_randn
|
||||
lambda inp: torch_allreduce(inp, group), inp_randn, tp_group
|
||||
)
|
||||
msccl_eager_output, msccl_eager_time = _bench_eager_time(
|
||||
lambda inp: msccl_allreduce(inp, pymscclpp_comm), inp_randn
|
||||
lambda inp: msccl_allreduce(inp, pymscclpp_comm),
|
||||
inp_randn,
|
||||
tp_group,
|
||||
)
|
||||
msccl_graph_output, msccl_graph_time = _bench_graph_time(
|
||||
lambda inp: msccl_allreduce(inp, pymscclpp_comm), inp_randn
|
||||
lambda inp: msccl_allreduce(inp, pymscclpp_comm),
|
||||
inp_randn,
|
||||
tp_group,
|
||||
)
|
||||
# since pynccl is inplace op, this return result is not correct if graph loop > 1
|
||||
_, pynccl_graph_time = _bench_graph_time(
|
||||
lambda inp: pynccl_allreduce(inp, pynccl_comm), inp_randn
|
||||
lambda inp: pynccl_allreduce(inp, pynccl_comm),
|
||||
inp_randn,
|
||||
tp_group,
|
||||
)
|
||||
torch.testing.assert_close(torch_eager_output, msccl_graph_output)
|
||||
torch.testing.assert_close(torch_eager_output, msccl_eager_output)
|
||||
|
||||
@@ -39,11 +39,11 @@ from sglang.srt.distributed.device_communicators.torch_symm_mem import (
|
||||
TorchSymmMemCommunicator,
|
||||
)
|
||||
from sglang.srt.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_group,
|
||||
graph_capture,
|
||||
initialize_model_parallel,
|
||||
set_torch_symm_mem_all_reduce,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.test_utils import publish_build_topology
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
@@ -191,10 +191,10 @@ if __name__ == "__main__":
|
||||
)
|
||||
publish_build_topology(world_rank=rank, tp_size=world_size)
|
||||
initialize_model_parallel()
|
||||
group = get_tensor_model_parallel_group().device_group
|
||||
cpu_group = get_tensor_model_parallel_group().cpu_group
|
||||
pynccl_comm = get_tensor_model_parallel_group().pynccl_comm
|
||||
torch_symm_mem_comm = get_tensor_model_parallel_group().torch_symm_mem_comm
|
||||
group = get_parallel().tp_group.device_group
|
||||
cpu_group = get_parallel().tp_group.cpu_group
|
||||
pynccl_comm = get_parallel().tp_group.pynccl_comm
|
||||
torch_symm_mem_comm = get_parallel().tp_group.torch_symm_mem_comm
|
||||
dist.barrier()
|
||||
profile = False
|
||||
dtype = torch.bfloat16
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
# ViT attention with FA4 on SM103: row max from tcgen05.ld.red
|
||||
# (SGLANG_FA4_TMEM_LOAD_RED_MAX=1) vs FMNMX (=0). Shapes and call follow
|
||||
# VisionFlash4Attention (varlen, non-causal, no LSE). Each round CUDA-graph times both
|
||||
# back to back, in alternating order; the per-round ratio is the headline number.
|
||||
# Outputs must be bit-identical.
|
||||
#
|
||||
# Run on B300 / GB300 (elsewhere the flag is ignored):
|
||||
# python benchmark/kernels/attention/bench_flash_attention_tmem_red_max.py
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from typing import Iterable
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.flash_attn.cute.interface import (
|
||||
flash_attn_varlen_func,
|
||||
)
|
||||
|
||||
RED_MAX_ENV = "SGLANG_FA4_TMEM_LOAD_RED_MAX"
|
||||
|
||||
# images, tokens per image, heads, head dim
|
||||
SHAPES = [
|
||||
(8, 4096, 16, 80), # Qwen2.5-VL
|
||||
(8, 4096, 16, 72), # Qwen3-VL
|
||||
(8, 4096, 16, 64), # InternViT, GLM / DeepSeek-V4 ViT
|
||||
(64, 1024, 16, 80),
|
||||
(64, 1024, 16, 64),
|
||||
(1, 32768, 16, 80), # one large image or a video
|
||||
(1, 32768, 16, 72),
|
||||
(1, 32768, 16, 64),
|
||||
]
|
||||
|
||||
|
||||
def _bits(t: torch.Tensor) -> torch.Tensor:
|
||||
int_dtype = {1: torch.uint8, 2: torch.int16, 4: torch.int32}[t.element_size()]
|
||||
return t.contiguous().view(int_dtype)
|
||||
|
||||
|
||||
def time_graph(fn, iters: int, calls_per_graph: int = 5) -> float:
|
||||
"""Mean seconds per call, replaying a CUDA graph of `calls_per_graph` calls."""
|
||||
for _ in range(3):
|
||||
fn()
|
||||
torch.cuda.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
for _ in range(calls_per_graph):
|
||||
fn()
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
graph.replay()
|
||||
torch.cuda.synchronize()
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
start.record()
|
||||
for _ in range(iters):
|
||||
graph.replay()
|
||||
end.record()
|
||||
torch.cuda.synchronize()
|
||||
return start.elapsed_time(end) / 1e3 / (iters * calls_per_graph)
|
||||
|
||||
|
||||
def main(argv: Iterable[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--rounds",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Rounds per shape; each round times both variants back to back.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--iters", type=int, default=50, help="Graph replays per round."
|
||||
)
|
||||
parser.add_argument("--seed", type=int, default=0)
|
||||
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("CUDA is required")
|
||||
major, minor = torch.cuda.get_device_capability()
|
||||
if major != 10:
|
||||
raise RuntimeError(f"This benchmark is for SM10x. Got {major}.{minor}.")
|
||||
if minor < 3:
|
||||
print(
|
||||
f"Note: SM{major}{minor} has no tcgen05.ld.red; {RED_MAX_ENV} is ignored "
|
||||
"and both variants run the same kernel."
|
||||
)
|
||||
|
||||
torch.manual_seed(args.seed)
|
||||
device = "cuda"
|
||||
mismatches = []
|
||||
print(
|
||||
f"{'shape':28s} {'FMNMX us':>9s} {'ld.red us':>9s} {'ld.red TF':>9s} "
|
||||
f"{'best':>7s} {'paired':>7s} {'wins':>5s} bitwise"
|
||||
)
|
||||
for images, tokens, heads, head_dim in SHAPES:
|
||||
torch.cuda.empty_cache()
|
||||
q, k, v = [
|
||||
torch.randn(
|
||||
images * tokens, heads, head_dim, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
for _ in range(3)
|
||||
]
|
||||
cu_seqlens = torch.arange(images + 1, dtype=torch.int32, device=device) * tokens
|
||||
kwargs = dict(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
cu_seqlens_q=cu_seqlens,
|
||||
cu_seqlens_k=cu_seqlens,
|
||||
max_seqlen_q=tokens,
|
||||
max_seqlen_k=tokens,
|
||||
)
|
||||
|
||||
outputs, times = {}, {False: [], True: []}
|
||||
for red_max in (False, True):
|
||||
os.environ[RED_MAX_ENV] = "1" if red_max else "0"
|
||||
outputs[red_max] = flash_attn_varlen_func(**kwargs)[0]
|
||||
# Swap the order every round so clock and thermal drift hit both equally.
|
||||
for r in range(args.rounds):
|
||||
for red_max in (False, True) if r % 2 == 0 else (True, False):
|
||||
os.environ[RED_MAX_ENV] = "1" if red_max else "0"
|
||||
times[red_max].append(
|
||||
time_graph(lambda: flash_attn_varlen_func(**kwargs), args.iters)
|
||||
)
|
||||
same = torch.equal(_bits(outputs[True]), _bits(outputs[False]))
|
||||
name = f"{images} x {tokens} tok, h{heads} d{head_dim}"
|
||||
if not same:
|
||||
mismatches.append(name)
|
||||
t0, t1 = min(times[False]), min(times[True])
|
||||
# Ratio of the two variants within each round, then the median over rounds.
|
||||
paired = sorted(a / c for a, c in zip(times[False], times[True]))
|
||||
wins = f"{sum(p > 1 for p in paired)}/{len(paired)}"
|
||||
flops = 4.0 * images * heads * head_dim * tokens * tokens
|
||||
print(
|
||||
f"{name:28s} {t0 * 1e6:9.1f} {t1 * 1e6:9.1f} {flops / t1 / 1e12:9.1f} "
|
||||
f"{t0 / t1:6.3f}x {paired[len(paired) // 2]:6.3f}x {wins:>5s} "
|
||||
f"{'identical' if same else 'DIFFERENT'}",
|
||||
flush=True,
|
||||
)
|
||||
del q, k, v, outputs
|
||||
|
||||
os.environ.pop(RED_MAX_ENV, None)
|
||||
if mismatches:
|
||||
print(f"\nOutputs differ between the variants for: {', '.join(mismatches)}")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -7,7 +7,7 @@ import torch.utils.benchmark as benchmark
|
||||
from flashinfer import BatchDecodeWithPagedKVCacheWrapper
|
||||
|
||||
from sglang.kernels.ops.attention.decode_attention import decode_attention_fwd
|
||||
from sglang.kernels.ops.attention.flashinfer_backend import should_use_tensor_core
|
||||
from sglang.srt.layers.attention.flashinfer_backend import should_use_tensor_core
|
||||
|
||||
|
||||
def benchmark_forward(
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.benchmark.deepseek_utils import (
|
||||
per_block_cast_to_fp8,
|
||||
per_token_cast_to_fp8,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
from sglang.kernels.ops.gemm.fp8_kernel import (
|
||||
w8a8_block_fp8_matmul_deepgemm as w8a8_block_fp8_matmul,
|
||||
)
|
||||
|
||||
|
||||
@@ -9,10 +9,8 @@ from sglang.benchmark.deepseek_utils import (
|
||||
get_weight_shapes,
|
||||
per_block_cast_to_fp8,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
sglang_per_token_group_quant_fp8,
|
||||
w8a8_block_fp8_matmul_deepgemm,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.fp8_kernel import w8a8_block_fp8_matmul_deepgemm
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import sglang_per_token_group_quant_fp8
|
||||
from sglang.srt.layers.quantization.fp8_utils import requant_weight_ue8m0
|
||||
|
||||
BLOCK_SIZE = 128
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Compare default, H200, and H20-3e block-FP8 configs on an H20-3e.
|
||||
|
||||
Example:
|
||||
python benchmark/kernels/deepseek/benchmark_h20_block_fp8.py \
|
||||
--output h20_block_fp8.jsonl
|
||||
|
||||
This measures individual GEMMs, not end-to-end model throughput. Run on an
|
||||
otherwise idle GPU. Inputs are synthetic; no model weights are needed.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import pathlib
|
||||
import statistics
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
import triton
|
||||
|
||||
from sglang.kernels.ops.gemm import fp8_kernel
|
||||
|
||||
SHAPES = [
|
||||
(1792, 5120),
|
||||
(25600, 6144),
|
||||
(4096, 1280),
|
||||
(5120, 1024),
|
||||
(5120, 288),
|
||||
(576, 5120),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output", type=pathlib.Path, required=True)
|
||||
parser.add_argument(
|
||||
"--batch-sizes",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64],
|
||||
)
|
||||
parser.add_argument("--trials", type=int, default=5)
|
||||
parser.add_argument("--rep-ms", type=int, default=50)
|
||||
parser.add_argument("--seed", type=int, default=98765)
|
||||
args = parser.parse_args()
|
||||
assert torch.cuda.get_device_name() == "NVIDIA H20-3e"
|
||||
assert args.trials > 0 and args.rep_ms > 0 and all(m > 0 for m in args.batch_sizes)
|
||||
torch.backends.cuda.matmul.allow_tf32 = False
|
||||
torch.manual_seed(args.seed)
|
||||
config_dir = pathlib.Path(fp8_kernel.__file__).parent / "configs"
|
||||
with args.output.open("x") as output:
|
||||
|
||||
def emit(row):
|
||||
text = json.dumps(row)
|
||||
output.write(text + "\n")
|
||||
output.flush()
|
||||
print(text, flush=True)
|
||||
|
||||
emit(
|
||||
{
|
||||
"kind": "environment",
|
||||
"device": torch.cuda.get_device_name(),
|
||||
"torch": torch.__version__,
|
||||
"triton": triton.__version__,
|
||||
"seed": args.seed,
|
||||
"batch_sizes": args.batch_sizes,
|
||||
}
|
||||
)
|
||||
for n, k in SHAPES:
|
||||
b = (torch.randn(n, k, device="cuda") * 0.2).to(torch.float8_e4m3fn)
|
||||
bs = torch.rand(n // 32, k // 32, device="cuda") + 0.1
|
||||
b_dequant = b.float() * bs.repeat_interleave(32, 0).repeat_interleave(32, 1)
|
||||
tables = {"default": None}
|
||||
for device in ("NVIDIA_H200", "NVIDIA_H20-3e"):
|
||||
path = (
|
||||
config_dir
|
||||
/ f"N={n},K={k},device_name={device},dtype=fp8_w8a8,block_shape=[32, 32].json"
|
||||
)
|
||||
if path.exists():
|
||||
tables[device] = {
|
||||
int(m): cfg for m, cfg in json.loads(path.read_text()).items()
|
||||
}
|
||||
assert "NVIDIA_H20-3e" in tables, (
|
||||
"Install the H20-3e configuration files before benchmarking"
|
||||
)
|
||||
for m in args.batch_sizes:
|
||||
a = (torch.randn(m, k, device="cuda") * 0.2).to(torch.float8_e4m3fn)
|
||||
scales = (torch.rand(k // 32, m, device="cuda") + 0.1).T
|
||||
reference = (
|
||||
(a.float() * scales.repeat_interleave(32, 1)) @ b_dequant.T
|
||||
).to(torch.bfloat16)
|
||||
results = {}
|
||||
# Compile and validate every variant before timing any variant.
|
||||
for label, table in tables.items():
|
||||
with patch.object(
|
||||
fp8_kernel, "get_w8a8_block_fp8_configs", return_value=table
|
||||
):
|
||||
actual = fp8_kernel.w8a8_block_fp8_matmul_triton(
|
||||
a, b, scales, bs, [32, 32], torch.bfloat16
|
||||
)
|
||||
relative_l2 = (
|
||||
torch.linalg.vector_norm(actual.float() - reference.float())
|
||||
/ torch.linalg.vector_norm(reference.float())
|
||||
).item()
|
||||
assert torch.isfinite(actual).all() and relative_l2 < 0.01
|
||||
results[label] = {
|
||||
"config": None
|
||||
if table is None
|
||||
else table[min(table, key=lambda key: abs(key - m))],
|
||||
"relative_l2": relative_l2,
|
||||
"trials_us": [],
|
||||
}
|
||||
for trial in range(args.trials):
|
||||
labels = list(tables)
|
||||
# Rotate variant order to reduce systematic timing-order bias.
|
||||
labels = (
|
||||
labels[trial % len(labels) :] + labels[: trial % len(labels)]
|
||||
)
|
||||
for label in labels:
|
||||
with patch.object(
|
||||
fp8_kernel,
|
||||
"get_w8a8_block_fp8_configs",
|
||||
return_value=tables[label],
|
||||
):
|
||||
|
||||
def invoke():
|
||||
return fp8_kernel.w8a8_block_fp8_matmul_triton(
|
||||
a, b, scales, bs, [32, 32], torch.bfloat16
|
||||
)
|
||||
|
||||
elapsed = (
|
||||
triton.testing.do_bench_cudagraph(
|
||||
invoke, rep=args.rep_ms
|
||||
)
|
||||
* 1000
|
||||
)
|
||||
results[label]["trials_us"].append(elapsed)
|
||||
for result in results.values():
|
||||
result["median_us"] = statistics.median(result["trials_us"])
|
||||
emit(
|
||||
{"kind": "measurement", "m": m, "n": n, "k": k, "results": results}
|
||||
)
|
||||
emit({"kind": "complete"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -78,7 +78,7 @@ python benchmark/kernels/quantization/tuning_block_wise_kernel.py --N 5120 --K 1
|
||||
|
||||
## Output
|
||||
|
||||
Generates JSON config files saved to `python/sglang/srt/layers/quantization/configs/`:
|
||||
Generates JSON config files saved to `python/sglang/kernels/ops/gemm/configs/`:
|
||||
```
|
||||
N={N},K={K},device_name={DEVICE},dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
```
|
||||
|
||||
@@ -28,11 +28,11 @@ from tqdm import tqdm
|
||||
mp.set_start_method("spawn", force=True)
|
||||
|
||||
from sglang.benchmark.deepseek_utils import get_weight_shapes
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
from sglang.kernels.ops.gemm.fp8_kernel import (
|
||||
_w8a8_block_fp8_matmul,
|
||||
_w8a8_block_fp8_matmul_unrolledx4,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.int8_kernel import _w8a8_block_int8_matmul
|
||||
from sglang.kernels.ops.gemm.int8_kernel import _w8a8_block_int8_matmul
|
||||
from sglang.srt.utils import (
|
||||
get_device,
|
||||
get_device_core_count,
|
||||
@@ -491,7 +491,7 @@ if __name__ == "__main__":
|
||||
parser.add_argument(
|
||||
"--save-path",
|
||||
type=str,
|
||||
default="python/sglang/kernels/ops/quantization/configs",
|
||||
default="python/sglang/kernels/ops/gemm/configs",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Tune the gfx950 native MXFP8 dense route for one (N, K) and write its rows into
|
||||
python/sglang/kernels/ops/quantization/mxfp8_gemv_gfx95_configs.json.
|
||||
|
||||
Two tables: the scaled-MFMA GEMV config per M bucket (1 ... 32) and the dot_scaled tile per
|
||||
M bucket (64 ... 16384). Each candidate is timed by HIP-graph replay rotating over weight
|
||||
copies that together exceed the last-level cache, so the weight streams from HBM as it does
|
||||
in decode. The GEMV pick minimizes the fp8-input plus the bf16-input time: fused producers
|
||||
hand it fp8, other callers bf16.
|
||||
|
||||
python benchmark/kernels/quantization/tuning_mxfp8_native_gfx95.py --N 1792 --K 5120
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import itertools
|
||||
import json
|
||||
|
||||
import torch
|
||||
from triton.runtime.errors import OutOfResources
|
||||
|
||||
from sglang.kernels.jit.utils import empty_sentinel
|
||||
from sglang.kernels.ops.quantization import mxfp8_native_amd_gfx95 as native
|
||||
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import fp8_grid_quantize
|
||||
|
||||
GRAPH_CALLS = 20
|
||||
ROTATION_BYTES = 512 << 20
|
||||
|
||||
|
||||
def graph_time_us(calls, replays: int) -> float:
|
||||
"""Mean time of one call, replaying a graph of GRAPH_CALLS calls cycling through calls."""
|
||||
for call in calls:
|
||||
call()
|
||||
torch.cuda.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
for i in range(GRAPH_CALLS):
|
||||
calls[i % len(calls)]()
|
||||
graph.replay()
|
||||
torch.cuda.synchronize()
|
||||
start, end = (
|
||||
torch.cuda.Event(enable_timing=True),
|
||||
torch.cuda.Event(enable_timing=True),
|
||||
)
|
||||
start.record()
|
||||
for _ in range(replays):
|
||||
graph.replay()
|
||||
end.record()
|
||||
torch.cuda.synchronize()
|
||||
return start.elapsed_time(end) * 1000 / (GRAPH_CALLS * replays)
|
||||
|
||||
|
||||
def make_weights(n: int, k: int):
|
||||
copies = max(2, min(16, -(-ROTATION_BYTES // (n * k))))
|
||||
weights = []
|
||||
for _ in range(copies):
|
||||
weight = (torch.randn(n, k, device="cuda") * 0.05).to(torch.float8_e4m3fn)
|
||||
scale = torch.exp2(
|
||||
torch.randint(-3, 3, (n // 32, k // 32), device="cuda").float()
|
||||
)
|
||||
weights.append(native.prepare_mxfp8_native_weight(weight, scale, [32, 32]))
|
||||
return weights
|
||||
|
||||
|
||||
def tune_gemv(n: int, k: int, weights) -> dict:
|
||||
rows = {}
|
||||
for bucket in native._M_BUCKETS:
|
||||
x = torch.randn(bucket, k, device="cuda", dtype=torch.bfloat16)
|
||||
xq, xs = fp8_grid_quantize(x)
|
||||
xq = xq.view(torch.uint8)
|
||||
no_scale = empty_sentinel(x.device, torch.uint8)
|
||||
out = torch.empty(bucket, n, dtype=torch.bfloat16, device="cuda")
|
||||
times = {}
|
||||
for waves, steps, tile_rows, tokens in itertools.product(
|
||||
(4, 8, 16), (1, 2, 4), (16, 32), (16, 32)
|
||||
):
|
||||
config = native._GemvConfig(waves, steps, tile_rows, tokens)
|
||||
if not config.valid_for(bucket, n):
|
||||
continue
|
||||
fp8_kernel = native._jit_mxfp8_gemv_module(config, False)
|
||||
bf16_kernel = native._jit_mxfp8_gemv_module(config, True)
|
||||
fp8_us = graph_time_us(
|
||||
[lambda w=w: fp8_kernel.run(w[0], w[1], xq, xs, out) for w in weights],
|
||||
replays=10,
|
||||
)
|
||||
bf16_us = graph_time_us(
|
||||
[
|
||||
lambda w=w: bf16_kernel.run(w[0], w[1], x, no_scale, out)
|
||||
for w in weights
|
||||
],
|
||||
replays=10,
|
||||
)
|
||||
times[f"w{waves}s{steps}r{tile_rows}t{tokens}k"] = fp8_us + bf16_us
|
||||
best = min(times, key=times.get)
|
||||
rows[f"gfx950:{n}:{k}:{bucket}"] = best
|
||||
print(f"gemv M={bucket:5d}: {best} {times[best]:.2f} us (fp8 + bf16)")
|
||||
return rows
|
||||
|
||||
|
||||
def tune_large_m(n: int, k: int, weights) -> dict:
|
||||
k_pad = weights[0][0].shape[1] * 128
|
||||
rows = {}
|
||||
for bucket in native._LARGE_M_BUCKETS:
|
||||
xq, xs = fp8_grid_quantize(
|
||||
torch.randn(bucket, k, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
times = {}
|
||||
for bm, bn, bk, warps, split_k in itertools.product(
|
||||
(32, 64, 128, 256), (64, 128, 256), (128, 256), (4, 8), (1, 2, 4, 5, 8)
|
||||
):
|
||||
if k_pad % bk or (k_pad // bk) % split_k or bm > max(64, 2 * bucket):
|
||||
continue
|
||||
if split_k > 1 and bucket > 1024:
|
||||
continue
|
||||
tile = (bm, bn, bk, warps)
|
||||
try:
|
||||
times[f"{bm},{bn},{bk},{warps},{split_k}"] = graph_time_us(
|
||||
[
|
||||
lambda w=w: native._mxfp8_shuffled_gemm(
|
||||
xq, xs, w[0], w[1], tile, split_k
|
||||
)
|
||||
for w in weights
|
||||
],
|
||||
replays=5,
|
||||
)
|
||||
except OutOfResources:
|
||||
continue # the tile's LDS footprint exceeds the CU's
|
||||
best = min(times, key=times.get)
|
||||
rows[f"gfx950:{n}:{k}:{bucket}"] = best
|
||||
print(f"large_m M={bucket:5d}: {best} {times[best]:.2f} us")
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--N", type=int, required=True)
|
||||
parser.add_argument("--K", type=int, required=True)
|
||||
args = parser.parse_args()
|
||||
assert native.native_route_supports(args.N, args.K), (args.N, args.K)
|
||||
assert torch.cuda.get_device_properties(0).gcnArchName.startswith("gfx950")
|
||||
|
||||
torch.manual_seed(0)
|
||||
weights = make_weights(args.N, args.K)
|
||||
gemv_rows = tune_gemv(args.N, args.K, weights)
|
||||
large_m_rows = tune_large_m(args.N, args.K, weights)
|
||||
|
||||
with open(native._CONFIG_FILE) as f:
|
||||
table = json.load(f)
|
||||
for section, rows in (("configs", gemv_rows), ("large_m", large_m_rows)):
|
||||
table[section] = dict(sorted({**table[section], **rows}.items()))
|
||||
with open(native._CONFIG_FILE, "w") as f:
|
||||
json.dump(table, f, indent=1)
|
||||
f.write("\n")
|
||||
print(f"wrote {len(gemv_rows) + len(large_m_rows)} rows to {native._CONFIG_FILE}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,20 +0,0 @@
|
||||
## Download data
|
||||
```
|
||||
bash download_data.sh
|
||||
```
|
||||
|
||||
## Run benchmark
|
||||
|
||||
### Benchmark sglang
|
||||
```
|
||||
python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000
|
||||
```
|
||||
|
||||
```
|
||||
python3 bench_sglang.py --nsub 10
|
||||
```
|
||||
|
||||
```
|
||||
# OpenAI models
|
||||
python3 bench_sglang.py --backend gpt-3.5-turbo --parallel 8
|
||||
```
|
||||
@@ -1,151 +0,0 @@
|
||||
"""
|
||||
Usage:
|
||||
python3 bench_hf.py --model-path meta-llama/Llama-2-7b-hf --data-dir data --ntrain 5
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
from tqdm import tqdm
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
choices = ["A", "B", "C", "D"]
|
||||
|
||||
|
||||
def format_subject(subject):
|
||||
l = subject.split("_")
|
||||
s = ""
|
||||
for entry in l:
|
||||
s += " " + entry
|
||||
return s
|
||||
|
||||
|
||||
def format_example(df, idx, include_answer=True):
|
||||
prompt = df.iloc[idx, 0]
|
||||
k = df.shape[1] - 2
|
||||
for j in range(k):
|
||||
prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1])
|
||||
prompt += "\nAnswer:"
|
||||
if include_answer:
|
||||
prompt += " {}\n\n".format(df.iloc[idx, k + 1])
|
||||
return prompt
|
||||
|
||||
|
||||
def gen_prompt(train_df, subject, k=-1):
|
||||
prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format(
|
||||
format_subject(subject)
|
||||
)
|
||||
if k == -1:
|
||||
k = train_df.shape[0]
|
||||
for i in range(k):
|
||||
prompt += format_example(train_df, i)
|
||||
return prompt
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def main(args):
|
||||
print(f"Loading model: {args.model_path}")
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
args.model_path,
|
||||
torch_dtype=torch.bfloat16,
|
||||
trust_remote_code=True,
|
||||
device_map="auto",
|
||||
).eval()
|
||||
|
||||
subjects = sorted(
|
||||
[
|
||||
f.split("_test.csv")[0]
|
||||
for f in os.listdir(os.path.join(args.data_dir, "test"))
|
||||
if "_test.csv" in f
|
||||
]
|
||||
)
|
||||
|
||||
all_cors = []
|
||||
num_requests = 0
|
||||
total_latency = 0
|
||||
|
||||
for subject in tqdm(subjects[: args.nsub]):
|
||||
dev_df = pd.read_csv(
|
||||
os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None
|
||||
)[: args.ntrain]
|
||||
test_df = pd.read_csv(
|
||||
os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None
|
||||
)
|
||||
|
||||
k = args.ntrain
|
||||
few_shot_examples = gen_prompt(dev_df, subject, k)
|
||||
while len(tokenizer.encode(few_shot_examples)) > 1536:
|
||||
k -= 1
|
||||
if k < 0:
|
||||
break
|
||||
few_shot_examples = gen_prompt(dev_df, subject, k)
|
||||
|
||||
preds = []
|
||||
labels = []
|
||||
tic = time.perf_counter()
|
||||
|
||||
for i in range(test_df.shape[0]):
|
||||
prompt_end = format_example(test_df, i, include_answer=False)
|
||||
prompt = few_shot_examples + prompt_end
|
||||
|
||||
input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.device)
|
||||
output_ids = model.generate(
|
||||
input_ids,
|
||||
max_new_tokens=1,
|
||||
do_sample=False,
|
||||
pad_token_id=tokenizer.eos_token_id,
|
||||
)
|
||||
|
||||
output_str = tokenizer.decode(
|
||||
output_ids[0][input_ids.shape[-1] :], skip_special_tokens=True
|
||||
)
|
||||
preds.append(output_str.strip()[0] if len(output_str.strip()) > 0 else "")
|
||||
labels.append(test_df.iloc[i, test_df.shape[1] - 1])
|
||||
|
||||
latency = time.perf_counter() - tic
|
||||
total_latency += latency
|
||||
|
||||
cors = [pred == label for pred, label in zip(preds, labels)]
|
||||
all_cors.append(cors)
|
||||
num_requests += len(test_df)
|
||||
|
||||
print(
|
||||
f"Subject: {subject}, Accuracy: {np.mean(cors):.3f}, Latency: {latency:.3f}s"
|
||||
)
|
||||
|
||||
weighted_acc = np.mean(np.concatenate(all_cors))
|
||||
print(f"Total Latency: {total_latency:.3f}s")
|
||||
print(f"Average Accuracy: {weighted_acc:.3f}")
|
||||
|
||||
if args.output:
|
||||
with open(args.output, "a") as fout:
|
||||
value = {
|
||||
"task": "mmlu",
|
||||
"backend": "hf",
|
||||
"model": args.model_path,
|
||||
"latency": round(total_latency, 3),
|
||||
"accuracy": round(weighted_acc, 3),
|
||||
"num_requests": num_requests,
|
||||
"other": {
|
||||
"nsub": args.nsub,
|
||||
"ntrain": args.ntrain,
|
||||
},
|
||||
}
|
||||
fout.write(json.dumps(value) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model-path", type=str, required=True)
|
||||
parser.add_argument("--ntrain", type=int, default=5)
|
||||
parser.add_argument("--data-dir", type=str, default="data")
|
||||
parser.add_argument("--nsub", type=int, default=60)
|
||||
parser.add_argument("--output", type=str, help="Output file path")
|
||||
args = parser.parse_args()
|
||||
main(args)
|
||||
@@ -1,210 +0,0 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tarfile
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import tiktoken
|
||||
|
||||
from sglang.test.test_utils import (
|
||||
add_common_sglang_args_and_parse,
|
||||
dump_bench_raw_result,
|
||||
select_sglang_backend,
|
||||
)
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
choices = ["A", "B", "C", "D"]
|
||||
|
||||
tokenizer = tiktoken.encoding_for_model("gpt-3.5-turbo")
|
||||
|
||||
|
||||
def format_subject(subject):
|
||||
l = subject.split("_")
|
||||
s = ""
|
||||
for entry in l:
|
||||
s += " " + entry
|
||||
return s
|
||||
|
||||
|
||||
def format_example(df, idx, include_answer=True):
|
||||
prompt = df.iloc[idx, 0]
|
||||
k = df.shape[1] - 2
|
||||
for j in range(k):
|
||||
prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1])
|
||||
prompt += "\nAnswer:"
|
||||
if include_answer:
|
||||
prompt += " {}\n\n".format(df.iloc[idx, k + 1])
|
||||
return prompt
|
||||
|
||||
|
||||
def gen_prompt(train_df, subject, k=-1):
|
||||
prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format(
|
||||
format_subject(subject)
|
||||
)
|
||||
if k == -1:
|
||||
k = train_df.shape[0]
|
||||
for i in range(k):
|
||||
prompt += format_example(train_df, i)
|
||||
return prompt
|
||||
|
||||
|
||||
def download_data(data_dir):
|
||||
"""Download and extract MMLU data if it doesn't exist."""
|
||||
if os.path.isdir(os.path.join(data_dir, "test")):
|
||||
return
|
||||
print(f"Data not found at {data_dir}. Downloading...")
|
||||
os.makedirs(data_dir, exist_ok=True)
|
||||
tar_path = os.path.join(data_dir, "data.tar")
|
||||
subprocess.check_call(
|
||||
["wget", "-O", tar_path, "https://people.eecs.berkeley.edu/~hendrycks/data.tar"]
|
||||
)
|
||||
with tarfile.open(tar_path) as tar:
|
||||
tar.extractall(path=data_dir, filter="data")
|
||||
# The tarball extracts into a "data/" subdirectory; move contents up if needed
|
||||
nested = os.path.join(data_dir, "data")
|
||||
if os.path.isdir(nested):
|
||||
for item in os.listdir(nested):
|
||||
os.rename(os.path.join(nested, item), os.path.join(data_dir, item))
|
||||
os.rmdir(nested)
|
||||
os.remove(tar_path)
|
||||
print("Download complete.")
|
||||
|
||||
|
||||
def main(args):
|
||||
subjects = sorted(
|
||||
[
|
||||
f.split("_test.csv")[0]
|
||||
for f in os.listdir(os.path.join(args.data_dir, "test"))
|
||||
if "_test.csv" in f
|
||||
]
|
||||
)
|
||||
|
||||
# Build prompts
|
||||
arguments = []
|
||||
labels = []
|
||||
num_questions = []
|
||||
|
||||
for subject in subjects[: args.nsub]:
|
||||
dev_df = pd.read_csv(
|
||||
os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None
|
||||
)[: args.ntrain]
|
||||
test_df = pd.read_csv(
|
||||
os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None
|
||||
)
|
||||
num_questions.append(test_df.shape[0])
|
||||
|
||||
k = args.ntrain
|
||||
few_shot_examples = gen_prompt(dev_df, subject, k)
|
||||
while len(tokenizer.encode(few_shot_examples)) > 1536:
|
||||
k -= 1
|
||||
few_shot_examples = gen_prompt(dev_df, subject, k)
|
||||
|
||||
for i in range(test_df.shape[0]):
|
||||
prompt_end = format_example(test_df, i, include_answer=False)
|
||||
|
||||
arguments.append(
|
||||
{
|
||||
"examples": few_shot_examples,
|
||||
"question": prompt_end,
|
||||
}
|
||||
)
|
||||
|
||||
label = test_df.iloc[i, test_df.shape[1] - 1]
|
||||
labels.append(label)
|
||||
|
||||
#####################################
|
||||
######### SGL Program Begin #########
|
||||
#####################################
|
||||
|
||||
import sglang as sgl
|
||||
|
||||
if args.backend.startswith("gpt-"):
|
||||
|
||||
@sgl.function
|
||||
def few_shot_mmlu(s, examples, question):
|
||||
s += sgl.user(examples + question)
|
||||
s += sgl.assistant(sgl.gen("answer"))
|
||||
|
||||
else:
|
||||
|
||||
@sgl.function
|
||||
def few_shot_mmlu(s, examples, question):
|
||||
s += examples + question + sgl.gen("answer")
|
||||
|
||||
#####################################
|
||||
########## SGL Program End ##########
|
||||
#####################################
|
||||
|
||||
# Select backend
|
||||
backend = select_sglang_backend(args)
|
||||
|
||||
# Run
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_mmlu.run_batch(
|
||||
arguments,
|
||||
temperature=0,
|
||||
max_new_tokens=1,
|
||||
backend=backend,
|
||||
num_threads=args.parallel,
|
||||
progress_bar=True,
|
||||
)
|
||||
preds = [
|
||||
s["answer"].strip()[0] if len(s["answer"].strip()) > 0 else "" for s in states
|
||||
]
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
# Compute accuracy
|
||||
cors = [pred == label for pred, label in zip(preds, labels)]
|
||||
|
||||
pt = 0
|
||||
for subject, num_qs in zip(subjects[: args.nsub], num_questions):
|
||||
print(
|
||||
f"subject: {subject}, #q:{num_qs}, acc: {np.mean(cors[pt : pt + num_qs]):.3f}"
|
||||
)
|
||||
pt += num_qs
|
||||
assert pt == len(cors)
|
||||
weighted_acc = np.mean(cors)
|
||||
|
||||
dump_bench_raw_result(
|
||||
path=args.raw_result_file,
|
||||
states=states,
|
||||
preds=preds,
|
||||
labels=labels,
|
||||
)
|
||||
|
||||
# Print results
|
||||
print("Total latency: {:.3f}".format(latency))
|
||||
print("Average accuracy: {:.3f}".format(weighted_acc))
|
||||
|
||||
# Write results
|
||||
with open(args.result_file, "a") as fout:
|
||||
value = {
|
||||
"task": "mmlu",
|
||||
"backend": args.backend,
|
||||
"num_gpus": 1,
|
||||
"latency": round(latency, 3),
|
||||
"accuracy": round(weighted_acc, 3),
|
||||
"num_requests": len(arguments),
|
||||
"other": {
|
||||
"nsub": args.nsub,
|
||||
"parallel": args.parallel,
|
||||
},
|
||||
}
|
||||
fout.write(json.dumps(value) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--ntrain", "-k", type=int, default=5)
|
||||
parser.add_argument(
|
||||
"--data_dir", "-d", type=str, default=os.path.join(SCRIPT_DIR, "data")
|
||||
)
|
||||
parser.add_argument("--save_dir", "-s", type=str, default="results")
|
||||
parser.add_argument("--nsub", type=int, default=60)
|
||||
args = add_common_sglang_args_and_parse(parser)
|
||||
download_data(args.data_dir)
|
||||
main(args)
|
||||
@@ -1,2 +0,0 @@
|
||||
wget https://people.eecs.berkeley.edu/~hendrycks/data.tar
|
||||
tar xf data.tar
|
||||
@@ -0,0 +1,271 @@
|
||||
"""Engine benchmark for setwise (multi-position) scoring.
|
||||
|
||||
Benchmarks `Engine.async_score(..., score_extraction_token_id=...)` — the setwise
|
||||
path that pools the classification head at each candidate's anchor token and
|
||||
returns an ``[N, num_labels]`` matrix in a single forward pass.
|
||||
|
||||
Unlike the general ``bench_score.py`` (label-token / MIS scoring), this is a
|
||||
focused, self-contained harness: it sweeps the **set size** (number of
|
||||
candidates packed into one item) and reports latency percentiles and throughput
|
||||
(requests/s and candidates/s) so you can see how a single-forward setwise pass
|
||||
scales with N.
|
||||
|
||||
It talks to the Engine directly (no HTTP) and requires the setwise deployment
|
||||
constraints (``--disable-radix-cache``, ``--chunked-prefill-size -1``), which it
|
||||
sets automatically.
|
||||
|
||||
Usage:
|
||||
# Default sweep over set sizes on the target model
|
||||
python bench_setwise_score.py --model-path /path/to/seqcls_model
|
||||
|
||||
# Custom sweep + concurrency + per-candidate length
|
||||
python bench_setwise_score.py --model-path /path/to/model \
|
||||
--set-sizes 8 16 32 64 --concurrency 16 --num-requests 100 \
|
||||
--item-tokens 180 --query-tokens 64
|
||||
|
||||
# Include pooled hidden states in the response (per-anchor pre-head vectors)
|
||||
python bench_setwise_score.py --model-path /path/to/model --return-pooled-hidden-states
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import csv
|
||||
import random
|
||||
import statistics
|
||||
import time
|
||||
from typing import List, Optional
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Benchmark setwise scoring via Engine.async_score",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model-path",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Path to a SequenceClassification model (e.g. Qwen3ForSequenceClassification).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--score-extraction-token",
|
||||
type=str,
|
||||
default="<|object_ref_start|>",
|
||||
help="Anchor token pooled at each candidate (default: <|object_ref_start|>).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--set-sizes",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=[8, 16, 32, 64],
|
||||
help="Candidates per set (each is one anchor in one item). Swept in order.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--item-tokens",
|
||||
type=int,
|
||||
default=180,
|
||||
help="Content tokens per candidate before its anchor (default: 180).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--query-tokens",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Shared query-prefix tokens (default: 0).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-requests",
|
||||
type=int,
|
||||
default=50,
|
||||
help="Timed requests per set size (default: 50).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--concurrency",
|
||||
type=int,
|
||||
default=8,
|
||||
help="Max in-flight requests (default: 8).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--warmup",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Warmup requests per set size, not timed (default: 3).",
|
||||
)
|
||||
parser.add_argument("--apply-softmax", action="store_true")
|
||||
parser.add_argument("--return-pooled-hidden-states", action="store_true")
|
||||
|
||||
# Engine configuration
|
||||
parser.add_argument("--dtype", type=str, default="float16")
|
||||
parser.add_argument("--mem-fraction-static", type=float, default=0.5)
|
||||
parser.add_argument("--max-prefill-tokens", type=int, default=60000)
|
||||
parser.add_argument("--attention-backend", type=str, default="flashinfer")
|
||||
parser.add_argument("--seed", type=int, default=0)
|
||||
parser.add_argument(
|
||||
"--output", "-o", type=str, default=None, help="Optional CSV output path."
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _random_content(num_tokens: int, anchor_id: int, lo: int = 10, hi: int = 30000):
|
||||
"""Random content token ids that never collide with the anchor id."""
|
||||
ids = [random.randint(lo, hi) for _ in range(num_tokens)]
|
||||
return [t + 1 if t == anchor_id else t for t in ids]
|
||||
|
||||
|
||||
def _build_setwise_request(args, anchor_id: int, n_candidates: int):
|
||||
"""One setwise request: shared query prefix + ONE item with N anchors."""
|
||||
query_ids = (
|
||||
_random_content(args.query_tokens, anchor_id) if args.query_tokens else []
|
||||
)
|
||||
item_ids: List[int] = []
|
||||
for _ in range(n_candidates):
|
||||
item_ids.extend(_random_content(args.item_tokens, anchor_id))
|
||||
item_ids.append(anchor_id)
|
||||
return query_ids, [item_ids]
|
||||
|
||||
|
||||
def _percentile(sorted_vals: List[float], pct: float) -> float:
|
||||
if not sorted_vals:
|
||||
return 0.0
|
||||
k = max(0, min(len(sorted_vals) - 1, round((pct / 100.0) * (len(sorted_vals) - 1))))
|
||||
return sorted_vals[k]
|
||||
|
||||
|
||||
async def _run_set_size(engine, args, anchor_id: int, n: int) -> Optional[dict]:
|
||||
# Pre-build requests so generation cost is outside the timed section.
|
||||
requests = [
|
||||
_build_setwise_request(args, anchor_id, n)
|
||||
for _ in range(args.num_requests + args.warmup)
|
||||
]
|
||||
prompt_tokens = len(requests[0][0]) + len(requests[0][1][0])
|
||||
|
||||
async def _one(query_ids, items):
|
||||
t0 = time.perf_counter()
|
||||
res = await engine.async_score(
|
||||
query=query_ids,
|
||||
items=items,
|
||||
score_extraction_token_id=anchor_id,
|
||||
apply_softmax=args.apply_softmax,
|
||||
return_pooled_hidden_states=args.return_pooled_hidden_states,
|
||||
)
|
||||
# Setwise scores are nested (one [Ni x num_labels] matrix per item) and
|
||||
# this harness sends exactly one item, so the per-candidate row count is
|
||||
# len(res.scores[0]), not len(res.scores) (which is always 1 = num items).
|
||||
return time.perf_counter() - t0, len(res.scores[0]) if res.scores else 0
|
||||
|
||||
# Warmup (also validates the row count == N).
|
||||
for q, it in requests[: args.warmup]:
|
||||
_, rows = await _one(q, it)
|
||||
if rows != n:
|
||||
print(f"[warn] set_size={n}: expected {n} score rows, got {rows}")
|
||||
|
||||
sem = asyncio.Semaphore(args.concurrency)
|
||||
|
||||
async def _guarded(q, it):
|
||||
async with sem:
|
||||
return await _one(q, it)
|
||||
|
||||
timed = requests[args.warmup :]
|
||||
wall_start = time.perf_counter()
|
||||
results = await asyncio.gather(*(_guarded(q, it) for q, it in timed))
|
||||
wall = time.perf_counter() - wall_start
|
||||
|
||||
latencies_ms = sorted(1000.0 * lat for lat, _ in results)
|
||||
n_req = len(results)
|
||||
return {
|
||||
"set_size": n,
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"requests": n_req,
|
||||
"concurrency": args.concurrency,
|
||||
"avg_ms": statistics.mean(latencies_ms),
|
||||
"p50_ms": _percentile(latencies_ms, 50),
|
||||
"p95_ms": _percentile(latencies_ms, 95),
|
||||
"p99_ms": _percentile(latencies_ms, 99),
|
||||
"req_per_s": n_req / wall if wall > 0 else 0.0,
|
||||
"cand_per_s": (n_req * n) / wall if wall > 0 else 0.0,
|
||||
}
|
||||
|
||||
|
||||
def _print_table(rows: List[dict]):
|
||||
headers = [
|
||||
"set_size",
|
||||
"prompt_tokens",
|
||||
"requests",
|
||||
"avg_ms",
|
||||
"p50_ms",
|
||||
"p95_ms",
|
||||
"p99_ms",
|
||||
"req_per_s",
|
||||
"cand_per_s",
|
||||
]
|
||||
print("\n" + " | ".join(f"{h:>13}" for h in headers))
|
||||
print("-" * (16 * len(headers)))
|
||||
for r in rows:
|
||||
cells = []
|
||||
for h in headers:
|
||||
v = r[h]
|
||||
cells.append(f"{v:>13.2f}" if isinstance(v, float) else f"{v:>13}")
|
||||
print(" | ".join(cells))
|
||||
|
||||
|
||||
async def main():
|
||||
args = parse_args()
|
||||
random.seed(args.seed)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True)
|
||||
anchor_id = tokenizer.convert_tokens_to_ids(args.score_extraction_token)
|
||||
unk_id = tokenizer.unk_token_id
|
||||
if anchor_id is None or (unk_id is not None and anchor_id == unk_id):
|
||||
raise ValueError(
|
||||
f"score_extraction_token {args.score_extraction_token!r} did not resolve "
|
||||
f"to a dedicated token id."
|
||||
)
|
||||
|
||||
from sglang.srt.entrypoints.engine import Engine
|
||||
|
||||
# Setwise deployment constraints: positions are full-prompt coordinates, so
|
||||
# radix reuse / chunked prefill must be off (the engine also validates this).
|
||||
engine = Engine(
|
||||
model_path=args.model_path,
|
||||
dtype=args.dtype,
|
||||
mem_fraction_static=args.mem_fraction_static,
|
||||
max_prefill_tokens=args.max_prefill_tokens,
|
||||
attention_backend=args.attention_backend,
|
||||
disable_radix_cache=True,
|
||||
chunked_prefill_size=-1,
|
||||
)
|
||||
|
||||
print(
|
||||
f"Setwise scoring benchmark | model={args.model_path}\n"
|
||||
f"anchor={args.score_extraction_token!r} (id={anchor_id}) | "
|
||||
f"item_tokens={args.item_tokens} query_tokens={args.query_tokens} | "
|
||||
f"concurrency={args.concurrency} num_requests={args.num_requests}"
|
||||
)
|
||||
|
||||
rows = []
|
||||
try:
|
||||
for n in args.set_sizes:
|
||||
row = await _run_set_size(engine, args, anchor_id, n)
|
||||
if row is not None:
|
||||
rows.append(row)
|
||||
print(
|
||||
f"set_size={n:>4}: avg={row['avg_ms']:.1f}ms "
|
||||
f"p95={row['p95_ms']:.1f}ms "
|
||||
f"{row['req_per_s']:.1f} req/s {row['cand_per_s']:.0f} cand/s"
|
||||
)
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
_print_table(rows)
|
||||
|
||||
if args.output and rows:
|
||||
with open(args.output, "w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
print(f"\nWrote {len(rows)} rows to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Finite cold-cache decode benchmark using synthetic token IDs.
|
||||
|
||||
Run against an otherwise idle SGLang server. This clears its prefix cache.
|
||||
Example:
|
||||
python benchmark/serving/benchmark_h20_decode.py \
|
||||
--model-path /path/to/DeepSeek-V4.1-Flash --output before.jsonl
|
||||
|
||||
Repeat after installing the H20-3e configurations with identical server flags.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
import statistics
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
from tokenizers import Tokenizer
|
||||
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("--label", default="measurement")
|
||||
p.add_argument("--url", default="http://127.0.0.1:8000")
|
||||
p.add_argument("--model-path", type=pathlib.Path, required=True)
|
||||
p.add_argument("--output", type=pathlib.Path, required=True)
|
||||
args = p.parse_args()
|
||||
out = args.output
|
||||
assert not out.exists()
|
||||
tokenizer = Tokenizer.from_file(str(args.model_path / "tokenizer.json"))
|
||||
filler = tokenizer.encode(
|
||||
(
|
||||
"The research server processes independent records. Each record contains an identifier, a timestamp, and a measured value. We compare latency using the same data and the same output budget.\n"
|
||||
)
|
||||
* 16000,
|
||||
add_special_tokens=False,
|
||||
).ids
|
||||
|
||||
|
||||
def emit(row):
|
||||
row = {
|
||||
"time": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"label": args.label,
|
||||
**row,
|
||||
}
|
||||
with out.open("a") as f:
|
||||
f.write(json.dumps(row) + "\n")
|
||||
print(json.dumps(row), flush=True)
|
||||
|
||||
|
||||
def request(path, body=None):
|
||||
return urllib.request.urlopen(
|
||||
urllib.request.Request(
|
||||
args.url.rstrip("/") + path,
|
||||
data=None if body is None else json.dumps(body).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
),
|
||||
timeout=None,
|
||||
)
|
||||
|
||||
|
||||
def flush():
|
||||
with request("/flush_cache?timeout=30", {}) as r:
|
||||
r.read()
|
||||
|
||||
|
||||
def native(case, isl, osl, index):
|
||||
ids = (
|
||||
tokenizer.encode(
|
||||
f"Independent record {index}: {case}.\n", add_special_tokens=False
|
||||
).ids
|
||||
+ filler
|
||||
)[:isl]
|
||||
start = time.perf_counter()
|
||||
first = last = None
|
||||
final = None
|
||||
events = 0
|
||||
body = {
|
||||
"input_ids": ids,
|
||||
"sampling_params": {
|
||||
"temperature": 0.0,
|
||||
"max_new_tokens": osl,
|
||||
"ignore_eos": True,
|
||||
},
|
||||
"stream": True,
|
||||
}
|
||||
with request("/generate", body) as response:
|
||||
for line in response:
|
||||
if not line.startswith(b"data:"):
|
||||
continue
|
||||
value = line[5:].strip()
|
||||
if value == b"[DONE]":
|
||||
break
|
||||
obj = json.loads(value)
|
||||
now = time.perf_counter()
|
||||
if obj.get("error"):
|
||||
raise RuntimeError(str(obj["error"]))
|
||||
if obj.get("meta_info", {}).get("completion_tokens", 0) > 0:
|
||||
first = first or now
|
||||
last = now
|
||||
final = obj
|
||||
events += 1
|
||||
elapsed = time.perf_counter() - start
|
||||
assert final is not None, (case, index)
|
||||
meta = final["meta_info"]
|
||||
assert meta["completion_tokens"] == osl and meta["prompt_tokens"] == isl, meta
|
||||
assert meta.get("cached_tokens") == 0, meta
|
||||
return {
|
||||
"kind": "request",
|
||||
"case": case,
|
||||
"index": index,
|
||||
"input_hash": hashlib.sha256(json.dumps(ids).encode()).hexdigest(),
|
||||
"prompt_tokens": isl,
|
||||
"completion_tokens": osl,
|
||||
"cached_tokens": meta["cached_tokens"],
|
||||
"ttft_s": first - start,
|
||||
"elapsed_s": elapsed,
|
||||
"decode_tokens_per_s": (osl - 1) / (last - first) if osl > 1 else None,
|
||||
"tpot_ms": 1000 * (last - first) / (osl - 1) if osl > 1 else None,
|
||||
"events": events,
|
||||
}
|
||||
|
||||
|
||||
cases = [
|
||||
("8k_1000_c1", 8192, 1000, 1),
|
||||
("1k_128_c1", 1024, 128, 1),
|
||||
("1k_128_c8", 1024, 128, 8),
|
||||
("1k_128_c32", 1024, 128, 32),
|
||||
("8k_128_c8", 8192, 128, 8),
|
||||
("32k_128_c1", 32768, 128, 1),
|
||||
]
|
||||
emit(
|
||||
{
|
||||
"kind": "start",
|
||||
"cases": cases,
|
||||
"repetitions": 3,
|
||||
}
|
||||
)
|
||||
try:
|
||||
for case, isl, osl, concurrency in cases:
|
||||
flush()
|
||||
emit({"kind": "warmup_start", "case": case})
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
list(
|
||||
pool.map(
|
||||
lambda i: native("warm_" + case, isl, 16, i), range(concurrency)
|
||||
)
|
||||
)
|
||||
for repeat in range(3):
|
||||
flush()
|
||||
start = time.perf_counter()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
rows = list(
|
||||
pool.map(
|
||||
lambda i: native(case, isl, osl, i + repeat * 100),
|
||||
range(concurrency),
|
||||
)
|
||||
)
|
||||
wall = time.perf_counter() - start
|
||||
for row in rows:
|
||||
emit({**row, "repeat": repeat})
|
||||
emit(
|
||||
{
|
||||
"kind": "batch",
|
||||
"case": case,
|
||||
"repeat": repeat,
|
||||
"concurrency": concurrency,
|
||||
"wall_s": wall,
|
||||
"output_tokens_per_s": concurrency * osl / wall,
|
||||
"median_ttft_s": statistics.median(x["ttft_s"] for x in rows),
|
||||
"median_decode_tokens_per_s": statistics.median(
|
||||
x["decode_tokens_per_s"] for x in rows
|
||||
),
|
||||
"median_tpot_ms": statistics.median(x["tpot_ms"] for x in rows),
|
||||
}
|
||||
)
|
||||
emit({"kind": "complete"})
|
||||
except Exception as exc:
|
||||
emit({"kind": "failed", "error": repr(exc)})
|
||||
raise
|
||||
@@ -1,77 +0,0 @@
|
||||
# 优化版(2026-10-08 v3):torch/nvidia 全家桶永远复用基底,全量路径零重下。
|
||||
# 要点:
|
||||
# 1. pyproject build-system.requires 含 torch==2.13.0,pip 隔离构建环境每次都重下 ~6GB。
|
||||
# 基底镜像已装 torch 2.13 → 用 --no-build-isolation 复用,构建环境零下载。
|
||||
# 2. 老全量路径让 pip 解析全部依赖:bare `torchvision` 等会让解析器弃用基底已装的
|
||||
# 本地版本(torch 2.13.0+cu130),改用 PyPI 的 torch==2.13.0,连带重下整套 nvidia
|
||||
# 依赖(9/23 实测 nvidia-nccl-cu13 等 6GB+,限流下 2h40m);且基底 NCCL 是
|
||||
# 刻意装高的 2.30.7(torch 元数据钉 ==2.29.7),任何全量解析都会把它降级回去。
|
||||
# → v3 不再让 pip 碰重型依赖:check_deps.py 在构建期对照 pyproject 与基底
|
||||
# 已装版本,把增量分三类——重型(torch*/nvidia-*/cuda-*)永不进解析、
|
||||
# 不满足即让构建失败并提示决策(显式失败好过静默重装);
|
||||
# kernel wheel(sglang-kernel/flashinfer*/sgl-deep-gemm)要跟随源码版本,
|
||||
# 钉版变动时以 pip --no-deps 单独装(防止顺藤摸瓜解析 torch);
|
||||
# 其余轻量依赖只装未满足的增量。
|
||||
# 3. 基底缺 setuptools-scm(版本打戳要用),单层预装 + pip cache mount,只下载一次。
|
||||
# 4. RUN 的 pip cache mount 持久化在宿主机,轻量依赖增量不再重复下载。
|
||||
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
|
||||
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV PIP_INDEX_URL=${PIP_INDEX}
|
||||
|
||||
# 构建后端(pyproject build-system.requires,torch 除外——基底已有)
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
|
||||
|
||||
# 依赖增量计算脚本(对照 pyproject 与基底已装版本,产出轻量/kernel 增量清单)
|
||||
# 注意 build context 是 /data/ymk 且 .dockerignore 排除了 build/,只能从 sglang/ 树内 COPY
|
||||
COPY sglang/deploy/b300-build/check_deps.py /tmp/check_deps.py
|
||||
|
||||
# 整个源码树(含 .git,用于 setuptools-scm 版本打戳)
|
||||
COPY sglang/ /sgl-workspace/sglang/
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译。基底自带 rustup stable(实测 cargo 1.98.1),
|
||||
# RUSTUP_TOOLCHAIN=stable 绕过仓库 rust-toolchain.toml 钉的 1.90(避免重复下载工具链);
|
||||
# b300 直连 crates.io(稀疏索引 + static.crates.io 均通),无需任何镜像源。
|
||||
# cache mount 持久化 registry/target:冷编译 ~15min(二进制 ~11min + python 扩展 ~4min),
|
||||
# 增量重建秒~分钟级。python 扩展必须显式 -p(workspace 根包默认不编译扩展),
|
||||
# 产物 libsglang_router_rs.so 需重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件
|
||||
# (与 H20 线上部署方式一致)。2026-10-09 b300 实测验证。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
# 计算依赖增量(重型 torch/nvidia/cuda 永不进入清单;不满足则 check_deps 非零、构建失败)
|
||||
RUN python /tmp/check_deps.py /sgl-workspace/sglang/python/pyproject.toml \
|
||||
/tmp/missing_light.txt /tmp/missing_kernel.txt \
|
||||
&& echo "== kernel wheel delta (pip --no-deps) ==" && cat /tmp/missing_kernel.txt \
|
||||
&& echo "== lightweight dep delta ==" && cat /tmp/missing_light.txt
|
||||
|
||||
# kernel wheel 增量(如 sglang-kernel 升版):--no-deps 防止 pip 顺藤摸瓜解析 torch
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_kernel.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_kernel.txt; \
|
||||
else \
|
||||
echo "== no kernel wheel delta"; \
|
||||
fi
|
||||
|
||||
# 轻量依赖增量(通常为空或几个小包),重型全家桶原样复用基底。
|
||||
# ⚠️ 必须 --no-deps:即使目标只是 xgrammar 这种小包,只要依赖图里出现 torch,
|
||||
# pip 就会校验 torch+cu130 元数据钉死的 nvidia-nccl-cu13==2.29.7,发现基底装的是
|
||||
# 2.30.7 便"好心"下载 206MB 降级(10-08 v3 首跑实测踩中)。基底的传递依赖本来就
|
||||
# 齐全,增量小包的二级依赖也一样;若有漏网,b300 冒烟会在 import 阶段显式暴露。
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_light.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_light.txt; \
|
||||
else \
|
||||
echo "== no lightweight delta, skipping dependency install"; \
|
||||
fi
|
||||
|
||||
# editable 安装 sglang 本体,--no-deps 保证 pip 完全不碰依赖图
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-deps --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
@@ -1,65 +0,0 @@
|
||||
# fast 路径(2026-10-08 v3.1):python/pyproject.toml 未变化时使用(build.sh 自动判断)。
|
||||
# 与 Dockerfile 全量路径同构:同样跑 check_deps.py 增量安装。
|
||||
# 原因:全量构建若装过 kernel wheel 增量(如 sglang-kernel 0.4.7),随后 pyproject 未变的
|
||||
# fast 构建若只 --no-deps,会静默退回基底的旧 kernel(0.4.6.post1)——镜像内容漂移。
|
||||
# check_deps 本地比对 ~0.5s,增量命中 pip cache 秒级,成本可忽略。
|
||||
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
|
||||
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV PIP_INDEX_URL=${PIP_INDEX}
|
||||
|
||||
# 构建后端(pyproject build-system.requires,torch 除外——基底已有)
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
|
||||
|
||||
# 依赖增量计算脚本(对照 pyproject 与基底已装版本,产出轻量/kernel 增量清单)
|
||||
# 注意 build context 是 /data/ymk 且 .dockerignore 排除了 build/,只能从 sglang/ 树内 COPY
|
||||
COPY sglang/deploy/b300-build/check_deps.py /tmp/check_deps.py
|
||||
|
||||
# 整个源码树(含 .git,用于 setuptools-scm 版本打戳)
|
||||
COPY sglang/ /sgl-workspace/sglang/
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译。基底自带 rustup stable(实测 cargo 1.98.1),
|
||||
# RUSTUP_TOOLCHAIN=stable 绕过仓库 rust-toolchain.toml 钉的 1.90(避免重复下载工具链);
|
||||
# b300 直连 crates.io(稀疏索引 + static.crates.io 均通),无需任何镜像源。
|
||||
# cache mount 持久化 registry/target:冷编译 ~15min,增量重建秒~分钟级。
|
||||
# python 扩展必须显式 -p(workspace 根包默认不编译扩展),产物 libsglang_router_rs.so
|
||||
# 需重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件(与 H20 线上部署方式一致)。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
# 计算依赖增量(重型 torch/nvidia/cuda 永不进入清单;不满足则 check_deps 非零、构建失败)
|
||||
RUN python /tmp/check_deps.py /sgl-workspace/sglang/python/pyproject.toml \
|
||||
/tmp/missing_light.txt /tmp/missing_kernel.txt \
|
||||
&& echo "== kernel wheel delta (pip --no-deps) ==" && cat /tmp/missing_kernel.txt \
|
||||
&& echo "== lightweight dep delta ==" && cat /tmp/missing_light.txt
|
||||
|
||||
# kernel wheel 增量(如 sglang-kernel 升版):--no-deps 防止 pip 顺藤摸瓜解析 torch
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_kernel.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_kernel.txt; \
|
||||
else \
|
||||
echo "== no kernel wheel delta"; \
|
||||
fi
|
||||
|
||||
# 轻量依赖增量(通常为空或几个小包),重型全家桶原样复用基底。
|
||||
# ⚠️ 必须 --no-deps:即使目标只是 xgrammar 这种小包,只要依赖图里出现 torch,
|
||||
# pip 就会校验 torch+cu130 元数据钉死的 nvidia-nccl-cu13==2.29.7,发现基底装的是
|
||||
# 2.30.7 便"好心"下载 206MB 降级(10-08 v3 首跑实测踩中)。基底的传递依赖本来就
|
||||
# 齐全,增量小包的二级依赖也一样;若有漏网,b300 冒烟会在 import 阶段显式暴露。
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_light.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_light.txt; \
|
||||
else \
|
||||
echo "== no lightweight delta, skipping dependency install"; \
|
||||
fi
|
||||
|
||||
# editable 安装 sglang 本体,--no-deps 保证 pip 完全不碰依赖图
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-deps --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
@@ -1,33 +0,0 @@
|
||||
#!/bin/bash
|
||||
# 用法: /data/ymk/build/build.sh [tag]
|
||||
# 默认 tag: ymkymx/sglang:<分支>-<sha9>-local-<UTC构建日期时间>(命名规则见 AGENTS.md)
|
||||
# 另打 <分支>-latest-local 别名。
|
||||
#
|
||||
# 依赖路径自动选择:
|
||||
# python/pyproject.toml 的 sha256 与上次成功构建一致 → Dockerfile.fast(--no-deps,秒级)
|
||||
# 不一致(或 FULL=1)→ Dockerfile 全量依赖解析,成功后记录新 hash
|
||||
set -euo pipefail
|
||||
cd /data/ymk
|
||||
SHA=$(git -C sglang rev-parse --short=9 HEAD)
|
||||
BR=$(git -C sglang rev-parse --abbrev-ref HEAD)
|
||||
DT=$(date -u +%Y%m%d-%H%M)
|
||||
TAG=${1:-ymkymx/sglang:$BR-$SHA-local-$DT}
|
||||
|
||||
HASH_FILE=build/.last_pyproject_sha256
|
||||
CUR=$(sha256sum sglang/python/pyproject.toml | cut -d' ' -f1)
|
||||
PREV=$(cat "$HASH_FILE" 2>/dev/null || echo none)
|
||||
|
||||
DOCKERFILE=build/Dockerfile
|
||||
if [ "$CUR" = "$PREV" ] && [ "${FULL:-0}" != "1" ]; then
|
||||
DOCKERFILE=build/Dockerfile.fast
|
||||
echo "== pyproject 未变,走 fast 路径(--no-deps)"
|
||||
else
|
||||
echo "== pyproject 有变化或 FULL=1,走全量依赖解析"
|
||||
fi
|
||||
|
||||
echo "== building $TAG from sglang@$(git -C sglang log --oneline -1) [$DOCKERFILE]"
|
||||
docker build -f "$DOCKERFILE" -t "$TAG" .
|
||||
# 构建成功才记录 hash / 打别名
|
||||
echo "$CUR" > "$HASH_FILE"
|
||||
docker tag "$TAG" "ymkymx/sglang:$BR-latest-local"
|
||||
echo "== done: $TAG (别名 $BR-latest-local)"
|
||||
@@ -1,90 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compute dependency deltas for a b300 sglang image build, in three classes.
|
||||
|
||||
Reads pyproject.toml, evaluates every requirement against the *installed*
|
||||
distributions in the base image, and writes two requirements files:
|
||||
|
||||
argv[2] (light): unsatisfied lightweight requirements → `pip install -r`
|
||||
argv[3] (kernel): unsatisfied kernel wheels (sglang-kernel, flashinfer*,
|
||||
sgl-deep-gemm) → `pip install --no-deps -r`
|
||||
These track the sglang source being built and are meant
|
||||
to be upgraded from the index when pyproject bumps them;
|
||||
--no-deps keeps pip from walking torch's metadata.
|
||||
|
||||
The PyTorch stack (torch/torchvision/torchaudio/torchcodec/
|
||||
torch_memory_saver), the nvidia-* CUDA family and cuda-* packages are NEVER
|
||||
emitted. If one of them is missing or its installed version no longer
|
||||
satisfies the pinned specifier (e.g. upstream bumps nvidia-cutlass-dsl), the
|
||||
script fails the build loudly instead of letting pip download ~6GB or
|
||||
"fix" the base image's deliberately newer nvidia-nccl-cu13 2.30.7 back to
|
||||
torch's metadata pin ==2.29.7 (any pip full-resolution would do both —
|
||||
2026-09-23/24 builds did, throttled to 2h40m).
|
||||
"""
|
||||
|
||||
import sys
|
||||
import tomllib
|
||||
from importlib import metadata as md
|
||||
|
||||
from packaging.requirements import Requirement
|
||||
|
||||
# Never resolved by pip: reused from the base image or the build fails.
|
||||
HEAVY_PREFIXES = ("torch", "nvidia-", "cuda-")
|
||||
# Kernel wheels: upgraded via pip --no-deps when pyproject pins move.
|
||||
KERNEL_PREFIXES = ("sglang-kernel", "flashinfer", "sgl-deep-gemm")
|
||||
|
||||
|
||||
def norm(name: str) -> str:
|
||||
return name.lower().replace("_", "-")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
pyproject, light_out, kernel_out = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
with open(pyproject, "rb") as f:
|
||||
deps = tomllib.load(f)["project"]["dependencies"]
|
||||
|
||||
light: list[str] = []
|
||||
kernel: list[str] = []
|
||||
heavy_unsat: list[str] = []
|
||||
for dep in deps:
|
||||
req = Requirement(dep)
|
||||
if req.marker is not None and not req.marker.evaluate():
|
||||
continue
|
||||
try:
|
||||
installed = md.version(req.name)
|
||||
except md.PackageNotFoundError:
|
||||
installed = None
|
||||
satisfied = installed is not None and req.specifier.contains(
|
||||
installed, prereleases=True
|
||||
)
|
||||
if satisfied:
|
||||
continue
|
||||
name = norm(req.name)
|
||||
if name.startswith(HEAVY_PREFIXES):
|
||||
heavy_unsat.append(f"{dep} (installed: {installed})")
|
||||
elif name.startswith(KERNEL_PREFIXES):
|
||||
kernel.append(dep)
|
||||
else:
|
||||
light.append(dep)
|
||||
|
||||
if heavy_unsat:
|
||||
print("=" * 70, file=sys.stderr)
|
||||
print("FATAL: heavy base-image deps no longer satisfy pyproject:", file=sys.stderr)
|
||||
for item in heavy_unsat:
|
||||
print(f" {item}", file=sys.stderr)
|
||||
print(
|
||||
"Refusing to let pip resolve these (would re-download ~6GB and/or\n"
|
||||
"downgrade NCCL). Decide explicitly: bump the base image, or adjust\n"
|
||||
"the pin in pyproject.toml.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
with open(light_out, "w") as f:
|
||||
f.write("\n".join(light) + ("\n" if light else ""))
|
||||
with open(kernel_out, "w") as f:
|
||||
f.write("\n".join(kernel) + ("\n" if kernel else ""))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,33 +0,0 @@
|
||||
# B300 ds41 部署 compose 档案
|
||||
|
||||
b300-01 上 DeepSeek-V4.1-Flash 各部署方案的 docker compose 存档。
|
||||
**本目录是 source of truth**;b300 上的运行目录 `/data/ymk/ds41/` 是工作副本。
|
||||
|
||||
方案说明、参数矩阵、特殊 env、已知坑见飞书部署手册(组内共享):
|
||||
《B300 sglang 部署手册》 https://u04wb5irxz.feishu.cn/docx/FOi8dbnybodHr0xIfkPcQ6gtnRx
|
||||
|
||||
## 同步流程
|
||||
|
||||
```bash
|
||||
# 本机(D:\B300\sglang)
|
||||
git add deploy/ && git commit -m "deploy: ..."
|
||||
git push origin dsv41-pd-visioncp # Gitea(paths-ignore,不触发镜像构建)
|
||||
git push b300 dsv41-pd-visioncp # b300 裸仓库
|
||||
|
||||
# b300-01
|
||||
git -C /data/ymk/sglang pull # deploy/ 出现在 /data/ymk/sglang/deploy/b300-ds41/
|
||||
```
|
||||
|
||||
## CI 说明
|
||||
|
||||
`.gitea/workflows/build-image.yaml` 的 push 触发器带 `paths-ignore: ['deploy/**', '.gitea/**']`,
|
||||
改本目录不会触发镜像构建。注意:workflow 自身的改动也不再自动触发,需要重建镜像时
|
||||
伴随代码改动 push 或手动触发。
|
||||
|
||||
## 目录规则
|
||||
|
||||
- 命名:`dockerserve-<拓扑>-<特性>-<卡位>.yml`(如 `-b4` = 后 4 卡)
|
||||
- 退役文件不要留在主目录:移入 `archive-YYYYMMDD/`(b300 侧同样执行)
|
||||
- 禁止提交 `.bak` 备份文件
|
||||
- `mooncake-store.json` 是 hicache L3 mooncake store 配置,被
|
||||
`dockerserve-pd2-p-r1/r2.yml` 通过 `SGLANG_HICACHE_MOONCAKE_CONFIG_PATH` 引用
|
||||
@@ -1,27 +0,0 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
# 一律 --warmup-requests 0:默认 warmup 会把 prompt#0 种进 L1/L2 污染 L3 测量(已踩过)
|
||||
run() {
|
||||
echo "===== [$(date +%H:%M:%S)] RUN: $1 (np=$2) ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--warmup-requests 0 --num-prompts "$2" --max-concurrency "$2" --seed "$3" --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" 2>&1 | grep -E 'Input token throughput|Mean TTFT|P99 TTFT|Successful'
|
||||
sleep 2
|
||||
}
|
||||
flush() { curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache > /dev/null && echo "[$(date +%H:%M:%S)] flushed"; sleep 3; }
|
||||
|
||||
echo "===== PRIME COLD mc8(冷基线 + 写入 L3) ====="
|
||||
run PRIME_COLD 8 1541391
|
||||
flush
|
||||
run MC1_L3 1 1541391
|
||||
flush
|
||||
run MC2_L3 2 1541391
|
||||
flush
|
||||
run MC4_L3 4 1541391
|
||||
flush
|
||||
run MC8_L3 8 1541391
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -1,204 +0,0 @@
|
||||
#!/bin/bash
|
||||
# bench_suite_v2.sh — DeepSeek-V4.1-Flash on b300-01,2026-10-09 新版基线方法
|
||||
#
|
||||
# 相对 v1 的方法变化:
|
||||
# 1. 服务刚起数据不准 → 每个测试模式自带 warmup 轮(结果丢弃,标记 WARM)
|
||||
# 2. prefill 冷/热双轮:冷轮写入 hicache(L1/L3),同 seed 热轮测纯 cache-hit TPS
|
||||
# 3. decode 阶梯并发:步长 32,上限 = cuda-graph-max-bs-decode 的 80%;
|
||||
# 每档先 warm(灌 L1)再测,prefill 全 cache hit;
|
||||
# 报告 TPOT + 整实例输出 TPS + 推算实际并发(= 输出TPS × TPOT / 1000)
|
||||
# 4. l3 模式(3p1d 经 router):大语料溢出 L1 后重放,验证 L3 命中
|
||||
#
|
||||
# 在宿主机执行,bench 客户端通过 docker exec 进容器跑(需要 /data 挂载)。
|
||||
#
|
||||
# 用法:
|
||||
# bench_suite_v2.sh warmup <container> <port>
|
||||
# bench_suite_v2.sh prefill <tag> <container> <port>
|
||||
# bench_suite_v2.sh decode <tag> <container> <port> [cg_max_bs=256]
|
||||
# bench_suite_v2.sh l3 <tag> <container> <port> [num_prompts=24]
|
||||
# bench_suite_v2.sh report <log>
|
||||
# 产物: /data/ymk/ds41/bench_v2_<tag>.log
|
||||
set -u
|
||||
|
||||
BASE=/data/ymk/ds41
|
||||
MODEL=${MODEL:-/data/models/DeepSeek-V4.1-Flash}
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
|
||||
usage() { sed -n 2,20p "$0" >&2; exit 1; }
|
||||
|
||||
# $1=container $2=port $3=tag -> 生成可执行脚本路径并 echo
|
||||
emit_common() {
|
||||
cat <<EOF
|
||||
cd $BASE
|
||||
HDR="$HDR"
|
||||
PORT=$2
|
||||
run() {
|
||||
local name="\$1"; shift
|
||||
echo "===== [\$(date +%H:%M:%S)] START: \$name ====="
|
||||
python3 -m sglang.bench_serving "\$@"
|
||||
echo "===== [\$(date +%H:%M:%S)] END(exit=\$?): \$name ====="
|
||||
sleep 5
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
# ---------------- warmup ----------------
|
||||
cmd_warmup() {
|
||||
local c=${1:?container} port=${2:?port}
|
||||
local gen=$BASE/.bench_v2_warmup.sh
|
||||
{ emit_common "$c" "$port"; cat <<'EOF'
|
||||
run WARMUP_a --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777001 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
run WARMUP_b --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 16000 --random-output-len 1 \
|
||||
--num-prompts 2 --max-concurrency 1 --seed 777002 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
EOF
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
docker exec "$c" bash "$gen"
|
||||
}
|
||||
|
||||
# ---------------- prefill(冷/热双轮) ----------------
|
||||
cmd_prefill() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port}
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
{ emit_common "$c" "$port"
|
||||
# warmup(丢弃)
|
||||
cat <<'EOF'
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
EOF
|
||||
# 冷轮
|
||||
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
|
||||
set -- $stage
|
||||
cat <<EOF
|
||||
run COLD_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
|
||||
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
# 热轮(同 seed → cache hit)
|
||||
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
|
||||
set -- $stage
|
||||
cat <<EOF
|
||||
run HIT_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
|
||||
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! seeds=${p}1..3 log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- decode 阶梯 ----------------
|
||||
cmd_decode() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port} cg=${4:-256}
|
||||
local maxc=$(( cg * 80 / 100 / 32 * 32 )) # 80% 向下取整到 32 的倍数
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
echo "ladder: 32..$maxc step 32 (cg_max_bs=$cg)"
|
||||
{ emit_common "$c" "$port"
|
||||
cat <<'EOF'
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 128 \
|
||||
--num-prompts 16 --max-concurrency 16 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
EOF
|
||||
# bs1 延迟地板(warm + measure)
|
||||
cat <<EOF
|
||||
run WARM_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run DEC_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
local cc
|
||||
for (( cc=32; cc<=maxc; cc+=32 )); do
|
||||
local np=$(( cc * 2 ))
|
||||
cat <<EOF
|
||||
run WARM_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run DEC_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- L3 语料溢出(3p1d 经 router) ----------------
|
||||
cmd_l3() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port} np=${4:-48}
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
{ emit_common "$c" "$port"
|
||||
cat <<EOF
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \\
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run L3_COLD_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
|
||||
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
echo "===== [\$(date +%H:%M:%S)] DRAIN: sleep 120 等 L2→L3 备份排空 ====="
|
||||
sleep 120
|
||||
run L3_HIT_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
|
||||
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! corpus=${np}x160k log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- report ----------------
|
||||
cmd_report() {
|
||||
local log=${1:?log required}
|
||||
awk '
|
||||
/START:/ { name=$4; sub(/:$/, "", name); tps=""; itps=""; tpot=""; ttft=""; acc="" }
|
||||
/Input token throughput/ { itps=$NF }
|
||||
/Output token throughput/ { tps=$NF }
|
||||
/Mean TPOT/ { tpot=$NF }
|
||||
/Mean TTFT/ { ttft=$NF }
|
||||
/[Aa]ccept/ { acc=$0; gsub(/^ +/, "", acc) }
|
||||
/END\(exit/ {
|
||||
line=sprintf("%-22s in_tps=%-10s out_tps=%-10s tpot_ms=%-8s ttft_ms=%-8s", name, itps, tps, tpot, ttft)
|
||||
if (tps != "" && tpot != "" && tpot+0 > 0)
|
||||
line=line sprintf(" eff_concurrency=%.1f", tps * tpot / 1000)
|
||||
print line
|
||||
if (acc != "") print " " acc
|
||||
}
|
||||
' "$log"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
warmup) shift; cmd_warmup "$@" ;;
|
||||
prefill) shift; cmd_prefill "$@" ;;
|
||||
decode) shift; cmd_decode "$@" ;;
|
||||
l3) shift; cmd_l3 "$@" ;;
|
||||
report) shift; cmd_report "$@" ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
@@ -1,61 +0,0 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
run() {
|
||||
local name="$1"; shift
|
||||
echo "===== [$(date +%H:%M:%S)] START: $name ====="
|
||||
python3 -m sglang.bench_serving "$@" 2>&1 | grep -vE '^Namespace|warn'
|
||||
echo "===== [$(date +%H:%M:%S)] END(exit=$?): $name ====="
|
||||
sleep 3
|
||||
}
|
||||
|
||||
r160() { # $1=name $2=seed $3=num_prompts $4=mc
|
||||
run "$1" --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts "$3" --max-concurrency "$4" --seed "$2" --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
}
|
||||
|
||||
# WARMUP
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
|
||||
# R0 纯计算极限(集合 A 首次出现,L1/L2/L3 全空)
|
||||
r160 R0_COMPUTE_COLD 1541391 8 8
|
||||
# R1 L1 极限(A 全量在 L1 设备池:1.28M < 1.6M)
|
||||
r160 R1_L1_HIT 1541391 8 8
|
||||
# R2 填充集 B(12×160k=1.92M > L1 1.6M),把 A 挤出 L1 到 L2
|
||||
r160 R2_FILLER_EVICT 999000 12 8
|
||||
# R3 L2 极限(A 已被挤出 L1,从 L2 host 池 load-back)
|
||||
r160 R3_L2_HIT 1541391 8 8
|
||||
# R4 flush L1+L2(L3 保留),走 L3 prefetch
|
||||
echo "===== [$(date +%H:%M:%S)] FLUSH L1+L2 ====="
|
||||
curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache && echo "flush ok"
|
||||
sleep 5
|
||||
r160 R4_L3_HIT 1541391 8 8
|
||||
# R5 嫁接后 L2 复核
|
||||
r160 R5_L2_AFTER_GRAFT 1541391 8 8
|
||||
|
||||
# R6 混合负载:新集合 C(纯计算,4×160k mc4)+ 集合 A(命中,8×160k mc8)并发打
|
||||
echo "===== [$(date +%H:%M:%S)] START: R6_MIX_PARALLEL ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 4 --max-concurrency 4 --seed 555000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r6_coldclient.log 2>&1 &
|
||||
COLD_PID=$!
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 1541391 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r6_hitclient.log 2>&1 &
|
||||
HIT_PID=$!
|
||||
wait $COLD_PID $HIT_PID
|
||||
echo "===== [$(date +%H:%M:%S)] END: R6_MIX_PARALLEL ====="
|
||||
echo "--- R6 cold client (compute) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r6_coldclient.log
|
||||
echo "--- R6 hit client (cache) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r6_hitclient.log
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -1,27 +0,0 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
echo "===== [$(date +%H:%M:%S)] FLUSH L1+L2(A 只剩 L3) ====="
|
||||
curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache && echo "flush ok"
|
||||
sleep 5
|
||||
|
||||
echo "===== [$(date +%H:%M:%S)] START: R7_MIX_L3_PLUS_COMPUTE ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 4 --max-concurrency 4 --seed 666000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r7_coldclient.log 2>&1 &
|
||||
COLD_PID=$!
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 1541391 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r7_hitclient.log 2>&1 &
|
||||
HIT_PID=$!
|
||||
wait $COLD_PID $HIT_PID
|
||||
echo "===== [$(date +%H:%M:%S)] END: R7 ====="
|
||||
echo "--- R7 cold client (compute, 4x160k mc4) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r7_coldclient.log
|
||||
echo "--- R7 hit client (L3, 8x160k mc8) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r7_hitclient.log
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -1,172 +0,0 @@
|
||||
# 3x cp2-P + 1x dp2-D PD 分离 + L3 mooncake 互联缓存 + engram host table
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-d
|
||||
environment:
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -1,225 +0,0 @@
|
||||
# 【L3 dummy client 变体 2026-10-09 深夜,定型=shm+best_effort】sglang→standalone_storage,数据面经本机 mooncake_client(50052 IPC+进程映射),不碰 TCP 传输引擎
|
||||
# 实测 24x160k 热/冷:shm+best_effort +26.4%(唯一正收益);wait_complete 因 all-or-nothing discard 等待白等反而 -17.7%。TCP 传输引擎两组皆负。
|
||||
# 【L3 cache-hit 测试变体 2026-10-09】基于 dockerserve-3p1d-otlp.yml,仅改 P:
|
||||
# --max-total-tokens 800000(L1 缩到 ~0.8M token)+ --hicache-ratio 1.0(L2=L1=0.8M token。⚠️ 教训:ratio 0.05 时 L2 太小,build_storage_backup_spec 拿不到 host 页会静默跳过 L3 写入,热轮无数据可中;V4 hicache 不支持 --hicache-size)
|
||||
# 使 ~2.6M token/P 的语料(48×160k)溢出 L1+L2(1.6M),热轮强制走 L3 mooncake 命中。
|
||||
# 【2026-10-10 L2 放大】hicache-ratio 1.0→4.0(L2=3.2M token/P,宿主可用内存 ~1.9TB 充裕):
|
||||
# #38348 fail-soft 移植后 bench 显示收益被容量掩盖——24x160k 下 host_capacity 拒绝
|
||||
# 0.75M~2.97M token/P(L2 装不下,prefetch 未发起即被拒退回重算)。放大后验证:
|
||||
# ①热轮请求 L3/L2 命中不被容量拒绝;②L3 hit 嫁接进 L2 树后在 L2 长期留存。
|
||||
# 与主模板同容器名/端口,用前必须先 down 主栈。D 与 router 不变。
|
||||
# 原注释: 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-baba18bc7d-local-20261010-2310
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
RUSTUP_TOOLCHAIN: "1.92-x86_64-unknown-linux-gnu"
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
# G9 P0 验证(2026-10-11):D 单独升镜像拿 /query_cache_hit 修复(71c6d8bc5),
|
||||
# 并开 decode radix + hicache L3 客户端(复用 dummy store,只读查询)。
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
RUSTUP_TOOLCHAIN: "1.92-x86_64-unknown-linux-gnu"
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--disaggregation-decode-enable-radix-cache
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -1,239 +0,0 @@
|
||||
# 【G9 P0 验证栈:真实 store 3p1d 2026-10-11】基于 dockerserve-3p1d-otlp.yml。
|
||||
# 用途:验证 P0 改动(/v1/loads 排队 token + drain rate 字段、P 侧 /query_cache_hit)。
|
||||
# 与主模板同容器名/端口,用前必须先 down 主栈;router(ds41-router-3p1d)独立不在本文件。
|
||||
# 2026-10-11 改造要点(vs 原 l3test):
|
||||
# - 镜像升 6612af9e0(含 G9 P0 + /query_cache_hit KeyError 修复)+ scheduler.py
|
||||
# 热补丁(L3 查询门限从 enable_decode_hicache 放宽到 enable_hicache_storage,
|
||||
# 否则 P 节点永远查不到 L3)。
|
||||
# - 每实例独立 MOONCAKE_LOCAL_HOSTNAME(a/b/c/d),extra_hosts 互配全部 rank 别名
|
||||
# (真实 store 跨实例副本写要求对端 rank 主机名可解析,见 realstore-x2 注释;
|
||||
# rank 命名规则 {hostname}-d{dp}t{tp}p{pp}c{attn_cp},commit 84fc632088;
|
||||
# D 为 dp2 dp-attention(attn_tp=1),后缀推断为 -d0t0p0c0/-d1t1p0c0,未实证)。
|
||||
# - ⚠️ D 侧 cache 方案实证否决(2026-10-11):SWA 模型下
|
||||
# --disaggregation-decode-enable-radix-cache 先要 SGLANG_ENABLE_UNIFIED_RADIX_TREE=1,
|
||||
# 开了又报 "supports only device-resident cache and is incompatible with
|
||||
# --enable-hierarchical-cache"(kv_cache_builder.py 硬 ValueError)——D 挂不了 L3。
|
||||
# 结论:/query_cache_hit 放 P(L3 全局统一,P 有完整 hicache),D 保持生产配置。
|
||||
# - D mem-fraction 0.80:曾因崩溃 d1 泄漏 ~61GiB 临时降 0.75,全栈 down 掉旧容器后
|
||||
# 驱动即回收(GPU 全零),已恢复 0.80。D max-running 保持 256(512 时 SWA 池挤爆 KV 池)。
|
||||
# P 侧保持 l3test 原状:--max-total-tokens 800000 + ratio 1.0(L1/L2 各 ~0.8M token),
|
||||
# write_through 保证 prefill 后 L3 有数据,供 /query_cache_hit 对拍。
|
||||
# 原注释: 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 依赖: mooncake-master 容器(50051,起栈前 restart 清旧 segment 注册);
|
||||
# b300-01 k8s cw jaeger-trace(4317,导出失败只丢 span 不影响服务)
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-e7dc175e9-local-20261011-0940
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-rank-hosts: &rank-hosts
|
||||
# mooncake rank 主机名别名:P(tp2/cp2 interleave)-d0t0p0c0/-d0t1p0c1;
|
||||
# D(dp2 dp-attention)-d0t0p0c0/-d1t1p0c0(推断,起栈后核对 master 注册名)
|
||||
- "localhost-a-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-a-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-b-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-b-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-c-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-c-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-d-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d-d1t1p0c0:127.0.0.1"
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-a
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-b
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-c
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
extra_hosts: *rank-hosts
|
||||
# ⚠️ D 保持生产配置(2026-10-11 实证):SWA 模型下
|
||||
# --disaggregation-decode-enable-radix-cache 与 --enable-hierarchical-cache 互斥
|
||||
# (kv_cache_builder.py 硬 ValueError,"supports only device-resident cache"),
|
||||
# D 无法挂 L3 —— /query_cache_hit 改由 P 节点回答(L3 全局统一,P 有完整 hicache)。
|
||||
# MOONCAKE_LOCAL_HOSTNAME 保留备用(无害)。
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-d
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -1,196 +0,0 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -1,196 +0,0 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -1,174 +0,0 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram(2026-09-25,mr256,nsys on p1/d1)
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
nsys launch --session-new=p1prof --cuda-graph-trace=node sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
nsys launch --session-new=d1prof --cuda-graph-trace=node sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -1,174 +0,0 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram(2026-09-25,mr256,nsys on p1/d1)
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -1,43 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.60
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,43 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.60
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
|
||||
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,42 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,160 +0,0 @@
|
||||
# 4×cp2 独立实例(非 PD)+ hicache L3(mooncake) + dspark + engram host table 卸载(2026-09-23 晚)
|
||||
# 用户指定组合:4 cp2 hicache l3 dspark roundrobin loadbalance engram offload
|
||||
# 每实例:tp2 ep2 + interleave CP2 + dspark b5 + hicache L3 write_through(size 0) + engram host table
|
||||
# (tp2 权重必须靠 engram 卸载才放得下,见 goals-track G1.9 反转)
|
||||
# GPU: a=0,1 b=2,3 c=4,5 d=6,7;端口: 30000/30010/30020/30030;rr router=30002
|
||||
# 参考:dockerserve-cpdspark-l3.yml(hicache 参数)、dockerserve-3cp2-pd.yml(cp2/engram 参数)
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
# 热补丁(2026-09-23):CP 对称 idle 检查,修 health-check × hicache drain 死锁
|
||||
# 源文件:/data/ymk/sglang(dsv41-pd-visioncp 分支工作区已改);补丁脚本见
|
||||
# D:\B300\experiments\cp2x4\patch_healthcheck_cp_idle.py
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
environment: &env
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_MS_AUTO_DISC: "0"
|
||||
MOONCAKE_MASTER: 127.0.0.1:50051
|
||||
MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE
|
||||
MOONCAKE_PROTOCOL: tcp
|
||||
MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
a:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-a
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
b:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-b
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
c:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-c
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
d:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-d2
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30030
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -1,44 +0,0 @@
|
||||
# cp8 单机非 PD 测试(2026-09-23 晚)
|
||||
# 背景:dp8 单机 dspark 健康但长上下文 prefill 结构性慢(单请求只落 1 rank),
|
||||
# 换 cp8(tp8 + interleave prefill CP8,单请求 prefill 切到 8 卡)对照。
|
||||
# 关 engram、无 hicache L3、无 PD。cp+dspark 无 replay 是已知健康组合
|
||||
# (cp4+dspark+replay 三元组必崩,本配置不开 replay)。
|
||||
# GPU 0-7,端口 30000,容器 ds41-cp8,project cp8
|
||||
services:
|
||||
cp8:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-cp8
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
SGLANG_RAGGED_VERIFY_MODE: "static"
|
||||
CUDA_VISIBLE_DEVICES: "0,1,2,3,4,5,6,7"
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 8 --ep-size 8
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -1,50 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-b
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,51 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-b
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,50 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-a
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,51 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-a
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,44 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-92632a60b-20260922-2100
|
||||
container_name: ds41-cpdspark-nr-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30001"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,44 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
container_name: ds41-cpdspark-nr
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,44 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-cpdspark
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,49 +0,0 @@
|
||||
# 单独 D 节点基线(非 PD role):3p1d 的 d1 配置去掉 disaggregation,直接对外服务
|
||||
# 用途:测单 D 节点 decode 基线(阶梯并发 32..80%*cuda-graph-max-bs-decode,
|
||||
# 报 TPOT / 实际并发 / 整实例输出 TPS;prefill 走 L1 radix cache-hit)
|
||||
# GPU 6,7;端口 30001;无 hicache 依赖(L1 即可支撑重复 prompt 命中)
|
||||
# 2026-10-09 建:配合 bench_suite_v2.sh decode
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
d1s:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-d1-standalone
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -1,62 +0,0 @@
|
||||
# dp2 对照(2026-09-24 上午):验证 dp2(dp-attention)下 dspark 是否正常。
|
||||
# 与 dockerserve-tp2.yml 逐参数对齐,唯一差异 = 加 --dp-size 2 --enable-dp-attention --enable-dp-lm-head。
|
||||
# GPU 6-7,端口 30030 直连。镜像/补丁挂载/L3/engram/tok8 全部相同。
|
||||
# 注:dp+dspark 不能开 --enable-decoder-swa-bounded-replay(本配置本来就没开)。
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
environment: &env
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_MS_AUTO_DISC: "0"
|
||||
MOONCAKE_MASTER: 127.0.0.1:50051
|
||||
MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE
|
||||
MOONCAKE_PROTOCOL: tcp
|
||||
MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
dp2:
|
||||
<<: *common
|
||||
container_name: ds41-dp2
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--dp-size 2 --enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30030
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -1,42 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-dp4test
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,44 +0,0 @@
|
||||
# dp8+tp8 单机非 PD 对照测试(2026-09-23 晚)
|
||||
# 背景:3cp2+dp2 PD 部署 dspark accept 回归(32k ctx accept len 3.14→1.31,
|
||||
# 见 experiments/3cp2-pd/bench-20260923.md「未决问题」)。本配置退回单机验证
|
||||
# dspark 健康度:engram host table 关闭、无 hicache L3、无 PD、无 mooncake。
|
||||
# 参数基准:dockerserve-3cp2-pd.yml 的 d 服务(dp 路径),去掉 PD/L3/engram 相关。
|
||||
# GPU 0-7,端口 30000,容器 ds41-dp8,project dp8
|
||||
services:
|
||||
dp8:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-dp8
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
SGLANG_RAGGED_VERIFY_MODE: "static"
|
||||
CUDA_VISIBLE_DEVICES: "0,1,2,3,4,5,6,7"
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 8 --ep-size 8 --dp-size 8
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -1,61 +0,0 @@
|
||||
# 单独 P 节点基线(非 PD role):3p1d 的 p1 配置去掉 disaggregation,直接对外服务
|
||||
# 用途:测单 P 节点 prefill 基线吞吐(冷)+ hicache cache-hit TPS(热)
|
||||
# GPU 0,1;端口 30000;依赖 mooncake-master(hicache L3,127.0.0.1:50051)
|
||||
# 2026-10-09 建:配合 bench_suite_v2.sh prefill
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
p1s:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-p1-standalone
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -1,47 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--disaggregation-mode prefill
|
||||
--optimistic-prefill-attempts 4
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,51 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd-decode
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30001
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -1,50 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-pd-decode
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30001
|
||||
--enable-cache-report
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,55 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-c74a4037f-20260921-1300
|
||||
container_name: ds41-pd-prefill-dp
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--load-balance-method total_tokens
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30010
|
||||
--enable-cache-report
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-bootstrap-port 8918
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,53 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -1,52 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-pd-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,57 +0,0 @@
|
||||
# PD2 二分 Round 2 D 侧(2026-09-24):干净 D + 嫌疑项②
|
||||
# SGLANG_DISAGGREGATION_QUEUE_SIZE=16 + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-decode
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=6,7
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
- SGLANG_DISAGGREGATION_QUEUE_SIZE=16
|
||||
- SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--dp-size 2
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30030
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user