Compare commits
587
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
310ad35430 | ||
|
|
bdbd8dffb8 | ||
|
|
f363a6ab7b | ||
|
|
e4c7e540be | ||
|
|
b8b205b367 | ||
|
|
b3d6d19c7d | ||
|
|
8f53c1fb73 | ||
|
|
42f228bfb8 | ||
|
|
242a7ad23a | ||
|
|
2eb163ad73 | ||
|
|
d3240c745e | ||
|
|
433257c7b0 | ||
|
|
7980a09c0a | ||
|
|
a7c668a936 | ||
|
|
0c6849e378 | ||
|
|
85605b8c72 | ||
|
|
b6a5d70cba | ||
|
|
cac60f7c2e | ||
|
|
68c2af7957 | ||
|
|
497aeff5f6 | ||
|
|
5fa8c23227 | ||
|
|
f4de9f2141 | ||
|
|
7f42a0806a | ||
|
|
17b922c84f | ||
|
|
5ad9ff6a5f | ||
|
|
fec10023e0 | ||
|
|
9f9dce1ff6 | ||
|
|
c401c614fa | ||
|
|
24b284d4e7 | ||
|
|
fef023b64f | ||
|
|
15218b2f65 | ||
|
|
883f7d08e1 | ||
|
|
e40c6dfe83 | ||
|
|
32e5e0f30f | ||
|
|
a5fbe834e8 | ||
|
|
d6f1e1efbd | ||
|
|
520ca548ea | ||
|
|
5f7be7fab4 | ||
|
|
97c087778b | ||
|
|
53267ee18f | ||
|
|
aa53670fcd | ||
|
|
7b3f34cbc0 | ||
|
|
3b0baa3455 | ||
|
|
fe9c6574fe | ||
|
|
72e556ec81 | ||
|
|
23053730fc | ||
|
|
bf8cf7f1dc | ||
|
|
c911d1b572 | ||
|
|
145e05c44d | ||
|
|
5d5870f3ea | ||
|
|
309af6e6e1 | ||
|
|
1a274df27e | ||
|
|
cb7988f849 | ||
|
|
0866012b80 | ||
|
|
85f4f9576c | ||
|
|
2434d09848 | ||
|
|
d47a99b2ad | ||
|
|
2ba9b8431a | ||
|
|
ee9170e7bb | ||
|
|
17505e6cf6 | ||
|
|
81f7398855 | ||
|
|
0a56a5f6e7 | ||
|
|
7f6fe22445 | ||
|
|
f6fc6ac3e7 | ||
|
|
0a3c370451 | ||
|
|
b1b303c9f2 | ||
|
|
7d996300b9 | ||
|
|
54df642fe6 | ||
|
|
564aec61c8 | ||
|
|
a7183a8246 | ||
|
|
af3f9e4c8f | ||
|
|
f09cec65e9 | ||
|
|
f4ecaae888 | ||
|
|
cdab6c1bcc | ||
|
|
f2c634961b | ||
|
|
3b95120f00 | ||
|
|
55f23a8b59 | ||
|
|
3c6a6c8068 | ||
|
|
17d92ea671 | ||
|
|
7cefa60765 | ||
|
|
25ba24e9a7 | ||
|
|
42ac81c655 | ||
|
|
de7055f596 | ||
|
|
88ac7c02a6 | ||
|
|
a77998e157 | ||
|
|
79701a2549 | ||
|
|
327c906764 | ||
|
|
5bef70c0a2 | ||
|
|
1d4dcc81cc | ||
|
|
47c0791ad1 | ||
|
|
bbc77e7db4 | ||
|
|
4e2d5b17ad | ||
|
|
be8600d02d | ||
|
|
a08bb0ec9e | ||
|
|
e66c9e981d | ||
|
|
2e6d87e2ca | ||
|
|
386fa9ecfd | ||
|
|
5b8955e70f | ||
|
|
738e4535ea | ||
|
|
3712b5944d | ||
|
|
2d5ad3243f | ||
|
|
e22a530859 | ||
|
|
49a2fbf817 | ||
|
|
ee2dd727a3 | ||
|
|
79e5c4bb9b | ||
|
|
5994c5597d | ||
|
|
6a319e7b34 | ||
|
|
0d592c475b | ||
|
|
855c092deb | ||
|
|
197be2203a | ||
|
|
f05ad5ec01 | ||
|
|
15dc838f9b | ||
|
|
ea226cf782 | ||
|
|
5f7542d762 | ||
|
|
27ac614f05 | ||
|
|
4542fc8f59 | ||
|
|
194abb8e54 | ||
|
|
1d605b7ece | ||
|
|
005333d029 | ||
|
|
d0022e1df3 | ||
|
|
ac3db731b3 | ||
|
|
8b9b992e41 | ||
|
|
0931dfedc9 | ||
|
|
e6451c1a95 | ||
|
|
1340044e97 | ||
|
|
9d7ad308bd | ||
|
|
baf80c61b7 | ||
|
|
1e894eb92d | ||
|
|
16aa816060 | ||
|
|
74c08f3dc5 | ||
|
|
30dc2f9430 | ||
|
|
5a86d6ee17 | ||
|
|
eb8ef80842 | ||
|
|
efd2ce8a9d | ||
|
|
c677171c1d | ||
|
|
02b983603d | ||
|
|
0e34962884 | ||
|
|
82ff16dad0 | ||
|
|
3e5f5505ed | ||
|
|
2fbd094a8c | ||
|
|
9b46f27090 | ||
|
|
2a46e32fbc | ||
|
|
86933b1f33 | ||
|
|
0347b2bb4e | ||
|
|
4b1941e889 | ||
|
|
f967418a35 | ||
|
|
535aeae2a7 | ||
|
|
69f830756e | ||
|
|
611a703061 | ||
|
|
494eeecb6c | ||
|
|
f9129915a3 | ||
|
|
8b9f72417f | ||
|
|
bb72b47daf | ||
|
|
77bc8cab18 | ||
|
|
7d1a16d3eb | ||
|
|
2b8e4cde95 | ||
|
|
4b273537de | ||
|
|
02d2445138 | ||
|
|
472208fec0 | ||
|
|
6d6c9cd4ae | ||
|
|
6f466f655a | ||
|
|
8708d2ee99 | ||
|
|
4f47c48b33 | ||
|
|
1cd0ebb3d3 | ||
|
|
4da6f585f3 | ||
|
|
79ab8f6d7c | ||
|
|
6e667c4310 | ||
|
|
6c844eb12e | ||
|
|
6dd0dfabf0 | ||
|
|
2be6577ba9 | ||
|
|
76c382e80f | ||
|
|
904691e0b5 | ||
|
|
3f36fc0833 | ||
|
|
0e338777b0 | ||
|
|
7c680a0d45 | ||
|
|
7baacd1d1b | ||
|
|
c7994e7076 | ||
|
|
5bdbec0d48 | ||
|
|
e83f585ae6 | ||
|
|
49933325cc | ||
|
|
f4a2df3dac | ||
|
|
fa628c5359 | ||
|
|
79eb10e182 | ||
|
|
f2873f1613 | ||
|
|
698651acd0 | ||
|
|
ce2261863f | ||
|
|
a4838160b3 | ||
|
|
c4fa35b829 | ||
|
|
2da6c0c237 | ||
|
|
363e6c6b4d | ||
|
|
5daa1352ba | ||
|
|
e5dc0c363a | ||
|
|
c82f7141ad | ||
|
|
94e4611eac | ||
|
|
92e57d5de7 | ||
|
|
b588aa39e9 | ||
|
|
f36ee5de93 | ||
|
|
26396bf566 | ||
|
|
a95ae5f1c5 | ||
|
|
97b79f0796 | ||
|
|
9c7dd3c45f | ||
|
|
ae4258715c | ||
|
|
77f90ade97 | ||
|
|
bb9e9d5632 | ||
|
|
5849ff5337 | ||
|
|
8d47d515e4 | ||
|
|
035ed73b9d | ||
|
|
c394a90c2b | ||
|
|
f9b7225f2e | ||
|
|
9c0f6d6a61 | ||
|
|
818d7932d7 | ||
|
|
3b0003dac1 | ||
|
|
a904e4d31a | ||
|
|
840a6313c9 | ||
|
|
25ad569ff9 | ||
|
|
e5db41b329 | ||
|
|
786cdfc040 | ||
|
|
13999673ef | ||
|
|
154e29c1ec | ||
|
|
ba83415eea | ||
|
|
bf0e193bfd | ||
|
|
aae2145eb2 | ||
|
|
357323e912 | ||
|
|
4f96807ea8 | ||
|
|
d67d1ae02c | ||
|
|
0d6b283d7e | ||
|
|
8d8916815b | ||
|
|
913e91ed51 | ||
|
|
c80285ca9a | ||
|
|
cc811ff0c3 | ||
|
|
8cd3c494a3 | ||
|
|
afd0d7b47b | ||
|
|
0f7d3a0451 | ||
|
|
dcf05724f7 | ||
|
|
698fa949ec | ||
|
|
635836129b | ||
|
|
a9902f47e3 | ||
|
|
851419717a | ||
|
|
91fc46a017 | ||
|
|
0ec3fee429 | ||
|
|
e43f6128a0 | ||
|
|
f4ccfd4183 | ||
|
|
178febef5d | ||
|
|
81841d6bdc | ||
|
|
8a6770d566 | ||
|
|
96f013b452 | ||
|
|
d0a7565436 | ||
|
|
3f6c313389 | ||
|
|
5c2cb60790 | ||
|
|
0a87f163dc | ||
|
|
1c967f5043 | ||
|
|
b5137693a5 | ||
|
|
24e45961c6 | ||
|
|
c64886f9e3 | ||
|
|
f727076ed0 | ||
|
|
9d5386fd13 | ||
|
|
33a0734ca1 | ||
|
|
569dda1e61 | ||
|
|
779adb4b82 | ||
|
|
f68e82e7bc | ||
|
|
82f7249ce6 | ||
|
|
a56024c267 | ||
|
|
9569a36561 | ||
|
|
ce66b3ab59 | ||
|
|
12370da115 | ||
|
|
a6a6beffac | ||
|
|
d9ce511646 | ||
|
|
3472e16f96 | ||
|
|
80263b5580 | ||
|
|
f91b81b82a | ||
|
|
8d7df07a56 | ||
|
|
17babd67b4 | ||
|
|
45de4adde9 | ||
|
|
9ee9858805 | ||
|
|
f1e26dc658 | ||
|
|
1962cbeebe | ||
|
|
5f62e23d30 | ||
|
|
6929afb528 | ||
|
|
8dc7f98005 | ||
|
|
3468a65b51 | ||
|
|
780a5b1acc | ||
|
|
b057d5b3f1 | ||
|
|
b453996b7d | ||
|
|
3d508d614f | ||
|
|
67bbfcee3c | ||
|
|
d61f618276 | ||
|
|
4049941a82 | ||
|
|
aa10927271 | ||
|
|
59149553b9 | ||
|
|
9ec74a53cc | ||
|
|
9301ca37a0 | ||
|
|
5fff58a169 | ||
|
|
eacd0a07d7 | ||
|
|
ed89693c4a | ||
|
|
da7b97a737 | ||
|
|
3438ed660d | ||
|
|
ed1f56fead | ||
|
|
f4035cab0b | ||
|
|
42db2db4a8 | ||
|
|
991b846227 | ||
|
|
4b9936b5e3 | ||
|
|
99cf403c9e | ||
|
|
aa5880e2b8 | ||
|
|
51baa1f6e9 | ||
|
|
f3f0f7b68c | ||
|
|
f5eca05b05 | ||
|
|
29e78045fb | ||
|
|
68623cae92 | ||
|
|
f7d4f06242 | ||
|
|
bc412213c1 | ||
|
|
620718c8ea | ||
|
|
97945831f9 | ||
|
|
2c7a9e3b50 | ||
|
|
028de2c5d4 | ||
|
|
a2e4ffbcea | ||
|
|
7e0e259435 | ||
|
|
a3d00eff58 | ||
|
|
7e5256d1ba | ||
|
|
f06d1bb1f4 | ||
|
|
740d5f9f9e | ||
|
|
5e79333c95 | ||
|
|
b57b5e2286 | ||
|
|
ed268dd320 | ||
|
|
6e98c84c60 | ||
|
|
4b1a44bc77 | ||
|
|
75878f27c8 | ||
|
|
cdbd478422 | ||
|
|
af49ada16d | ||
|
|
7163e97462 | ||
|
|
8e3e2980db | ||
|
|
7e5d3780ef | ||
|
|
dd9ea06c00 | ||
|
|
e109bc1832 | ||
|
|
2815eed49c | ||
|
|
fda84f80b7 | ||
|
|
a175e39b84 | ||
|
|
62b4127668 | ||
|
|
43a3f956cb | ||
|
|
43728298fe | ||
|
|
ca72dad953 | ||
|
|
5bf10d4c79 | ||
|
|
2e7c9d851c | ||
|
|
9c10e0dd61 | ||
|
|
7554be5a69 | ||
|
|
131c42ba33 | ||
|
|
be65391abd | ||
|
|
1278087336 | ||
|
|
2c9d919071 | ||
|
|
21f775cd97 | ||
|
|
7a632cf998 | ||
|
|
8bd6aa9a6c | ||
|
|
dbe0a9a7c9 | ||
|
|
6efdd33d07 | ||
|
|
6cac4dc3fe | ||
|
|
fee32a5c1f | ||
|
|
a361f0c410 | ||
|
|
f7b8c6fe40 | ||
|
|
8acfabbd67 | ||
|
|
cd61db49c7 | ||
|
|
132836b369 | ||
|
|
4449d7ecb0 | ||
|
|
844d2b5dec | ||
|
|
8599b7251f | ||
|
|
fb6e6a09ce | ||
|
|
c3b2b55bbb | ||
|
|
da7e3f1c12 | ||
|
|
bd5a445bf7 | ||
|
|
574f9cdb2e | ||
|
|
956b9a24ee | ||
|
|
1689cb2d52 | ||
|
|
4b5c9c4b5e | ||
|
|
32c26788f3 | ||
|
|
573b5716f5 | ||
|
|
1c2f42470c | ||
|
|
6bd1c76735 | ||
|
|
aba892fbdf | ||
|
|
eb91a41f38 | ||
|
|
f1ba723c64 | ||
|
|
93088901fb | ||
|
|
c0d18c0254 | ||
|
|
c1c287a429 | ||
|
|
b326e8349d | ||
|
|
5b1dbc2c80 | ||
|
|
9686edcf80 | ||
|
|
027208ae02 | ||
|
|
299a75ac5d | ||
|
|
533020b500 | ||
|
|
4f8f06faf3 | ||
|
|
7b1d5cd02d | ||
|
|
bf145750a6 | ||
|
|
d5e83ea4f6 | ||
|
|
c1e43b95f4 | ||
|
|
f1eec1232d | ||
|
|
e2e1a97fc3 | ||
|
|
b008ec3de6 | ||
|
|
1dd0f361b8 | ||
|
|
583cfc4d55 | ||
|
|
f0b6590722 | ||
|
|
7d9bc10493 | ||
|
|
cc7ae372ac | ||
|
|
0b536ca112 | ||
|
|
468e26f253 | ||
|
|
bab4a808d8 | ||
|
|
0d7638b035 | ||
|
|
dc62c3effd | ||
|
|
194876bea0 | ||
|
|
afe3e1df31 | ||
|
|
d69966f9c0 | ||
|
|
cfec1b91bb | ||
|
|
29274b41c6 | ||
|
|
82ac83a8d0 | ||
|
|
f82465718b | ||
|
|
c2e5742464 | ||
|
|
89860fbd2e | ||
|
|
7591118794 | ||
|
|
eb4b167d27 | ||
|
|
7656071309 | ||
|
|
e262b44eef | ||
|
|
952c63b1bb | ||
|
|
10d5293546 | ||
|
|
3bf4cbc6cf | ||
|
|
aa91108f64 | ||
|
|
ee2f95a522 | ||
|
|
df13d5dce3 | ||
|
|
eae134d61f | ||
|
|
07eb671b6a | ||
|
|
ca473ef3e4 | ||
|
|
800e60e345 | ||
|
|
306ccf6f57 | ||
|
|
9db58ab547 | ||
|
|
a70b1b73d3 | ||
|
|
16eb87c06f | ||
|
|
2229345dff | ||
|
|
f8fd41c36e | ||
|
|
06314dc2e8 | ||
|
|
0788f1fdd0 | ||
|
|
fdd89d182a | ||
|
|
78185f4b3f | ||
|
|
1739d962a4 | ||
|
|
15e8e687db | ||
|
|
9e23107772 | ||
|
|
1ec902e00f | ||
|
|
407bfc94f9 | ||
|
|
4b83f215f4 | ||
|
|
4a651146de | ||
|
|
36978d6c4d | ||
|
|
50fa3727eb | ||
|
|
472f0d540b | ||
|
|
476fe86a12 | ||
|
|
35c873cff4 | ||
|
|
c7687a0e7c | ||
|
|
602c92ccd8 | ||
|
|
553836c164 | ||
|
|
43a896933e | ||
|
|
d1c3346f74 | ||
|
|
72d886efb8 | ||
|
|
fdd53418a0 | ||
|
|
c53b98d1fe | ||
|
|
5b683d6b72 | ||
|
|
4a0c6f2fe0 | ||
|
|
e3205080ce | ||
|
|
1140a8dfd9 | ||
|
|
3c07ff2654 | ||
|
|
3eb4bd598f | ||
|
|
ec1fdc22a9 | ||
|
|
8683f8ad4a | ||
|
|
5e630fa571 | ||
|
|
af856717c2 | ||
|
|
55b2aef7ad | ||
|
|
2c00b5d129 | ||
|
|
67d9ee156c | ||
|
|
ce329c85fa | ||
|
|
79ed13fc2c | ||
|
|
1897575ce1 | ||
|
|
ebc71e650e | ||
|
|
71ee2dc950 | ||
|
|
693d333306 | ||
|
|
ab580b2e06 | ||
|
|
6b90e5716d | ||
|
|
964ab2b280 | ||
|
|
48ea486928 | ||
|
|
3a059de94c | ||
|
|
28448a86d4 | ||
|
|
1238edd954 | ||
|
|
9a949403f1 | ||
|
|
ab258b6453 | ||
|
|
05d7f7a166 | ||
|
|
49766a74f9 | ||
|
|
80ea5ee638 | ||
|
|
fdc2dd0d05 | ||
|
|
81ac15391c | ||
|
|
7656ea0e74 | ||
|
|
b20cb3665d | ||
|
|
642a671c30 | ||
|
|
8b1d1e0d74 | ||
|
|
aaa9388c91 | ||
|
|
f1a2ee763d | ||
|
|
5385866108 | ||
|
|
9ab74b8517 | ||
|
|
0ce41c5a06 | ||
|
|
722669e606 | ||
|
|
678913a0f0 | ||
|
|
80b86dbecb | ||
|
|
5ed306bd69 | ||
|
|
4ef9765e76 | ||
|
|
716c9151f2 | ||
|
|
8e7601ae4a | ||
|
|
806e7423af | ||
|
|
c04f9a61f0 | ||
|
|
7c4cc5a51c | ||
|
|
96ba108072 | ||
|
|
417606527d | ||
|
|
8c4680cb65 | ||
|
|
62cee1b15c | ||
|
|
ed0b8f433f | ||
|
|
0025f93944 | ||
|
|
e45cbce599 | ||
|
|
a90e9c5dd8 | ||
|
|
8111841103 | ||
|
|
33dd8f6a2e | ||
|
|
5d9c956f8e | ||
|
|
af9bb2a32d | ||
|
|
3d588779e1 | ||
|
|
f86a04a331 | ||
|
|
a5117beb11 | ||
|
|
b6d3b7e163 | ||
|
|
97857f9f96 | ||
|
|
dc81f17b90 | ||
|
|
a3330237db | ||
|
|
04c8c9703f | ||
|
|
7c9a78efbe | ||
|
|
f726461287 | ||
|
|
9a93a17c08 | ||
|
|
2b75048e2a | ||
|
|
58ad0748a6 | ||
|
|
266550048a | ||
|
|
82c9a8b037 | ||
|
|
834bfe4ef0 | ||
|
|
8d3eee46a7 | ||
|
|
06342acad2 | ||
|
|
f02458fff1 | ||
|
|
617c8b67b1 | ||
|
|
7b1710324c | ||
|
|
7f96dc6874 | ||
|
|
642f43ce59 | ||
|
|
a7fd2e1158 | ||
|
|
69076b9f9e | ||
|
|
fea48754de | ||
|
|
377cd867fd | ||
|
|
df078cc729 | ||
|
|
10ec53ac2f | ||
|
|
0dc6de2e74 | ||
|
|
cff88d6eac | ||
|
|
299da60131 | ||
|
|
329072d99a | ||
|
|
59fec011dd | ||
|
|
3c560089e1 | ||
|
|
1637a4e305 | ||
|
|
6708426728 | ||
|
|
3f13f5d966 | ||
|
|
1461152cb0 | ||
|
|
fe89a7c890 | ||
|
|
da414adda7 | ||
|
|
0f4aedfecc | ||
|
|
4ec19ca27f | ||
|
|
bb1d4cddd9 | ||
|
|
d4e3c163c9 | ||
|
|
e9b22a0818 | ||
|
|
57db5089c2 | ||
|
|
2e9b2cacda | ||
|
|
2a83471676 | ||
|
|
d318a546da | ||
|
|
a5b21145e1 | ||
|
|
3edb156dec | ||
|
|
19cb24f1e1 | ||
|
|
1d17066bad | ||
|
|
ba50ea3d77 | ||
|
|
47eeddcd18 | ||
|
|
fe1851c3c2 | ||
|
|
e17fed6ea7 | ||
|
|
680a2ff27e | ||
|
|
1fd46ddd56 | ||
|
|
8dee3f6bdb | ||
|
|
f8e32706fc | ||
|
|
a3abea3560 | ||
|
|
3b932cde26 | ||
|
|
5a930cfd88 |
@@ -742,7 +742,7 @@ cd test && python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1
|
||||
- `docs/docs/developer_guide/development_jit_kernel_guide.mdx`
|
||||
- `test/run_suite.py` — suite names, discovery of `test/registered/`, execution entrypoint for CI
|
||||
- `python/sglang/test/ci/ci_register.py` — `register_cuda_ci` and AST registration rules
|
||||
- `python/sglang/kernels/jit/utils/compile.py` — `load_jit`, `make_cpp_args`
|
||||
- `python/sglang/kernels/jit/utils/compile/` — `load_jit` (`loader.py`), `make_cpp_args` (`cpp_args.py`)
|
||||
- `python/sglang/kernels/jit/utils/common.py` — `cache_once`, `should_run_full_tests`, `get_ci_test_range`
|
||||
- `python/sglang/kernels/jit/include/sgl_kernel/tensor.h` — `TensorMatcher`, `SymbolicSize/DType/Device`, `is_type`
|
||||
- `python/sglang/kernels/jit/include/sgl_kernel/ffi.h` — `ffi::empty`, `ffi::empty_like`, `ffi::from_blob`
|
||||
|
||||
@@ -398,6 +398,7 @@ group: pr-test-{event_name}-{branch}-{pr_sha}-{stage}
|
||||
| `/run-extra-ci` | Adds both labels + reruns **only** `PR Test Extra`, leaving baseline runs alone. Alias: `/rerun-extra-ci` |
|
||||
| `/rerun-test <test-file> [<test-file> ...]` | Reruns specific test file(s) via `rerun-test.yml`. A file arg containing a glob metacharacter (`*`, `?`, `[...]`) expands against `test/registered/` and the multimodal test dir to every matching `test_*.py` (e.g. `/rerun-test test_*backend*.py` — wrap in backticks so GitHub doesn't italicize the `*`); matches are deduped, grouped by dispatch shape, and can't carry a `::test` selector. No match → single ⛔ reply, nothing dispatched. |
|
||||
| `/rerun-group <group> [<group> ...]` | Expands registered test groups, then reuses `/rerun-test` |
|
||||
| `/run-cpu-ci` | Dispatches `run-cpu-ci.yml` on main: only the `base-a-test-cpu` suite, at the PR head SHA, no caches saved. Same gate as `/rerun-test`. Alias: `/rerun-cpu-ci` |
|
||||
|
||||
Handled by `scripts/ci/utils/slash_command_handler.py` → `.github/workflows/slash-command-handler.yml`.
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ These are investigation hints, not a list of changes to apply.
|
||||
| `No platform detected. Using base SRTPlatform with defaults.` | Platform detection | Decide whether fallback is expected on this machine or indicates a missing plugin. |
|
||||
| `Unexpected error during package walk` in CUTE_DSL | Installed cutlass package, its handler, root propagation, and `CUTE_DSL_LOG_LEVEL` | Inspect the actual exception and duplicate-handler path; do not assume the current log level or hide all warnings. |
|
||||
| Repeated dtype, hybrid-model, or tokenizer fallback messages | Model-config and tokenizer construction in launcher, scheduler, and detokenizer | Multiple constructors may explain repetition. Retain meaningful per-process information and distinguish these from handler duplication. |
|
||||
| Repeated template detection messages | `managers/template_detection.py` and `managers/template_manager.py` | Check whether a summary already contains the same information. |
|
||||
| Repeated template detection messages | `parser/template_detection.py` and `parser/template_manager.py` | Check whether a summary already contains the same information. |
|
||||
| KV cache dtype logged separately from allocation | Model runner and memory pool | Check whether the allocation summary already carries dtype; retain useful allocation and SWA details. |
|
||||
| CUTLASS backend disabled during graph capture | Attention backend selection | Expected fallback may be useful information; retain the reason and selected behavior. |
|
||||
| `Ignore import error when loading ...` | Model, multimodal, or algorithm registries | Distinguish an unused optional model dependency from failure to load the requested model. |
|
||||
|
||||
@@ -205,7 +205,7 @@ than restating.
|
||||
`python/sglang/srt/function_call/function_call_parser.py` (ToolCallParserEnum) —
|
||||
prose naming a near-miss id (e.g. the reasoning id where the tool id differs) is a
|
||||
factual bug. `--…-parser auto` is acceptable ONLY if the template-detection rules
|
||||
(`python/sglang/srt/managers/template_detection.py`) actually resolve THIS model's
|
||||
(`python/sglang/srt/parser/template_detection.py`) actually resolve THIS model's
|
||||
chat template to the right parser — no rule match means auto silently disables the
|
||||
parser; when in doubt require explicit ids (the DSv4 page pins explicit ids).
|
||||
|
||||
|
||||
@@ -17,25 +17,47 @@ logical grouping and remove the old `_jit_` filename prefix.
|
||||
The `sgl_kernel` wheel retains its own public API and packaging tests.
|
||||
- Group by computation, not model or GPU: GEMM and GEMV in `gemm`, expert
|
||||
routing in `moe`, attention index selection in `attention`, sampling in
|
||||
`sampling`, normalization in `layernorm`. A quantized GEMM is still a GEMM;
|
||||
quantization alone belongs in `quantization`.
|
||||
`sampling`, normalization in `layernorm`.
|
||||
- Classify a fused operator by its complete contract; do not split a fused
|
||||
kernel into separate launches just to separate its parts. Split unrelated
|
||||
public entry points that happen to share a source file. Decided rules:
|
||||
- Any cross-rank collective or peer-memory access (all-reduce, all-gather,
|
||||
reduce-scatter, a2a, multicast/NVLS) puts the op in `communication`,
|
||||
whatever compute it fuses.
|
||||
- Producer + output quantization stays with the producer (norm + quant in
|
||||
`layernorm`, activation + quant in `activation`, GEMM with a quant epilogue
|
||||
in `gemm`, MoE dispatch/permute + quant in `moe`). `quantization` holds only
|
||||
pure format conversion: quant, dequant, scale-layout conversion, repack.
|
||||
- RoPE stays in `attention` (standalone RoPE, position ids, QK-norm + RoPE).
|
||||
- `kvcache` owns ops that read/write KV-cache contents; `attention` owns the
|
||||
index/shape/schedule tables (page tables, `cu_seqlens`, `kv_indices`, split
|
||||
metadata) and index selection. A fused compute + store stays with its
|
||||
compute group.
|
||||
- Model-specific files/subpackages and tuning data are allowed **inside** a
|
||||
logical group. Do not add a model bundle or an implementation file directly
|
||||
under `ops/`. Propose a new logical group only for a distinct responsibility,
|
||||
and update `ops._GROUPS`, documentation, and tests together.
|
||||
- Classify a fused operator by its complete contract. Do not split a fused
|
||||
kernel into separate launches just to separate norm, RoPE, or quantization.
|
||||
Split unrelated public entry points that happen to share a source file.
|
||||
- Keep shared CUDA build/runtime infrastructure in `kernels/jit`; operator
|
||||
wrappers call it from their logical group. Keep process groups, communicator
|
||||
state, model dispatch, and buffer ownership in `srt`. K3-specific adapters in
|
||||
`srt/layers/communication/` need not pretend to be generic interfaces.
|
||||
and update `ops._GROUPS`, the README group list, and tests together
|
||||
(`test_kernels_namespace.py` checks the README against `_GROUPS` and the
|
||||
README layout against the directories under `kernels/`).
|
||||
- Agent-generated implementations live in `kernels/kda_kernels/` (Kernel Design
|
||||
Agents, `KernelBackend.KDA`) and `kernels/cake_kernels/` (Cake adapters over
|
||||
FlashInfer). They are backends, not groups: register their public ops in the
|
||||
logical group with a `KernelSpec` whose target points into them, and do not
|
||||
reorganize those directories.
|
||||
- Keep shared build/runtime infrastructure (including the Helion runtime
|
||||
helper) in `kernels/jit`; operator wrappers call it from their logical group.
|
||||
Keep process groups, communicator state, workspace classes, model dispatch,
|
||||
and buffer ownership in `srt` (`srt/layers/communication/` for the K3 and Muse
|
||||
Spark adapters); kernels must not import srt runtime modules to inspect
|
||||
model objects, so pass flags in. Adapters there need not pretend to be generic
|
||||
interfaces.
|
||||
|
||||
## Register the public entry point
|
||||
|
||||
Add lazy `KernelSpec` metadata in the owning group's `__init__.py`, or use the
|
||||
existing `BaseFusedOp` registration when the operation has interchangeable
|
||||
backends. Group imports must not import GPU implementations or compile kernels.
|
||||
backends (a `BaseFusedOp` built per layer registers `module:Class.forward_<backend>`
|
||||
targets). Group imports must not import GPU implementations or compile kernels.
|
||||
|
||||
Use `<group>.<name>` for the op id and `module:callable` for the target. Preserve
|
||||
existing backend dispatch and describe actual device/architecture restrictions
|
||||
@@ -53,8 +75,9 @@ factories, reference helpers, and runtime classes are not separate kernel APIs.
|
||||
runtime module. A CUDA allocation does not make a runtime test a kernel test.
|
||||
- Non-CI smoke scripts and benchmarks: `test/manual/kernels/`; do not leave
|
||||
standalone test/benchmark entry points in production operator modules.
|
||||
- Shared test helpers: `sglang.test.kernels`. Preserve vendored upstream trees
|
||||
and AOT wheel packaging boundaries instead of reorganizing them incidentally.
|
||||
- Shared test helpers and reference oracles: `sglang.test.kernels`. Preserve
|
||||
vendored upstream trees and AOT wheel packaging boundaries instead of
|
||||
reorganizing them incidentally.
|
||||
|
||||
For a move/split, preserve assertions, parametrization, fixtures, platform skips,
|
||||
execution entry points, and CI stages/runners. Apportion existing time estimates
|
||||
|
||||
@@ -49,15 +49,15 @@ in-flight row as shipped.
|
||||
| FlashInfer unified `allreduce_fusion` | `cross_device_reduce_1stage*`<br>`all_reduce`<br>`FusedAddRMSNormKernel`<br>`rmsnorm*` | `python/sglang/srt/layers/flashinfer_comm_fusion.py`<br>`python/sglang/srt/layers/layernorm.py::forward_with_allreduce_fusion`<br>`python/sglang/srt/layers/layer_boundary/residual/add_norm.py::apply_flashinfer_allreduce_fusion` | FlashInfer workspace creation plus `allreduce_fusion(..., pattern=AllReduceFusionPattern.kARResidualRMSNorm, ...)` | First suspect missing / disabled / unsupported FlashInfer allreduce fusion, not a brand new TP fusion idea. |
|
||||
| AITER allreduce fusion | ROCm all-reduce plus RMSNorm still split | `python/sglang/srt/layers/layernorm.py::forward_with_allreduce_fusion`<br>`python/sglang/srt/distributed/communication_op.py::tensor_model_parallel_fused_allreduce_rmsnorm`<br>`python/sglang/srt/layers/layer_boundary/residual/add_norm.py::apply_aiter_all_reduce_fusion` | ROCm-side fused TP all-reduce + RMSNorm with fallback to plain all-reduce plus norm | On AMD, rule out existing AITER fusion before proposing a new communication fusion. |
|
||||
| Fused activation-and-mul (`SwiGLU` / `GeGLU`) | `silu_and_mul`<br>`gelu_and_mul`<br>`npu_swiglu` | `python/sglang/srt/layers/activation.py` | Single op covers activation plus elementwise multiply across CUDA / CPU / NPU / XPU backends | Treat separate activation + mul on packed MLP outputs as missing existing fusion. |
|
||||
| Fused dual residual RMSNorm | residual add plus two RMSNorm-like kernels around Grok blocks | `python/sglang/srt/layers/elementwise.py::fused_dual_residual_rmsnorm`<br>`python/sglang/srt/models/grok.py` | One Triton kernel computes intermediate residual update and next RMSNorm output together | On Grok-like residual layouts, treat split residual + norm as missing existing fusion. |
|
||||
| Fused dual residual RMSNorm | residual add plus two RMSNorm-like kernels around Grok blocks | `python/sglang/kernels/ops/layernorm/fused_rmsnorm.py::fused_dual_residual_rmsnorm`<br>`python/sglang/srt/models/grok.py` | One Triton kernel computes intermediate residual update and next RMSNorm output together | On Grok-like residual layouts, treat split residual + norm as missing existing fusion. |
|
||||
| In-place QK RMSNorm | split `q_norm` / `k_norm` kernels | `python/sglang/srt/models/utils.py::apply_qk_norm`<br>`python/sglang/kernels/ops/layernorm/norm.py::fused_inplace_qknorm` | In-place JIT QK norm plus optional `alt_stream` overlap for K | Check shape, dtype, deterministic mode, and in-place legality before proposing a new QK fuse. |
|
||||
| TorchInductor horizontal Q/K norm combo-kernels | `combo_kernels`<br>`benchmark_combo_kernel`<br>`q_norm`<br>`k_norm`<br>`split_with_sizes` | `torch._inductor.config.combo_kernels` | TorchInductor can horizontally fuse sibling Q-norm and K-norm kernels in compiled traces, often deleting `split_with_sizes` / `clone` ladders | Treat separate Q/K norm ladders in compile-heavy traces as an existing compiler-fusion family first. |
|
||||
| MiniMax TP fused QK RMSNorm | `MiniMaxM2RMSNormTP`<br>`rms_sumsq_serial`<br>`rms_apply_serial`<br>`forward_qk` | `python/sglang/srt/models/minimax_m2.py` | Triton kernels compute Q / K sumsq together, TP all-reduces shared stats, then apply both RMSNorms together | On MiniMax traces, separate Q norm and K norm are usually a missed model-specific Triton fusion. |
|
||||
| Fused QK RMSNorm + RoPE | `qknorm*` + `rope*` + `rotary*` as separate steps | `python/sglang/kernels/ops/attention/fused_qknorm_rope.py`<br>`python/sglang/srt/models/qwen3_moe.py` | One JIT kernel applies QK RMSNorm and RoPE in-place on packed QKV | For compatible LLMs, classify split QK norm + RoPE as a missing existing fusion. |
|
||||
| Fused QK RoPE reshape + KV cache write | `fused_qk_rope_reshape_and_cache*`<br>RoPE followed by reshape / cache DtoD | `python/sglang/srt/layers/attention/utils.py::fused_qk_rope_reshape_and_cache` | One Triton kernel applies RoPE to Q / K, reshapes cache layout, and writes K / V directly to paged cache | Treat separate RoPE + reshape + cache-write ladders as an existing attention-prep fusion family. |
|
||||
| Fused QK RoPE reshape + KV cache write | `fused_qk_rope_reshape_and_cache*`<br>RoPE followed by reshape / cache DtoD | `python/sglang/kernels/ops/attention/fused_qk_rope_reshape_and_cache.py::fused_qk_rope_reshape_and_cache` | One Triton kernel applies RoPE to Q / K, reshapes cache layout, and writes K / V directly to paged cache | Treat separate RoPE + reshape + cache-write ladders as an existing attention-prep fusion family. |
|
||||
| Fused RoPE + KV cache store | `fused_set_kv_buffer`<br>RoPE followed by KV-store, DtoD, or cache-write kernels | `python/sglang/kernels/ops/attention/rope.py`<br>`python/sglang/srt/models/utils.py::enable_fused_set_kv_buffer` | Shared entrypoints can route to fused RoPE + KV-store or model-side `fused_set_kv_buffer` fast paths | Compare against the fused cache-store path before proposing a new KV rewrite. |
|
||||
| Fused decode metadata setup | `normal_decode_set_metadata`<br>`cache_seqlens_int32`<br>`cu_seqlens_k`<br>`page_table`<br>`swa_page_table` | `python/sglang/srt/layers/attention/flashattention_backend.py::normal_decode_set_metadata` | Triton decode path fuses seq-len cast/add, prefix-sum, req-to-token gather, page-table divide, and optional SWA metadata build into 1-2 kernels | If decode exposes multiple tiny metadata kernels before attention, first compare against this existing fused metadata-prep path. |
|
||||
| NSA fused metadata copy for graph replay | `fused_metadata_copy`<br>`fused_metadata_copy_multi`<br>`fused_nsa_cache_seqlens`<br>`fused_flashmla_metadata` | `python/sglang/kernels/ops/attention/fused_metadata_copy.py` | CUDA graph replay path fuses multiple metadata copies into one kernel or one multi-destination kernel | Treat bursts of tiny metadata-copy kernels around NSA replay as a missed existing replay fusion. |
|
||||
| NSA fused metadata copy for graph replay | `fused_metadata_copy`<br>`fused_metadata_copy_multi`<br>`fused_nsa_cache_seqlens`<br>`fused_flashmla_metadata` | `python/sglang/kernels/ops/attention/dsa/fused_metadata_copy.py` | CUDA graph replay path fuses multiple metadata copies into one kernel or one multi-destination kernel | Treat bursts of tiny metadata-copy kernels around NSA replay as a missed existing replay fusion. |
|
||||
| DeepSeek MLA fused projection + norm + RoPE | `qkv_proj_with_rope_fused_weight`<br>`fused_qkv_a_proj_with_mqa`<br>`forward_absorb_fused_mla_rope*` | `python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_fused_rope_cpu.py`<br>`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_fused_rope_rocm.py`<br>`python/sglang/srt/models/deepseek_v2.py` | CPU / ROCm paths fuse DeepSeek MLA projection packing with q / k norm, RoPE, and cache-oriented MLA prep | For DeepSeek MLA, split proj / norm / rope prep is usually an existing backend-specific fuse that did not fire. |
|
||||
| Fused QK RoPE concat + MLA cache write | `fused_qk_rope_cat_and_cache_mla`<br>`set_mla_kv_buffer` | `python/sglang/srt/layers/rocm_linear_utils.py`<br>`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py` | ROCm MLA path can fuse Q / K RoPE packing, concat, and MLA cache write in one backend-specific op | On DeepSeek / MLA traces, separate RoPE-cat-cache steps are not automatically novel. |
|
||||
| Qwen3 decode fused QK norm + 3D mRoPE + KV cache write | `fused_qk_norm_mrope_3d_cache_pts_quant_shuffle`<br>`mrope`<br>decode cache write | `python/sglang/srt/models/qwen3.py` | ROCm / AITER decode path fuses QK norm, 3D mRoPE, and paged KV cache write | On Qwen3-style decode, separate norm + mRoPE + cache-store kernels are not a novel opportunity. |
|
||||
@@ -69,18 +69,18 @@ in-flight row as shipped.
|
||||
| Qwen-style shared-expert append into routed top-k output | `_append_shared_to_topk_output`<br>`fused_append_shared_experts_with_weights`<br>`num_fused_shared_experts` | `python/sglang/srt/models/qwen2_moe.py`<br>`python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe_triton_kernels.py` | Qwen-style MoE paths can append shared-expert ids and sigmoid gate weights to routed top-k output in one Triton kernel so the shared experts execute inside the fused MoE path | Treat routed top-k plus shared-expert pad / concat ladders as an existing MoE-prep fusion family first. |
|
||||
| Fused MoE dispatch / permute / combine | token permutation<br>dispatch / combine<br>grouped top-k<br>many small MoE support kernels | `python/sglang/srt/layers/moe/fused_moe_triton/layer.py`<br>`python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py` | `FusedMoE` plus DeepEP / FlashInfer / FuseEP / standard dispatch backends and `permute_fusion=True` | First ask whether the model is missing an existing `FusedMoE`-style path or backend-specific dispatcher path. |
|
||||
| Fused MoE sum + all-reduce | routed MoE followed by explicit sum-reduce kernels | `python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py`<br>`python/sglang/srt/layers/moe/fused_moe_triton/fused_moe_triton_kernels.py` | `fuse_sum_all_reduce=True` path in the second MoE GEMM | Before inventing a new MoE reduction fuse, check whether `enable_fused_moe_sum_all_reduce` is simply off or the quant path is incompatible. |
|
||||
| Fused MoE activation + quant / re-quant | `silu_and_mul_*quant*`<br>`npu_dequant_swiglu_quant`<br>`swiglu_quant` | `python/sglang/srt/layers/moe/ep_moe/kernels.py`<br>`python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py`<br>`python/sglang/srt/layers/moe/cutlass_w4a8_moe.py`<br>`python/sglang/srt/hardware_backend/npu/quantization/moe_methods.py` | Quantized MoE backends fuse SwiGLU / SiLU-and-mul with FP8 / FP4 / NPU re-quant before the second expert GEMM | If MoE traces show standalone activation then quant kernels, first check whether the quantized fused path is missing. |
|
||||
| Fused MoE activation + quant / re-quant | `silu_and_mul_*quant*`<br>`npu_dequant_swiglu_quant`<br>`swiglu_quant` | `python/sglang/srt/layers/moe/ep_moe/kernels.py`<br>`python/sglang/kernels/ops/gemm/nvfp4_gemm_swiglu_nvfp4_quant.py`<br>`python/sglang/srt/layers/moe/cutlass_w4a8_moe.py`<br>`python/sglang/srt/hardware_backend/npu/quantization/moe_methods.py` | Quantized MoE backends fuse SwiGLU / SiLU-and-mul with FP8 / FP4 / NPU re-quant before the second expert GEMM | If MoE traces show standalone activation then quant kernels, first check whether the quantized fused path is missing. |
|
||||
| DeepSeek comm-prep fused RMSNorm + quant / flatten-quant | `fused_rms_fp8_group_quant`<br>`fused_rms_mxfp4_quant`<br>`fused_flatten_fp8_group_quant`<br>`fused_flatten_mxfp4_quant` | `python/sglang/srt/layers/layer_boundary/residual/add_norm.py`<br>`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py`<br>`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py` | DeepSeek MLA / MHA ROCm paths fuse RMSNorm or flatten with FP8 / MXFP4 quantization for comm / attention prep | On DeepSeek quant traces, split norm + quant or flatten + quant is an existing family, not a new idea. |
|
||||
| NSA fused top-k transform / page-table build | `fast_topk_transform_fused`<br>`fast_topk_transform_ragged_fused` | `python/sglang/srt/layers/attention/dsa_backend.py` | NSA can fuse top-k selection with paged / ragged index transform instead of separate top-k plus metadata scatter | If NSA top-k metadata work is split, check `SGLANG_DSA_FUSE_TOPK` and backend support first. |
|
||||
| NSA fused quantize + indexed K-cache store | `fused_store_index_k_cache`<br>`act_quant`<br>`index_k_with_scale_buffer` | `python/sglang/kernels/ops/attention/fused_store_index_cache.py`<br>`python/sglang/srt/layers/attention/nsa/nsa_indexer.py` | Single JIT kernel quantizes bf16 K to fp8 + scale and writes directly into NSA index cache | Treat split `act_quant` + buffer-store on CUDA as missing an existing fused store path. |
|
||||
| NSA fused quantize + indexed K-cache store | `fused_store_index_k_cache`<br>`act_quant`<br>`index_k_with_scale_buffer` | `python/sglang/kernels/ops/kvcache/dsa_fused_store_index_cache.py`<br>`python/sglang/srt/layers/attention/nsa/nsa_indexer.py` | Single JIT kernel quantizes bf16 K to fp8 + scale and writes directly into NSA index cache | Treat split `act_quant` + buffer-store on CUDA as missing an existing fused store path. |
|
||||
| Fused sampling temperature + softmax | `fused_temperature_softmax*` | `python/sglang/srt/layers/fused_sampling.py`<br>`python/sglang/srt/layers/sampler.py` | Triton single-pass / multi-pass kernels fuse temperature scaling and softmax during decode | Separate temp-divide + softmax at decode batch sizes is often a missed existing fusion. |
|
||||
| Fused logit softcap | `fused_softcap`<br>`final_logit_softcapping` | `python/sglang/srt/layers/elementwise.py`<br>`python/sglang/srt/layers/logits_processor.py` | Triton kernels fuse cast-to-float and softcap / tanh math for logits or generic elementwise softcapping | Treat exposed cast + softcap ladders as an existing Triton fuse family. |
|
||||
| Linear-attention packed projection reshuffle | `fused_qkvzba_split_reshape_cat*`<br>`qkvz_proj`<br>`ba_proj`<br>`qkvabz_proj`<br>`fused_qkvbfg_a_proj` | `python/sglang/kernels/ops/attention/triton_gdn_fused_proj.py`<br>`python/sglang/srt/models/qwen3_next.py`<br>`python/sglang/srt/models/qwen3_5.py`<br>`python/sglang/srt/models/kimi_linear.py`<br>`python/sglang/srt/models/jet_nemotron.py` | GDN / Kimi / Jet-style linear-attn models pack multiple projections, then fuse split / reshape / cat into one kernel | Treat split reshape / transpose / cat ladders as an existing linear-attention fusion family. |
|
||||
| Fused GDN gating prep | `fused_gdn_gating`<br>`softplus`<br>`beta_output` | `python/sglang/srt/layers/attention/fla/fused_gdn_gating.py` | Triton kernel computes GDN gate preparation such as `-exp(A_log) * softplus(...)` and `sigmoid(b)` together | On GDN traces, treat split gate-prep elementwise kernels as missing existing fusion first. |
|
||||
| Fused GDN gating prep | `fused_gdn_gating`<br>`softplus`<br>`beta_output` | `python/sglang/kernels/ops/attention/linear/fused_gdn_gating.py` | Triton kernel computes GDN gate preparation such as `-exp(A_log) * softplus(...)` and `sigmoid(b)` together | On GDN traces, treat split gate-prep elementwise kernels as missing existing fusion first. |
|
||||
| Fused RMSNorm-gated linear-attention output | `FusedRMSNormGated`<br>`layer_norm_gated_fwd` | `python/sglang/srt/layers/attention/fla/fused_norm_gate.py`<br>`python/sglang/srt/models/qwen3_next.py`<br>`python/sglang/srt/models/kimi_linear.py` | One Triton op covers residual-aware (RMS)Norm plus sigmoid / swish gating | If norm and output gate appear as separate kernels in GDN / Kimi-like blocks, first suspect a missing existing fusion. |
|
||||
| Fused gated RMSNorm / LayerNorm | `rms_norm_gated`<br>`layer_norm_gated` | `python/sglang/srt/layers/attention/mamba/ops/layernorm_gated.py` | Mamba-derived kernels can fuse normalization with the gating branch `z * sigmoid(z)` | Treat split norm and gate post-processing on Mamba-style blocks as an existing fusion family. |
|
||||
| Fused linear-attention chunk KKT + solve_tril | `chunk_gated_delta_rule_fwd_kkt_solve_kernel`<br>`scaled_dot_kkt`<br>`solve_tril`<br>`recompute_w_u` | `python/sglang/srt/layers/attention/fla/chunk_fwd.py`<br>`python/sglang/srt/layers/attention/fla/kda.py` | GDN / KDA chunk forward fuses `scaled_dot_kkt + solve_tril` in the prefill / intra-chunk path, then finishes `recompute_w_u` as the next step | Treat split KKT + triangular-solve ladders as an existing linear-attention fusion family first. |
|
||||
| Fused linear-attention recurrent / KDA update | `fused_sigmoid_gating_delta_rule_update`<br>`fused_recurrent_gated_delta_rule_update`<br>`fused_kda_gate` | `python/sglang/srt/layers/attention/fla/fused_sigmoid_gating_recurrent.py`<br>`python/sglang/srt/layers/attention/fla/fused_recurrent.py`<br>`python/sglang/srt/models/kimi_linear.py`<br>`python/sglang/srt/models/jet_nemotron.py` | Triton / CuTeDSL kernels fuse gating math, optional QK l2norm, recurrent state update, and output generation | Treat split gating + recurrent-update chains as existing linear-attention fusion, not a novel opportunity. |
|
||||
| Fused linear-attention recurrent / KDA update | `fused_sigmoid_gating_delta_rule_update`<br>`fused_recurrent_gated_delta_rule_update`<br>`fused_kda_gate` | `python/sglang/kernels/ops/attention/linear/fused_sigmoid_gating_recurrent.py`<br>`python/sglang/kernels/ops/attention/fla/fused_recurrent.py`<br>`python/sglang/srt/models/kimi_linear.py`<br>`python/sglang/srt/models/jet_nemotron.py` | Triton / CuTeDSL kernels fuse gating math, optional QK l2norm, recurrent state update, and output generation | Treat split gating + recurrent-update chains as existing linear-attention fusion, not a novel opportunity. |
|
||||
| Fused Mamba state gather/scatter with mask | `fused_mamba_state_scatter_with_mask`<br>`index_elementwise_kernel` | `python/sglang/srt/layers/attention/mamba/mamba_state_scatter_triton.py` | Triton kernel replaces multiple masked gather / scatter index kernels with one fused update | If Mamba verify/update shows many tiny index kernels, first compare against this existing fused path. |
|
||||
| Staging-buffer fused gather / scatter | `_fused_gather_to_staging_kernel`<br>`_fused_scatter_from_staging_kernel` | `python/sglang/srt/disaggregation/common/staging_buffer.py` | Triton kernels gather scattered KV slices into contiguous staging memory and scatter them back into KV cache on decode | Treat ladders of tiny gather/scatter/copy kernels in heterogeneous TP staging as missing an existing Triton fusion. |
|
||||
|
||||
@@ -146,7 +146,7 @@ Stable entries should be folded into the mainline family rows above.
|
||||
| PR `#22005` fused add + RMSNorm + per-token FP8 quant | `fused_add_rmsnorm_per_token_quant`<br>`per_token_quant_fp8` | `PR #22005`<br>`python/sglang/kernels/jit/csrc/elementwise/fused_add_rmsnorm_per_token_quant.cuh`<br>`python/sglang/kernels/jit/fused_add_rmsnorm_per_token_quant.py` | CUDA JIT kernel keeps normed values in registers and emits BF16 + FP8 outputs plus per-token scales | If FP8 online-quant traces show add+norm followed by per-token quant, treat this as an in-flight upstream CUDA fuse family. |
|
||||
| PR `#20667` Qwen3.5 fused QK norm + RoPE + KV cache write | `fused_qk_norm_rope_cache_pts_quant_shuffle`<br>`fused_qk_norm_mrope_3d_cache_pts_quant_shuffle`<br>`rotary_dim` | `PR #20667`<br>`python/sglang/srt/models/qwen3_5.py`<br>`python/sglang/srt/models/utils.py` | ROCm / AITER path fuses Q / K RMSNorm, partial or 3D RoPE, and direct KV cache write for Qwen3.5 attention | Treat split QK-norm + RoPE + cache-store on Qwen3.5 as a concrete in-flight upstream family, not a novel idea. |
|
||||
| PR `#22392` CUTLASS FP8 GEMM replacing nvjet | `cutlass_scaled_mm`<br>`fp8_scaled_mm`<br>`nvjet`<br>`cudaMemsetAsync` | `PR #22392`<br>`python/sglang/kernels/aot/python/sgl_kernel/gemm.py`<br>`python/sglang/srt/layers/quantization/fp8_utils.py` | Runtime replacement swaps nvjet FP8 GEMMs for CUTLASS kernels, removing per-launch memset bubbles and extra output-copy kernels | Treat nvjet GEMM + memset bubble ladders as an in-flight SGLang linear-kernel family before calling them novel. |
|
||||
| PR `#18612` NVFP4 CUTLASS MoE fused SiLU+Mul+quant | `silu_and_mul_scaled_nvfp4`<br>`nvfp4 expert quant`<br>`cutlass moe` | `PR #18612`<br>`python/sglang/srt/layers/moe/cutlass_w4a8_moe.py`<br>`python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py` | Fuses MoE activation epilogue and NVFP4 expert quantization before the CUTLASS MoE second GEMM | Treat split SiLU+Mul then NVFP4 expert quant in CUTLASS MoE traces as an in-flight upstream SGLang family. |
|
||||
| PR `#18612` NVFP4 CUTLASS MoE fused SiLU+Mul+quant | `silu_and_mul_scaled_nvfp4`<br>`nvfp4 expert quant`<br>`cutlass moe` | `PR #18612`<br>`python/sglang/srt/layers/moe/cutlass_w4a8_moe.py`<br>`python/sglang/kernels/ops/gemm/nvfp4_gemm_swiglu_nvfp4_quant.py` | Fuses MoE activation epilogue and NVFP4 expert quantization before the CUTLASS MoE second GEMM | Treat split SiLU+Mul then NVFP4 expert quant in CUTLASS MoE traces as an in-flight upstream SGLang family. |
|
||||
| PR `#22918` FlashInfer per-token NVFP4 MoE | `per_token_nvfp4`<br>`trtllm_fp4_block_scale_moe`<br>`FlashInfer MoE` | `PR #22918`<br>`python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py` | Adds FlashInfer-backed per-token NVFP4 MoE execution so expert quant/dequant work can move into the fused MoE backend | Treat standalone per-token NVFP4 MoE support kernels as a candidate missing backend-selection path, not an automatically novel kernel idea. |
|
||||
| PR `#22851` NSA top-k backend and FlashInfer / PyTorch top-k split | `nsa topk`<br>`flashinfer_topk`<br>`pytorch_topk`<br>`fast_topk_transform` | `PR #22851`<br>`python/sglang/srt/layers/attention/dsa_backend.py` | Makes NSA top-k backend selection explicit and aligns fused top-k transform with FlashInfer / PyTorch fallbacks | When NSA top-k dominates decode, first classify it as backend selection or fused-transform eligibility work. |
|
||||
| PR `#24125` GLM5 NSA decode CatArrayBatchedCopy removal | `CatArrayBatchedCopy`<br>`GLM-5`<br>`NSA`<br>`TileLang decode` | `PR #24125`<br>`python/sglang/srt/layers/attention/dsa_backend.py` | Skips redundant cat/copy work in the GLM5 NSA TileLang decode path | Treat cat/copy bursts in GLM5 NSA decode as a concrete in-flight cleanup opportunity. |
|
||||
|
||||
@@ -18,7 +18,7 @@ Use these upstream files when the workflow or behavior needs to be justified fro
|
||||
|
||||
## Scheduler-side trace writing
|
||||
|
||||
- `python/sglang/srt/managers/scheduler_profiler_mixin.py`
|
||||
- `python/sglang/srt/managers/scheduler_components/profiler_manager.py`
|
||||
- actual trace start/stop behavior
|
||||
- filename pattern for `TP/DP/PP/EP` and optional stage suffixes
|
||||
- `CUDA_PROFILER` and torch profiler handling
|
||||
@@ -37,6 +37,3 @@ Use these upstream files when the workflow or behavior needs to be justified fro
|
||||
|
||||
- `test/registered/profiling/test_start_profile.py`
|
||||
- validates `/start_profile` behavior, including `CUDA_PROFILER`
|
||||
|
||||
- `test/registered/profiling/test_profile_v2.py`
|
||||
- validates stage-scoped trace outputs under `SGLANG_PROFILE_V2`
|
||||
|
||||
@@ -482,7 +482,9 @@ FUSION_PATTERN_REGISTRY: Tuple[FusionPatternSpec, ...] = (
|
||||
),
|
||||
FusionPatternSpec(
|
||||
pattern="Fused QK RoPE reshape + KV cache write",
|
||||
candidate_path="python/sglang/srt/layers/attention/utils.py",
|
||||
candidate_path=(
|
||||
"python/sglang/kernels/ops/attention/fused_qk_rope_reshape_and_cache.py"
|
||||
),
|
||||
active_keywords=("fused_qk_rope_reshape_and_cache",),
|
||||
split_groups=(
|
||||
("rotary", "rope", "mrope"),
|
||||
@@ -533,7 +535,7 @@ FUSION_PATTERN_REGISTRY: Tuple[FusionPatternSpec, ...] = (
|
||||
),
|
||||
FusionPatternSpec(
|
||||
pattern="NSA fused metadata copy for graph replay",
|
||||
candidate_path="python/sglang/kernels/ops/attention/fused_metadata_copy.py",
|
||||
candidate_path="python/sglang/kernels/ops/attention/dsa/fused_metadata_copy.py",
|
||||
active_keywords=(
|
||||
"fused_metadata_copy",
|
||||
"fused_metadata_copy_multi",
|
||||
@@ -700,7 +702,7 @@ FUSION_PATTERN_REGISTRY: Tuple[FusionPatternSpec, ...] = (
|
||||
pattern="Fused MoE activation + quant / re-quant",
|
||||
candidate_path=(
|
||||
"python/sglang/srt/layers/moe/ep_moe/kernels.py"
|
||||
"<br>python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py"
|
||||
"<br>python/sglang/kernels/ops/gemm/nvfp4_gemm_swiglu_nvfp4_quant.py"
|
||||
"<br>python/sglang/srt/layers/moe/cutlass_w4a8_moe.py"
|
||||
),
|
||||
active_keywords=(
|
||||
@@ -761,7 +763,7 @@ FUSION_PATTERN_REGISTRY: Tuple[FusionPatternSpec, ...] = (
|
||||
FusionPatternSpec(
|
||||
pattern="NSA fused quantize + indexed K-cache store",
|
||||
candidate_path=(
|
||||
"python/sglang/kernels/ops/attention/fused_store_index_cache.py"
|
||||
"python/sglang/kernels/ops/kvcache/dsa_fused_store_index_cache.py"
|
||||
"<br>python/sglang/srt/layers/attention/nsa/nsa_indexer.py"
|
||||
),
|
||||
active_keywords=("fused_store_index_k_cache",),
|
||||
|
||||
@@ -15,7 +15,7 @@ One container owns process-static runtime state: `sglang.srt.runtime_context.Run
|
||||
| runtime flags | `get_flags()` | state that is *not* a pure function of config: `capture` (cuda-graph lifecycle), `moe` (ACTIVE backends, swappable), `dp` (DP-attention runtime flags) | materialized at subsystem init; groups offer `override()` for tests |
|
||||
| resources | `get_resources()`, `get_stream(name)`, `get_buffer(name, factory)` | process-level handles: graph pools, EPLB state, EP dispatcher state, named side streams, workspace buffers | lazy; cleared by `reset_context()` |
|
||||
| per-forward | `get_forward()` | forward-scoped flags (multi-stream switch, MoE output buffer, attn-TP inputs, extend-in-batch) | contextvar-backed; `scoped(**kw)` restores on exit; new threads see defaults |
|
||||
| parallel | `get_parallel()` | one spelling per name: ranks and group handles are the live topology (`@property`, read-through); every other name, sizes included, is a leaf of the parallel config bag | ranks/groups: after dist init; leaves: after publish |
|
||||
| parallel | `get_parallel()` | one spelling per name: scoped override, then permanent stamp, then published parallel config leaf | ranks: stamped at publish; group handles: stamped at distributed init; configured and derived widths: after publish |
|
||||
|
||||
`reset_context()` (unit-test teardown) drops the published config and installs fresh
|
||||
flags/resources/forward tiers.
|
||||
@@ -87,13 +87,12 @@ with what the operator typed, not with what resolution decided.**
|
||||
`ModelRunner.update_model_fields` overrides `model_path` / `load_format` for
|
||||
the target runner.
|
||||
- **Late launcher-stage resolution (pre-publish)**: a few rules cannot run inside
|
||||
`__post_init__` — LoRA normalization, and the auto-parser detection that needs a
|
||||
`ServerArgs.resolve_once` — LoRA normalization, and the auto-parser detection that needs a
|
||||
tokenizer/chat-template load. They are resolution, not mutation, and they
|
||||
**declare** via `arg_groups.overrides.declare_resolution(server_args, source,
|
||||
**fields)`, the same call the rest of the pipeline makes; there is no
|
||||
`declare_late_resolution` any more. *When* a declaration is made is not
|
||||
something the code marks — the guardrails that used to read that marker
|
||||
cover these sites through the ordinary keyword scan instead. The declaration lands
|
||||
something the code marks. The declaration lands
|
||||
in the stash on that very object, so every holder of it carries the decision —
|
||||
the HTTP server, the multi-tokenizer workers it is serialized for, the
|
||||
schedulers it forks — and each of them publishes bags projected from it. The
|
||||
@@ -135,16 +134,23 @@ bag to override at all.
|
||||
instance: **constructor arguments** (`ModelRunner(draft_attention_backend=...)`,
|
||||
`MMEncoder(gpu_id=...)`) and **runner attributes holding the resolved value**
|
||||
(`model_runner.kv_cache_dtype_str`, `prefill_attention_backend_str`,
|
||||
`num_fused_shared_experts`, `linear_attn_backends`) — threaded to consumers as
|
||||
`linear_attn_backends`) — threaded to consumers as
|
||||
arguments, never backfilled onto a shared object. A per-runner choice also stays
|
||||
*out* of the bags: recording it there is how a second runner inherits the first
|
||||
one's answer, which is exactly the bug `linear_attn_backends` replaced. The one sanctioned bend in that rule is
|
||||
*scoped*: `ModelRunner._load_format_scope` exposes the draft's
|
||||
`--speculative-draft-load-format` through `get_model().override(load_format=...)`
|
||||
for exactly the duration of the draft build, because model construction
|
||||
reads that bag leaf — the override restores on exit, so nothing outlives
|
||||
the scope. When there is a runner in hand, read its
|
||||
stamp; that is a different rule from "read the instance".
|
||||
one's answer, which is exactly the bug `linear_attn_backends` replaced. The two
|
||||
sanctioned bends in that rule are *scoped*. `ModelRunner._load_format_scope`
|
||||
exposes the draft's `--speculative-draft-load-format` through
|
||||
`get_model().override(load_format=...)` for exactly the duration of the draft
|
||||
build, because model construction reads that bag leaf — the override restores
|
||||
on exit, so nothing outlives the scope. The shared-expert layout is another scoped view:
|
||||
`get_model().shared_experts` selects the immutable `SharedExpertsConfig` owned
|
||||
by the loaded model (`model.shared_experts_config`). The loader binds it for
|
||||
construction, and runners bind it for execution, warmup and graph capture.
|
||||
Both the fused count and ordinary TopK slot count belong to this object;
|
||||
neither is a separately mutable flag. A scope selects a built layout and
|
||||
restores the previous reference on exit, including exceptions. It cannot
|
||||
resize weights, buffers or a captured graph. When the model is in hand, read
|
||||
its owned config directly (as the routed-experts capturer does).
|
||||
- **Per-instance boundaries** — the tokenizer-manager family, everything under
|
||||
`entrypoints/`, and the tokenizer-process multimodal processors read the bags.
|
||||
The old justification for keeping them on `self.server_args` ("several
|
||||
@@ -178,8 +184,8 @@ bag to override at all.
|
||||
|
||||
### Four ways a config sweep breaks something no test runs
|
||||
|
||||
Each of these shipped in a review round and cost a real defect; each now has a
|
||||
guard, named here so the next sweep checks the same four things by hand first.
|
||||
Each of these shipped in a review round and cost a real defect. Where a test
|
||||
guards one it is named; the next sweep checks all four by hand first.
|
||||
|
||||
1. **The other implementations of an interface.** Dropping a parameter means
|
||||
auditing implementers, not just callers: `CustomSpecAlgo` is the plugin
|
||||
@@ -191,7 +197,8 @@ guard, named here so the next sweep checks the same four things by hand first.
|
||||
`publish` says nothing about whether a given read runs before it. Spawned
|
||||
workers (`MMEncoder` for encoder DP/TP, the Ray scheduler actor) start with
|
||||
an empty context, so a bag read above the publish raises only there.
|
||||
Guard: `test_publish_precedes_bag_reads.py`.
|
||||
Guarded on the launch path only (`test_pre_publish_readers.py`); other
|
||||
entries, spawned workers included, are checked by review.
|
||||
3. **The role namespace a process publishes under.** `ROLE_NAMESPACE_SETS`
|
||||
narrows what each role may read; the DP controller is audited for `exec`
|
||||
alone. A helper that reaches for another namespace passes every default-mode
|
||||
@@ -201,7 +208,9 @@ guard, named here so the next sweep checks the same four things by hand first.
|
||||
means enumerating the others: HTTP, gRPC and in-process `Engine` each have
|
||||
their own server-info and model-info, and each passes its own tests while
|
||||
its users lose the field.
|
||||
Guard: `test_effective_state_surfaces.py`.
|
||||
Guarded for server info across HTTP, both gRPC roles and the Rust frontend
|
||||
(`TestServerInfoTransportParity` in `entrypoints/test_server_info.py`);
|
||||
`Engine.get_server_info` and model info are checked by review.
|
||||
|
||||
A fifth, from the same rounds: the accessor **name itself**. Called as an
|
||||
object member (`manager.get_disagg()`), or shadowed by a same-named import
|
||||
@@ -223,12 +232,13 @@ slack (`>= len(...) - 1`) or compared key names instead of value sources.
|
||||
|
||||
### `get_parallel()`: one spelling per name
|
||||
|
||||
**There is no `.config` hop.** Ranks and group handles are `@property`
|
||||
read-through over the canonical getters, so they answer with the live process
|
||||
groups. Everything else — `tp_size`, `pp_size`, `attn_cp_size`, `dcp_size`,
|
||||
`moe_dp_size` included, alongside config-only leaves like `nccl_port`,
|
||||
`dp_size`, `ep_size`, `dwdp_size` — is answered from the
|
||||
published `parallel` bag. Reading a leaf before publish raises a `ValueError`
|
||||
**There is no `.config` hop.** `ParallelContext._read` answers every name in
|
||||
one order: a scoped override, then a permanent stamp, then the published
|
||||
`parallel` bag leaf. `publish` stamps ranks from the launcher; distributed init
|
||||
stamps group handles and validates their widths. Configured widths such as
|
||||
`tp_size`, `pp_size`, `attn_cp_size`, `dcp_size` and `moe_dp_size`, and config-only
|
||||
leaves such as `nccl_port`, `dp_size`, `ep_size` and `dwdp_size`, fall back to the
|
||||
published bag. Reading a config leaf before publish raises a `ValueError`
|
||||
naming the namespace; an unknown name is an `AttributeError`.
|
||||
|
||||
A size reads from the configuration because the groups are built at exactly the
|
||||
@@ -245,38 +255,33 @@ configured widths — checked at every assignment to `_TP` / `_PP` / `_ATTN_CP`
|
||||
coordinators keep their construction width. Those are different names, not two
|
||||
answers to one name.
|
||||
|
||||
DCP keeps its own pair: `get_parallel().attn_dcp_size` / `.dcp_enabled` answer the
|
||||
*effective* topology (`1` / `False` with no group installed), while `dcp_size` is
|
||||
DCP keeps its own pair: `get_parallel().attn_dcp_size` / `.dcp_enabled` are derived
|
||||
from the published configuration unless explicitly overridden; `dcp_size` is
|
||||
what the launch requested.
|
||||
|
||||
A process-global seed field-read of one of these sizes
|
||||
(`get_server_args().tp_size`, or an alias of it) is a read-ratchet failure. A
|
||||
`server_args` the object was *handed* is a different thing and not a ratchet
|
||||
matter — see "Reads that legitimately stay on a ServerArgs instance".
|
||||
Fail-loud is narrower: before dist init, a live *rank/group* read raises. The six
|
||||
Fail-loud is narrower: an unstamped rank or group handle with no declared default
|
||||
raises; ranks are available after publish, before dist init. The six
|
||||
parallel quotients are not live reads at all — `attn_tp_size`, `attn_dp_size`,
|
||||
`attn_dcp_size`, `moe_ep_size`, `moe_tp_size`, `dcp_enabled` are a function of the
|
||||
configured leaves, computed once at publish into bag leaves, and answered
|
||||
override → stamp → published leaf. So `dcp_enabled` means "the launch configured
|
||||
DCP" (`dcp_size > 1`), not "a DCP group is installed here"; in a scheduler the
|
||||
stamp makes the two identical, in a process that publishes without dist init they
|
||||
differ. `test_a_topology_is_stated_by_naming_the_width` and its neighbours in
|
||||
`test_runtime_context.py` pin this; they replaced
|
||||
DCP" (`dcp_size > 1`), not "a DCP group is installed here". Normal distributed
|
||||
initialization builds the groups to match the configured widths; it does not
|
||||
restamp these DCP leaves. `test_a_topology_is_stated_by_naming_the_width` and its
|
||||
neighbours in `test_runtime_context.py` pin this; they replaced
|
||||
`test_attn_dcp_defaults_when_group_is_uninitialized`. One consequence for tests:
|
||||
overriding a leaf no longer moves its quotient — state a topology by publishing a
|
||||
config, or by naming the width. After init,
|
||||
only the DCP group is optional (`_DCP` exists only when `dcp_size > 1`; attn-CP and
|
||||
moe-DP always install, as size-1 aliases if unused). The `config` hop is
|
||||
deliberately dynamo-traceable (a plain property over a slot, no
|
||||
`object.__getattribute__`); gate helpers like `enable_moe_dense_fully_dp()` run inside
|
||||
moe-DP always install, as size-1 aliases if unused). Bare parallel leaf reads are
|
||||
dynamo-traceable; gate helpers like `enable_moe_dense_fully_dp()` run inside
|
||||
compiled model forwards (`test_parallel_config_leaves_trace_under_torch_compile` pins
|
||||
this).
|
||||
|
||||
A third surface carries the same names: `ParallelState` (`self.ps` / `mr.ps`), the
|
||||
frozen per-process snapshot built once in `Scheduler.__init__` from these configured
|
||||
sizes plus this process's ranks, and handed down (draft runners included). Prefer it
|
||||
where an object was handed one; it is not a global accessor.
|
||||
|
||||
### Reading config: the seed is off limits
|
||||
|
||||
`get_server_args().field` in business code is a ratchet failure. Read:
|
||||
@@ -322,12 +327,13 @@ where an object was handed one; it is not a global accessor.
|
||||
follows a post-publish override. Two questions are *not* that, and have their
|
||||
own spelling: the width of the MoE communicator you are about to collectively
|
||||
operate on is `get_moe_cp_size()` (the `_MOE_DP = _ATTN_CP` alias makes it
|
||||
differ), and the effective DCP topology is `get_parallel().attn_dcp_size` /
|
||||
`.dcp_enabled` (`1` / `False` when no group is installed), which does not need
|
||||
dist init to answer.
|
||||
differ), and DCP's derived width and enablement are
|
||||
`get_parallel().attn_dcp_size` / `.dcp_enabled` (published configuration
|
||||
unless explicitly overridden).
|
||||
- **this runner's resolved value** → the runner
|
||||
(`prefill_attention_backend_str`, `kv_cache_dtype_str`,
|
||||
`draft_attention_backend`, `num_fused_shared_experts` on the model).
|
||||
`draft_attention_backend`); for what the model's layers bake in at
|
||||
construction, the layer (`num_fused_shared_experts` on a MoE block).
|
||||
|
||||
`self.server_args.field` is still right for handed per-instance config (see
|
||||
"Reads that legitimately stay on a ServerArgs instance" above for the full set —
|
||||
@@ -335,10 +341,9 @@ per-instance boundaries and whole-object passes; there are no per-runner config
|
||||
copies to read any more). The allow-list is `GrammarManager` and `MMEncoder`;
|
||||
what sits beside it is residue, not a family — and not for one single reason:
|
||||
|
||||
- the tokenizer-manager family and `entrypoints/` **read the bags**; what is
|
||||
left of them in the exposure ratchet is a handful of individually-dispositioned
|
||||
pairs, not a family awaiting conversion. Read the ratchet for the current set
|
||||
rather than assuming a directory is off-limits;
|
||||
- the tokenizer-manager family and `entrypoints/` **read the bags**; review a new
|
||||
instance read there against the raw-input contract rather than assuming a
|
||||
directory is off-limits;
|
||||
- `GrammarManager` is a handed instance for its residual `self.server_args`
|
||||
reads, but backend selection is **not** on the instance any more:
|
||||
`create_grammar_backend` reads `get_exec().kernel.grammar_backend`, and
|
||||
@@ -384,15 +389,15 @@ if you do it, say so in the test.
|
||||
|
||||
### Mid-resolution reads (inside the pipeline only)
|
||||
|
||||
Resolution runs in `__post_init__` and **writes nothing onto the record**: a
|
||||
handler declares (`self._declare` / `declare_resolution`), the declaration goes
|
||||
Resolution runs in `ServerArgs.resolve_once` and **writes nothing onto the record**: a
|
||||
handler declares (`declare_resolution`), the declaration goes
|
||||
into the stash, and the fields keep what the caller passed. So a mid-resolution
|
||||
read of a field answers with the *raw input* — every reader in the pipeline goes
|
||||
through a view instead:
|
||||
|
||||
- `resolving_view(server_args)` / `self._resolved()` — the live view (walks the
|
||||
- `resolving_view(server_args)` — the live view (walks the
|
||||
stash per read). This is what handlers and hooks bind, conventionally as
|
||||
`cfg = resolving_view(self)` at the top of the handler.
|
||||
`cfg = resolving_view(server_args)` at the top of the handler.
|
||||
- `resolved_view(server_args)` — snapshots the overlay when built, which is what
|
||||
a post-process pass wants: it reads the state at *its* slot.
|
||||
|
||||
@@ -451,13 +456,18 @@ Never assign `server_args` fields from model code. Declare instead
|
||||
its checkpoint drives (shared-experts fusion) is a question the *loader* asks
|
||||
the model class — `shared_experts_fusion_disable_reason(hf_config,
|
||||
quant_config)`, a classmethod answering without an instance — at the single
|
||||
model-instantiation point, and
|
||||
`install_shared_experts_fusion_decision` writes the answer to the ACTIVE moe
|
||||
flag before that model's layers build and read it
|
||||
(`is_shared_experts_fusion_disabled`, config-intent fallback).
|
||||
`draft_model_build_scope` brackets every draft build and routes the draft's
|
||||
answer to the speculative leaf, so a draft's decision never overwrites the
|
||||
target's. A process-level load-time fact (the sm80 dtype fallback —
|
||||
model-instantiation point, together with how many shared experts it fuses
|
||||
(`num_fusable_shared_experts(hf_config)`; an entry class without both hooks
|
||||
fuses none). `resolve_shared_experts_config` returns one immutable config;
|
||||
the loader scopes it through `get_model().override(shared_experts=config)`
|
||||
while building and retains it on the resulting model. `num_shared_topk_slots`
|
||||
derives from the fused count and frozen Waterfill policy, so ordinary routing
|
||||
and Waterfill cannot both claim the shared slots. `TopK`/`HashTopK` retain the
|
||||
same config; `top_k` counts only routed experts at construction.
|
||||
`draft_model_build_scope` still selects draft boundary/MMA preferences; it
|
||||
does not save shared-expert flags. Target and draft each own their layout.
|
||||
These values have model lifetime, so they do not belong in `get_forward()`.
|
||||
A process-level load-time fact (the sm80 dtype fallback —
|
||||
device-driven, identical for every runner) records directly via
|
||||
`get_context().override`.
|
||||
|
||||
@@ -466,12 +476,12 @@ dataclass. A declaration against a non-whitelisted field fails at its slot.
|
||||
|
||||
### Load-time vs resolution-time (critical)
|
||||
|
||||
`__post_init__` runs in the launcher process before any model/platform import. Logic that
|
||||
`ServerArgs.resolve_once` runs in the launcher process before any model/platform import. Logic that
|
||||
consults an **extensible registry** (e.g. out-of-tree platforms registering attention
|
||||
backends in `init_backend()`, which runs at `model_runner` import) must stay at load time
|
||||
(ModelRunner init), writing through `get_context().override()`. Before moving any
|
||||
load-time logic into resolution, verify everything it reads is already complete at
|
||||
construction time.
|
||||
resolution time.
|
||||
|
||||
## Runtime flags (`get_flags()`)
|
||||
|
||||
@@ -499,8 +509,8 @@ Named slots + two keyed-lazy registries:
|
||||
side stream is `"lora_side"`. Two call sites may share a name only if their work belongs
|
||||
on one stream — sharing across roles serializes intended overlap.
|
||||
- `get_buffer(name, factory)` — get-or-create a named persistent buffer. Grow-only or
|
||||
per-device semantics manage their `resources.buffers` entries directly (see tokenspeed /
|
||||
SM120 split / Marlin workspace). Buffer names are per-backend today; do not silently
|
||||
per-device semantics manage their `resources.buffers` entries directly (see SM120
|
||||
split / Marlin workspace). Buffer names are per-backend today; do not silently
|
||||
share.
|
||||
- Singletons with manager semantics (EP dispatcher buffers, EPLB recorder/metadata, graph
|
||||
memory pool) keep their owning accessors/classes as facades; only the *state* lives in a
|
||||
@@ -543,8 +553,9 @@ ONE thread — do not design for TBO threads that don't exist.
|
||||
them explicitly on the mock; `MagicMock(spec=...)` raises on attributes that only
|
||||
exist post-`__init__`, which is the fastest way to find a missed stub.
|
||||
- `reset_context()` in teardown when a test publishes outside a scoped override.
|
||||
- `ServerArgs(model_path="dummy")` early-returns the pipeline (few declarations, no
|
||||
strict guard) — fine for lightweight fixtures.
|
||||
- `ServerArgs(model_path="dummy")` resolves through the pipeline's early return (few
|
||||
declarations) — fine for lightweight fixtures. The strict guard still applies once
|
||||
`resolve_once` has run, as it does at `publish`.
|
||||
- **Asserting what resolution decided reads `resolution_result(sa, "field")`**, not
|
||||
`sa.field`: the field is the raw input. Assert the field only when the point of
|
||||
the case *is* that the record stayed pristine (the FA4 page-size and waterfill
|
||||
@@ -553,7 +564,7 @@ ONE thread — do not design for TBO threads that don't exist.
|
||||
pytest run lets a context published by an earlier file mask a missing-publish bug in a
|
||||
later one.
|
||||
|
||||
## Guardrails (these fail CI; what to do when they fire)
|
||||
## Guardrails and review rules
|
||||
|
||||
1. **Strict mutation guard** (always on, and with no exception): bare
|
||||
`server_args.x = ...` after resolution raises unconditionally in
|
||||
@@ -563,10 +574,9 @@ ONE thread — do not design for TBO threads that don't exist.
|
||||
resolved config with `get_context().override`; hand a per-runner value to its
|
||||
runner as a constructor argument. Projected bags are sealed the same way (leaf
|
||||
assignment raises).
|
||||
2. **Mutation ratchet** (`test_server_args_mutation_ratchet.py`, exact pin 0 over the whole
|
||||
package minus the pipeline / multimodal_gen): textual scan for assignment forms. Never
|
||||
raise the baseline.
|
||||
3. **No-copy contract** (`test_server_args_no_instance_mutation_entry.py`): neither
|
||||
2. **No textual mutation ratchet**: the strict guard above is the only mutation
|
||||
check.
|
||||
3. **No-copy contract** (review only, no automated guard): neither
|
||||
`ServerArgs.override` nor `ServerArgs.derive` exists, and nothing in the package
|
||||
calls either form. Rerouting a writer to the bags means flipping **all its readers
|
||||
in the same commit** (no transitional dual-write).
|
||||
@@ -577,23 +587,23 @@ ONE thread — do not design for TBO threads that don't exist.
|
||||
runner's stamp — not `get_server_args().field`, which the read ratchet below pins
|
||||
at zero. `runtime_context.get_server_args()` is only for the
|
||||
whole-object shapes (dumps, provenance, a hand-off to a callee that takes a config).
|
||||
5. **Global config read ratchet** (`test_global_config_read_ratchet.py`): baselines are
|
||||
**0** for both the direct `get_server_args().field` and the alias form (function-local
|
||||
5. **Global config read check** (`test_global_config_read_ratchet.py`): no business code
|
||||
reads a field off `get_server_args()`, directly or through an alias (function-local
|
||||
— including local copies of an alias, `cfg = sa` — module-level, or parked on an
|
||||
instance attribute, plus the `getattr(..., "field")` spelling of each; a name
|
||||
computed at runtime or indirection deeper than a local name copy is census-tool
|
||||
territory, per the test's docstring). The scanner matches `get_server_args` by its
|
||||
computed at runtime or indirection deeper than a local name copy is outside the
|
||||
check, per the test's docstring). The scanner matches `get_server_args` by its
|
||||
literal name — bare or module-qualified (`ctx.get_server_args()`) — and
|
||||
`TestNoRenamedAccessorImports` in the same file *bans* `import ... as` renames of it,
|
||||
which is what makes literal-name matching sound. Exempt by owner
|
||||
module only (`runtime_context.py`, `server_args.py`, `arg_groups/`). Two classes,
|
||||
no more: `TestGlobalConfigReadRatchet` holds the two baselines and
|
||||
no more: `TestGlobalConfigReads` holds the zero check and
|
||||
`TestNoRenamedAccessorImports` holds the ban. There is no configured-size registry
|
||||
here any longer — `get_parallel()` has one spelling per name, so a size read is not a
|
||||
choice between two answers and nothing needs registering.
|
||||
6. **Module-state ratchet** (`test_module_state_ratchet.py`): `global` statements in the
|
||||
flag-owning layers are pinned by name. A new module-level runtime global belongs on a
|
||||
flags group / resources slot instead; migrating a pinned survivor must shrink the pin.
|
||||
6. **Module-state rule** (review only, no automated guard): a new module-level
|
||||
runtime global in the flag-owning layers belongs on a flags group / resources slot
|
||||
instead.
|
||||
7. **Namespace coverage** (`test_server_args_namespaces.py`,
|
||||
`test_runtime_context_config_bags.py`): every `ServerArgs` field resolves to a
|
||||
namespace — from the `arg_groups/fields/` class that declares it — and the
|
||||
@@ -649,8 +659,7 @@ Key source files: `python/sglang/srt/runtime_context.py` (the container, every t
|
||||
`python/sglang/srt/arg_groups/overrides.py` (override registry, passes,
|
||||
`declare_resolution` and the spellings around it), `python/sglang/srt/server_args.py` (`NS` metadata,
|
||||
`Arg(..., resolvable=True)`, `__setattr__` strict guard), and the guardrail tests under
|
||||
`test/registered/unit/` (`test_server_args_mutation_ratchet.py`,
|
||||
`test_global_config_read_ratchet.py`,
|
||||
`test_module_state_ratchet.py`, `test_server_args_namespaces.py`,
|
||||
`test/registered/unit/` (`test_global_config_read_ratchet.py`,
|
||||
`test_server_args_namespaces.py`,
|
||||
`test_runtime_context.py` — the last one doubles
|
||||
as executable documentation of every tier's semantics).
|
||||
|
||||
@@ -36,4 +36,4 @@ a real `AttributeError` if the field is ever renamed:
|
||||
revision=getattr(server_args, "revision", None), # BAD
|
||||
revision=server_args.revision, # GOOD
|
||||
```
|
||||
(see `python/sglang/srt/managers/template_detection.py`)
|
||||
(see `python/sglang/srt/parser/template_detection.py`)
|
||||
|
||||
@@ -27,8 +27,8 @@ A new unit test case must fall into one of these categories:
|
||||
3. **Critical-path bookkeeping.** Defends conventions that are easy to break by
|
||||
forgetting to sync -- registry completeness, field lifecycle, serialization
|
||||
compatibility. Enumerating assertions are fine here; the guarded failure
|
||||
mode is "someone extended X without updating Y". Example: the ratchet tests
|
||||
(`test/registered/unit/test_module_state_ratchet.py`).
|
||||
mode is "someone extended X without updating Y". Example:
|
||||
`test/registered/unit/test_global_config_read_ratchet.py`.
|
||||
|
||||
Not admissible:
|
||||
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
# Gitea Actions 自动构建 sglang 镜像(海外节点,原版源)
|
||||
# 基底 lmsysorg/sglang:dev-dsv41(docker.io),依赖走 pypi.org 默认源。
|
||||
# 触发:push 到 dsv41-pd 分支。
|
||||
#
|
||||
# 前置条件(一次性,在 Gitea 实例上配置):
|
||||
# 1. 实例已注册 act_runner(Gitea Actions runner,标签含 ubuntu-latest)
|
||||
# 2. 仓库 Settings → Secrets 添加:
|
||||
# REGISTRY_USERNAME / REGISTRY_PASSWORD(推送镜像的账号,如 Gitea 访问令牌)
|
||||
# 3. 可选:Settings → Actions → Variables 添加 REGISTRY(默认 git.agentwithu.com,
|
||||
# 即 Gitea 自带容器 registry;也可填 docker.io 等)
|
||||
name: build-sglang-image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dsv41-pd, dsv41-pd-visioncp]
|
||||
# compose/部署配置 与 workflow 自身的改动不触发镜像构建(省 runner 与推送带宽)。
|
||||
# 改了 workflow 想重建镜像时,需伴随任意代码改动或手动触发。
|
||||
paths-ignore:
|
||||
- 'deploy/**'
|
||||
- '.gitea/**'
|
||||
|
||||
env:
|
||||
# 直接写死 Gitea 自带 registry(vars context 在该实例上求值异常会导致回退 docker.io)
|
||||
REGISTRY: git.agentwithu.com
|
||||
IMAGE_NAME: minke.yu/sglang
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Resolve tag
|
||||
id: meta
|
||||
run: |
|
||||
SHA9=$(git rev-parse --short=9 HEAD)
|
||||
echo "tag=${{ github.ref_name }}-${SHA9}-ci-$(date -u +%Y%m%d-%H%M)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ secrets.REGISTRY_USERNAME }}
|
||||
password: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
|
||||
- uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile.gitea
|
||||
push: true
|
||||
# cache-from: type=registry
|
||||
# cache-to: type=registry,mode=max
|
||||
tags: |
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.tag }}
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}-latest
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
.github @merrymercy @Fridge003 @ispobock @Kangyan-Zhou @HaiShaw @bingxche
|
||||
/docker @Fridge003 @ispobock @HaiShaw @ishandhanani @yctseng0211 @sogalin
|
||||
/docker/npu.Dockerfile @ping1jing2 @iforgetmyname @whybeyoung
|
||||
/docker/sgl-router.Dockerfile @sherlockwu @Kangyan-Zhou @ShangmingCai
|
||||
/docs @wisclmy0611 @zijiexia @sogalin
|
||||
/docs @wisclmy0611 @zijiexia @Richardczl98 @JustinTong0323 @sogalin
|
||||
/python/pyproject.toml @merrymercy @Fridge003 @ispobock
|
||||
/python/sglang/kernels @DarkSharpness @HaiShaw @BBuf @celve @HydraQYH @yuan-luo
|
||||
/python/sglang/kernels/ops/diffusion @yingluosanqian @BBuf @mickqian @triple-mu @niehen6174
|
||||
/python/sglang/kernels/ops/attention/fla @yizhang2077 @hebiao064 @yuan-luo
|
||||
/python/sglang/kernels/ops/attention/linear @yizhang2077 @hebiao064 @yuan-luo
|
||||
/python/sglang/kernels/ops/lora @Ying1123 @Fridge003 @lifuhuang @yushengsu-thu @jybsuper
|
||||
/python/sglang/kernels/jit/csrc/lora @Ying1123 @Fridge003 @lifuhuang @yushengsu-thu @jybsuper
|
||||
/python/sglang/multimodal_gen @mickqian @ping1jing2 @HaiShaw @yichiche @AgainstEntropy @BBuf @triple-mu @kevin-mii @niehen6174
|
||||
@@ -81,6 +83,11 @@
|
||||
/rust/sglang-processor @sagearc @sherlockwu
|
||||
/rust/sglang-radix-tree @Jialin @alphabetc1
|
||||
/rust/sglang-server @rainj-me @merrymercy
|
||||
/sgl-router @sherlockwu @Kangyan-Zhou @ShangmingCai @hzh0425
|
||||
/sgl-router/sgl-kv-indexer @sherlockwu @lixiufei-leo @ShangmingCai @wuyl1 @hzh0425
|
||||
/sgl-router/src/policies @sherlockwu @ShangmingCai @Bo-Vincent
|
||||
/sgl-router/src/workers @sherlockwu @AMD-yanfeiwang
|
||||
/sgl-router/src/protocol @sherlockwu @whybeyoung
|
||||
/sgl-model-gateway @slin1237 @CatherineSue
|
||||
/sgl-model-gateway/benches @slin1237
|
||||
/sgl-model-gateway/bindings/python @CatherineSue @key4ng @slin1237
|
||||
@@ -107,6 +114,7 @@
|
||||
/python/sglang/srt/speculative/cpp_ngram @hnyls2002 @Qiaolin-Yu @kpham-sgl
|
||||
/python/sglang/srt/speculative/frozen_kv_mtp_*.py @hnyls2002 @Qiaolin-Yu @kpham-sgl @pyc96
|
||||
/python/sglang/kernels/ops/speculative/ngram_*.py @hnyls2002 @Qiaolin-Yu @kpham-sgl
|
||||
/python/sglang/kernels/ops/embeddings/ngram_embedding.py @hnyls2002 @Qiaolin-Yu @kpham-sgl
|
||||
/python/sglang/kernels/jit/csrc/ngram_corpus @hnyls2002 @Qiaolin-Yu @kpham-sgl
|
||||
/python/sglang/srt/models/laguna.py @Jiminator
|
||||
/python/sglang/srt/configs/laguna.py @Jiminator
|
||||
|
||||
@@ -14,9 +14,9 @@ Recognized permission keys:
|
||||
| --- | --- |
|
||||
| `can_tag_run_ci_label` | `/tag-run-ci-label`, `/tag-and-rerun-ci`, `/run-full-ci`, `/run-extra-ci` |
|
||||
| `can_rerun_failed_ci` | `/rerun-failed-ci`, `/tag-and-rerun-ci`, `/run-full-ci`, `/run-extra-ci` |
|
||||
| `cooldown_interval_minutes` | rate limit in `pr-gate.yml`; `0` also grants `/rerun-test`, `/rerun-group` |
|
||||
| `cooldown_interval_minutes` | rate limit in `pr-gate.yml`; `0` also grants `/rerun-test`, `/rerun-group`, `/run-cpu-ci` |
|
||||
|
||||
`/rerun-test` and `/rerun-group` are gated on the commenter alone: either
|
||||
`/rerun-test`, `/rerun-group` and `/run-cpu-ci` are gated on the commenter alone: either
|
||||
`cooldown_interval_minutes: 0`, or `write`/`admin` permission on the repo. Where
|
||||
the PR comes from makes no difference, and authoring it grants nothing.
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ inputs:
|
||||
required: false
|
||||
default: ''
|
||||
cache_key_prefix:
|
||||
description: 'Must match what _pr-test-rust-ext-build.yml saves under.'
|
||||
description: 'Must match what _pr-test-rust-ext-build.yml saves under. ARM64 runners always restore rust-ext-aarch64-cp312.'
|
||||
required: false
|
||||
default: rust-ext-x86_64-cp312
|
||||
|
||||
@@ -53,7 +53,7 @@ runs:
|
||||
path: |
|
||||
python/sglang/srt/rust_extensions/_*.so
|
||||
python/sglang/srt/mem_cache/rust_tree_core/mem_cache*.so
|
||||
key: ${{ inputs.cache_key_prefix }}-${{ hashFiles('rust/**', 'proto/**', 'python/setup.py', 'python/pyproject.toml', 'python/sglang/srt/rust_extensions/torch_build.py', '.github/workflows/_pr-test-rust-ext-build.yml', 'scripts/ci/utils/stage_rust_ext_modules.sh') }}
|
||||
key: ${{ runner.arch == 'ARM64' && 'rust-ext-aarch64-cp312' || inputs.cache_key_prefix }}-${{ hashFiles('rust/**', 'proto/**', 'python/setup.py', 'python/pyproject.toml', 'python/sglang/srt/rust_extensions/torch_build.py', '.github/workflows/_pr-test-rust-ext-build.yml', 'scripts/ci/utils/stage_rust_ext_modules.sh') }}
|
||||
|
||||
# Job-wide, but only setup.py reads it, and only while building.
|
||||
# Whether the modules suit this interpreter is not decided here:
|
||||
|
||||
@@ -135,6 +135,28 @@ hicache:
|
||||
- 'python/sglang/srt/mem_cache/**/*storage*'
|
||||
- 'python/sglang/srt/mem_cache/l2_transfer.py'
|
||||
|
||||
# A *mooncake* filename glob also matches benchmark/datasets/mooncake.py.
|
||||
mooncake:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
# PD KV transfer.
|
||||
- 'python/sglang/srt/disaggregation/mooncake/**/*'
|
||||
- 'python/sglang/srt/distributed/device_communicators/mooncake_transfer_engine.py'
|
||||
- 'python/sglang/srt/observability/mooncake_trace.py'
|
||||
- 'test/registered/unit/disaggregation/**/*mooncake*'
|
||||
- 'test/manual/kv_transfer/**/*mooncake*'
|
||||
# HiCache store.
|
||||
- 'python/sglang/srt/mem_cache/storage/mooncake_store/**/*'
|
||||
- 'python/sglang/test/mooncake_utils.py'
|
||||
- 'test/registered/unit/mem_cache/**/*mooncake*'
|
||||
- 'test/registered/hicache/**/*mooncake*'
|
||||
# MoE EP. elastic_ep.py also implements nixl.
|
||||
- 'python/sglang/srt/layers/moe/token_dispatcher/mooncake.py'
|
||||
- 'python/sglang/srt/elastic_ep/expert_backup_client.py'
|
||||
- 'python/sglang/srt/elastic_ep/expert_backup_manager.py'
|
||||
- 'test/registered/ep/**/*mooncake*'
|
||||
- 'test/manual/ep/**/*mooncake*'
|
||||
|
||||
# Deterministic
|
||||
deterministic:
|
||||
- changed-files:
|
||||
|
||||
@@ -1,133 +0,0 @@
|
||||
# Reusable workflow: cross-reference CI test failures with coverage-based
|
||||
# test recommendations. Called by pr-test-npu.yml's analyze-failure-report job.
|
||||
name: Analyze Failure Report
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
log_artifact_pattern:
|
||||
type: string
|
||||
required: true
|
||||
description: 'Glob pattern for test log artifacts (e.g. selected-test-logs-*)'
|
||||
diffusion_artifact_pattern:
|
||||
type: string
|
||||
required: false
|
||||
default: 'diffusion-failures-*'
|
||||
description: 'Glob pattern for diffusion failure artifacts (consistency failures)'
|
||||
recommendations_content:
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
description: 'Content of recommendations from upstream job output'
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -el {0}
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: cross-reference failures with recommendations
|
||||
continue-on-error: true # Non-blocking on parse/read failure.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout sglang repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# ----- Step 1: Locate recommendations -----
|
||||
- name: Locate recommendations file
|
||||
id: locate-recs
|
||||
run: |
|
||||
mkdir -p ./test-logs
|
||||
# Always create the file so the analyzer can run on diffusion-only failures.
|
||||
touch ./test-logs/_recommended.txt
|
||||
echo "RECS_FILE=./test-logs/_recommended.txt" >> $GITHUB_ENV
|
||||
|
||||
if [ -n "${{ inputs.recommendations_content }}" ]; then
|
||||
echo "has_recommendations=true" >> $GITHUB_OUTPUT
|
||||
echo "Using workflow output recommendations"
|
||||
echo "${{ inputs.recommendations_content }}" > ./test-logs/_recommended.txt
|
||||
echo "RECS_SOURCE=output" >> $GITHUB_ENV
|
||||
COUNT=$(wc -l < ./test-logs/_recommended.txt)
|
||||
echo "Recommendations count: ${COUNT}"
|
||||
else
|
||||
echo "has_recommendations=false" >> $GITHUB_OUTPUT
|
||||
echo "RECS_SOURCE=none" >> $GITHUB_ENV
|
||||
echo "::notice::No recommended test cases detected; analyzing log and diffusion failures only."
|
||||
fi
|
||||
|
||||
# ----- Step 2: Download test logs -----
|
||||
# continue-on-error is a step keyword, not a download-artifact input.
|
||||
- name: Download test logs
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
pattern: ${{ inputs.log_artifact_pattern }}
|
||||
path: ./test-logs
|
||||
merge-multiple: true
|
||||
continue-on-error: true
|
||||
|
||||
# ----- Step 2b: Download diffusion failure artifacts -----
|
||||
# Keep merge-multiple off so each partition's consistency_failures/summary.json
|
||||
# lands in its own <artifact-name>/ directory instead of clobbering the others.
|
||||
- name: Download diffusion failure artifacts
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
pattern: ${{ inputs.diffusion_artifact_pattern }}
|
||||
path: ./diffusion-failures
|
||||
continue-on-error: true
|
||||
|
||||
- name: Check for diffusion failure artifacts
|
||||
id: check-diffusion
|
||||
run: |
|
||||
if compgen -G "./diffusion-failures/*/consistency_failures/summary.json" > /dev/null; then
|
||||
echo "found=true" >> $GITHUB_OUTPUT
|
||||
echo "Diffusion consistency failure artifacts found"
|
||||
else
|
||||
echo "found=false" >> $GITHUB_OUTPUT
|
||||
echo "No diffusion consistency failure artifacts"
|
||||
fi
|
||||
|
||||
- name: Show downloaded artifacts structure
|
||||
run: |
|
||||
echo "Downloaded log files:"
|
||||
find ./test-logs -type f | head -50 || echo " (no logs found)"
|
||||
echo "Downloaded diffusion failure artifacts:"
|
||||
if compgen -G "./diffusion-failures/*" > /dev/null; then
|
||||
find ./diffusion-failures -type f | head -50
|
||||
else
|
||||
echo " (no diffusion failure artifacts found)"
|
||||
fi
|
||||
|
||||
# ----- Step 3: Run analysis -----
|
||||
- name: Ensure regex is available
|
||||
if: ${{ steps.locate-recs.outputs.has_recommendations != 'false' || steps.check-diffusion.outputs.found == 'true' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! python3 -c "import regex" >/dev/null 2>&1; then
|
||||
echo "regex not found for $(python3 -V); bootstrapping pip and installing regex"
|
||||
curl -fsSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py
|
||||
python3 get-pip.py --break-system-packages
|
||||
python3 -m pip install --break-system-packages regex
|
||||
fi
|
||||
python3 -c "import regex; print('regex ok:', regex.__file__)"
|
||||
|
||||
- name: Run failure analysis
|
||||
if: ${{ steps.locate-recs.outputs.has_recommendations != 'false' || steps.check-diffusion.outputs.found == 'true' }}
|
||||
id: analysis
|
||||
continue-on-error: true
|
||||
run: |
|
||||
python3 scripts/ci/npu/precise-test/analyze_failure_report.py \
|
||||
--log-dir ./test-logs \
|
||||
--diffusion-dir ./diffusion-failures \
|
||||
--recommendations-file "${RECS_FILE}" \
|
||||
--recommendations-source "${RECS_SOURCE:-none}" \
|
||||
--output ./test-logs/failure_report.md
|
||||
|
||||
# ----- Step 4: Upload report -----
|
||||
- name: Upload failure analysis report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: failure-analysis-report
|
||||
path: ./test-logs/failure_report.md
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
@@ -50,10 +50,10 @@ on:
|
||||
description: 'If true, delegate test execution to scripts/ci/npu/precise-test/run_tests_with_coverage.sh instead of running via run_suite.py directly'
|
||||
type: boolean
|
||||
default: false
|
||||
upload_test_logs:
|
||||
description: 'If true, upload the test logs as a GitHub Actions artifact (only enabled by pr-test-npu)'
|
||||
type: boolean
|
||||
default: false
|
||||
only_tests:
|
||||
description: 'Coverage-recommended test files for this suite (one repo-relative path per line, from recommend-tests-from-coverage). Empty = run the full suite.'
|
||||
type: string
|
||||
default: ''
|
||||
github-token:
|
||||
description: 'GitHub token for API calls'
|
||||
type: string
|
||||
@@ -70,6 +70,7 @@ env:
|
||||
|
||||
jobs:
|
||||
run:
|
||||
name: ${{ inputs.use_coverage_runner && inputs.self_name || 'run' }} (${{ matrix.partition }})
|
||||
runs-on: ${{ inputs.runner_config }}
|
||||
container:
|
||||
image: ${{ inputs.image }}
|
||||
@@ -241,6 +242,7 @@ jobs:
|
||||
env:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
TEST_LOG_DIR: /tmp/test-logs
|
||||
ONLY_TESTS: ${{ inputs.only_tests }}
|
||||
shell: bash
|
||||
run: |
|
||||
# Fail fast on any command error, undefined variable, or pipe failure.
|
||||
@@ -284,6 +286,16 @@ jobs:
|
||||
NIGHTLY_FLAG="--nightly --continue-on-error"
|
||||
fi
|
||||
|
||||
# Precision-test mode: run only the coverage-recommended files for
|
||||
# this suite. The bucketed list comes from recommend-tests-from-
|
||||
# coverage; empty input keeps the full-suite behavior.
|
||||
ONLY_FILE_ARGS=""
|
||||
if [ -n "${ONLY_TESTS}" ]; then
|
||||
printf '%s\n' "${ONLY_TESTS}" > /tmp/precision_tests.txt
|
||||
ONLY_FILE_ARGS="--only-file-list /tmp/precision_tests.txt"
|
||||
echo "Precision mode: running $(wc -l < /tmp/precision_tests.txt) recommended file(s)"
|
||||
fi
|
||||
|
||||
# Optional: delegate execution to scripts/ci/npu/precise-test/run_tests_with_coverage.sh when the
|
||||
# caller sets use_coverage_runner=true. list_tests.py (same directory) enumerates the
|
||||
# selected test files; the original run_suite.py execution below is preserved
|
||||
@@ -308,13 +320,4 @@ jobs:
|
||||
--auto-partition-id ${{ matrix.partition }} \
|
||||
--auto-partition-size ${{ fromJson(inputs.partitions).size }} \
|
||||
${{ inputs.timeout_per_file && format('--timeout-per-file {0}', inputs.timeout_per_file) || '' }} \
|
||||
$NIGHTLY_FLAG $CONTINUE_ON_ERROR_FLAG 2>&1 | tee "${LOG_FILE}"
|
||||
|
||||
- name: Upload test logs
|
||||
if: always() && inputs.use_coverage_runner != true && inputs.upload_test_logs == true
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: selected-test-logs-${{ inputs.self_name }}-part${{ matrix.partition }}
|
||||
path: /tmp/test-logs/
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
$ONLY_FILE_ARGS $NIGHTLY_FLAG $CONTINUE_ON_ERROR_FLAG 2>&1 | tee "${LOG_FILE}"
|
||||
|
||||
@@ -71,10 +71,10 @@ on:
|
||||
description: 'If true, delegate test execution to scripts/ci/npu/precise-test/run_tests_with_coverage.sh instead of running via run_suite.py directly'
|
||||
type: boolean
|
||||
default: false
|
||||
upload_test_logs:
|
||||
description: 'If true, upload the test logs as a GitHub Actions artifact (only enabled by pr-test-npu)'
|
||||
type: boolean
|
||||
default: false
|
||||
only_tests:
|
||||
description: 'Coverage-recommended test files for this suite (one repo-relative path per line, from recommend-tests-from-coverage). Empty = run the full suite.'
|
||||
type: string
|
||||
default: ''
|
||||
|
||||
env:
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check }}
|
||||
@@ -85,7 +85,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
e2e:
|
||||
name: ${{ inputs.test_suite }}
|
||||
name: ${{ inputs.test_suite }}${{ inputs.use_coverage_runner && format(' ({0})', matrix.partition) || '' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -238,6 +238,7 @@ jobs:
|
||||
SGLANG_IS_IN_CI: true
|
||||
TRANSFORMERS_VERBOSITY: "error"
|
||||
GDN_ATTN_BACKEND_TRITON: 1
|
||||
ONLY_TESTS: ${{ inputs.only_tests }}
|
||||
shell: bash
|
||||
run: |
|
||||
# Fail fast on any command error, undefined variable, or pipe failure.
|
||||
@@ -401,6 +402,16 @@ jobs:
|
||||
PARTITION_ARGS="--auto-partition-id ${PARTITION_ID} --auto-partition-size ${PARTITION_SIZE}"
|
||||
fi
|
||||
|
||||
# Precision-test mode: run only the coverage-recommended files for
|
||||
# this suite. The bucketed list comes from recommend-tests-from-
|
||||
# coverage; empty input keeps the full-suite behavior.
|
||||
ONLY_FILE_ARGS=""
|
||||
if [ -n "${ONLY_TESTS}" ]; then
|
||||
printf '%s\n' "${ONLY_TESTS}" > /tmp/precision_tests.txt
|
||||
ONLY_FILE_ARGS="--only-file-list /tmp/precision_tests.txt"
|
||||
echo "Precision mode: running $(wc -l < /tmp/precision_tests.txt) recommended file(s)"
|
||||
fi
|
||||
|
||||
# Append `--timeout-from-est‑time` when the job belongs to the nightly pipeline.
|
||||
# per‑case timeout is derived from each case's est_time.
|
||||
TIMEOUT_FLAG=""
|
||||
@@ -435,7 +446,7 @@ jobs:
|
||||
# Run NPU test suite with run_suite.py.
|
||||
# Capture mode (--enable-retry --max-attempts 1) keeps test subprocesses off the
|
||||
# tee pipe, so leftover server processes cannot block the pipeline on exit.
|
||||
${PYTHON_FOR_SGLANG} -u run_suite.py --hw npu --suite ${test_suite} ${NIGHTLY_FLAG} ${PARTITION_ARGS} ${TIMEOUT_FLAG} --enable-retry --max-attempts 1 --timeout-per-file 7200 2>&1 | tee "${LOG_TEE_TARGETS[@]}" || test_exit_code=$?
|
||||
${PYTHON_FOR_SGLANG} -u run_suite.py --hw npu --suite ${test_suite} ${NIGHTLY_FLAG} ${PARTITION_ARGS} ${ONLY_FILE_ARGS} ${TIMEOUT_FLAG} --enable-retry --max-attempts 1 --timeout-per-file 7200 2>&1 | tee "${LOG_TEE_TARGETS[@]}" || test_exit_code=$?
|
||||
|
||||
echo "Finished test: ${tc_name}"
|
||||
|
||||
@@ -467,12 +478,3 @@ jobs:
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
exit ${test_exit_code}
|
||||
|
||||
- name: Upload test logs
|
||||
if: always() && inputs.use_coverage_runner != true && inputs.upload_test_logs == true
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: selected-test-logs-${{ inputs.test_suite }}-part${{ matrix.partition }}
|
||||
path: /tmp/test-logs/
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
- "python/setup.py"
|
||||
- "python/sglang/!(multimodal_gen|kernels)/**/!(*.md)"
|
||||
# Diffusion-only kernels and tests are left to multimodal_gen and
|
||||
# jit_kernel; ops/diffusion/common stays since srt imports it via kda_kernels.
|
||||
# jit_kernel; ops/diffusion/common stays since kda_kernels (an srt path) imports it.
|
||||
- "python/sglang/kernels/{,ops/,jit/,jit/csrc/}!(*.md)"
|
||||
- "python/sglang/kernels/!(aot|ops|jit)/**/!(*.md)"
|
||||
- "python/sglang/kernels/jit/!(csrc)/**/!(*.md)"
|
||||
@@ -130,6 +130,7 @@ jobs:
|
||||
simulator:
|
||||
- ".github/workflows/pr-test-extra.yml"
|
||||
- ".github/workflows/_pr-test-simulator-cpu.yml"
|
||||
- ".github/workflows/_pr-test-simulator-gpu.yml"
|
||||
- "benchmark/simulator/**/!(*.md)"
|
||||
- "tools/sglang-simulator/**/!(*.md)"
|
||||
multimodal_gen:
|
||||
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- python-version: "3.10"
|
||||
- python-version: "3.11"
|
||||
cuda-version: "13.0"
|
||||
name: ${{ inputs.job_display_name }}
|
||||
steps:
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
run: |
|
||||
uv pip install -e "python" --index-strategy unsafe-best-match --prerelease allow
|
||||
uv pip install pytest
|
||||
uv pip install -e "tools/sglang-simulator[aic]"
|
||||
uv pip install "./tools/sglang-simulator[aic]"
|
||||
|
||||
- name: Prebuild HiCache native hash extension
|
||||
timeout-minutes: 5
|
||||
@@ -91,6 +91,7 @@ jobs:
|
||||
PYTEST_ADDOPTS: "-s"
|
||||
TORCH_EXTENSIONS_DIR: ${{ runner.temp }}/torch-extensions
|
||||
run: |
|
||||
OMP_NUM_THREADS=2 python3 -m pytest -q tools/sglang-simulator/test/test_train_latency_model.py
|
||||
python3 -m pytest -q tools/sglang-simulator/test/test_simulation_sglang_runner.py
|
||||
python3 -m pytest -q tools/sglang-simulator/test/test_simulation_sglang_serving.py
|
||||
python3 -m pytest -q tools/sglang-simulator/test/test_simulation_cache_hit_ratio.py
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
name: PR Test SGLang Simulator (GPU)
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
check_changes:
|
||||
description: 'toJson(needs.check-changes.outputs).'
|
||||
type: string
|
||||
required: true
|
||||
caller_inputs:
|
||||
description: 'toJson(inputs) from the caller workflow.'
|
||||
type: string
|
||||
required: true
|
||||
rust_ext_artifact:
|
||||
description: 'Artifact of prebuilt Rust extension modules.'
|
||||
type: string
|
||||
default: ''
|
||||
|
||||
env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ (fromJson(inputs.caller_inputs).skip_pr_test_health_check || fromJson(inputs.caller_inputs).test_parallel_dispatch || fromJson(inputs.caller_inputs).run_all_tests) && 'true' || 'false' }}
|
||||
PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
|
||||
USE_VENV: "1"
|
||||
|
||||
jobs:
|
||||
run:
|
||||
name: simulator-test-gpu
|
||||
if: fromJson(inputs.check_changes).main_package == 'true' || fromJson(inputs.check_changes).simulator == 'true' || fromJson(inputs.check_changes).sgl_kernel == 'true'
|
||||
runs-on: ${{ fromJson(fromJson(inputs.check_changes).runs_on_map)['1-gpu-small'] }}
|
||||
timeout-minutes: 40
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ fromJson(inputs.caller_inputs).git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
- name: Download kernel wheels
|
||||
if: fromJson(inputs.check_changes).sgl_kernel == 'true'
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.11-cuda*
|
||||
|
||||
- uses: ./.github/actions/download-rust-ext
|
||||
with:
|
||||
artifact_name: ${{ inputs.rust_ext_artifact }}
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
CUSTOM_BUILD_SGL_KERNEL: ${{ fromJson(inputs.check_changes).sgl_kernel }}
|
||||
run: bash scripts/ci/cuda/ci_install_dependency.sh
|
||||
|
||||
- name: Test baseline collection
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
|
||||
run: |
|
||||
uv pip install "./tools/sglang-simulator"
|
||||
python3 -m pytest -q tools/sglang-simulator/test/test_baseline_collection_sglang.py
|
||||
|
||||
- name: Cleanup venv
|
||||
if: always()
|
||||
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
||||
@@ -17,7 +17,7 @@ on:
|
||||
type: string
|
||||
required: true
|
||||
caller_inputs:
|
||||
description: 'toJson(inputs) from pr-test.yml. Read via fromJson(...).git_ref / skip_pr_test_health_check / test_parallel_dispatch / run_all_tests.'
|
||||
description: 'toJson(inputs) from pr-test.yml. Read via fromJson(...).git_ref / skip_pr_test_health_check / test_parallel_dispatch / run_all_tests / skip_cache_save.'
|
||||
type: string
|
||||
required: true
|
||||
partitions:
|
||||
@@ -110,7 +110,7 @@ jobs:
|
||||
with:
|
||||
workspaces: rust
|
||||
shared-key: "sglang-grpc-cpu"
|
||||
save-if: ${{ matrix.partition == 0 }}
|
||||
save-if: ${{ matrix.partition == 0 && !fromJson(inputs.caller_inputs).skip_cache_save }}
|
||||
|
||||
# uv pip targets a venv by default; setup-python has no venv — install into that interpreter (see UV_SYSTEM_PYTHON in https://docs.astral.sh/uv/guides/integration/github/)
|
||||
- name: Install dependencies
|
||||
@@ -156,9 +156,12 @@ jobs:
|
||||
python3 run_suite.py --hw cpu --suite ${{ inputs.self_name }} --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(inputs.partitions)[inputs.self_name].size }} --partition-model-file /tmp/partition-model.json --fork-worker-batch-size 20 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
# PRs save only what the base lacks; the implicit if: success() keeps a
|
||||
# half-downloaded cache unpublished.
|
||||
# half-downloaded cache unpublished. skip_cache_save is for callers that
|
||||
# dispatch on main but test PR code (run-cpu-ci.yml): a save there would
|
||||
# publish PR-written files under main's cache scope.
|
||||
- name: Decide whether to save the HF cache
|
||||
id: hf_cache_save
|
||||
if: ${{ !fromJson(inputs.caller_inputs).skip_cache_save }}
|
||||
run: |
|
||||
size=$(du -sb "$HF_HOME" 2>/dev/null | cut -f1)
|
||||
after=${size:-0}
|
||||
|
||||
@@ -147,7 +147,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda*
|
||||
pattern: wheel-python3.11-cuda*
|
||||
|
||||
- name: Download artifacts (v6)
|
||||
if: ${{ fromJson(inputs.check_changes).sgl_kernel == 'true' && steps.rc.outputs.artifact_version == 'v6' }}
|
||||
@@ -155,7 +155,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda*
|
||||
pattern: wheel-python3.11-cuda*
|
||||
|
||||
- uses: ./.github/actions/download-rust-ext
|
||||
if: ${{ !inputs.skip_prebuilt_rust_ext }}
|
||||
@@ -216,7 +216,7 @@ jobs:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ fromJson(inputs.check_changes).continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
RUNNER_LABELS: ${{ fromJson(fromJson(inputs.check_changes).runs_on_map)[inputs.runner_config] }}
|
||||
GPU_CONFIG: ${{ inputs.runner_config }}
|
||||
HF_MUSE_SPARK_TOKEN: ${{ inputs.self_name == 'base-c-test-8-gpu-b300' && secrets.HF_MUSE_SPARK_TOKEN || '' }}
|
||||
HF_MUSE_SPARK_TOKEN: ${{ (inputs.self_name == 'base-c-test-8-gpu-b300' || inputs.self_name == 'extra-b-test-8-gpu-b300') && secrets.HF_MUSE_SPARK_TOKEN || '' }}
|
||||
# Read only by test_runai_model_loader.py (nightly 1-gpu); inert elsewhere.
|
||||
# Left unconditional because an `env:` key cannot be omitted, and an empty
|
||||
# value would reach the runai streamer as a malformed limit.
|
||||
|
||||
@@ -58,11 +58,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
pip install tomli
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Configure Git and branch
|
||||
run: |
|
||||
|
||||
@@ -24,11 +24,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
pip install tomli
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Configure Git and branch
|
||||
run: |
|
||||
|
||||
@@ -22,11 +22,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
pip install tomli
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Check if sync is needed
|
||||
id: check_sync
|
||||
|
||||
@@ -24,11 +24,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
pip install tomli
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Configure Git and branch
|
||||
run: |
|
||||
|
||||
@@ -26,11 +26,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
pip install tomli
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Configure Git and branch
|
||||
id: set_output
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Generate Summary Report
|
||||
run: |
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Generate Tests by Folder Report
|
||||
run: |
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Generate Tests by Suite Report
|
||||
run: |
|
||||
@@ -73,7 +73,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Generate Tests by GPU Count Report
|
||||
run: |
|
||||
@@ -184,7 +184,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Generate JSON Report
|
||||
run: |
|
||||
|
||||
@@ -1,15 +1,17 @@
|
||||
# Coverage data collection pipeline for the NPU precision test selection.
|
||||
# Coverage baseline collection for the NPU precision test selection.
|
||||
# Delegates to pr-test-npu.yml with coverage_mode=true, which switches:
|
||||
# - Runners to the coverage pool (shared /root/.cache with setup-covstub)
|
||||
# - use_coverage_runner=true on all test jobs
|
||||
# - Skips pr-gate, multimodal-gen, recommend, analyze
|
||||
# - Runs setup-covstub to assemble coverage data and publish it to the
|
||||
# shared disk at a fixed path (/root/.cache/tests/precise-test/coverage)
|
||||
# pr-test-npu-finish is the final gate in the called workflow.
|
||||
#
|
||||
# Manually triggered only (workflow_dispatch): PRs run the standard
|
||||
# pr-test-npu.yml instead; this pipeline builds the full coverage baseline
|
||||
# on demand.
|
||||
# - Test jobs to the coverage pool with use_coverage_runner=true
|
||||
# - base-a to a hosted runner (base-a-test-npu-cov, data merged via artifact)
|
||||
# - Runs the multimodal-gen suites too. Their runner is asked for this
|
||||
# shard's targets (case-level pytest node ids) and each target is run in
|
||||
# its own coverage invocation, so diffusion coverage is attributed per
|
||||
# test case instead of per shard.
|
||||
# - Skips pr-gate, recommend
|
||||
# - setup-covstub assembles and publishes the baseline to the shared disk
|
||||
# at /root/.cache/tests/precise-test/coverage
|
||||
# pr-test-npu-finish is the final gate in the called workflow. Manually
|
||||
# triggered only (workflow_dispatch): PRs run the standard pr-test-npu.yml
|
||||
# instead; this pipeline builds the full coverage baseline on demand.
|
||||
name: Coverage Collection (NPU)
|
||||
|
||||
on:
|
||||
|
||||
@@ -24,7 +24,7 @@ on:
|
||||
default: ''
|
||||
type: string
|
||||
kernel_artifact_run_id:
|
||||
description: "Optional sgl-kernel wheel source: GitHub Actions run ID of a pr-test.yml run on the SAME commit as 'ref' that uploaded wheel-python3.10-cuda13.0. Use only when this branch's torch/ABI change makes the PyPI sglang-kernel wheel incompatible. Find it under Actions > pr-test; artifacts expire after 90 days."
|
||||
description: "Optional sgl-kernel wheel source: GitHub Actions run ID of a pr-test.yml run on the SAME commit as 'ref' that uploaded wheel-python3.11-cuda13.0. Use only when this branch's torch/ABI change makes the PyPI sglang-kernel wheel incompatible. Find it under Actions > pr-test; artifacts expire after 90 days."
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
@@ -204,7 +204,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
name: wheel-python3.10-cuda13.0
|
||||
name: wheel-python3.11-cuda13.0
|
||||
run-id: ${{ inputs.kernel_artifact_run_id }}
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -351,7 +351,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Verify write access to sgl-project/ci-data-diffusion
|
||||
if: ${{ inputs.publish }}
|
||||
@@ -403,7 +403,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
name: wheel-python3.10-cuda13.0
|
||||
name: wheel-python3.11-cuda13.0
|
||||
run-id: ${{ inputs.kernel_artifact_run_id }}
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -479,7 +479,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
name: wheel-python3.10-cuda13.0
|
||||
name: wheel-python3.11-cuda13.0
|
||||
run-id: ${{ inputs.kernel_artifact_run_id }}
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -557,7 +557,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
name: wheel-python3.10-cuda13.0
|
||||
name: wheel-python3.11-cuda13.0
|
||||
run-id: ${{ inputs.kernel_artifact_run_id }}
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -630,7 +630,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
name: wheel-python3.10-cuda13.0
|
||||
name: wheel-python3.11-cuda13.0
|
||||
run-id: ${{ inputs.kernel_artifact_run_id }}
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
name: Nightly Experimental sgl-router Docker Image
|
||||
name: Nightly sgl-router Docker Image
|
||||
|
||||
on:
|
||||
schedule:
|
||||
@@ -34,15 +34,18 @@ jobs:
|
||||
- suite: nightly-stage-c-2-gpu-mi350
|
||||
runner: linux-mi300-2gpu-sglang
|
||||
timeout: 180
|
||||
# Single-request decode of Qwen3.8-27B-FP8 runs at ~20 tok/s on MI300X, so long
|
||||
# generations outlast the test client's 180 s request timeout.
|
||||
skip_tests: tests/e2e/sglang/test_session_server_multi_role/test_qwen38small.py
|
||||
# test_qwen38small: single-request decode of Qwen3.8-27B-FP8 runs at ~20 tok/s on
|
||||
# MI300X, so long generations outlast the test client's 180 s request timeout.
|
||||
# test_glm5_2_744b_a40b_5layer_ci: its tilelang indexer kernel needs 98 KiB of LDS
|
||||
# per block; gfx942 has 64 KiB.
|
||||
skip_tests: >-
|
||||
tests/e2e/sglang/test_session_server_multi_role/test_qwen38small.py
|
||||
tests/e2e/megatron/model_scripts/test_glm5_2_744b_a40b_5layer_ci.py
|
||||
# The 4- and 8-GPU suites run past 180 min on MI300X; they take ~155 and ~112 min on MI355X.
|
||||
- suite: nightly-stage-c-4-gpu-mi350
|
||||
runner: linux-mi300-4gpu-sglang
|
||||
timeout: 300
|
||||
# Its tilelang indexer kernel needs 98 KiB of LDS per block; gfx942 has 64 KiB.
|
||||
skip_tests: tests/e2e/megatron/model_scripts/test_glm5_2_744b_a40b_5layer_ci.py
|
||||
skip_tests: ""
|
||||
- suite: nightly-stage-c-8-gpu-mi350
|
||||
runner: linux-mi300-8gpu-sglang
|
||||
timeout: 300
|
||||
|
||||
@@ -24,15 +24,28 @@ concurrency:
|
||||
jobs:
|
||||
test-miles-mi35x:
|
||||
if: github.repository == 'sgl-project/sglang'
|
||||
name: test-miles-mi35x (${{ matrix.suite }}, ${{ matrix.runner }}${{ matrix.partition_size && format(', part {0}', matrix.part) || '' }})
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# All suites use the 8-GPU MI35X pool. The 2-GPU suite includes the
|
||||
# GLM-5.2 5-layer test, which takes the linux-mi35x-gpu-2 runner down
|
||||
# ("runner lost communication") but passes on an 8-GPU runner.
|
||||
include:
|
||||
- suite: nightly-stage-c-2-gpu-mi350
|
||||
runner: linux-mi35x-gpu-2
|
||||
# Use 8-GPU MI35X pool for 4-GPU suite.
|
||||
runner: linux-mi35x-gpu-8
|
||||
# The 4-GPU suite is ~6 h of estimated test time; split it so each
|
||||
# half fits in the step timeout.
|
||||
- suite: nightly-stage-c-4-gpu-mi350
|
||||
runner: linux-mi35x-gpu-8
|
||||
partition_id: 0
|
||||
partition_size: 2
|
||||
part: 1/2
|
||||
- suite: nightly-stage-c-4-gpu-mi350
|
||||
runner: linux-mi35x-gpu-8
|
||||
partition_id: 1
|
||||
partition_size: 2
|
||||
part: 2/2
|
||||
- suite: nightly-stage-c-8-gpu-mi350
|
||||
runner: linux-mi35x-gpu-8
|
||||
runs-on: ${{ matrix.runner }}
|
||||
@@ -88,6 +101,7 @@ jobs:
|
||||
-e PYTHONPATH=/root/miles:/opt/tilelang \
|
||||
-e GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
|
||||
python3 tests/ci/run_suite.py --hw rocm --suite ${{ matrix.suite }} --cadence nightly \
|
||||
${{ matrix.partition_size && format('--auto-partition-id {0} --auto-partition-size {1}', matrix.partition_id, matrix.partition_size) || '' }} \
|
||||
${{ (github.event_name != 'workflow_dispatch' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
|
||||
cat "${GITHUB_WORKSPACE}/github_summary.md" >> "$GITHUB_STEP_SUMMARY" || true
|
||||
exit ${TEST_EXIT_CODE}
|
||||
|
||||
@@ -1000,6 +1000,11 @@ jobs:
|
||||
# HF download env vars:
|
||||
# - HF_HUB_ENABLE_HF_TRANSFER=1: Use faster hf_transfer for downloads (if available)
|
||||
# - HF_HUB_DISABLE_SYMLINKS_WARNING=1: Suppress symlink warnings
|
||||
#
|
||||
# Skips ltx_2_5_diffusion_decoder_2gpus: on mi300 its server has never
|
||||
# finished warmup inside the 1200s startup deadline, and the harness
|
||||
# retries that TimeoutError six times, running the shard into the
|
||||
# step timeout.
|
||||
docker exec \
|
||||
-e HF_TOKEN \
|
||||
-e HUGGING_FACE_HUB_TOKEN \
|
||||
@@ -1019,6 +1024,7 @@ jobs:
|
||||
--suite 2-gpu \
|
||||
--partition-id ${{ matrix.part }} \
|
||||
--total-partitions 2 \
|
||||
-k "not ltx_2_5_diffusion_decoder" \
|
||||
${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
# Post-test diagnostics
|
||||
@@ -1456,9 +1462,13 @@ jobs:
|
||||
run: |
|
||||
docker exec ci_sglang rocm-smi --showtopotype || true
|
||||
docker exec ci_sglang bash -c 'strings $(python3 -c "import mooncake,os;print(os.path.dirname(mooncake.__file__))")/engine*.so | grep -iE "rdma,hip|Unsupported multi-protocol"' || true
|
||||
# With Mooncake v0.3.14-rc1, registering the KV cache through HIP dmabuf
|
||||
# leaves the server's next HIP stream creation failing (out of memory on
|
||||
# rocm10, segfault on rocm720). Register through ibv_reg_mr instead.
|
||||
bash scripts/ci/amd/amd_ci_exec.sh \
|
||||
-e SGLANG_TEST_RDMA_DEVICE="${{ env.SGLANG_TEST_RDMA_DEVICE }}" \
|
||||
-e MC_LOG_LEVEL=TRACE \
|
||||
-e MOONCAKE_DISABLE_HIP_DMABUF=1 \
|
||||
-e GLOG_logtostderr=1 \
|
||||
-w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
|
||||
|
||||
@@ -150,6 +150,15 @@ jobs:
|
||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
||||
# No `secrets: inherit`: this hosted CPU job has no secret consumer.
|
||||
|
||||
simulator-test-gpu:
|
||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
|
||||
uses: ./.github/workflows/_pr-test-simulator-gpu.yml
|
||||
with:
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
||||
|
||||
# =============================================== extra-a / extra-b ===============================================
|
||||
# `name:` keeps the job names wait-for-jobs and check-pr-test-health match on.
|
||||
extra-test:
|
||||
@@ -191,6 +200,7 @@ jobs:
|
||||
sgl-kernel-build-wheels,
|
||||
rust-ext-build,
|
||||
simulator-test-cpu,
|
||||
simulator-test-gpu,
|
||||
extra-test,
|
||||
]
|
||||
if: always()
|
||||
|
||||
@@ -99,7 +99,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda13.0
|
||||
pattern: wheel-python3.11-cuda13.0
|
||||
|
||||
- uses: ./.github/actions/download-rust-ext
|
||||
with:
|
||||
|
||||
@@ -65,7 +65,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Compute partitions
|
||||
id: compute
|
||||
@@ -101,7 +101,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda*
|
||||
pattern: wheel-python3.11-cuda*
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
@@ -168,13 +168,19 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda*
|
||||
pattern: wheel-python3.11-cuda*
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
|
||||
|
||||
- name: Run SM120-only diffusion unit tests
|
||||
timeout-minutes: 10
|
||||
working-directory: python
|
||||
run: |
|
||||
python3 -m pytest -v sglang/multimodal_gen/test/unit/test_fp8_fa_sm120_attn.py
|
||||
|
||||
- name: Run 5090 diffusion canary tests
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
@@ -233,7 +239,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda*
|
||||
pattern: wheel-python3.11-cuda*
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
@@ -298,7 +304,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda*
|
||||
pattern: wheel-python3.11-cuda*
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
@@ -368,7 +374,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda*
|
||||
pattern: wheel-python3.11-cuda*
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
@@ -413,7 +419,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda*
|
||||
pattern: wheel-python3.11-cuda*
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
@@ -427,6 +433,22 @@ jobs:
|
||||
run: |
|
||||
python3 -m pytest -v python/sglang/multimodal_gen/test/single_test_file/test_wan_vae_parallel_groups_4_gpu.py
|
||||
|
||||
- name: Run Blackwell-only diffusion unit tests
|
||||
timeout-minutes: 10
|
||||
working-directory: python
|
||||
run: |
|
||||
python3 -m pytest -v \
|
||||
sglang/multimodal_gen/test/unit/test_mxfp8_online_gemm.py \
|
||||
sglang/multimodal_gen/test/unit/test_wan_gelu_mlp.py
|
||||
|
||||
- name: Run 4-GPU diffusion unit tests
|
||||
timeout-minutes: 15
|
||||
working-directory: python
|
||||
run: |
|
||||
python3 -m pytest -v \
|
||||
sglang/multimodal_gen/test/unit/test_kandinsky6_sequence_parallel.py::test_joint_video_audio_tensor_sequence_parallel \
|
||||
sglang/multimodal_gen/test/unit/test_kandinsky6_sr_sequence_parallel.py::test_sr_four_gpus
|
||||
|
||||
- name: Run diffusion server tests
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
@@ -483,13 +505,20 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda*
|
||||
pattern: wheel-python3.11-cuda*
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
|
||||
|
||||
- name: Install Quant-VideoGen
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
# quant-videogen pins an incompatible torch, so install it without deps
|
||||
# plus the import-time deps of the diffusion-qvg extra.
|
||||
uv pip install --no-deps quant-videogen==0.1.0 "loguru>=0.7" "termcolor>=2.3"
|
||||
|
||||
- name: Run diffusion unit tests
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
@@ -519,7 +548,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Download all execution reports
|
||||
uses: actions/download-artifact@v4
|
||||
|
||||
+596
-108
@@ -1,11 +1,13 @@
|
||||
name: PR Test (NPU)
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
schedule:
|
||||
- cron: '0 12 * * *' # Run daily at 12:00 UTC
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- 'npu/**'
|
||||
- 'npu-*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
coverage_mode:
|
||||
@@ -169,9 +171,84 @@ jobs:
|
||||
cd test
|
||||
python3 run_suite.py --hw npu --suite base-a-test-npu
|
||||
|
||||
# Coverage-mode-only counterpart of the base-a PR gate above. Stays on
|
||||
# ubuntu-latest like the gate (its UTs mock the NPU stack and need a CPU
|
||||
# triton; the NPU image's triton-ascend breaks those mocks), so the shared
|
||||
# template (unconditional CANN container:) cannot be used and coverage
|
||||
# data reaches setup-covstub via artifact instead of the shared disk.
|
||||
base-a-test-npu-cov:
|
||||
needs: [set-image-config]
|
||||
if: ${{ inputs.coverage_mode == true }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Pin to the event SHA so this job tests the same commit as the rest of the run.
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: 'python/pyproject_npu.toml'
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
# uv pip targets a venv by default; setup-python has no venv.
|
||||
UV_SYSTEM_PYTHON: "1"
|
||||
run: |
|
||||
# Same dependency set as the base-a PR gate above (CPU torch + CPU
|
||||
# triton, no Ascend wheels); keep the two blocks in sync.
|
||||
rm -f python/pyproject.toml
|
||||
mv python/pyproject_npu.toml python/pyproject.toml
|
||||
uv pip install torch==2.10.0 torchvision==0.25.0 torchaudio==2.10.0 triton \
|
||||
--index-url https://download.pytorch.org/whl/cpu \
|
||||
--extra-index-url https://pypi.org/simple/ \
|
||||
--index-strategy unsafe-best-match
|
||||
uv pip install -e "python[dev_npu]" --index-strategy unsafe-best-match
|
||||
pip install coverage==7.8.*
|
||||
|
||||
- name: Run test with coverage
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
# Hosted runners cannot write the shared disk, so stage locally;
|
||||
# run_tests_with_coverage.sh derives RUN_DIR/date tag from run-level
|
||||
# env vars, so the artifact merges 1:1 into the shared-disk layout.
|
||||
export PRECISE_TEST_CACHE_ROOT=/tmp/precise-test
|
||||
cd test
|
||||
python3 ../scripts/ci/npu/precise-test/list_tests.py --hw npu --suite base-a-test-npu \
|
||||
-o /tmp/selected_tests.txt
|
||||
echo "Selected test files:"
|
||||
cat /tmp/selected_tests.txt
|
||||
if [ -s /tmp/selected_tests.txt ]; then
|
||||
bash ../scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
|
||||
fi
|
||||
|
||||
- name: Upload coverage outputs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-outputs-base-a-test-npu
|
||||
path: /tmp/precise-test/
|
||||
retention-days: 1
|
||||
if-no-files-found: warn
|
||||
|
||||
base-b-test-1-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu, base-a-test-npu-cov, recommend-tests-from-coverage]
|
||||
if: >-
|
||||
${{ !failure() && !cancelled() &&
|
||||
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
||||
needs.recommend-tests-from-coverage.outputs.should_run_base_b_test_1_npu_a3 != 'false' }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
self_name: base-b-test-1-npu-a3
|
||||
@@ -179,13 +256,16 @@ jobs:
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
run_timeout_minutes: '60'
|
||||
timeout_per_file: '3600'
|
||||
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_b_test_1_npu_a3 }}
|
||||
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
||||
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
||||
secrets: inherit
|
||||
|
||||
base-b-test-2-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu, base-a-test-npu-cov, recommend-tests-from-coverage]
|
||||
if: >-
|
||||
${{ !failure() && !cancelled() &&
|
||||
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
||||
needs.recommend-tests-from-coverage.outputs.should_run_base_b_test_2_npu_a3 != 'false' }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
self_name: base-b-test-2-npu-a3
|
||||
@@ -193,13 +273,16 @@ jobs:
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
run_timeout_minutes: '60'
|
||||
timeout_per_file: '3600'
|
||||
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_b_test_2_npu_a3 }}
|
||||
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
||||
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
||||
secrets: inherit
|
||||
|
||||
base-b-test-4-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu, base-a-test-npu-cov, recommend-tests-from-coverage]
|
||||
if: >-
|
||||
${{ !failure() && !cancelled() &&
|
||||
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
||||
needs.recommend-tests-from-coverage.outputs.should_run_base_b_test_4_npu_a3 != 'false' }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
self_name: base-b-test-4-npu-a3
|
||||
@@ -208,13 +291,16 @@ jobs:
|
||||
run_timeout_minutes: '120'
|
||||
timeout_per_file: '3600'
|
||||
partitions: '{"size":2,"arr":[0, 1]}'
|
||||
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_b_test_4_npu_a3 }}
|
||||
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
||||
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
||||
secrets: inherit
|
||||
|
||||
base-b-test-8-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu, base-a-test-npu-cov, recommend-tests-from-coverage]
|
||||
if: >-
|
||||
${{ !failure() && !cancelled() &&
|
||||
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
||||
needs.recommend-tests-from-coverage.outputs.should_run_base_b_test_8_npu_a3 != 'false' }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
self_name: base-b-test-8-npu-a3
|
||||
@@ -222,13 +308,16 @@ jobs:
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
run_timeout_minutes: '60'
|
||||
timeout_per_file: '3600'
|
||||
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_b_test_8_npu_a3 }}
|
||||
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
||||
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
||||
secrets: inherit
|
||||
|
||||
base-b-test-16-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu, base-a-test-npu-cov, recommend-tests-from-coverage]
|
||||
if: >-
|
||||
${{ !failure() && !cancelled() &&
|
||||
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
||||
needs.recommend-tests-from-coverage.outputs.should_run_base_b_test_16_npu_a3 != 'false' }}
|
||||
uses: ./.github/workflows/_npu-pr-test-stage.yml
|
||||
with:
|
||||
self_name: base-b-test-16-npu-a3
|
||||
@@ -236,13 +325,24 @@ jobs:
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
run_timeout_minutes: '120'
|
||||
timeout_per_file: '3600'
|
||||
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_b_test_16_npu_a3 }}
|
||||
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
||||
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
||||
secrets: inherit
|
||||
|
||||
multimodal-gen-test-1-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
|
||||
needs:
|
||||
[
|
||||
check-changes,
|
||||
pr-gate,
|
||||
set-image-config,
|
||||
base-a-test-npu,
|
||||
base-a-test-npu-cov,
|
||||
recommend-tests-from-coverage,
|
||||
]
|
||||
if: >-
|
||||
${{ !failure() && !cancelled() &&
|
||||
(inputs.coverage_mode == true || needs.check-changes.outputs.multimodal_gen == 'true') &&
|
||||
needs.recommend-tests-from-coverage.outputs.should_run_multimodal_gen_test_1_npu_a3 != 'false' }}
|
||||
runs-on: linux-aarch64-a3-800t-2
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -290,7 +390,8 @@ jobs:
|
||||
run: python3 scripts/ci/utils/diffusion/prefetch_input_image.py
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: ${{ inputs.coverage_mode == true && 150 || 60 }}
|
||||
shell: bash
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
SGLANG_IS_IN_CI: true
|
||||
@@ -299,12 +400,75 @@ jobs:
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
STREAMS_PER_DEVICE: 32
|
||||
SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
|
||||
ONLY_TESTS: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_multimodal_gen_test_1_npu_a3 }}
|
||||
run: |
|
||||
# Coverage mode: enumerate this shard's targets with an NPU-only CI
|
||||
# helper (it reuses the shared diffusion runner read-only) and run
|
||||
# each target in its own `coverage run -m pytest` invocation. The
|
||||
# runner normally batches a whole shard into one pytest call, which
|
||||
# cannot be attributed per test case by the precise-test selector.
|
||||
if [[ "${{ inputs.coverage_mode }}" == "true" ]]; then
|
||||
set -euo pipefail
|
||||
pip install coverage==7.8.*
|
||||
# Coverage tracing slows down the Python glue, which shows up in
|
||||
# wall-clock assertions. Relax only the timing tolerances for the
|
||||
# collection run; PR runs (no coverage) keep the strict profile and
|
||||
# the memory/VRAM checks are untouched.
|
||||
export SGLANG_E2E_TOLERANCE=1.0
|
||||
export SGLANG_STAGE_TIME_TOLERANCE=1.0
|
||||
export SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=2.0
|
||||
export SGLANG_DENOISE_STEP_TOLERANCE=1.0
|
||||
export SGLANG_DENOISE_AGG_TOLERANCE=1.0
|
||||
python3 scripts/ci/npu/precise-test/list_multimodal_tests.py \
|
||||
--suite 1-npu \
|
||||
--partition-id ${{ matrix.part }} \
|
||||
--total-partitions 2 \
|
||||
-o /tmp/selected_tests.txt
|
||||
echo "Selected test targets:"
|
||||
cat /tmp/selected_tests.txt
|
||||
if [ -s /tmp/selected_tests.txt ]; then
|
||||
bash scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Precision mode: ONLY_TESTS holds one pytest -k token per line
|
||||
# ('[case_id]' for parametrized cases, a file stem for standalone
|
||||
# files). It is empty when nothing was recommended, precision mode
|
||||
# is off, or a safety gate forced the full suite; then no filter is
|
||||
# applied and the shard runs its whole assignment.
|
||||
FILTER=""
|
||||
while IFS= read -r line; do
|
||||
if [ -z "${line}" ]; then
|
||||
continue
|
||||
fi
|
||||
# Every line must be a -k token ('[case_id]' or a standalone file
|
||||
# stem). A path means the bucket contract drifted: with a path the
|
||||
# -k expression matches nothing, pytest exits 5, and the shard
|
||||
# would report success after running zero tests.
|
||||
if [[ "${line}" == */* || "${line}" == *.py ]]; then
|
||||
echo "::error::Unexpected only_tests entry '${line}' (expected a pytest -k token)"
|
||||
exit 1
|
||||
fi
|
||||
if [ -n "${FILTER}" ]; then
|
||||
FILTER="${FILTER} or "
|
||||
fi
|
||||
FILTER="${FILTER}${line}"
|
||||
done <<< "${ONLY_TESTS}"
|
||||
FILTER_ARGS=()
|
||||
if [ -n "${FILTER}" ]; then
|
||||
FILTER_ARGS+=(-k "${FILTER}")
|
||||
echo "Precision mode: filtered to ${FILTER}"
|
||||
else
|
||||
echo "Precision mode: no filter (full suite)"
|
||||
fi
|
||||
|
||||
cd python
|
||||
python3 sglang/multimodal_gen/test/run_suite.py \
|
||||
--suite 1-npu \
|
||||
--partition-id ${{ matrix.part }} \
|
||||
--total-partitions 2
|
||||
--total-partitions 2 \
|
||||
"${FILTER_ARGS[@]}"
|
||||
|
||||
- name: Upload diffusion failure artifacts
|
||||
if: always()
|
||||
@@ -316,8 +480,19 @@ jobs:
|
||||
retention-days: 7
|
||||
|
||||
multimodal-gen-test-4-npu-a3:
|
||||
needs: [check-changes, pr-gate, set-image-config, base-a-test-npu]
|
||||
if: ${{ !failure() && !cancelled() && inputs.coverage_mode != true && needs.check-changes.outputs.multimodal_gen == 'true' }}
|
||||
needs:
|
||||
[
|
||||
check-changes,
|
||||
pr-gate,
|
||||
set-image-config,
|
||||
base-a-test-npu,
|
||||
base-a-test-npu-cov,
|
||||
recommend-tests-from-coverage,
|
||||
]
|
||||
if: >-
|
||||
${{ !failure() && !cancelled() &&
|
||||
(inputs.coverage_mode == true || needs.check-changes.outputs.multimodal_gen == 'true') &&
|
||||
needs.recommend-tests-from-coverage.outputs.should_run_multimodal_gen_test_4_npu_a3 != 'false' }}
|
||||
runs-on: linux-aarch64-a3-800t-4
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -360,7 +535,8 @@ jobs:
|
||||
python3 scripts/ci/utils/diffusion/prefetch_consistency_gt.py
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: ${{ inputs.coverage_mode == true && 150 || 60 }}
|
||||
shell: bash
|
||||
env:
|
||||
SGLANG_USE_MODELSCOPE: true
|
||||
SGLANG_IS_IN_CI: true
|
||||
@@ -369,12 +545,75 @@ jobs:
|
||||
PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
|
||||
STREAMS_PER_DEVICE: 32
|
||||
SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
|
||||
ONLY_TESTS: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_multimodal_gen_test_4_npu_a3 }}
|
||||
run: |
|
||||
# Coverage mode: enumerate this shard's targets with an NPU-only CI
|
||||
# helper (it reuses the shared diffusion runner read-only) and run
|
||||
# each target in its own `coverage run -m pytest` invocation. The
|
||||
# runner normally batches a whole shard into one pytest call, which
|
||||
# cannot be attributed per test case by the precise-test selector.
|
||||
if [[ "${{ inputs.coverage_mode }}" == "true" ]]; then
|
||||
set -euo pipefail
|
||||
pip install coverage==7.8.*
|
||||
# Coverage tracing slows down the Python glue, which shows up in
|
||||
# wall-clock assertions. Relax only the timing tolerances for the
|
||||
# collection run; PR runs (no coverage) keep the strict profile and
|
||||
# the memory/VRAM checks are untouched.
|
||||
export SGLANG_E2E_TOLERANCE=1.0
|
||||
export SGLANG_STAGE_TIME_TOLERANCE=1.0
|
||||
export SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE=2.0
|
||||
export SGLANG_DENOISE_STEP_TOLERANCE=1.0
|
||||
export SGLANG_DENOISE_AGG_TOLERANCE=1.0
|
||||
python3 scripts/ci/npu/precise-test/list_multimodal_tests.py \
|
||||
--suite 2-npu \
|
||||
--partition-id ${{ matrix.part }} \
|
||||
--total-partitions 2 \
|
||||
-o /tmp/selected_tests.txt
|
||||
echo "Selected test targets:"
|
||||
cat /tmp/selected_tests.txt
|
||||
if [ -s /tmp/selected_tests.txt ]; then
|
||||
bash scripts/ci/npu/precise-test/run_tests_with_coverage.sh $(cat /tmp/selected_tests.txt)
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Precision mode: ONLY_TESTS holds one pytest -k token per line
|
||||
# ('[case_id]' for parametrized cases, a file stem for standalone
|
||||
# files). It is empty when nothing was recommended, precision mode
|
||||
# is off, or a safety gate forced the full suite; then no filter is
|
||||
# applied and the shard runs its whole assignment.
|
||||
FILTER=""
|
||||
while IFS= read -r line; do
|
||||
if [ -z "${line}" ]; then
|
||||
continue
|
||||
fi
|
||||
# Every line must be a -k token ('[case_id]' or a standalone file
|
||||
# stem). A path means the bucket contract drifted: with a path the
|
||||
# -k expression matches nothing, pytest exits 5, and the shard
|
||||
# would report success after running zero tests.
|
||||
if [[ "${line}" == */* || "${line}" == *.py ]]; then
|
||||
echo "::error::Unexpected only_tests entry '${line}' (expected a pytest -k token)"
|
||||
exit 1
|
||||
fi
|
||||
if [ -n "${FILTER}" ]; then
|
||||
FILTER="${FILTER} or "
|
||||
fi
|
||||
FILTER="${FILTER}${line}"
|
||||
done <<< "${ONLY_TESTS}"
|
||||
FILTER_ARGS=()
|
||||
if [ -n "${FILTER}" ]; then
|
||||
FILTER_ARGS+=(-k "${FILTER}")
|
||||
echo "Precision mode: filtered to ${FILTER}"
|
||||
else
|
||||
echo "Precision mode: no filter (full suite)"
|
||||
fi
|
||||
|
||||
cd python
|
||||
python3 sglang/multimodal_gen/test/run_suite.py \
|
||||
--suite 2-npu \
|
||||
--partition-id ${{ matrix.part }} \
|
||||
--total-partitions 2
|
||||
--total-partitions 2 \
|
||||
"${FILTER_ARGS[@]}"
|
||||
|
||||
- name: Upload diffusion failure artifacts
|
||||
if: always()
|
||||
@@ -387,8 +626,11 @@ jobs:
|
||||
|
||||
base-c-test-acc-2-npu-a3:
|
||||
name: base-c-test-acc-2-npu-a3
|
||||
needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3 ]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3, recommend-tests-from-coverage ]
|
||||
if: >-
|
||||
${{ !failure() && !cancelled() &&
|
||||
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
||||
needs.recommend-tests-from-coverage.outputs.should_run_base_c_test_acc_2_npu_a3 != 'false' }}
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-2' || 'linux-aarch64-a3-2-' }}
|
||||
@@ -398,13 +640,16 @@ jobs:
|
||||
install_sglang_deps: true
|
||||
device_type_for_deps: 'a3'
|
||||
partitions: '{"size":2,"arr":[0,1]}'
|
||||
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_c_test_acc_2_npu_a3 }}
|
||||
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
||||
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
||||
|
||||
base-c-test-acc-16-npu-a3:
|
||||
name: base-c-test-acc-16-npu-a3
|
||||
needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3 ]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
needs: [ check-changes, pr-gate, set-image-config, base-b-test-1-npu-a3, base-b-test-2-npu-a3, base-b-test-4-npu-a3, base-b-test-8-npu-a3, base-b-test-16-npu-a3, recommend-tests-from-coverage ]
|
||||
if: >-
|
||||
${{ !failure() && !cancelled() &&
|
||||
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
||||
needs.recommend-tests-from-coverage.outputs.should_run_base_c_test_acc_16_npu_a3 != 'false' }}
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: ${{ inputs.coverage_mode == true && 'linux-aarch64-a3-800t-16' || 'linux-aarch64-a3-16-' }}
|
||||
@@ -413,13 +658,16 @@ jobs:
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
install_sglang_deps: true
|
||||
device_type_for_deps: 'a3'
|
||||
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_c_test_acc_16_npu_a3 }}
|
||||
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
||||
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
||||
|
||||
base-c-test-perf-2-npu-a3:
|
||||
name: base-c-test-perf-2-npu-a3
|
||||
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3 ]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3, recommend-tests-from-coverage ]
|
||||
if: >-
|
||||
${{ !failure() && !cancelled() &&
|
||||
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
||||
needs.recommend-tests-from-coverage.outputs.should_run_base_c_test_perf_2_npu_a3 != 'false' }}
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: linux-aarch64-a3-800t-2
|
||||
@@ -428,13 +676,16 @@ jobs:
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
install_sglang_deps: true
|
||||
device_type_for_deps: 'a3'
|
||||
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_c_test_perf_2_npu_a3 }}
|
||||
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
||||
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
||||
|
||||
base-c-test-perf-16-npu-a3:
|
||||
name: base-c-test-perf-16-npu-a3
|
||||
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3 ]
|
||||
if: ${{ !failure() && !cancelled() && (inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') }}
|
||||
needs: [ check-changes, pr-gate, set-image-config, base-c-test-acc-2-npu-a3, base-c-test-acc-16-npu-a3, recommend-tests-from-coverage ]
|
||||
if: >-
|
||||
${{ !failure() && !cancelled() &&
|
||||
(inputs.coverage_mode == true || needs.check-changes.outputs.main_package == 'true') &&
|
||||
needs.recommend-tests-from-coverage.outputs.should_run_base_c_test_perf_16_npu_a3 != 'false' }}
|
||||
uses: ./.github/workflows/_npu-single-node-test-stage.yml
|
||||
with:
|
||||
runner: linux-aarch64-a3-800t-16
|
||||
@@ -444,8 +695,8 @@ jobs:
|
||||
install_sglang_deps: true
|
||||
device_type_for_deps: 'a3'
|
||||
test_timeout_minutes: '180'
|
||||
only_tests: ${{ needs.recommend-tests-from-coverage.outputs.only_tests_base_c_test_perf_16_npu_a3 }}
|
||||
use_coverage_runner: ${{ inputs.coverage_mode == true }}
|
||||
upload_test_logs: ${{ inputs.coverage_mode != true }}
|
||||
|
||||
|
||||
# ==================== Recommend Tests from Coverage ==================== #
|
||||
@@ -453,25 +704,63 @@ jobs:
|
||||
# continue-on-error so it never blocks the PR gate.
|
||||
recommend-tests-from-coverage:
|
||||
name: Recommend tests from coverage
|
||||
needs: [check-changes, pr-gate]
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
# TEMP-DISABLED: OBS coverage baseline returns 403; re-enable once fixed.
|
||||
if: false
|
||||
# if: ${{ !cancelled() && github.event_name == 'pull_request' && inputs.coverage_mode != true }}
|
||||
if: >-
|
||||
${{ !failure() && !cancelled() &&
|
||||
github.event_name == 'pull_request' &&
|
||||
inputs.coverage_mode != true &&
|
||||
needs.check-changes.outputs.changes_exist == 'true' }}
|
||||
outputs:
|
||||
coverage_paths: ${{ steps.export-paths.outputs.coverage_paths }}
|
||||
should_run_base_b_test_1_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_b_test_1_npu_a3 }}
|
||||
only_tests_base_b_test_1_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_b_test_1_npu_a3 }}
|
||||
should_run_base_b_test_2_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_b_test_2_npu_a3 }}
|
||||
only_tests_base_b_test_2_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_b_test_2_npu_a3 }}
|
||||
should_run_base_b_test_4_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_b_test_4_npu_a3 }}
|
||||
only_tests_base_b_test_4_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_b_test_4_npu_a3 }}
|
||||
should_run_base_b_test_8_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_b_test_8_npu_a3 }}
|
||||
only_tests_base_b_test_8_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_b_test_8_npu_a3 }}
|
||||
should_run_base_b_test_16_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_b_test_16_npu_a3 }}
|
||||
only_tests_base_b_test_16_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_b_test_16_npu_a3 }}
|
||||
should_run_base_c_test_acc_2_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_c_test_acc_2_npu_a3 }}
|
||||
only_tests_base_c_test_acc_2_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_acc_2_npu_a3 }}
|
||||
should_run_base_c_test_acc_16_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_c_test_acc_16_npu_a3 }}
|
||||
only_tests_base_c_test_acc_16_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_acc_16_npu_a3 }}
|
||||
should_run_base_c_test_perf_2_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_c_test_perf_2_npu_a3 }}
|
||||
only_tests_base_c_test_perf_2_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_perf_2_npu_a3 }}
|
||||
should_run_base_c_test_perf_16_npu_a3: ${{ steps.bucket-tests.outputs.should_run_base_c_test_perf_16_npu_a3 }}
|
||||
only_tests_base_c_test_perf_16_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_base_c_test_perf_16_npu_a3 }}
|
||||
should_run_multimodal_gen_test_1_npu_a3: ${{ steps.bucket-tests.outputs.should_run_multimodal_gen_test_1_npu_a3 }}
|
||||
only_tests_multimodal_gen_test_1_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_multimodal_gen_test_1_npu_a3 }}
|
||||
should_run_multimodal_gen_test_4_npu_a3: ${{ steps.bucket-tests.outputs.should_run_multimodal_gen_test_4_npu_a3 }}
|
||||
only_tests_multimodal_gen_test_4_npu_a3: ${{ steps.bucket-tests.outputs.only_tests_multimodal_gen_test_4_npu_a3 }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
# Every step carries step-level continue-on-error so the job conclusion
|
||||
# is ALWAYS 'success' — the fast-fail health check in other jobs filters
|
||||
# by job conclusion, and job-level continue-on-error still reports
|
||||
# 'failure' via the API. Failed steps stay visible (red ✗ + annotation)
|
||||
# but never cascade to test jobs.
|
||||
- name: Checkout sglang scripts
|
||||
continue-on-error: true
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# test/registered + python/sglang/test/ci are needed by
|
||||
# list_tests.py to enumerate each suite's test files for bucketing.
|
||||
# The diffusion case config is read (AST only) by
|
||||
# bucket_recommended_tests.py to enumerate multimodal cases.
|
||||
sparse-checkout: |
|
||||
scripts
|
||||
test/registered
|
||||
python/sglang/test/ci
|
||||
python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py
|
||||
sparse-checkout-cone-mode: false
|
||||
|
||||
- name: Download test case map
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -euo pipefail
|
||||
curl -fsSL \
|
||||
@@ -485,6 +774,7 @@ jobs:
|
||||
du -h test_case_map.json
|
||||
|
||||
- name: Download coverage package and extract covstub
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -euo pipefail
|
||||
curl -fsSL \
|
||||
@@ -502,6 +792,7 @@ jobs:
|
||||
find outputs/covstub -maxdepth 2 -type d | head -20 || true
|
||||
|
||||
- name: Recommend tests for current PR
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! python3 -c "import regex" >/dev/null 2>&1; then
|
||||
@@ -520,52 +811,42 @@ jobs:
|
||||
--source-dir "$PWD/outputs/covstub" \
|
||||
--map-file "$PWD/test_case_map.json"
|
||||
|
||||
- name: Export coverage paths output
|
||||
id: export-paths
|
||||
# Bucket the recommended tests per suite. Downstream jobs consume two
|
||||
# outputs per suite: should_run_<suite> ('false' -> skip the job, e.g.
|
||||
# empty bucket in precision mode) and only_tests_<suite> (non-empty only
|
||||
# in precision mode, gated inside the script). On failure outputs stay
|
||||
# empty -> callers run full suites.
|
||||
- name: Bucket recommended tests per suite
|
||||
id: bucket-tests
|
||||
continue-on-error: true
|
||||
if: always()
|
||||
env:
|
||||
# Precision-test switch: 'true' = the a3 suites run only the
|
||||
# recommended subset, 'false' = full suites.
|
||||
PRECISION_MODE: 'true'
|
||||
run: |
|
||||
FILE="scripts/ci/npu/precise-test/recommended_pytest_paths.txt"
|
||||
if [ -f "$FILE" ]; then
|
||||
{
|
||||
echo "coverage_paths<<EOF"
|
||||
cat "$FILE"
|
||||
echo "EOF"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
echo "Exported $(wc -l < "$FILE") recommended paths"
|
||||
else
|
||||
echo "coverage_paths=" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice::recommended_pytest_paths.txt not found"
|
||||
fi
|
||||
set -euo pipefail
|
||||
python3 scripts/ci/npu/precise-test/bucket_recommended_tests.py \
|
||||
--recommended scripts/ci/npu/precise-test/recommended_pytest_paths.txt \
|
||||
--changed-files scripts/ci/npu/precise-test/pr_changed_files.txt \
|
||||
--suites "base-b-test-1-npu-a3,base-b-test-2-npu-a3,base-b-test-4-npu-a3,base-b-test-8-npu-a3,base-b-test-16-npu-a3,base-c-test-acc-2-npu-a3,base-c-test-acc-16-npu-a3,base-c-test-perf-2-npu-a3,base-c-test-perf-16-npu-a3" \
|
||||
--diffusion-suites "multimodal-gen-test-1-npu-a3:1-npu,multimodal-gen-test-4-npu-a3:2-npu" \
|
||||
--case-map "$PWD/test_case_map.json" \
|
||||
--precision-mode "${PRECISION_MODE}" \
|
||||
--github-output "${GITHUB_OUTPUT}" \
|
||||
--github-summary "${GITHUB_STEP_SUMMARY:-}" \
|
||||
--report-file recommended-tests-report.md
|
||||
|
||||
# ==================== Analyze Failure Report ==================== #
|
||||
# Cross-references test failures with coverage recommendations.
|
||||
# !cancelled() only: skipped jobs in needs poison success().
|
||||
analyze-failure-report:
|
||||
name: Analyze failure report
|
||||
needs:
|
||||
[
|
||||
base-a-test-npu,
|
||||
base-b-test-1-npu-a3,
|
||||
base-b-test-2-npu-a3,
|
||||
base-b-test-4-npu-a3,
|
||||
base-b-test-8-npu-a3,
|
||||
base-b-test-16-npu-a3,
|
||||
|
||||
multimodal-gen-test-1-npu-a3,
|
||||
multimodal-gen-test-4-npu-a3,
|
||||
|
||||
base-c-test-acc-2-npu-a3,
|
||||
base-c-test-acc-16-npu-a3,
|
||||
base-c-test-perf-2-npu-a3,
|
||||
base-c-test-perf-16-npu-a3,
|
||||
|
||||
recommend-tests-from-coverage,
|
||||
]
|
||||
if: ${{ !cancelled() && inputs.coverage_mode != true }}
|
||||
uses: ./.github/workflows/_npu-analyze-failure.yml
|
||||
with:
|
||||
log_artifact_pattern: selected-test-logs-*
|
||||
recommendations_content: ${{ needs.recommend-tests-from-coverage.outputs.coverage_paths || '' }}
|
||||
# Same report as the step summary, downloadable from the run page.
|
||||
- name: Upload recommended tests report
|
||||
continue-on-error: true
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: recommended-tests-report
|
||||
path: recommended-tests-report.md
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
# ==================== Coverage Assembly & Publish ==================== #
|
||||
# Assembles coverage data and publishes the baseline to the shared disk.
|
||||
@@ -573,7 +854,7 @@ jobs:
|
||||
setup-covstub:
|
||||
needs:
|
||||
[
|
||||
set-image-config,
|
||||
base-a-test-npu-cov,
|
||||
base-b-test-1-npu-a3,
|
||||
base-b-test-2-npu-a3,
|
||||
base-b-test-4-npu-a3,
|
||||
@@ -583,11 +864,13 @@ jobs:
|
||||
base-c-test-acc-16-npu-a3,
|
||||
base-c-test-perf-2-npu-a3,
|
||||
base-c-test-perf-16-npu-a3,
|
||||
multimodal-gen-test-1-npu-a3,
|
||||
multimodal-gen-test-4-npu-a3,
|
||||
]
|
||||
if: ${{ !failure() && !cancelled() && inputs.coverage_mode == true }}
|
||||
runs-on: linux-aarch64-a3-800t-2
|
||||
runs-on: linux-amd64-cpu-4
|
||||
container:
|
||||
image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
|
||||
image: swr.ap-southeast-1.myhuaweicloud.com/base_image/ascend-ci/sglang:main-x86
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -596,12 +879,31 @@ jobs:
|
||||
# checks out the same commit even if the branch advances mid-run.
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- name: Collect base-a coverage artifact
|
||||
# Runs on ubuntu-latest (no shared disk) — hence the artifact relay.
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: coverage-outputs-base-a-test-npu
|
||||
path: /tmp/base-a-coverage
|
||||
|
||||
- name: Merge base-a coverage into shared run dir
|
||||
run: |
|
||||
# Artifact stages {RUN_DIR}/outputs/... — merges 1:1 into the shared layout.
|
||||
CACHE_ROOT="/root/.cache/tests/precise-test"
|
||||
mkdir -p "${CACHE_ROOT}"
|
||||
cp -a /tmp/base-a-coverage/. "${CACHE_ROOT}/"
|
||||
echo "Merged base-a coverage into ${CACHE_ROOT}:"
|
||||
find "${CACHE_ROOT}" -maxdepth 3 -type d | head -20
|
||||
|
||||
- name: Copy sglang source to covstub
|
||||
run: |
|
||||
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
||||
COVSTUB="${RUN_DIR}/outputs/covstub"
|
||||
mkdir -p "${COVSTUB}"
|
||||
cp -r python/sglang "${COVSTUB}/"
|
||||
# _version.py is build-generated (git-ignored); the placeholder keeps
|
||||
# coverage html from failing on a missing recorded source file.
|
||||
echo '__version__ = "0.0.0"' > "${COVSTUB}/sglang/_version.py"
|
||||
|
||||
- name: Build test case map
|
||||
shell: bash
|
||||
@@ -609,11 +911,9 @@ jobs:
|
||||
set -euo pipefail
|
||||
|
||||
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
||||
# Date tag derivation must match scripts/ci/npu/precise-test/run_tests_with_coverage.sh.
|
||||
# The custom k8s runner only injects explicitly-set env vars, so
|
||||
# GITHUB_RUN_STARTED_AT may be absent (set -u would abort); fall back
|
||||
# to local date exactly like run_tests_with_coverage.sh does, so both
|
||||
# sides derive the same tag.
|
||||
# Must match run_tests_with_coverage.sh: GITHUB_RUN_STARTED_AT may be
|
||||
# unset on the k8s runner (only explicitly-set envs are injected), so
|
||||
# fall back to local date the same way.
|
||||
if [ -n "${GITHUB_RUN_STARTED_AT:-}" ]; then
|
||||
COV_DATE_TAG="${GITHUB_RUN_STARTED_AT:0:10}"
|
||||
COV_DATE_TAG="${COV_DATE_TAG//-/}"
|
||||
@@ -622,14 +922,14 @@ jobs:
|
||||
fi
|
||||
COV_ROOT="${RUN_DIR}/outputs/sglang@${COV_DATE_TAG}"
|
||||
COVSTUB="${RUN_DIR}/outputs/covstub"
|
||||
# Placed outside outputs/ so it is NOT included in outputs.tar.gz.
|
||||
# Standalone top-level deliverable, never packed inside outputs/.
|
||||
MAP_FILE="${RUN_DIR}/test_case_map.json"
|
||||
|
||||
echo "Coverage dir: ${COV_ROOT}"
|
||||
echo "Source dir: ${COVSTUB}"
|
||||
|
||||
# Coverage data may be absent when all test jobs were skipped
|
||||
# (e.g. by the changes filter). Skip map building in that case.
|
||||
# Coverage data may be absent when no test targets were enumerated
|
||||
# (run_tests_with_coverage.sh never ran). Skip map building then.
|
||||
if [ ! -d "${COV_ROOT}" ]; then
|
||||
echo "::warning::Coverage dir not found: ${COV_ROOT}, skip building test case map"
|
||||
exit 0
|
||||
@@ -643,11 +943,11 @@ jobs:
|
||||
fi
|
||||
|
||||
if ! python3 -c "import regex" 2>/dev/null; then
|
||||
pip install regex \
|
||||
python3 -m pip install regex \
|
||||
-i http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
||||
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
||||
--retries 3 --timeout 60 \
|
||||
|| pip install regex -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
||||
|| python3 -m pip install regex -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
||||
fi
|
||||
|
||||
python3 scripts/ci/npu/precise-test/test_selector.py \
|
||||
@@ -658,10 +958,182 @@ jobs:
|
||||
test -s "${MAP_FILE}"
|
||||
du -h "${MAP_FILE}"
|
||||
|
||||
- name: Generate HTML coverage report
|
||||
# Best-effort add-on: continue-on-error keeps the job green and later
|
||||
# steps running, while a failed generation still shows a red ✗ in the UI.
|
||||
continue-on-error: true
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
||||
# Date tag derivation must match run_tests_with_coverage.sh.
|
||||
if [ -n "${GITHUB_RUN_STARTED_AT:-}" ]; then
|
||||
COV_DATE_TAG="${GITHUB_RUN_STARTED_AT:0:10}"
|
||||
COV_DATE_TAG="${COV_DATE_TAG//-/}"
|
||||
else
|
||||
COV_DATE_TAG="$(date +%Y%m%d)"
|
||||
fi
|
||||
COV_ROOT="${RUN_DIR}/outputs/sglang@${COV_DATE_TAG}"
|
||||
COVSTUB="${RUN_DIR}/outputs/covstub"
|
||||
# Standalone top-level dir, packaged into htmlcov.tar.gz at publish.
|
||||
REPORT_DIR="${RUN_DIR}/htmlcov"
|
||||
|
||||
# Same skip conditions as map building: no data -> nothing to report.
|
||||
if [ ! -d "${COV_ROOT}" ] \
|
||||
|| [ "$(find "${COV_ROOT}" -name 'coverage.*' -type f | wc -l)" -eq 0 ]; then
|
||||
echo "::warning::No coverage data found, skip HTML report"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Failures below exit non-zero (errexit) — red ✗, job stays green.
|
||||
# Do NOT convert failures to exit 0 behind an `if ! ( set -e; ... )`
|
||||
# guard: bash ignores -e inside a subshell in a condition context,
|
||||
# which silently swallowed three failures in run 38713909169.
|
||||
|
||||
# Pin matches the test jobs' coverage==7.8.* so combine can read
|
||||
# their SQLite data.
|
||||
if ! python3 -c "import coverage" 2>/dev/null; then
|
||||
python3 -m pip install coverage==7.8.* \
|
||||
-i http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
||||
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
||||
--retries 3 --timeout 60 \
|
||||
|| python3 -m pip install coverage==7.8.* -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
||||
fi
|
||||
|
||||
# Alias runner-specific source paths onto covstub; first entry must
|
||||
# be a real local path (no wildcards).
|
||||
{
|
||||
echo "[run]"
|
||||
echo "branch = True"
|
||||
echo "[paths]"
|
||||
echo "sglang ="
|
||||
echo " ${COVSTUB}/sglang/"
|
||||
echo " */python/sglang/"
|
||||
} > /tmp/report-rc
|
||||
|
||||
# --keep preserves the per-test raw coverage files: coverage-data.tar.gz
|
||||
# archives them after this step for manual map rebuilds.
|
||||
COVERAGE_FILE=/tmp/.coverage-all \
|
||||
python3 -m coverage combine --rcfile=/tmp/report-rc --keep \
|
||||
$(find "${COV_ROOT}" -name 'coverage.*' -type f)
|
||||
|
||||
rm -rf "${REPORT_DIR}"
|
||||
COVERAGE_FILE=/tmp/.coverage-all \
|
||||
python3 -m coverage html --rcfile=/tmp/report-rc \
|
||||
--directory="${REPORT_DIR}"
|
||||
|
||||
# NPU-only view: [run] source does not filter at report time (a
|
||||
# source-based rcfile yields the full report), so instead touch the
|
||||
# unexecuted npu .py files into a data copy (os.walk, so the PEP 420
|
||||
# namespace tree counts) and filter with --include, which works.
|
||||
cp /tmp/.coverage-all /tmp/.coverage-npu
|
||||
python3 - "${COVSTUB}" <<'PYEOF'
|
||||
import os
|
||||
import sys
|
||||
from coverage import CoverageData
|
||||
|
||||
npu_root = os.path.join(sys.argv[1], "sglang", "srt", "hardware_backend", "npu")
|
||||
data = CoverageData(basename="/tmp/.coverage-npu")
|
||||
data.read()
|
||||
measured = set(data.measured_files())
|
||||
added = 0
|
||||
for root, _, files in os.walk(npu_root):
|
||||
for f in files:
|
||||
if f.endswith(".py"):
|
||||
p = os.path.join(root, f)
|
||||
if p not in measured:
|
||||
data.touch_file(p)
|
||||
added += 1
|
||||
data.write()
|
||||
print(f"NPU view: {len(measured)} measured, touched {added} unexecuted files")
|
||||
PYEOF
|
||||
rm -rf "${REPORT_DIR}-npu"
|
||||
COVERAGE_FILE=/tmp/.coverage-npu \
|
||||
python3 -m coverage html --rcfile=/tmp/report-rc \
|
||||
--include='*/sglang/srt/hardware_backend/npu/*' \
|
||||
--directory="${REPORT_DIR}-npu"
|
||||
|
||||
# Surface the totals directly on the run page.
|
||||
# GITHUB_STEP_SUMMARY may be unset on the k8s runner pool.
|
||||
if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
|
||||
{
|
||||
echo "### Coverage"
|
||||
echo '```'
|
||||
echo "python/sglang (measured files only):"
|
||||
COVERAGE_FILE=/tmp/.coverage-all \
|
||||
python3 -m coverage report --rcfile=/tmp/report-rc | tail -n 1
|
||||
echo "python/sglang/srt/hardware_backend/npu (full-tree denominator):"
|
||||
COVERAGE_FILE=/tmp/.coverage-npu \
|
||||
python3 -m coverage report --rcfile=/tmp/report-rc \
|
||||
--include='*/sglang/srt/hardware_backend/npu/*' | tail -n 1
|
||||
echo '```'
|
||||
echo "Full line-by-line report: download the \`coverage-html-report\` artifact and open \`htmlcov/index.html\` (NPU-only view: \`htmlcov-npu/index.html\`)."
|
||||
} >> "${GITHUB_STEP_SUMMARY}"
|
||||
fi
|
||||
|
||||
echo "HTML coverage report: ${REPORT_DIR}"
|
||||
echo "NPU-only HTML coverage report: ${REPORT_DIR}-npu"
|
||||
|
||||
- name: Upload HTML coverage report
|
||||
# Report upload must never block the baseline publish pipeline.
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-html-report
|
||||
# Two paths -> artifact root becomes the run dir, so the archive
|
||||
# holds htmlcov/ and htmlcov-npu/ side by side.
|
||||
path: |
|
||||
/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}/htmlcov
|
||||
/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}/htmlcov-npu
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
- name: Archive outputs
|
||||
run: |
|
||||
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
||||
tar -czf "${RUN_DIR}/outputs.tar.gz" -C "${RUN_DIR}" outputs
|
||||
# Source-only baseline: recommend mode needs covstub only, and it
|
||||
# shrinks the package from ~300MB to a few MB.
|
||||
tar -czf "${RUN_DIR}/outputs.tar.gz" -C "${RUN_DIR}" outputs/covstub
|
||||
# Raw coverage data for manual map rebuilds: extract both archives
|
||||
# under one root to reconstruct outputs/. Absent when no data.
|
||||
# Date tag derivation must match run_tests_with_coverage.sh.
|
||||
if [ -n "${GITHUB_RUN_STARTED_AT:-}" ]; then
|
||||
COV_DATE_TAG="${GITHUB_RUN_STARTED_AT:0:10}"
|
||||
COV_DATE_TAG="${COV_DATE_TAG//-/}"
|
||||
else
|
||||
COV_DATE_TAG="$(date +%Y%m%d)"
|
||||
fi
|
||||
if [ -d "${RUN_DIR}/outputs/sglang@${COV_DATE_TAG}" ]; then
|
||||
tar -czf "${RUN_DIR}/coverage-data.tar.gz" -C "${RUN_DIR}" "outputs/sglang@${COV_DATE_TAG}"
|
||||
fi
|
||||
|
||||
- name: Upload outputs and test case map
|
||||
# Never blocks the publish pipeline; must run before publish mv's
|
||||
# the files to the fixed PUBLISH_DIR (races with concurrent runs).
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: precise-test-outputs
|
||||
# Two paths -> artifact root becomes RUN_DIR, so the archive
|
||||
# holds outputs.tar.gz and test_case_map.json side by side.
|
||||
path: |
|
||||
/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}/outputs.tar.gz
|
||||
/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}/test_case_map.json
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload raw coverage data
|
||||
# Separate artifact keeps the ~300MB bulk out of the baseline package;
|
||||
# must run before publish mv's the file to the fixed PUBLISH_DIR.
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: precise-test-coverage-data
|
||||
path: /root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}/coverage-data.tar.gz
|
||||
# Absent when all test jobs were skipped (no coverage data).
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
- name: Publish outputs and map to shared disk
|
||||
shell: bash
|
||||
@@ -669,14 +1141,14 @@ jobs:
|
||||
set -euo pipefail
|
||||
|
||||
RUN_DIR="/root/.cache/tests/precise-test/${{ github.run_id }}-attempt-${{ github.run_attempt }}"
|
||||
# Fixed publish dir with no RUN_DIR layer: other pipelines have a
|
||||
# different GITHUB_RUN_ID and must be able to read the baseline.
|
||||
# Must match PUBLISH_DIR in the recommend-tests-from-coverage job.
|
||||
# Fixed-name copy for manual use; the recommend job's baseline lives
|
||||
# in OBS (upload outputs.tar.gz + test_case_map.json manually after
|
||||
# each run). coverage-data.tar.gz has no automatic consumer — no OBS
|
||||
# copy, only this dir and the precise-test-coverage-data artifact.
|
||||
PUBLISH_DIR="/root/.cache/tests/precise-test/coverage"
|
||||
|
||||
# Skip publish when no coverage data was collected this run (map
|
||||
# build was skipped), so we never overwrite the historical
|
||||
# baseline on the shared disk with an incomplete one.
|
||||
# Skip publish when no coverage data this run — never overwrite
|
||||
# the historical baseline with an incomplete one.
|
||||
if [ ! -s "${RUN_DIR}/test_case_map.json" ]; then
|
||||
echo "::warning::test_case_map.json missing (no coverage data this run), skip publish"
|
||||
exit 0
|
||||
@@ -684,14 +1156,29 @@ jobs:
|
||||
|
||||
mkdir -p "${PUBLISH_DIR}"
|
||||
|
||||
# mv (rename) within the same shared filesystem is atomic, so a
|
||||
# concurrent reader sees either the old or the new baseline,
|
||||
# never a partial file.
|
||||
# Same-filesystem mv is atomic per file, so readers never see a
|
||||
# partial file (files from two concurrent publishes can interleave).
|
||||
mv "${RUN_DIR}/outputs.tar.gz" "${PUBLISH_DIR}/outputs.tar.gz"
|
||||
mv "${RUN_DIR}/test_case_map.json" "${PUBLISH_DIR}/test_case_map.json"
|
||||
# Guarded: absent in the unlikely case this run collected no
|
||||
# coverage data while still producing a map.
|
||||
if [ -f "${RUN_DIR}/coverage-data.tar.gz" ]; then
|
||||
mv "${RUN_DIR}/coverage-data.tar.gz" "${PUBLISH_DIR}/coverage-data.tar.gz"
|
||||
fi
|
||||
|
||||
# HTML reports too, fixed names — latest wins, like the baseline.
|
||||
# [ -d ] guards the continue-on-error generation step. A tar failure
|
||||
# aborts red after the baseline mv completed; Cleanup stays skipped,
|
||||
# keeping RUN_DIR for recovery.
|
||||
if [ -d "${RUN_DIR}/htmlcov" ]; then
|
||||
tar -czf "${PUBLISH_DIR}/htmlcov.tar.gz" -C "${RUN_DIR}" htmlcov
|
||||
fi
|
||||
if [ -d "${RUN_DIR}/htmlcov-npu" ]; then
|
||||
tar -czf "${PUBLISH_DIR}/htmlcov-npu.tar.gz" -C "${RUN_DIR}" htmlcov-npu
|
||||
fi
|
||||
|
||||
echo "Published baseline to ${PUBLISH_DIR}"
|
||||
du -h "${PUBLISH_DIR}/outputs.tar.gz" "${PUBLISH_DIR}/test_case_map.json"
|
||||
ls -lh "${PUBLISH_DIR}" | tail -n +2
|
||||
|
||||
- name: Clean up per-run directory
|
||||
# Skipped on publish failure, keeping RUN_DIR for manual recovery.
|
||||
@@ -708,6 +1195,7 @@ jobs:
|
||||
check-changes,
|
||||
|
||||
base-a-test-npu,
|
||||
base-a-test-npu-cov,
|
||||
base-b-test-1-npu-a3,
|
||||
base-b-test-2-npu-a3,
|
||||
base-b-test-4-npu-a3,
|
||||
|
||||
@@ -9,14 +9,14 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
run_all_tests:
|
||||
description: "Run the preflight even if the last commit touched no PPU path"
|
||||
description: "Build and run both sanity files even without PPU path changes"
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
||||
description: "Git ref to test; defaults to the triggering event SHA"
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
@@ -27,8 +27,7 @@ on:
|
||||
default: false
|
||||
|
||||
concurrency:
|
||||
# Newer runs cancel older runs for the same ref; unrelated label events
|
||||
# use isolated groups to avoid interfering with active runs.
|
||||
# Unrelated label events must not cancel or replace the active PPU checks.
|
||||
group: >-
|
||||
pr-test-ppu-${{
|
||||
(github.event_name == 'pull_request' &&
|
||||
@@ -40,11 +39,15 @@ concurrency:
|
||||
}}
|
||||
cancel-in-progress: true
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
GIT_CONFIG_NOSYSTEM: "1"
|
||||
|
||||
jobs:
|
||||
# ==================== Check Changes ==================== #
|
||||
check-changes:
|
||||
# Give skipped checks from unrelated labels distinct names so they cannot
|
||||
# replace the corresponding checks from an active run.
|
||||
name: >-
|
||||
${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' &&
|
||||
github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') &&
|
||||
@@ -59,25 +62,28 @@ jobs:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
# Scope only; the shared pr-gate checks both required labels.
|
||||
ppu: ${{ steps.filter.outputs.ppu == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
||||
source-sha: ${{ steps.source.outputs.sha }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# github.sha (not github.ref) so a runner that dequeues after main
|
||||
# advances still validates the SHA that triggered the run.
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Pin the source revision for every job
|
||||
id: source
|
||||
run: printf 'sha=%s\n' "$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Determine run mode
|
||||
id: run-mode
|
||||
env:
|
||||
RUN_ALL_TESTS: ${{ inputs.run_all_tests }}
|
||||
run: |
|
||||
if [[ "${{ inputs.run_all_tests }}" == "true" ]]; then
|
||||
if [[ "$RUN_ALL_TESTS" == "true" ]]; then
|
||||
echo "run_all_tests=true" >> "$GITHUB_OUTPUT"
|
||||
echo "Run mode: ALL TESTS"
|
||||
else
|
||||
echo "run_all_tests=false" >> "$GITHUB_OUTPUT"
|
||||
echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
|
||||
fi
|
||||
|
||||
- name: Detect file changes
|
||||
@@ -85,16 +91,18 @@ jobs:
|
||||
uses: dorny/paths-filter@v3
|
||||
if: steps.run-mode.outputs.run_all_tests != 'true'
|
||||
with:
|
||||
# PPU-specific paths only.
|
||||
filters: |
|
||||
ppu:
|
||||
- ".github/workflows/pr-test-ppu.yml"
|
||||
- "scripts/ci/ppu/**"
|
||||
- "python/sglang/srt/platforms/ppu.py"
|
||||
- "python/sglang/srt/hardware_backend/ppu/**"
|
||||
- "python/sglang/kernels/aot/csrc/ppu/**"
|
||||
- "python/sglang/srt/platforms/sail.py"
|
||||
- "python/sglang/kernels/aot/setup_sail.py"
|
||||
- "python/sglang/kernels/aot/pyproject_sail.toml"
|
||||
- "python/sglang/kernels/aot/csrc/common_extension_sail.cc"
|
||||
- "python/sglang/kernels/jit/utils/compile/sailify.py"
|
||||
- "test/registered/core/test_basic_sanity_eagle3.py"
|
||||
- "test/registered/core/test_basic_sanity_dflash.py"
|
||||
|
||||
# ==================== PR Gate ==================== #
|
||||
pr-gate:
|
||||
name: >-
|
||||
${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' &&
|
||||
@@ -108,62 +116,168 @@ jobs:
|
||||
require-label: run-ci-ppu
|
||||
secrets: inherit
|
||||
|
||||
# ==================== PPU Runner Preflight ==================== #
|
||||
# Validates the runner contract only; the SRT platform and AOT kernels land
|
||||
# in later PRs and build their first real test on top of this job.
|
||||
ppu-preflight:
|
||||
ppu-build:
|
||||
name: >-
|
||||
${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' &&
|
||||
github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') &&
|
||||
'ignored-label-ppu-preflight' || 'ppu-preflight' }}
|
||||
'ignored-label-ppu-build' || 'ppu-build' }}
|
||||
needs: [check-changes, pr-gate]
|
||||
if: needs.check-changes.outputs.ppu == 'true'
|
||||
runs-on: 1-ppu-m890p
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 120
|
||||
permissions:
|
||||
contents: read
|
||||
container:
|
||||
image: download.t-head.cn/docker_release/pytorch:sdk2.2.0-2.13.0-ubuntu24.04-hggcrt3-py312-20261005
|
||||
env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
PPU_CI_SOURCE_SHA: ${{ needs.check-changes.outputs.source-sha }}
|
||||
MAX_JOBS: "2"
|
||||
SGL_KERNEL_COMPILE_THREADS: "2"
|
||||
outputs:
|
||||
wheel-artifact-id: ${{ steps.wheels.outputs.artifact-id }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
- name: Initialize private build directories and Git configuration
|
||||
run: |
|
||||
work="$(mktemp -d "${RUNNER_TEMP}/ppu-build.XXXXXX")"
|
||||
touch "$work/gitconfig"
|
||||
mkdir "$work/wheels" "$work/artifacts"
|
||||
{
|
||||
printf 'GIT_CONFIG_GLOBAL=%s\n' "$work/gitconfig"
|
||||
printf 'PPU_CI_WHEEL_DIR=%s\n' "$work/wheels"
|
||||
printf 'PPU_CI_ARTIFACT_DIR=%s\n' "$work/artifacts"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
# Same ref as check-changes. build_and_test.sh rejects any HEAD other than
|
||||
# PPU_CI_SOURCE_SHA, so a workflow_call branch that moves after
|
||||
# check-changes fails the run instead of building different code.
|
||||
- name: Checkout the triggering source revision
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Build native SAIL wheels
|
||||
run: |
|
||||
mkdir -p "$PPU_CI_WHEEL_DIR"
|
||||
bash scripts/ci/ppu/build_and_test.sh build
|
||||
|
||||
- name: Upload PPU wheels
|
||||
id: wheels
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ppu-wheels-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ env.PPU_CI_WHEEL_DIR }}
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Upload build diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ppu-build-logs-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ env.PPU_CI_ARTIFACT_DIR }}
|
||||
if-no-files-found: warn
|
||||
|
||||
ppu-sanity:
|
||||
name: >-
|
||||
${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' &&
|
||||
github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') &&
|
||||
format('ignored-label-ppu-sanity ({0})', matrix.runner) ||
|
||||
format('ppu-sanity ({0})', matrix.runner) }}
|
||||
needs: [check-changes, pr-gate, ppu-build]
|
||||
if: needs.check-changes.outputs.ppu == 'true'
|
||||
# Both PPU generations reuse the same wheels; each host supplies its own
|
||||
# runtime image and device, while checkpoints come from the shared NAS.
|
||||
strategy:
|
||||
# A failure on one generation must not hide the other generation's result.
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: [8-ppu-m890p, 16-ppu-zw810e]
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 60
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
PPU_CI_SOURCE_SHA: ${{ needs.check-changes.outputs.source-sha }}
|
||||
steps:
|
||||
- name: Initialize private runtime directories and Git configuration
|
||||
run: |
|
||||
work="$(mktemp -d "${RUNNER_TEMP}/ppu-runtime.XXXXXX")"
|
||||
touch "$work/gitconfig"
|
||||
mkdir "$work/wheels" "$work/artifacts"
|
||||
{
|
||||
printf 'GIT_CONFIG_GLOBAL=%s\n' "$work/gitconfig"
|
||||
printf 'PPU_CI_WHEEL_DIR=%s\n' "$work/wheels"
|
||||
printf 'PPU_CI_ARTIFACT_DIR=%s\n' "$work/artifacts"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
# Same ref as check-changes; run_test.sh rejects any HEAD other than
|
||||
# PPU_CI_SOURCE_SHA.
|
||||
- name: Checkout the triggering source revision
|
||||
timeout-minutes: 10
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Validate PPU runner contract
|
||||
timeout-minutes: 10
|
||||
run: python3 scripts/ci/ppu/check_ppu_environment.py
|
||||
- name: Download the matching PPU wheel artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
artifact-ids: ${{ needs.ppu-build.outputs.wheel-artifact-id }}
|
||||
merge-multiple: true
|
||||
path: ${{ env.PPU_CI_WHEEL_DIR }}
|
||||
|
||||
- name: Run both speculative sanity files in isolated Docker
|
||||
env:
|
||||
CONFIG_PPU_CI_DEVICE: ${{ vars.PPU_CI_DEVICE }}
|
||||
CONFIG_PPU_CI_TARGET_MODEL_PATH: ${{ vars.PPU_CI_TARGET_MODEL_PATH }}
|
||||
CONFIG_PPU_CI_EAGLE3_MODEL_PATH: ${{ vars.PPU_CI_EAGLE3_MODEL_PATH }}
|
||||
CONFIG_PPU_CI_DFLASH_MODEL_PATH: ${{ vars.PPU_CI_DFLASH_MODEL_PATH }}
|
||||
CONFIG_PPU_CI_EVAL_CACHE: ${{ vars.PPU_CI_EVAL_CACHE }}
|
||||
run: |
|
||||
for key in PPU_CI_DEVICE PPU_CI_TARGET_MODEL_PATH PPU_CI_EAGLE3_MODEL_PATH PPU_CI_DFLASH_MODEL_PATH PPU_CI_EVAL_CACHE; do
|
||||
override="CONFIG_$key"
|
||||
if [[ -n "${!override}" ]]; then
|
||||
export "$key=${!override}"
|
||||
fi
|
||||
done
|
||||
exec bash scripts/ci/ppu/run_test.sh
|
||||
|
||||
- name: Upload runtime diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ppu-runtime-logs-${{ matrix.runner }}-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ env.PPU_CI_ARTIFACT_DIR }}
|
||||
if-no-files-found: warn
|
||||
|
||||
# ==================== Finish ==================== #
|
||||
pr-test-ppu-finish:
|
||||
name: >-
|
||||
${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' &&
|
||||
github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') &&
|
||||
'ignored-label-pr-test-ppu-finish' || 'pr-test-ppu-finish' }}
|
||||
needs: [pr-gate, check-changes, ppu-preflight]
|
||||
needs: [pr-gate, check-changes, ppu-build, ppu-sanity]
|
||||
if: always() && needs.check-changes.result != 'skipped'
|
||||
runs-on: ubuntu-latest
|
||||
permissions: {}
|
||||
steps:
|
||||
- name: Check all dependent job statuses
|
||||
- name: Require successful native build and both PPU sanity files when requested
|
||||
env:
|
||||
NEEDS_JSON: ${{ toJson(needs) }}
|
||||
PPU_IN_SCOPE: ${{ needs.check-changes.outputs.ppu }}
|
||||
BUILD_RESULT: ${{ needs.ppu-build.result }}
|
||||
SANITY_RESULT: ${{ needs.ppu-sanity.result }}
|
||||
run: |
|
||||
json_needs='${{ toJson(needs) }}'
|
||||
for job in $(echo "$json_needs" | jq -r 'keys_unsorted[]'); do
|
||||
result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
|
||||
echo "$job: $result"
|
||||
if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
|
||||
echo "The above jobs failed."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "All jobs completed successfully"
|
||||
|
||||
- name: Require the preflight when PPU is in scope
|
||||
run: |
|
||||
in_scope='${{ needs.check-changes.outputs.ppu }}'
|
||||
preflight='${{ needs.ppu-preflight.result }}'
|
||||
echo "PPU in scope: '$in_scope' / ppu-preflight: '$preflight'"
|
||||
|
||||
if [[ "$in_scope" == "true" && "$preflight" != "success" ]]; then
|
||||
echo "PPU was in scope but ppu-preflight did not succeed."
|
||||
if jq -e 'any(.[]; .result == "failure" or .result == "cancelled")' <<< "$NEEDS_JSON" >/dev/null; then
|
||||
printf '%s\n' '::error::A PPU CI dependency failed or was cancelled.'
|
||||
exit 1
|
||||
fi
|
||||
echo "Preflight status is consistent with the requested scope."
|
||||
if [[ "$PPU_IN_SCOPE" == "true" ]]; then
|
||||
if [[ "$BUILD_RESULT" != "success" || "$SANITY_RESULT" != "success" ]]; then
|
||||
printf '%s\n' '::error::PPU scope requires a successful native build and both PPU sanity files.'
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\n' 'Native build and both PPU sanity files passed.'
|
||||
else
|
||||
printf '%s\n' 'PPU is outside this run scope; no PPU tests were requested.'
|
||||
fi
|
||||
|
||||
@@ -56,7 +56,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda*
|
||||
pattern: wheel-python3.11-cuda*
|
||||
|
||||
- uses: ./.github/actions/download-rust-ext
|
||||
with:
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda*
|
||||
pattern: wheel-python3.11-cuda*
|
||||
|
||||
- uses: ./.github/actions/download-rust-ext
|
||||
with:
|
||||
@@ -146,7 +146,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda*
|
||||
pattern: wheel-python3.11-cuda*
|
||||
|
||||
- uses: ./.github/actions/download-rust-ext
|
||||
with:
|
||||
@@ -181,7 +181,7 @@ jobs:
|
||||
# with:
|
||||
# path: python/sglang/kernels/aot/dist/
|
||||
# merge-multiple: true
|
||||
# pattern: wheel-python3.10-cuda*
|
||||
# pattern: wheel-python3.11-cuda*
|
||||
|
||||
# - name: Install dependencies
|
||||
# run: |
|
||||
|
||||
@@ -6,10 +6,10 @@ name: PR Test (sgl-router)
|
||||
# fires on every PR synchronize / push, so the run record always
|
||||
# appears as a check on the PR (no more "workflow silently didn't
|
||||
# fire" mystery debugging). Path-based skip decisions are made
|
||||
# inside the `sgl-router-gate` job below, where we can log the
|
||||
# inside the `sgl-router-changes` job below, where we can log the
|
||||
# reason. Same for the `run-ci` label gate — moved off the
|
||||
# workflow trigger and into the gate job so its outcome is
|
||||
# observable.
|
||||
# observable, and now fails (rather than skips) when it is missing.
|
||||
#
|
||||
# * `push` on `main` keeps firing so post-merge runs still record
|
||||
# against the default branch.
|
||||
@@ -29,21 +29,17 @@ env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
|
||||
jobs:
|
||||
# Stage 0 — decide whether to run the heavy tiers. Always runs
|
||||
# (cheap, ubuntu-latest), and emits a single `should_run` output
|
||||
# consumed by every tier below. Reasons it might evict downstream
|
||||
# work:
|
||||
# * `pull_request` event without the `run-ci` label (budget gate)
|
||||
# * No files matching the sgl-router paths changed in this PR
|
||||
# Either outcome is logged on the gate job's page, so operators can
|
||||
# see *why* the tiers were skipped instead of guessing.
|
||||
sgl-router-gate:
|
||||
name: gate
|
||||
# Stage 0a — path detection. Always runs (cheap, ubuntu-latest) and
|
||||
# emits a single `should_run` output consumed by every tier below.
|
||||
# It only answers "does this PR touch sgl-router?"; the `run-ci`
|
||||
# label is checked by the separate gate job so that a missing label
|
||||
# fails that job instead of silently skipping the tiers (see below).
|
||||
sgl-router-changes:
|
||||
name: changes
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
should_run: ${{ steps.decide.outputs.should_run }}
|
||||
paths_changed: ${{ steps.paths.outputs.sgl_router }}
|
||||
has_run_ci_label: ${{ steps.label.outputs.has_run_ci }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# `dorny/paths-filter` computes the diff against the PR base
|
||||
@@ -53,53 +49,62 @@ jobs:
|
||||
# the gate is observable.
|
||||
- name: Detect sgl-router path changes
|
||||
id: paths
|
||||
if: github.event_name != 'workflow_dispatch'
|
||||
uses: dorny/paths-filter@v3
|
||||
with:
|
||||
filters: |
|
||||
sgl_router:
|
||||
- 'experimental/sgl-router/**'
|
||||
- 'sgl-router/**'
|
||||
- 'rust/Cargo.toml'
|
||||
- 'rust/sglang-processor/**'
|
||||
- 'rust/sglang-api-types/**'
|
||||
- 'proto/**'
|
||||
- '.github/workflows/pr-test-sgl-router.yml'
|
||||
- 'docker/sgl-router.Dockerfile'
|
||||
- 'scripts/ci/cuda/ci_install_dependency.sh'
|
||||
# `run-ci` label gate. Cheap workflow runs (e.g. docs-only PRs
|
||||
# that happen to touch the sgl-router dir) still need the
|
||||
# opt-in label before consuming the H100 / kind tiers below.
|
||||
# `push` events on main and `workflow_dispatch` skip this gate.
|
||||
- name: Check run-ci label
|
||||
id: label
|
||||
run: |
|
||||
if [[ "${{ github.event_name }}" != "pull_request" ]]; then
|
||||
echo "has_run_ci=true" >> "$GITHUB_OUTPUT"
|
||||
echo "Non-PR event (${{ github.event_name }}); label gate bypassed."
|
||||
exit 0
|
||||
fi
|
||||
if [[ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci') }}" == "true" ]]; then
|
||||
echo "has_run_ci=true" >> "$GITHUB_OUTPUT"
|
||||
echo "PR has run-ci label; downstream tiers will run."
|
||||
else
|
||||
echo "has_run_ci=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::warning::PR is missing the 'run-ci' label; skipping sgl-router tiers. Apply the label to opt in."
|
||||
fi
|
||||
- name: Decide
|
||||
id: decide
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
PATHS_CHANGED: ${{ steps.paths.outputs.sgl_router }}
|
||||
run: |
|
||||
paths='${{ steps.paths.outputs.sgl_router }}'
|
||||
label='${{ steps.label.outputs.has_run_ci }}'
|
||||
if [[ "$paths" == "true" && "$label" == "true" ]]; then
|
||||
if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then
|
||||
echo "should_run=true" >> "$GITHUB_OUTPUT"
|
||||
echo "Both path filter and run-ci label match; running tiers."
|
||||
else
|
||||
echo "Manual dispatch; running sgl-router tiers."
|
||||
elif [[ "$PATHS_CHANGED" != "true" ]]; then
|
||||
echo "should_run=false" >> "$GITHUB_OUTPUT"
|
||||
echo "Skipping tiers (paths_changed=$paths, has_run_ci=$label)."
|
||||
echo "No sgl-router-related changes; skipping tiers."
|
||||
else
|
||||
echo "should_run=true" >> "$GITHUB_OUTPUT"
|
||||
echo "sgl-router-related changes detected; running tiers."
|
||||
fi
|
||||
|
||||
# Stage 0b — `run-ci` label gate, shared with `pr-test.yml`. Cheap
|
||||
# workflow runs (e.g. docs-only PRs that happen to touch the
|
||||
# sgl-router dir) still need the opt-in label before consuming the
|
||||
# H100 / kind tiers below. `push` on main and `workflow_dispatch`
|
||||
# pass straight through.
|
||||
#
|
||||
# `pr-gate.yml` fetches the PR's labels live and FAILS when `run-ci`
|
||||
# is missing. Both matter for `/tag-and-rerun-ci`: the slash handler
|
||||
# adds the label with GITHUB_TOKEN, which fires no `labeled` event,
|
||||
# then reruns the PR's unsuccessful runs. A rerun replays the original
|
||||
# event payload, so a gate reading `github.event.pull_request.labels`
|
||||
# would skip again, and a run that concluded `success` with every
|
||||
# tier skipped is never rerun at all. Failing here on the live label
|
||||
# set makes the rerun both happen and pick the label up.
|
||||
sgl-router-gate:
|
||||
name: gate
|
||||
needs: sgl-router-changes
|
||||
if: needs.sgl-router-changes.outputs.should_run == 'true'
|
||||
uses: ./.github/workflows/pr-gate.yml
|
||||
with:
|
||||
require-run-ci: true
|
||||
|
||||
sgl-router-lint:
|
||||
name: tier-1 — lint
|
||||
needs: sgl-router-gate
|
||||
if: needs.sgl-router-gate.outputs.should_run == 'true'
|
||||
needs: [sgl-router-changes, sgl-router-gate]
|
||||
if: needs.sgl-router-changes.outputs.should_run == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# Scoped per-job (not workflow-wide) to avoid SMG's documented breakage
|
||||
@@ -133,22 +138,22 @@ jobs:
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: experimental/sgl-router
|
||||
workspaces: sgl-router
|
||||
shared-key: "sgl-router-cache"
|
||||
cache-all-crates: true
|
||||
cache-on-failure: true
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: cargo check
|
||||
working-directory: experimental/sgl-router
|
||||
working-directory: sgl-router
|
||||
run: cargo check --locked --workspace --all-targets
|
||||
|
||||
- name: cargo clippy
|
||||
working-directory: experimental/sgl-router
|
||||
working-directory: sgl-router
|
||||
run: cargo clippy --locked --workspace --all-targets -- -D warnings
|
||||
|
||||
- name: cargo fmt
|
||||
working-directory: experimental/sgl-router
|
||||
working-directory: sgl-router
|
||||
run: |
|
||||
rustup toolchain install nightly --profile minimal --component rustfmt
|
||||
cargo +nightly fmt --all -- --check
|
||||
@@ -158,7 +163,7 @@ jobs:
|
||||
run: cargo install --locked cargo-deny@0.19.6
|
||||
|
||||
- name: cargo deny check
|
||||
working-directory: experimental/sgl-router
|
||||
working-directory: sgl-router
|
||||
run: cargo deny check licenses bans sources advisories
|
||||
|
||||
- name: Show sccache stats
|
||||
@@ -167,8 +172,8 @@ jobs:
|
||||
|
||||
sgl-router-build-and-test:
|
||||
name: tier-2 — build + test
|
||||
needs: [sgl-router-gate, sgl-router-lint]
|
||||
if: needs.sgl-router-gate.outputs.should_run == 'true'
|
||||
needs: [sgl-router-changes, sgl-router-gate, sgl-router-lint]
|
||||
if: needs.sgl-router-changes.outputs.should_run == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
@@ -206,14 +211,14 @@ jobs:
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: experimental/sgl-router
|
||||
workspaces: sgl-router
|
||||
shared-key: "sgl-router-cache"
|
||||
cache-all-crates: true
|
||||
cache-on-failure: true
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: cargo build (record wall time)
|
||||
working-directory: experimental/sgl-router
|
||||
working-directory: sgl-router
|
||||
run: |
|
||||
START=$(date +%s)
|
||||
cargo build --locked --release --all-targets
|
||||
@@ -223,7 +228,7 @@ jobs:
|
||||
echo "::notice title=Tier-2 build duration::${BUILD_SECONDS}s"
|
||||
|
||||
- name: cargo test
|
||||
working-directory: experimental/sgl-router
|
||||
working-directory: sgl-router
|
||||
# Skip the tokenizer parity_matrix test here: ubuntu-latest has
|
||||
# no HuggingFace cache, so the matrix would hard-fail (see the
|
||||
# test docstring). The e2e job runs it after pytest populates
|
||||
@@ -245,13 +250,13 @@ jobs:
|
||||
# through to a no-diff "pass" when no fixture was actually
|
||||
# regenerated.
|
||||
- name: kv_events hash parity fixture (drift check)
|
||||
working-directory: experimental/sgl-router
|
||||
working-directory: sgl-router
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 tests/scripts/generate_kv_events_hash_parity.py
|
||||
if ! git diff --exit-code -- tests/fixtures/kv_events_hash_parity.json; then
|
||||
echo "::error::kv_events hash parity fixture drifted." \
|
||||
"Re-run experimental/sgl-router/tests/scripts/generate_kv_events_hash_parity.py" \
|
||||
"Re-run sgl-router/tests/scripts/generate_kv_events_hash_parity.py" \
|
||||
"and commit the updated tests/fixtures/kv_events_hash_parity.json."
|
||||
exit 1
|
||||
fi
|
||||
@@ -268,8 +273,8 @@ jobs:
|
||||
|
||||
sgl-router-docker-build-test:
|
||||
name: tier-3 — docker (placeholder)
|
||||
needs: [sgl-router-gate, sgl-router-build-and-test]
|
||||
if: needs.sgl-router-gate.outputs.should_run == 'true'
|
||||
needs: [sgl-router-changes, sgl-router-gate, sgl-router-build-and-test]
|
||||
if: needs.sgl-router-changes.outputs.should_run == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
@@ -286,8 +291,8 @@ jobs:
|
||||
# the GPU/cluster runners from spinning up.
|
||||
sgl-router-k8s-integration:
|
||||
name: tier-3 — k8s integration
|
||||
needs: [sgl-router-gate, sgl-router-build-and-test]
|
||||
if: needs.sgl-router-gate.outputs.should_run == 'true'
|
||||
needs: [sgl-router-changes, sgl-router-gate, sgl-router-build-and-test]
|
||||
if: needs.sgl-router-changes.outputs.should_run == 'true'
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
@@ -303,20 +308,20 @@ jobs:
|
||||
curl -Lo /tmp/kubectl https://dl.k8s.io/release/v1.31.0/bin/linux/amd64/kubectl
|
||||
chmod +x /tmp/kubectl && sudo mv /tmp/kubectl /usr/local/bin/kubectl
|
||||
- name: Build router image
|
||||
run: docker build -t sgl-router:e2e -f experimental/sgl-router/tests/e2e/k8s_integration/Dockerfile.router .
|
||||
run: docker build -t sgl-router:e2e -f sgl-router/tests/e2e/k8s_integration/Dockerfile.router .
|
||||
- name: Build fake_worker image
|
||||
run: |
|
||||
docker build -t sgl-router-fake-worker:e2e \
|
||||
-f experimental/sgl-router/tests/e2e/k8s_integration/Dockerfile.fake_worker \
|
||||
experimental/sgl-router/tests/e2e/k8s_integration/
|
||||
-f sgl-router/tests/e2e/k8s_integration/Dockerfile.fake_worker \
|
||||
sgl-router/tests/e2e/k8s_integration/
|
||||
- name: Bootstrap kind + deploy
|
||||
run: bash experimental/sgl-router/tests/e2e/k8s_integration/setup.sh
|
||||
run: bash sgl-router/tests/e2e/k8s_integration/setup.sh
|
||||
- name: Set up Python
|
||||
run: |
|
||||
python3 -m venv /tmp/e2e-venv
|
||||
/tmp/e2e-venv/bin/pip install -r experimental/sgl-router/tests/e2e/k8s_integration/requirements.txt
|
||||
/tmp/e2e-venv/bin/pip install -r sgl-router/tests/e2e/k8s_integration/requirements.txt
|
||||
- name: Run E2E
|
||||
run: /tmp/e2e-venv/bin/pytest experimental/sgl-router/tests/e2e/k8s_integration/ -v --tb=short
|
||||
run: /tmp/e2e-venv/bin/pytest sgl-router/tests/e2e/k8s_integration/ -v --tb=short
|
||||
- name: Dump cluster + router diagnostics on failure
|
||||
if: failure()
|
||||
run: |
|
||||
@@ -333,8 +338,8 @@ jobs:
|
||||
|
||||
sgl-router-e2e:
|
||||
name: tier-3 — e2e
|
||||
needs: [sgl-router-gate, sgl-router-build-and-test]
|
||||
if: needs.sgl-router-gate.outputs.should_run == 'true'
|
||||
needs: [sgl-router-changes, sgl-router-gate, sgl-router-build-and-test]
|
||||
if: needs.sgl-router-changes.outputs.should_run == 'true'
|
||||
runs-on: 2-gpu-h100
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
@@ -344,10 +349,10 @@ jobs:
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: experimental/sgl-router
|
||||
workspaces: sgl-router
|
||||
shared-key: "sgl-router-cache"
|
||||
- name: cargo build (release)
|
||||
working-directory: experimental/sgl-router
|
||||
working-directory: sgl-router
|
||||
run: |
|
||||
source "$HOME/.cargo/env"
|
||||
cargo build --locked --release --workspace
|
||||
@@ -365,7 +370,7 @@ jobs:
|
||||
|
||||
- name: Install e2e test dependencies
|
||||
run: |
|
||||
python3 -m pip install -r experimental/sgl-router/tests/e2e/requirements.txt
|
||||
python3 -m pip install -r sgl-router/tests/e2e/requirements.txt
|
||||
|
||||
# IMPORTANT: --ignore=tests/e2e/k8s_integration. The k8s integration
|
||||
# suite is owned by the `sgl-router-k8s-integration` job which has
|
||||
@@ -376,7 +381,7 @@ jobs:
|
||||
# image pull) must not surface as e2e failures on H100, and e2e
|
||||
# flakes (model load, HF auth) must not surface as k8s failures.
|
||||
- name: Run e2e
|
||||
working-directory: experimental/sgl-router
|
||||
working-directory: sgl-router
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
run: |
|
||||
@@ -384,7 +389,7 @@ jobs:
|
||||
--ignore=tests/e2e/k8s_integration
|
||||
|
||||
- name: Tokenizer parity matrix (uses HF cache populated by e2e)
|
||||
working-directory: experimental/sgl-router
|
||||
working-directory: sgl-router
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
run: |
|
||||
@@ -394,25 +399,43 @@ jobs:
|
||||
sgl-router-finish:
|
||||
name: finish
|
||||
needs:
|
||||
- sgl-router-changes
|
||||
- sgl-router-gate
|
||||
- sgl-router-lint
|
||||
- sgl-router-build-and-test
|
||||
- sgl-router-docker-build-test
|
||||
- sgl-router-k8s-integration
|
||||
- sgl-router-e2e
|
||||
# `always()` lets `finish` run after upstream `skipped` outcomes
|
||||
# (when the gate decided to skip). The downstream-skipped path is
|
||||
# an explicit success — the workflow was correctly bypassed for a
|
||||
# PR that didn't need full CI. The downstream-failure /
|
||||
# downstream-cancelled path is a real fail. We also accept the
|
||||
# gate's `skipped` (impossible today, kept for symmetry) and
|
||||
# require the gate itself to have succeeded.
|
||||
if: |
|
||||
always() &&
|
||||
needs.sgl-router-gate.result == 'success' &&
|
||||
!contains(needs.*.result, 'failure') &&
|
||||
!contains(needs.*.result, 'cancelled')
|
||||
# `always()` lets `finish` run after upstream `skipped` outcomes.
|
||||
# Two outcomes are a success: every tier passed, or the PR touches
|
||||
# no sgl-router path and every tier was skipped. Anything else,
|
||||
# including a gate that failed for a missing `run-ci` label, fails
|
||||
# the run, so `/tag-and-rerun-ci` sees an unsuccessful run to rerun.
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: All required checks completed
|
||||
run: echo "OK"
|
||||
- name: Check required results
|
||||
env:
|
||||
CHANGES_RESULT: ${{ needs.sgl-router-changes.result }}
|
||||
SHOULD_RUN: ${{ needs.sgl-router-changes.outputs.should_run }}
|
||||
GATE_RESULT: ${{ needs.sgl-router-gate.result }}
|
||||
LINT_RESULT: ${{ needs.sgl-router-lint.result }}
|
||||
BUILD_RESULT: ${{ needs.sgl-router-build-and-test.result }}
|
||||
DOCKER_RESULT: ${{ needs.sgl-router-docker-build-test.result }}
|
||||
K8S_RESULT: ${{ needs.sgl-router-k8s-integration.result }}
|
||||
E2E_RESULT: ${{ needs.sgl-router-e2e.result }}
|
||||
run: |
|
||||
if [[ "$CHANGES_RESULT" != "success" ]]; then
|
||||
echo "::error::sgl-router path detection did not succeed ($CHANGES_RESULT)."
|
||||
exit 1
|
||||
fi
|
||||
tiers="$GATE_RESULT $LINT_RESULT $BUILD_RESULT $DOCKER_RESULT $K8S_RESULT $E2E_RESULT"
|
||||
if [[ "$SHOULD_RUN" == "true" && "$tiers" == "success success success success success success" ]]; then
|
||||
echo "All sgl-router checks passed."
|
||||
elif [[ "$SHOULD_RUN" == "false" && "$tiers" == "skipped skipped skipped skipped skipped skipped" ]]; then
|
||||
echo "No sgl-router-related changes; tiers skipped."
|
||||
else
|
||||
echo "::error::Unexpected sgl-router results: should_run=$SHOULD_RUN, gate=$GATE_RESULT, lint=$LINT_RESULT, build=$BUILD_RESULT, docker=$DOCKER_RESULT, k8s=$K8S_RESULT, e2e=$E2E_RESULT."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -93,31 +93,27 @@ jobs:
|
||||
suite: "stage-a-tp-test-cpu-intel"
|
||||
name: gnr
|
||||
|
||||
# Three SPR boxes each run ONE full-machine container taking one
|
||||
# partition of a 3-way split over the merged non-TP pool
|
||||
# (base-b non-TP + base-c). Single container per box (no socket
|
||||
# split): sglang auto-computes mem_fraction_static from host memory
|
||||
# and does not honor cgroup limits, so two parallel containers on
|
||||
# one box exhaust host RAM and OOM (tracked upstream). One container
|
||||
# per box keeps each server within the box's memory budget.
|
||||
# Each SPR box (2 sockets / 2 NUMA, 32 physical cores each) runs TWO
|
||||
# socket-pinned containers in parallel, each taking one partition of a
|
||||
# 4-way split over the merged non-TP pool. Each container is capped
|
||||
# with --memory so sglang (which reads cgroup memory.max, PR #38260 +
|
||||
# the empty-NUMA-node fix) sizes its KV pool against one socket's
|
||||
# budget, not the whole host -- so two parallel servers no longer
|
||||
# exhaust host RAM. 2 boxes x 2 sockets = 4 partitions.
|
||||
- runner: xeon-spr
|
||||
role: spr
|
||||
suite: "stage-a-test-cpu-intel,stage-b-test-cpu-intel"
|
||||
name: spr1
|
||||
part_id: 0
|
||||
part_size: 3
|
||||
part_lo: 0
|
||||
part_hi: 1
|
||||
part_size: 4
|
||||
- runner: xeon-spr
|
||||
role: spr
|
||||
suite: "stage-a-test-cpu-intel,stage-b-test-cpu-intel"
|
||||
name: spr2
|
||||
part_id: 1
|
||||
part_size: 3
|
||||
- runner: xeon-spr
|
||||
role: spr
|
||||
suite: "stage-a-test-cpu-intel,stage-b-test-cpu-intel"
|
||||
name: spr3
|
||||
part_id: 2
|
||||
part_size: 3
|
||||
part_lo: 2
|
||||
part_hi: 3
|
||||
part_size: 4
|
||||
steps:
|
||||
- name: Cleanup workspace
|
||||
run: |
|
||||
@@ -169,43 +165,70 @@ jobs:
|
||||
timeout-minutes: 120
|
||||
run: |
|
||||
docker exec -w /sglang-checkout/ ci_sglang_${{ matrix.name }} \
|
||||
bash -c "source /opt/.venv/bin/activate && cd ./test && python3 run_suite.py --hw cpu --suite ${{ matrix.suite }} --timeout-from-est-time"
|
||||
bash -c "source /opt/.venv/bin/activate && hf auth login --token \$HF_TOKEN && cd ./test && python3 run_suite.py --hw cpu --suite ${{ matrix.suite }} --timeout-from-est-time"
|
||||
|
||||
# ============ SPR: single full-machine container, one partition ====== #
|
||||
- name: Run container (SPR)
|
||||
if: matrix.role == 'spr'
|
||||
run: |
|
||||
docker rm -f ci_sglang_${{ matrix.name }} || true
|
||||
docker run -dt \
|
||||
-v ${{ github.workspace }}:/sglang-checkout/ --shm-size=64g \
|
||||
-v $HOME/.cache/huggingface:/root/.cache/huggingface \
|
||||
-v $HOME/.cache/sgl_eval:/root/.cache/sgl_eval \
|
||||
-e HF_TOKEN="$(cat ~/huggingface_token.txt)" \
|
||||
--name ci_sglang_${{ matrix.name }} \
|
||||
sglang_xeon
|
||||
|
||||
- name: Check AMX support (SPR)
|
||||
if: matrix.role == 'spr'
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
docker exec -w /sglang-checkout/ ci_sglang_${{ matrix.name }} \
|
||||
bash -c "source /opt/.venv/bin/activate && python3 -c 'import torch; import sgl_kernel; assert torch.cpu._is_amx_tile_supported(); assert hasattr(torch.ops.sgl_kernel, \"convert_weight_packed\"); '"
|
||||
|
||||
- name: Run unit tests (SPR)
|
||||
# ========= SPR: two socket-pinned containers, run in parallel ======== #
|
||||
- name: Run unit tests (SPR, two sockets in parallel)
|
||||
if: matrix.role == 'spr'
|
||||
timeout-minutes: 120
|
||||
run: |
|
||||
docker exec -w /sglang-checkout/ ci_sglang_${{ matrix.name }} \
|
||||
bash -c "source /opt/.venv/bin/activate && cd ./test && python3 run_suite.py --hw cpu --suite ${{ matrix.suite }} --auto-partition-id ${{ matrix.part_id }} --auto-partition-size ${{ matrix.part_size }} --timeout-from-est-time"
|
||||
set -uo pipefail
|
||||
HF_TOKEN_VAL="$(cat ~/huggingface_token.txt)"
|
||||
|
||||
# One socket-pinned container. Physical cores only (0-31 = socket0,
|
||||
# 32-63 = socket1); logical/HT siblings (64-127) are never bound.
|
||||
# --memory caps the cgroup so sglang sizes KV against one socket, and
|
||||
# --cpuset-mems keeps allocations on the local memory bank.
|
||||
# $1 socket id, $2 cpuset-cpus, $3 cpuset-mems, $4 partition id
|
||||
run_socket () {
|
||||
local sid=$1 cpus=$2 mems=$3 pid=$4
|
||||
local cname=ci_sglang_${{ matrix.name }}_s${sid}
|
||||
docker rm -f "$cname" >/dev/null 2>&1 || true
|
||||
docker run -dt \
|
||||
--cpuset-cpus="$cpus" --cpuset-mems="$mems" --memory=240g \
|
||||
-v ${{ github.workspace }}:/sglang-checkout/ --shm-size=32g \
|
||||
-v $HOME/.cache/huggingface:/root/.cache/huggingface \
|
||||
-v $HOME/.cache/sgl_eval:/root/.cache/sgl_eval \
|
||||
-e HF_TOKEN="$HF_TOKEN_VAL" \
|
||||
-e SGLANG_CPU_OMP_THREADS_BIND="$cpus" \
|
||||
--name "$cname" \
|
||||
sglang_xeon
|
||||
docker exec -w /sglang-checkout/ "$cname" bash -c \
|
||||
"source /opt/.venv/bin/activate && python3 -c 'import torch; import sgl_kernel; assert torch.cpu._is_amx_tile_supported(); assert hasattr(torch.ops.sgl_kernel, \"convert_weight_packed\")' && \
|
||||
hf auth login --token \$HF_TOKEN && \
|
||||
cd ./test && python3 run_suite.py --hw cpu --suite ${{ matrix.suite }} --auto-partition-id $pid --auto-partition-size ${{ matrix.part_size }} --timeout-from-est-time"
|
||||
}
|
||||
|
||||
# socket0 -> part_lo, socket1 -> part_hi; run concurrently, wait both.
|
||||
run_socket 0 "0-31" 0 ${{ matrix.part_lo }} > s0.log 2>&1 &
|
||||
P0=$!
|
||||
run_socket 1 "32-63" 1 ${{ matrix.part_hi }} > s1.log 2>&1 &
|
||||
P1=$!
|
||||
|
||||
rc=0
|
||||
wait $P0 || rc=1
|
||||
wait $P1 || rc=1
|
||||
|
||||
echo "==================== socket0 (partition ${{ matrix.part_lo }}) ===================="
|
||||
cat s0.log || true
|
||||
echo "==================== socket1 (partition ${{ matrix.part_hi }}) ===================="
|
||||
cat s1.log || true
|
||||
exit $rc
|
||||
|
||||
# ============================ Common ================================= #
|
||||
- name: Change permission
|
||||
if: always()
|
||||
timeout-minutes: 2
|
||||
run: |
|
||||
docker exec -u root ci_sglang_${{ matrix.name }} bash -c "rm -rf /tmp/ci-home && chown -R $(id -u):$(id -g) /sglang-checkout/ 2>/dev/null || true" 2>/dev/null || true
|
||||
# GNR has one container (ci_sglang_<name>); SPR has two socket
|
||||
# containers (ci_sglang_<name>_s0/_s1). Chown from whichever exist.
|
||||
for c in ci_sglang_${{ matrix.name }} ci_sglang_${{ matrix.name }}_s0 ci_sglang_${{ matrix.name }}_s1; do
|
||||
docker exec -u root "$c" bash -c "rm -rf /tmp/ci-home && chown -R $(id -u):$(id -g) /sglang-checkout/ 2>/dev/null || true" 2>/dev/null || true
|
||||
done
|
||||
|
||||
- name: Cleanup container
|
||||
if: always()
|
||||
run: |
|
||||
docker rm -f ci_sglang_${{ matrix.name }} || true
|
||||
for c in ci_sglang_${{ matrix.name }} ci_sglang_${{ matrix.name }}_s0 ci_sglang_${{ matrix.name }}_s1; do
|
||||
docker rm -f "$c" || true
|
||||
done
|
||||
|
||||
@@ -446,6 +446,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { runner_config: 2-gpu-gb300, run_timeout: '30', timeout_per_file: '1800' }
|
||||
- { runner_config: 4-gpu-gb300, run_timeout: '30', timeout_per_file: '1800' }
|
||||
uses: ./.github/workflows/_pr-test-stage.yml
|
||||
with:
|
||||
|
||||
@@ -2,10 +2,20 @@ name: Release Docker Images Nightly gfx1151 (AMD)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
push:
|
||||
description: "Push the image to Docker Hub (uncheck to only check that the branch builds)"
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
schedule:
|
||||
# Stagger this build from the CDNA ROCm nightlies at 12:00 UTC.
|
||||
- cron: "0 14 * * *"
|
||||
|
||||
# Scheduled runs always push; a manual dispatch can turn the push off.
|
||||
env:
|
||||
PUSH_IMAGES: ${{ github.event_name != 'workflow_dispatch' || inputs.push }}
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
@@ -59,5 +69,9 @@ jobs:
|
||||
--no-cache \
|
||||
--progress=plain \
|
||||
-t "rocm/sgl-dev:${IMAGE_TAG}"
|
||||
docker push "rocm/sgl-dev:${IMAGE_TAG}"
|
||||
echo "Published rocm/sgl-dev:${IMAGE_TAG}" >> "$GITHUB_STEP_SUMMARY"
|
||||
if [ "${PUSH_IMAGES}" = "true" ]; then
|
||||
docker push "rocm/sgl-dev:${IMAGE_TAG}"
|
||||
echo "Published rocm/sgl-dev:${IMAGE_TAG}" >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
echo "Build-only run: not pushing rocm/sgl-dev:${IMAGE_TAG}" >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
|
||||
@@ -21,8 +21,17 @@ on:
|
||||
- 'all'
|
||||
- gfx942-rocm1000
|
||||
- gfx950-rocm1000
|
||||
push:
|
||||
description: 'Push the images to Docker Hub (uncheck to only check that the branch builds)'
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
schedule:
|
||||
- cron: '0 12 * * *'
|
||||
- cron: '0 10 * * *' # before the ROCm 7.2 build, so it is not queued behind it
|
||||
|
||||
# Scheduled runs always push; a manual dispatch can turn the push off.
|
||||
env:
|
||||
PUSH_IMAGES: ${{ github.event_name != 'workflow_dispatch' || inputs.push }}
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
|
||||
@@ -106,15 +115,21 @@ jobs:
|
||||
--build-arg UBUNTU_MIRROR=https://archive.ubuntu.com \
|
||||
-t "rocm/sgl-dev:${image_tag}" \
|
||||
--no-cache
|
||||
docker push "rocm/sgl-dev:${image_tag}"
|
||||
if [ "${PUSH_IMAGES}" = "true" ]; then
|
||||
docker push "rocm/sgl-dev:${image_tag}"
|
||||
else
|
||||
echo "Build-only run: not pushing rocm/sgl-dev:${image_tag}"
|
||||
fi
|
||||
|
||||
- name: Login to Docker Hub (lmsys)
|
||||
if: env.PUSH_IMAGES == 'true'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Push to lmsysorg/sglang-rocm
|
||||
if: env.PUSH_IMAGES == 'true'
|
||||
run: |
|
||||
docker tag "rocm/sgl-dev:${IMAGE_TAG}" "lmsysorg/sglang-rocm:${IMAGE_TAG}"
|
||||
docker push "lmsysorg/sglang-rocm:${IMAGE_TAG}"
|
||||
@@ -178,15 +193,21 @@ jobs:
|
||||
--build-arg UBUNTU_MIRROR=https://archive.ubuntu.com \
|
||||
-t "rocm/sgl-dev:${image_tag}" \
|
||||
--no-cache
|
||||
docker push "rocm/sgl-dev:${image_tag}"
|
||||
if [ "${PUSH_IMAGES}" = "true" ]; then
|
||||
docker push "rocm/sgl-dev:${image_tag}"
|
||||
else
|
||||
echo "Build-only run: not pushing rocm/sgl-dev:${image_tag}"
|
||||
fi
|
||||
|
||||
- name: Login to Docker Hub (lmsys)
|
||||
if: env.PUSH_IMAGES == 'true'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Push to lmsysorg/sglang-rocm
|
||||
if: env.PUSH_IMAGES == 'true'
|
||||
run: |
|
||||
docker tag "rocm/sgl-dev:${IMAGE_TAG}" "lmsysorg/sglang-rocm:${IMAGE_TAG}"
|
||||
docker push "lmsysorg/sglang-rocm:${IMAGE_TAG}"
|
||||
@@ -253,15 +274,21 @@ jobs:
|
||||
--build-arg UBUNTU_MIRROR=https://archive.ubuntu.com \
|
||||
-t "rocm/sgl-dev:${image_tag}" \
|
||||
--no-cache
|
||||
docker push "rocm/sgl-dev:${image_tag}"
|
||||
if [ "${PUSH_IMAGES}" = "true" ]; then
|
||||
docker push "rocm/sgl-dev:${image_tag}"
|
||||
else
|
||||
echo "Build-only run: not pushing rocm/sgl-dev:${image_tag}"
|
||||
fi
|
||||
|
||||
- name: Login to Docker Hub (lmsys)
|
||||
if: env.PUSH_IMAGES == 'true'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Push to lmsysorg/sglang-rocm
|
||||
if: env.PUSH_IMAGES == 'true'
|
||||
run: |
|
||||
docker tag "rocm/sgl-dev:${IMAGE_TAG}" "lmsysorg/sglang-rocm:${IMAGE_TAG}"
|
||||
docker push "lmsysorg/sglang-rocm:${IMAGE_TAG}"
|
||||
|
||||
@@ -21,8 +21,17 @@ on:
|
||||
- gfx950-rocm724
|
||||
- gfx942-rocm720
|
||||
- gfx950-rocm720
|
||||
push:
|
||||
description: 'Push the images to Docker Hub (uncheck to only check that the branch builds)'
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
schedule:
|
||||
- cron: '0 12 * * *'
|
||||
- cron: '0 11 * * *' # an hour after ROCm 10, ahead of the 12:00 UTC MILES builds
|
||||
|
||||
# Scheduled runs always push; a manual dispatch can turn the push off.
|
||||
env:
|
||||
PUSH_IMAGES: ${{ github.event_name != 'workflow_dispatch' || inputs.push }}
|
||||
|
||||
concurrency:
|
||||
# A PR number if a pull request and otherwise the commit hash. This cancels
|
||||
@@ -112,18 +121,24 @@ jobs:
|
||||
# UBUNTU_MIRROR forces apt over HTTPS to dodge port-80 reachability flakes
|
||||
# to Canonical's archive.ubuntu.com mirror IPs from the amd-docker-scale runner.
|
||||
docker build . -f docker/rocm.Dockerfile --build-arg SGL_BRANCH=${{ github.ref_name }} --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }} --build-arg ENABLE_MORI=1 --build-arg ENABLE_NIXL=1 --build-arg SETUPTOOLS_SCM_PRETEND_VERSION=${pretend_version} --build-arg UBUNTU_MIRROR=https://archive.ubuntu.com -t rocm/sgl-dev:${tag}-${{ env.DATE }} --no-cache
|
||||
docker push rocm/sgl-dev:${tag}-${{ env.DATE }}
|
||||
if [ "${PUSH_IMAGES}" = "true" ]; then
|
||||
docker push rocm/sgl-dev:${tag}-${{ env.DATE }}
|
||||
else
|
||||
echo "Build-only run: not pushing rocm/sgl-dev:${tag}-${{ env.DATE }}"
|
||||
fi
|
||||
|
||||
# Persist the tag right after rocm/sgl-dev push succeeds so the local
|
||||
# registry mirror can run even if a later step in this job (lmsys push)
|
||||
# fails. By default this step only runs when the previous step succeeded,
|
||||
# so the artifact only exists when an image actually landed on Docker Hub.
|
||||
- name: Save published image tag
|
||||
if: env.PUSH_IMAGES == 'true'
|
||||
run: |
|
||||
mkdir -p image-tag
|
||||
echo "${{ env.IMAGE_TAG }}" > "image-tag/${{ matrix.gpu_arch }}.txt"
|
||||
|
||||
- name: Upload image tag artifact
|
||||
if: env.PUSH_IMAGES == 'true'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: image-tag-${{ matrix.gpu_arch }}
|
||||
@@ -131,12 +146,14 @@ jobs:
|
||||
retention-days: 1
|
||||
|
||||
- name: Login to Docker Hub (lmsys)
|
||||
if: env.PUSH_IMAGES == 'true'
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Push to lmsysorg/sglang-rocm
|
||||
if: env.PUSH_IMAGES == 'true'
|
||||
run: |
|
||||
docker tag rocm/sgl-dev:${{ env.IMAGE_TAG }} lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
|
||||
docker push lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
|
||||
|
||||
@@ -57,6 +57,7 @@ jobs:
|
||||
kernel_source=wheel
|
||||
kernel_ref=main
|
||||
latest_tag=latest
|
||||
image_tag="${image_tag}-main-whl"
|
||||
else
|
||||
# Pin the kernel commit so the tag names exactly what was built.
|
||||
kernel_ref=$(git ls-remote https://github.com/sgl-project/sgl-kernel-xpu.git refs/heads/main | cut -f1)
|
||||
@@ -66,7 +67,7 @@ jobs:
|
||||
fi
|
||||
kernel_source=source
|
||||
latest_tag=latest-kernel-main
|
||||
image_tag="${image_tag}-kernel-main-${kernel_ref:0:7}"
|
||||
image_tag="${image_tag}-main-main-${kernel_ref:0:7}"
|
||||
fi
|
||||
echo "DATE=${date_tag}" >> $GITHUB_ENV
|
||||
echo "commit_hash=${commit_hash}" >> $GITHUB_OUTPUT
|
||||
|
||||
@@ -83,7 +83,7 @@ jobs:
|
||||
provenance: false
|
||||
sbom: false
|
||||
build-args: |
|
||||
SGLANG_KERNEL_NPU_TAG=2026.9.0.post9
|
||||
SGLANG_KERNEL_NPU_TAG=2026.9.0.post10
|
||||
CANN_VERSION=${{ matrix.cann_version }}
|
||||
DEVICE_TYPE=${{ matrix.device_type }}
|
||||
|
||||
|
||||
@@ -87,7 +87,7 @@ jobs:
|
||||
push: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }}
|
||||
provenance: false
|
||||
build-args: |
|
||||
SGLANG_KERNEL_NPU_TAG=2026.9.0.post9
|
||||
SGLANG_KERNEL_NPU_TAG=2026.9.0.post10
|
||||
CANN_VERSION=${{ matrix.cann_version }}
|
||||
DEVICE_TYPE=${{ matrix.device_type }}
|
||||
SGLANG_TAG=${{ steps.version.outputs.version }}
|
||||
|
||||
@@ -161,7 +161,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Update wheel index
|
||||
run: |
|
||||
|
||||
@@ -170,7 +170,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Update wheel index
|
||||
run: |
|
||||
|
||||
@@ -104,7 +104,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Upload to pypi
|
||||
run: |
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
||||
python-version: ["3.11", "3.12", "3.13"]
|
||||
arch: [x86_64, aarch64]
|
||||
include:
|
||||
- arch: x86_64
|
||||
|
||||
@@ -168,7 +168,6 @@ jobs:
|
||||
# so bind eagerly to fail here on any missing symbol.
|
||||
LD_BIND_NOW=1 python3 - <<'PY'
|
||||
import ctypes
|
||||
import sys
|
||||
import torch
|
||||
import deep_ep
|
||||
import deep_gemm
|
||||
@@ -179,9 +178,6 @@ jobs:
|
||||
assert ctypes.CDLL("libnccl.so.2").ncclGetVersion(ctypes.byref(nccl_version)) == 0
|
||||
assert nccl_version.value == 23007, nccl_version.value
|
||||
assert hasattr(deep_ep, "ElasticBuffer"), "DeepEP ElasticBuffer is required"
|
||||
sys.path.insert(0, "DeepGEMM/third-party")
|
||||
from tilelang_ops import ref_mhc
|
||||
assert ref_mhc.has_baseline(), "Mega mHC reference operators are unavailable"
|
||||
print("DeepGEMM test dependencies ready; NCCL:", nccl_version.value)
|
||||
PY
|
||||
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
name: Release sgl-flash-mla
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: "Wheel version (e.g. 1.0.0, 1.0.1rc0)"
|
||||
type: string
|
||||
required: true
|
||||
branch:
|
||||
description: "FlashMLA branch to build from (default: sgl)"
|
||||
type: string
|
||||
required: false
|
||||
default: 'sgl'
|
||||
validation_only:
|
||||
description: "Build and test without publishing wheels"
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
skip_tests:
|
||||
description: "Skip the GPU test suite (publishes untested wheels)"
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
|
||||
concurrency:
|
||||
group: release-sgl-flash-mla-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# The wheel pins this; it is also what the test jobs install.
|
||||
# The extensions use PyTorch's stable C++ ABI and would load against any 2.10+,
|
||||
# but a release is tested against one version.
|
||||
TORCH_VER: "2.14.1"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: github.repository == 'sgl-project/sglang'
|
||||
strategy:
|
||||
matrix:
|
||||
# One wheel per architecture. No Python matrix: the wheel is abi3
|
||||
# (cp310-abi3) and serves CPython 3.10+, so 3.12 only drives the build.
|
||||
python-version: ["3.12"]
|
||||
cuda-version: ["13.4"]
|
||||
arch: [x86_64, aarch64]
|
||||
include:
|
||||
- arch: x86_64
|
||||
runner: x64-kernel-build-node
|
||||
- arch: aarch64
|
||||
runner: arm-kernel-build-node
|
||||
runs-on: ${{ matrix.runner }}
|
||||
steps:
|
||||
- name: Clean workspace (remove root-owned files from prior runs)
|
||||
run: |
|
||||
docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
|
||||
sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Checkout FlashMLA
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: sgl-project/FlashMLA
|
||||
ref: ${{ inputs.branch || 'sgl' }}
|
||||
path: FlashMLA
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Set wheel version
|
||||
run: |
|
||||
echo -n "${{ inputs.version }}" > FlashMLA/flash_mla/VERSION
|
||||
cat FlashMLA/flash_mla/VERSION
|
||||
|
||||
- name: Build wheel
|
||||
run: |
|
||||
chmod +x ./scripts/build_sgl_flash_mla.sh ./scripts/rename_sgl_flash_mla_whl.sh
|
||||
./scripts/build_sgl_flash_mla.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" "${{ github.workspace }}/FlashMLA" "${{ matrix.arch }}"
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: flashmla-wheel-${{ matrix.arch }}
|
||||
path: FlashMLA/dist/*.whl
|
||||
|
||||
- name: Upload PyPI artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: flashmla-pypi-${{ matrix.arch }}
|
||||
path: FlashMLA/dist-pypi/*.whl
|
||||
|
||||
test:
|
||||
needs: build
|
||||
if: ${{ !inputs.skip_tests }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# SM90 runs only the frozen kernels: the current ones are sm_100a/sm_103a.
|
||||
# The frozen ones are built for sm_90a/sm_100a/sm_103a, so SM100 runs both.
|
||||
- arch_label: sm90
|
||||
runner: 8-gpu-h200
|
||||
wheel_arch: x86_64
|
||||
tests: legacy
|
||||
timeout: 90
|
||||
- arch_label: sm100
|
||||
runner: 8-gpu-b200
|
||||
wheel_arch: x86_64
|
||||
tests: current legacy
|
||||
timeout: 180
|
||||
- arch_label: sm103-aarch64
|
||||
runner: 4-gpu-gb300
|
||||
wheel_arch: aarch64
|
||||
tests: current legacy
|
||||
timeout: 180
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: ${{ matrix.timeout }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Checkout FlashMLA
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: sgl-project/FlashMLA
|
||||
ref: ${{ inputs.branch || 'sgl' }}
|
||||
path: FlashMLA
|
||||
submodules: recursive
|
||||
|
||||
# The tests need only tests/. The source package would shadow the installed
|
||||
# wheel (its directory has no built extensions), so it is removed.
|
||||
- name: Remove the FlashMLA source package
|
||||
run: rm -rf FlashMLA/flash_mla
|
||||
|
||||
- name: Download wheel
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: dist/
|
||||
merge-multiple: true
|
||||
pattern: flashmla-wheel-${{ matrix.wheel_arch }}
|
||||
|
||||
- name: Install wheel and deps
|
||||
run: |
|
||||
python3 -m pip install --upgrade pip
|
||||
python3 -m pip install "torch==${TORCH_VER}" --index-url https://download.pytorch.org/whl/cu130 --force-reinstall
|
||||
python3 -m pip install numpy einops rich pytest
|
||||
# --force-reinstall: a base image may already carry this version, in
|
||||
# which case pip skips the wheel and the job tests a stale extension.
|
||||
python3 -m pip install dist/*.whl --force-reinstall --no-deps
|
||||
# Both extensions register through STABLE_TORCH_LIBRARY at dlopen time,
|
||||
# so a missing symbol surfaces here rather than at the first launch.
|
||||
python3 -c "
|
||||
import flash_mla
|
||||
from flash_mla._loader import get_ops
|
||||
print('flash_mla:', flash_mla.__file__)
|
||||
import os
|
||||
assert not flash_mla.__file__.startswith(os.environ['GITHUB_WORKSPACE']), 'not the installed wheel'
|
||||
print('version:', flash_mla.__version__)
|
||||
print('has_legacy:', flash_mla.has_legacy())
|
||||
assert get_ops(use_legacy=False).sparse_decode_fwd is not None
|
||||
assert get_ops(use_legacy=True).dense_decode_fwd is not None
|
||||
"
|
||||
|
||||
- name: Run tests (current kernels)
|
||||
if: contains(matrix.tests, 'current')
|
||||
working-directory: FlashMLA
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}/FlashMLA/tests
|
||||
run: |
|
||||
python3 -c "import os, flash_mla; assert not flash_mla.__file__.startswith(os.environ['GITHUB_WORKSPACE']), flash_mla.__file__"
|
||||
python3 tests/test-sparse-prefill.py
|
||||
python3 tests/test-sparse-decode.py
|
||||
|
||||
- name: Run tests (frozen kernels)
|
||||
if: contains(matrix.tests, 'legacy')
|
||||
working-directory: FlashMLA
|
||||
env:
|
||||
# conftest.py only fixes sys.path under pytest; these are scripts.
|
||||
PYTHONPATH: ${{ github.workspace }}/FlashMLA/tests
|
||||
run: |
|
||||
python3 -c "import os, flash_mla; assert not flash_mla.__file__.startswith(os.environ['GITHUB_WORKSPACE']), flash_mla.__file__"
|
||||
python3 -m pytest tests/legacy/ -q
|
||||
# Dense MLA decoding is SM90-only.
|
||||
if [ "${{ matrix.arch_label }}" = sm90 ]; then
|
||||
python3 tests/legacy/test_flash_mla_dense_decoding.py
|
||||
fi
|
||||
python3 tests/legacy/test_flash_mla_sparse_prefill.py
|
||||
python3 tests/legacy/test_flash_mla_sparse_decoding.py
|
||||
python3 tests/legacy/test_flash_mla_sparse_decoding_v32_bit_equal.py
|
||||
|
||||
release:
|
||||
if: |
|
||||
!cancelled() && !inputs.validation_only &&
|
||||
needs.build.result == 'success' &&
|
||||
(needs.test.result == 'success' || needs.test.result == 'skipped')
|
||||
needs: [build, test]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: dist/
|
||||
merge-multiple: true
|
||||
pattern: flashmla-wheel-*
|
||||
|
||||
- name: Download PyPI artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: dist-pypi/
|
||||
merge-multiple: true
|
||||
pattern: flashmla-pypi-*
|
||||
|
||||
- name: Record that the GPU test suite was skipped
|
||||
if: ${{ inputs.skip_tests }}
|
||||
run: |
|
||||
echo "Releasing v${{ inputs.version }} with the GPU test suite skipped." \
|
||||
>> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload to PyPI
|
||||
run: |
|
||||
pip install --upgrade twine "packaging>=24.2"
|
||||
python3 -m twine upload --skip-existing dist-pypi/* -u __token__ -p ${{ secrets.SGL_FLASH_MLA_PYPI_TOKEN }}
|
||||
|
||||
- name: Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: v${{ inputs.version }}
|
||||
repository: sgl-project/whl
|
||||
token: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
|
||||
files: |
|
||||
dist/*
|
||||
|
||||
- name: Clone wheel index
|
||||
run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl
|
||||
env:
|
||||
WHL_TOKEN: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
|
||||
|
||||
- name: Update wheel index
|
||||
run: python3 scripts/update_flashmla_whl_index.py --cuda 130
|
||||
|
||||
- name: Push wheel index
|
||||
run: |
|
||||
cd sgl-whl
|
||||
git config --local user.name "sglang-bot"
|
||||
git config --local user.email "sglangbot@gmail.com"
|
||||
git add -A
|
||||
git commit -m "update sgl-flash-mla whl index for v${{ inputs.version }}${{ inputs.skip_tests && ' (GPU tests skipped)' || '' }}"
|
||||
git push
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
(github.event_name == 'push' || github.event.inputs.target == 'all' || github.event.inputs.target == 'cu130')
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.10"]
|
||||
python-version: ["3.11"]
|
||||
cuda-version: ["13.0"]
|
||||
arch: [x86_64, aarch64]
|
||||
include:
|
||||
@@ -174,8 +174,9 @@ jobs:
|
||||
matrix:
|
||||
python-version: ["3.10"]
|
||||
# ROCm 10.0.0 builds inside a Python 3.12 container; the wheel is still
|
||||
# tagged cp310-abi3 (wheel.py-api in pyproject.toml), so it stays under
|
||||
# the same python-version leg as the 7.x wheels.
|
||||
# tagged cp310-abi3 (build_rocm.sh sets wheel.py-api to cp310 for every
|
||||
# ROCm flavor), so it stays under the same python-version leg as the 7.x
|
||||
# wheels.
|
||||
#
|
||||
# A dispatch that names one flavor builds only that flavor. Every extra
|
||||
# leg is another image build on the shared AMD node, and its release job
|
||||
|
||||
@@ -183,7 +183,7 @@ jobs:
|
||||
env:
|
||||
TEST_COMMAND: ${{ matrix.test_command }}
|
||||
REQUESTED_TEST_COMMAND: ${{ inputs.test_command }}
|
||||
HF_MUSE_SPARK_TOKEN: ${{ inputs.runs_on == '8-gpu-b300' && inputs.pr_head_sha == '' && contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_12_b300_pd.py') && secrets.HF_MUSE_SPARK_TOKEN || '' }}
|
||||
HF_MUSE_SPARK_TOKEN: ${{ inputs.pr_head_sha == '' && inputs.runs_on == '8-gpu-b300' && (contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_12_b300_pd.py') || contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_unified_memory.py')) && secrets.HF_MUSE_SPARK_TOKEN || '' }}
|
||||
SGLANG_PRECISION_HF_TOKEN: ${{ inputs.refresh_precision_baseline && inputs.mode == 'cuda' && inputs.runs_on == '8-gpu-h200' && inputs.test_command == 'registered/debug_utils/test_nightly_precision_regression.py' && inputs.pr_head_sha == '' && secrets.HF_TOKEN_PRECISION_STORE || '' }}
|
||||
run: |
|
||||
if [[ "${{ inputs.refresh_precision_baseline }}" == "true" ]]; then
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
# Runs only the per-commit CPU unit suite (base-a-test-cpu) against a PR head,
|
||||
# dispatched by the `/run-cpu-ci` slash command. A cheap check for small PRs:
|
||||
# pair it with `/rerun-group <module>` instead of a full `run-ci` round.
|
||||
#
|
||||
# Always dispatched on main, so the workflow definition and every reusable
|
||||
# workflow it calls are trusted; only the code under test comes from the PR.
|
||||
# That code runs on hosted runners with a read-only token and no secrets
|
||||
# (nothing here inherits them), the same exposure as `/rerun-test` cpu mode.
|
||||
name: Run CPU CI
|
||||
run-name: "[run-cpu-ci] ${{ inputs.git_ref }}"
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
git_ref:
|
||||
description: "PR head SHA to test"
|
||||
required: true
|
||||
type: string
|
||||
pr_number:
|
||||
description: "PR number (informational)"
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
# Read by _pr-test-stage-cpu.yml via caller_inputs. Its fail-fast health
|
||||
# check inspects sibling jobs of a pull_request run, which this is not.
|
||||
skip_pr_test_health_check:
|
||||
description: "Skip the PR test health check fail-fast"
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
# Also read via caller_inputs. This run is on main's ref, so a cache it
|
||||
# saved would be restored by main and every PR, yet hold PR-written files.
|
||||
skip_cache_save:
|
||||
description: "Do not save the HF / Rust caches"
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
concurrency:
|
||||
group: run-cpu-ci-${{ inputs.git_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: read
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
# pr_test_yml stays pr-test.yml: compute_partitions.py reads it from the PR
|
||||
# checkout, which may predate this file, and sizing shards off the per-commit
|
||||
# stage's run_timeout_minutes keeps the split identical to PR CI.
|
||||
check-changes:
|
||||
uses: ./.github/workflows/_pr-test-check-changes.yml
|
||||
with:
|
||||
git_ref: ${{ inputs.git_ref }}
|
||||
run_all_tests: true
|
||||
|
||||
# self_name must be the suite name: the stage passes it to run_suite.py.
|
||||
# No rust_ext_artifact: its compile job runs on a self-hosted build node, so
|
||||
# the stage falls back to its cache and compiles on a miss.
|
||||
base-a-test-cpu:
|
||||
needs: check-changes
|
||||
uses: ./.github/workflows/_pr-test-stage-cpu.yml
|
||||
with:
|
||||
self_name: base-a-test-cpu
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
||||
run_timeout_minutes: '15'
|
||||
@@ -7,6 +7,7 @@ on:
|
||||
paths:
|
||||
- '.github/workflows/runner-utilization.yml'
|
||||
- 'scripts/ci/utils/runner_utilization_report.py'
|
||||
- 'scripts/ci/utils/test_runner_utilization_report.py'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
hours:
|
||||
@@ -50,7 +51,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Unit-test report logic
|
||||
# Pure-logic tests (stdlib only, no API calls) — guards the
|
||||
@@ -70,7 +71,7 @@ jobs:
|
||||
run: |
|
||||
# PR-trigger is just a script smoke check; scan a 20-minute window
|
||||
# (0.34h) to bound API cost. schedule / dispatch run the full report.
|
||||
python scripts/ci/utils/runner_utilization_report.py \
|
||||
python -u scripts/ci/utils/runner_utilization_report.py \
|
||||
--repo ${{ github.repository }} \
|
||||
--hours ${{ (github.event_name == 'pull_request' && '0.34') || inputs.hours || '24' }} \
|
||||
${{ inputs.filter && format('--filter {0}', inputs.filter) || '' }} \
|
||||
|
||||
@@ -27,6 +27,8 @@ jobs:
|
||||
contains(github.event.comment.body, '/run-extra-ci') ||
|
||||
contains(github.event.comment.body, '/rerun-full-ci') ||
|
||||
contains(github.event.comment.body, '/rerun-extra-ci') ||
|
||||
contains(github.event.comment.body, '/run-cpu-ci') ||
|
||||
contains(github.event.comment.body, '/rerun-cpu-ci') ||
|
||||
contains(github.event.comment.body, '/rerun-group') ||
|
||||
contains(github.event.comment.body, '/rerun-test') ||
|
||||
startsWith(github.event.comment.body, '/tag-') ||
|
||||
@@ -92,7 +94,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Update est_time values
|
||||
env:
|
||||
|
||||
+3
-2
@@ -73,7 +73,8 @@ instance/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
# `**/` because .dockerignore links here and Docker anchors bare patterns at the context root.
|
||||
**/target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
@@ -232,7 +233,7 @@ work_dirs/
|
||||
Cargo.lock
|
||||
!rust/Cargo.lock
|
||||
!rust/sglang-radix-tree/Cargo.lock
|
||||
!experimental/sgl-router/Cargo.lock
|
||||
!sgl-router/Cargo.lock
|
||||
|
||||
# Other repos
|
||||
lmms-eval
|
||||
|
||||
+15
-7
@@ -11,10 +11,6 @@ repos:
|
||||
- id: end-of-file-fixer
|
||||
- id: check-yaml
|
||||
args: [--allow-multiple-documents]
|
||||
# Helm chart templates embed Go template syntax ({{- ... -}})
|
||||
# that is not valid YAML on its own; the rendered output is
|
||||
# validated by `helm template` / `helm lint`.
|
||||
exclude: ^experimental/sgl-router/helm/.*/templates/.*\.(yaml|tpl)$
|
||||
- id: check-toml
|
||||
- id: check-ast
|
||||
- id: check-added-large-files
|
||||
@@ -74,6 +70,12 @@ repos:
|
||||
- '--extra-keys=metadata.kernelspec metadata.language_info.version'
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: forbid-test-srt-path
|
||||
name: forbid test/srt/ (use test/registered/ or test/manual/)
|
||||
entry: >-
|
||||
test/srt/ is retired. Move these tests under test/registered/ or test/manual/ instead.
|
||||
language: fail
|
||||
files: ^test/srt/
|
||||
- id: forbid-docs-new-path
|
||||
name: forbid docs_new/ (renamed to docs/)
|
||||
# docs_new/ was renamed to docs/ in #32123. `language: fail` fires only
|
||||
@@ -110,6 +112,12 @@ repos:
|
||||
language: system
|
||||
files: ^(\.github/actions/download-rust-ext/action\.yml|\.github/workflows/_pr-test-rust-ext-build\.yml)$
|
||||
pass_filenames: false
|
||||
- id: check-dockerfile-wheel-pins
|
||||
name: check Dockerfile sgl-deep-gemm/sgl-deep-ep versions match python/pyproject.toml
|
||||
entry: python3 scripts/lint/check_dockerfile_wheel_pins.py
|
||||
language: system
|
||||
files: ^(python/pyproject\.toml|docker/.*Dockerfile.*|scripts/lint/check_dockerfile_wheel_pins\.py)$
|
||||
pass_filenames: false
|
||||
- id: check-no-bare-pytest-main
|
||||
name: reject bare pytest.main calls in __main__ blocks
|
||||
entry: python3 scripts/lint/check_no_bare_pytest_main.py
|
||||
@@ -134,10 +142,10 @@ repos:
|
||||
files: ^sgl-model-gateway/.*\.rs$
|
||||
pass_filenames: false
|
||||
- id: rustfmt-sgl-router
|
||||
name: rustfmt experimental/sgl-router
|
||||
entry: bash -c 'cd experimental/sgl-router && cargo fmt --all -- --check'
|
||||
name: rustfmt sgl-router
|
||||
entry: bash -c 'cd sgl-router && cargo fmt --all -- --check'
|
||||
language: system
|
||||
files: ^experimental/sgl-router/.*\.rs$
|
||||
files: ^sgl-router/.*\.rs$
|
||||
pass_filenames: false
|
||||
- id: clippy-rust-workspace
|
||||
name: clippy rust/ workspace (auto-fix)
|
||||
|
||||
+15
-1
@@ -56,6 +56,19 @@ if [[ "${ROCM_VERSION}" == "1000" ]]; then
|
||||
WHEEL_ROCM_VERSION="1000"
|
||||
fi
|
||||
|
||||
# The build reuses the CUDA pyproject.toml, which requires Python 3.11 and tags
|
||||
# its wheel cp311-abi3. The ROCm 7.2.0 image and amd-sglang still run on Python
|
||||
# 3.10, so the ROCm wheels keep the Python 3.10 floor and the cp310-abi3 tag.
|
||||
sed -i \
|
||||
-e 's/^requires-python = ">=3.11"$/requires-python = ">=3.10"/' \
|
||||
-e 's/^wheel.py-api = "cp311"$/wheel.py-api = "cp310"/' \
|
||||
pyproject.toml
|
||||
if ! grep -qx 'requires-python = ">=3.10"' pyproject.toml \
|
||||
|| ! grep -qx 'wheel.py-api = "cp310"' pyproject.toml; then
|
||||
echo "ERROR: could not set the ROCm Python 3.10 / cp310 metadata in pyproject.toml." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Pull and run the latest image
|
||||
echo "Pulling Docker image: ${IMAGE}"
|
||||
docker pull "${IMAGE}"
|
||||
@@ -86,5 +99,6 @@ ${ROCM_SETUP}
|
||||
rm -rf CMakeLists.txt && mv CMakeLists_rocm.txt CMakeLists.txt && \
|
||||
${PYTHON_ROOT_PATH}/python rocm_hipify.py && \
|
||||
${PYTHON_ROOT_PATH}/python -m uv build --wheel -Cbuild-dir=build . --color=always --no-build-isolation && \
|
||||
./rename_wheels_rocm.sh ${WHEEL_ROCM_VERSION}
|
||||
./rename_wheels_rocm.sh ${WHEEL_ROCM_VERSION} && \
|
||||
${PYTHON_ROOT_PATH}/pip install --dry-run --no-deps dist/*.whl
|
||||
"
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
# Gitea Actions 用(海外节点):原版 docker.io 基底 + 原版 pypi
|
||||
# 与 b300 离线版(/data/ymk/build/Dockerfile)的区别:
|
||||
# - 基底直用 docker.io 的 lmsysorg/sglang:dev-dsv41(不走 umirror/DaoCloud)
|
||||
# - 不设 PIP_INDEX_URL,用默认 pypi.org
|
||||
# - 源码由 CI checkout 后经 COPY 进镜像(不用 in-image clone)
|
||||
FROM lmsysorg/sglang:dev-dsv41
|
||||
|
||||
# checkout 含 .git,setuptools-scm 可打戳(main 系分支 version 显示 dev 属正常)
|
||||
COPY . /sgl-workspace/sglang
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译,与 b300 离线版 deploy/b300-build/Dockerfile 同构。
|
||||
# 基底自带 rustup stable,RUSTUP_TOOLCHAIN=stable 绕过 rust-toolchain.toml 的 1.90 钉版;
|
||||
# python 扩展必须显式 -p,产物重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件。
|
||||
# 海外 runner 直连 crates.io 无网络问题;cache mount 海外环境同样无害。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
RUN pip install --no-cache-dir -e /sgl-workspace/sglang/python
|
||||
@@ -22,6 +22,11 @@ SGLang is an open-source inference framework for large language, vision-language
|
||||
|
||||
👋 Get started below, or meet the community at [SGLang Events](https://www.sglang.io/events), including meetups, developer meetings, workshops, and office hours.
|
||||
|
||||
> **[SGLang Summit 2026: Meet the builders of open AI](https://www.sglang.io/summit)**<br>
|
||||
> **Our flagship annual gathering · November 12–13 · San Francisco**
|
||||
>
|
||||
> Go deep on inference, training, and AI hardware through technical talks, live demos, and hands-on workshops. Bring your deployment challenges and meet the engineers and researchers building the stack.
|
||||
|
||||
## Get Started
|
||||
|
||||
Pull the Docker image, which includes SGLang and its dependencies:
|
||||
@@ -49,7 +54,7 @@ SGLang supports a wide range of GPUs, TPUs, NPUs, CPUs, and Apple Silicon platfo
|
||||
| --- | --- |
|
||||
| [NVIDIA](https://docs.sglang.io/docs/hardware-platforms/nvidia-gpus) | B200/B300/GB200/GB300; H100/H200/H800/H20; A100; RTX 30/40/50 series, RTX 6000 Ada / PRO 6000; DGX Spark, Jetson Orin |
|
||||
| [AMD](https://docs.sglang.io/docs/hardware-platforms/amd_gpu) | Instinct MI300X, MI325X, MI350X, MI355X |
|
||||
| [Google TPU](https://docs.sglang.io/docs/hardware-platforms/tpu) | v6e, v7; [SGL-JAX](https://github.com/sgl-project/sglang-jax) / [SGL-torchtpu](https://lmsys.org/blog/2026-07-30-sglang-google-tpu/) |
|
||||
| [Google TPU](https://docs.sglang.io/docs/hardware-platforms/tpu) | v6e, v7; [SGLang-TorchTPU](https://github.com/sgl-project/sglang-torchtpu) / [SGL-JAX](https://github.com/sgl-project/sglang-jax) |
|
||||
| Intel ([GPU](https://docs.sglang.io/docs/hardware-platforms/xpu) / [CPU](https://docs.sglang.io/docs/hardware-platforms/cpu_server)) | Arc / Arc Pro B-Series GPUs, Xeon CPUs |
|
||||
| [Apple Silicon](https://docs.sglang.io/docs/hardware-platforms/apple_metal) | Macs via Metal / MLX |
|
||||
| [Huawei Ascend](https://docs.sglang.io/docs/hardware-platforms/ascend-npus/getting-started/installation) | A2, A3, 950PR/DT NPUs |
|
||||
@@ -57,6 +62,8 @@ SGLang supports a wide range of GPUs, TPUs, NPUs, CPUs, and Apple Silicon platfo
|
||||
|
||||
Integrations in progress: AWS Trainium, [Alibaba T-Head PPU](https://github.com/sgl-project/sglang/issues/37519), [Cambricon MLU](https://github.com/sgl-project/sglang/pull/26898), Qualcomm QAIC, MetaX, Hygon HCU/DCU, Iluvatar CoreX, and more.
|
||||
|
||||
SGLang-TorchTPU brings the SGLang serving engine to Google TPUs through PyTorch, torch_tpu, and Pallas kernels. It supports LLM and vision-language serving on TPU7x (Ironwood), including tensor and expert parallelism across hosts. SGL-JAX provides the JAX-based TPU serving path.
|
||||
|
||||
See the [Cookbook](https://cookbook.sglang.io/) and platform guides for model compatibility and setup.
|
||||
|
||||
## SGL Ecosystem
|
||||
|
||||
@@ -20,10 +20,10 @@ import triton
|
||||
from sglang.kernels.ops.attention.cutedsl_kda import (
|
||||
cutedsl_fused_sigmoid_gating_kda_update,
|
||||
)
|
||||
from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import (
|
||||
from sglang.kernels.ops.attention.fla.kda import chunk_kda
|
||||
from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import (
|
||||
fused_sigmoid_gating_delta_rule_update,
|
||||
)
|
||||
from sglang.kernels.ops.attention.fla.kda import chunk_kda
|
||||
|
||||
|
||||
def make_inputs(
|
||||
|
||||
@@ -32,7 +32,7 @@ import triton
|
||||
from sglang.kernels.ops.attention.fla.fused_recurrent import (
|
||||
fused_recurrent_gated_delta_rule_packed_decode,
|
||||
)
|
||||
from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import (
|
||||
from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import (
|
||||
fused_sigmoid_gating_delta_rule_update,
|
||||
)
|
||||
|
||||
|
||||
@@ -26,12 +26,12 @@ import torch
|
||||
from sglang.kernels.ops.attention.fla.fused_recurrent import (
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import (
|
||||
fused_sigmoid_gating_delta_rule_update,
|
||||
)
|
||||
from sglang.kernels.ops.attention.helion.kda_decode import (
|
||||
helion_fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import (
|
||||
fused_sigmoid_gating_delta_rule_update,
|
||||
)
|
||||
|
||||
|
||||
def make_inputs(
|
||||
|
||||
@@ -37,7 +37,7 @@ import triton
|
||||
from sglang.kernels.ops.attention.fla.fused_recurrent import (
|
||||
fused_recurrent_gated_delta_rule_packed_decode,
|
||||
)
|
||||
from sglang.kernels.ops.attention.fla.fused_recurrent_linear_replayssm import (
|
||||
from sglang.kernels.ops.attention.linear.fused_recurrent_linear_replayssm import (
|
||||
fused_recurrent_gdn_replayssm_decode,
|
||||
)
|
||||
|
||||
|
||||
@@ -22,11 +22,11 @@ import argparse
|
||||
|
||||
import torch
|
||||
|
||||
import sglang.kernels.ops.attention.fla.fused_kda_conv_recurrent_verify as fused_mod
|
||||
from sglang.kernels.ops.attention.fla.fused_kda_conv_recurrent_verify import (
|
||||
import sglang.kernels.ops.attention.linear.fused_kda_conv_recurrent_verify as fused_mod
|
||||
from sglang.kernels.ops.attention.linear.fused_kda_conv_recurrent_verify import (
|
||||
fused_kda_conv_gating_verify,
|
||||
)
|
||||
from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import (
|
||||
from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import (
|
||||
fused_sigmoid_gating_delta_rule_update,
|
||||
)
|
||||
from sglang.kernels.ops.mamba.causal_conv1d_triton import causal_conv1d_update
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
Compares the legacy gather (``gather_dequant_requant_fp8_paged_legacy``:
|
||||
fresh ``torch.zeros`` destination + one program per (token, 128-elem
|
||||
slice)) against the new gather (``gather_dequant_requant_fp8_paged``:
|
||||
slice)) against the new gather (``gather_dequant_requant_fp8_paged_dsa``:
|
||||
no pre-zeroing needed, fused pad-row zero-fill, TOKENS_PER_PROG tokens
|
||||
per program with 16B-vectorized access), in three flavors:
|
||||
|
||||
@@ -43,6 +43,8 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
_MOD_PATH = _REPO_ROOT / "python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py"
|
||||
@@ -54,8 +56,127 @@ _spec = importlib.util.spec_from_file_location("dequant_k_cache", _MOD_PATH)
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(_mod)
|
||||
|
||||
gather_new = _mod.gather_dequant_requant_fp8_paged
|
||||
gather_legacy = _mod.gather_dequant_requant_fp8_paged_legacy
|
||||
gather_new = _mod.gather_dequant_requant_fp8_paged_dsa
|
||||
_infer_dsa_dims = _mod._infer_dsa_dims
|
||||
|
||||
|
||||
def gather_dequant_requant_fp8_paged_legacy(
|
||||
quant_k_cache: torch.Tensor,
|
||||
page_table_1_flattened: torch.Tensor,
|
||||
group_size: int = 128,
|
||||
extra_rows: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""Legacy (pre-vectorization) gather + dequant + requant.
|
||||
|
||||
Kept as the bit-exactness / performance reference for
|
||||
``gather_dequant_requant_fp8_paged_dsa`` (see
|
||||
``benchmark/kernels/deepseek/benchmark_q8kv8_kv_gather.py``). Allocates and zero-fills the
|
||||
full destination each call, then launches one program per
|
||||
(token, 128-elem slice).
|
||||
"""
|
||||
dim_quant = quant_k_cache.shape[-1]
|
||||
dim_nope, dim_rope = _infer_dsa_dims(dim_quant)
|
||||
quant_k_cache = quant_k_cache.view((-1, dim_quant))
|
||||
|
||||
num_tokens = page_table_1_flattened.shape[0]
|
||||
assert quant_k_cache.dtype == torch.float8_e4m3fn
|
||||
num_tiles = dim_nope // group_size
|
||||
out_dim = dim_nope + dim_rope
|
||||
assert num_tiles * group_size == dim_nope
|
||||
|
||||
total_rows = num_tokens + extra_rows
|
||||
# Allocate a fresh zero-filled buffer. The extra landing-pad rows at
|
||||
# the tail must read as zeros (the kernel may over-read past
|
||||
# num_tokens for masked indices).
|
||||
output = torch.zeros(
|
||||
(total_rows, 1, out_dim),
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=quant_k_cache.device,
|
||||
)
|
||||
|
||||
num_blocks_per_token = num_tiles + triton.cdiv(dim_rope, group_size)
|
||||
|
||||
input_nope_q = quant_k_cache[:, :dim_nope]
|
||||
input_nope_s = quant_k_cache[:, dim_nope : dim_nope + num_tiles * 4].view(
|
||||
torch.float32
|
||||
)
|
||||
input_rope = quant_k_cache[:, dim_nope + num_tiles * 4 :].view(torch.bfloat16)
|
||||
|
||||
_gather_dequant_requant_fp8_paged_kernel[(num_tokens, num_blocks_per_token)](
|
||||
output,
|
||||
input_nope_q,
|
||||
input_nope_s,
|
||||
input_rope,
|
||||
page_table_1_flattened,
|
||||
output.stride(0),
|
||||
input_nope_q.stride(0),
|
||||
input_nope_s.stride(0),
|
||||
input_rope.stride(0),
|
||||
NUM_NOPE_BLOCKS=num_tiles,
|
||||
GROUP_SIZE=group_size,
|
||||
DIM_NOPE=dim_nope,
|
||||
DIM_ROPE=dim_rope,
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_dequant_requant_fp8_paged_kernel(
|
||||
output_ptr,
|
||||
input_nope_q_ptr,
|
||||
input_nope_s_ptr,
|
||||
input_rope_ptr,
|
||||
page_table_1_ptr,
|
||||
output_stride_0: int,
|
||||
input_nope_q_stride_0: int,
|
||||
input_nope_s_stride_0: int,
|
||||
input_rope_stride_0: int,
|
||||
NUM_NOPE_BLOCKS: tl.constexpr,
|
||||
GROUP_SIZE: tl.constexpr,
|
||||
DIM_NOPE: tl.constexpr,
|
||||
DIM_ROPE: tl.constexpr,
|
||||
):
|
||||
"""Fused gather + dequant(per-group) + requant(per-tensor) -> fp8."""
|
||||
token_id = tl.program_id(0)
|
||||
token_id_paged = tl.load(page_table_1_ptr + token_id).to(tl.int32)
|
||||
raw_block_id = tl.program_id(1)
|
||||
|
||||
if raw_block_id < NUM_NOPE_BLOCKS:
|
||||
# nope: read fp8, mul group scale -> f32, cast to fp8_e4m3fn
|
||||
effective_block_id = raw_block_id
|
||||
offs_q = effective_block_id * GROUP_SIZE + tl.arange(0, GROUP_SIZE)
|
||||
mask = offs_q < DIM_NOPE
|
||||
|
||||
ptr_q = input_nope_q_ptr + token_id_paged * input_nope_q_stride_0 + offs_q
|
||||
ptr_s = (
|
||||
input_nope_s_ptr
|
||||
+ token_id_paged * input_nope_s_stride_0
|
||||
+ effective_block_id
|
||||
)
|
||||
|
||||
y_q = tl.load(ptr_q, mask=mask, other=0.0).to(tl.float32)
|
||||
y_s = tl.load(ptr_s)
|
||||
|
||||
# dequant -> f32 -> requant to fp8
|
||||
y = (y_q * y_s).to(tl.float8e4nv)
|
||||
|
||||
dst_ptr = output_ptr + token_id * output_stride_0 + offs_q
|
||||
tl.store(dst_ptr, y, mask=mask)
|
||||
else:
|
||||
# rope: read bf16, cast to fp8
|
||||
effective_block_id = raw_block_id - NUM_NOPE_BLOCKS
|
||||
offs = effective_block_id * GROUP_SIZE + tl.arange(0, GROUP_SIZE)
|
||||
mask = offs < DIM_ROPE
|
||||
|
||||
src_ptr = input_rope_ptr + token_id_paged * input_rope_stride_0 + offs
|
||||
dst_ptr = output_ptr + token_id * output_stride_0 + DIM_NOPE + offs
|
||||
|
||||
data = tl.load(src_ptr, mask=mask).to(tl.float8e4nv)
|
||||
tl.store(dst_ptr, data, mask=mask)
|
||||
|
||||
|
||||
gather_legacy = gather_dequant_requant_fp8_paged_legacy
|
||||
|
||||
PAGE_SIZE = 64
|
||||
DIM_QUANT = 656 # 512 nope fp8 + 16 scale bytes + 128 rope bytes
|
||||
|
||||
@@ -54,7 +54,7 @@ import argparse
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.kvcache.cache_ops import (
|
||||
from sglang.kernels.ops.attention.mla_qk_prep import (
|
||||
absorbed_bmm_concat_cast_q_fp8,
|
||||
concat_and_cast_q_fp8_pad,
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""Tune the gfx950 native MXFP8 dense route for one (N, K) and write its rows into
|
||||
python/sglang/kernels/ops/quantization/mxfp8_gemv_gfx95_configs.json.
|
||||
python/sglang/kernels/ops/gemm/mxfp8_gemv_gfx95_configs.json.
|
||||
|
||||
Two tables: the scaled-MFMA GEMV config per M bucket (1 ... 32) and the dot_scaled tile per
|
||||
M bucket (64 ... 16384). Each candidate is timed by HIP-graph replay rotating over weight
|
||||
@@ -18,7 +18,7 @@ import torch
|
||||
from triton.runtime.errors import OutOfResources
|
||||
|
||||
from sglang.kernels.jit.utils import empty_sentinel
|
||||
from sglang.kernels.ops.quantization import mxfp8_native_amd_gfx95 as native
|
||||
from sglang.kernels.ops.gemm import mxfp8_native_amd_gfx95 as native
|
||||
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import fp8_grid_quantize
|
||||
|
||||
GRAPH_CALLS = 20
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Benchmark seeded versus unseeded EAGLE draft Gumbel sampling.
|
||||
|
||||
Example:
|
||||
python benchmark/kernels/sampling/bench_deterministic_fast_sample.py \
|
||||
--batch-sizes 1 8 32 128 --vocab-sizes 32000 128256 163840
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
||||
import torch
|
||||
import triton
|
||||
|
||||
from sglang.srt.speculative.spec_utils import fast_sample
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--batch-sizes", type=int, nargs="+", default=[1, 8, 32, 128])
|
||||
parser.add_argument(
|
||||
"--vocab-sizes", type=int, nargs="+", default=[32000, 128256, 163840]
|
||||
)
|
||||
parser.add_argument("--warmup-ms", type=int, default=100)
|
||||
parser.add_argument("--rep-ms", type=int, default=500)
|
||||
parser.add_argument(
|
||||
"--modes", nargs="+", choices=("eager", "graph"), default=["graph"]
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def capture(fn):
|
||||
fn()
|
||||
torch.cuda.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
fn()
|
||||
return graph.replay
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
device = torch.device("cuda")
|
||||
print(f"device={torch.cuda.get_device_name(device)}")
|
||||
print("mode\tbatch\tvocab\tunseeded_us\tseeded_us\tslowdown")
|
||||
|
||||
for vocab_size in args.vocab_sizes:
|
||||
for batch_size in args.batch_sizes:
|
||||
logits = torch.randn(
|
||||
(batch_size, vocab_size), device=device, dtype=torch.float32
|
||||
)
|
||||
probs = torch.softmax(logits, dim=-1)
|
||||
seeds = torch.arange(batch_size, device=device, dtype=torch.int64) + 12345
|
||||
positions = (
|
||||
torch.arange(batch_size, device=device, dtype=torch.int64) + 4096
|
||||
)
|
||||
|
||||
def unseeded():
|
||||
return fast_sample(probs)
|
||||
|
||||
def seeded():
|
||||
return fast_sample(
|
||||
probs,
|
||||
sampling_seed=seeds,
|
||||
positions=positions,
|
||||
draft_step=1,
|
||||
)
|
||||
|
||||
# Compile/JIT and populate allocator caches before timing.
|
||||
unseeded()
|
||||
seeded()
|
||||
torch.cuda.synchronize()
|
||||
|
||||
for mode in args.modes:
|
||||
unseeded_fn = capture(unseeded) if mode == "graph" else unseeded
|
||||
seeded_fn = capture(seeded) if mode == "graph" else seeded
|
||||
unseeded_ms = triton.testing.do_bench(
|
||||
unseeded_fn, warmup=args.warmup_ms, rep=args.rep_ms
|
||||
)
|
||||
seeded_ms = triton.testing.do_bench(
|
||||
seeded_fn, warmup=args.warmup_ms, rep=args.rep_ms
|
||||
)
|
||||
print(
|
||||
f"{mode}\t{batch_size}\t{vocab_size}\t"
|
||||
f"{unseeded_ms * 1000:.1f}\t{seeded_ms * 1000:.1f}\t"
|
||||
f"{seeded_ms / unseeded_ms:.2f}x"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,77 +0,0 @@
|
||||
# 优化版(2026-10-08 v3):torch/nvidia 全家桶永远复用基底,全量路径零重下。
|
||||
# 要点:
|
||||
# 1. pyproject build-system.requires 含 torch==2.13.0,pip 隔离构建环境每次都重下 ~6GB。
|
||||
# 基底镜像已装 torch 2.13 → 用 --no-build-isolation 复用,构建环境零下载。
|
||||
# 2. 老全量路径让 pip 解析全部依赖:bare `torchvision` 等会让解析器弃用基底已装的
|
||||
# 本地版本(torch 2.13.0+cu130),改用 PyPI 的 torch==2.13.0,连带重下整套 nvidia
|
||||
# 依赖(9/23 实测 nvidia-nccl-cu13 等 6GB+,限流下 2h40m);且基底 NCCL 是
|
||||
# 刻意装高的 2.30.7(torch 元数据钉 ==2.29.7),任何全量解析都会把它降级回去。
|
||||
# → v3 不再让 pip 碰重型依赖:check_deps.py 在构建期对照 pyproject 与基底
|
||||
# 已装版本,把增量分三类——重型(torch*/nvidia-*/cuda-*)永不进解析、
|
||||
# 不满足即让构建失败并提示决策(显式失败好过静默重装);
|
||||
# kernel wheel(sglang-kernel/flashinfer*/sgl-deep-gemm)要跟随源码版本,
|
||||
# 钉版变动时以 pip --no-deps 单独装(防止顺藤摸瓜解析 torch);
|
||||
# 其余轻量依赖只装未满足的增量。
|
||||
# 3. 基底缺 setuptools-scm(版本打戳要用),单层预装 + pip cache mount,只下载一次。
|
||||
# 4. RUN 的 pip cache mount 持久化在宿主机,轻量依赖增量不再重复下载。
|
||||
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
|
||||
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV PIP_INDEX_URL=${PIP_INDEX}
|
||||
|
||||
# 构建后端(pyproject build-system.requires,torch 除外——基底已有)
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
|
||||
|
||||
# 依赖增量计算脚本(对照 pyproject 与基底已装版本,产出轻量/kernel 增量清单)
|
||||
# 注意 build context 是 /data/ymk 且 .dockerignore 排除了 build/,只能从 sglang/ 树内 COPY
|
||||
COPY sglang/deploy/b300-build/check_deps.py /tmp/check_deps.py
|
||||
|
||||
# 整个源码树(含 .git,用于 setuptools-scm 版本打戳)
|
||||
COPY sglang/ /sgl-workspace/sglang/
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译。基底自带 rustup stable(实测 cargo 1.98.1),
|
||||
# RUSTUP_TOOLCHAIN=stable 绕过仓库 rust-toolchain.toml 钉的 1.90(避免重复下载工具链);
|
||||
# b300 直连 crates.io(稀疏索引 + static.crates.io 均通),无需任何镜像源。
|
||||
# cache mount 持久化 registry/target:冷编译 ~15min(二进制 ~11min + python 扩展 ~4min),
|
||||
# 增量重建秒~分钟级。python 扩展必须显式 -p(workspace 根包默认不编译扩展),
|
||||
# 产物 libsglang_router_rs.so 需重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件
|
||||
# (与 H20 线上部署方式一致)。2026-10-09 b300 实测验证。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
# 计算依赖增量(重型 torch/nvidia/cuda 永不进入清单;不满足则 check_deps 非零、构建失败)
|
||||
RUN python /tmp/check_deps.py /sgl-workspace/sglang/python/pyproject.toml \
|
||||
/tmp/missing_light.txt /tmp/missing_kernel.txt \
|
||||
&& echo "== kernel wheel delta (pip --no-deps) ==" && cat /tmp/missing_kernel.txt \
|
||||
&& echo "== lightweight dep delta ==" && cat /tmp/missing_light.txt
|
||||
|
||||
# kernel wheel 增量(如 sglang-kernel 升版):--no-deps 防止 pip 顺藤摸瓜解析 torch
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_kernel.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_kernel.txt; \
|
||||
else \
|
||||
echo "== no kernel wheel delta"; \
|
||||
fi
|
||||
|
||||
# 轻量依赖增量(通常为空或几个小包),重型全家桶原样复用基底。
|
||||
# ⚠️ 必须 --no-deps:即使目标只是 xgrammar 这种小包,只要依赖图里出现 torch,
|
||||
# pip 就会校验 torch+cu130 元数据钉死的 nvidia-nccl-cu13==2.29.7,发现基底装的是
|
||||
# 2.30.7 便"好心"下载 206MB 降级(10-08 v3 首跑实测踩中)。基底的传递依赖本来就
|
||||
# 齐全,增量小包的二级依赖也一样;若有漏网,b300 冒烟会在 import 阶段显式暴露。
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_light.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_light.txt; \
|
||||
else \
|
||||
echo "== no lightweight delta, skipping dependency install"; \
|
||||
fi
|
||||
|
||||
# editable 安装 sglang 本体,--no-deps 保证 pip 完全不碰依赖图
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-deps --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
@@ -1,65 +0,0 @@
|
||||
# fast 路径(2026-10-08 v3.1):python/pyproject.toml 未变化时使用(build.sh 自动判断)。
|
||||
# 与 Dockerfile 全量路径同构:同样跑 check_deps.py 增量安装。
|
||||
# 原因:全量构建若装过 kernel wheel 增量(如 sglang-kernel 0.4.7),随后 pyproject 未变的
|
||||
# fast 构建若只 --no-deps,会静默退回基底的旧 kernel(0.4.6.post1)——镜像内容漂移。
|
||||
# check_deps 本地比对 ~0.5s,增量命中 pip cache 秒级,成本可忽略。
|
||||
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
|
||||
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV PIP_INDEX_URL=${PIP_INDEX}
|
||||
|
||||
# 构建后端(pyproject build-system.requires,torch 除外——基底已有)
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
|
||||
|
||||
# 依赖增量计算脚本(对照 pyproject 与基底已装版本,产出轻量/kernel 增量清单)
|
||||
# 注意 build context 是 /data/ymk 且 .dockerignore 排除了 build/,只能从 sglang/ 树内 COPY
|
||||
COPY sglang/deploy/b300-build/check_deps.py /tmp/check_deps.py
|
||||
|
||||
# 整个源码树(含 .git,用于 setuptools-scm 版本打戳)
|
||||
COPY sglang/ /sgl-workspace/sglang/
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译。基底自带 rustup stable(实测 cargo 1.98.1),
|
||||
# RUSTUP_TOOLCHAIN=stable 绕过仓库 rust-toolchain.toml 钉的 1.90(避免重复下载工具链);
|
||||
# b300 直连 crates.io(稀疏索引 + static.crates.io 均通),无需任何镜像源。
|
||||
# cache mount 持久化 registry/target:冷编译 ~15min,增量重建秒~分钟级。
|
||||
# python 扩展必须显式 -p(workspace 根包默认不编译扩展),产物 libsglang_router_rs.so
|
||||
# 需重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件(与 H20 线上部署方式一致)。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
# 计算依赖增量(重型 torch/nvidia/cuda 永不进入清单;不满足则 check_deps 非零、构建失败)
|
||||
RUN python /tmp/check_deps.py /sgl-workspace/sglang/python/pyproject.toml \
|
||||
/tmp/missing_light.txt /tmp/missing_kernel.txt \
|
||||
&& echo "== kernel wheel delta (pip --no-deps) ==" && cat /tmp/missing_kernel.txt \
|
||||
&& echo "== lightweight dep delta ==" && cat /tmp/missing_light.txt
|
||||
|
||||
# kernel wheel 增量(如 sglang-kernel 升版):--no-deps 防止 pip 顺藤摸瓜解析 torch
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_kernel.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_kernel.txt; \
|
||||
else \
|
||||
echo "== no kernel wheel delta"; \
|
||||
fi
|
||||
|
||||
# 轻量依赖增量(通常为空或几个小包),重型全家桶原样复用基底。
|
||||
# ⚠️ 必须 --no-deps:即使目标只是 xgrammar 这种小包,只要依赖图里出现 torch,
|
||||
# pip 就会校验 torch+cu130 元数据钉死的 nvidia-nccl-cu13==2.29.7,发现基底装的是
|
||||
# 2.30.7 便"好心"下载 206MB 降级(10-08 v3 首跑实测踩中)。基底的传递依赖本来就
|
||||
# 齐全,增量小包的二级依赖也一样;若有漏网,b300 冒烟会在 import 阶段显式暴露。
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_light.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_light.txt; \
|
||||
else \
|
||||
echo "== no lightweight delta, skipping dependency install"; \
|
||||
fi
|
||||
|
||||
# editable 安装 sglang 本体,--no-deps 保证 pip 完全不碰依赖图
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-deps --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
@@ -1,33 +0,0 @@
|
||||
#!/bin/bash
|
||||
# 用法: /data/ymk/build/build.sh [tag]
|
||||
# 默认 tag: ymkymx/sglang:<分支>-<sha9>-local-<UTC构建日期时间>(命名规则见 AGENTS.md)
|
||||
# 另打 <分支>-latest-local 别名。
|
||||
#
|
||||
# 依赖路径自动选择:
|
||||
# python/pyproject.toml 的 sha256 与上次成功构建一致 → Dockerfile.fast(--no-deps,秒级)
|
||||
# 不一致(或 FULL=1)→ Dockerfile 全量依赖解析,成功后记录新 hash
|
||||
set -euo pipefail
|
||||
cd /data/ymk
|
||||
SHA=$(git -C sglang rev-parse --short=9 HEAD)
|
||||
BR=$(git -C sglang rev-parse --abbrev-ref HEAD)
|
||||
DT=$(date -u +%Y%m%d-%H%M)
|
||||
TAG=${1:-ymkymx/sglang:$BR-$SHA-local-$DT}
|
||||
|
||||
HASH_FILE=build/.last_pyproject_sha256
|
||||
CUR=$(sha256sum sglang/python/pyproject.toml | cut -d' ' -f1)
|
||||
PREV=$(cat "$HASH_FILE" 2>/dev/null || echo none)
|
||||
|
||||
DOCKERFILE=build/Dockerfile
|
||||
if [ "$CUR" = "$PREV" ] && [ "${FULL:-0}" != "1" ]; then
|
||||
DOCKERFILE=build/Dockerfile.fast
|
||||
echo "== pyproject 未变,走 fast 路径(--no-deps)"
|
||||
else
|
||||
echo "== pyproject 有变化或 FULL=1,走全量依赖解析"
|
||||
fi
|
||||
|
||||
echo "== building $TAG from sglang@$(git -C sglang log --oneline -1) [$DOCKERFILE]"
|
||||
docker build -f "$DOCKERFILE" -t "$TAG" .
|
||||
# 构建成功才记录 hash / 打别名
|
||||
echo "$CUR" > "$HASH_FILE"
|
||||
docker tag "$TAG" "ymkymx/sglang:$BR-latest-local"
|
||||
echo "== done: $TAG (别名 $BR-latest-local)"
|
||||
@@ -1,90 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compute dependency deltas for a b300 sglang image build, in three classes.
|
||||
|
||||
Reads pyproject.toml, evaluates every requirement against the *installed*
|
||||
distributions in the base image, and writes two requirements files:
|
||||
|
||||
argv[2] (light): unsatisfied lightweight requirements → `pip install -r`
|
||||
argv[3] (kernel): unsatisfied kernel wheels (sglang-kernel, flashinfer*,
|
||||
sgl-deep-gemm) → `pip install --no-deps -r`
|
||||
These track the sglang source being built and are meant
|
||||
to be upgraded from the index when pyproject bumps them;
|
||||
--no-deps keeps pip from walking torch's metadata.
|
||||
|
||||
The PyTorch stack (torch/torchvision/torchaudio/torchcodec/
|
||||
torch_memory_saver), the nvidia-* CUDA family and cuda-* packages are NEVER
|
||||
emitted. If one of them is missing or its installed version no longer
|
||||
satisfies the pinned specifier (e.g. upstream bumps nvidia-cutlass-dsl), the
|
||||
script fails the build loudly instead of letting pip download ~6GB or
|
||||
"fix" the base image's deliberately newer nvidia-nccl-cu13 2.30.7 back to
|
||||
torch's metadata pin ==2.29.7 (any pip full-resolution would do both —
|
||||
2026-09-23/24 builds did, throttled to 2h40m).
|
||||
"""
|
||||
|
||||
import sys
|
||||
import tomllib
|
||||
from importlib import metadata as md
|
||||
|
||||
from packaging.requirements import Requirement
|
||||
|
||||
# Never resolved by pip: reused from the base image or the build fails.
|
||||
HEAVY_PREFIXES = ("torch", "nvidia-", "cuda-")
|
||||
# Kernel wheels: upgraded via pip --no-deps when pyproject pins move.
|
||||
KERNEL_PREFIXES = ("sglang-kernel", "flashinfer", "sgl-deep-gemm")
|
||||
|
||||
|
||||
def norm(name: str) -> str:
|
||||
return name.lower().replace("_", "-")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
pyproject, light_out, kernel_out = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
with open(pyproject, "rb") as f:
|
||||
deps = tomllib.load(f)["project"]["dependencies"]
|
||||
|
||||
light: list[str] = []
|
||||
kernel: list[str] = []
|
||||
heavy_unsat: list[str] = []
|
||||
for dep in deps:
|
||||
req = Requirement(dep)
|
||||
if req.marker is not None and not req.marker.evaluate():
|
||||
continue
|
||||
try:
|
||||
installed = md.version(req.name)
|
||||
except md.PackageNotFoundError:
|
||||
installed = None
|
||||
satisfied = installed is not None and req.specifier.contains(
|
||||
installed, prereleases=True
|
||||
)
|
||||
if satisfied:
|
||||
continue
|
||||
name = norm(req.name)
|
||||
if name.startswith(HEAVY_PREFIXES):
|
||||
heavy_unsat.append(f"{dep} (installed: {installed})")
|
||||
elif name.startswith(KERNEL_PREFIXES):
|
||||
kernel.append(dep)
|
||||
else:
|
||||
light.append(dep)
|
||||
|
||||
if heavy_unsat:
|
||||
print("=" * 70, file=sys.stderr)
|
||||
print("FATAL: heavy base-image deps no longer satisfy pyproject:", file=sys.stderr)
|
||||
for item in heavy_unsat:
|
||||
print(f" {item}", file=sys.stderr)
|
||||
print(
|
||||
"Refusing to let pip resolve these (would re-download ~6GB and/or\n"
|
||||
"downgrade NCCL). Decide explicitly: bump the base image, or adjust\n"
|
||||
"the pin in pyproject.toml.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
with open(light_out, "w") as f:
|
||||
f.write("\n".join(light) + ("\n" if light else ""))
|
||||
with open(kernel_out, "w") as f:
|
||||
f.write("\n".join(kernel) + ("\n" if kernel else ""))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,33 +0,0 @@
|
||||
# B300 ds41 部署 compose 档案
|
||||
|
||||
b300-01 上 DeepSeek-V4.1-Flash 各部署方案的 docker compose 存档。
|
||||
**本目录是 source of truth**;b300 上的运行目录 `/data/ymk/ds41/` 是工作副本。
|
||||
|
||||
方案说明、参数矩阵、特殊 env、已知坑见飞书部署手册(组内共享):
|
||||
《B300 sglang 部署手册》 https://u04wb5irxz.feishu.cn/docx/FOi8dbnybodHr0xIfkPcQ6gtnRx
|
||||
|
||||
## 同步流程
|
||||
|
||||
```bash
|
||||
# 本机(D:\B300\sglang)
|
||||
git add deploy/ && git commit -m "deploy: ..."
|
||||
git push origin dsv41-pd-visioncp # Gitea(paths-ignore,不触发镜像构建)
|
||||
git push b300 dsv41-pd-visioncp # b300 裸仓库
|
||||
|
||||
# b300-01
|
||||
git -C /data/ymk/sglang pull # deploy/ 出现在 /data/ymk/sglang/deploy/b300-ds41/
|
||||
```
|
||||
|
||||
## CI 说明
|
||||
|
||||
`.gitea/workflows/build-image.yaml` 的 push 触发器带 `paths-ignore: ['deploy/**', '.gitea/**']`,
|
||||
改本目录不会触发镜像构建。注意:workflow 自身的改动也不再自动触发,需要重建镜像时
|
||||
伴随代码改动 push 或手动触发。
|
||||
|
||||
## 目录规则
|
||||
|
||||
- 命名:`dockerserve-<拓扑>-<特性>-<卡位>.yml`(如 `-b4` = 后 4 卡)
|
||||
- 退役文件不要留在主目录:移入 `archive-YYYYMMDD/`(b300 侧同样执行)
|
||||
- 禁止提交 `.bak` 备份文件
|
||||
- `mooncake-store.json` 是 hicache L3 mooncake store 配置,被
|
||||
`dockerserve-pd2-p-r1/r2.yml` 通过 `SGLANG_HICACHE_MOONCAKE_CONFIG_PATH` 引用
|
||||
@@ -1,27 +0,0 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
# 一律 --warmup-requests 0:默认 warmup 会把 prompt#0 种进 L1/L2 污染 L3 测量(已踩过)
|
||||
run() {
|
||||
echo "===== [$(date +%H:%M:%S)] RUN: $1 (np=$2) ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--warmup-requests 0 --num-prompts "$2" --max-concurrency "$2" --seed "$3" --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" 2>&1 | grep -E 'Input token throughput|Mean TTFT|P99 TTFT|Successful'
|
||||
sleep 2
|
||||
}
|
||||
flush() { curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache > /dev/null && echo "[$(date +%H:%M:%S)] flushed"; sleep 3; }
|
||||
|
||||
echo "===== PRIME COLD mc8(冷基线 + 写入 L3) ====="
|
||||
run PRIME_COLD 8 1541391
|
||||
flush
|
||||
run MC1_L3 1 1541391
|
||||
flush
|
||||
run MC2_L3 2 1541391
|
||||
flush
|
||||
run MC4_L3 4 1541391
|
||||
flush
|
||||
run MC8_L3 8 1541391
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -1,204 +0,0 @@
|
||||
#!/bin/bash
|
||||
# bench_suite_v2.sh — DeepSeek-V4.1-Flash on b300-01,2026-10-09 新版基线方法
|
||||
#
|
||||
# 相对 v1 的方法变化:
|
||||
# 1. 服务刚起数据不准 → 每个测试模式自带 warmup 轮(结果丢弃,标记 WARM)
|
||||
# 2. prefill 冷/热双轮:冷轮写入 hicache(L1/L3),同 seed 热轮测纯 cache-hit TPS
|
||||
# 3. decode 阶梯并发:步长 32,上限 = cuda-graph-max-bs-decode 的 80%;
|
||||
# 每档先 warm(灌 L1)再测,prefill 全 cache hit;
|
||||
# 报告 TPOT + 整实例输出 TPS + 推算实际并发(= 输出TPS × TPOT / 1000)
|
||||
# 4. l3 模式(3p1d 经 router):大语料溢出 L1 后重放,验证 L3 命中
|
||||
#
|
||||
# 在宿主机执行,bench 客户端通过 docker exec 进容器跑(需要 /data 挂载)。
|
||||
#
|
||||
# 用法:
|
||||
# bench_suite_v2.sh warmup <container> <port>
|
||||
# bench_suite_v2.sh prefill <tag> <container> <port>
|
||||
# bench_suite_v2.sh decode <tag> <container> <port> [cg_max_bs=256]
|
||||
# bench_suite_v2.sh l3 <tag> <container> <port> [num_prompts=24]
|
||||
# bench_suite_v2.sh report <log>
|
||||
# 产物: /data/ymk/ds41/bench_v2_<tag>.log
|
||||
set -u
|
||||
|
||||
BASE=/data/ymk/ds41
|
||||
MODEL=${MODEL:-/data/models/DeepSeek-V4.1-Flash}
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
|
||||
usage() { sed -n 2,20p "$0" >&2; exit 1; }
|
||||
|
||||
# $1=container $2=port $3=tag -> 生成可执行脚本路径并 echo
|
||||
emit_common() {
|
||||
cat <<EOF
|
||||
cd $BASE
|
||||
HDR="$HDR"
|
||||
PORT=$2
|
||||
run() {
|
||||
local name="\$1"; shift
|
||||
echo "===== [\$(date +%H:%M:%S)] START: \$name ====="
|
||||
python3 -m sglang.bench_serving "\$@"
|
||||
echo "===== [\$(date +%H:%M:%S)] END(exit=\$?): \$name ====="
|
||||
sleep 5
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
# ---------------- warmup ----------------
|
||||
cmd_warmup() {
|
||||
local c=${1:?container} port=${2:?port}
|
||||
local gen=$BASE/.bench_v2_warmup.sh
|
||||
{ emit_common "$c" "$port"; cat <<'EOF'
|
||||
run WARMUP_a --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777001 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
run WARMUP_b --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 16000 --random-output-len 1 \
|
||||
--num-prompts 2 --max-concurrency 1 --seed 777002 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
EOF
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
docker exec "$c" bash "$gen"
|
||||
}
|
||||
|
||||
# ---------------- prefill(冷/热双轮) ----------------
|
||||
cmd_prefill() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port}
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
{ emit_common "$c" "$port"
|
||||
# warmup(丢弃)
|
||||
cat <<'EOF'
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
EOF
|
||||
# 冷轮
|
||||
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
|
||||
set -- $stage
|
||||
cat <<EOF
|
||||
run COLD_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
|
||||
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
# 热轮(同 seed → cache hit)
|
||||
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
|
||||
set -- $stage
|
||||
cat <<EOF
|
||||
run HIT_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
|
||||
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! seeds=${p}1..3 log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- decode 阶梯 ----------------
|
||||
cmd_decode() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port} cg=${4:-256}
|
||||
local maxc=$(( cg * 80 / 100 / 32 * 32 )) # 80% 向下取整到 32 的倍数
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
echo "ladder: 32..$maxc step 32 (cg_max_bs=$cg)"
|
||||
{ emit_common "$c" "$port"
|
||||
cat <<'EOF'
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 128 \
|
||||
--num-prompts 16 --max-concurrency 16 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
EOF
|
||||
# bs1 延迟地板(warm + measure)
|
||||
cat <<EOF
|
||||
run WARM_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run DEC_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
local cc
|
||||
for (( cc=32; cc<=maxc; cc+=32 )); do
|
||||
local np=$(( cc * 2 ))
|
||||
cat <<EOF
|
||||
run WARM_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run DEC_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- L3 语料溢出(3p1d 经 router) ----------------
|
||||
cmd_l3() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port} np=${4:-48}
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
{ emit_common "$c" "$port"
|
||||
cat <<EOF
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \\
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run L3_COLD_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
|
||||
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
echo "===== [\$(date +%H:%M:%S)] DRAIN: sleep 120 等 L2→L3 备份排空 ====="
|
||||
sleep 120
|
||||
run L3_HIT_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
|
||||
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! corpus=${np}x160k log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- report ----------------
|
||||
cmd_report() {
|
||||
local log=${1:?log required}
|
||||
awk '
|
||||
/START:/ { name=$4; sub(/:$/, "", name); tps=""; itps=""; tpot=""; ttft=""; acc="" }
|
||||
/Input token throughput/ { itps=$NF }
|
||||
/Output token throughput/ { tps=$NF }
|
||||
/Mean TPOT/ { tpot=$NF }
|
||||
/Mean TTFT/ { ttft=$NF }
|
||||
/[Aa]ccept/ { acc=$0; gsub(/^ +/, "", acc) }
|
||||
/END\(exit/ {
|
||||
line=sprintf("%-22s in_tps=%-10s out_tps=%-10s tpot_ms=%-8s ttft_ms=%-8s", name, itps, tps, tpot, ttft)
|
||||
if (tps != "" && tpot != "" && tpot+0 > 0)
|
||||
line=line sprintf(" eff_concurrency=%.1f", tps * tpot / 1000)
|
||||
print line
|
||||
if (acc != "") print " " acc
|
||||
}
|
||||
' "$log"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
warmup) shift; cmd_warmup "$@" ;;
|
||||
prefill) shift; cmd_prefill "$@" ;;
|
||||
decode) shift; cmd_decode "$@" ;;
|
||||
l3) shift; cmd_l3 "$@" ;;
|
||||
report) shift; cmd_report "$@" ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
@@ -1,61 +0,0 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
run() {
|
||||
local name="$1"; shift
|
||||
echo "===== [$(date +%H:%M:%S)] START: $name ====="
|
||||
python3 -m sglang.bench_serving "$@" 2>&1 | grep -vE '^Namespace|warn'
|
||||
echo "===== [$(date +%H:%M:%S)] END(exit=$?): $name ====="
|
||||
sleep 3
|
||||
}
|
||||
|
||||
r160() { # $1=name $2=seed $3=num_prompts $4=mc
|
||||
run "$1" --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts "$3" --max-concurrency "$4" --seed "$2" --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
}
|
||||
|
||||
# WARMUP
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
|
||||
# R0 纯计算极限(集合 A 首次出现,L1/L2/L3 全空)
|
||||
r160 R0_COMPUTE_COLD 1541391 8 8
|
||||
# R1 L1 极限(A 全量在 L1 设备池:1.28M < 1.6M)
|
||||
r160 R1_L1_HIT 1541391 8 8
|
||||
# R2 填充集 B(12×160k=1.92M > L1 1.6M),把 A 挤出 L1 到 L2
|
||||
r160 R2_FILLER_EVICT 999000 12 8
|
||||
# R3 L2 极限(A 已被挤出 L1,从 L2 host 池 load-back)
|
||||
r160 R3_L2_HIT 1541391 8 8
|
||||
# R4 flush L1+L2(L3 保留),走 L3 prefetch
|
||||
echo "===== [$(date +%H:%M:%S)] FLUSH L1+L2 ====="
|
||||
curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache && echo "flush ok"
|
||||
sleep 5
|
||||
r160 R4_L3_HIT 1541391 8 8
|
||||
# R5 嫁接后 L2 复核
|
||||
r160 R5_L2_AFTER_GRAFT 1541391 8 8
|
||||
|
||||
# R6 混合负载:新集合 C(纯计算,4×160k mc4)+ 集合 A(命中,8×160k mc8)并发打
|
||||
echo "===== [$(date +%H:%M:%S)] START: R6_MIX_PARALLEL ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 4 --max-concurrency 4 --seed 555000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r6_coldclient.log 2>&1 &
|
||||
COLD_PID=$!
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 1541391 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r6_hitclient.log 2>&1 &
|
||||
HIT_PID=$!
|
||||
wait $COLD_PID $HIT_PID
|
||||
echo "===== [$(date +%H:%M:%S)] END: R6_MIX_PARALLEL ====="
|
||||
echo "--- R6 cold client (compute) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r6_coldclient.log
|
||||
echo "--- R6 hit client (cache) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r6_hitclient.log
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -1,27 +0,0 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
echo "===== [$(date +%H:%M:%S)] FLUSH L1+L2(A 只剩 L3) ====="
|
||||
curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache && echo "flush ok"
|
||||
sleep 5
|
||||
|
||||
echo "===== [$(date +%H:%M:%S)] START: R7_MIX_L3_PLUS_COMPUTE ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 4 --max-concurrency 4 --seed 666000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r7_coldclient.log 2>&1 &
|
||||
COLD_PID=$!
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 1541391 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r7_hitclient.log 2>&1 &
|
||||
HIT_PID=$!
|
||||
wait $COLD_PID $HIT_PID
|
||||
echo "===== [$(date +%H:%M:%S)] END: R7 ====="
|
||||
echo "--- R7 cold client (compute, 4x160k mc4) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r7_coldclient.log
|
||||
echo "--- R7 hit client (L3, 8x160k mc8) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r7_hitclient.log
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -1,172 +0,0 @@
|
||||
# 3x cp2-P + 1x dp2-D PD 分离 + L3 mooncake 互联缓存 + engram host table
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-d
|
||||
environment:
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -1,225 +0,0 @@
|
||||
# 【L3 dummy client 变体 2026-10-09 深夜,定型=shm+best_effort】sglang→standalone_storage,数据面经本机 mooncake_client(50052 IPC+进程映射),不碰 TCP 传输引擎
|
||||
# 实测 24x160k 热/冷:shm+best_effort +26.4%(唯一正收益);wait_complete 因 all-or-nothing discard 等待白等反而 -17.7%。TCP 传输引擎两组皆负。
|
||||
# 【L3 cache-hit 测试变体 2026-10-09】基于 dockerserve-3p1d-otlp.yml,仅改 P:
|
||||
# --max-total-tokens 800000(L1 缩到 ~0.8M token)+ --hicache-ratio 1.0(L2=L1=0.8M token。⚠️ 教训:ratio 0.05 时 L2 太小,build_storage_backup_spec 拿不到 host 页会静默跳过 L3 写入,热轮无数据可中;V4 hicache 不支持 --hicache-size)
|
||||
# 使 ~2.6M token/P 的语料(48×160k)溢出 L1+L2(1.6M),热轮强制走 L3 mooncake 命中。
|
||||
# 【2026-10-10 L2 放大】hicache-ratio 1.0→4.0(L2=3.2M token/P,宿主可用内存 ~1.9TB 充裕):
|
||||
# #38348 fail-soft 移植后 bench 显示收益被容量掩盖——24x160k 下 host_capacity 拒绝
|
||||
# 0.75M~2.97M token/P(L2 装不下,prefetch 未发起即被拒退回重算)。放大后验证:
|
||||
# ①热轮请求 L3/L2 命中不被容量拒绝;②L3 hit 嫁接进 L2 树后在 L2 长期留存。
|
||||
# 与主模板同容器名/端口,用前必须先 down 主栈。D 与 router 不变。
|
||||
# 原注释: 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-baba18bc7d-local-20261010-2310
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
RUSTUP_TOOLCHAIN: "1.92-x86_64-unknown-linux-gnu"
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
# G9 P0 验证(2026-10-11):D 单独升镜像拿 /query_cache_hit 修复(71c6d8bc5),
|
||||
# 并开 decode radix + hicache L3 客户端(复用 dummy store,只读查询)。
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
RUSTUP_TOOLCHAIN: "1.92-x86_64-unknown-linux-gnu"
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--disaggregation-decode-enable-radix-cache
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -1,239 +0,0 @@
|
||||
# 【G9 P0 验证栈:真实 store 3p1d 2026-10-11】基于 dockerserve-3p1d-otlp.yml。
|
||||
# 用途:验证 P0 改动(/v1/loads 排队 token + drain rate 字段、P 侧 /query_cache_hit)。
|
||||
# 与主模板同容器名/端口,用前必须先 down 主栈;router(ds41-router-3p1d)独立不在本文件。
|
||||
# 2026-10-11 改造要点(vs 原 l3test):
|
||||
# - 镜像升 6612af9e0(含 G9 P0 + /query_cache_hit KeyError 修复)+ scheduler.py
|
||||
# 热补丁(L3 查询门限从 enable_decode_hicache 放宽到 enable_hicache_storage,
|
||||
# 否则 P 节点永远查不到 L3)。
|
||||
# - 每实例独立 MOONCAKE_LOCAL_HOSTNAME(a/b/c/d),extra_hosts 互配全部 rank 别名
|
||||
# (真实 store 跨实例副本写要求对端 rank 主机名可解析,见 realstore-x2 注释;
|
||||
# rank 命名规则 {hostname}-d{dp}t{tp}p{pp}c{attn_cp},commit 84fc632088;
|
||||
# D 为 dp2 dp-attention(attn_tp=1),后缀推断为 -d0t0p0c0/-d1t1p0c0,未实证)。
|
||||
# - ⚠️ D 侧 cache 方案实证否决(2026-10-11):SWA 模型下
|
||||
# --disaggregation-decode-enable-radix-cache 先要 SGLANG_ENABLE_UNIFIED_RADIX_TREE=1,
|
||||
# 开了又报 "supports only device-resident cache and is incompatible with
|
||||
# --enable-hierarchical-cache"(kv_cache_builder.py 硬 ValueError)——D 挂不了 L3。
|
||||
# 结论:/query_cache_hit 放 P(L3 全局统一,P 有完整 hicache),D 保持生产配置。
|
||||
# - D mem-fraction 0.80:曾因崩溃 d1 泄漏 ~61GiB 临时降 0.75,全栈 down 掉旧容器后
|
||||
# 驱动即回收(GPU 全零),已恢复 0.80。D max-running 保持 256(512 时 SWA 池挤爆 KV 池)。
|
||||
# P 侧保持 l3test 原状:--max-total-tokens 800000 + ratio 1.0(L1/L2 各 ~0.8M token),
|
||||
# write_through 保证 prefill 后 L3 有数据,供 /query_cache_hit 对拍。
|
||||
# 原注释: 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 依赖: mooncake-master 容器(50051,起栈前 restart 清旧 segment 注册);
|
||||
# b300-01 k8s cw jaeger-trace(4317,导出失败只丢 span 不影响服务)
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-e7dc175e9-local-20261011-0940
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-rank-hosts: &rank-hosts
|
||||
# mooncake rank 主机名别名:P(tp2/cp2 interleave)-d0t0p0c0/-d0t1p0c1;
|
||||
# D(dp2 dp-attention)-d0t0p0c0/-d1t1p0c0(推断,起栈后核对 master 注册名)
|
||||
- "localhost-a-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-a-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-b-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-b-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-c-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-c-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-d-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d-d1t1p0c0:127.0.0.1"
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-a
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-b
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-c
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
extra_hosts: *rank-hosts
|
||||
# ⚠️ D 保持生产配置(2026-10-11 实证):SWA 模型下
|
||||
# --disaggregation-decode-enable-radix-cache 与 --enable-hierarchical-cache 互斥
|
||||
# (kv_cache_builder.py 硬 ValueError,"supports only device-resident cache"),
|
||||
# D 无法挂 L3 —— /query_cache_hit 改由 P 节点回答(L3 全局统一,P 有完整 hicache)。
|
||||
# MOONCAKE_LOCAL_HOSTNAME 保留备用(无害)。
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-d
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -1,196 +0,0 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -1,196 +0,0 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -1,174 +0,0 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram(2026-09-25,mr256,nsys on p1/d1)
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
nsys launch --session-new=p1prof --cuda-graph-trace=node sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
nsys launch --session-new=d1prof --cuda-graph-trace=node sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -1,174 +0,0 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram(2026-09-25,mr256,nsys on p1/d1)
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -1,43 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.60
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,43 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.60
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
|
||||
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,42 +0,0 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -1,148 +0,0 @@
|
||||
# 【阶段A 新旧基线 2026-10-18】4×2卡 standalone 并行微基准(aiperf 口径)
|
||||
# 目的:同一台 b300-01 上旧版(tag 26.10.18 线上镜像)vs 新版(g9i-est-td @ e4391748e,
|
||||
# 对齐上游 main 6c3db0f520,torch 2.14.1 自制基底)同配置对比。
|
||||
# 配置来源:线上 k8s ds41-3p1d-g9i 的 P/D pod spec,去掉 --disaggregation-* 与
|
||||
# mooncake L3 storage backend(微基准只测 L1/L2 冷/热;L3 在阶段 B PD 合测再测)。
|
||||
# 布局:old-cp2=GPU0,1:30500 old-dp2=GPU2,3:30600 new-cp2=GPU4,5:30700 new-dp2=GPU6,7:30800
|
||||
# 注意:
|
||||
# - 新镜像 flashinfer 0.7.2 首次 autotune 暖机 ~15-20min,用独立 cache 目录
|
||||
# (/data/ymk/cache/sglang-e43917)防与旧版缓存串扰;旧版复用 /data/ymk/cache/sglang。
|
||||
# - standalone 无 PD 传输,不带 pid:host / NVLink mooncake 四件套。
|
||||
# - 启动后核对三件套(max_total_num_tokens / max_running_requests / available_gpu_mem)。
|
||||
# 用法:docker compose -p bencha -f dockerserve-bencha-4inst.yml up -d
|
||||
# bench: docker exec aiperf-cmp bash /data/ymk/ds41/bench_aiperf.sh <mode> <tag> <port> [args]
|
||||
|
||||
x-common: &common
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment: &common-env
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
old-cp2:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-baba18bc7d-local-20261010-2310
|
||||
container_name: bencha-old-cp2
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *common-env
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30500
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
|
||||
old-dp2:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740
|
||||
container_name: bencha-old-dp2
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *common-env
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--dp-size 2 --enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 160
|
||||
--max-running-requests 300
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30600
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
|
||||
new-cp2:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:g9i-est-td-1d2610bbc-local-20261018-1506
|
||||
container_name: bencha-new-cp2
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang-e43917:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *common-env
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30700
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
|
||||
new-dp2:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:g9i-est-td-1d2610bbc-local-20261018-1506
|
||||
container_name: bencha-new-dp2
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang-e43917:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *common-env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--dp-size 2 --enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 160
|
||||
--max-running-requests 300
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30800
|
||||
--enable-cache-report --enable-metrics
|
||||
--skip-server-warmup
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -1,229 +0,0 @@
|
||||
# 【阶段B 新旧基线 2026-10-18】两套 1P1D PD 并行:旧版 vs 新版同部署对比
|
||||
# 旧栈:P=GPU0,1:32100 D=GPU2,3:32200 router=32300 master=50051(镜像 ∈ tag 26.10.18)
|
||||
# 新栈:P=GPU4,5:32400 D=GPU6,7:32500 router=32600 master=50061(g9i-est-td-e4391748e)
|
||||
# 配置 = 线上 k8s P/D spec 原样(mem 0.85、ratio 1.0、write_through、wait_complete、
|
||||
# dspark b5、cp2-P / dp2-D),仅改端口/bootstrap/master;OTLP trace 不陪跑。
|
||||
# NVLink-intra:pid:host + MC_INTRA 双拼写 + AUX_TCP(与线上 k8s spec 逐字对齐;CUSTOM_MEM_POOL 实测触发段错误,不设)。
|
||||
# 两套各挂各的 mooncake master(rank 名在同 master 下必须唯一,混挂会撞 localhost-d0tXpYcZ)。
|
||||
# 新栈复用阶段A已暖的 cache /data/ymk/cache/sglang-e43917(flashinfer 0.7.2 autotune 已跑过)。
|
||||
# ⚠️ 用前必须 down 阶段A(docker compose -p bencha down)。
|
||||
# 用法:docker compose -p benchb -f dockerserve-benchb-1p1d-x2.yml up -d
|
||||
# 入口:旧栈 http://127.0.0.1:32300 新栈 http://127.0.0.1:32600(api-key deepseek-v4-flash-0731-key)
|
||||
|
||||
x-common: &common
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-pd-env: &pd-env
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
x-p-extra-hosts: &p-extra-hosts
|
||||
# mooncake rank 后缀主机名(84fc632088 要求可解析)
|
||||
- "localhost-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d0t1p0c1:127.0.0.1"
|
||||
|
||||
services:
|
||||
# ---------- 旧栈(tag 26.10.18) ----------
|
||||
master-old:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-baba18bc7d-local-20261010-2310
|
||||
container_name: benchb-master-old
|
||||
network_mode: host
|
||||
command: >
|
||||
/opt/sglang/bin/mooncake_master --port 50051
|
||||
--enable_http_metadata_server=true --http_metadata_server_port=8080
|
||||
--eviction_high_watermark_ratio=0.90
|
||||
|
||||
p-old:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-baba18bc7d-local-20261010-2310
|
||||
container_name: benchb-p-old
|
||||
extra_hosts: *p-extra-hosts
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *pd-env
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-tcp.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 32100
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9601
|
||||
|
||||
d-old:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740
|
||||
container_name: benchb-d-old
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *pd-env
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--dp-size 2 --enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 160
|
||||
--max-running-requests 300
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 32200
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9602
|
||||
|
||||
router-old:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-baba18bc7d-local-20261010-2310
|
||||
container_name: benchb-router-old
|
||||
network_mode: host
|
||||
entrypoint: sglang-router
|
||||
command: >
|
||||
launch --mini-lb --pd-disaggregation
|
||||
--prefill http://127.0.0.1:32100 9601
|
||||
--decode http://127.0.0.1:32200
|
||||
--host 0.0.0.0 --port 32300
|
||||
--api-key deepseek-v4-flash-0731-key
|
||||
|
||||
# ---------- 新栈(g9i-est-td @ e4391748e) ----------
|
||||
master-new:
|
||||
image: ymkymx/sglang:g9i-est-td-148936bcb-local-20261018-1803
|
||||
container_name: benchb-master-new
|
||||
network_mode: host
|
||||
command: >
|
||||
/opt/sglang/bin/mooncake_master --port 50061
|
||||
--enable_http_metadata_server=true --http_metadata_server_port=8081
|
||||
--metrics_port 9004
|
||||
--eviction_high_watermark_ratio=0.90
|
||||
|
||||
p-new:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:g9i-est-td-148936bcb-local-20261018-1803
|
||||
container_name: benchb-p-new
|
||||
extra_hosts: *p-extra-hosts
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang-e43917:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *pd-env
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-tcp-50061.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 32400
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9611
|
||||
|
||||
d-new:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:g9i-est-td-148936bcb-local-20261018-1803
|
||||
container_name: benchb-d-new
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang-e43917:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *pd-env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--dp-size 2 --enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 160
|
||||
--max-running-requests 300
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 32500
|
||||
--enable-cache-report --enable-metrics
|
||||
--skip-server-warmup
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9612
|
||||
|
||||
router-new:
|
||||
image: ymkymx/sglang:g9i-est-td-148936bcb-local-20261018-1803
|
||||
container_name: benchb-router-new
|
||||
network_mode: host
|
||||
entrypoint: sglang-router
|
||||
command: >
|
||||
launch --mini-lb --pd-disaggregation
|
||||
--prefill http://127.0.0.1:32400 9611
|
||||
--decode http://127.0.0.1:32500
|
||||
--host 0.0.0.0 --port 32600
|
||||
--api-key deepseek-v4-flash-0731-key
|
||||
@@ -1,160 +0,0 @@
|
||||
# 4×cp2 独立实例(非 PD)+ hicache L3(mooncake) + dspark + engram host table 卸载(2026-09-23 晚)
|
||||
# 用户指定组合:4 cp2 hicache l3 dspark roundrobin loadbalance engram offload
|
||||
# 每实例:tp2 ep2 + interleave CP2 + dspark b5 + hicache L3 write_through(size 0) + engram host table
|
||||
# (tp2 权重必须靠 engram 卸载才放得下,见 goals-track G1.9 反转)
|
||||
# GPU: a=0,1 b=2,3 c=4,5 d=6,7;端口: 30000/30010/30020/30030;rr router=30002
|
||||
# 参考:dockerserve-cpdspark-l3.yml(hicache 参数)、dockerserve-3cp2-pd.yml(cp2/engram 参数)
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
# 热补丁(2026-09-23):CP 对称 idle 检查,修 health-check × hicache drain 死锁
|
||||
# 源文件:/data/ymk/sglang(dsv41-pd-visioncp 分支工作区已改);补丁脚本见
|
||||
# D:\B300\experiments\cp2x4\patch_healthcheck_cp_idle.py
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
environment: &env
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_MS_AUTO_DISC: "0"
|
||||
MOONCAKE_MASTER: 127.0.0.1:50051
|
||||
MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE
|
||||
MOONCAKE_PROTOCOL: tcp
|
||||
MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
a:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-a
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
b:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-b
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
c:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-c
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
d:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-d2
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30030
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user