[Refactor] Cuda Graph Runner/Backend Refactor (#23906)
Co-authored-by: BBuf <1182563586@qq.com> Co-authored-by: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Co-authored-by: Lianmin Zheng <lianminzheng@gmail.com>
This commit is contained in:
co-authored by
BBuf
Cheng Wan
Lianmin Zheng
parent
56f06278c6
commit
2495c02c2c
+4
-4
@@ -29,8 +29,8 @@ class TestPiecewiseGraphPrefillCorrectness(GSM8KAscendMixin, CustomTestCase):
|
||||
"ascend",
|
||||
"--cuda-graph-bs",
|
||||
128,
|
||||
"--enforce-piecewise-cuda-graph",
|
||||
"--piecewise-cuda-graph-tokens",
|
||||
"--cuda-graph-backend-prefill=tc_piecewise",
|
||||
"--cuda-graph-bs-prefill",
|
||||
*TOKENS_TO_CAPTURE,
|
||||
]
|
||||
accuracy = 0.84
|
||||
@@ -45,8 +45,8 @@ class TestPiecewiseGraphPrefillBenchmark(CustomTestCase):
|
||||
0.8,
|
||||
"--attention-backend",
|
||||
"ascend",
|
||||
"--enforce-piecewise-cuda-graph",
|
||||
"--piecewise-cuda-graph-tokens",
|
||||
"--cuda-graph-backend-prefill=tc_piecewise",
|
||||
"--cuda-graph-bs-prefill",
|
||||
] + TOKENS_TO_CAPTURE
|
||||
|
||||
latency = 0.045
|
||||
|
||||
Reference in New Issue
Block a user