minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

fix(grpc): derive choice count before normalization (#35778)

Browse Source 
Signed-off-by: Connor Carpenter <connorc@nvidia.com>
This commit is contained in:
Connor Carpenter
2026-08-21 11:29:52 -07:00
committed by GitHub
parent 61c2da42bb
commit c3735625de
2 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/grpc_bridge.py
+3 -1
View File
@@ -299,7 +299,9 @@ class RuntimeHandle:
gen = self.tokenizer_manager.generate_request(obj, request=request) gen = self.tokenizer_manager.generate_request(obj, request=request)
if stream: if stream:
completed_choices = set() completed_choices = set()
expected_choices = obj.batch_size * obj.parallel_sample_num # generate_request does not normalize obj until iteration begins.
sampling_params = obj.sampling_params or {}
expected_choices = max(1, int(sampling_params.get("n", 1)))
async for chunk in gen: async for chunk in gen:
choice_finished = ( choice_finished = (
chunk.get("meta_info", {}).get("finish_reason") is not None chunk.get("meta_info", {}).get("finish_reason") is not None
test/registered/unit/entrypoints/test_grpc_bridge.py
+1 -1
View File
@@ -93,7 +93,7 @@ class TestNativeGrpcParallelResponses(CustomTestCase):
}, },
] ]
handle = _make_runtime_handle(responses) handle = _make_runtime_handle(responses)
obj = SimpleNamespace(rid="logical", batch_size=1, parallel_sample_num=2) obj = SimpleNamespace(rid="logical", sampling_params={"n": 2})
asyncio.run( asyncio.run(
handle._run_generate( handle._run_generate(