diff --git a/python/sglang/srt/entrypoints/grpc_bridge.py b/python/sglang/srt/entrypoints/grpc_bridge.py index dbab46081..dc62075bb 100644 --- a/python/sglang/srt/entrypoints/grpc_bridge.py +++ b/python/sglang/srt/entrypoints/grpc_bridge.py @@ -299,7 +299,9 @@ class RuntimeHandle: gen = self.tokenizer_manager.generate_request(obj, request=request) if stream: completed_choices = set() - expected_choices = obj.batch_size * obj.parallel_sample_num + # generate_request does not normalize obj until iteration begins. + sampling_params = obj.sampling_params or {} + expected_choices = max(1, int(sampling_params.get("n", 1))) async for chunk in gen: choice_finished = ( chunk.get("meta_info", {}).get("finish_reason") is not None diff --git a/test/registered/unit/entrypoints/test_grpc_bridge.py b/test/registered/unit/entrypoints/test_grpc_bridge.py index dccc7b24b..6aa741bb8 100644 --- a/test/registered/unit/entrypoints/test_grpc_bridge.py +++ b/test/registered/unit/entrypoints/test_grpc_bridge.py @@ -93,7 +93,7 @@ class TestNativeGrpcParallelResponses(CustomTestCase): }, ] handle = _make_runtime_handle(responses) - obj = SimpleNamespace(rid="logical", batch_size=1, parallel_sample_num=2) + obj = SimpleNamespace(rid="logical", sampling_params={"n": 2}) asyncio.run( handle._run_generate(