avoid host sync in DSpark prefill slot expansion (#40111)
This commit is contained in:
@@ -640,11 +640,16 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
state_slot = final_pos = None
|
||||
if is_unified_kv_triton():
|
||||
repeats = ctx_lens.to(torch.int64)
|
||||
num_tokens = sum(batch.extend_lens)
|
||||
state_slot = torch.repeat_interleave(
|
||||
batch.req_pool_indices.to(device=device, dtype=torch.int64), repeats
|
||||
batch.req_pool_indices.to(device=device, dtype=torch.int64),
|
||||
repeats,
|
||||
output_size=num_tokens,
|
||||
)
|
||||
final_pos = torch.repeat_interleave(
|
||||
(draft_seq_lens + ctx_lens - 1).to(torch.int64), repeats
|
||||
(draft_seq_lens + ctx_lens - 1).to(torch.int64),
|
||||
repeats,
|
||||
output_size=num_tokens,
|
||||
)
|
||||
cache_loc = batch.out_cache_loc
|
||||
token_indices = logits_output.hidden_states_token_indices
|
||||
|
||||
Reference in New Issue
Block a user