[CP V1 Deprecation 3/5] Remove generic prefill CP v1 runtime (#36228)
This commit is contained in:
@@ -380,7 +380,7 @@ class EagerRunner(BaseRunner):
|
||||
def _execute_extend_cp_v2(
|
||||
self, forward_batch: ForwardBatch, kwargs: dict
|
||||
) -> Union[LogitsProcessorOutput, PPProxyTensors]:
|
||||
"""CP-v2 extend: shard inputs at the model boundary, run the body on the
|
||||
"""CP extend: shard inputs at the model boundary, run the body on the
|
||||
rank-local slice, then gather hidden states before the logits step.
|
||||
"""
|
||||
model = self.model_runner.model
|
||||
@@ -389,13 +389,16 @@ class EagerRunner(BaseRunner):
|
||||
if input_embeds is None:
|
||||
input_embeds = model.get_input_embeddings()(forward_batch.input_ids)
|
||||
with cp_shard_model_inputs(
|
||||
input_embeds, forward_batch.positions, forward_batch
|
||||
) as (sharded_input_embeds, sharded_positions):
|
||||
input_embeds,
|
||||
forward_batch.positions,
|
||||
forward_batch,
|
||||
forward_batch.input_ids,
|
||||
) as (sharded_input_embeds, sharded_positions, model_input_ids):
|
||||
model_kwargs = {"input_embeds": sharded_input_embeds}
|
||||
if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None:
|
||||
model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors
|
||||
hidden_states = model.model(
|
||||
forward_batch.input_ids,
|
||||
model_input_ids,
|
||||
sharded_positions,
|
||||
forward_batch,
|
||||
**model_kwargs,
|
||||
|
||||
Reference in New Issue
Block a user