Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
3d31ac2672
commit
6575aea128
@@ -1003,9 +1003,7 @@ class ModelRunner(ModelRunnerKVCacheMixin):
|
||||
sv = torch.empty(0, dtype=torch.uint8, device=t.device).set_(
|
||||
t.untyped_storage()
|
||||
)
|
||||
if sv.data_ptr() not in {
|
||||
v.data_ptr() for v in model_tensors.values()
|
||||
}:
|
||||
if sv.data_ptr() not in {v.data_ptr() for v in model_tensors.values()}:
|
||||
model_tensors[f"{name}.__storage"] = sv
|
||||
|
||||
nixl_metadata = nixl_mgr.register_tensors(model_tensors)
|
||||
|
||||
@@ -2408,9 +2408,7 @@ class RemoteInstanceModelLoader(BaseModelLoader):
|
||||
|
||||
# --- Transport-specific transfer ---
|
||||
if transport == "nixl":
|
||||
self._transfer_via_nixl(
|
||||
model, nixl_mgr, source_worker, tp_rank
|
||||
)
|
||||
self._transfer_via_nixl(model, nixl_mgr, source_worker, tp_rank)
|
||||
else:
|
||||
self._transfer_via_transfer_engine(
|
||||
model, transfer_engine, source_worker, tp_rank
|
||||
@@ -2555,8 +2553,7 @@ class RemoteInstanceModelLoader(BaseModelLoader):
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"ModelExpress [nixl]: transferred %d tensors, "
|
||||
"%.2f GB in %.2fs",
|
||||
"ModelExpress [nixl]: transferred %d tensors, " "%.2f GB in %.2fs",
|
||||
matched,
|
||||
total_bytes / 1e9,
|
||||
duration,
|
||||
|
||||
@@ -7109,7 +7109,10 @@ class ServerArgs:
|
||||
if self.remote_instance_weight_loader_start_seed_via_transfer_engine:
|
||||
return True
|
||||
# ModelExpress source mode needs TransferEngine init only if transport is transfer_engine.
|
||||
if self.modelexpress_source and self.modelexpress_transport == "transfer_engine":
|
||||
if (
|
||||
self.modelexpress_source
|
||||
and self.modelexpress_transport == "transfer_engine"
|
||||
):
|
||||
return True
|
||||
# Use TransferEngine as client backend.
|
||||
elif (
|
||||
|
||||
Reference in New Issue
Block a user