diff --git a/python/sglang/srt/layers/engram.py b/python/sglang/srt/layers/engram.py index 71fc553dce..024a4b9fb2 100644 --- a/python/sglang/srt/layers/engram.py +++ b/python/sglang/srt/layers/engram.py @@ -192,6 +192,15 @@ class EngramLayout(msgspec.Struct, frozen=True): head_dim=config.engram_head_dim, ) + def column_bounds(self, hash_index: int) -> tuple[int, ...]: + """Prefix sums of one layer's column segments, in the hash's (n-gram + size, head) column order: hash column c owns rows [bounds[c], bounds[c + 1]).""" + bounds = [0] + for per_ngram in self.primes[hash_index]: + for prime in per_ngram: + bounds.append(bounds[-1] + prime) + return tuple(bounds) + def compute_engram_hash_ids( tokens: torch.Tensor, @@ -745,22 +754,55 @@ class _HostTable: logger.info(msg) +def shard_range( + num_embeddings: int, + column_bounds: tuple[int, ...], + tp_rank: int, + tp_size: int, +) -> tuple[int, int]: + """This rank's [start, end) rows: on the layer's hash-column boundaries when + the column count divides evenly, so a rank owns whole per-head subtables, + else the equal-row split. Exactly one rank owns each row either way, so the + zero-fill + all-reduce lookup is bitwise independent of the choice.""" + assert column_bounds[-1] == num_embeddings, ( + "the config's table size disagrees with its primes" + ) + if (len(column_bounds) - 1) % tp_size == 0: + cols_per_rank = (len(column_bounds) - 1) // tp_size + return ( + column_bounds[tp_rank * cols_per_rank], + column_bounds[(tp_rank + 1) * cols_per_rank], + ) + return ( + num_embeddings * tp_rank // tp_size, + num_embeddings * (tp_rank + 1) // tp_size, + ) + + class EngramEmbedding(nn.Module): """One layer's fp8 hash table with e8m0 block scales, dequantized on lookup. - Rows are sharded over the TP group in device memory; with + Rows are sharded over the TP group in device memory, aligned to the hash + columns where they divide evenly (`shard_range`); with SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE they live in host memory instead, as one shared copy or one shard per rank (see _HostTable). Loading is sharded in every layout: a rank writes only its own row range. """ - def __init__(self, num_embeddings: int, dim: int, layer_id: int): + def __init__( + self, + num_embeddings: int, + dim: int, + layer_id: int, + column_bounds: tuple[int, ...], + ): super().__init__() self.dim = dim self.tp_size = get_parallel().tp_size tp_rank = get_parallel().tp_rank - self.row_start = num_embeddings * tp_rank // self.tp_size - row_end = num_embeddings * (tp_rank + 1) // self.tp_size + self.row_start, row_end = shard_range( + num_embeddings, column_bounds, tp_rank, self.tp_size + ) self.rows = row_end - self.row_start self.host_table: Optional[_HostTable] = None if envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get(): @@ -990,7 +1032,10 @@ class Engram(nn.Module): self.clamp_value = 1e-6 dim, hc_mult = config.hidden_size, config.hc_mult self.embed = EngramEmbedding( - layout.num_embeddings[self.layer_hash_index], layout.head_dim, layer_id + layout.num_embeddings[self.layer_hash_index], + layout.head_dim, + layer_id, + column_bounds=layout.column_bounds(self.layer_hash_index), ) n_hash_cols = (layout.max_ngram_size - 1) * layout.n_heads self.wkv = ReplicatedLinear( diff --git a/test/registered/unit/models/test_deepseek_v4_amd_tp4.py b/test/registered/unit/models/test_deepseek_v4_amd_tp4.py index 2cf1f45573..a717ebb2d1 100644 --- a/test/registered/unit/models/test_deepseek_v4_amd_tp4.py +++ b/test/registered/unit/models/test_deepseek_v4_amd_tp4.py @@ -390,7 +390,9 @@ def _engram_table(rows): envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.override(False), torch.device("cuda"), ): - embed = EngramEmbedding(rows, 128, layer_id=1) + # One hash column per rank, so TP4 takes the column-aligned split. + bounds = tuple(rows * c // 4 for c in range(5)) + embed = EngramEmbedding(rows, 128, layer_id=1, column_bounds=bounds) embed.weight.weight_loader(embed.weight, weight) embed.scale.weight_loader(embed.scale, scale.view(torch.float8_e8m0fnu)) embed.finish_load(label="test")