[Spec][Ngram] Support multiple SAMs with dynamic HTTP API (#22203)
This commit is contained in:
@@ -118,8 +118,8 @@ def get_ngram_corpus_cls():
|
||||
state_ids_t = torch.tensor(state_ids, dtype=torch.int64)
|
||||
self.erase_match_state(state_ids_t) # type: ignore
|
||||
|
||||
def load_external_corpus(
|
||||
self, chunks: Iterable[Sequence[int]]
|
||||
def load_external_corpus_named(
|
||||
self, corpus_id: str, chunks: Iterable[Sequence[int]]
|
||||
) -> Tuple[int, int]:
|
||||
self.start_external_corpus_load() # type: ignore
|
||||
chunk_count = 0
|
||||
@@ -130,10 +130,19 @@ def get_ngram_corpus_cls():
|
||||
loaded_token_count += len(tokens_t)
|
||||
self.append_external_corpus_tokens(tokens_t) # type: ignore
|
||||
chunk_count += 1
|
||||
self.finish_external_corpus_load() # type: ignore
|
||||
self.finish_external_corpus_load(corpus_id) # type: ignore
|
||||
except Exception:
|
||||
self.clear_external_corpus() # type: ignore
|
||||
raise
|
||||
return chunk_count, loaded_token_count
|
||||
|
||||
def remove_corpus(self, corpus_id: str) -> None:
|
||||
self.remove_external_corpus(corpus_id) # type: ignore
|
||||
|
||||
def list_corpora(self) -> List[str]:
|
||||
result = self.list_external_corpora() # type: ignore
|
||||
if not result:
|
||||
return []
|
||||
return result.split("\n")
|
||||
|
||||
return NgramCorpusFFI
|
||||
|
||||
Reference in New Issue
Block a user