[Spec][Ngram] Return token counts in list_external_corpora API (#22471)

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

python/sglang/jit_kernel/csrc/ngram_corpus/ngram.cpp

@@ -115,14 +115,14 @@ void Ngram::clearExternalCorpus() {
   staging_sam_.reset();
 }
 
-std::vector<std::string> Ngram::listExternalCorpora() const {
+std::vector<std::pair<std::string, int64_t>> Ngram::listExternalCorpora() const {
   std::unique_lock<std::mutex> lock(mutex_);
-  std::vector<std::string> ids;
-  ids.reserve(sams_.size());
-  for (const auto& [id, _] : sams_) {
-    ids.push_back(id);
+  std::vector<std::pair<std::string, int64_t>> entries;
+  entries.reserve(sams_.size());
+  for (const auto& [id, sam] : sams_) {
+    entries.emplace_back(id, sam->tokenCount());
   }
-  return ids;
+  return entries;
 }
 
 void Ngram::insertWorker() {

python/sglang/jit_kernel/csrc/ngram_corpus/ngram.h

@@ -59,7 +59,7 @@ class Ngram {
 
   void clearExternalCorpus();
 
-  std::vector<std::string> listExternalCorpora() const;
+  std::vector<std::pair<std::string, int64_t>> listExternalCorpora() const;
 
   Result batchMatch(const std::vector<std::vector<int32_t>>& tokens);
 

python/sglang/jit_kernel/csrc/ngram_corpus/ngram_corpus_ffi.cpp

@@ -129,11 +129,11 @@ struct NgramCorpusObj : public tvm::ffi::Object {
   }
 
   std::string list_external_corpora() {
-    auto ids = ngram_->listExternalCorpora();
+    auto entries = ngram_->listExternalCorpora();
     std::string result;
-    for (size_t i = 0; i < ids.size(); ++i) {
+    for (size_t i = 0; i < entries.size(); ++i) {
       if (i > 0) result += "\n";
-      result += ids[i];
+      result += entries[i].first + "\t" + std::to_string(entries[i].second);
     }
     return result;
   }

python/sglang/jit_kernel/csrc/ngram_corpus/suffix_automaton.h

@@ -39,6 +39,10 @@ class SuffixAutomaton {
     return !loaded_;
   }
 
+  int64_t tokenCount() const {
+    return pos_;
+  }
+
   Result buildRecency(
       const int32_t* context, size_t len, int32_t last_token, size_t draft_token_num, const Param& param) const;