refactor: replace oversized 1.3MB tiny_tokenizer.json fixture with a genuinely tiny byte-level BPE fixture (#27235)

Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>

experimental/sgl-router/src/tokenizer/mod.rs

@@ -102,15 +102,17 @@ mod tests {
 
     /// Forces `decode_complete` through its `DecodeResult::Partial` branch.
     ///
-    /// Strategy A: the fixture is a GPT-2 byte-level BPE. The 4-byte UTF-8
+    /// The fixture is a no-merge byte-level BPE. The 4-byte UTF-8 emoji
-    /// emoji `😀` (`\xF0\x9F\x98\x80`) encodes into 2 byte-level BPE tokens
+    /// `😀` (`\xF0\x9F\x98\x80`) encodes to its raw byte token ids:
-    /// with this fixture: `[47249, 222]`. Decoding just the first token
+    /// `[240, 159, 152, 128]`. Decoding only a prefix yields leading bytes
-    /// yields a leading-bytes-only prefix that the HF adapter passes through
+    /// that the HF adapter passes through `String::from_utf8_lossy`,
-    /// `String::from_utf8_lossy`, producing a trailing U+FFFD. dynamo's
+    /// producing a trailing U+FFFD. dynamo's `DecodeResult::from_decoded`
-    /// `DecodeResult::from_decoded` then classifies that as `Partial`.
+    /// then classifies that as `Partial`.
-    /// Pinning the literal token id keeps the test deterministic — if the
+    ///
-    /// fixture or upstream BPE merges ever shift, this fails loudly rather
+    /// Pinning the literal token ids keeps the test deterministic: if the
-    /// than silently dropping back into `Complete` and losing coverage.
+    /// fixture shape or upstream byte-level handling ever shifts, this fails
+    /// loudly rather than silently dropping back into `Complete` and losing
+    /// coverage.
     #[test]
     fn decode_complete_returns_string_on_partial_utf8() {
         let r = TokenizerRegistry::load_from_config(&cfg()).unwrap();
@@ -121,13 +123,13 @@ mod tests {
         let full = adapter::encode(&t, "😀").unwrap();
         assert_eq!(
             full,
-            vec![47249, 222],
+            vec![240, 159, 152, 128],
-            "fixture tokenisation drift: '😀' no longer encodes to [47249, 222]"
+            "fixture tokenisation drift: '😀' no longer encodes to [240, 159, 152, 128]"
         );
 
-        // Feed only the first token — its bytes are the leading 3 of a
+        // Feed only the first three bytes of a 4-byte UTF-8 codepoint,
-        // 4-byte UTF-8 codepoint, which is incomplete.
+        // which is incomplete.
-        let s = adapter::decode_complete(&t, &full[..1], false).unwrap();
+        let s = adapter::decode_complete(&t, &full[..3], false).unwrap();
 
         // We pin the exact output: the lossy decoder folds the 3 leading
         // bytes into a single U+FFFD. Anything else (empty string, Err, or