[CI] Merge tokenizer worker tests and drop redundant triton attention e2e (#33641)

test/registered/tokenizer/test_multi_tokenizer.py

@@ -24,6 +24,9 @@ register_amd_ci(est_time=355, suite="stage-b-test-1-gpu-small-amd")
 
 
 class TestMultiTokenizer(CustomTestCase, MMLUMixin):
+    """One server covering both worker pools: multi-tokenizer and
+    multi-detokenizer (the flags are orthogonal)."""
+
     mmlu_score_threshold = 0.65
     mmlu_num_examples = 64
     mmlu_num_threads = 32
@@ -39,6 +42,8 @@ class TestMultiTokenizer(CustomTestCase, MMLUMixin):
             other_args=[
                 "--tokenizer-worker-num",
                 8,
+                "--detokenizer-worker-num",
+                4,
                 "--mem-fraction-static",
                 0.7,
             ],