[CI] Keep the KL tests' LongBench cache outside the checkout and bound its download (#43068)
This commit is contained in:
@@ -2,6 +2,7 @@ import inspect
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import requests
|
||||
@@ -14,7 +15,14 @@ LONGBENCH_V2_DATASET = "THUDM/LongBench-v2"
|
||||
LONGBENCH_V2_SPLIT = "train"
|
||||
DEFAULT_NUM_SAMPLES = 48 # Number of samples to use
|
||||
DEFAULT_PROMPT_TOKENS = 3000 # Maximum number of tokens to use
|
||||
CACHE_DIR = os.path.join(os.path.dirname(__file__), ".longbench_cache")
|
||||
# Outside the repo, where CI's checkout clean would wipe it before every job.
|
||||
CACHE_DIR = os.path.join(
|
||||
os.environ.get("XDG_CACHE_HOME", os.path.expanduser("~/.cache")),
|
||||
"sglang",
|
||||
"longbench",
|
||||
)
|
||||
# Fail the test instead of letting a slow stream run out the CI job.
|
||||
DOWNLOAD_TIMEOUT_S = 300
|
||||
|
||||
# In-memory cache for the current session
|
||||
_cached_input_ids = {}
|
||||
@@ -78,18 +86,26 @@ def get_input_ids(
|
||||
)
|
||||
|
||||
input_ids = []
|
||||
for i, example in enumerate(dataset):
|
||||
deadline = time.monotonic() + DOWNLOAD_TIMEOUT_S
|
||||
for example in dataset:
|
||||
if len(input_ids) >= num_samples:
|
||||
break
|
||||
if time.monotonic() > deadline:
|
||||
raise TimeoutError(
|
||||
f"LongBench V2 download exceeded {DOWNLOAD_TIMEOUT_S}s after "
|
||||
f"{len(input_ids)}/{num_samples} samples"
|
||||
)
|
||||
text = format_longbench_v2_example(example)
|
||||
tokens = tokenizer.encode(text)
|
||||
# Truncate to a random length between 0.5x and 1.5x of max_prompt_tokens
|
||||
truncate_len = int(max_prompt_tokens * random.uniform(0.5, 1.5))
|
||||
input_ids.append(tokens[:truncate_len])
|
||||
|
||||
# Save to local cache
|
||||
with open(cache_file, "w") as f:
|
||||
# Save to local cache; concurrent jobs may share the directory.
|
||||
tmp_file = f"{cache_file}.{os.getpid()}.tmp"
|
||||
with open(tmp_file, "w") as f:
|
||||
json.dump(input_ids, f)
|
||||
os.replace(tmp_file, cache_file)
|
||||
print(f"Saved {len(input_ids)} prompts to cache: {cache_file}")
|
||||
|
||||
# Also cache in memory
|
||||
|
||||
Reference in New Issue
Block a user