Increase flush cache timeout in hicache CI (#21305)

This commit is contained in:
Ke Bao
2026-03-24 19:00:59 -07:00
committed by GitHub
parent cd634abee4
commit c1d930c028
+12 -6
View File
@@ -169,21 +169,27 @@ def download_image_with_retry(image_url: str, max_retries: int = 3) -> Image.Ima
def flush_cache_with_retry(
base_url: str, retries: int = 5, interval: float = 2.0
base_url: str,
timeout: float = 30.0,
poll_interval: float = 0.5,
) -> bool:
"""Flush device cache with retry.
"""Flush device cache, polling until success or timeout.
The scheduler may still have in-flight HiCache async ops (GPU↔Host↔L3)
that prevent is_fully_idle() from returning True, so we retry.
flush_cache only succeeds when the scheduler is fully idle, but
HiCache async ops (write-through, backup) may still be in-flight
after a request completes. We poll with a short interval so idle
is detected quickly, while the generous timeout accommodates slow
CI environments.
"""
for _ in range(retries):
deadline = time.time() + timeout
while time.time() < deadline:
try:
response = requests.post(f"{base_url}/flush_cache", timeout=10)
if response.status_code == 200:
return True
except requests.RequestException:
pass
time.sleep(interval)
time.sleep(poll_interval)
return False