Increase flush cache timeout in hicache CI (#21305)
This commit is contained in:
@@ -169,21 +169,27 @@ def download_image_with_retry(image_url: str, max_retries: int = 3) -> Image.Ima
|
||||
|
||||
|
||||
def flush_cache_with_retry(
|
||||
base_url: str, retries: int = 5, interval: float = 2.0
|
||||
base_url: str,
|
||||
timeout: float = 30.0,
|
||||
poll_interval: float = 0.5,
|
||||
) -> bool:
|
||||
"""Flush device cache with retry.
|
||||
"""Flush device cache, polling until success or timeout.
|
||||
|
||||
The scheduler may still have in-flight HiCache async ops (GPU↔Host↔L3)
|
||||
that prevent is_fully_idle() from returning True, so we retry.
|
||||
flush_cache only succeeds when the scheduler is fully idle, but
|
||||
HiCache async ops (write-through, backup) may still be in-flight
|
||||
after a request completes. We poll with a short interval so idle
|
||||
is detected quickly, while the generous timeout accommodates slow
|
||||
CI environments.
|
||||
"""
|
||||
for _ in range(retries):
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
response = requests.post(f"{base_url}/flush_cache", timeout=10)
|
||||
if response.status_code == 200:
|
||||
return True
|
||||
except requests.RequestException:
|
||||
pass
|
||||
time.sleep(interval)
|
||||
time.sleep(poll_interval)
|
||||
return False
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user