From f3c3eea6083cd2842b48f2f23cde87e36eb8ad90 Mon Sep 17 00:00:00 2001 From: Alison Shao <54658187+alisonshao@users.noreply.github.com> Date: Tue, 7 Jul 2026 19:56:32 -0700 Subject: [PATCH] ci: make multi-GPU jit test hangs attributable from the CI log (#29925) --- python/sglang/jit_kernel/mp.py | 4 ++++ python/sglang/jit_kernel/tests/utils.py | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/python/sglang/jit_kernel/mp.py b/python/sglang/jit_kernel/mp.py index 9afd10135..61da78e6c 100644 --- a/python/sglang/jit_kernel/mp.py +++ b/python/sglang/jit_kernel/mp.py @@ -168,6 +168,10 @@ def multigpu_launch( os.environ[pid_key] = str(os.getpid()) os.environ.setdefault("OMP_NUM_THREADS", "1") os.environ.setdefault("GLOO_SOCKET_IFNAME", "lo") # single-machine setup + # Unbuffered child stdout: when a worker is killed on timeout, pytest's + # block-buffered progress output is otherwise lost or flushed out of + # order into the CI log, making it impossible to tell which test hung. + os.environ.setdefault("PYTHONUNBUFFERED", "1") signal.signal(signal.SIGINT, signal.default_int_handler) runnable: List[int] = [] for N in sorted(num_gpus): diff --git a/python/sglang/jit_kernel/tests/utils.py b/python/sglang/jit_kernel/tests/utils.py index 695be5380..493df2c9a 100644 --- a/python/sglang/jit_kernel/tests/utils.py +++ b/python/sglang/jit_kernel/tests/utils.py @@ -43,7 +43,14 @@ def multigpu_pytest_main( # CI's run_unittest_files invokes `python3 -f` (legacy # unittest failfast). Translate to pytest's `-x` so it survives. pytest_args = ["-x" if a == "-f" else a for a in sys.argv[1:]] - return pytest.main([file] + pytest_args) + # Dump all thread stacks (every rank; stderr is not redirected) if a + # single test exceeds half the harness budget, so a hung collective + # is attributable from the CI log before the outer timeout kills the + # process group. Non-fatal: the test keeps running after the dump. + dump_after = (timeout // 2) if timeout else 300 + return pytest.main( + [file, "-o", f"faulthandler_timeout={dump_after}"] + pytest_args + ) return multigpu_launch( name,