minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Test] Consolidate test cleanup and CI taxonomy (net -11.4K lines) (#37436)

Browse Source 
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Xiaoyu Zhang
2026-09-07 15:13:59 +08:00
committed by GitHub
co-authored by Mick Qian
parent 6a1ff90f2d
commit 4d23a4fa6d
199 changed files with 1185 additions and 11812 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/kernels/ops/diffusion/README.md
+2 -2
View File
@@ -17,7 +17,7 @@ from sglang.kernels.ops.diffusion import fused_rmsnorm_scale_shift_bitexact
```
**Import from the package, never from a submodule.** The internal layout is
free to move; the facade is not. `test_import_surface.py` enforces this, with
free to move; the facade is not. Callers should use the facade, with
a small allowlist for tests that deliberately exercise one backend.
Resolution is lazy (PEP 562): the backends have disjoint heavy dependencies
@@ -189,7 +189,7 @@ inspecting model modules is its whole job.
generated by the KDA workflow in `sglang.kernels.kda_kernels`, together
with its source revision and any JIT CUDA source files.
2. Export it from `__init__.py` (`_EXPORTS`) and register a `KernelSpec`
(`_SPECS`) — `test_import_surface.py` checks both resolve.
(`_SPECS`).
3. Give it a `can_use_*` predicate; raise, don't return `None`.
4. State the numerical contract in the module docstring, including which
shapes it was verified on.
python/sglang/kernels/ops/diffusion/__init__.py
+2 -2
View File
@@ -5,8 +5,8 @@ This module is the **only** supported import surface for these kernels::
from sglang.kernels.ops.diffusion import fused_rmsnorm_scale_shift_bitexact
Importing a submodule directly (``...diffusion.norm.norm_triton``) couples the
caller to the file layout; ``test_import_surface.py`` guards against it. The
one exception is a test that deliberately exercises a single backend.
caller to the file layout. The one exception is a test that deliberately
exercises a single backend.
Layout -- ordinary implementations use one subpackage per **operator domain**
(``norm``, ``modulate``, ``rope``, ``activation``, ``attention``, ``routing``,
python/sglang/multimodal_gen/test/run_suite.py
+2 -687
View File
@@ -1,691 +1,6 @@
"""
Test runner for multimodal_gen that manages test suites and parallel execution.
For diffusion 1-gpu/2-gpu suites, cases are partitioned by estimated runtime
using LPT so each CI shard has a similar total runtime.
"""
import argparse
import copy
import json
import os
import random
import subprocess
import sys
import time
from dataclasses import dataclass
from pathlib import Path
import tabulate
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.test.partitioning import PartitionItem, assign_partition
from sglang.multimodal_gen.test.runner.pytest_runner import (
partition_items_by_index,
run_pytest,
)
from sglang.multimodal_gen.test.server.testcase_configs import (
BASELINE_CONFIG,
DiffusionTestCase,
)
# TODO: remove duplicated code
if current_platform.is_npu():
from sglang.multimodal_gen.test.server.ascend.testcase_configs_npu import (
_UPDATE_WEIGHTS_FROM_DISK_TEST_FILE,
COMPONENT_ACCURACY_SUITES,
DEFAULT_EST_TIME_SECONDS,
DEFAULT_STANDALONE_EST_TIME_SECONDS,
FILE_SUITES,
PARAMETRIZED_CASE_GROUPS,
STANDALONE_FILE_EST_TIMES,
STANDALONE_FILES,
STARTUP_OVERHEAD_SECONDS,
SUITES,
)
else:
from sglang.multimodal_gen.test.server.gpu_cases import ( # noqa: F401 It is used by ci scripts
_UPDATE_WEIGHTS_FROM_DISK_TEST_FILE,
_UPDATE_WEIGHTS_MODEL_PAIR_ENV,
_UPDATE_WEIGHTS_MODEL_PAIR_IDS,
COMPONENT_ACCURACY_FILE_NUM_GPUS,
COMPONENT_ACCURACY_SUITES,
DEFAULT_EST_TIME_SECONDS,
DEFAULT_STANDALONE_EST_TIME_SECONDS,
FILE_SUITES,
ONE_GPU_CASES,
PARAMETRIZED_CASE_GROUPS,
STANDALONE_FILE_EST_TIMES,
STANDALONE_FILES,
STARTUP_OVERHEAD_SECONDS,
STRICT_SUITES,
SUITES,
TWO_GPU_CASES,
)
logger = init_logger(__name__)
@dataclass(frozen=True)
class PartitionAssignment:
case_ids: list[str]
standalone_files: list[str]
estimated_time: float | None = None
missing_standalone_estimates: list[str] | None = None
def get_case_est_time(case_id: str) -> float:
scenario = BASELINE_CONFIG.scenarios.get(case_id)
if scenario is None:
return DEFAULT_EST_TIME_SECONDS
if scenario.estimated_full_test_time_s is not None:
return scenario.estimated_full_test_time_s
return scenario.expected_e2e_ms / 1000.0 + STARTUP_OVERHEAD_SECONDS
def get_standalone_file_est_time(
suite: str, standalone_file: str
) -> tuple[float, bool]:
suite_est_times = STANDALONE_FILE_EST_TIMES.get(suite, {})
if standalone_file not in suite_est_times:
return DEFAULT_STANDALONE_EST_TIME_SECONDS, True
return suite_est_times[standalone_file], False
def get_all_standalone_file_est_times() -> dict[str, dict[str, float]]:
return copy.deepcopy(STANDALONE_FILE_EST_TIMES)
def validate_standalone_file_est_times() -> dict[str, list[str]]:
missing_by_suite: dict[str, list[str]] = {}
for suite, standalone_files in STANDALONE_FILES.items():
suite_est_times = STANDALONE_FILE_EST_TIMES.get(suite, {})
missing = [
standalone_file
for standalone_file in standalone_files
if standalone_file not in suite_est_times
]
if missing:
missing_by_suite[suite] = missing
return missing_by_suite
def get_suite_files_rel(suite: str, parametrized_only: bool = False) -> list[str]:
if parametrized_only and suite in PARAMETRIZED_CASE_GROUPS:
return [filename for filename, _ in PARAMETRIZED_CASE_GROUPS[suite]]
return SUITES[suite]
def _normalize_standalone_key(standalone_file: str) -> str:
return f"standalone:{standalone_file}"
def parse_partition_plan(
suite: str,
partition_id: int,
total_partitions: int,
plan_json: str,
) -> PartitionAssignment:
plan = json.loads(plan_json)
if plan.get("suite") != suite:
raise ValueError(
f"Partition plan suite mismatch: expected {suite!r}, "
f"got {plan.get('suite')!r}"
)
partition_count = plan.get("partition_count")
if partition_count != total_partitions:
raise ValueError(
f"Partition count mismatch for suite {suite!r}: "
f"plan={partition_count}, matrix={total_partitions}"
)
partitions = plan.get("partitions", [])
selected_partition = None
for partition in partitions:
if partition.get("part") == partition_id:
selected_partition = partition
break
if selected_partition is None:
raise ValueError(
f"Partition {partition_id} not found in plan for suite {suite!r}"
)
return PartitionAssignment(
case_ids=list(selected_partition.get("case_ids", [])),
standalone_files=list(selected_partition.get("standalone_files", [])),
estimated_time=selected_partition.get("estimated_time"),
missing_standalone_estimates=list(
selected_partition.get("missing_standalone_estimates", [])
),
)
def build_local_partition_assignment(
suite: str,
partition_id: int,
total_partitions: int,
) -> PartitionAssignment:
"""Assign this shard's work when CI did not precompute a partition plan.
Lanes with a hardcoded ``--total-partitions`` (the AMD ones) cannot give
every standalone file a shard of its own, so standalone files are LPT
balanced together with the parametrized cases instead.
"""
items = [
PartitionItem(kind="case", item_id=case.id, est_time=get_case_est_time(case.id))
for case in _get_dynamic_suite_cases(suite)
]
for standalone_file in STANDALONE_FILES.get(suite, []):
items.append(
PartitionItem(
kind="standalone",
item_id=standalone_file,
est_time=get_standalone_file_est_time(suite, standalone_file)[0],
)
)
my_items = assign_partition(items, partition_id, total_partitions)
return PartitionAssignment(
case_ids=[item.item_id for item in my_items if item.kind == "case"],
standalone_files=[
item.item_id for item in my_items if item.kind == "standalone"
],
)
def _merge_execution_results(
executed_cases: list[str],
case_results: dict[str, str],
new_executed_cases: list[str],
new_case_results: dict[str, str],
) -> None:
executed_cases.extend(
case_id for case_id in new_executed_cases if case_id not in executed_cases
)
case_results.update(new_case_results)
def _format_standalone_estimate_snippet(
suite: str, standalone_file: str, measured_full_test_time_s: float
) -> str:
return (
f'"{suite}": {{\n "{standalone_file}": {measured_full_test_time_s:.1f},\n}}'
)
def _print_missing_standalone_estimate_message(
suite: str,
standalone_file: str,
measured_full_test_time_s: float,
) -> None:
snippet = _format_standalone_estimate_snippet(
suite, standalone_file, measured_full_test_time_s
)
logger.error(
f"\n{'=' * 60}\n"
f'Add standalone estimate for suite "{suite}" and file "{standalone_file}":\n\n'
f"File: python/sglang/multimodal_gen/test/run_suite.py\n\n"
f"Current partition used fallback estimate: "
f"{DEFAULT_STANDALONE_EST_TIME_SECONDS:.1f}s\n\n"
f"{snippet}\n"
f"{'=' * 60}\n"
)
def _run_standalone_file(
suite: str,
standalone_rel: str,
target_dir: Path,
extra_filter: str | None = None,
) -> tuple[int, list[str], dict[str, str], dict]:
if standalone_rel == _UPDATE_WEIGHTS_FROM_DISK_TEST_FILE:
_maybe_pin_update_weights_model_pair([standalone_rel])
est_time, used_fallback_estimate = get_standalone_file_est_time(
suite, standalone_rel
)
standalone_file = _resolve_suite_files(target_dir, [standalone_rel], strict=True)[0]
junit_xml_path = str(
target_dir / f"junit_results_{suite}_{Path(standalone_rel).stem}.xml"
)
start_time = time.perf_counter()
exit_code, _, _ = run_pytest(
[standalone_file],
filter_expr=extra_filter,
junit_xml_path=junit_xml_path,
)
measured_full_test_time_s = round(time.perf_counter() - start_time, 1)
standalone_key = _normalize_standalone_key(standalone_rel)
measurement = {
"suite": suite,
"standalone_file": standalone_rel,
"measured_full_test_time_s": measured_full_test_time_s,
"used_fallback_estimate": used_fallback_estimate,
"fallback_estimate_s": DEFAULT_STANDALONE_EST_TIME_SECONDS,
"had_configured_estimate": not used_fallback_estimate,
"configured_or_fallback_estimate_s": est_time,
}
if used_fallback_estimate:
_print_missing_standalone_estimate_message(
suite, standalone_rel, measured_full_test_time_s
)
return (
exit_code,
[standalone_key],
{standalone_key: "pass" if exit_code == 0 else "fail"},
measurement,
)
def parse_args():
suite_choices = sorted(set(FILE_SUITES) | set(PARAMETRIZED_CASE_GROUPS))
parser = argparse.ArgumentParser(description="Run multimodal_gen test suite")
parser.add_argument(
"--suite",
type=str,
required=True,
choices=suite_choices,
help="The test suite to run.",
)
parser.add_argument(
"--partition-id",
type=int,
default=0,
help="Index of the current partition (for parallel execution)",
)
parser.add_argument(
"--total-partitions",
type=int,
default=1,
help="Total number of partitions",
)
parser.add_argument(
"--base-dir",
type=str,
default="server",
help="Base directory for tests relative to this script's parent",
)
parser.add_argument(
"-k",
"--filter",
type=str,
default=None,
help="Pytest filter expression (passed to pytest -k)",
)
parser.add_argument(
"--continue-on-error",
action="store_true",
default=False,
help="Continue running remaining tests even if one fails.",
)
parser.add_argument(
"--partition-plan-json",
type=str,
default=None,
help="Full partition plan JSON for the current suite.",
)
return parser.parse_args()
def write_execution_report(
suite: str,
partition_id: int,
total_partitions: int,
executed_cases: list[str],
is_standalone: bool = False,
standalone_file: str | None = None,
case_results: dict[str, str] | None = None,
missing_standalone_estimates: list[str] | None = None,
standalone_measurements: list[dict] | None = None,
) -> str:
report = {
"suite": suite,
"partition_id": partition_id,
"total_partitions": total_partitions,
"is_standalone": is_standalone,
"standalone_file": standalone_file,
"executed_cases": executed_cases,
"case_results": case_results or {},
"missing_standalone_estimates": missing_standalone_estimates or [],
"standalone_measurements": standalone_measurements or [],
}
report_filename = f"execution_report_{suite}_{partition_id}.json"
report_path = Path(__file__).parent / report_filename
with open(report_path, "w", encoding="utf-8") as f:
json.dump(report, f, indent=2)
logger.info("Execution report written to: %s", report_path)
return str(report_path)
def run_component_accuracy_files(files, filter_expr=None, continue_on_error=False):
exit_code = 0
for file_path in files:
file_name = Path(file_path).name
num_gpus = COMPONENT_ACCURACY_FILE_NUM_GPUS.get(file_name, 1)
if num_gpus > 1:
cmd = [
sys.executable,
"-m",
"torch.distributed.run",
f"--nproc_per_node={num_gpus}",
"-m",
"pytest",
"-s",
"-v",
]
else:
cmd = [sys.executable, "-m", "pytest", "-s", "-v"]
if filter_expr:
cmd.extend(["-k", filter_expr])
cmd.append(file_path)
print(f"Running command: {' '.join(cmd)}")
file_exit_code = subprocess.call(cmd)
if file_exit_code == 5:
print(
"No tests collected (exit code 5). This is expected when filters "
"deselect all tests in a file. Treating as success."
)
file_exit_code = 0
if file_exit_code != 0 and exit_code == 0:
exit_code = file_exit_code
if file_exit_code != 0 and not continue_on_error:
return file_exit_code
return exit_code
def _is_in_ci() -> bool:
return os.environ.get("SGLANG_IS_IN_CI", "").lower() in ("1", "true", "yes", "on")
def _maybe_pin_update_weights_model_pair(suite_files_rel: list[str]) -> None:
if not _is_in_ci():
return
if _UPDATE_WEIGHTS_FROM_DISK_TEST_FILE not in suite_files_rel:
return
if os.environ.get(_UPDATE_WEIGHTS_MODEL_PAIR_ENV):
print(
f"Using preset {_UPDATE_WEIGHTS_MODEL_PAIR_ENV}="
f"{os.environ[_UPDATE_WEIGHTS_MODEL_PAIR_ENV]}"
)
return
selected_pair = random.choice(_UPDATE_WEIGHTS_MODEL_PAIR_IDS)
os.environ[_UPDATE_WEIGHTS_MODEL_PAIR_ENV] = selected_pair
print(f"Selected {_UPDATE_WEIGHTS_MODEL_PAIR_ENV}={selected_pair} for this CI run")
def _resolve_suite_files(
target_dir: Path, suite_files_rel: list[str], strict: bool
) -> list[str]:
suite_files_abs = []
for f_rel in suite_files_rel:
f_abs = target_dir / f_rel
if not f_abs.exists():
msg = f"Test file {f_rel} not found in {target_dir}."
if strict:
print(f"Error: {msg}")
sys.exit(1)
print(f"Warning: {msg} Skipping.")
continue
suite_files_abs.append(str(f_abs))
return suite_files_abs
def _run_file_suite(args, target_dir: Path) -> int:
suite_files_rel = FILE_SUITES[args.suite]
_maybe_pin_update_weights_model_pair(suite_files_rel)
suite_files_abs = _resolve_suite_files(
target_dir, suite_files_rel, args.suite in STRICT_SUITES
)
if not suite_files_abs:
print(f"No valid test files found for suite '{args.suite}'.")
return 1 if args.suite in STRICT_SUITES else 0
exit_code, _, _ = run_pytest(
suite_files_abs,
filter_expr=args.filter,
junit_xml_path=None,
)
return exit_code
def _get_dynamic_suite_cases(suite: str) -> list[DiffusionTestCase]:
cases = []
for _, case_group in PARAMETRIZED_CASE_GROUPS[suite]:
cases.extend(case_group)
return cases
def _get_parametrized_files_for_case_ids(
suite: str, case_ids: set[str], target_dir: Path
) -> list[str]:
files = []
for filename, case_group in PARAMETRIZED_CASE_GROUPS[suite]:
if any(case.id in case_ids for case in case_group):
file_path = target_dir / filename
if file_path.exists():
files.append(str(file_path))
else:
logger.warning("Test file %s not found in %s", filename, target_dir)
return files
def _run_dynamic_suite(args, target_dir: Path) -> int:
if args.partition_plan_json:
assignment = parse_partition_plan(
suite=args.suite,
partition_id=args.partition_id,
total_partitions=args.total_partitions,
plan_json=args.partition_plan_json,
)
else:
assignment = build_local_partition_assignment(
suite=args.suite,
partition_id=args.partition_id,
total_partitions=args.total_partitions,
)
return _run_partition_assignment(args, target_dir, assignment)
def _run_partition_assignment(
args, target_dir: Path, assignment: PartitionAssignment
) -> int:
rows = [[args.suite, f"{args.partition_id + 1}/{args.total_partitions}"]]
print(tabulate.tabulate(rows, headers=["Suite", "Partition"], tablefmt="psql"))
total_est_time = 0.0
executed_cases: list[str] = []
case_results: dict[str, str] = {}
missing_standalone_estimates: list[str] = []
standalone_measurements: list[dict] = []
overall_exit_code = 0
if assignment.case_ids:
case_id_set = set(assignment.case_ids)
total_est_time += sum(
get_case_est_time(case_id) for case_id in assignment.case_ids
)
suite_files = _get_parametrized_files_for_case_ids(
args.suite, case_id_set, target_dir
)
if not suite_files:
print(f"No valid parametrized test files found for suite '{args.suite}'.")
return 0
partition_filter = " or ".join(
f"[{case_id}]" for case_id in assignment.case_ids
)
filter_expr = (
f"({partition_filter}) and ({args.filter})"
if args.filter
else partition_filter
)
print(
f"Running {len(assignment.case_ids)} parametrized cases with estimated total "
f"{sum(get_case_est_time(case_id) for case_id in assignment.case_ids):.1f}s:"
)
for case_id in assignment.case_ids:
print(f" - case: {case_id} ({get_case_est_time(case_id):.1f}s)")
print(f"Test files: {[Path(f).name for f in suite_files]}")
print(f"Filter expression: {filter_expr}")
junit_xml_path = str(
target_dir / f"junit_results_{args.suite}_{args.partition_id}.xml"
)
exit_code, new_executed_cases, new_case_results = run_pytest(
suite_files,
filter_expr=filter_expr,
junit_xml_path=junit_xml_path,
)
_merge_execution_results(
executed_cases, case_results, new_executed_cases, new_case_results
)
# A failing case must not swallow this shard's standalone files: they
# are separate pytest runs, and they only share a shard because the
# shard count is fixed. --continue-on-error still decides whether a
# failing standalone file stops the ones queued behind it.
if exit_code != 0 and overall_exit_code == 0:
overall_exit_code = exit_code
if assignment.standalone_files:
standalone_estimate = sum(
get_standalone_file_est_time(args.suite, standalone_file)[0]
for standalone_file in assignment.standalone_files
)
total_est_time += standalone_estimate
print(
f"Running {len(assignment.standalone_files)} standalone file(s) with estimated total "
f"{standalone_estimate:.1f}s:"
)
for standalone_file in assignment.standalone_files:
est_time, used_fallback_estimate = get_standalone_file_est_time(
args.suite, standalone_file
)
fallback_suffix = (
f", fallback estimate {DEFAULT_STANDALONE_EST_TIME_SECONDS:.1f}s"
if used_fallback_estimate
else ""
)
print(
f" - standalone: {standalone_file} ({est_time:.1f}s{fallback_suffix})"
)
for standalone_file in assignment.standalone_files:
exit_code, new_executed_cases, new_case_results, measurement = (
_run_standalone_file(
args.suite,
standalone_file,
target_dir,
extra_filter=args.filter,
)
)
if measurement["used_fallback_estimate"]:
missing_standalone_estimates.append(standalone_file)
standalone_measurements.append(measurement)
_merge_execution_results(
executed_cases,
case_results,
new_executed_cases,
new_case_results,
)
if exit_code != 0 and overall_exit_code == 0:
overall_exit_code = exit_code
if exit_code != 0 and not args.continue_on_error:
break
if not assignment.case_ids and not assignment.standalone_files:
print(f"No work assigned to partition {args.partition_id}. Exiting success.")
print(f"Partition estimated total time: {total_est_time:.1f}s")
write_execution_report(
suite=args.suite,
partition_id=args.partition_id,
total_partitions=args.total_partitions,
executed_cases=executed_cases,
is_standalone=False,
standalone_file=None,
case_results=case_results,
missing_standalone_estimates=missing_standalone_estimates,
standalone_measurements=standalone_measurements,
)
return overall_exit_code
def main():
args = parse_args()
validate_standalone_file_est_times()
test_root_dir = Path(__file__).resolve().parent
target_dir = test_root_dir / args.base_dir
if not target_dir.exists():
print(f"Error: Target directory {target_dir} does not exist.")
sys.exit(1)
if args.suite in COMPONENT_ACCURACY_SUITES:
suite_files_rel = FILE_SUITES[args.suite]
suite_files_abs = _resolve_suite_files(
target_dir, suite_files_rel, args.suite in STRICT_SUITES
)
if not suite_files_abs:
print(f"No valid test files found for suite '{args.suite}'.")
sys.exit(1 if args.suite in STRICT_SUITES else 0)
my_files = partition_items_by_index(
suite_files_abs, args.partition_id, args.total_partitions
)
partition_info = (
f"{args.partition_id + 1}/{args.total_partitions} "
f"(0-based id={args.partition_id})"
)
headers = ["Suite", "Partition"]
rows = [[args.suite, partition_info]]
msg = tabulate.tabulate(rows, headers=headers, tablefmt="psql") + "\n"
msg += f"Enabled {len(my_files)} file(s):\n"
for file_path in my_files:
msg += f" - {file_path}\n"
print(msg, flush=True)
print(
f"Suite: {args.suite} | Partition: {args.partition_id}/{args.total_partitions}"
)
print(f"Selected {len(suite_files_abs)} files:")
for f in suite_files_abs:
print(f" - {os.path.basename(f)}")
if not my_files:
print("No files assigned to this partition. Exiting success.")
sys.exit(0)
print(f"Running {len(my_files)} files in this shard: {', '.join(my_files)}")
exit_code = run_component_accuracy_files(
my_files,
filter_expr=args.filter,
continue_on_error=args.continue_on_error,
)
msg = "\n" + tabulate.tabulate(rows, headers=headers, tablefmt="psql") + "\n"
msg += f"Executed {len(my_files)} file(s):\n"
for file_path in my_files:
msg += f" - {file_path}\n"
print(msg, flush=True)
elif args.suite in PARAMETRIZED_CASE_GROUPS:
exit_code = _run_dynamic_suite(args, target_dir)
else:
exit_code = _run_file_suite(args, target_dir)
sys.exit(exit_code)
"""Compatibility entry point; CI dispatches diffusion via ``test/run_suite.py``."""
from sglang.multimodal_gen.test.runner.diffusion_suite_runner import main
if __name__ == "__main__":
main()
python/sglang/multimodal_gen/test/runner/diffusion_suite_runner.py
+690
View File
@@ -0,0 +1,690 @@
"""Internal diffusion-suite adapter used by ``test/run_suite.py`` bridges.
For diffusion 1-gpu/2-gpu suites, cases are partitioned by estimated runtime
using LPT so each CI shard has a similar total runtime.
"""
import argparse
import copy
import json
import os
import random
import subprocess
import sys
import time
from dataclasses import dataclass
from pathlib import Path
import tabulate
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.test.partitioning import PartitionItem, assign_partition
from sglang.multimodal_gen.test.runner.pytest_runner import (
partition_items_by_index,
run_pytest,
)
from sglang.multimodal_gen.test.server.testcase_configs import (
BASELINE_CONFIG,
DiffusionTestCase,
)
# TODO: remove duplicated code
if current_platform.is_npu():
from sglang.multimodal_gen.test.server.ascend.testcase_configs_npu import (
_UPDATE_WEIGHTS_FROM_DISK_TEST_FILE,
COMPONENT_ACCURACY_SUITES,
DEFAULT_EST_TIME_SECONDS,
DEFAULT_STANDALONE_EST_TIME_SECONDS,
FILE_SUITES,
PARAMETRIZED_CASE_GROUPS,
STANDALONE_FILE_EST_TIMES,
STANDALONE_FILES,
STARTUP_OVERHEAD_SECONDS,
SUITES,
)
else:
from sglang.multimodal_gen.test.server.gpu_cases import ( # noqa: F401 It is used by ci scripts
_UPDATE_WEIGHTS_FROM_DISK_TEST_FILE,
_UPDATE_WEIGHTS_MODEL_PAIR_ENV,
_UPDATE_WEIGHTS_MODEL_PAIR_IDS,
COMPONENT_ACCURACY_FILE_NUM_GPUS,
COMPONENT_ACCURACY_SUITES,
DEFAULT_EST_TIME_SECONDS,
DEFAULT_STANDALONE_EST_TIME_SECONDS,
FILE_SUITES,
ONE_GPU_CASES,
PARAMETRIZED_CASE_GROUPS,
STANDALONE_FILE_EST_TIMES,
STANDALONE_FILES,
STARTUP_OVERHEAD_SECONDS,
STRICT_SUITES,
SUITES,
TWO_GPU_CASES,
)
logger = init_logger(__name__)
@dataclass(frozen=True)
class PartitionAssignment:
case_ids: list[str]
standalone_files: list[str]
estimated_time: float | None = None
missing_standalone_estimates: list[str] | None = None
def get_case_est_time(case_id: str) -> float:
scenario = BASELINE_CONFIG.scenarios.get(case_id)
if scenario is None:
return DEFAULT_EST_TIME_SECONDS
if scenario.estimated_full_test_time_s is not None:
return scenario.estimated_full_test_time_s
return scenario.expected_e2e_ms / 1000.0 + STARTUP_OVERHEAD_SECONDS
def get_standalone_file_est_time(
suite: str, standalone_file: str
) -> tuple[float, bool]:
suite_est_times = STANDALONE_FILE_EST_TIMES.get(suite, {})
if standalone_file not in suite_est_times:
return DEFAULT_STANDALONE_EST_TIME_SECONDS, True
return suite_est_times[standalone_file], False
def get_all_standalone_file_est_times() -> dict[str, dict[str, float]]:
return copy.deepcopy(STANDALONE_FILE_EST_TIMES)
def validate_standalone_file_est_times() -> dict[str, list[str]]:
missing_by_suite: dict[str, list[str]] = {}
for suite, standalone_files in STANDALONE_FILES.items():
suite_est_times = STANDALONE_FILE_EST_TIMES.get(suite, {})
missing = [
standalone_file
for standalone_file in standalone_files
if standalone_file not in suite_est_times
]
if missing:
missing_by_suite[suite] = missing
return missing_by_suite
def get_suite_files_rel(suite: str, parametrized_only: bool = False) -> list[str]:
if parametrized_only and suite in PARAMETRIZED_CASE_GROUPS:
return [filename for filename, _ in PARAMETRIZED_CASE_GROUPS[suite]]
return SUITES[suite]
def _normalize_standalone_key(standalone_file: str) -> str:
return f"standalone:{standalone_file}"
def parse_partition_plan(
suite: str,
partition_id: int,
total_partitions: int,
plan_json: str,
) -> PartitionAssignment:
plan = json.loads(plan_json)
if plan.get("suite") != suite:
raise ValueError(
f"Partition plan suite mismatch: expected {suite!r}, "
f"got {plan.get('suite')!r}"
)
partition_count = plan.get("partition_count")
if partition_count != total_partitions:
raise ValueError(
f"Partition count mismatch for suite {suite!r}: "
f"plan={partition_count}, matrix={total_partitions}"
)
partitions = plan.get("partitions", [])
selected_partition = None
for partition in partitions:
if partition.get("part") == partition_id:
selected_partition = partition
break
if selected_partition is None:
raise ValueError(
f"Partition {partition_id} not found in plan for suite {suite!r}"
)
return PartitionAssignment(
case_ids=list(selected_partition.get("case_ids", [])),
standalone_files=list(selected_partition.get("standalone_files", [])),
estimated_time=selected_partition.get("estimated_time"),
missing_standalone_estimates=list(
selected_partition.get("missing_standalone_estimates", [])
),
)
def build_local_partition_assignment(
suite: str,
partition_id: int,
total_partitions: int,
) -> PartitionAssignment:
"""Assign this shard's work when CI did not precompute a partition plan.
Lanes with a hardcoded ``--total-partitions`` (the AMD ones) cannot give
every standalone file a shard of its own, so standalone files are LPT
balanced together with the parametrized cases instead.
"""
items = [
PartitionItem(kind="case", item_id=case.id, est_time=get_case_est_time(case.id))
for case in _get_dynamic_suite_cases(suite)
]
for standalone_file in STANDALONE_FILES.get(suite, []):
items.append(
PartitionItem(
kind="standalone",
item_id=standalone_file,
est_time=get_standalone_file_est_time(suite, standalone_file)[0],
)
)
my_items = assign_partition(items, partition_id, total_partitions)
return PartitionAssignment(
case_ids=[item.item_id for item in my_items if item.kind == "case"],
standalone_files=[
item.item_id for item in my_items if item.kind == "standalone"
],
)
def _merge_execution_results(
executed_cases: list[str],
case_results: dict[str, str],
new_executed_cases: list[str],
new_case_results: dict[str, str],
) -> None:
executed_cases.extend(
case_id for case_id in new_executed_cases if case_id not in executed_cases
)
case_results.update(new_case_results)
def _format_standalone_estimate_snippet(
suite: str, standalone_file: str, measured_full_test_time_s: float
) -> str:
return (
f'"{suite}": {{\n "{standalone_file}": {measured_full_test_time_s:.1f},\n}}'
)
def _print_missing_standalone_estimate_message(
suite: str,
standalone_file: str,
measured_full_test_time_s: float,
) -> None:
snippet = _format_standalone_estimate_snippet(
suite, standalone_file, measured_full_test_time_s
)
logger.error(
f"\n{'=' * 60}\n"
f'Add standalone estimate for suite "{suite}" and file "{standalone_file}":\n\n'
"File: python/sglang/multimodal_gen/test/server/gpu_cases.py\n\n"
f"Current partition used fallback estimate: "
f"{DEFAULT_STANDALONE_EST_TIME_SECONDS:.1f}s\n\n"
f"{snippet}\n"
f"{'=' * 60}\n"
)
def _run_standalone_file(
suite: str,
standalone_rel: str,
target_dir: Path,
extra_filter: str | None = None,
) -> tuple[int, list[str], dict[str, str], dict]:
if standalone_rel == _UPDATE_WEIGHTS_FROM_DISK_TEST_FILE:
_maybe_pin_update_weights_model_pair([standalone_rel])
est_time, used_fallback_estimate = get_standalone_file_est_time(
suite, standalone_rel
)
standalone_file = _resolve_suite_files(target_dir, [standalone_rel], strict=True)[0]
junit_xml_path = str(
target_dir / f"junit_results_{suite}_{Path(standalone_rel).stem}.xml"
)
start_time = time.perf_counter()
exit_code, _, _ = run_pytest(
[standalone_file],
filter_expr=extra_filter,
junit_xml_path=junit_xml_path,
)
measured_full_test_time_s = round(time.perf_counter() - start_time, 1)
standalone_key = _normalize_standalone_key(standalone_rel)
measurement = {
"suite": suite,
"standalone_file": standalone_rel,
"measured_full_test_time_s": measured_full_test_time_s,
"used_fallback_estimate": used_fallback_estimate,
"fallback_estimate_s": DEFAULT_STANDALONE_EST_TIME_SECONDS,
"had_configured_estimate": not used_fallback_estimate,
"configured_or_fallback_estimate_s": est_time,
}
if used_fallback_estimate:
_print_missing_standalone_estimate_message(
suite, standalone_rel, measured_full_test_time_s
)
return (
exit_code,
[standalone_key],
{standalone_key: "pass" if exit_code == 0 else "fail"},
measurement,
)
def parse_args():
suite_choices = sorted(set(FILE_SUITES) | set(PARAMETRIZED_CASE_GROUPS))
parser = argparse.ArgumentParser(description="Run multimodal_gen test suite")
parser.add_argument(
"--suite",
type=str,
required=True,
choices=suite_choices,
help="The test suite to run.",
)
parser.add_argument(
"--partition-id",
type=int,
default=0,
help="Index of the current partition (for parallel execution)",
)
parser.add_argument(
"--total-partitions",
type=int,
default=1,
help="Total number of partitions",
)
parser.add_argument(
"--base-dir",
type=str,
default="server",
help="Base directory for tests relative to this script's parent",
)
parser.add_argument(
"-k",
"--filter",
type=str,
default=None,
help="Pytest filter expression (passed to pytest -k)",
)
parser.add_argument(
"--continue-on-error",
action="store_true",
default=False,
help="Continue running remaining tests even if one fails.",
)
parser.add_argument(
"--partition-plan-json",
type=str,
default=None,
help="Full partition plan JSON for the current suite.",
)
return parser.parse_args()
def write_execution_report(
suite: str,
partition_id: int,
total_partitions: int,
executed_cases: list[str],
is_standalone: bool = False,
standalone_file: str | None = None,
case_results: dict[str, str] | None = None,
missing_standalone_estimates: list[str] | None = None,
standalone_measurements: list[dict] | None = None,
) -> str:
report = {
"suite": suite,
"partition_id": partition_id,
"total_partitions": total_partitions,
"is_standalone": is_standalone,
"standalone_file": standalone_file,
"executed_cases": executed_cases,
"case_results": case_results or {},
"missing_standalone_estimates": missing_standalone_estimates or [],
"standalone_measurements": standalone_measurements or [],
}
report_filename = f"execution_report_{suite}_{partition_id}.json"
report_path = Path(__file__).resolve().parents[1] / report_filename
with open(report_path, "w", encoding="utf-8") as f:
json.dump(report, f, indent=2)
logger.info("Execution report written to: %s", report_path)
return str(report_path)
def run_component_accuracy_files(files, filter_expr=None, continue_on_error=False):
exit_code = 0
for file_path in files:
file_name = Path(file_path).name
num_gpus = COMPONENT_ACCURACY_FILE_NUM_GPUS.get(file_name, 1)
if num_gpus > 1:
cmd = [
sys.executable,
"-m",
"torch.distributed.run",
f"--nproc_per_node={num_gpus}",
"-m",
"pytest",
"-s",
"-v",
]
else:
cmd = [sys.executable, "-m", "pytest", "-s", "-v"]
if filter_expr:
cmd.extend(["-k", filter_expr])
cmd.append(file_path)
print(f"Running command: {' '.join(cmd)}")
file_exit_code = subprocess.call(cmd)
if file_exit_code == 5:
print(
"No tests collected (exit code 5). This is expected when filters "
"deselect all tests in a file. Treating as success."
)
file_exit_code = 0
if file_exit_code != 0 and exit_code == 0:
exit_code = file_exit_code
if file_exit_code != 0 and not continue_on_error:
return file_exit_code
return exit_code
def _is_in_ci() -> bool:
return os.environ.get("SGLANG_IS_IN_CI", "").lower() in ("1", "true", "yes", "on")
def _maybe_pin_update_weights_model_pair(suite_files_rel: list[str]) -> None:
if not _is_in_ci():
return
if _UPDATE_WEIGHTS_FROM_DISK_TEST_FILE not in suite_files_rel:
return
if os.environ.get(_UPDATE_WEIGHTS_MODEL_PAIR_ENV):
print(
f"Using preset {_UPDATE_WEIGHTS_MODEL_PAIR_ENV}="
f"{os.environ[_UPDATE_WEIGHTS_MODEL_PAIR_ENV]}"
)
return
selected_pair = random.choice(_UPDATE_WEIGHTS_MODEL_PAIR_IDS)
os.environ[_UPDATE_WEIGHTS_MODEL_PAIR_ENV] = selected_pair
print(f"Selected {_UPDATE_WEIGHTS_MODEL_PAIR_ENV}={selected_pair} for this CI run")
def _resolve_suite_files(
target_dir: Path, suite_files_rel: list[str], strict: bool
) -> list[str]:
suite_files_abs = []
for f_rel in suite_files_rel:
f_abs = target_dir / f_rel
if not f_abs.exists():
msg = f"Test file {f_rel} not found in {target_dir}."
if strict:
print(f"Error: {msg}")
sys.exit(1)
print(f"Warning: {msg} Skipping.")
continue
suite_files_abs.append(str(f_abs))
return suite_files_abs
def _run_file_suite(args, target_dir: Path) -> int:
suite_files_rel = FILE_SUITES[args.suite]
_maybe_pin_update_weights_model_pair(suite_files_rel)
suite_files_abs = _resolve_suite_files(
target_dir, suite_files_rel, args.suite in STRICT_SUITES
)
if not suite_files_abs:
print(f"No valid test files found for suite '{args.suite}'.")
return 1 if args.suite in STRICT_SUITES else 0
exit_code, _, _ = run_pytest(
suite_files_abs,
filter_expr=args.filter,
junit_xml_path=None,
)
return exit_code
def _get_dynamic_suite_cases(suite: str) -> list[DiffusionTestCase]:
cases = []
for _, case_group in PARAMETRIZED_CASE_GROUPS[suite]:
cases.extend(case_group)
return cases
def _get_parametrized_files_for_case_ids(
suite: str, case_ids: set[str], target_dir: Path
) -> list[str]:
files = []
for filename, case_group in PARAMETRIZED_CASE_GROUPS[suite]:
if any(case.id in case_ids for case in case_group):
file_path = target_dir / filename
if file_path.exists():
files.append(str(file_path))
else:
logger.warning("Test file %s not found in %s", filename, target_dir)
return files
def _run_dynamic_suite(args, target_dir: Path) -> int:
if args.partition_plan_json:
assignment = parse_partition_plan(
suite=args.suite,
partition_id=args.partition_id,
total_partitions=args.total_partitions,
plan_json=args.partition_plan_json,
)
else:
assignment = build_local_partition_assignment(
suite=args.suite,
partition_id=args.partition_id,
total_partitions=args.total_partitions,
)
return _run_partition_assignment(args, target_dir, assignment)
def _run_partition_assignment(
args, target_dir: Path, assignment: PartitionAssignment
) -> int:
rows = [[args.suite, f"{args.partition_id + 1}/{args.total_partitions}"]]
print(tabulate.tabulate(rows, headers=["Suite", "Partition"], tablefmt="psql"))
total_est_time = 0.0
executed_cases: list[str] = []
case_results: dict[str, str] = {}
missing_standalone_estimates: list[str] = []
standalone_measurements: list[dict] = []
overall_exit_code = 0
if assignment.case_ids:
case_id_set = set(assignment.case_ids)
total_est_time += sum(
get_case_est_time(case_id) for case_id in assignment.case_ids
)
suite_files = _get_parametrized_files_for_case_ids(
args.suite, case_id_set, target_dir
)
if not suite_files:
print(f"No valid parametrized test files found for suite '{args.suite}'.")
return 0
partition_filter = " or ".join(
f"[{case_id}]" for case_id in assignment.case_ids
)
filter_expr = (
f"({partition_filter}) and ({args.filter})"
if args.filter
else partition_filter
)
print(
f"Running {len(assignment.case_ids)} parametrized cases with estimated total "
f"{sum(get_case_est_time(case_id) for case_id in assignment.case_ids):.1f}s:"
)
for case_id in assignment.case_ids:
print(f" - case: {case_id} ({get_case_est_time(case_id):.1f}s)")
print(f"Test files: {[Path(f).name for f in suite_files]}")
print(f"Filter expression: {filter_expr}")
junit_xml_path = str(
target_dir / f"junit_results_{args.suite}_{args.partition_id}.xml"
)
exit_code, new_executed_cases, new_case_results = run_pytest(
suite_files,
filter_expr=filter_expr,
junit_xml_path=junit_xml_path,
)
_merge_execution_results(
executed_cases, case_results, new_executed_cases, new_case_results
)
# A failing case must not swallow this shard's standalone files: they
# are separate pytest runs, and they only share a shard because the
# shard count is fixed. --continue-on-error still decides whether a
# failing standalone file stops the ones queued behind it.
if exit_code != 0 and overall_exit_code == 0:
overall_exit_code = exit_code
if assignment.standalone_files:
standalone_estimate = sum(
get_standalone_file_est_time(args.suite, standalone_file)[0]
for standalone_file in assignment.standalone_files
)
total_est_time += standalone_estimate
print(
f"Running {len(assignment.standalone_files)} standalone file(s) with estimated total "
f"{standalone_estimate:.1f}s:"
)
for standalone_file in assignment.standalone_files:
est_time, used_fallback_estimate = get_standalone_file_est_time(
args.suite, standalone_file
)
fallback_suffix = (
f", fallback estimate {DEFAULT_STANDALONE_EST_TIME_SECONDS:.1f}s"
if used_fallback_estimate
else ""
)
print(
f" - standalone: {standalone_file} ({est_time:.1f}s{fallback_suffix})"
)
for standalone_file in assignment.standalone_files:
exit_code, new_executed_cases, new_case_results, measurement = (
_run_standalone_file(
args.suite,
standalone_file,
target_dir,
extra_filter=args.filter,
)
)
if measurement["used_fallback_estimate"]:
missing_standalone_estimates.append(standalone_file)
standalone_measurements.append(measurement)
_merge_execution_results(
executed_cases,
case_results,
new_executed_cases,
new_case_results,
)
if exit_code != 0 and overall_exit_code == 0:
overall_exit_code = exit_code
if exit_code != 0 and not args.continue_on_error:
break
if not assignment.case_ids and not assignment.standalone_files:
print(f"No work assigned to partition {args.partition_id}. Exiting success.")
print(f"Partition estimated total time: {total_est_time:.1f}s")
write_execution_report(
suite=args.suite,
partition_id=args.partition_id,
total_partitions=args.total_partitions,
executed_cases=executed_cases,
is_standalone=False,
standalone_file=None,
case_results=case_results,
missing_standalone_estimates=missing_standalone_estimates,
standalone_measurements=standalone_measurements,
)
return overall_exit_code
def main():
args = parse_args()
validate_standalone_file_est_times()
test_root_dir = Path(__file__).resolve().parents[1]
target_dir = test_root_dir / args.base_dir
if not target_dir.exists():
print(f"Error: Target directory {target_dir} does not exist.")
sys.exit(1)
if args.suite in COMPONENT_ACCURACY_SUITES:
suite_files_rel = FILE_SUITES[args.suite]
suite_files_abs = _resolve_suite_files(
target_dir, suite_files_rel, args.suite in STRICT_SUITES
)
if not suite_files_abs:
print(f"No valid test files found for suite '{args.suite}'.")
sys.exit(1 if args.suite in STRICT_SUITES else 0)
my_files = partition_items_by_index(
suite_files_abs, args.partition_id, args.total_partitions
)
partition_info = (
f"{args.partition_id + 1}/{args.total_partitions} "
f"(0-based id={args.partition_id})"
)
headers = ["Suite", "Partition"]
rows = [[args.suite, partition_info]]
msg = tabulate.tabulate(rows, headers=headers, tablefmt="psql") + "\n"
msg += f"Enabled {len(my_files)} file(s):\n"
for file_path in my_files:
msg += f" - {file_path}\n"
print(msg, flush=True)
print(
f"Suite: {args.suite} | Partition: {args.partition_id}/{args.total_partitions}"
)
print(f"Selected {len(suite_files_abs)} files:")
for f in suite_files_abs:
print(f" - {os.path.basename(f)}")
if not my_files:
print("No files assigned to this partition. Exiting success.")
sys.exit(0)
print(f"Running {len(my_files)} files in this shard: {', '.join(my_files)}")
exit_code = run_component_accuracy_files(
my_files,
filter_expr=args.filter,
continue_on_error=args.continue_on_error,
)
msg = "\n" + tabulate.tabulate(rows, headers=headers, tablefmt="psql") + "\n"
msg += f"Executed {len(my_files)} file(s):\n"
for file_path in my_files:
msg += f" - {file_path}\n"
print(msg, flush=True)
elif args.suite in PARAMETRIZED_CASE_GROUPS:
exit_code = _run_dynamic_suite(args, target_dir)
else:
exit_code = _run_file_suite(args, target_dir)
sys.exit(exit_code)
if __name__ == "__main__":
main()
python/sglang/multimodal_gen/test/scripts/gen_diffusion_ci_outputs.py
+2 -2
View File
@@ -2,7 +2,7 @@
"""
Generate diffusion CI outputs for consistency testing.
This script reuses the CI test code by calling run_suite.py with SGLANG_GEN_GT=1,
This script reuses the CI test adapter with SGLANG_GEN_GT=1,
ensuring that GT generation uses exactly the same code path as CI tests.
Usage:
@@ -21,7 +21,7 @@ from sglang.multimodal_gen.test.partitioning import (
PartitionItem,
partition_items_by_lpt,
)
from sglang.multimodal_gen.test.run_suite import (
from sglang.multimodal_gen.test.runner.diffusion_suite_runner import (
SUITES,
_maybe_pin_update_weights_model_pair,
get_case_est_time,
python/sglang/multimodal_gen/test/unit/realtime/test_realtime_webui.py
-112
View File
@@ -1,112 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
from pathlib import Path
def test_realtime_webui_presets_do_not_emit_camera_scripts():
repo_root = Path(__file__).resolve().parents[6]
app_js = (
repo_root / "python/sglang/multimodal_gen/apps/realtime_webui/app.js"
).read_text()
index_html = (
repo_root / "python/sglang/multimodal_gen/apps/realtime_webui/index.html"
).read_text()
styles_css = (
repo_root / "python/sglang/multimodal_gen/apps/realtime_webui/styles.css"
).read_text()
playback_js = (
repo_root
/ "python/sglang/multimodal_gen/apps/realtime_webui/playback_controller.js"
).read_text()
assert "preset.actions" not in app_js
assert "repeatActions" not in app_js
assert 'id="eventFrames"' not in index_html
assert "ControlStateController" in app_js
assert 'const DEFAULT_PREVIEW_OUTPUT_FORMAT = "webp";' in app_js
assert 'id="transportFormat"' in index_html
assert 'id="fps" type="number" value="25"' in index_html
assert 'id="superResolution" type="checkbox"' in index_html
assert 'id="upscalingScale"' in index_html
assert 'class="workspace"' in index_html
assert 'class="preview-frame"' in index_html
assert 'id="previewOverlay" class="preview-overlay"' in index_html
assert 'id="previewScale" type="range" min="80" max="170" value="120"' in index_html
assert 'id="previewScaleText"' in index_html
assert 'id="outputSizeText"' in index_html
assert 'id="frameInterpolation" type="checkbox" />' in index_html
assert (
'id="serverUrl" value="ws://127.0.0.1:30000/v1/realtime_video/generate"'
in index_html
)
assert '<option value="webp" selected>WebP preview</option>' in index_html
assert 'id="serverSendText"' in index_html
assert 'id="theoreticalFpsText"' in index_html
assert 'id="renderFps"' in index_html
assert 'id="stageRenderFps"' not in index_html
assert "sglang-diffusion Realtime Studio" in index_html
assert "SGLD" not in index_html
assert 'class="tabs"' not in index_html
assert "Recordings" not in index_html
assert "API" not in index_html
assert "Info" not in index_html
assert 'id="steps" type="number" value="4"' in index_html
assert 'id="guidance" type="number" value="1"' in index_html
assert "styles.css?v=realtime-record-v49" in index_html
assert "app.js?v=realtime-record-v75" in index_html
assert (
'const DECODER_WORKER_URL = "./decoder_worker.js?v=rgb-worker-v10";' in app_js
)
assert "const DEFAULT_TARGET_FPS = 25;" in app_js
assert "const DEFAULT_FRAME_INTERPOLATION_EXP = 1;" in app_js
assert "const DEFAULT_FRAME_INTERPOLATION_SCALE = 1.0;" in app_js
assert "const DEFAULT_UPSCALING_SCALE = 2;" in app_js
assert "const DEFAULT_PREVIEW_SCALE = 120;" in app_js
assert 'setPreviewState("waiting")' in app_js
assert "stage.dataset.previewState = state" in app_js
assert "previewProgressSpin" in styles_css
assert "previewDotPulse" not in styles_css
assert 'document.querySelector(".preview-frame")' in app_js
assert 'previewFrame.style.setProperty("--preview-scale"' in app_js
assert "cancelAnimationFrame(previewScaleFrame)" in app_js
assert "enable_frame_interpolation: true" in app_js
assert "frame_interpolation_exp: DEFAULT_FRAME_INTERPOLATION_EXP" in app_js
assert "frame_interpolation_scale: DEFAULT_FRAME_INTERPOLATION_SCALE" in app_js
assert "readSuperResolutionParams()" in app_js
assert "enable_upscaling: true" in app_js
assert "upscaling_scale: readUpscalingScale()" in app_js
assert "updateOutputSizeFromHeader(header)" in app_js
assert "setPreviewScale(DEFAULT_PREVIEW_SCALE)" in app_js
assert "preview_scale" in app_js
assert "sr_scale" in app_js
assert "elapsedMs < targetMs" in playback_js
assert "queuedDecodeFrames > maxQueuedFrames" in app_js
assert (
'const REACTOR_PRESET_BASE_URL = "https://www.reactor.inc/lingbot-world-fast-v1";'
in app_js
)
assert "Dragon Dolly" in app_js
assert "no creature morphing" in app_js
assert "the Plastic Beach island stays centered" in app_js
assert "no camera descent, no push-in, no orbit" in app_js
assert "Ziggy Stardust" in app_js
assert "blue K. West sign" in app_js
assert "wet pavement reflecting a yellow streetlamp" in app_js
assert "ZiggyStardust.jpg" in app_js
assert "A slow aerial orbit around a pastel floating island hotel" not in app_js
assert app_js.index("Dragon Ride") < app_js.index("Dragon Dolly")
assert app_js.index("Ziggy Stardust") < app_js.index("Plastic Beach")
assert app_js.index("Dragon Dolly") < app_js.index("Kid A")
assert "dragon-ride.jpg" in app_js
assert "stageRenderFps" not in app_js
assert 'setStatus("Receiving", "live")' in app_js
assert "decodeQueue.push(" in app_js
assert "receiveChain" not in app_js
assert 'message.type === "chunk_stats"' in app_js
assert "chunkTotal > 0 ? numFrames / chunkTotal" in app_js
assert ".stage-stat" in styles_css
assert ".workspace" in styles_css
assert ".preview-frame" in styles_css
assert ".preview-overlay" in styles_css
assert ".preview-scale-control" in styles_css
assert "--preview-scale" in styles_css
python/sglang/multimodal_gen/test/unit/test_consistency_metrics.py
-12
View File
@@ -47,18 +47,6 @@ def test_consistency_gt_urls_are_pinned_to_ci_data_revision():
assert pinned_revision_path in test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE
def test_remote_file_exists_returns_false_for_definitive_404(monkeypatch):
class Response:
status_code = 404
def close(self):
pass
monkeypatch.setattr(test_utils.requests, "head", lambda *args, **kwargs: Response())
assert test_utils._remote_file_exists("https://example.com/missing.png") is False
def test_remote_video_gt_candidates_survive_inconclusive_probe(monkeypatch):
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "h100")
monkeypatch.setattr(test_utils, "_remote_file_exists", lambda url: None)
python/sglang/multimodal_gen/test/unit/test_cosmos3.py
-12
View File
@@ -1907,18 +1907,6 @@ class TestCosmos3ModalitySamplingParams(unittest.TestCase):
self.assertEqual(sp.condition_frame_indexes, [0, 1])
self.assertEqual(sp.condition_video_keep, "first")
def test_action_fields_default_none(self):
sp = Cosmos3SamplingParams(prompt="t")
for field in (
"action_mode",
"domain_id",
"domain_name",
"raw_action_dim",
"action_fps",
"action",
):
self.assertIsNone(getattr(sp, field))
class TestCosmos3CaptionMetadata(unittest.TestCase):
"""Structured captions get generation metadata; prose prompts opt out."""
python/sglang/multimodal_gen/test/unit/test_diffusion_benchmark_skill.py
-658
View File
@@ -1,658 +0,0 @@
import importlib.util
import json
import sys
import tempfile
import types
import unittest
from pathlib import Path
from unittest.mock import patch
def _load_benchmark_module(temp_root: Path):
multimodal_gen_root = Path(__file__).resolve().parents[2]
script_path = (
multimodal_gen_root
/ ".claude"
/ "skills"
/ "sglang-diffusion-benchmark-profile"
/ "scripts"
/ "bench_diffusion_denoise.py"
)
fake_env = types.ModuleType("diffusion_skill_env")
fake_env.ensure_dir = lambda path: (
Path(path).mkdir(parents=True, exist_ok=True) or Path(path)
)
fake_env.get_assets_dir = lambda _root: temp_root / "assets"
fake_env.get_output_dir = lambda _kind, _root: temp_root / "outputs"
fake_env.get_repo_root = lambda: temp_root / "repo"
fake_env.pick_idle_gpus = lambda count: list(range(count))
spec = importlib.util.spec_from_file_location(
"test_bench_diffusion_denoise", script_path
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
with patch.dict(sys.modules, {"diffusion_skill_env": fake_env}):
spec.loader.exec_module(module)
return module
def _load_skill_env_module():
script_path = (
Path(__file__).resolve().parents[2]
/ ".claude"
/ "skills"
/ "sglang-diffusion-benchmark-profile"
/ "scripts"
/ "diffusion_skill_env.py"
)
spec = importlib.util.spec_from_file_location(
"test_diffusion_skill_env", script_path
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
class TestDiffusionBenchmarkSkill(unittest.TestCase):
def test_skill_env_prefers_own_worktree_over_installed_package(self):
module = _load_skill_env_module()
installed = types.ModuleType("sglang")
installed.__file__ = "/sgl-workspace/sglang/python/sglang/__init__.py"
with patch.dict(sys.modules, {"sglang": installed}):
repo_root = module.get_repo_root()
self.assertEqual(repo_root, Path(__file__).resolve().parents[5])
def test_nightly_presets_remain_aligned(self):
with tempfile.TemporaryDirectory() as tmpdir:
module = _load_benchmark_module(Path(tmpdir))
repo_root = Path(__file__).resolve().parents[5]
module.NIGHTLY_CONFIG_PATH = (
repo_root
/ "scripts"
/ "ci"
/ "utils"
/ "diffusion"
/ "comparison_configs.json"
)
self.assertEqual(module.validate_nightly_alignment(), 0)
def test_recent_model_presets_are_eager_by_default(self):
with tempfile.TemporaryDirectory() as tmpdir:
module = _load_benchmark_module(Path(tmpdir))
expected = {
"longcat-image",
"longcat-image-edit",
"longcat-image-edit-turbo",
"qwen-edit-base",
"qwen-image-layered",
"stable-diffusion-3.5-medium",
"sana-video",
"sana-wm-bidirectional",
"sana-wm-streaming",
"lingbot-video-moe",
"lingbot-world",
"lingbot-world-v2",
"fastwan21-t2v-1.3b",
"fasth3-t2va-vsa",
"wan22-t2v-nvfp4",
"krea2-turbo",
"krea2-raw",
"ideogram4-fast",
"ideogram4-instant",
"longlive2-t2v",
"longlive2-i2v",
"fast-hunyuan",
"turbowan21-t2v-1.3b",
"helios-mid",
"helios-distilled",
"joy-echo",
"cosmos3-edge-t2i",
"cosmos3-super-t2v-cfg2tp2",
"cosmos3-super-i2v",
"cosmos3-super-t2i-distilled",
"ltx25",
"ltx25-diffusion-decoder",
}
self.assertTrue(expected.issubset(module.MODELS))
eager_cmd = module.build_sglang_cmd("longcat-image")
self.assertNotIn("--enable-torch-compile", eager_cmd)
self.assertIn("--enable-prompt-rewrite=false", eager_cmd)
self.assertIn("--quality=lossless", eager_cmd)
compiled_cmd = module.build_sglang_cmd("longcat-image", torch_compile=True)
self.assertIn("--enable-torch-compile", compiled_cmd)
longcat_edit_cmd = module.build_sglang_cmd("longcat-image-edit")
self.assertIn(
"--model-path=meituan-longcat/LongCat-Image-Edit",
longcat_edit_cmd,
)
self.assertTrue(
any(arg.startswith("--image-path=") for arg in longcat_edit_cmd)
)
self.assertIn("--enable-prompt-rewrite=false", longcat_edit_cmd)
longcat_edit_bcg_cmd = module.build_sglang_cmd(
"longcat-image-edit", breakable_cuda_graph=True
)
resolution_index = longcat_edit_bcg_cmd.index("--warmup-resolutions")
self.assertEqual(longcat_edit_bcg_cmd[resolution_index + 1], "1264x848")
longcat_edit_turbo_cmd = module.build_sglang_cmd("longcat-image-edit-turbo")
self.assertIn(
"--model-path=meituan-longcat/LongCat-Image-Edit-Turbo",
longcat_edit_turbo_cmd,
)
layered_cmd = module.build_sglang_cmd("qwen-image-layered")
self.assertIn("--model-path=Qwen/Qwen-Image-Layered", layered_cmd)
self.assertIn("--num-frames=4", layered_cmd)
sd35_cmd = module.build_sglang_cmd("stable-diffusion-3.5-medium")
self.assertIn(
"--model-path=stabilityai/stable-diffusion-3.5-medium-diffusers",
sd35_cmd,
)
self.assertIn("stable-diffusion-3.5-medium", module.GATED_MODELS)
h3_cmd = module.build_sglang_cmd("minimax-h3-t2va", torch_compile=True)
self.assertNotIn("--enable-torch-compile", h3_cmd)
fastwan_cmd = module.build_sglang_cmd("fastwan21-t2v-1.3b")
self.assertIn("--num-frames=61", fastwan_cmd)
self.assertIn("--num-inference-steps=3", fastwan_cmd)
self.assertIn("--dit-layerwise-offload=false", fastwan_cmd)
wan_nvfp4_cmd = module.build_sglang_cmd("wan22-t2v-nvfp4")
self.assertIn(
"--model-path=nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4",
wan_nvfp4_cmd,
)
self.assertIn("--num-frames=81", wan_nvfp4_cmd)
self.assertIn("--dit-layerwise-offload=false", wan_nvfp4_cmd)
self.assertEqual(module.required_gpus_for_model("wan22-t2v-nvfp4"), 1)
krea_raw_cmd = module.build_sglang_cmd("krea2-raw")
self.assertIn("--num-inference-steps=50", krea_raw_cmd)
self.assertIn("--guidance-scale=4.5", krea_raw_cmd)
cosmos_i2v_cmd = module.build_sglang_cmd("cosmos3-super-i2v")
self.assertIn(
"--model-path=nvidia/Cosmos3-Super-Image2Video", cosmos_i2v_cmd
)
self.assertIn("--num-gpus=2", cosmos_i2v_cmd)
self.assertIn("--tp-size=2", cosmos_i2v_cmd)
self.assertIn("--num-frames=81", cosmos_i2v_cmd)
cosmos_cfg_cmd = module.build_sglang_cmd("cosmos3-super-t2v-cfg2tp2")
self.assertIn("--model-path=nvidia/Cosmos3-Super", cosmos_cfg_cmd)
self.assertIn("--num-gpus=4", cosmos_cfg_cmd)
self.assertIn("--tp-size=2", cosmos_cfg_cmd)
sana_wm_dense_cmd = module.build_sglang_cmd("sana-wm-bidirectional")
self.assertIn(
"--model-path=Efficient-Large-Model/SANA-WM_bidirectional",
sana_wm_dense_cmd,
)
self.assertIn("--num-inference-steps=20", sana_wm_dense_cmd)
self.assertNotIn("--streaming", sana_wm_dense_cmd)
sana_wm_streaming_cmd = module.build_sglang_cmd("sana-wm-streaming")
self.assertIn("--streaming", sana_wm_streaming_cmd)
self.assertIn("--refiner-chunked", sana_wm_streaming_cmd)
self.assertIn("--action=w-16,wl-16,l-16", sana_wm_streaming_cmd)
lingbot_world_cmd = module.build_sglang_cmd("lingbot-world")
self.assertIn(
"--model-path=robbyant/lingbot-world-fast-diffusers",
lingbot_world_cmd,
)
self.assertIn("--num-frames=9", lingbot_world_cmd)
self.assertIn("--warmup-mode=off", lingbot_world_cmd)
self.assertIn("--config=", " ".join(lingbot_world_cmd))
self.assertEqual(
module.MODELS["lingbot-world"]["config_overrides"]["actions"],
[["w"] for _ in range(9)],
)
lingbot_world_v2_cmd = module.build_sglang_cmd("lingbot-world-v2")
self.assertIn(
"--model-path=robbyant/lingbot-world-v2-14b-causal-fast-diffusers",
lingbot_world_v2_cmd,
)
self.assertIn("--num-frames=9", lingbot_world_v2_cmd)
self.assertIn("--num-inference-steps=4", lingbot_world_v2_cmd)
ideogram_cmd = module.build_sglang_cmd("ideogram4-instant")
self.assertFalse(
any(arg.startswith("--num-inference-steps") for arg in ideogram_cmd)
)
longlive_i2v_cmd = module.build_sglang_cmd("longlive2-i2v")
self.assertIn("--num-frames=61", longlive_i2v_cmd)
self.assertTrue(
any(arg.startswith("--image-path=") for arg in longlive_i2v_cmd)
)
joy_echo_cmd = module.build_sglang_cmd("joy-echo")
self.assertIn("--num-gpus=2", joy_echo_cmd)
self.assertIn("--ulysses-degree=2", joy_echo_cmd)
config_arg = next(
arg for arg in joy_echo_cmd if arg.startswith("--config=")
)
config = json.loads(Path(config_arg.removeprefix("--config=")).read_text())
self.assertFalse(config["enable_memory_bank"])
def test_quality_and_bcg_comparators_are_explicit_and_exclusive(self):
with tempfile.TemporaryDirectory() as tmpdir:
module = _load_benchmark_module(Path(tmpdir))
high_cmd = module.build_sglang_cmd("longcat-image", quality="high")
self.assertIn("--quality=high", high_cmd)
self.assertNotIn("--enable-breakable-cuda-graph", high_cmd)
extra_high_cmd = module.build_sglang_cmd(
"longcat-image", quality="extra-high"
)
self.assertIn("--quality=extra-high", extra_high_cmd)
self.assertNotIn("--enable-breakable-cuda-graph", extra_high_cmd)
bcg_cmd = module.build_sglang_cmd(
"longcat-image",
breakable_cuda_graph=True,
bcg_text_buckets=[256, 512],
)
self.assertIn("--enable-breakable-cuda-graph", bcg_cmd)
self.assertEqual(
bcg_cmd[bcg_cmd.index("--warmup-resolutions") + 1], "1024x1024"
)
bucket_index = bcg_cmd.index("--bcg-text-buckets")
self.assertEqual(
bcg_cmd[bucket_index + 1 : bucket_index + 3], ["256", "512"]
)
sana_video_bcg_cmd = module.build_sglang_cmd(
"sana-video", breakable_cuda_graph=True
)
self.assertEqual(
sana_video_bcg_cmd[sana_video_bcg_cmd.index("--warmup-num-frames") + 1],
"17",
)
for _, quality, breakable_cuda_graph in module.QUALITY_BCG_ABBA_MATRIX:
module.build_sglang_cmd(
"longcat-image",
quality=quality,
breakable_cuda_graph=breakable_cuda_graph,
bcg_text_buckets=[256, 512] if breakable_cuda_graph else None,
)
with self.assertRaisesRegex(ValueError, "comparators"):
module.build_sglang_cmd(
"longcat-image",
torch_compile=True,
breakable_cuda_graph=True,
)
with self.assertRaisesRegex(ValueError, "requires"):
module.build_sglang_cmd("longcat-image", bcg_text_buckets=[256])
def test_isolated_cache_cleanup_writes_zero_residual_ledger(self):
with tempfile.TemporaryDirectory() as tmpdir:
temp_root = Path(tmpdir)
module = _load_benchmark_module(temp_root)
cache_root = temp_root / "model-caches"
cache_dir = module._prepare_model_cache(
cache_root, "longcat-image", "baseline"
)
weight_path = cache_dir / "huggingface" / "hub" / "model.safetensors"
weight_path.parent.mkdir(parents=True)
weight_path.write_bytes(b"weights")
env = module._model_cache_env(cache_dir)
self.assertTrue(env["HF_HOME"].startswith(str(cache_dir)))
self.assertTrue(env["HF_XET_CACHE"].startswith(str(cache_dir)))
self.assertTrue(env["TRANSFORMERS_CACHE"].startswith(str(cache_dir)))
self.assertTrue(env["MODELSCOPE_CACHE"].startswith(str(cache_dir)))
ledger_path = temp_root / "artifacts" / "cleanup.jsonl"
record = module._cleanup_model_cache(
cache_root,
cache_dir,
ledger_path,
"longcat-image",
"baseline",
"success",
)
self.assertFalse(cache_dir.exists())
self.assertEqual(record["before"]["weight_file_count"], 1)
self.assertEqual(record["after"]["file_count"], 0)
ledger = json.loads(ledger_path.read_text(encoding="utf-8"))
self.assertEqual(ledger["exit_reason"], "success")
self.assertEqual(ledger["after"]["weight_file_count"], 0)
def test_isolated_cache_refuses_to_reuse_existing_run_directory(self):
with tempfile.TemporaryDirectory() as tmpdir:
temp_root = Path(tmpdir)
module = _load_benchmark_module(temp_root)
cache_root = temp_root / "model-caches"
module._prepare_model_cache(cache_root, "sana-video", "baseline")
with self.assertRaises(FileExistsError):
module._prepare_model_cache(cache_root, "sana-video", "baseline")
def test_isolated_cache_seeds_read_only_hf_cache_with_writable_overlay(self):
with tempfile.TemporaryDirectory() as tmpdir:
temp_root = Path(tmpdir)
module = _load_benchmark_module(temp_root)
seed_root = temp_root / "shared-hf"
source_model = seed_root / "hub" / "models--org--model"
source_weight = source_model / "snapshots" / "abc" / "model.safetensors"
source_weight.parent.mkdir(parents=True)
source_weight.write_bytes(b"shared weights")
source_ref = source_model / "refs" / "main"
source_ref.parent.mkdir()
source_ref.write_text("abc")
cache_root = temp_root / "model-caches"
cache_dir = module._prepare_model_cache(
cache_root,
"sana-video",
"baseline",
seed_model_cache_roots=[seed_root],
)
seeded_model = cache_dir / "huggingface" / "hub" / "models--org--model"
self.assertTrue(seeded_model.is_dir())
self.assertFalse(seeded_model.is_symlink())
seeded_weight = seeded_model / "snapshots" / "abc" / "model.safetensors"
self.assertTrue(seeded_weight.is_symlink())
self.assertEqual(
seeded_weight.read_bytes(),
b"shared weights",
)
new_blob = seeded_model / "blobs" / "downloaded"
new_blob.parent.mkdir(exist_ok=True)
new_blob.write_bytes(b"new download")
(seeded_model / "refs" / "main").write_text("new-revision")
self.assertEqual(new_blob.read_bytes(), b"new download")
self.assertEqual(source_ref.read_text(), "abc")
module._cleanup_model_cache(
cache_root,
cache_dir,
temp_root / "cleanup.jsonl",
"sana-video",
"baseline",
"success",
)
self.assertFalse(cache_dir.exists())
self.assertEqual(source_weight.read_bytes(), b"shared weights")
def test_interrupted_run_cleans_isolated_cache_in_finally(self):
with tempfile.TemporaryDirectory() as tmpdir:
temp_root = Path(tmpdir)
module = _load_benchmark_module(temp_root)
cache_root = temp_root / "model-caches"
output_dir = temp_root / "outputs"
output_dir.mkdir()
with (
patch.object(
module, "_run_benchmark_once_impl", side_effect=KeyboardInterrupt
),
self.assertRaises(KeyboardInterrupt),
):
module.run_benchmark_once(
"sana-video",
"baseline",
output_dir,
model_cache_root=cache_root,
cleanup_model_cache=True,
)
self.assertFalse((cache_root / "sana-video-baseline").exists())
ledger = json.loads(
(output_dir / "cleanup.jsonl").read_text(encoding="utf-8")
)
self.assertEqual(ledger["exit_reason"], "interrupted")
self.assertEqual(ledger["after"]["weight_file_count"], 0)
def test_failed_run_is_recorded_as_error_and_cleaned(self):
with tempfile.TemporaryDirectory() as tmpdir:
temp_root = Path(tmpdir)
module = _load_benchmark_module(temp_root)
cache_root = temp_root / "model-caches"
output_dir = temp_root / "outputs"
output_dir.mkdir()
with (
patch.object(
module, "_run_benchmark_once_impl", side_effect=RuntimeError("boom")
),
self.assertRaisesRegex(RuntimeError, "boom"),
):
module.run_benchmark_once(
"sana-video",
"baseline",
output_dir,
model_cache_root=cache_root,
cleanup_model_cache=True,
)
self.assertFalse((cache_root / "sana-video-baseline").exists())
ledger = json.loads(
(output_dir / "cleanup.jsonl").read_text(encoding="utf-8")
)
self.assertEqual(ledger["exit_reason"], "error")
def test_zero_exit_without_artifacts_is_invalid(self):
with tempfile.TemporaryDirectory() as tmpdir:
temp_root = Path(tmpdir)
module = _load_benchmark_module(temp_root)
output_dir = temp_root / "outputs"
output_dir.mkdir()
with patch.object(module.subprocess, "Popen") as popen:
popen.return_value.stdout = iter(())
popen.return_value.wait.return_value = 0
result = module._run_benchmark_once_impl(
"sana-video",
"missing-artifacts",
output_dir,
warmup=False,
cuda_visible_devices="0",
)
command = popen.call_args.args[0]
self.assertIn("--output-path", command)
self.assertIn("--output-file-name", command)
self.assertTrue(result["error"])
self.assertEqual(
result["missing_artifacts"], ["perf dump", "generated output"]
)
def test_mesh_artifacts_are_accepted_and_hashed(self):
with tempfile.TemporaryDirectory() as tmpdir:
temp_root = Path(tmpdir)
module = _load_benchmark_module(temp_root)
output_dir = temp_root / "outputs"
output_dir.mkdir()
def finish_run():
(output_dir / "hunyuan3d-shape_mesh-output.json").write_text(
json.dumps({"total_duration_ms": 1000, "steps": []}),
encoding="utf-8",
)
(output_dir / "hunyuan3d-shape-mesh-output.obj").write_bytes(
b"v 0 0 0\n"
)
return 0
with patch.object(module.subprocess, "Popen") as popen:
popen.return_value.stdout = iter(())
popen.return_value.wait.side_effect = finish_run
result = module._run_benchmark_once_impl(
"hunyuan3d-shape",
"mesh-output",
output_dir,
warmup=False,
cuda_visible_devices="0",
)
self.assertFalse(result["error"])
self.assertEqual(
result["output_artifacts"],
[str(output_dir / "hunyuan3d-shape-mesh-output.obj")],
)
self.assertEqual(len(result["output_sha256"]), 1)
def test_high_bcg_rejects_quality_fusion_mounted_after_capture(self):
with tempfile.TemporaryDirectory() as tmpdir:
temp_root = Path(tmpdir)
module = _load_benchmark_module(temp_root)
output_dir = temp_root / "outputs"
output_dir.mkdir()
with patch.object(module.subprocess, "Popen") as popen:
popen.return_value.stdout = iter(
(
"[Diffusion BCG] captured 3 segment(s)\n",
"Mounted LTX-2 fused RMSNorm+modulate for quality=high\n",
)
)
popen.return_value.wait.return_value = 0
result = module._run_benchmark_once_impl(
"longcat-image",
"bcg-high",
output_dir,
warmup=False,
quality="high",
breakable_cuda_graph=True,
cuda_visible_devices="0",
)
self.assertTrue(result["error"])
self.assertEqual(
result["bcg_invalid_signals"],
[module.BCG_LATE_QUALITY_FUSION_SIGNAL],
)
def test_extra_high_bcg_rejects_quality_fusion_mounted_after_capture(self):
with tempfile.TemporaryDirectory() as tmpdir:
temp_root = Path(tmpdir)
module = _load_benchmark_module(temp_root)
output_dir = temp_root / "outputs"
output_dir.mkdir()
with patch.object(module.subprocess, "Popen") as popen:
popen.return_value.stdout = iter(
(
"[Diffusion BCG] captured 3 segment(s)\n",
"Mounted Qwen fused added-QKV for quality=extra-high\n",
)
)
popen.return_value.wait.return_value = 0
result = module._run_benchmark_once_impl(
"longcat-image",
"bcg-extra-high",
output_dir,
warmup=False,
quality="extra-high",
breakable_cuda_graph=True,
cuda_visible_devices="0",
)
self.assertTrue(result["error"])
self.assertEqual(
result["bcg_invalid_signals"],
[module.BCG_LATE_QUALITY_FUSION_SIGNAL],
)
def test_quality_bcg_matrix_reuses_one_gpu_set_and_cleans_once(self):
with tempfile.TemporaryDirectory() as tmpdir:
temp_root = Path(tmpdir)
module = _load_benchmark_module(temp_root)
cache_root = temp_root / "model-caches"
output_dir = temp_root / "outputs"
output_dir.mkdir()
calls = []
def fake_run(model_key, label, _output_dir, **kwargs):
calls.append((model_key, label, kwargs))
cache_dir = kwargs["model_cache_dir"]
weight_path = cache_dir / "hub" / "model.safetensors"
weight_path.parent.mkdir(parents=True, exist_ok=True)
weight_path.write_bytes(b"weights")
return {"model": model_key, "label": label, "error": False}
with patch.object(module, "_run_benchmark_once_impl", side_effect=fake_run):
results = module.run_quality_bcg_matrix(
"sana-video",
"h200",
output_dir,
model_cache_root=cache_root,
cleanup_model_cache=True,
)
self.assertEqual(len(results), 12)
self.assertEqual(
[
(call[2]["quality"], call[2]["breakable_cuda_graph"])
for call in calls
],
[
(quality, breakable_cuda_graph)
for _, quality, breakable_cuda_graph in module.QUALITY_BCG_ABBA_MATRIX
],
)
self.assertEqual({call[2]["cuda_visible_devices"] for call in calls}, {"0"})
self.assertEqual(
{call[2]["model_cache_dir"] for call in calls},
{calls[0][2]["model_cache_dir"]},
)
self.assertFalse(calls[0][2]["model_cache_dir"].exists())
ledger = json.loads(
(output_dir / "cleanup.jsonl").read_text(encoding="utf-8")
)
self.assertEqual(ledger["exit_reason"], "success")
self.assertEqual(ledger["before"]["weight_file_count"], 1)
self.assertEqual(ledger["after"]["weight_file_count"], 0)
def test_quality_bcg_matrix_rejects_output_hash_mismatch(self):
with tempfile.TemporaryDirectory() as tmpdir:
module = _load_benchmark_module(Path(tmpdir))
results = [
{
"quality": "lossless",
"breakable_cuda_graph": False,
"output_sha256": ["eager"],
"error": False,
},
{
"quality": "lossless",
"breakable_cuda_graph": True,
"output_sha256": ["bcg"],
"error": False,
},
]
module._validate_quality_bcg_output_hashes(results)
self.assertFalse(results[0]["error"])
self.assertTrue(results[1]["error"])
self.assertEqual(
results[1]["output_hash_error"],
"BCG lossless output hash differs from eager",
)
if __name__ == "__main__":
unittest.main()
python/sglang/multimodal_gen/test/unit/test_disagg_roles.py
-7
View File
@@ -617,13 +617,6 @@ class TestStageAffinityAndValidation(_GlobalStageArgsMixin, unittest.TestCase):
pipeline.create_pipeline_stages(pipeline.server_args)
self.assertEqual(list(pipeline._stage_name_mapping.keys()), stage_names)
def test_hunyuan3d_shape_stage_no_longer_stores_model_dtype(self):
pipeline = self._make_hunyuan_pipeline(RoleType.ENCODER, paint_enable=False)
pipeline.create_pipeline_stages(pipeline.server_args)
stage = pipeline._stage_name_mapping["shape_before_denoising"]
self.assertIsInstance(stage, Hunyuan3DShapeBeforeDenoisingStage)
self.assertFalse(hasattr(stage, "model_dtype"))
def test_ltx2_refinement_stage_keeps_class_name_stage_key(self):
stage = object.__new__(LTX2RefinementStage)
self.assertEqual(
python/sglang/multimodal_gen/test/unit/test_nvtx_pytorch_hooks.py
-11
View File
@@ -29,17 +29,6 @@ from sglang.multimodal_gen.runtime.utils.nvtx_pytorch_hooks import (
class TestMaybeNvtxRange(unittest.TestCase):
def test_disabled_returns_noop_context_manager(self) -> None:
ran = False
with maybe_nvtx_range("never", enabled=False):
ran = True
self.assertTrue(ran)
def test_disabled_propagates_exception(self) -> None:
with self.assertRaises(RuntimeError):
with maybe_nvtx_range("never", enabled=False):
raise RuntimeError("boom")
def test_disabled_does_not_call_nvtx(self) -> None:
with (
patch.object(nvtx_pytorch_hooks.nvtx, "range_push") as push,
python/sglang/multimodal_gen/test/unit/test_qwen2_5vl_generation.py
-52
View File
@@ -15,8 +15,6 @@ from sglang.multimodal_gen.runtime.models.encoders.qwen2_5vl import (
Qwen2_5_VLAttention,
Qwen2_5_VLForConditionalGeneration,
_apply_repetition_penalty,
_make_column_linear,
_make_row_linear,
_select_next_token,
)
from sglang.multimodal_gen.runtime.models.encoders.qwen2_5vl_vision import (
@@ -29,14 +27,7 @@ from sglang.multimodal_gen.runtime.pipelines.longcat_image import LongCatImagePi
from sglang.srt.layers.linear import (
ColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from sglang.srt.models.qwen2_5_vl import (
Qwen2_5_VisionPatchEmbed,
Qwen2_5_VisionPatchMerger,
Qwen2_5_VLMLP,
)
from sglang.srt.runtime_context import get_parallel
class _StubQwen2_5VL(Qwen2_5_VLForConditionalGeneration):
@@ -79,49 +70,6 @@ class _AttentionRecorder(nn.Module):
return query
def test_native_vision_reuses_srt_modules():
config = SimpleNamespace(
hidden_size=16,
intermediate_size=24,
hidden_act="silu",
num_heads=2,
depth=0,
patch_size=2,
temporal_patch_size=1,
in_channels=3,
spatial_merge_size=2,
out_hidden_size=12,
fullatt_block_indexes=[],
window_size=8,
)
with get_parallel().override(tp_size=1, tp_rank=0):
model = Qwen2_5VLVisionTransformer(config)
mlp = Qwen2_5_VLMLP(
16,
24,
fuse_gate_up=False,
)
fused_mlp = Qwen2_5_VLMLP(16, 24)
assert isinstance(model.patch_embed, Qwen2_5_VisionPatchEmbed)
assert isinstance(model.merger, Qwen2_5_VisionPatchMerger)
assert not mlp.fuse_gate_up
assert isinstance(mlp.gate_proj, ColumnParallelLinear)
assert isinstance(mlp.up_proj, ColumnParallelLinear)
assert mlp.gate_proj.tp_size == mlp.up_proj.tp_size == 1
assert isinstance(mlp.down_proj, ReplicatedLinear)
assert isinstance(fused_mlp.down_proj, RowParallelLinear)
assert mlp.act is not None
assert isinstance(
_make_column_linear(16, 24, bias=False, use_tensor_parallel=False),
ReplicatedLinear,
)
assert isinstance(
_make_row_linear(24, 16, bias=False, use_tensor_parallel=False),
ReplicatedLinear,
)
def test_text_mlp_uses_single_rank_when_intermediate_size_is_not_tp_divisible(
monkeypatch,
):
python/sglang/multimodal_gen/test/unit/test_qwen3vl_vision.py
-9
View File
@@ -12,7 +12,6 @@ from sglang.multimodal_gen.runtime.models.encoders.minimax_h3_qwen3vl import (
)
from sglang.multimodal_gen.runtime.models.encoders.qwen3vl import (
Qwen3VLForConditionalGeneration,
_make_text_rms_norm,
)
from sglang.multimodal_gen.runtime.models.encoders.qwen3vl_vision import (
Qwen3VLVisionRotaryEmbedding,
@@ -20,7 +19,6 @@ from sglang.multimodal_gen.runtime.models.encoders.qwen3vl_vision import (
_vision_cu_seqlens,
_vision_position_ids,
)
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.models.qwen3_vl import (
Qwen3VLMoeVisionPatchMerger,
Qwen3VLVisionPatchEmbed,
@@ -48,13 +46,6 @@ def test_native_vision_layout_matches_qwen3_merge_order():
assert cu_seqlens.tolist() == [0, 24, 32, 40]
def test_qwen3vl_text_reuses_srt_rms_norm():
norm = _make_text_rms_norm(16, 1e-6)
assert isinstance(norm, RMSNorm)
assert norm.cast_x_before_out_mul
def test_native_vision_keeps_checkpoint_parameter_names():
config = SimpleNamespace(
hidden_size=16,
python/sglang/multimodal_gen/test/unit/test_server_args.py
-5
View File
@@ -770,11 +770,6 @@ class TestWarmupModeNormalization(unittest.TestCase):
sa = self._resolve(warmup_mode="server")
self.assertEqual(sa.warmup_mode, "server")
def test_defaulted_mode_applies_without_legacy_flags(self):
# Bare `sglang serve` defaults to server-based warmup.
sa = self._resolve(warmup_mode="server")
self.assertEqual(sa.warmup_mode, "server")
def test_resolutions_force_warmup_on(self):
sa = self._resolve(
warmup_mode="off",
python/sglang/multimodal_gen/test/unit/test_server_warmup_progress.py
-36
View File
@@ -1,36 +0,0 @@
"""Unit tests for server warmup progress reporting."""
import unittest
from unittest.mock import MagicMock, patch
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
from sglang.multimodal_gen.runtime.server_warmup import SchedulerWarmupMixin
class TestServerWarmupProgress(unittest.TestCase):
def test_ci_progress_uses_scheduler_counter_when_tqdm_is_disabled(self):
scheduler = SchedulerWarmupMixin()
scheduler._show_warmup_progress = True
scheduler._warmup_total = 1
scheduler._warmup_processed = 1
progress_bar = MagicMock(total=1, n=0)
scheduler._warmup_progress_bar = progress_bar
with (
patch(
"sglang.multimodal_gen.runtime.server_warmup._is_ci_log_env",
return_value=True,
),
patch("sglang.multimodal_gen.runtime.server_warmup.logger") as logger,
):
scheduler._advance_warmup_progress_bar(object(), OutputBatch())
logger.info.assert_called_once_with(
"Warmup requests: %s/%s %s", 1, 1, "warmup req"
)
progress_bar.close.assert_called_once_with()
self.assertIsNone(scheduler._warmup_progress_bar)
if __name__ == "__main__":
unittest.main()
python/sglang/multimodal_gen/test/unit/test_single_rank_device_group.py
-5
View File
@@ -25,11 +25,6 @@ class TestSingleRankDeviceGroup(unittest.TestCase):
new_device_group(ranks, requested)
new_group.assert_called_once_with(ranks, backend=requested)
def test_backend_defaults_to_none_for_multi_rank(self):
with patch(NEW_GROUP_PATH) as new_group:
new_device_group([0, 1])
new_group.assert_called_once_with([0, 1], backend=None)
if __name__ == "__main__":
unittest.main()
python/sglang/multimodal_gen/test/unit/test_sol_attn_backend.py
-48
View File
@@ -1,46 +1,14 @@
import importlib.util
import unittest
from unittest.mock import MagicMock, patch
import torch
from sglang.multimodal_gen.runtime.layers.attention.backends.sol_attn import (
SolAttnBackend,
SolAttnImpl,
_get_sol_attn_runtime_config,
_parse_layer_ranges,
)
from sglang.multimodal_gen.runtime.platforms.cuda import CudaPlatformBase
from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEnum
class FakeCudaPlatform(CudaPlatformBase):
is_sm120_device = False
is_blackwell_device = False
supports_flash_attention = True
@classmethod
def is_sm120(cls):
return cls.is_sm120_device
@classmethod
def is_blackwell(cls):
return cls.is_blackwell_device
@classmethod
def has_device_capability(
cls,
capability: tuple[int, int] | int,
device_id: int = 0,
) -> bool:
return cls.supports_flash_attention
class TestSolAttnBackend(unittest.TestCase):
def test_enum_name(self):
self.assertEqual(str(AttentionBackendEnum.SOL_ATTN), "sol_attn")
self.assertTrue(AttentionBackendEnum.SOL_ATTN.is_sparse)
def test_parse_layer_ranges(self):
self.assertEqual(_parse_layer_ranges("0,1,3-5"), frozenset({0, 1, 3, 4, 5}))
@@ -70,9 +38,6 @@ class TestSolAttnBackend(unittest.TestCase):
):
_get_sol_attn_runtime_config()
def test_backend_head_size(self):
self.assertEqual(SolAttnBackend.get_supported_head_sizes(), [128])
def test_dense_guard_uses_early_steps(self):
impl = SolAttnImpl(
num_heads=8,
@@ -127,19 +92,6 @@ class TestSolAttnBackend(unittest.TestCase):
):
self.assertFalse(impl._should_use_dense())
def test_cuda_resolver(self):
if importlib.util.find_spec("sol_attn") is None:
self.skipTest("sol_attn package is not available")
cls_str = FakeCudaPlatform.get_attn_backend_cls_str(
selected_backend=AttentionBackendEnum.SOL_ATTN,
head_size=128,
dtype=torch.bfloat16,
)
self.assertTrue(cls_str.endswith("SolAttnBackend"))
def test_supports_packed_varlen(self):
self.assertTrue(SolAttnBackend.supports_packed_varlen())
if __name__ == "__main__":
unittest.main()
python/sglang/multimodal_gen/test/unit/test_sp_shard.py
-6
View File
@@ -156,12 +156,6 @@ def test_strategy_sp1_replicates(monkeypatch):
assert sps.plan_text_strategy(100) == "replicate"
def test_strategy_shard_when_legal(monkeypatch):
_fake_sp(monkeypatch, 2)
assert sps.plan_text_strategy(15) == "shard"
assert sps.plan_text_strategy(16) == "shard"
def test_strategy_replicates_when_padding_spans_multiple_shards(monkeypatch):
_fake_sp(monkeypatch, 8)
assert sps.plan_text_strategy(1) == "replicate"
python/sglang/multimodal_gen/test/unit/test_srt_clip_reuse.py
-6
View File
@@ -34,12 +34,6 @@ class _FakeProjection(nn.Module):
return hidden_states, None
def test_mmgen_clip_reuses_srt_components():
assert mmgen_clip.CLIPEncoder is srt_clip.CLIPEncoder
assert mmgen_clip.CLIPTextEmbeddings is srt_clip.CLIPTextEmbeddings
assert mmgen_clip.CLIPVisionEmbeddings is srt_clip.CLIPVisionEmbeddings
def test_clip_encoder_propagates_causal_semantics():
with (
patch.object(srt_clip, "CLIPAttention", return_value=nn.Identity()) as attn,
python/sglang/multimodal_gen/test/unit/test_subblock_sparse_attention.py
-12
View File
@@ -24,7 +24,6 @@ from sglang.multimodal_gen.runtime.layers.attention.backends.subblock_sparse.rou
_snap_up_to_8,
)
from sglang.multimodal_gen.runtime.layers.attention.backends.subblock_sparse_attn import (
SubBlockSparseAttentionBackend,
SubBlockSparseAttentionImpl,
SubBlockSparseSchedule,
_dit_layer_index,
@@ -187,17 +186,6 @@ class TestBudgetGranularity(unittest.TestCase):
class TestSubBlockSparseBackend(unittest.TestCase):
def test_the_advertised_builder_can_be_built(self):
"""`AttentionMetadataBuilder.__init__` is abstract; a builder that does
not override it makes `get_builder_cls()()` a TypeError."""
builder = SubBlockSparseAttentionBackend.get_builder_cls()()
builder.prepare()
metadata = builder.build(current_timestep=7)
self.assertIsInstance(
metadata, SubBlockSparseAttentionBackend.get_metadata_cls()
)
self.assertEqual(metadata.current_timestep, 7)
def test_sm90_adapter_uses_presorted_indices_and_64x64_blocks(self):
captured = {}
python/sglang/multimodal_gen/test/unit/test_suite_partitioning.py
+2 -2
View File
@@ -11,9 +11,9 @@ from types import SimpleNamespace
import pytest
from sglang.multimodal_gen.test import run_suite
from sglang.multimodal_gen.test.partitioning import PartitionItem, assign_partition
from sglang.multimodal_gen.test.run_suite import (
from sglang.multimodal_gen.test.runner import diffusion_suite_runner as run_suite
from sglang.multimodal_gen.test.runner.diffusion_suite_runner import (
PartitionAssignment,
build_local_partition_assignment,
)
python/sglang/multimodal_gen/test/unit/test_vae_loader.py
-9
View File
@@ -71,9 +71,6 @@ class _FakeServerArgs:
def should_start_component_on_cpu(self, _component_name):
return False
def should_use_fsdp_for_component(self, _component_name):
return False
def should_configure_layerwise_offload_for_lazy_component(self, component_name):
return component_name in self.layerwise_components
@@ -615,12 +612,6 @@ class TestVAELoader(unittest.TestCase):
native_load.assert_not_called()
def test_pipeline_config_declares_an_empty_native_only_default(self):
loader = vae_loader.VAELoader()
server_args = _FakeServerArgs(QwenImagePipelineConfig())
self.assertFalse(loader.should_raise_customized_load_error(server_args, "vae"))
def test_backfill_ltx2_audio_vae_latent_stats_maps_official_keys(self):
loaded = {
"per_channel_statistics.mean-of-means": torch.tensor([1.0, 2.0]),
python/sglang/multimodal_gen/test/unit/test_vae_spatial_parallel_decode.py
-6
View File
@@ -77,12 +77,6 @@ class _DispatchProbeVAE(ParallelTiledVAE):
class TestVAESpatialParallelDecode(unittest.TestCase):
def test_base_vae_config_defaults_to_auto_parallel_decode(self):
config = VAEConfig()
self.assertTrue(config.use_parallel_decode)
self.assertEqual(config.parallel_decode_mode, "auto")
def test_image_video_vae_configs_default_to_auto_parallel_decode(self):
configs = (
ErnieImageVAEConfig(),
python/sglang/multimodal_gen/test/unit/test_wan_attention_backend.py
-36
View File
@@ -1,36 +0,0 @@
import unittest
from unittest.mock import patch
from torch import nn
from sglang.multimodal_gen.runtime.models.dits.wanvideo import WanSelfAttention
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
_WAN = "sglang.multimodal_gen.runtime.models.dits.wanvideo"
class TestWanAttentionBackendRole(unittest.TestCase):
def test_cross_attention_role_is_forwarded_to_usp(self):
with (
patch(f"{_WAN}.ColumnParallelLinear", return_value=nn.Identity()),
patch(f"{_WAN}.RowParallelLinear", return_value=nn.Identity()),
patch(f"{_WAN}.get_tp_world_size", return_value=1),
patch(f"{_WAN}.USPAttention") as usp_attention,
):
WanSelfAttention(
dim=128,
num_heads=1,
qk_norm=False,
is_cross_attention=True,
supported_attention_backends={
AttentionBackendEnum.FA,
AttentionBackendEnum.TORCH_SDPA,
},
)
self.assertTrue(usp_attention.call_args.kwargs["is_cross_attention"])
self.assertTrue(usp_attention.call_args.kwargs["skip_sequence_parallel"])
if __name__ == "__main__":
unittest.main()
python/sglang/srt/arg_groups/model_overrides/__init__.py
+1 -2
View File
@@ -5,8 +5,7 @@ Importing this package is what registers them. An architecture may be claimed
by more than one module here -- one supplies its attention shape, another its
MoE runner -- but two of them must never declare the *same* field for it:
nobody would own that value, and which module supplied it would come down to
the order of the imports below. ``test_model_override_split.py`` forbids the
overlap, which is why this list needs no particular order.
the order of the imports below. Keep each field owned by one family module.
"""
from sglang.srt.arg_groups.model_overrides import cohere2_moe # noqa: F401
python/sglang/srt/arg_groups/model_overrides/minicpm.py
+2 -2
View File
@@ -33,8 +33,8 @@ def _minicpm_sala_overrides(server_args: Any, hf_config: Any) -> dict:
"minicpm_flashattn": ("fa4" if get_platform().is_blackwell else "fa3"),
"minicpm_flashinfer": "flashinfer",
}
# Literal keys keep the written-field set statically derivable; a loop
# variable hides it from the census in test_chain_read_ratchet.py.
# Keep the three backend decisions explicit so each resolved field is
# easy to review independently.
dense_attention = dense_backends.get(cfg.attention_backend)
if dense_attention is not None:
overrides["attention_backend"] = dense_attention
python/sglang/srt/server_args.py
+1 -2
View File
@@ -797,8 +797,7 @@ def m3_fp8_attn_gemm_enabled(args) -> bool:
# NOTE: The process-wide ServerArgs is owned by the runtime context
# (sglang.srt.runtime_context). The two functions below are LEGACY shims kept
# for the existing call-sites; they publish/read the same live object by
# reference. Do not add new call-sites — the counts are ratcheted
# (decrease-only) by test/registered/unit/test_legacy_global_ratchet.py.
# reference. Do not add new call-sites.
# Imports are in-function so the two modules stay cycle-free at import time.
@functools.lru_cache(maxsize=1)
def _underscore_field_names() -> frozenset:
python/sglang/test/ci/ci_register.py
-13
View File
@@ -98,19 +98,6 @@ def register_amd_ci(
return None
def register_musa_ci(
est_time: float,
suite: Optional[str] = None,
nightly: bool = False,
disabled: Optional[str] = None,
*,
stage: Optional[str] = None,
runner_config: Optional[str] = None,
):
"""Marker for MUSA CI registration (parsed via AST; runtime no-op)."""
return None
def register_npu_ci(
est_time: float,
suite: Optional[str] = None,
python/sglang/test/ci/diffusion_suite_bridge.py
+35
View File
@@ -0,0 +1,35 @@
"""Bridge registered diffusion suites to their case-aware pytest adapter."""
import os
import sys
from pathlib import Path
def _enabled(name: str) -> bool:
return os.environ.get(name, "").lower() in {"1", "true", "yes", "on"}
def run_diffusion_suite(suite: str) -> None:
"""Run one legacy-named diffusion suite without exposing a second CI CLI."""
from sglang.multimodal_gen.test.runner.diffusion_suite_runner import main
args = [sys.argv[0], "--suite", suite]
optional_values = (
("DIFFUSION_PARTITION_ID", "--partition-id"),
("DIFFUSION_TOTAL_PARTITIONS", "--total-partitions"),
("DIFFUSION_PARTITION_PLAN_JSON", "--partition-plan-json"),
("DIFFUSION_PYTEST_FILTER", "--filter"),
)
for environment_name, option in optional_values:
value = os.environ.get(environment_name)
if value:
args.extend([option, value])
if _enabled("DIFFUSION_CONTINUE_ON_ERROR"):
args.append("--continue-on-error")
# Preserve the historical cwd: several diffusion fixtures emit artifacts
# relative to ``python/`` rather than to their source file.
os.chdir(Path(__file__).resolve().parents[4] / "python")
sys.argv = args
main()