[diffusion] chore: adjust layer wise-offload strategy (#25930)
This commit is contained in:
@@ -95,7 +95,6 @@ class WanT2V480PConfig(PipelineConfig):
|
|||||||
def get_model_deployment_config(self) -> ModelDeploymentConfig:
|
def get_model_deployment_config(self) -> ModelDeploymentConfig:
|
||||||
return ModelDeploymentConfig(
|
return ModelDeploymentConfig(
|
||||||
auto_dit_layerwise_offload=True,
|
auto_dit_layerwise_offload=True,
|
||||||
auto_dit_layerwise_offload_high_memory_disable_gb=130,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -146,7 +145,6 @@ class WanI2V480PConfig(WanT2V480PConfig, WanI2VCommonConfig):
|
|||||||
def get_model_deployment_config(self) -> ModelDeploymentConfig:
|
def get_model_deployment_config(self) -> ModelDeploymentConfig:
|
||||||
return ModelDeploymentConfig(
|
return ModelDeploymentConfig(
|
||||||
auto_dit_layerwise_offload=True,
|
auto_dit_layerwise_offload=True,
|
||||||
auto_dit_layerwise_offload_high_memory_disable_gb=130,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -201,7 +201,7 @@ class ServerArgsAutoTuner:
|
|||||||
self._enable_cfg_parallel_if_supported()
|
self._enable_cfg_parallel_if_supported()
|
||||||
|
|
||||||
def maybe_adjust_auto_default_layerwise_offload(self) -> None:
|
def maybe_adjust_auto_default_layerwise_offload(self) -> None:
|
||||||
"""Enable verified non-DiT layerwise defaults for unset component placement."""
|
"""Enable verified layerwise defaults for unset component placement."""
|
||||||
args = self.server_args
|
args = self.server_args
|
||||||
if args.performance_mode != "auto":
|
if args.performance_mode != "auto":
|
||||||
return
|
return
|
||||||
@@ -220,7 +220,7 @@ class ServerArgsAutoTuner:
|
|||||||
return
|
return
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"Automatically enable default non-DiT layerwise offload for %s: %s",
|
"Automatically enable default layerwise offload for %s: %s",
|
||||||
args.pipeline_config.__class__.__name__,
|
args.pipeline_config.__class__.__name__,
|
||||||
layerwise_components,
|
layerwise_components,
|
||||||
)
|
)
|
||||||
@@ -367,17 +367,70 @@ class ServerArgsAutoTuner:
|
|||||||
or args.dit_layerwise_offload is True
|
or args.dit_layerwise_offload is True
|
||||||
):
|
):
|
||||||
# The legacy --dit-layerwise-offload flag is a DiT-only selector.
|
# The legacy --dit-layerwise-offload flag is a DiT-only selector.
|
||||||
# Do not merge implicit non-DiT defaults into that explicit mode.
|
# Do not merge implicit defaults into that explicit mode.
|
||||||
return []
|
return []
|
||||||
|
|
||||||
# `*_cpu_offload` is the component placement knob. If a user explicitly
|
# `*_cpu_offload` is the component placement knob. If a user explicitly
|
||||||
# set it to either true or false, keep that component out of default
|
# set it to either true or false, keep that component out of default
|
||||||
# layerwise selection.
|
# layerwise selection.
|
||||||
return [
|
components = [
|
||||||
component_name
|
component_name
|
||||||
for component_name, arg_name in DEFAULT_LAYERWISE_COMPONENT_ARG_NAMES
|
for component_name, arg_name in DEFAULT_LAYERWISE_COMPONENT_ARG_NAMES
|
||||||
if not args.is_arg_explicitly_set(arg_name)
|
if not args.is_arg_explicitly_set(arg_name)
|
||||||
]
|
]
|
||||||
|
if self._should_auto_enable_dit_layerwise_offload():
|
||||||
|
components.insert(0, LAYERWISE_OFFLOAD_DIT_GROUP)
|
||||||
|
self._set_default_wan_dit_offload_prefetch_size()
|
||||||
|
return components
|
||||||
|
|
||||||
|
def _should_auto_enable_dit_layerwise_offload(self) -> bool:
|
||||||
|
args = self.server_args
|
||||||
|
|
||||||
|
# only for wan for now
|
||||||
|
if not self._is_wan_pipeline_config():
|
||||||
|
return False
|
||||||
|
if not self._deployment_config().auto_dit_layerwise_offload:
|
||||||
|
return False
|
||||||
|
|
||||||
|
if (
|
||||||
|
args.pipeline_config.dmd_denoising_steps is not None
|
||||||
|
or not current_platform.enable_dit_layerwise_offload_for_wan_by_default()
|
||||||
|
or envs.SGLANG_CACHE_DIT_ENABLED
|
||||||
|
or args.use_fsdp_inference
|
||||||
|
or args.is_arg_explicitly_set("dit_cpu_offload")
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
|
||||||
|
# memory mode is memory-first: keep the broad Wan DiT layerwise policy
|
||||||
|
# unless a guard above says it conflicts with another placement path
|
||||||
|
if args.performance_mode == "memory":
|
||||||
|
return True
|
||||||
|
|
||||||
|
# auto mode is performance-first: profiling only showed clear wins for
|
||||||
|
# Wan2.2 A14B, where coarse DiT CPU offload creates large step spikes
|
||||||
|
return (
|
||||||
|
args.performance_mode == "auto" and self._is_wan2_2_a14b_pipeline_config()
|
||||||
|
)
|
||||||
|
|
||||||
|
def _is_wan2_2_a14b_pipeline_config(self) -> bool:
|
||||||
|
config_name = self.server_args.pipeline_config.__class__.__name__
|
||||||
|
return config_name.startswith("Wan2_2_") and "A14B" in config_name
|
||||||
|
|
||||||
|
def _set_default_wan_dit_offload_prefetch_size(self) -> None:
|
||||||
|
args = self.server_args
|
||||||
|
if (
|
||||||
|
args.performance_mode == "auto"
|
||||||
|
and self._is_wan2_2_a14b_pipeline_config()
|
||||||
|
and not args.is_arg_explicitly_set("dit_offload_prefetch_size")
|
||||||
|
):
|
||||||
|
# p2 was the fastest stable default in the Wan2.2 A14B sweep
|
||||||
|
args.dit_offload_prefetch_size = 2
|
||||||
|
|
||||||
|
def _is_wan_pipeline_config(self) -> bool:
|
||||||
|
return any(
|
||||||
|
cls.__module__.endswith(".wan")
|
||||||
|
for cls in self.server_args.pipeline_config.__class__.mro()
|
||||||
|
)
|
||||||
|
|
||||||
def _auto_uses_dit_offload(self) -> bool:
|
def _auto_uses_dit_offload(self) -> bool:
|
||||||
args = self.server_args
|
args = self.server_args
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ Each collected request prints a performance log before validation.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import queue
|
||||||
|
import threading
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
@@ -56,6 +58,21 @@ logger = init_logger(__name__)
|
|||||||
# Track test cases missing estimated_full_test_time_s for time measurement output
|
# Track test cases missing estimated_full_test_time_s for time measurement output
|
||||||
_MISSING_ESTIMATED_TIME_CASES: set[str] = set()
|
_MISSING_ESTIMATED_TIME_CASES: set[str] = set()
|
||||||
_PENDING_BASELINE_DUMPS: dict[str, tuple["PerformanceSummary", bool]] = {}
|
_PENDING_BASELINE_DUMPS: dict[str, tuple["PerformanceSummary", bool]] = {}
|
||||||
|
_OPENAI_REQUEST_TIMEOUT_SECS = float(
|
||||||
|
os.environ.get("SGLANG_TEST_OPENAI_REQUEST_TIMEOUT_SECS", "600")
|
||||||
|
)
|
||||||
|
_SERVER_EXIT_POLL_INTERVAL_SECS = float(
|
||||||
|
os.environ.get("SGLANG_TEST_SERVER_EXIT_POLL_INTERVAL_SECS", "1")
|
||||||
|
)
|
||||||
|
_CONTROL_API_TIMEOUT_SECS = float(
|
||||||
|
os.environ.get("SGLANG_TEST_CONTROL_API_TIMEOUT_SECS", "300")
|
||||||
|
)
|
||||||
|
_SERVER_FATAL_LOG_PATTERNS = (
|
||||||
|
"terminate called after throwing an instance of",
|
||||||
|
"Fatal Python error:",
|
||||||
|
"Segmentation fault",
|
||||||
|
"Aborted (core dumped)",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
@@ -259,8 +276,80 @@ class DiffusionServerBase:
|
|||||||
return OpenAI(
|
return OpenAI(
|
||||||
api_key="sglang-anything",
|
api_key="sglang-anything",
|
||||||
base_url=f"http://localhost:{ctx.port}/v1",
|
base_url=f"http://localhost:{ctx.port}/v1",
|
||||||
|
timeout=_OPENAI_REQUEST_TIMEOUT_SECS,
|
||||||
|
max_retries=0,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _fail_if_server_stopped_or_crashed(
|
||||||
|
self, ctx: ServerContext, case_id: str
|
||||||
|
) -> None:
|
||||||
|
returncode = ctx.process.poll()
|
||||||
|
if returncode is None:
|
||||||
|
tail = ctx.log_tail()
|
||||||
|
for pattern in _SERVER_FATAL_LOG_PATTERNS:
|
||||||
|
if pattern in tail:
|
||||||
|
pytest.fail(
|
||||||
|
f"{case_id}: server reported a fatal backend error during "
|
||||||
|
f"generation: {pattern}\n\nServer log tail:\n{tail}",
|
||||||
|
pytrace=False,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
tail = ctx.log_tail()
|
||||||
|
message = (
|
||||||
|
f"{case_id}: server process exited during generation "
|
||||||
|
f"(code {returncode})."
|
||||||
|
)
|
||||||
|
if tail:
|
||||||
|
message += f"\n\nServer log tail:\n{tail}"
|
||||||
|
pytest.fail(message, pytrace=False)
|
||||||
|
|
||||||
|
def _run_generation_with_server_watchdog(
|
||||||
|
self,
|
||||||
|
ctx: ServerContext,
|
||||||
|
case_id: str,
|
||||||
|
generate_fn: Callable[[str, openai.Client], tuple[str, bytes]],
|
||||||
|
client: openai.Client,
|
||||||
|
) -> tuple[str, bytes]:
|
||||||
|
result_queue: queue.Queue[tuple[str, tuple[str, bytes] | BaseException]] = (
|
||||||
|
queue.Queue(maxsize=1)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _target() -> None:
|
||||||
|
try:
|
||||||
|
result_queue.put(("ok", generate_fn(case_id, client)))
|
||||||
|
except BaseException as exc:
|
||||||
|
result_queue.put(("error", exc))
|
||||||
|
|
||||||
|
# native backend crashes can leave the HTTP client blocked until its read
|
||||||
|
# timeout; keep the request in a daemon thread so the main test thread can
|
||||||
|
# fail as soon as the server subprocess exits
|
||||||
|
thread = threading.Thread(
|
||||||
|
target=_target,
|
||||||
|
name=f"diffusion-generation-{case_id}",
|
||||||
|
daemon=True,
|
||||||
|
)
|
||||||
|
thread.start()
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
state, payload = result_queue.get(
|
||||||
|
timeout=_SERVER_EXIT_POLL_INTERVAL_SECS
|
||||||
|
)
|
||||||
|
except queue.Empty:
|
||||||
|
self._fail_if_server_stopped_or_crashed(ctx, case_id)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if state == "ok":
|
||||||
|
if isinstance(payload, BaseException):
|
||||||
|
raise payload
|
||||||
|
return payload
|
||||||
|
|
||||||
|
self._fail_if_server_stopped_or_crashed(ctx, case_id)
|
||||||
|
if not isinstance(payload, BaseException):
|
||||||
|
pytest.fail(f"{case_id}: invalid generation result state: {state}")
|
||||||
|
raise payload
|
||||||
|
|
||||||
def run_and_collect(
|
def run_and_collect(
|
||||||
self,
|
self,
|
||||||
ctx: ServerContext,
|
ctx: ServerContext,
|
||||||
@@ -274,7 +363,9 @@ class DiffusionServerBase:
|
|||||||
Tuple of (performance_record, content_bytes)
|
Tuple of (performance_record, content_bytes)
|
||||||
"""
|
"""
|
||||||
client = self._client(ctx)
|
client = self._client(ctx)
|
||||||
rid, content = generate_fn(case_id, client)
|
rid, content = self._run_generation_with_server_watchdog(
|
||||||
|
ctx, case_id, generate_fn, client
|
||||||
|
)
|
||||||
|
|
||||||
if not collect_perf:
|
if not collect_perf:
|
||||||
return None, content
|
return None, content
|
||||||
@@ -680,41 +771,55 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
|||||||
This test verifies that each API call succeeds AND that generation works after each operation.
|
This test verifies that each API call succeeds AND that generation works after each operation.
|
||||||
"""
|
"""
|
||||||
base_url = f"http://localhost:{ctx.port}/v1"
|
base_url = f"http://localhost:{ctx.port}/v1"
|
||||||
client = OpenAI(base_url=base_url, api_key="dummy")
|
client = self._client(ctx)
|
||||||
|
|
||||||
# Test 1: unmerge_lora_weights - API should succeed and generation should work
|
# Test 1: unmerge_lora_weights - API should succeed and generation should work
|
||||||
logger.info("[LoRA E2E] Testing unmerge_lora_weights for %s", case.id)
|
logger.info("[LoRA E2E] Testing unmerge_lora_weights for %s", case.id)
|
||||||
resp = requests.post(f"{base_url}/unmerge_lora_weights")
|
resp = requests.post(
|
||||||
|
f"{base_url}/unmerge_lora_weights", timeout=_CONTROL_API_TIMEOUT_SECS
|
||||||
|
)
|
||||||
assert resp.status_code == 200, f"unmerge_lora_weights failed: {resp.text}"
|
assert resp.status_code == 200, f"unmerge_lora_weights failed: {resp.text}"
|
||||||
|
|
||||||
logger.info("[LoRA E2E] Verifying generation after unmerge for %s", case.id)
|
logger.info("[LoRA E2E] Verifying generation after unmerge for %s", case.id)
|
||||||
rid_after_unmerge, _ = generate_fn(case.id, client)
|
rid_after_unmerge, _ = self._run_generation_with_server_watchdog(
|
||||||
|
ctx, case.id, generate_fn, client
|
||||||
|
)
|
||||||
assert rid_after_unmerge is not None, "Generation after unmerge failed"
|
assert rid_after_unmerge is not None, "Generation after unmerge failed"
|
||||||
logger.info("[LoRA E2E] Generation after unmerge succeeded")
|
logger.info("[LoRA E2E] Generation after unmerge succeeded")
|
||||||
|
|
||||||
# Test 2: merge_lora_weights - API should succeed and generation should work
|
# Test 2: merge_lora_weights - API should succeed and generation should work
|
||||||
logger.info("[LoRA E2E] Testing merge_lora_weights for %s", case.id)
|
logger.info("[LoRA E2E] Testing merge_lora_weights for %s", case.id)
|
||||||
resp = requests.post(f"{base_url}/merge_lora_weights")
|
resp = requests.post(
|
||||||
|
f"{base_url}/merge_lora_weights", timeout=_CONTROL_API_TIMEOUT_SECS
|
||||||
|
)
|
||||||
assert resp.status_code == 200, f"merge_lora_weights failed: {resp.text}"
|
assert resp.status_code == 200, f"merge_lora_weights failed: {resp.text}"
|
||||||
|
|
||||||
logger.info("[LoRA E2E] Verifying generation after re-merge for %s", case.id)
|
logger.info("[LoRA E2E] Verifying generation after re-merge for %s", case.id)
|
||||||
rid_after_merge, _ = generate_fn(case.id, client)
|
rid_after_merge, _ = self._run_generation_with_server_watchdog(
|
||||||
|
ctx, case.id, generate_fn, client
|
||||||
|
)
|
||||||
assert rid_after_merge is not None, "Generation after merge failed"
|
assert rid_after_merge is not None, "Generation after merge failed"
|
||||||
logger.info("[LoRA E2E] Generation after merge succeeded")
|
logger.info("[LoRA E2E] Generation after merge succeeded")
|
||||||
|
|
||||||
# Test 3: set_lora (re-set the same adapter) - API should succeed and generation should work
|
# Test 3: set_lora (re-set the same adapter) - API should succeed and generation should work
|
||||||
logger.info("[LoRA E2E] Testing set_lora for %s", case.id)
|
logger.info("[LoRA E2E] Testing set_lora for %s", case.id)
|
||||||
resp = requests.post(f"{base_url}/set_lora", json={"lora_nickname": "default"})
|
resp = requests.post(
|
||||||
|
f"{base_url}/set_lora",
|
||||||
|
json={"lora_nickname": "default"},
|
||||||
|
timeout=_CONTROL_API_TIMEOUT_SECS,
|
||||||
|
)
|
||||||
assert resp.status_code == 200, f"set_lora failed: {resp.text}"
|
assert resp.status_code == 200, f"set_lora failed: {resp.text}"
|
||||||
|
|
||||||
logger.info("[LoRA E2E] Verifying generation after set_lora for %s", case.id)
|
logger.info("[LoRA E2E] Verifying generation after set_lora for %s", case.id)
|
||||||
rid_after_set, _ = generate_fn(case.id, client)
|
rid_after_set, _ = self._run_generation_with_server_watchdog(
|
||||||
|
ctx, case.id, generate_fn, client
|
||||||
|
)
|
||||||
assert rid_after_set is not None, "Generation after set_lora failed"
|
assert rid_after_set is not None, "Generation after set_lora failed"
|
||||||
logger.info("[LoRA E2E] Generation after set_lora succeeded")
|
logger.info("[LoRA E2E] Generation after set_lora succeeded")
|
||||||
|
|
||||||
# Test 4: list_loras - API should return the expected list of LoRA adapters
|
# Test 4: list_loras - API should return the expected list of LoRA adapters
|
||||||
logger.info("[LoRA E2E] Testing list_loras for %s", case.id)
|
logger.info("[LoRA E2E] Testing list_loras for %s", case.id)
|
||||||
resp = requests.get(f"{base_url}/list_loras")
|
resp = requests.get(f"{base_url}/list_loras", timeout=_CONTROL_API_TIMEOUT_SECS)
|
||||||
assert resp.status_code == 200, f"list_loras failed: {resp.text}"
|
assert resp.status_code == 200, f"list_loras failed: {resp.text}"
|
||||||
lora_info = resp.json()
|
lora_info = resp.json()
|
||||||
logger.info("[LoRA E2E] list_loras returned %s", lora_info)
|
logger.info("[LoRA E2E] list_loras returned %s", lora_info)
|
||||||
@@ -742,13 +847,15 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
|||||||
and generation succeeds after each switch.
|
and generation succeeds after each switch.
|
||||||
"""
|
"""
|
||||||
base_url = f"http://localhost:{ctx.port}/v1"
|
base_url = f"http://localhost:{ctx.port}/v1"
|
||||||
client = OpenAI(base_url=base_url, api_key="dummy")
|
client = self._client(ctx)
|
||||||
|
|
||||||
# Test 1: Generate with initial LoRA
|
# Test 1: Generate with initial LoRA
|
||||||
logger.info(
|
logger.info(
|
||||||
"[LoRA Switch E2E] Testing generation with initial LoRA for %s", case.id
|
"[LoRA Switch E2E] Testing generation with initial LoRA for %s", case.id
|
||||||
)
|
)
|
||||||
rid_initial, _ = generate_fn(case.id, client)
|
rid_initial, _ = self._run_generation_with_server_watchdog(
|
||||||
|
ctx, case.id, generate_fn, client
|
||||||
|
)
|
||||||
assert rid_initial is not None, "Generation with initial LoRA failed"
|
assert rid_initial is not None, "Generation with initial LoRA failed"
|
||||||
logger.info("[LoRA Switch E2E] Generation with initial LoRA succeeded")
|
logger.info("[LoRA Switch E2E] Generation with initial LoRA succeeded")
|
||||||
|
|
||||||
@@ -759,6 +866,7 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
|||||||
resp = requests.post(
|
resp = requests.post(
|
||||||
f"{base_url}/set_lora",
|
f"{base_url}/set_lora",
|
||||||
json={"lora_nickname": "lora2", "lora_path": second_lora_path},
|
json={"lora_nickname": "lora2", "lora_path": second_lora_path},
|
||||||
|
timeout=_CONTROL_API_TIMEOUT_SECS,
|
||||||
)
|
)
|
||||||
assert (
|
assert (
|
||||||
resp.status_code == 200
|
resp.status_code == 200
|
||||||
@@ -767,20 +875,28 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
|||||||
logger.info(
|
logger.info(
|
||||||
"[LoRA Switch E2E] Verifying generation with second LoRA for %s", case.id
|
"[LoRA Switch E2E] Verifying generation with second LoRA for %s", case.id
|
||||||
)
|
)
|
||||||
rid_second, _ = generate_fn(case.id, client)
|
rid_second, _ = self._run_generation_with_server_watchdog(
|
||||||
|
ctx, case.id, generate_fn, client
|
||||||
|
)
|
||||||
assert rid_second is not None, "Generation with second LoRA failed"
|
assert rid_second is not None, "Generation with second LoRA failed"
|
||||||
logger.info("[LoRA Switch E2E] Generation with second LoRA succeeded")
|
logger.info("[LoRA Switch E2E] Generation with second LoRA succeeded")
|
||||||
|
|
||||||
# Test 3: Switch back to original LoRA and generate
|
# Test 3: Switch back to original LoRA and generate
|
||||||
logger.info("[LoRA Switch E2E] Switching back to original LoRA for %s", case.id)
|
logger.info("[LoRA Switch E2E] Switching back to original LoRA for %s", case.id)
|
||||||
resp = requests.post(f"{base_url}/set_lora", json={"lora_nickname": "default"})
|
resp = requests.post(
|
||||||
|
f"{base_url}/set_lora",
|
||||||
|
json={"lora_nickname": "default"},
|
||||||
|
timeout=_CONTROL_API_TIMEOUT_SECS,
|
||||||
|
)
|
||||||
assert resp.status_code == 200, f"set_lora back to default failed: {resp.text}"
|
assert resp.status_code == 200, f"set_lora back to default failed: {resp.text}"
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"[LoRA Switch E2E] Verifying generation after switching back for %s",
|
"[LoRA Switch E2E] Verifying generation after switching back for %s",
|
||||||
case.id,
|
case.id,
|
||||||
)
|
)
|
||||||
rid_switched_back, _ = generate_fn(case.id, client)
|
rid_switched_back, _ = self._run_generation_with_server_watchdog(
|
||||||
|
ctx, case.id, generate_fn, client
|
||||||
|
)
|
||||||
assert rid_switched_back is not None, "Generation after switching back failed"
|
assert rid_switched_back is not None, "Generation after switching back failed"
|
||||||
logger.info("[LoRA Switch E2E] Generation after switching back succeeded")
|
logger.info("[LoRA Switch E2E] Generation after switching back succeeded")
|
||||||
|
|
||||||
@@ -812,6 +928,7 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
|||||||
resp = requests.post(
|
resp = requests.post(
|
||||||
f"{base_url}/set_lora",
|
f"{base_url}/set_lora",
|
||||||
json={"lora_nickname": "default", "lora_path": dynamic_lora_path},
|
json={"lora_nickname": "default", "lora_path": dynamic_lora_path},
|
||||||
|
timeout=_CONTROL_API_TIMEOUT_SECS,
|
||||||
)
|
)
|
||||||
assert resp.status_code == 200, f"Dynamic set_lora failed: {resp.text}"
|
assert resp.status_code == 200, f"Dynamic set_lora failed: {resp.text}"
|
||||||
logger.info("[Dynamic LoRA] set_lora succeeded for %s", case.id)
|
logger.info("[Dynamic LoRA] set_lora succeeded for %s", case.id)
|
||||||
@@ -829,7 +946,7 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
|||||||
Tests: basic multi-LoRA, different strengths, cached adapters, switch back to single.
|
Tests: basic multi-LoRA, different strengths, cached adapters, switch back to single.
|
||||||
"""
|
"""
|
||||||
base_url = f"http://localhost:{ctx.port}/v1"
|
base_url = f"http://localhost:{ctx.port}/v1"
|
||||||
client = OpenAI(base_url=base_url, api_key="dummy")
|
client = self._client(ctx)
|
||||||
|
|
||||||
# Test 1: Basic multi-LoRA with list format
|
# Test 1: Basic multi-LoRA with list format
|
||||||
resp = requests.post(
|
resp = requests.post(
|
||||||
@@ -840,11 +957,14 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
|||||||
"target": "all",
|
"target": "all",
|
||||||
"strength": [1.0, 1.0],
|
"strength": [1.0, 1.0],
|
||||||
},
|
},
|
||||||
|
timeout=_CONTROL_API_TIMEOUT_SECS,
|
||||||
)
|
)
|
||||||
assert (
|
assert (
|
||||||
resp.status_code == 200
|
resp.status_code == 200
|
||||||
), f"set_lora with multiple adapters failed: {resp.text}"
|
), f"set_lora with multiple adapters failed: {resp.text}"
|
||||||
rid, _ = generate_fn(case.id, client)
|
rid, _ = self._run_generation_with_server_watchdog(
|
||||||
|
ctx, case.id, generate_fn, client
|
||||||
|
)
|
||||||
assert rid is not None
|
assert rid is not None
|
||||||
|
|
||||||
# Test 2: Different strengths
|
# Test 2: Different strengths
|
||||||
@@ -856,15 +976,22 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
|||||||
"target": "all",
|
"target": "all",
|
||||||
"strength": [0.8, 0.5],
|
"strength": [0.8, 0.5],
|
||||||
},
|
},
|
||||||
|
timeout=_CONTROL_API_TIMEOUT_SECS,
|
||||||
)
|
)
|
||||||
assert (
|
assert (
|
||||||
resp.status_code == 200
|
resp.status_code == 200
|
||||||
), f"set_lora with different strengths failed: {resp.text}"
|
), f"set_lora with different strengths failed: {resp.text}"
|
||||||
rid, _ = generate_fn(case.id, client)
|
rid, _ = self._run_generation_with_server_watchdog(
|
||||||
|
ctx, case.id, generate_fn, client
|
||||||
|
)
|
||||||
assert rid is not None
|
assert rid is not None
|
||||||
|
|
||||||
# Test 3: Different targets
|
# Test 3: Different targets
|
||||||
requests.post(f"{base_url}/set_lora", json={"lora_nickname": "default"})
|
requests.post(
|
||||||
|
f"{base_url}/set_lora",
|
||||||
|
json={"lora_nickname": "default"},
|
||||||
|
timeout=_CONTROL_API_TIMEOUT_SECS,
|
||||||
|
)
|
||||||
resp = requests.post(
|
resp = requests.post(
|
||||||
f"{base_url}/set_lora",
|
f"{base_url}/set_lora",
|
||||||
json={
|
json={
|
||||||
@@ -873,19 +1000,28 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
|||||||
"target": ["transformer", "transformer_2"],
|
"target": ["transformer", "transformer_2"],
|
||||||
"strength": [0.8, 0.5],
|
"strength": [0.8, 0.5],
|
||||||
},
|
},
|
||||||
|
timeout=_CONTROL_API_TIMEOUT_SECS,
|
||||||
)
|
)
|
||||||
assert (
|
assert (
|
||||||
resp.status_code == 200
|
resp.status_code == 200
|
||||||
), f"set_lora with cached adapters failed: {resp.text}"
|
), f"set_lora with cached adapters failed: {resp.text}"
|
||||||
rid, _ = generate_fn(case.id, client)
|
rid, _ = self._run_generation_with_server_watchdog(
|
||||||
|
ctx, case.id, generate_fn, client
|
||||||
|
)
|
||||||
assert rid is not None
|
assert rid is not None
|
||||||
|
|
||||||
# Test 4: Switch back to single LoRA
|
# Test 4: Switch back to single LoRA
|
||||||
resp = requests.post(f"{base_url}/set_lora", json={"lora_nickname": "default"})
|
resp = requests.post(
|
||||||
|
f"{base_url}/set_lora",
|
||||||
|
json={"lora_nickname": "default"},
|
||||||
|
timeout=_CONTROL_API_TIMEOUT_SECS,
|
||||||
|
)
|
||||||
assert (
|
assert (
|
||||||
resp.status_code == 200
|
resp.status_code == 200
|
||||||
), f"set_lora back to single adapter failed: {resp.text}"
|
), f"set_lora back to single adapter failed: {resp.text}"
|
||||||
rid, _ = generate_fn(case.id, client)
|
rid, _ = self._run_generation_with_server_watchdog(
|
||||||
|
ctx, case.id, generate_fn, client
|
||||||
|
)
|
||||||
assert rid is not None
|
assert rid is not None
|
||||||
|
|
||||||
logger.info("[Multi-LoRA] All multi-LoRA tests passed for %s", case.id)
|
logger.info("[Multi-LoRA] All multi-LoRA tests passed for %s", case.id)
|
||||||
@@ -901,7 +1037,7 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
|||||||
|
|
||||||
# Test GET /v1/models
|
# Test GET /v1/models
|
||||||
logger.info("[Models API] Testing GET /v1/models for %s", case.id)
|
logger.info("[Models API] Testing GET /v1/models for %s", case.id)
|
||||||
resp = requests.get(f"{base_url}/v1/models")
|
resp = requests.get(f"{base_url}/v1/models", timeout=_CONTROL_API_TIMEOUT_SECS)
|
||||||
assert resp.status_code == 200, f"/v1/models failed: {resp.text}"
|
assert resp.status_code == 200, f"/v1/models failed: {resp.text}"
|
||||||
|
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
@@ -948,7 +1084,9 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
|||||||
# Test GET /v1/models/{model_path}
|
# Test GET /v1/models/{model_path}
|
||||||
model_path = model["id"]
|
model_path = model["id"]
|
||||||
logger.info("[Models API] Testing GET /v1/models/%s", model_path)
|
logger.info("[Models API] Testing GET /v1/models/%s", model_path)
|
||||||
resp = requests.get(f"{base_url}/v1/models/{model_path}")
|
resp = requests.get(
|
||||||
|
f"{base_url}/v1/models/{model_path}", timeout=_CONTROL_API_TIMEOUT_SECS
|
||||||
|
)
|
||||||
assert resp.status_code == 200, f"/v1/models/{model_path} failed: {resp.text}"
|
assert resp.status_code == 200, f"/v1/models/{model_path} failed: {resp.text}"
|
||||||
|
|
||||||
single_model = resp.json()
|
single_model = resp.json()
|
||||||
@@ -968,7 +1106,10 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
|||||||
|
|
||||||
# Test GET /v1/models/{non_existent_model} returns 404
|
# Test GET /v1/models/{non_existent_model} returns 404
|
||||||
logger.info("[Models API] Testing GET /v1/models/non_existent_model")
|
logger.info("[Models API] Testing GET /v1/models/non_existent_model")
|
||||||
resp = requests.get(f"{base_url}/v1/models/non_existent_model")
|
resp = requests.get(
|
||||||
|
f"{base_url}/v1/models/non_existent_model",
|
||||||
|
timeout=_CONTROL_API_TIMEOUT_SECS,
|
||||||
|
)
|
||||||
assert resp.status_code == 404, f"Expected 404, got {resp.status_code}"
|
assert resp.status_code == 404, f"Expected 404, got {resp.status_code}"
|
||||||
error_data = resp.json()
|
error_data = resp.json()
|
||||||
assert "error" in error_data, "404 response missing 'error' field"
|
assert "error" in error_data, "404 response missing 'error' field"
|
||||||
@@ -986,7 +1127,7 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
|||||||
return
|
return
|
||||||
|
|
||||||
base_url = f"http://localhost:{ctx.port}"
|
base_url = f"http://localhost:{ctx.port}"
|
||||||
resp = requests.get(f"{base_url}/v1/models")
|
resp = requests.get(f"{base_url}/v1/models", timeout=_CONTROL_API_TIMEOUT_SECS)
|
||||||
assert resp.status_code == 200, f"/v1/models failed: {resp.text}"
|
assert resp.status_code == 200, f"/v1/models failed: {resp.text}"
|
||||||
data = resp.json().get("data", [])
|
data = resp.json().get("data", [])
|
||||||
if not data:
|
if not data:
|
||||||
@@ -1001,7 +1142,11 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
|||||||
if case.sampling_params.output_size:
|
if case.sampling_params.output_size:
|
||||||
payload["size"] = case.sampling_params.output_size
|
payload["size"] = case.sampling_params.output_size
|
||||||
|
|
||||||
resp = requests.post(f"{base_url}/v1/videos", json=payload)
|
resp = requests.post(
|
||||||
|
f"{base_url}/v1/videos",
|
||||||
|
json=payload,
|
||||||
|
timeout=_CONTROL_API_TIMEOUT_SECS,
|
||||||
|
)
|
||||||
assert (
|
assert (
|
||||||
resp.status_code == 400
|
resp.status_code == 400
|
||||||
), f"Expected 400 for T2V input_reference, got {resp.status_code}: {resp.text}"
|
), f"Expected 400 for T2V input_reference, got {resp.status_code}: {resp.text}"
|
||||||
|
|||||||
@@ -160,6 +160,14 @@ class ServerContext:
|
|||||||
_stdout_fh: Any = field(repr=False)
|
_stdout_fh: Any = field(repr=False)
|
||||||
_log_thread: threading.Thread | None = field(default=None, repr=False)
|
_log_thread: threading.Thread | None = field(default=None, repr=False)
|
||||||
|
|
||||||
|
def log_tail(self, lines: int = 200) -> str:
|
||||||
|
"""Return recent server output for failure diagnostics."""
|
||||||
|
try:
|
||||||
|
content = self.stdout_file.read_text(encoding="utf-8", errors="ignore")
|
||||||
|
return "\n".join(content.splitlines()[-lines:])
|
||||||
|
except Exception:
|
||||||
|
return ""
|
||||||
|
|
||||||
def cleanup(self) -> None:
|
def cleanup(self) -> None:
|
||||||
"""Clean up server resources."""
|
"""Clean up server resources."""
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -17,7 +17,16 @@ from sglang.multimodal_gen.configs.pipeline_configs.mova import MOVAPipelineConf
|
|||||||
from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import (
|
from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import (
|
||||||
QwenImagePipelineConfig,
|
QwenImagePipelineConfig,
|
||||||
)
|
)
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.wan import WanT2V480PConfig
|
from sglang.multimodal_gen.configs.pipeline_configs.wan import (
|
||||||
|
FastWan2_2_TI2V_5B_Config,
|
||||||
|
TurboWanT2V480PConfig,
|
||||||
|
Wan2_2_I2V_A14B_Config,
|
||||||
|
Wan2_2_T2V_A14B_Config,
|
||||||
|
WanI2V480PConfig,
|
||||||
|
WanI2V720PConfig,
|
||||||
|
WanT2V480PConfig,
|
||||||
|
WanT2V720PConfig,
|
||||||
|
)
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.zimage import ZImagePipelineConfig
|
from sglang.multimodal_gen.configs.pipeline_configs.zimage import ZImagePipelineConfig
|
||||||
from sglang.multimodal_gen.registry import _get_config_info
|
from sglang.multimodal_gen.registry import _get_config_info
|
||||||
from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
|
from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
|
||||||
@@ -510,6 +519,54 @@ class TestOffloadDefaults(unittest.TestCase):
|
|||||||
["text_encoder", "image_encoder", "vae"],
|
["text_encoder", "image_encoder", "vae"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_auto_wan2_2_a14b_layerwise_offload_adds_dit(self):
|
||||||
|
for pipeline_config, model_path in (
|
||||||
|
(Wan2_2_T2V_A14B_Config(), "Wan-AI/Wan2.2-T2V-A14B-Diffusers"),
|
||||||
|
(Wan2_2_I2V_A14B_Config(), "Wan-AI/Wan2.2-I2V-A14B-Diffusers"),
|
||||||
|
):
|
||||||
|
with self.subTest(pipeline_config=pipeline_config.__class__.__name__):
|
||||||
|
args = self._from_dict_with_pipeline_config(
|
||||||
|
pipeline_config,
|
||||||
|
kwargs={
|
||||||
|
"model_path": model_path,
|
||||||
|
"performance_mode": "auto",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertTrue(args.layerwise_offload_components)
|
||||||
|
self.assertFalse(args.use_fsdp_inference)
|
||||||
|
self.assertFalse(args.dit_cpu_offload)
|
||||||
|
self.assertFalse(args.text_encoder_cpu_offload)
|
||||||
|
self.assertFalse(args.image_encoder_cpu_offload)
|
||||||
|
self.assertEqual(args.dit_offload_prefetch_size, 2)
|
||||||
|
self.assertEqual(
|
||||||
|
args.layerwise_offload_components,
|
||||||
|
["dit", "text_encoder", "image_encoder", "vae"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_auto_wan2_1_14b_layerwise_offload_uses_non_dit_default(self):
|
||||||
|
for pipeline_config, model_path in (
|
||||||
|
(WanT2V720PConfig(), "Wan-AI/Wan2.1-T2V-14B-Diffusers"),
|
||||||
|
(WanI2V480PConfig(), "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers"),
|
||||||
|
(WanI2V720PConfig(), "Wan-AI/Wan2.1-I2V-14B-720P-Diffusers"),
|
||||||
|
):
|
||||||
|
with self.subTest(pipeline_config=pipeline_config.__class__.__name__):
|
||||||
|
args = self._from_dict_with_pipeline_config(
|
||||||
|
pipeline_config,
|
||||||
|
kwargs={
|
||||||
|
"model_path": model_path,
|
||||||
|
"performance_mode": "auto",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertTrue(args.layerwise_offload_components)
|
||||||
|
self.assertTrue(args.dit_cpu_offload)
|
||||||
|
self.assertEqual(args.dit_offload_prefetch_size, 0.0)
|
||||||
|
self.assertEqual(
|
||||||
|
args.layerwise_offload_components,
|
||||||
|
["text_encoder", "image_encoder", "vae"],
|
||||||
|
)
|
||||||
|
|
||||||
def test_memory_wan_layerwise_offload_is_enabled_without_fsdp(self):
|
def test_memory_wan_layerwise_offload_is_enabled_without_fsdp(self):
|
||||||
args = self._from_dict_with_pipeline_config(
|
args = self._from_dict_with_pipeline_config(
|
||||||
WanT2V480PConfig(),
|
WanT2V480PConfig(),
|
||||||
@@ -518,12 +575,12 @@ class TestOffloadDefaults(unittest.TestCase):
|
|||||||
|
|
||||||
self.assertTrue(args.layerwise_offload_components)
|
self.assertTrue(args.layerwise_offload_components)
|
||||||
self.assertFalse(args.use_fsdp_inference)
|
self.assertFalse(args.use_fsdp_inference)
|
||||||
self.assertTrue(args.dit_cpu_offload)
|
self.assertFalse(args.dit_cpu_offload)
|
||||||
self.assertFalse(args.text_encoder_cpu_offload)
|
self.assertFalse(args.text_encoder_cpu_offload)
|
||||||
self.assertFalse(args.image_encoder_cpu_offload)
|
self.assertFalse(args.image_encoder_cpu_offload)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
args.layerwise_offload_components,
|
args.layerwise_offload_components,
|
||||||
["text_encoder", "image_encoder", "vae"],
|
["dit", "text_encoder", "image_encoder", "vae"],
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_auto_wan_layerwise_offload_does_not_disable_explicit_fsdp(self):
|
def test_auto_wan_layerwise_offload_does_not_disable_explicit_fsdp(self):
|
||||||
@@ -543,6 +600,79 @@ class TestOffloadDefaults(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
self.assertTrue(args.use_fsdp_inference)
|
self.assertTrue(args.use_fsdp_inference)
|
||||||
|
|
||||||
|
def test_auto_wan_layerwise_offload_preserves_explicit_dit_cpu_offload(self):
|
||||||
|
args = self._from_dict_with_pipeline_config(
|
||||||
|
WanT2V480PConfig(),
|
||||||
|
kwargs={
|
||||||
|
"model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
|
||||||
|
"performance_mode": "auto",
|
||||||
|
"dit_cpu_offload": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertTrue(args.dit_cpu_offload)
|
||||||
|
self.assertEqual(
|
||||||
|
args.layerwise_offload_components,
|
||||||
|
["text_encoder", "image_encoder", "vae"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_auto_mova_layerwise_offload_does_not_implicitly_add_dit(self):
|
||||||
|
args = self._from_dict_with_pipeline_config(
|
||||||
|
MOVAPipelineConfig(),
|
||||||
|
kwargs={
|
||||||
|
"model_path": "OpenMOSS-Team/MOVA-360p",
|
||||||
|
"performance_mode": "auto",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertTrue(args.dit_cpu_offload)
|
||||||
|
self.assertEqual(
|
||||||
|
args.layerwise_offload_components,
|
||||||
|
["text_encoder", "image_encoder", "vae"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_auto_fastwan_layerwise_offload_does_not_implicitly_add_dit(self):
|
||||||
|
args = self._from_dict_with_pipeline_config(
|
||||||
|
FastWan2_2_TI2V_5B_Config(),
|
||||||
|
kwargs={
|
||||||
|
"model_path": "FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers",
|
||||||
|
"performance_mode": "auto",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertTrue(args.dit_cpu_offload)
|
||||||
|
self.assertEqual(
|
||||||
|
args.layerwise_offload_components,
|
||||||
|
["text_encoder", "image_encoder", "vae"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_auto_turbo_wan_layerwise_offload_does_not_implicitly_add_dit(self):
|
||||||
|
args = self._from_dict_with_pipeline_config(
|
||||||
|
TurboWanT2V480PConfig(),
|
||||||
|
kwargs={
|
||||||
|
"model_path": "IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers",
|
||||||
|
"performance_mode": "auto",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertTrue(args.dit_cpu_offload)
|
||||||
|
self.assertEqual(
|
||||||
|
args.layerwise_offload_components,
|
||||||
|
["text_encoder", "image_encoder", "vae"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_explicit_fastwan_dit_layerwise_still_selects_dit_group(self):
|
||||||
|
args = self._from_dict_with_pipeline_config(
|
||||||
|
FastWan2_2_TI2V_5B_Config(),
|
||||||
|
kwargs={
|
||||||
|
"model_path": "FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers",
|
||||||
|
"dit_layerwise_offload": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertFalse(args.dit_cpu_offload)
|
||||||
|
self.assertEqual(args.layerwise_offload_components, ["dit"])
|
||||||
|
|
||||||
def test_auto_multi_gpu_wan_uses_layerwise_offload_without_cfg(self):
|
def test_auto_multi_gpu_wan_uses_layerwise_offload_without_cfg(self):
|
||||||
with patch.object(ServerArgs, "_model_default_uses_cfg", return_value=False):
|
with patch.object(ServerArgs, "_model_default_uses_cfg", return_value=False):
|
||||||
args = self._from_dict_with_pipeline_config(
|
args = self._from_dict_with_pipeline_config(
|
||||||
@@ -825,12 +955,12 @@ class TestOffloadDefaults(unittest.TestCase):
|
|||||||
|
|
||||||
self.assertFalse(args.use_fsdp_inference)
|
self.assertFalse(args.use_fsdp_inference)
|
||||||
self.assertTrue(args.layerwise_offload_components)
|
self.assertTrue(args.layerwise_offload_components)
|
||||||
self.assertTrue(args.dit_cpu_offload)
|
self.assertFalse(args.dit_cpu_offload)
|
||||||
self.assertFalse(args.text_encoder_cpu_offload)
|
self.assertFalse(args.text_encoder_cpu_offload)
|
||||||
self.assertFalse(args.image_encoder_cpu_offload)
|
self.assertFalse(args.image_encoder_cpu_offload)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
args.layerwise_offload_components,
|
args.layerwise_offload_components,
|
||||||
["text_encoder", "image_encoder", "vae"],
|
["dit", "text_encoder", "image_encoder", "vae"],
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_memory_mode_preserves_explicit_fsdp(self):
|
def test_memory_mode_preserves_explicit_fsdp(self):
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ CI_DATA_REPO_OWNER = "sgl-project"
|
|||||||
CI_DATA_REPO_NAME = "ci-data"
|
CI_DATA_REPO_NAME = "ci-data"
|
||||||
CI_DATA_BRANCH = "main"
|
CI_DATA_BRANCH = "main"
|
||||||
HISTORY_PREFIX = "diffusion-comparisons"
|
HISTORY_PREFIX = "diffusion-comparisons"
|
||||||
MAX_HISTORY_RUNS = 14
|
MAX_HISTORY_RUNS = 29
|
||||||
|
|
||||||
# Base URL for chart images pushed to sgl-project/ci-data
|
# Base URL for chart images pushed to sgl-project/ci-data
|
||||||
CHARTS_RAW_BASE_URL = (
|
CHARTS_RAW_BASE_URL = (
|
||||||
|
|||||||
Reference in New Issue
Block a user