diff --git a/.github/workflows/_npu-analyze-failure.yml b/.github/workflows/_npu-analyze-failure.yml index f801f45575..b50bc95b3a 100644 --- a/.github/workflows/_npu-analyze-failure.yml +++ b/.github/workflows/_npu-analyze-failure.yml @@ -9,6 +9,11 @@ on: type: string required: true description: 'Glob pattern for test log artifacts (e.g. selected-test-logs-*)' + diffusion_artifact_pattern: + type: string + required: false + default: 'diffusion-failures-*' + description: 'Glob pattern for diffusion failure artifacts (consistency failures)' recommendations_content: type: string required: false @@ -33,37 +38,68 @@ jobs: id: locate-recs run: | mkdir -p ./test-logs + # Always create the file so the analyzer can run on diffusion-only failures. + touch ./test-logs/_recommended.txt + echo "RECS_FILE=./test-logs/_recommended.txt" >> $GITHUB_ENV if [ -n "${{ inputs.recommendations_content }}" ]; then echo "has_recommendations=true" >> $GITHUB_OUTPUT echo "Using workflow output recommendations" echo "${{ inputs.recommendations_content }}" > ./test-logs/_recommended.txt - echo "RECS_FILE=./test-logs/_recommended.txt" >> $GITHUB_ENV echo "RECS_SOURCE=output" >> $GITHUB_ENV COUNT=$(wc -l < ./test-logs/_recommended.txt) echo "Recommendations count: ${COUNT}" else echo "has_recommendations=false" >> $GITHUB_OUTPUT - echo "::notice::No recommended test cases detected, skipping subsequent analysis tasks." + echo "RECS_SOURCE=none" >> $GITHUB_ENV + echo "::notice::No recommended test cases detected; analyzing log and diffusion failures only." fi # ----- Step 2: Download test logs ----- + # continue-on-error is a step keyword, not a download-artifact input. - name: Download test logs uses: actions/download-artifact@v7 with: pattern: ${{ inputs.log_artifact_pattern }} path: ./test-logs merge-multiple: true - continue-on-error: true + continue-on-error: true - - name: Show downloaded logs structure + # ----- Step 2b: Download diffusion failure artifacts ----- + # Keep merge-multiple off so each partition's consistency_failures/summary.json + # lands in its own / directory instead of clobbering the others. + - name: Download diffusion failure artifacts + uses: actions/download-artifact@v7 + with: + pattern: ${{ inputs.diffusion_artifact_pattern }} + path: ./diffusion-failures + continue-on-error: true + + - name: Check for diffusion failure artifacts + id: check-diffusion + run: | + if compgen -G "./diffusion-failures/*/consistency_failures/summary.json" > /dev/null; then + echo "found=true" >> $GITHUB_OUTPUT + echo "Diffusion consistency failure artifacts found" + else + echo "found=false" >> $GITHUB_OUTPUT + echo "No diffusion consistency failure artifacts" + fi + + - name: Show downloaded artifacts structure run: | echo "Downloaded log files:" find ./test-logs -type f | head -50 || echo " (no logs found)" + echo "Downloaded diffusion failure artifacts:" + if compgen -G "./diffusion-failures/*" > /dev/null; then + find ./diffusion-failures -type f | head -50 + else + echo " (no diffusion failure artifacts found)" + fi # ----- Step 3: Run analysis ----- - name: Ensure regex is available - if: steps.locate-recs.outputs.has_recommendations != 'false' + if: ${{ steps.locate-recs.outputs.has_recommendations != 'false' || steps.check-diffusion.outputs.found == 'true' }} run: | set -euo pipefail if ! python3 -c "import regex" >/dev/null 2>&1; then @@ -75,12 +111,13 @@ jobs: python3 -c "import regex; print('regex ok:', regex.__file__)" - name: Run failure analysis - if: steps.locate-recs.outputs.has_recommendations != 'false' + if: ${{ steps.locate-recs.outputs.has_recommendations != 'false' || steps.check-diffusion.outputs.found == 'true' }} id: analysis continue-on-error: true run: | python3 scripts/ci/npu/precise-test/analyze_failure_report.py \ --log-dir ./test-logs \ + --diffusion-dir ./diffusion-failures \ --recommendations-file "${RECS_FILE}" \ --recommendations-source "${RECS_SOURCE:-none}" \ --output ./test-logs/failure_report.md diff --git a/.github/workflows/pr-test-npu.yml b/.github/workflows/pr-test-npu.yml index afb2e531ab..f7a9b2f114 100644 --- a/.github/workflows/pr-test-npu.yml +++ b/.github/workflows/pr-test-npu.yml @@ -278,6 +278,11 @@ jobs: # copy gsm8k dataset from cache cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp + - name: Prefetch image-edit input + env: + GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" + run: python3 scripts/ci/utils/diffusion/prefetch_input_image.py + - name: Run test timeout-minutes: 60 env: diff --git a/proto/sglang/api/v1/generate.proto b/proto/sglang/api/v1/generate.proto index 446cbc38df..bdc9a9096c 100644 --- a/proto/sglang/api/v1/generate.proto +++ b/proto/sglang/api/v1/generate.proto @@ -184,6 +184,14 @@ message MediaRef { optional int64 max_image_tokens = 13; // Video: also decode the audio track. optional bool use_audio = 14; + // GLM-Image external AR uses grids with input_ids to compute MRoPE positions. + optional ImageGridThw image_grid_thw = 15; +} + +// One (t, h, w) row per image; the processor validates row arity. +message ImageGridThw { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated Int64List rows = 1; } // One media item. JSON: a bare string is the source; an object is a MediaRef. diff --git a/python/sglang/api/v1/api_types.py b/python/sglang/api/v1/api_types.py index 686c9c73ef..88cc0d063e 100644 --- a/python/sglang/api/v1/api_types.py +++ b/python/sglang/api/v1/api_types.py @@ -344,6 +344,7 @@ class MediaRef(msgspec.Struct, kw_only=True, frozen=True): max_tokens_per_frame: Optional[int] = None max_image_tokens: Optional[int] = None use_audio: Optional[bool] = None + image_grid_thw: Optional[ImageGridThw] = None @classmethod def from_json_value(cls, value: Any) -> "MediaRef": @@ -506,6 +507,9 @@ NullableTopLogprobs = Optional[TopLogprobRow] # sglang.api.v1.HiddenStateRow: JSON bare array. HiddenStateRow = List[float] +# sglang.api.v1.ImageGridThw: JSON bare array. +ImageGridThw = List[Int64List] + # sglang.api.v1.MediaItem: JSON: the value's shape picks the arm. MediaItem = Union[str, MediaRef] @@ -1267,7 +1271,7 @@ def encode_GenerateStreamError(v: GenerateStreamError) -> Dict[str, Any]: return d -_FIELDS_MediaRef = ("url", "detail", "max_dynamic_patch", "content_hash", "min_pixels", "max_pixels", "total_max_pixels", "fps", "num_frames", "min_frames", "max_frames", "max_tokens_per_frame", "max_image_tokens", "use_audio",) +_FIELDS_MediaRef = ("url", "detail", "max_dynamic_patch", "content_hash", "min_pixels", "max_pixels", "total_max_pixels", "fps", "num_frames", "min_frames", "max_frames", "max_tokens_per_frame", "max_image_tokens", "use_audio", "image_grid_thw",) def decode_MediaRef(value: Any) -> MediaRef: @@ -1302,6 +1306,8 @@ def decode_MediaRef(value: Any) -> MediaRef: kw["max_image_tokens"] = None if x is None else _expect_int(x, expected="i64") elif key == "use_audio": kw["use_audio"] = None if x is None else _expect_bool(x, expected="a boolean") + elif key == "image_grid_thw": + kw["image_grid_thw"] = None if x is None else decode_ImageGridThw(x) else: raise _unknown_field(key, fields=_FIELDS_MediaRef) return MediaRef(**kw) @@ -1336,9 +1342,19 @@ def encode_MediaRef(v: MediaRef) -> Dict[str, Any]: d["max_image_tokens"] = v.max_image_tokens if v.use_audio is not None: d["use_audio"] = v.use_audio + if v.image_grid_thw is not None: + d["image_grid_thw"] = encode_ImageGridThw(v.image_grid_thw) return d +def decode_ImageGridThw(value: Any) -> ImageGridThw: + return [decode_Int64List(e) for e in _expect_seq(value, expected="a sequence")] + + +def encode_ImageGridThw(v: ImageGridThw) -> List[Any]: + return [encode_Int64List(e) for e in v] + + def decode_MediaItem(value: Any) -> MediaItem: if isinstance(value, str): return value diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/glm_image.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/glm_image.py index 638ecad0ef..410d207ea3 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/glm_image.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/glm_image.py @@ -362,6 +362,23 @@ class GlmImageAR(PipelineStage): e, ) raise + except requests.HTTPError as e: + # HTTPError alone omits the server's request-validation details. + response = e.response + if response is not None: + logger.error( + "SGLang encoder at %s returned HTTP %s: %s", + server_args.srt_encoder_url, + response.status_code, + response.text[:2000], + ) + else: + logger.error( + "SGLang encoder at %s failed with no response: %s", + server_args.srt_encoder_url, + e, + ) + raise except requests.RequestException as e: logger.error( "SGLang encoder request to %s failed: %s", diff --git a/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py b/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py index d3aac8fa62..4ec8ee4635 100644 --- a/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py +++ b/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py @@ -203,6 +203,7 @@ TWO_NPU_CASES: list[DiffusionTestCase] = [ "seed": 1101, }, ), + pin_consistency_quality=False, run_perf_check=True, run_consistency_check=True, run_component_accuracy_check=False, diff --git a/python/sglang/multimodal_gen/test/server/testcase_configs.py b/python/sglang/multimodal_gen/test/server/testcase_configs.py index 7bb78d0258..3cd3c6829d 100644 --- a/python/sglang/multimodal_gen/test/server/testcase_configs.py +++ b/python/sglang/multimodal_gen/test/server/testcase_configs.py @@ -40,6 +40,7 @@ from sglang.multimodal_gen.test.test_utils import ( SGL_TEST_FILES_CI_DATA_REPO, SGL_TEST_FILES_CI_DATA_REVISION, ) +from sglang.srt.environ import envs @dataclass @@ -348,6 +349,8 @@ class DiffusionTestCase: run_lora_dynamic_load_check: bool = False run_lora_dynamic_switch_check: bool = False run_multi_lora_api_check: bool = False + # Cached MiniMax goldens require its implicit-quality cache mode. + pin_consistency_quality: bool = True def __post_init__(self) -> None: if self.perf_repeat_requests < 1: @@ -377,7 +380,19 @@ class DiffusionTestCase: # removes the pin. # Replaces rather than mutates: several cases share one module-level # sampling-params instance. - if self.run_consistency_check and "quality" not in self.sampling_params.extras: + cache_dit_requested = self.sampling_params.extras.get("enable_cache_dit") + if cache_dit_requested is None: + cache_dit_requested = self.server_args.enable_cache_dit or ( + self.server_args.env_vars.get("SGLANG_CACHE_DIT_ENABLED", "").lower() + == "true" + ) + # Cached goldens require the cached path; an explicit exact tier disables + # MiniMax-H3's environment-selected Cache-DiT configuration. + if ( + self.run_consistency_check + and "quality" not in self.sampling_params.extras + and (self.pin_consistency_quality or not cache_dit_requested) + ): object.__setattr__( self, "sampling_params", @@ -714,16 +729,22 @@ MODELOPT_QWEN_IMAGE_2512_NVFP4_CI_sampling_params = replace( extras={"num_inference_steps": 50, "seed": 0}, ) +# CI prefetches the input; standalone runs retain the release URL fallback. +TI2I_QWEN_IMAGE_EDIT_INPUT = ( + envs.SGLANG_TEST_TI2I_INPUT_IMAGE.get() + or "https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg" +) + MODELOPT_TI2I_CI_sampling_params = DiffusionSamplingParams( prompt="Convert 2D style to 3D style", - image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", + image_path=TI2I_QWEN_IMAGE_EDIT_INPUT, output_size="512x512", extras={"num_inference_steps": 8, "seed": 0}, ) TI2I_sampling_params = DiffusionSamplingParams( prompt="Convert 2D style to 3D style", - image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", + image_path=TI2I_QWEN_IMAGE_EDIT_INPUT, ) MULTI_IMAGE_TI2I_sampling_params = DiffusionSamplingParams( diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py index 4accd433b3..cb5b1270bd 100644 --- a/python/sglang/multimodal_gen/test/test_utils.py +++ b/python/sglang/multimodal_gen/test/test_utils.py @@ -40,9 +40,14 @@ SGL_TEST_FILES_CI_DATA_REPO = "sgl-project/ci-data-diffusion" SGL_TEST_FILES_CI_DATA_REVISION = "75992dbf1d6eaeacbc1c2cd6beb48f77dd7ae4f7" # The NPU pin is kept as a separate branch so ascend GT can be bumped independently -# when it's regenerated on its own cadence. +# when it's regenerated on its own cadence. 2173af2f re-generated the MiniMax-H3 +# frames after #38671 made num_inference_steps=50 run 50 denoising updates +# (previously 49). The regeneration must run on a branch where +# pin_consistency_quality=False for this case (npu_diffusion_ci_fixes_2): generating +# on main pins quality='exact', which disables MiniMax-H3's env-selected Cache-DiT +# and produces a golden from a different code path than the test runs. if current_platform.is_npu(): - SGL_TEST_FILES_CI_DATA_REVISION = "609c4cef31864d75dbd11fe3c26b8ae09469204d" + SGL_TEST_FILES_CI_DATA_REVISION = "2173af2fe762e2c95e18b06d017604d520196eff" SGL_TEST_FILES_CONSISTENCY_GT_ROOT = ( "https://raw.githubusercontent.com/" diff --git a/python/sglang/multimodal_gen/test/unit/test_performance_failure_policy.py b/python/sglang/multimodal_gen/test/unit/test_performance_failure_policy.py index 34cb583268..10211180e5 100644 --- a/python/sglang/multimodal_gen/test/unit/test_performance_failure_policy.py +++ b/python/sglang/multimodal_gen/test/unit/test_performance_failure_policy.py @@ -22,6 +22,42 @@ from sglang.multimodal_gen.test.server.testcase_configs import ( ) +@pytest.mark.parametrize("request_override", [None, True, False]) +@pytest.mark.parametrize("pin_consistency_quality", [True, False]) +def test_cached_consistency_case_preserves_cache_selection( + request_override, pin_consistency_quality +): + extras = {} if request_override is None else {"enable_cache_dit": request_override} + case = DiffusionTestCase( + "cached_consistency", + DiffusionServerArgs( + model_path="test", + modality="video", + env_vars={"SGLANG_CACHE_DIT_ENABLED": "true"}, + ), + DiffusionSamplingParams(prompt="test", extras=extras), + run_consistency_check=True, + pin_consistency_quality=pin_consistency_quality, + ) + if pin_consistency_quality or request_override is False: + assert case.sampling_params.extras["quality"] == "exact" + else: + # An explicit quality tier suppresses the legacy MiniMax cache mode. + assert "quality" not in case.sampling_params.extras + + +@pytest.mark.parametrize("pin_consistency_quality", [True, False]) +def test_cached_consistency_case_preserves_explicit_quality(pin_consistency_quality): + case = DiffusionTestCase( + "explicit_quality", + DiffusionServerArgs(model_path="test", modality="video", enable_cache_dit=True), + DiffusionSamplingParams(prompt="test", extras={"quality": "exact"}), + run_consistency_check=True, + pin_consistency_quality=pin_consistency_quality, + ) + assert case.sampling_params.extras["quality"] == "exact" + + @pytest.mark.parametrize("generate_baseline", [False, True]) def test_e2e_only_does_not_require_stage_metrics(monkeypatch, generate_baseline): monkeypatch.setenv("SGLANG_GEN_BASELINE", str(int(generate_baseline))) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 01cdb859bb..fc3993f33b 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -376,6 +376,7 @@ class Envs: SGLANG_IS_IN_CI = EnvBool(False) SGLANG_IS_IN_CI_AMD = EnvBool(False) SGLANG_TEST_MAX_RETRY = EnvInt(None) + SGLANG_TEST_TI2I_INPUT_IMAGE = EnvStr(None) # Expand jit_kernel test grids to their full parameter ranges (nightly). SGLANG_JIT_KERNEL_RUN_FULL_TESTS = EnvBool(False) SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK = EnvBool(False) diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.rs index 8c26c921bb..6bc8e1cd5a 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.rs @@ -535,6 +535,15 @@ pub struct MediaRef { /// Video: also decode the audio track. #[prost(bool, optional, tag = "14")] pub use_audio: ::core::option::Option, + /// GLM-Image external AR uses grids with input_ids to compute MRoPE positions. + #[prost(message, optional, tag = "15")] + pub image_grid_thw: ::core::option::Option, +} +/// One (t, h, w) row per image; the processor validates row arity. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ImageGridThw { + #[prost(message, repeated, tag = "1")] + pub rows: ::prost::alloc::vec::Vec, } /// One media item. JSON: a bare string is the source; an object is a MediaRef. #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs index d84aa80ab3..9c89b76a3c 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs @@ -2314,6 +2314,9 @@ impl ::serde::Serialize for MediaRef { if let Some(v) = &self.use_audio { ::serde::ser::SerializeMap::serialize_entry(&mut map, "use_audio", v)?; } + if let Some(v) = &self.image_grid_thw { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "image_grid_thw", v)?; + } ::serde::ser::SerializeMap::end(map) } } @@ -2335,6 +2338,7 @@ impl<'de> ::serde::Deserialize<'de> for MediaRef { "max_tokens_per_frame", "max_image_tokens", "use_audio", + "image_grid_thw", ]; struct V; impl<'de> ::serde::de::Visitor<'de> for V { @@ -2360,6 +2364,7 @@ impl<'de> ::serde::Deserialize<'de> for MediaRef { let mut f_max_tokens_per_frame = None; let mut f_max_image_tokens = None; let mut f_use_audio = None; + let mut f_image_grid_thw = None; while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { match key.as_ref() { "url" => { @@ -2456,6 +2461,12 @@ impl<'de> ::serde::Deserialize<'de> for MediaRef { } f_use_audio = Some(map.next_value::>()?); } + "image_grid_thw" => { + if f_image_grid_thw.is_some() { + return Err(::serde::de::Error::duplicate_field("image_grid_thw")); + } + f_image_grid_thw = Some(map.next_value::>()?); + } other => { return Err(::serde::de::Error::unknown_field(other, FIELDS)); } @@ -2476,6 +2487,7 @@ impl<'de> ::serde::Deserialize<'de> for MediaRef { max_tokens_per_frame: f_max_tokens_per_frame.flatten(), max_image_tokens: f_max_image_tokens.flatten(), use_audio: f_use_audio.flatten(), + image_grid_thw: f_image_grid_thw.flatten(), }) } } @@ -2483,6 +2495,22 @@ impl<'de> ::serde::Deserialize<'de> for MediaRef { } } +impl ::serde::Serialize for ImageGridThw { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.rows.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for ImageGridThw { + fn deserialize>(deserializer: D) -> Result { + Ok(ImageGridThw { + rows: <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + impl ::serde::Serialize for MediaItem { fn serialize(&self, serializer: S) -> Result { match &self.value { diff --git a/scripts/ci/npu/precise-test/analyze_failure_report.py b/scripts/ci/npu/precise-test/analyze_failure_report.py index ab760c4073..9a02023f38 100755 --- a/scripts/ci/npu/precise-test/analyze_failure_report.py +++ b/scripts/ci/npu/precise-test/analyze_failure_report.py @@ -9,12 +9,15 @@ Pipeline: a. TIMINGS JSON block (machine-readable, from ci_utils.py) b. ci_utils.py "✗ FAILED:" summary section (structured text) c. pytest "short test summary info" block (for pytest-style logs) - 2. Read recommended_pytest_paths.txt - 3. Match: exact match + file-level match - 4. Generate a Markdown report + 2. Collect diffusion consistency failures from diffusion-failures-* artifacts + (test_utils.py writes consistency_failures/summary.json per partition). + 3. Read recommended_pytest_paths.txt + 4. Match: exact match + file-level match + 5. Generate a Markdown report Usage: - python analyze_failure_report.py --log-dir LOG_DIR --recommendations-file RECOMMENDED.txt [--output report.md] + python analyze_failure_report.py --log-dir LOG_DIR --recommendations-file RECOMMENDED.txt \ + [--diffusion-dir DIFFUSION_DIR] [--output report.md] """ import argparse @@ -51,8 +54,11 @@ def clean_line(line): # ============================================================ -# Match pytest-style FAILED/ERROR lines with the test/ prefix (sglang convention). -FAILED_PATTERN = re.compile(r"^(?:FAILED|ERROR)\s+(test/\S+?\.py(?:::\S+?)?)\s") +# Match pytest-style FAILED/ERROR lines with a repo test path prefix. Diffusion +# tests live under python/sglang/multimodal_gen/test/, everything else under test/. +FAILED_PATTERN = re.compile( + r"^(?:FAILED|ERROR)\s+((?:test|sglang/multimodal_gen/test)/\S+?\.py(?:::\S+?)?)\s" +) SUMMARY_SEPARATOR_PATTERN = re.compile(r"^=+\s") CPU_LOG_PATH_PATTERN = re.compile(r"(?:^|-)cpu-\d+card(?:-|$)", re.IGNORECASE) CPU_FAILURE_LABEL = "cpu-ut" @@ -219,6 +225,71 @@ def extract_failed_from_logs(log_dir): return all_failed +# ============================================================ +# Step 1b: Extract diffusion consistency failures from +# diffusion-failures-* artifacts +# ============================================================ + + +def _dir_has_files(directory): + """Return whether a downloaded-artifact directory exists and holds any file.""" + base = Path(directory) + return base.is_dir() and any(item.is_file() for item in base.rglob("*")) + + +def extract_diffusion_failures(diffusion_dir): + """Collect consistency failure records from diffusion-failures-* artifacts. + + Artifacts are downloaded without merge-multiple, so every partition keeps + its own directory: //consistency_failures/summary.json. + Each summary is a list of records written by test_utils.py's + save_consistency_failure_artifact(). + + Returns a list of {case_id, num_gpus, metrics, thresholds, artifact} dicts, + deduplicated by case_id. + """ + base = Path(diffusion_dir) + if not base.is_dir(): + print(f"::notice:: Diffusion artifact directory not found: {diffusion_dir}") + return [] + + records = [] + seen = set() + for summary_path in sorted(base.rglob("summary.json")): + if summary_path.parent.name != "consistency_failures": + continue + try: + entries = json.loads(summary_path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + print(f"::warning:: Cannot read {summary_path}: {exc}") + continue + if not isinstance(entries, list): + continue + artifact = summary_path.parents[1].name + for entry in entries: + if not isinstance(entry, dict) or not entry.get("case_id"): + continue + case_id = entry["case_id"] + if case_id in seen: + continue + seen.add(case_id) + records.append( + { + "case_id": case_id, + "num_gpus": entry.get("num_gpus"), + "metrics": entry.get("metrics", {}), + "thresholds": entry.get("thresholds", {}), + "artifact": artifact, + } + ) + return records + + +def diffusion_failure_id(record): + """Failed-list identifier for one diffusion consistency failure record.""" + return f"diffusion-consistency/{record['case_id']}" + + # ============================================================ # Step 2: Read recommendations # ============================================================ @@ -309,13 +380,30 @@ def match_failed_vs_recommended(failed, recommended): # ============================================================ +def _format_metric(metrics, thresholds, metric_key, threshold_key): + """Render one metric as ``value (threshold X)``; n/a when absent.""" + value = metrics.get(metric_key) + if value is None: + return "n/a" + threshold = thresholds.get(threshold_key) + if threshold is None: + return f"{value}" + return f"{value} (threshold {threshold})" + + def generate_report( - failed, recommended, matched, log_dir, recommendations_source="none" + failed, + recommended, + matched, + log_dir, + recommendations_source="none", + diffusion_records=None, ): """Produce a Markdown summary table.""" hit = matched["hit"] miss = matched["miss"] untested = matched["untested"] + diffusion_records = diffusion_records or [] out = [] out.append("# Test Failure vs Recommendation Report") @@ -356,6 +444,39 @@ def generate_report( out.append("> No failed test cases") out.append("") + # ================================================================ + # Section 1b: Diffusion consistency failures + # ================================================================ + if diffusion_records: + out.append("---") + out.append("") + out.append( + f"## Diffusion Consistency Failures( {len(diffusion_records)} total)" + ) + out.append("") + out.append( + "> From `diffusion-failures-*` artifacts (sglang/multimodal_gen " + "consistency checks). These cases are outside the coverage " + "recommendation scope." + ) + out.append("") + out.append( + "| Case | GPUs | min CLIP | min SSIM | min PSNR | max mean abs diff | Artifact |" + ) + out.append("|---|---|---|---|---|---|---|") + for record in diffusion_records: + metrics = record["metrics"] + thresholds = record["thresholds"] + out.append( + f"| `{record['case_id']}` | {record['num_gpus']} " + f"| {_format_metric(metrics, thresholds, 'min_clip_similarity', 'clip_threshold')} " + f"| {_format_metric(metrics, thresholds, 'min_ssim', 'ssim_threshold')} " + f"| {_format_metric(metrics, thresholds, 'min_psnr', 'psnr_threshold')} " + f"| {_format_metric(metrics, thresholds, 'max_mean_abs_diff', 'mean_abs_diff_threshold')} " + f"| `{record['artifact']}` |" + ) + out.append("") + # ================================================================ # Section 2: Full Recommended Test List # ================================================================ @@ -457,6 +578,11 @@ def main(): parser.add_argument( "--log-dir", required=True, help="Directory containing CI .log files" ) + parser.add_argument( + "--diffusion-dir", + default=None, + help="Directory containing downloaded diffusion-failures-* artifacts", + ) parser.add_argument( "--recommendations-file", help="Path to recommended_pytest_paths.txt", @@ -486,7 +612,35 @@ def main(): print("Step 1: Extract failed tests from CI logs") print("=" * 50) failed = extract_failed_from_logs(args.log_dir) - print(f"Failed: {len(failed)}") + diffusion_records = [] + if args.diffusion_dir: + print() + print("=" * 50) + print("Step 1b: Extract diffusion consistency failures") + print("=" * 50) + diffusion_records = extract_diffusion_failures(args.diffusion_dir) + print(f"Diffusion consistency failures: {len(diffusion_records)}") + for record in diffusion_records: + failure_id = diffusion_failure_id(record) + if failure_id not in failed: + failed.append(failure_id) + + has_log_inputs = _dir_has_files(args.log_dir) + has_diffusion_inputs = bool(args.diffusion_dir) and _dir_has_files( + args.diffusion_dir + ) + if failed: + print(f"Failed: {len(failed)}") + elif not has_log_inputs and not has_diffusion_inputs: + # Empty result from empty input is not a clean-pass signal; say so + # instead of reporting a misleading 0. + print("Failed: 0 (no input artifacts found)") + print( + "::warning::No input artifacts found (no test logs, no diffusion " + "failures). This analysis has no data; it does not mean the run passed." + ) + else: + print("Failed: 0") print() print("=" * 50) @@ -505,7 +659,12 @@ def main(): print(f"Untested (recommended, no failure): {len(matched['untested'])}") report = generate_report( - failed, recommended, matched, args.log_dir, args.recommendations_source + failed, + recommended, + matched, + args.log_dir, + args.recommendations_source, + diffusion_records=diffusion_records, ) output_path = Path(args.output) output_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/scripts/ci/utils/diffusion/prefetch_input_image.py b/scripts/ci/utils/diffusion/prefetch_input_image.py new file mode 100755 index 0000000000..6049c087c9 --- /dev/null +++ b/scripts/ci/utils/diffusion/prefetch_input_image.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Cache the image-edit input before NPU CI starts loading models.""" + +import argparse +import hashlib +import os +import subprocess +import tempfile +from pathlib import Path + +IMAGE_URL = "https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg" +IMAGE_SHA256 = "0d56a63fca6ef8fe2a0ca27a0774c2bada9b2a198aeedbfdfb23bec09d0d3901" + + +def prefetch(cache_root: Path) -> Path: + cache_dir = cache_root.expanduser().resolve() / IMAGE_SHA256 + cache_dir.mkdir(parents=True, exist_ok=True) + image = cache_dir / "TI2I_Qwen_Image_Edit_Input.jpg" + if ( + image.is_file() + and hashlib.sha256(image.read_bytes()).hexdigest() == IMAGE_SHA256 + ): + return image + + # Unique staging files and atomic replacement allow concurrent partitions. + with tempfile.TemporaryDirectory(prefix="download-", dir=cache_dir) as staging: + downloaded = Path(staging) / image.name + subprocess.run( + [ + "curl", + "--fail", + "--location", + "--retry", + "2", + "--connect-timeout", + "30", + "--max-time", + "120", + "--output", + str(downloaded), + os.environ.get("GITHUB_PROXY_URL", "") + IMAGE_URL, + ], + check=True, + timeout=400, + ) + if hashlib.sha256(downloaded.read_bytes()).hexdigest() != IMAGE_SHA256: + raise ValueError("Image-edit input SHA-256 mismatch") + downloaded.replace(image) + return image + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--cache-root", type=Path, default=Path("~/.cache/sglang/inputs") + ) + args = parser.parse_args() + image = prefetch(args.cache_root) + line = f"SGLANG_TEST_TI2I_INPUT_IMAGE={image}" + if github_env := os.environ.get("GITHUB_ENV"): + with open(github_env, "a", encoding="utf-8") as output: + output.write(line + "\n") + print(line) + + +if __name__ == "__main__": + main() diff --git a/test/registered/unit/entrypoints/test_generate_api_contract.py b/test/registered/unit/entrypoints/test_generate_api_contract.py index c93741229d..22ae27fa2e 100644 --- a/test/registered/unit/entrypoints/test_generate_api_contract.py +++ b/test/registered/unit/entrypoints/test_generate_api_contract.py @@ -53,6 +53,29 @@ class TestGenerateContract(CustomTestCase): self.assertIsNotNone(error) self.assertIn("unknown field", error) + def test_glm_external_ar_grid_payload_is_accepted(self): + """GLM external AR sends grid metadata with pre-tokenized inputs.""" + for image_data in ( + # generate_prior_tokens: one request, one grid list + [{"image_grid_thw": [[1, 32, 32], [1, 64, 64]]}], + # generate_prior_tokens_batch: one grid list per batch entry + [ + [{"image_grid_thw": [[1, 32, 32], [1, 64, 64]]}], + [{"image_grid_thw": [[1, 32, 32], [1, 16, 16]]}], + ], + ): + with self.subTest(image_data=image_data): + self.assertIsNone( + generate_contract_error( + {"input_ids": [1, 2, 3], "image_data": image_data} + ) + ) + # A grid row of the wrong scalar type is still a contract error. + error = generate_contract_error( + {"input_ids": [1], "image_data": [{"image_grid_thw": [[1, "h", 32]]}]} + ) + self.assertIsNotNone(error) + def test_preprocessed_inputs_are_not_an_http_shape(self): """processor_output / precomputed_embedding carry tensors: Engine-only.""" error = generate_contract_error(