diff --git a/.github/workflows/pr-test-xpu.yml b/.github/workflows/pr-test-xpu.yml index d432697f7..88fe01566 100644 --- a/.github/workflows/pr-test-xpu.yml +++ b/.github/workflows/pr-test-xpu.yml @@ -251,7 +251,9 @@ jobs: - name: Run diffusion server tests (1-GPU) timeout-minutes: 60 run: | - docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/python && python3 sglang/multimodal_gen/test/run_suite.py --suite 1-gpu-xpu" + # xpu_b60.json is seeded with this flag on; without it the SYCL queue + # is deep enough that step timings record host enqueue, not device time. + docker exec -e SGLANG_DIFFUSION_SYNC_STAGE_PROFILING=1 ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/python && python3 sglang/multimodal_gen/test/run_suite.py --suite 1-gpu-xpu" - name: Cleanup container if: always() diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/xpu_b60.json b/python/sglang/multimodal_gen/test/server/perf_baselines/xpu_b60.json index 7bd8455ec..422eadcc5 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines/xpu_b60.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/xpu_b60.json @@ -2,8 +2,8 @@ "metadata": { "model": "Diffusion Server", "hardware": "CI Intel Arc Pro B60 (24 GiB) pool: bmg-multigen-models", - "description": "Reference numbers seeded from XPU multimodal_gen CI run 32736878259 job 97461821112 (PR #36100). wan2_1_t2v_1.3b re-seeded from run 35074312985 job 104723059209 (PR #36825); its expected_avg_denoise_ms still carries the step-0 lazy-init cost.", - "last_updated": "2026-09-17" + "description": "All scenarios re-seeded from XPU multimodal_gen CI run 35563559050 job 106220948594 (PR #39956), with SGLANG_DIFFUSION_SYNC_STAGE_PROFILING=1 so step timings record device time rather than host enqueue. Each value is the max over the 7 attempts of that job. Denoise numbers are stable across attempts (<=2% spread); expected_load_ms is host I/O bound and ranged 21.4-48.0s across attempts, so it is guarded only by the wide e2e tolerance.", + "last_updated": "2026-09-21" }, "tolerances": { "long_term": { @@ -42,181 +42,186 @@ "zimage_image_t2i": { "estimated_full_test_time_s": 65.1, "stages_ms": { - "InputValidationStage": 0.03, - "TextEncodingStage": 87.64, - "LatentPreparationStage": 0.21, - "TimestepPreparationStage": 136.9, - "DenoisingStage": 8361.1, - "DecodingStage": 8.39 + "InputValidationStage": 0.1, + "TextEncodingStage": 89.87, + "LatentPreparationStage": 0.28, + "TimestepPreparationStage": 2.44, + "DenoisingStage": 8614.3, + "DecodingStage": 293.13 }, "denoise_step_ms": { - "0": 99.57, - "1": 927.61, - "2": 926.72, - "3": 928.32, - "4": 929.29, - "5": 928.8, - "6": 925.2, - "7": 927.22, - "8": 931.28 + "0": 862.67, + "1": 859.76, + "2": 861.38, + "3": 860.12, + "4": 859.95, + "5": 859.53, + "6": 859.63, + "7": 860.1, + "8": 859.77 }, - "expected_e2e_ms": 9312.75, - "expected_avg_denoise_ms": 836.0, - "expected_median_denoise_ms": 927.61 + "expected_e2e_ms": 9799.49, + "expected_load_ms": 33283.41, + "expected_avg_denoise_ms": 860.1, + "expected_median_denoise_ms": 859.76 }, "flux_2_klein_image_t2i": { "estimated_full_test_time_s": 106.8, "stages_ms": { - "InputValidationStage": 0.04, - "TextEncodingStage": 37.83, - "ImageVAEEncodingStage": 0.0, - "LatentPreparationStage": 0.33, - "TimestepPreparationStage": 351.26, - "DenoisingStage": 2518.52, - "DecodingStage": 9.13 + "InputValidationStage": 0.11, + "TextEncodingStage": 88.26, + "ImageVAEEncodingStage": 0.06, + "LatentPreparationStage": 0.47, + "TimestepPreparationStage": 2.37, + "DenoisingStage": 2803.24, + "DecodingStage": 292.44 }, "denoise_step_ms": { - "0": 83.44, - "1": 16.73, - "2": 18.04, - "3": 17.11 + "0": 591.79, + "1": 536.64, + "2": 537.71, + "3": 537.7 }, - "expected_e2e_ms": 3509.48, - "expected_avg_denoise_ms": 33.83, - "expected_median_denoise_ms": 17.58 + "expected_e2e_ms": 3754.98, + "expected_load_ms": 26566.83, + "expected_avg_denoise_ms": 550.96, + "expected_median_denoise_ms": 537.7 }, "flux_2_klein_base_image_t2i": { "stages_ms": { - "InputValidationStage": 0.03, - "TextEncodingStage": 36.3, - "ImageVAEEncodingStage": 0.0, - "LatentPreparationStage": 0.28, - "TimestepPreparationStage": 355.5, - "DenoisingStage": 55237.43, - "DecodingStage": 11.09 + "InputValidationStage": 0.11, + "TextEncodingStage": 87.34, + "ImageVAEEncodingStage": 0.06, + "LatentPreparationStage": 0.44, + "TimestepPreparationStage": 2.49, + "DenoisingStage": 54080.81, + "DecodingStage": 287.66 }, "denoise_step_ms": { - "0": 106.73, - "1": 34.25, - "2": 34.59, - "3": 34.59, - "4": 35.08, - "5": 34.24, - "6": 34.03, - "7": 34.47, - "8": 34.17, - "9": 44.04, - "10": 34.83, - "11": 34.64, - "12": 35.38, - "13": 34.32, - "14": 35.51, - "15": 36.16, - "16": 35.32, - "17": 34.75, - "18": 35.28, - "19": 50.88, - "20": 35.41, - "21": 35.36, - "22": 34.53, - "23": 35.42, - "24": 34.75, - "25": 34.78, - "26": 34.95, - "27": 34.84, - "28": 35.38, - "29": 34.7, - "30": 34.42, - "31": 35.01, - "32": 34.98, - "33": 34.61, - "34": 34.78, - "35": 34.76, - "36": 34.61, - "37": 37.59, - "38": 35.0, - "39": 34.6, - "40": 35.52, - "41": 35.14, - "42": 34.84, - "43": 35.16, - "44": 33.86, - "45": 34.52, - "46": 34.94, - "47": 33.84, - "48": 35.14, - "49": 34.75 + "0": 1123.84, + "1": 1068.88, + "2": 1068.05, + "3": 1067.09, + "4": 1068.74, + "5": 1068.64, + "6": 1067.83, + "7": 1068.84, + "8": 1068.23, + "9": 1068.88, + "10": 1067.99, + "11": 1068.93, + "12": 1069.2, + "13": 1067.82, + "14": 1068.2, + "15": 1068.84, + "16": 1068.15, + "17": 1067.99, + "18": 1068.51, + "19": 1068.77, + "20": 1068.34, + "21": 1068.12, + "22": 1069.19, + "23": 1068.35, + "24": 1069.71, + "25": 1068.26, + "26": 1068.36, + "27": 1069.13, + "28": 1068.15, + "29": 1068.11, + "30": 1069.23, + "31": 1069.6, + "32": 1069.44, + "33": 1068.34, + "34": 1068.38, + "35": 1069.22, + "36": 1068.4, + "37": 1067.92, + "38": 1068.22, + "39": 1068.85, + "40": 1068.95, + "41": 1069.67, + "42": 1068.07, + "43": 1068.93, + "44": 1069.75, + "45": 1067.77, + "46": 1068.09, + "47": 1068.11, + "48": 1068.67, + "49": 1068.8 }, - "expected_e2e_ms": 56248.35, - "expected_avg_denoise_ms": 36.83, - "expected_median_denoise_ms": 34.84 + "expected_e2e_ms": 55028.04, + "expected_load_ms": 31539.71, + "expected_avg_denoise_ms": 1069.67, + "expected_median_denoise_ms": 1068.45, + "estimated_full_test_time_s": 103.3 }, "wan2_1_t2v_1.3b": { "estimated_full_test_time_s": 331.5, "stages_ms": { - "InputValidationStage": 0.05, - "TextEncodingStage": 761.88, - "LatentPreparationStage": 1.12, - "TimestepPreparationStage": 1.66, - "DenoisingStage": 73363.89, - "DecodingStage": 295.76, + "InputValidationStage": 0.11, + "TextEncodingStage": 764.42, + "LatentPreparationStage": 1.26, + "TimestepPreparationStage": 0.97, + "DenoisingStage": 70658.77, + "DecodingStage": 3102.93, "per_frame_generation": null }, "denoise_step_ms": { - "0": 3421.66, - "1": 103.21, - "2": 101.69, - "3": 102.97, - "4": 102.86, - "5": 104.51, - "6": 103.23, - "7": 102.74, - "8": 101.86, - "9": 102.28, - "10": 103.18, - "11": 105.43, - "12": 103.01, - "13": 102.33, - "14": 101.37, - "15": 102.32, - "16": 102.29, - "17": 103.88, - "18": 103.13, - "19": 103.36, - "20": 101.79, - "21": 101.59, - "22": 103.18, - "23": 106.52, - "24": 104.18, - "25": 103.53, - "26": 104.71, - "27": 103.92, - "28": 104.26, - "29": 105.03, - "30": 104.62, - "31": 104.0, - "32": 104.37, - "33": 103.74, - "34": 104.16, - "35": 105.51, - "36": 102.68, - "37": 101.05, - "38": 103.52, - "39": 101.97, - "40": 102.62, - "41": 108.73, - "42": 102.88, - "43": 102.2, - "44": 102.84, - "45": 102.31, - "46": 103.4, - "47": 102.44, - "48": 106.32, - "49": 101.64 + "0": 1677.96, + "1": 1401.03, + "2": 1400.7, + "3": 1401.14, + "4": 1400.29, + "5": 1401.15, + "6": 1400.76, + "7": 1400.83, + "8": 1399.92, + "9": 1401.37, + "10": 1400.79, + "11": 1401.01, + "12": 1400.59, + "13": 1400.05, + "14": 1400.17, + "15": 1400.44, + "16": 1400.18, + "17": 1400.41, + "18": 1399.86, + "19": 1400.23, + "20": 1400.58, + "21": 1400.54, + "22": 1400.3, + "23": 1400.26, + "24": 1400.32, + "25": 1400.66, + "26": 1400.31, + "27": 1400.01, + "28": 1400.15, + "29": 1400.61, + "30": 1400.58, + "31": 1400.79, + "32": 1400.66, + "33": 1400.92, + "34": 1401.43, + "35": 1400.94, + "36": 1400.91, + "37": 1400.93, + "38": 1401.66, + "39": 1400.7, + "40": 1400.75, + "41": 1401.24, + "42": 1400.85, + "43": 1400.86, + "44": 1400.98, + "45": 1400.81, + "46": 1400.27, + "47": 1400.9, + "48": 1400.3, + "49": 1400.39 }, - "expected_e2e_ms": 75344.43, - "expected_avg_denoise_ms": 169.74, - "expected_median_denoise_ms": 103.18 + "expected_e2e_ms": 75427.76, + "expected_load_ms": 48009.32, + "expected_avg_denoise_ms": 1406.13, + "expected_median_denoise_ms": 1400.58 } } }