[ci][xpu] Record device time in the multimodal_gen perf lane (#39956)
This commit is contained in:
ashwini rathi
@@ -251,7 +251,9 @@ jobs:
|
|||||||
- name: Run diffusion server tests (1-GPU)
|
- name: Run diffusion server tests (1-GPU)
|
||||||
timeout-minutes: 60
|
timeout-minutes: 60
|
||||||
run: |
|
run: |
|
||||||
docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/python && python3 sglang/multimodal_gen/test/run_suite.py --suite 1-gpu-xpu"
|
# xpu_b60.json is seeded with this flag on; without it the SYCL queue
|
||||||
|
# is deep enough that step timings record host enqueue, not device time.
|
||||||
|
docker exec -e SGLANG_DIFFUSION_SYNC_STAGE_PROFILING=1 ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/python && python3 sglang/multimodal_gen/test/run_suite.py --suite 1-gpu-xpu"
|
||||||
|
|
||||||
- name: Cleanup container
|
- name: Cleanup container
|
||||||
if: always()
|
if: always()
|
||||||
|
|||||||
@@ -2,8 +2,8 @@
|
|||||||
"metadata": {
|
"metadata": {
|
||||||
"model": "Diffusion Server",
|
"model": "Diffusion Server",
|
||||||
"hardware": "CI Intel Arc Pro B60 (24 GiB) pool: bmg-multigen-models",
|
"hardware": "CI Intel Arc Pro B60 (24 GiB) pool: bmg-multigen-models",
|
||||||
"description": "Reference numbers seeded from XPU multimodal_gen CI run 32736878259 job 97461821112 (PR #36100). wan2_1_t2v_1.3b re-seeded from run 35074312985 job 104723059209 (PR #36825); its expected_avg_denoise_ms still carries the step-0 lazy-init cost.",
|
"description": "All scenarios re-seeded from XPU multimodal_gen CI run 35563559050 job 106220948594 (PR #39956), with SGLANG_DIFFUSION_SYNC_STAGE_PROFILING=1 so step timings record device time rather than host enqueue. Each value is the max over the 7 attempts of that job. Denoise numbers are stable across attempts (<=2% spread); expected_load_ms is host I/O bound and ranged 21.4-48.0s across attempts, so it is guarded only by the wide e2e tolerance.",
|
||||||
"last_updated": "2026-09-17"
|
"last_updated": "2026-09-21"
|
||||||
},
|
},
|
||||||
"tolerances": {
|
"tolerances": {
|
||||||
"long_term": {
|
"long_term": {
|
||||||
@@ -42,181 +42,186 @@
|
|||||||
"zimage_image_t2i": {
|
"zimage_image_t2i": {
|
||||||
"estimated_full_test_time_s": 65.1,
|
"estimated_full_test_time_s": 65.1,
|
||||||
"stages_ms": {
|
"stages_ms": {
|
||||||
"InputValidationStage": 0.03,
|
"InputValidationStage": 0.1,
|
||||||
"TextEncodingStage": 87.64,
|
"TextEncodingStage": 89.87,
|
||||||
"LatentPreparationStage": 0.21,
|
"LatentPreparationStage": 0.28,
|
||||||
"TimestepPreparationStage": 136.9,
|
"TimestepPreparationStage": 2.44,
|
||||||
"DenoisingStage": 8361.1,
|
"DenoisingStage": 8614.3,
|
||||||
"DecodingStage": 8.39
|
"DecodingStage": 293.13
|
||||||
},
|
},
|
||||||
"denoise_step_ms": {
|
"denoise_step_ms": {
|
||||||
"0": 99.57,
|
"0": 862.67,
|
||||||
"1": 927.61,
|
"1": 859.76,
|
||||||
"2": 926.72,
|
"2": 861.38,
|
||||||
"3": 928.32,
|
"3": 860.12,
|
||||||
"4": 929.29,
|
"4": 859.95,
|
||||||
"5": 928.8,
|
"5": 859.53,
|
||||||
"6": 925.2,
|
"6": 859.63,
|
||||||
"7": 927.22,
|
"7": 860.1,
|
||||||
"8": 931.28
|
"8": 859.77
|
||||||
},
|
},
|
||||||
"expected_e2e_ms": 9312.75,
|
"expected_e2e_ms": 9799.49,
|
||||||
"expected_avg_denoise_ms": 836.0,
|
"expected_load_ms": 33283.41,
|
||||||
"expected_median_denoise_ms": 927.61
|
"expected_avg_denoise_ms": 860.1,
|
||||||
|
"expected_median_denoise_ms": 859.76
|
||||||
},
|
},
|
||||||
"flux_2_klein_image_t2i": {
|
"flux_2_klein_image_t2i": {
|
||||||
"estimated_full_test_time_s": 106.8,
|
"estimated_full_test_time_s": 106.8,
|
||||||
"stages_ms": {
|
"stages_ms": {
|
||||||
"InputValidationStage": 0.04,
|
"InputValidationStage": 0.11,
|
||||||
"TextEncodingStage": 37.83,
|
"TextEncodingStage": 88.26,
|
||||||
"ImageVAEEncodingStage": 0.0,
|
"ImageVAEEncodingStage": 0.06,
|
||||||
"LatentPreparationStage": 0.33,
|
"LatentPreparationStage": 0.47,
|
||||||
"TimestepPreparationStage": 351.26,
|
"TimestepPreparationStage": 2.37,
|
||||||
"DenoisingStage": 2518.52,
|
"DenoisingStage": 2803.24,
|
||||||
"DecodingStage": 9.13
|
"DecodingStage": 292.44
|
||||||
},
|
},
|
||||||
"denoise_step_ms": {
|
"denoise_step_ms": {
|
||||||
"0": 83.44,
|
"0": 591.79,
|
||||||
"1": 16.73,
|
"1": 536.64,
|
||||||
"2": 18.04,
|
"2": 537.71,
|
||||||
"3": 17.11
|
"3": 537.7
|
||||||
},
|
},
|
||||||
"expected_e2e_ms": 3509.48,
|
"expected_e2e_ms": 3754.98,
|
||||||
"expected_avg_denoise_ms": 33.83,
|
"expected_load_ms": 26566.83,
|
||||||
"expected_median_denoise_ms": 17.58
|
"expected_avg_denoise_ms": 550.96,
|
||||||
|
"expected_median_denoise_ms": 537.7
|
||||||
},
|
},
|
||||||
"flux_2_klein_base_image_t2i": {
|
"flux_2_klein_base_image_t2i": {
|
||||||
"stages_ms": {
|
"stages_ms": {
|
||||||
"InputValidationStage": 0.03,
|
"InputValidationStage": 0.11,
|
||||||
"TextEncodingStage": 36.3,
|
"TextEncodingStage": 87.34,
|
||||||
"ImageVAEEncodingStage": 0.0,
|
"ImageVAEEncodingStage": 0.06,
|
||||||
"LatentPreparationStage": 0.28,
|
"LatentPreparationStage": 0.44,
|
||||||
"TimestepPreparationStage": 355.5,
|
"TimestepPreparationStage": 2.49,
|
||||||
"DenoisingStage": 55237.43,
|
"DenoisingStage": 54080.81,
|
||||||
"DecodingStage": 11.09
|
"DecodingStage": 287.66
|
||||||
},
|
},
|
||||||
"denoise_step_ms": {
|
"denoise_step_ms": {
|
||||||
"0": 106.73,
|
"0": 1123.84,
|
||||||
"1": 34.25,
|
"1": 1068.88,
|
||||||
"2": 34.59,
|
"2": 1068.05,
|
||||||
"3": 34.59,
|
"3": 1067.09,
|
||||||
"4": 35.08,
|
"4": 1068.74,
|
||||||
"5": 34.24,
|
"5": 1068.64,
|
||||||
"6": 34.03,
|
"6": 1067.83,
|
||||||
"7": 34.47,
|
"7": 1068.84,
|
||||||
"8": 34.17,
|
"8": 1068.23,
|
||||||
"9": 44.04,
|
"9": 1068.88,
|
||||||
"10": 34.83,
|
"10": 1067.99,
|
||||||
"11": 34.64,
|
"11": 1068.93,
|
||||||
"12": 35.38,
|
"12": 1069.2,
|
||||||
"13": 34.32,
|
"13": 1067.82,
|
||||||
"14": 35.51,
|
"14": 1068.2,
|
||||||
"15": 36.16,
|
"15": 1068.84,
|
||||||
"16": 35.32,
|
"16": 1068.15,
|
||||||
"17": 34.75,
|
"17": 1067.99,
|
||||||
"18": 35.28,
|
"18": 1068.51,
|
||||||
"19": 50.88,
|
"19": 1068.77,
|
||||||
"20": 35.41,
|
"20": 1068.34,
|
||||||
"21": 35.36,
|
"21": 1068.12,
|
||||||
"22": 34.53,
|
"22": 1069.19,
|
||||||
"23": 35.42,
|
"23": 1068.35,
|
||||||
"24": 34.75,
|
"24": 1069.71,
|
||||||
"25": 34.78,
|
"25": 1068.26,
|
||||||
"26": 34.95,
|
"26": 1068.36,
|
||||||
"27": 34.84,
|
"27": 1069.13,
|
||||||
"28": 35.38,
|
"28": 1068.15,
|
||||||
"29": 34.7,
|
"29": 1068.11,
|
||||||
"30": 34.42,
|
"30": 1069.23,
|
||||||
"31": 35.01,
|
"31": 1069.6,
|
||||||
"32": 34.98,
|
"32": 1069.44,
|
||||||
"33": 34.61,
|
"33": 1068.34,
|
||||||
"34": 34.78,
|
"34": 1068.38,
|
||||||
"35": 34.76,
|
"35": 1069.22,
|
||||||
"36": 34.61,
|
"36": 1068.4,
|
||||||
"37": 37.59,
|
"37": 1067.92,
|
||||||
"38": 35.0,
|
"38": 1068.22,
|
||||||
"39": 34.6,
|
"39": 1068.85,
|
||||||
"40": 35.52,
|
"40": 1068.95,
|
||||||
"41": 35.14,
|
"41": 1069.67,
|
||||||
"42": 34.84,
|
"42": 1068.07,
|
||||||
"43": 35.16,
|
"43": 1068.93,
|
||||||
"44": 33.86,
|
"44": 1069.75,
|
||||||
"45": 34.52,
|
"45": 1067.77,
|
||||||
"46": 34.94,
|
"46": 1068.09,
|
||||||
"47": 33.84,
|
"47": 1068.11,
|
||||||
"48": 35.14,
|
"48": 1068.67,
|
||||||
"49": 34.75
|
"49": 1068.8
|
||||||
},
|
},
|
||||||
"expected_e2e_ms": 56248.35,
|
"expected_e2e_ms": 55028.04,
|
||||||
"expected_avg_denoise_ms": 36.83,
|
"expected_load_ms": 31539.71,
|
||||||
"expected_median_denoise_ms": 34.84
|
"expected_avg_denoise_ms": 1069.67,
|
||||||
|
"expected_median_denoise_ms": 1068.45,
|
||||||
|
"estimated_full_test_time_s": 103.3
|
||||||
},
|
},
|
||||||
"wan2_1_t2v_1.3b": {
|
"wan2_1_t2v_1.3b": {
|
||||||
"estimated_full_test_time_s": 331.5,
|
"estimated_full_test_time_s": 331.5,
|
||||||
"stages_ms": {
|
"stages_ms": {
|
||||||
"InputValidationStage": 0.05,
|
"InputValidationStage": 0.11,
|
||||||
"TextEncodingStage": 761.88,
|
"TextEncodingStage": 764.42,
|
||||||
"LatentPreparationStage": 1.12,
|
"LatentPreparationStage": 1.26,
|
||||||
"TimestepPreparationStage": 1.66,
|
"TimestepPreparationStage": 0.97,
|
||||||
"DenoisingStage": 73363.89,
|
"DenoisingStage": 70658.77,
|
||||||
"DecodingStage": 295.76,
|
"DecodingStage": 3102.93,
|
||||||
"per_frame_generation": null
|
"per_frame_generation": null
|
||||||
},
|
},
|
||||||
"denoise_step_ms": {
|
"denoise_step_ms": {
|
||||||
"0": 3421.66,
|
"0": 1677.96,
|
||||||
"1": 103.21,
|
"1": 1401.03,
|
||||||
"2": 101.69,
|
"2": 1400.7,
|
||||||
"3": 102.97,
|
"3": 1401.14,
|
||||||
"4": 102.86,
|
"4": 1400.29,
|
||||||
"5": 104.51,
|
"5": 1401.15,
|
||||||
"6": 103.23,
|
"6": 1400.76,
|
||||||
"7": 102.74,
|
"7": 1400.83,
|
||||||
"8": 101.86,
|
"8": 1399.92,
|
||||||
"9": 102.28,
|
"9": 1401.37,
|
||||||
"10": 103.18,
|
"10": 1400.79,
|
||||||
"11": 105.43,
|
"11": 1401.01,
|
||||||
"12": 103.01,
|
"12": 1400.59,
|
||||||
"13": 102.33,
|
"13": 1400.05,
|
||||||
"14": 101.37,
|
"14": 1400.17,
|
||||||
"15": 102.32,
|
"15": 1400.44,
|
||||||
"16": 102.29,
|
"16": 1400.18,
|
||||||
"17": 103.88,
|
"17": 1400.41,
|
||||||
"18": 103.13,
|
"18": 1399.86,
|
||||||
"19": 103.36,
|
"19": 1400.23,
|
||||||
"20": 101.79,
|
"20": 1400.58,
|
||||||
"21": 101.59,
|
"21": 1400.54,
|
||||||
"22": 103.18,
|
"22": 1400.3,
|
||||||
"23": 106.52,
|
"23": 1400.26,
|
||||||
"24": 104.18,
|
"24": 1400.32,
|
||||||
"25": 103.53,
|
"25": 1400.66,
|
||||||
"26": 104.71,
|
"26": 1400.31,
|
||||||
"27": 103.92,
|
"27": 1400.01,
|
||||||
"28": 104.26,
|
"28": 1400.15,
|
||||||
"29": 105.03,
|
"29": 1400.61,
|
||||||
"30": 104.62,
|
"30": 1400.58,
|
||||||
"31": 104.0,
|
"31": 1400.79,
|
||||||
"32": 104.37,
|
"32": 1400.66,
|
||||||
"33": 103.74,
|
"33": 1400.92,
|
||||||
"34": 104.16,
|
"34": 1401.43,
|
||||||
"35": 105.51,
|
"35": 1400.94,
|
||||||
"36": 102.68,
|
"36": 1400.91,
|
||||||
"37": 101.05,
|
"37": 1400.93,
|
||||||
"38": 103.52,
|
"38": 1401.66,
|
||||||
"39": 101.97,
|
"39": 1400.7,
|
||||||
"40": 102.62,
|
"40": 1400.75,
|
||||||
"41": 108.73,
|
"41": 1401.24,
|
||||||
"42": 102.88,
|
"42": 1400.85,
|
||||||
"43": 102.2,
|
"43": 1400.86,
|
||||||
"44": 102.84,
|
"44": 1400.98,
|
||||||
"45": 102.31,
|
"45": 1400.81,
|
||||||
"46": 103.4,
|
"46": 1400.27,
|
||||||
"47": 102.44,
|
"47": 1400.9,
|
||||||
"48": 106.32,
|
"48": 1400.3,
|
||||||
"49": 101.64
|
"49": 1400.39
|
||||||
},
|
},
|
||||||
"expected_e2e_ms": 75344.43,
|
"expected_e2e_ms": 75427.76,
|
||||||
"expected_avg_denoise_ms": 169.74,
|
"expected_load_ms": 48009.32,
|
||||||
"expected_median_denoise_ms": 103.18
|
"expected_avg_denoise_ms": 1406.13,
|
||||||
|
"expected_median_denoise_ms": 1400.58
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user