[NPU CI] Reorganize test output/log directory structure with workflow context (#33685)

2026-08-18 23:46:00 +08:00
parent 63d783bbe0
commit 499e90a125
139 changed files with 779 additions and 688 deletions
@@ -60,6 +60,12 @@ spec:
            value: "https://hf-mirror.com"
          - name: SGLANG_IS_IN_CI
            value: "{{ sglang_is_in_ci }}"
+          - name: RUN_LABEL
+            value: "{{ run_label }}"
+          - name: TROUBLE_SHOTTING
+            value: "{{ trouble_shotting }}"
+          - name: TRANSFORMERS_VERSION_FOR_SGLANG
+            value: "{{ transformers_version }}"
          command: ["/bin/bash", "-c"]
          args:
            - |
@@ -52,6 +52,8 @@ spec:
          value: "https://hf-mirror.com"
        - name: METRICS_DATA_FILE
          value: "{{ metrics_data_file }}"
+        - name: RUN_LABEL
+          value: "{{ run_label }}"
        - name: SGLANG_IS_IN_CI
          value: "{{ sglang_is_in_ci }}"
        - name: TRANSFORMERS_VERSION_FOR_SGLANG
@@ -60,6 +60,10 @@ spec:
            value: "{{ metrics_data_file }}"
          - name: SGLANG_IS_IN_CI
            value: "{{ sglang_is_in_ci }}"
+          - name: RUN_LABEL
+            value: "{{ run_label }}"
+          - name: TROUBLE_SHOTTING
+            value: "{{ trouble_shotting }}"
          - name: TRANSFORMERS_VERSION_FOR_SGLANG
            value: "{{ transformers_version }}"
          command: ["/bin/bash", "-c"]
@@ -172,6 +176,10 @@ spec:
            value: "{{ metrics_data_file }}"
          - name: SGLANG_IS_IN_CI
            value: "{{ sglang_is_in_ci }}"
+          - name: RUN_LABEL
+            value: "{{ run_label }}"
+          - name: TROUBLE_SHOTTING
+            value: "{{ trouble_shotting }}"
          - name: TRANSFORMERS_VERSION_FOR_SGLANG
            value: "{{ transformers_version }}"
          command: ["/bin/bash", "-c"]
@@ -274,6 +282,8 @@ spec:
            value: "{{ install_sglang_from_source }}"
          - name: METRICS_DATA_FILE
            value: "{{ metrics_data_file }}"
+          - name: RUN_LABEL
+            value: "{{ run_label }}"
          - name: KUBECONFIG
            value: "{{ kube_config }}"
          - name: NAMESPACE
@@ -284,6 +294,8 @@ spec:
            value: "https://hf-mirror.com"
          - name: SGLANG_IS_IN_CI
            value: "{{ sglang_is_in_ci }}"
+          - name: TROUBLE_SHOTTING
+            value: "{{ trouble_shotting }}"
          - name: TRANSFORMERS_VERSION_FOR_SGLANG
            value: "{{ transformers_version }}"
          command: ["/bin/bash", "-c"]
@@ -53,6 +53,8 @@ spec:
          value: "https://hf-mirror.com"
        - name: METRICS_DATA_FILE
          value: "{{ metrics_data_file }}"
+        - name: RUN_LABEL
+          value: "{{ run_label }}"
        - name: SGLANG_IS_IN_CI
          value: "{{ sglang_is_in_ci }}"
        command: ["/bin/bash", "-c"]
@@ -154,6 +156,8 @@ spec:
          value: "https://hf-mirror.com"
        - name: METRICS_DATA_FILE
          value: "{{ metrics_data_file }}"
+        - name: RUN_LABEL
+          value: "{{ run_label }}"
        - name: SGLANG_IS_IN_CI
          value: "{{ sglang_is_in_ci }}"
        command: ["/bin/bash", "-c"]
@@ -254,6 +258,8 @@ spec:
          value: "https://hf-mirror.com"
        - name: METRICS_DATA_FILE
          value: "{{ metrics_data_file }}"
+        - name: RUN_LABEL
+          value: "{{ run_label }}"
        - name: SGLANG_IS_IN_CI
          value: "{{ sglang_is_in_ci }}"
        command: ["/bin/bash", "-c"]
@@ -51,8 +51,13 @@ spec:
            value: "{{ kube_config }}"
          - name: HF_ENDPOINT
            value: "https://hf-mirror.com"
+          # Env vars consumed by run_npu_testcase.sh inside the pod
          - name: TROUBLE_SHOTTING
            value: "{{ trouble_shotting }}"
+          - name: RUN_LABEL
+            value: "{{ run_label }}"
+          - name: SGLANG_IS_IN_CI
+            value: "{{ sglang_is_in_ci }}"
          - name: TRANSFORMERS_VERSION_FOR_SGLANG
            value: "{{ transformers_version }}"
          command: ["/bin/bash", "-c"]
@@ -518,10 +518,17 @@ def generate_metrics_json(metrics_data_file, test_case, status):
    tc_name = test_case.rsplit("/", 1)[-1].rsplit(".", 1)[0]

    test_type = "unknown"
+    # nightly: .../output/{branch}-{date}-{run_id}-{run_attempt}/{workflow}/{test_type}/...
+    # PR:      .../output/{test_type}/{date}/{tc_name}
+    # After `output`, rest >= 4 segments -> test_type at parts[i+3]; otherwise at parts[i+1].
    parts = metrics_data_file.split("/")
    for i, part in enumerate(parts):
-        if part == "output" and i + 1 < len(parts):
-            test_type = parts[i + 1]
+        if part == "output":
+            rest = len(parts) - (i + 1)
+            if rest >= 4:
+                test_type = parts[i + 3]
+            elif rest >= 1:
+                test_type = parts[i + 1]
            break

    output = {
@@ -576,6 +583,22 @@ def run_npu_e2e_test_case(

    kube_config_map = f"sglang-configmap-{random_str}"
    final_kube_job_name = f"{kube_job_name_prefix}-{random_str}"
+    # run_label is injected into the pod as RUN_LABEL to build the pod log directory prefix.
+    # nightly (>=4 segments after `output`): first two segments {branch}-{date}-{run_id}-{run_attempt}/{workflow}
+    # PR legacy layout: fall back to the date segment to keep the original {date}/{tc_name}/{host} path.
+    parts = (
+        metrics_data_file.split("/output/")[-1]
+        if "/output/" in metrics_data_file
+        else ""
+    )
+    if parts:
+        segments = parts.split("/")
+        if len(segments) >= 4:
+            run_label = "/".join(segments[:2])
+        else:
+            run_label = segments[1] if len(segments) > 1 else "unknown"
+    else:
+        run_label = "unknown"

    kube_yaml_file_dict = {
        KUBE_JOB_SINGLE: f"k8s_single_{random_str}.yaml",
@@ -605,6 +628,7 @@ def run_npu_e2e_test_case(
                "env": env,
                "trouble_shotting": trouble_shotting,
                "transformers_version": transformers_version,
+                "run_label": run_label,
            }
            create_kube_yaml(
                kube_yaml_template=KUBE_YAML_TEMPLATE.get(kube_job_type),
@@ -627,6 +651,7 @@ def run_npu_e2e_test_case(
                "env": env,
                "trouble_shotting": trouble_shotting,
                "transformers_version": transformers_version,
+                "run_label": run_label,
            }
            template_key = (
                KUBE_JOB_MULTI_PD_MIX_GREEN if env == "green" else kube_job_type
@@ -654,6 +679,7 @@ def run_npu_e2e_test_case(
                "env": env,
                "trouble_shotting": trouble_shotting,
                "transformers_version": transformers_version,
+                "run_label": run_label,
            }
            template_key = (
                KUBE_JOB_MULTI_PD_SEPARATION_GREEN if env == "green" else kube_job_type
@@ -34,7 +34,7 @@ if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then
    echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally."
    TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
    mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}"
-    cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/"
+    cp "${TRANSFORMERS_PKG_PATH_SOURCE}/"* "${TRANSFORMERS_PKG_PATH_TARGET}/"
    pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}"
  fi
  echo "===== Install transformers for sglang in virtual env - End ====="
@@ -114,10 +114,11 @@ fi
 echo "Running test case ${test_case}"
 tc_name=${test_case##*/}
 tc_name=${tc_name%.*}
-current_date=$(date +%Y%m%d)
-log_path="/root/sglang/debug/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
+run_label="${RUN_LABEL:-unknown}"
+log_path="/root/sglang/debug/logs/log/${run_label}/${tc_name}/${HOSTNAME}"
 if [ "${SGLANG_IS_IN_CI}" = "true" ] || [ "${SGLANG_IS_IN_CI}" = "True" ];then
-    log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
+    # In CI, persist logs under /root/.cache/tests/logs so they can be collected
+    log_path="/root/.cache/tests/logs/log/${run_label}/${tc_name}/${HOSTNAME}"
 fi
 rm -rf "${log_path}"
 mkdir -p "${log_path}"
@@ -134,6 +135,7 @@ echo "Finished test case ${test_case}"

 if [ -n "${METRICS_DATA_FILE}" ]; then
    mkdir -p "${METRICS_DATA_FILE}"
+    # Archive the test log into the output directory for result collection
    cp "${log_path}/${tc_name}.log" "${METRICS_DATA_FILE}/test_output.log"
    echo "Metrics log saved to ${METRICS_DATA_FILE}/test_output.log"
 fi
@@ -311,16 +311,27 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
    def _setup_per_case_output(cls):
        """Set up per-case output directories and env vars.

-        Extracted from ``nightly-test-npu-e2e-single-node.yml`` so that when a
-        suite is executed, each case writes its metrics/plog to a path derived
-        from the case file rather than the suite name.
+        When the workflow sets METRICS_DATA_FILE to a suite-level directory
+        (e.g. .../output/{branch_label}-{create_date}-{run_id}-{run_attempt}/
+        {workflow_name}/{test_type}/{suite}), each case in the suite
+        writes to its own subdirectory under it, so results stay in the
+        structured layout and are keyed by the case id. Falls back to the
+        legacy per-case layout when the env var is not set.
        """
        cls.tc_name = cls._get_tc_name()
-        current_date = datetime.now().strftime("%Y%m%d")
-        test_type = getattr(cls, "test_type", "accuracy")
-        base_output = f"/root/.cache/tests/output/{test_type}/{current_date}"
-        os.makedirs(base_output, exist_ok=True)
-        cls.metrics_data_file = os.path.join(base_output, cls.tc_name)
+        suite_output = os.environ.get("METRICS_DATA_FILE")
+        if suite_output:
+            # Append the case id under the suite output prefix.
+            cls.metrics_data_file = os.path.join(suite_output, cls.tc_name)
+            # Mirror the output prefix to the plog location (drop the test_type/suite tail).
+            suite_plog = suite_output.replace("/output/", "/logs/plog/", 1)
+            cls.plog_base = os.path.dirname(os.path.dirname(suite_plog))
+        else:
+            current_date = datetime.now().strftime("%Y%m%d")
+            test_type = getattr(cls, "test_type", "accuracy")
+            base_output = f"/root/.cache/tests/output/{test_type}/{current_date}"
+            cls.metrics_data_file = os.path.join(base_output, cls.tc_name)
+            cls.plog_base = f"/root/.cache/tests/logs/plog"
        os.makedirs(cls.metrics_data_file, exist_ok=True)
        # Override env vars so evalscope/dump_metric write to per-case paths.
        os.environ["METRICS_DATA_FILE"] = cls.metrics_data_file
@@ -376,6 +387,12 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
            logger.info("Saved per-case metrics to %s", out_path)
        except Exception as e:
            logger.warning("Failed to write metrics.json: %s", e)
+        # Remove the intermediate JSONL records, keeping only the final metrics.json.
+        for jsonl_path in glob.glob(pattern):
+            try:
+                os.remove(jsonl_path)
+            except Exception as e:
+                logger.warning("Failed to remove %s: %s", jsonl_path, e)

    @classmethod
    def _backup_plog(cls):
@@ -391,7 +408,8 @@ class TestNpuAccuracyTestCaseBase(CustomTestCase):
        if not tc_name:
            return
        hostname = os.getenv("HOSTNAME", "unknown")
-        target = os.path.join("/root/.cache/tests/logs/plog", tc_name, hostname)
+        plog_base = getattr(cls, "plog_base", "/root/.cache/tests/logs/plog")
+        target = os.path.join(plog_base, tc_name, hostname)
        os.makedirs(target, exist_ok=True)
        for name in os.listdir(plog_path):
            src = os.path.join(plog_path, name)
@@ -930,16 +930,27 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
    def _setup_per_case_output(cls):
        """Set up per-case output directories and env vars.

-        Extracted from ``nightly-test-npu-e2e-single-node.yml`` so that when a
-        suite is executed, each case writes its metrics/plog to a path derived
-        from the case file rather than the suite name.
+        When the workflow sets METRICS_DATA_FILE to a suite-level directory
+        (e.g. .../output/{branch_label}-{create_date}-{run_id}-{run_attempt}/
+        {workflow_name}/{test_type}/{suite}), each case in the suite
+        writes to its own subdirectory under it, so results stay in the
+        structured layout and are keyed by the case id. Falls back to the
+        legacy per-case layout when the env var is not set.
        """
        cls.tc_name = cls._get_tc_name()
-        current_date = datetime.now().strftime("%Y%m%d")
-        test_type = getattr(cls, "test_type", "perf")
-        base_output = f"/root/.cache/tests/output/{test_type}/{current_date}"
-        os.makedirs(base_output, exist_ok=True)
-        cls.metrics_data_file = os.path.join(base_output, cls.tc_name)
+        suite_output = os.environ.get("METRICS_DATA_FILE")
+        if suite_output:
+            # Append the case id under the suite output prefix.
+            cls.metrics_data_file = os.path.join(suite_output, cls.tc_name)
+            # Mirror the output prefix to the plog location (drop the test_type/suite tail).
+            suite_plog = suite_output.replace("/output/", "/logs/plog/", 1)
+            cls.plog_base = os.path.dirname(os.path.dirname(suite_plog))
+        else:
+            current_date = datetime.now().strftime("%Y%m%d")
+            test_type = getattr(cls, "test_type", "perf")
+            base_output = f"/root/.cache/tests/output/{test_type}/{current_date}"
+            cls.metrics_data_file = os.path.join(base_output, cls.tc_name)
+            cls.plog_base = f"/root/.cache/tests/logs/plog"
        os.makedirs(cls.metrics_data_file, exist_ok=True)
        # Override env vars so evalscope/dump_metric write to per-case paths.
        os.environ["METRICS_DATA_FILE"] = cls.metrics_data_file
@@ -985,7 +996,7 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
        out_path = os.path.join(cls.metrics_data_file, "metrics.json")
        payload = {
            "test_case": cls.tc_name,
-            "test_type": getattr(cls, "test_type", "accuracy"),
+            "test_type": getattr(cls, "test_type", "perf"),
            "metrics": metrics,
            "baselines": baselines,
        }
@@ -995,6 +1006,12 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
            logger.info("Saved per-case metrics to %s", out_path)
        except Exception as e:
            logger.warning("Failed to write metrics.json: %s", e)
+        # Remove the intermediate JSONL records, keeping only the final metrics.json.
+        for jsonl_path in glob.glob(pattern):
+            try:
+                os.remove(jsonl_path)
+            except Exception as e:
+                logger.warning("Failed to remove %s: %s", jsonl_path, e)

    @classmethod
    def _backup_plog(cls):
@@ -1010,7 +1027,8 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
        if not tc_name:
            return
        hostname = os.getenv("HOSTNAME", "unknown")
-        target = os.path.join("/root/.cache/tests/logs/plog", tc_name, hostname)
+        plog_base = getattr(cls, "plog_base", "/root/.cache/tests/logs/plog")
+        target = os.path.join(plog_base, tc_name, hostname)
        os.makedirs(target, exist_ok=True)
        for name in os.listdir(plog_path):
            src = os.path.join(plog_path, name)