diff --git a/python/pyproject.toml b/python/pyproject.toml index 3f3b7a039..376db0878 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ "ninja", "easydict", # Required by remote model code (e.g. DeepSeek-OCR) loaded via trust_remote_code; validated by transformers 5.4+ check_imports "numpy", - "nvidia-cutlass-dsl[cu13]==4.5.0", + "nvidia-cutlass-dsl[cu13]==4.5.1", "nvidia-ml-py", "openai-harmony==0.0.4", "openai==2.6.1", diff --git a/scripts/ci/cuda/ci_install_dependency.sh b/scripts/ci/cuda/ci_install_dependency.sh index 485d5b63c..d37d2286e 100755 --- a/scripts/ci/cuda/ci_install_dependency.sh +++ b/scripts/ci/cuda/ci_install_dependency.sh @@ -335,6 +335,34 @@ download_flashinfer_cache() { mark_step_done "${FUNCNAME[0]}" } +force_reinstall_cutlass_dsl_libs_cu13() { + # nvidia-cutlass-dsl[cu13] has additive PyPI extras: installing it pulls in + # both -libs-base and -libs-cu13. The two wheels ship intentionally-different + # content for the same paths (cutlass/_mlir/dialects/_gpu_ops_gen.py and + # cutlass/_mlir/_mlir_libs/_cutlass_ir.cpython-*.so) -- each Python wrapper + # is paired with a matching pybind11 .so. If install order leaves the .py + # from one wheel and the .so from the other, GPUModuleOp.__init__ raises + # TypeError: incompatible function arguments at kernel-compile time. + # + # Force-reinstall -libs-cu13 LAST so both files come from the same wheel + # (BOTH-cu13 state), eliminating the mismatch. The version is parsed from + # pyproject.toml so this stays in sync with whatever nvidia-cutlass-dsl + # version the project pins. + if [ "$CU_MAJOR" != "13" ]; then + return + fi + + CUTLASS_DSL_VERSION=$(grep -Po -m1 'nvidia-cutlass-dsl(\[[^]]+\])?==\K[0-9A-Za-z\.\-]+' "${REPO_ROOT}/python/pyproject.toml" || echo "") + if [ -z "$CUTLASS_DSL_VERSION" ]; then + echo "WARNING: could not detect nvidia-cutlass-dsl version from pyproject.toml; skipping libs-cu13 force-reinstall" + return + fi + + $PIP_CMD install --force-reinstall --no-deps "nvidia-cutlass-dsl-libs-cu13==${CUTLASS_DSL_VERSION}" $PIP_INSTALL_SUFFIX + + mark_step_done "${FUNCNAME[0]}" +} + stabilize_flashinfer_jit_paths() { # In venv mode, FlashInfer JIT writes build.ninja with hardcoded -isystem # paths. Per-job venvs get unique paths, but the JIT cache is shared on the @@ -488,6 +516,7 @@ main() { install_sglang_kernel install_sglang_router download_flashinfer_cache + force_reinstall_cutlass_dsl_libs_cu13 stabilize_flashinfer_jit_paths install_extra_deps install_test_tools