Remove the torchao integration (--torchao-config) (#34304)

Co-authored-by: Brayden Zhong <brayden@radixark.ai>

docker/rocm.Dockerfile

@@ -282,7 +282,6 @@ ARG SETUPTOOLS_SCM_PRETEND_VERSION
 RUN pip install IPython \
     && pip install orjson \
     && pip install python-multipart \
-    && pip install torchao==0.9.0 \
     && pip install pybind11
 
 # Rust toolchain — needed by setuptools-rust to build the sglang-mm extension

docker/xpu.Dockerfile

@@ -69,7 +69,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 WORKDIR /sgl-workspace
 
 RUN  pip install --no-cache-dir msgspec blake3 py-cpuinfo compressed_tensors gguf partial_json_parser einops tabulate --root-user-action=ignore && \
-     pip install --no-cache-dir torch==2.12.0+xpu torchao==0.17.0+xpu torchvision==0.27.0+xpu torchaudio==2.11.0+xpu --index-url https://download.pytorch.org/whl/xpu
+     pip install --no-cache-dir torch==2.12.0+xpu torchvision==0.27.0+xpu torchaudio==2.11.0+xpu --index-url https://download.pytorch.org/whl/xpu
 
 RUN echo "Cloning ${SG_LANG_BRANCH} from ${SG_LANG_REPO}" && \
     git clone --branch ${SG_LANG_BRANCH} --single-branch ${SG_LANG_REPO} sglang && \