[Docs] Rename docs_new/ to docs/ (#32123)
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
zijiexia
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
c949e91f18
commit
b819d2fb5b
Executable
+203
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env node
|
||||
// Static guard for the cookbook deployment/playground engines and their configs.
|
||||
// Zero dependencies, no browser, no Mintlify — plain `node`.
|
||||
//
|
||||
// node docs/scripts/check_cookbook_configs.mjs
|
||||
//
|
||||
// What it protects, in order of how expensive the bug is to find by hand:
|
||||
//
|
||||
// 1. MIRROR drift. The overlay-resolution rule is written in both engines
|
||||
// because Mintlify snippets cannot import each other. If the copies drift,
|
||||
// the Deploy command and the playground's base disagree and the reader sees
|
||||
// phantom +/- lines in the diff — with no error anywhere.
|
||||
// 2. Sibling identity. Overlay resolution clones the base cell, so sibling
|
||||
// detection must compare match dimensions rather than object references.
|
||||
// 3. Config/engine contract. A cell keyed on a dimension the config no longer
|
||||
// declares silently stops matching; the panel just shows a different cell.
|
||||
// 4. Predicate safety. showWhen / disabled / flags run against selections the
|
||||
// author never clicked through; a throw there blanks the whole widget.
|
||||
|
||||
import { readFileSync, readdirSync } from "node:fs";
|
||||
import { dirname, join, relative } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const SNIPPETS = join(dirname(fileURLToPath(import.meta.url)), "..", "src", "snippets");
|
||||
const CONFIGS = join(SNIPPETS, "configs");
|
||||
const LEGACY_DIMS = ["variants", "quantizations", "strategies", "nodesOptions"];
|
||||
|
||||
const failures = [];
|
||||
const fail = (where, msg) => failures.push(`${where}: ${msg}`);
|
||||
|
||||
// ---------------------------------------------------------------- 1. MIRROR
|
||||
// Compare the marked blocks with comments and whitespace normalized away, so
|
||||
// wording may differ per file but the rule may not.
|
||||
const mirrorBody = (file) => {
|
||||
const src = readFileSync(join(SNIPPETS, file), "utf8");
|
||||
const start = src.indexOf("==== MIRROR");
|
||||
const end = src.indexOf("==== end MIRROR");
|
||||
if (start === -1 || end === -1) return null;
|
||||
return src
|
||||
.slice(src.indexOf("\n", start), end)
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter((l) => l && !l.startsWith("//"))
|
||||
.join(" ")
|
||||
.replace(/\s+/g, " ");
|
||||
};
|
||||
|
||||
const a = mirrorBody("_deployment.jsx");
|
||||
const b = mirrorBody("_playground.jsx");
|
||||
if (a === null) fail("_deployment.jsx", "MIRROR markers missing");
|
||||
if (b === null) fail("_playground.jsx", "MIRROR markers missing");
|
||||
if (a && b && a !== b) {
|
||||
fail("MIRROR", "overlay resolution has drifted between the two engines");
|
||||
const [la, lb] = [a.split(" "), b.split(" ")];
|
||||
const i = la.findIndex((t, k) => t !== lb[k]);
|
||||
fail("MIRROR", `first divergence near token ${i}: `
|
||||
+ `_deployment "${la.slice(i, i + 8).join(" ")}" vs `
|
||||
+ `_playground "${lb.slice(i, i + 8).join(" ")}"`);
|
||||
}
|
||||
|
||||
// `withOverlay` returns a clone, so object identity can never distinguish the
|
||||
// current base cell from a true sibling. This previously made every cookbook
|
||||
// show a spurious "matches … / switch base" hint before the reader changed
|
||||
// anything.
|
||||
const playgroundSource = readFileSync(join(SNIPPETS, "_playground.jsx"), "utf8");
|
||||
if (/\bmatchedCell\s*!==\s*baseCell\b/.test(playgroundSource)) {
|
||||
fail("_playground.jsx", "sibling detection compares cloned cells by object identity");
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------- 3/4. Configs
|
||||
// Configs are .jsx with a single `export const config` literal; import them
|
||||
// through a data: URL so no temp file is needed.
|
||||
const loadConfig = async (path) => {
|
||||
const src = readFileSync(path, "utf8");
|
||||
const mod = await import(
|
||||
"data:text/javascript," + encodeURIComponent(src)
|
||||
);
|
||||
return mod.config;
|
||||
};
|
||||
|
||||
// Every combination of match dims + overlay dims the reader can produce.
|
||||
const selectionSpace = (config) => {
|
||||
const dims = [
|
||||
{ id: "hw", options: (config.supportedHardware || []).map((id) => ({ id })) },
|
||||
...(config.matchDims || []),
|
||||
...(config.overlayDims || []),
|
||||
];
|
||||
let space = [{}];
|
||||
for (const d of dims) {
|
||||
const next = [];
|
||||
for (const partial of space) {
|
||||
for (const opt of (d.options || [])) next.push({ ...partial, [d.id]: opt.id });
|
||||
}
|
||||
space = next.length ? next : space;
|
||||
if (space.length > 20000) return space.slice(0, 20000); // cheap blow-up guard
|
||||
}
|
||||
return space;
|
||||
};
|
||||
|
||||
const walk = (dir) => readdirSync(dir, { withFileTypes: true }).flatMap((e) =>
|
||||
e.isDirectory() ? walk(join(dir, e.name))
|
||||
: (e.name.endsWith(".jsx")
|
||||
&& !e.name.includes("benchmark")
|
||||
&& e.name !== "popular-models.jsx"
|
||||
? [join(dir, e.name)] : []));
|
||||
|
||||
for (const path of walk(CONFIGS)) {
|
||||
const where = relative(join(SNIPPETS, ".."), path);
|
||||
let config;
|
||||
try {
|
||||
config = await loadConfig(path);
|
||||
} catch (e) {
|
||||
fail(where, `does not parse as a module: ${e.message}`);
|
||||
continue;
|
||||
}
|
||||
if (!config) { fail(where, "no `export const config`"); continue; }
|
||||
|
||||
const custom = Array.isArray(config.matchDims);
|
||||
|
||||
// A config either declares its own dims or carries the full legacy set —
|
||||
// half of each means the engine silently renders a dimension nobody authored.
|
||||
if (!custom) {
|
||||
for (const k of LEGACY_DIMS) {
|
||||
if (!Array.isArray(config[k])) fail(where, `legacy config is missing \`${k}\``);
|
||||
}
|
||||
}
|
||||
|
||||
const matchIds = ["hw", ...(custom
|
||||
? config.matchDims.map((d) => d.id)
|
||||
: LEGACY_DIMS.map((k) => ({ variants: "variant", quantizations: "quant",
|
||||
strategies: "strategy", nodesOptions: "nodes" })[k]))];
|
||||
|
||||
for (const [i, cell] of (config.cells || []).entries()) {
|
||||
const keys = Object.keys(cell.match || {}).sort();
|
||||
const want = [...matchIds].sort();
|
||||
if (keys.join(",") !== want.join(",")) {
|
||||
fail(where, `cells[${i}].match keys [${keys}] != declared dims [${want}]`);
|
||||
}
|
||||
for (const dim of (config.matchDims || [])) {
|
||||
const v = cell.match[dim.id];
|
||||
if (!(dim.options || []).some((o) => o.id === v)) {
|
||||
fail(where, `cells[${i}].match.${dim.id}="${v}" is not an option of that dim`);
|
||||
}
|
||||
}
|
||||
// Without a `nodes` dim the node count rides on the cell; a missing one
|
||||
// silently degrades a multi-node recipe to single-node.
|
||||
if (custom && !matchIds.includes("nodes") && cell.nnodes === undefined) {
|
||||
fail(where, `cells[${i}] has no \`nnodes\` and the config declares no nodes dim`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const dim of (config.overlayDims || [])) {
|
||||
const ids = (dim.options || []).map((o) => o.id);
|
||||
if (dim.default !== undefined && !ids.includes(dim.default)) {
|
||||
fail(where, `overlayDims.${dim.id}.default="${dim.default}" is not one of [${ids}]`);
|
||||
}
|
||||
}
|
||||
|
||||
// Predicates and flag builders must survive every reachable selection.
|
||||
const space = selectionSpace(config);
|
||||
const probe = (fn, label) => {
|
||||
for (const sel of space) {
|
||||
try { fn(sel); } catch (e) {
|
||||
fail(where, `${label} throws on ${JSON.stringify(sel)}: ${e.message}`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
for (const dim of [...(config.matchDims || []), ...(config.overlayDims || [])]) {
|
||||
if (typeof dim.showWhen === "function") probe(dim.showWhen, `${dim.id}.showWhen`);
|
||||
for (const opt of (dim.options || [])) {
|
||||
const tag = `${dim.id}.${opt.id}`;
|
||||
if (typeof opt.showWhen === "function") probe(opt.showWhen, `${tag}.showWhen`);
|
||||
if (typeof opt.disabled === "function") probe(opt.disabled, `${tag}.disabled`);
|
||||
for (const key of ["flags", "env", "hints"]) {
|
||||
if (typeof opt[key] !== "function") continue;
|
||||
probe((sel) => {
|
||||
const out = opt[key](sel);
|
||||
if (out !== undefined && !Array.isArray(out)) throw new Error(`${key} returned ${typeof out}, expected an array`);
|
||||
for (const f of (out || [])) {
|
||||
if (typeof f !== "string") throw new Error(`${key} yielded a non-string entry`);
|
||||
if (/undefined|NaN/.test(f)) throw new Error(`${key} produced "${f}"`);
|
||||
}
|
||||
}, `${tag}.${key}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (typeof config.curl === "function") {
|
||||
probe((sel) => {
|
||||
const out = config.curl(sel, null);
|
||||
if (typeof out !== "string") {
|
||||
throw new Error(`curl returned ${typeof out}, expected a string`);
|
||||
}
|
||||
}, "curl");
|
||||
}
|
||||
}
|
||||
|
||||
if (failures.length) {
|
||||
console.error(`FAIL (${failures.length})`);
|
||||
for (const f of failures) console.error(" - " + f);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log("cookbook config check: OK");
|
||||
Executable
+227
@@ -0,0 +1,227 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate Mintlify docs.json redirects from old Sphinx paths to new Mintlify paths."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
OLD_DOCS = REPO / "docs"
|
||||
NEW_DOCS = REPO / "docs" / "docs"
|
||||
|
||||
# Directory-level renames (old → new, under /docs/ prefix)
|
||||
SECTION_RENAMES = {
|
||||
"get_started": "get-started",
|
||||
"platforms": "hardware-platforms",
|
||||
"supported_models": "supported-models",
|
||||
"diffusion": "sglang-diffusion",
|
||||
}
|
||||
|
||||
# Explicit file-level mappings. Keys are old URL paths (no .html, with leading /).
|
||||
# Values are new URL paths (with /docs/ prefix, no extension).
|
||||
EXPLICIT = {
|
||||
# get_started → get-started
|
||||
"/get_started/install": "/docs/get-started/installation",
|
||||
# developer_guide rename
|
||||
"/developer_guide/development_jit_kernel_guide": "/docs/developer_guide/JIT_kernels",
|
||||
# platforms → hardware-platforms (with file renames)
|
||||
"/platforms/amd_gpu": "/docs/hardware-platforms/amd-gpus",
|
||||
"/platforms/cpu_server": "/docs/hardware-platforms/cpu-server",
|
||||
"/platforms/tpu": "/docs/hardware-platforms/tpu",
|
||||
"/platforms/xpu": "/docs/hardware-platforms/xpu",
|
||||
# platforms/ascend → hardware-platforms/ascend-npus (flattened, renamed)
|
||||
"/platforms/ascend/ascend_npu": "/docs/hardware-platforms/ascend-npus/SGLang-installation-with-NPUs-support",
|
||||
"/platforms/ascend/ascend_npu_best_practice": "/docs/hardware-platforms/ascend-npus/Best-Practice-on-Ascend-NPU",
|
||||
"/platforms/ascend/ascend_npu_deepseek_example": "/docs/hardware-platforms/ascend-npus/DeepSeek-Examples",
|
||||
"/platforms/ascend/ascend_npu_glm5_examples": "/docs/hardware-platforms/ascend-npus/GLM-5",
|
||||
"/platforms/ascend/ascend_npu_qwen3_examples": "/docs/hardware-platforms/ascend-npus/Qwen3-Examples",
|
||||
"/platforms/ascend/ascend_npu_qwen3_5_examples": "/docs/hardware-platforms/ascend-npus/Qwen3.5",
|
||||
"/platforms/ascend/ascend_npu_support_features": "/docs/hardware-platforms/ascend-npus/Support-Features-on-Ascend-NPU",
|
||||
"/platforms/ascend/ascend_npu_support_models": "/docs/hardware-platforms/ascend-npus/Support-Models-on-Ascend-NPU",
|
||||
# Old pages dropped — redirect to section overview
|
||||
"/platforms/ascend/ascend_contribution_guide": "/docs/hardware-platforms/overview",
|
||||
"/platforms/ascend/ascend_npu_environment_variables": "/docs/hardware-platforms/overview",
|
||||
"/platforms/ascend/ascend_npu_quantization": "/docs/hardware-platforms/overview",
|
||||
"/platforms/ascend/ascend_npu_support": "/docs/hardware-platforms/overview",
|
||||
"/platforms/ascend/mindspore_backend": "/docs/hardware-platforms/overview",
|
||||
"/platforms/ascend_npu_ring_sp_performance": "/docs/hardware-platforms/overview",
|
||||
"/platforms/apple_metal": "/docs/hardware-platforms/overview",
|
||||
"/platforms/mthreads_gpu": "/docs/hardware-platforms/overview",
|
||||
"/platforms/nvidia_jetson": "/docs/hardware-platforms/overview",
|
||||
"/platforms/plugin": "/docs/hardware-platforms/overview",
|
||||
# supported_models → supported-models (flattened, renamed)
|
||||
"/supported_models": "/docs/supported-models",
|
||||
"/supported_models/index": "/docs/supported-models",
|
||||
"/supported_models/extending/mindspore_models": "/docs/supported-models/mindspore-models",
|
||||
"/supported_models/extending/modelscope": "/docs/supported-models/modelscope",
|
||||
"/supported_models/extending/support_new_models": "/docs/supported-models/new-model-support",
|
||||
"/supported_models/extending/transformers_fallback": "/docs/supported-models/transformers-fallback",
|
||||
"/supported_models/extending/index": "/docs/supported-models",
|
||||
"/supported_models/retrieval_ranking/classify_models": "/docs/supported-models/classification-models",
|
||||
"/supported_models/retrieval_ranking/embedding_models": "/docs/supported-models/embedding-models",
|
||||
"/supported_models/retrieval_ranking/rerank_models": "/docs/supported-models/rerank-models",
|
||||
"/supported_models/retrieval_ranking/index": "/docs/supported-models",
|
||||
"/supported_models/specialized/reward_models": "/docs/supported-models/reward-models",
|
||||
"/supported_models/specialized/index": "/docs/supported-models",
|
||||
"/supported_models/text_generation/generative_models": "/docs/supported-models/large-language-models",
|
||||
"/supported_models/text_generation/multimodal_language_models": "/docs/supported-models/vision-language-models",
|
||||
"/supported_models/text_generation/diffusion_language_models": "/docs/supported-models/diffusion-language-models",
|
||||
"/supported_models/text_generation/index": "/docs/supported-models",
|
||||
# diffusion → sglang-diffusion (file renames snake_case → kebab-case)
|
||||
"/diffusion": "/docs/sglang-diffusion/installation",
|
||||
"/diffusion/index": "/docs/sglang-diffusion/installation",
|
||||
"/diffusion/installation": "/docs/sglang-diffusion/installation",
|
||||
"/diffusion/environment_variables": "/docs/sglang-diffusion/environment-variables",
|
||||
"/diffusion/ci_perf": "/docs/sglang-diffusion/ci-performance",
|
||||
"/diffusion/api/cli": "/docs/sglang-diffusion/api/cli",
|
||||
"/diffusion/api/openai_api": "/docs/sglang-diffusion/api/openai-api",
|
||||
"/diffusion/performance/attention_backends": "/docs/sglang-diffusion/attention-backends",
|
||||
"/diffusion/performance/cache/cache_dit": "/docs/sglang-diffusion/cache-dit",
|
||||
"/diffusion/performance/cache/index": "/docs/sglang-diffusion/caching-acceleration",
|
||||
"/diffusion/performance/cache/teacache": "/docs/sglang-diffusion/tea-cache",
|
||||
"/diffusion/performance/index": "/docs/sglang-diffusion/performance-optimization",
|
||||
"/diffusion/performance/profiling": "/docs/sglang-diffusion/profiling",
|
||||
# Diffusion pages dropped
|
||||
"/diffusion/api/post_processing": "/docs/sglang-diffusion/installation",
|
||||
"/diffusion/compatibility_matrix": "/docs/sglang-diffusion/installation",
|
||||
"/diffusion/contributing": "/docs/sglang-diffusion/installation",
|
||||
"/diffusion/development": "/docs/sglang-diffusion/installation",
|
||||
"/diffusion/disaggregation": "/docs/sglang-diffusion/installation",
|
||||
"/diffusion/performance/ring_sp_performance": "/docs/sglang-diffusion/performance-optimization",
|
||||
"/diffusion/quantization": "/docs/sglang-diffusion/installation",
|
||||
"/diffusion/reference": "/docs/sglang-diffusion/installation",
|
||||
"/diffusion/support_new_models": "/docs/sglang-diffusion/installation",
|
||||
"/diffusion/usage": "/docs/sglang-diffusion/installation",
|
||||
# basic_usage pages migrated to cookbook
|
||||
"/basic_usage/kimi_k2_5": "/cookbook/autoregressive/Moonshotai/Kimi-K2.5",
|
||||
"/basic_usage/deepseek_ocr": "/cookbook/autoregressive/DeepSeek/DeepSeek-OCR",
|
||||
"/basic_usage/deepseek_v3": "/cookbook/autoregressive/DeepSeek/DeepSeek-V3",
|
||||
"/basic_usage/deepseek_v32": "/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2",
|
||||
"/basic_usage/glm45": "/cookbook/autoregressive/GLM/GLM-4.5",
|
||||
"/basic_usage/glmv": "/cookbook/autoregressive/GLM/GLM-4.6V",
|
||||
"/basic_usage/gpt_oss": "/cookbook/autoregressive/OpenAI/GPT-OSS",
|
||||
"/basic_usage/llama4": "/cookbook/autoregressive/Llama/Llama4",
|
||||
"/basic_usage/minimax_m2": "/cookbook/autoregressive/MiniMax/MiniMax-M2",
|
||||
"/basic_usage/popular_model_usage": "/cookbook/autoregressive/intro",
|
||||
"/basic_usage/qwen3": "/cookbook/autoregressive/Qwen/Qwen3",
|
||||
"/basic_usage/qwen3_5": "/cookbook/autoregressive/Qwen/Qwen3.5",
|
||||
"/basic_usage/qwen3_vl": "/cookbook/autoregressive/Qwen/Qwen3-VL",
|
||||
# advanced_features dropped pages
|
||||
"/advanced_features/adaptive_speculative_decoding": "/docs/advanced_features/speculative_decoding",
|
||||
"/advanced_features/hisparse_guide": "/docs/advanced_features/overview",
|
||||
# references dropped
|
||||
"/references/learn_more": "/",
|
||||
"/references/release_lookup": "/docs/references/overview",
|
||||
# Root index
|
||||
"/index": "/",
|
||||
"/": "/",
|
||||
}
|
||||
|
||||
|
||||
def old_url_from_path(rel: Path) -> str | None:
|
||||
"""Convert old docs/<rel> to its Sphinx URL path (no .html, leading /)."""
|
||||
parts = list(rel.parts)
|
||||
stem = rel.stem
|
||||
# Skip README, release_lookup/README, top-level non-doc files
|
||||
if stem == "README":
|
||||
return None
|
||||
# Drop the extension → URL path
|
||||
new_parts = parts[:-1] + [stem]
|
||||
return "/" + "/".join(new_parts)
|
||||
|
||||
|
||||
def new_url_for(old_url: str, new_files_set: set[str]) -> str | None:
|
||||
"""Compute new URL from old URL using section rename + explicit overrides."""
|
||||
if old_url in EXPLICIT:
|
||||
return EXPLICIT[old_url]
|
||||
# Default rule: `/section/path` → `/docs/section/path`, applying section renames
|
||||
parts = old_url.strip("/").split("/")
|
||||
if not parts or not parts[0]:
|
||||
return None
|
||||
section = parts[0]
|
||||
section = SECTION_RENAMES.get(section, section)
|
||||
new_url = "/docs/" + "/".join([section] + parts[1:])
|
||||
# Verify destination exists in new file tree
|
||||
if new_url in new_files_set:
|
||||
return new_url
|
||||
return None # unmapped
|
||||
|
||||
|
||||
def list_new_urls() -> set[str]:
|
||||
urls = set()
|
||||
for p in NEW_DOCS.rglob("*"):
|
||||
if not p.is_file():
|
||||
continue
|
||||
if p.suffix not in (".mdx", ".ipynb", ".md"):
|
||||
continue
|
||||
rel = p.relative_to(NEW_DOCS)
|
||||
# Mintlify routes .mdx / .ipynb as `/docs/<path-without-ext>`
|
||||
url = "/docs/" + str(rel.with_suffix("")).replace(os.sep, "/")
|
||||
urls.add(url)
|
||||
return urls
|
||||
|
||||
|
||||
def main():
|
||||
new_urls = list_new_urls()
|
||||
redirects: list[dict] = []
|
||||
seen_sources: set[str] = set()
|
||||
unmapped: list[str] = []
|
||||
|
||||
# Iterate all old files
|
||||
old_files = []
|
||||
for p in sorted(OLD_DOCS.rglob("*")):
|
||||
if not p.is_file():
|
||||
continue
|
||||
if p.suffix not in (".md", ".rst", ".ipynb"):
|
||||
continue
|
||||
rel = p.relative_to(OLD_DOCS)
|
||||
# Skip non-doc dirs
|
||||
if rel.parts and rel.parts[0] in (
|
||||
"_static",
|
||||
"performance_dashboard",
|
||||
"release_lookup",
|
||||
):
|
||||
continue
|
||||
old_files.append(rel)
|
||||
|
||||
for rel in old_files:
|
||||
old_url = old_url_from_path(rel)
|
||||
if old_url is None:
|
||||
continue
|
||||
# Old Sphinx URLs end in .html
|
||||
source = old_url + ".html"
|
||||
if source in seen_sources:
|
||||
continue
|
||||
new_url = new_url_for(old_url, new_urls)
|
||||
if new_url is None:
|
||||
unmapped.append(source)
|
||||
continue
|
||||
redirects.append({"source": source, "destination": new_url})
|
||||
seen_sources.add(source)
|
||||
|
||||
# Also add explicit entries whose source key wasn't derived from a file (e.g. index variants)
|
||||
for old_key, new_val in EXPLICIT.items():
|
||||
source = old_key + ".html"
|
||||
if source in seen_sources:
|
||||
continue
|
||||
# Only add if old_key corresponds to an actual old page pattern we care about
|
||||
# Skip bare "/" and "/index" (handled by Mintlify default)
|
||||
if old_key in ("/", "/index"):
|
||||
continue
|
||||
redirects.append({"source": source, "destination": new_val})
|
||||
seen_sources.add(source)
|
||||
|
||||
# Output
|
||||
print(f"# Total redirects: {len(redirects)}")
|
||||
print(f"# Unmapped old URLs: {len(unmapped)}")
|
||||
if unmapped:
|
||||
print("# --- UNMAPPED ---")
|
||||
for u in unmapped:
|
||||
print(f"# {u}")
|
||||
print(json.dumps(redirects, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+286
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sync SGLang-related LMSYS blog cards into index.mdx."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
INDEX_PATH = ROOT / "index.mdx"
|
||||
|
||||
START_MARKER = "{/* BEGIN_LMSYS_SGLANG_BLOG_CARDS */}"
|
||||
END_MARKER = "{/* END_LMSYS_SGLANG_BLOG_CARDS */}"
|
||||
|
||||
LMSYS_BLOG_API_URL = (
|
||||
"https://api.github.com/repos/lm-sys/lm-sys.github.io/contents/blog"
|
||||
)
|
||||
LMSYS_BLOG_BASE_URL = "https://lmsys.org/blog"
|
||||
LMSYS_BASE_URL = "https://lmsys.org"
|
||||
DEFAULT_IMAGE_URL = "https://lmsys.org/social.png"
|
||||
|
||||
MAX_CARDS = int(os.getenv("LMSYS_SGLANG_MAX_CARDS", "6"))
|
||||
KEYWORDS = [
|
||||
"sglang",
|
||||
"sgl-project/sglang",
|
||||
"sgl-kernel",
|
||||
"sglang-jax",
|
||||
"sgl diffusion",
|
||||
"sglang diffusion",
|
||||
]
|
||||
|
||||
FRONTMATTER_RE = re.compile(r"\A---\s*\n(.*?)\n---\s*\n?", flags=re.DOTALL)
|
||||
HTML_IMG_RE = re.compile(r"<img[^>]*\ssrc=[\"']([^\"']+)[\"']", flags=re.IGNORECASE)
|
||||
MD_IMG_RE = re.compile(r"!\[[^\]]*]\(([^)]+)\)")
|
||||
|
||||
|
||||
@dataclass
|
||||
class BlogPost:
|
||||
slug: str
|
||||
title: str
|
||||
url: str
|
||||
image: str
|
||||
date: str
|
||||
|
||||
|
||||
def build_headers() -> dict[str, str]:
|
||||
headers = {
|
||||
"Accept": "application/vnd.github+json",
|
||||
"User-Agent": "sgl-docs-lmsys-blog-sync",
|
||||
}
|
||||
token = os.getenv("GITHUB_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
def download_blog_sources() -> list[tuple[str, str]]:
|
||||
# Fetch the directory listing for /blog only — no need to download the whole repo.
|
||||
request = urllib.request.Request(LMSYS_BLOG_API_URL, headers=build_headers())
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
items: list[dict] = json.loads(response.read())
|
||||
|
||||
sources: list[tuple[str, str]] = []
|
||||
for item in items:
|
||||
if item.get("type") != "file" or not item.get("name", "").endswith(".md"):
|
||||
continue
|
||||
download_url = item.get("download_url")
|
||||
if not download_url:
|
||||
continue
|
||||
raw_request = urllib.request.Request(download_url, headers=build_headers())
|
||||
with urllib.request.urlopen(raw_request, timeout=30) as raw_response:
|
||||
content = raw_response.read().decode("utf-8", errors="replace")
|
||||
sources.append((item["name"], content))
|
||||
|
||||
return sources
|
||||
|
||||
|
||||
def split_frontmatter(content: str) -> tuple[dict[str, str], str]:
|
||||
match = FRONTMATTER_RE.match(content)
|
||||
if not match:
|
||||
return {}, content
|
||||
|
||||
frontmatter: dict[str, str] = {}
|
||||
for raw_line in match.group(1).splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or ":" not in line:
|
||||
continue
|
||||
|
||||
key, value = line.split(":", 1)
|
||||
cleaned = value.strip()
|
||||
if (
|
||||
(cleaned.startswith('"') and cleaned.endswith('"'))
|
||||
or (cleaned.startswith("'") and cleaned.endswith("'"))
|
||||
) and len(cleaned) >= 2:
|
||||
cleaned = cleaned[1:-1]
|
||||
frontmatter[key.strip()] = cleaned
|
||||
|
||||
return frontmatter, content[match.end() :]
|
||||
|
||||
|
||||
def first_image_from_body(body: str) -> str | None:
|
||||
markdown_match = MD_IMG_RE.search(body)
|
||||
if markdown_match:
|
||||
candidate = markdown_match.group(1).strip()
|
||||
if candidate.startswith("<") and candidate.endswith(">"):
|
||||
candidate = candidate[1:-1]
|
||||
if " " in candidate:
|
||||
candidate = candidate.split(" ", 1)[0]
|
||||
return candidate
|
||||
|
||||
html_match = HTML_IMG_RE.search(body)
|
||||
if html_match:
|
||||
return html_match.group(1).strip()
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def to_absolute_url(url_or_path: str | None) -> str:
|
||||
if not url_or_path:
|
||||
return DEFAULT_IMAGE_URL
|
||||
|
||||
value = url_or_path.strip()
|
||||
if value.startswith(("http://", "https://")):
|
||||
return value
|
||||
if value.startswith("//"):
|
||||
return f"https:{value}"
|
||||
return f"{LMSYS_BASE_URL}/{value.lstrip('/')}"
|
||||
|
||||
|
||||
def is_relevant(slug: str, title: str, body: str) -> bool:
|
||||
searchable = f"{slug}\n{title}\n{body}".lower()
|
||||
return any(keyword in searchable for keyword in KEYWORDS)
|
||||
|
||||
|
||||
def parse_blog_post(filename: str, content: str) -> BlogPost | None:
|
||||
if not filename.endswith(".md"):
|
||||
return None
|
||||
|
||||
slug = filename[:-3]
|
||||
frontmatter, body = split_frontmatter(content)
|
||||
|
||||
title = frontmatter.get("title", "").strip() or slug.replace("-", " ").title()
|
||||
preview_img = frontmatter.get("previewImg") or first_image_from_body(body)
|
||||
image = to_absolute_url(preview_img)
|
||||
url = f"{LMSYS_BLOG_BASE_URL}/{slug}/"
|
||||
date = frontmatter.get("date", "").strip() or slug[:10]
|
||||
|
||||
if not is_relevant(slug=slug, title=title, body=body):
|
||||
return None
|
||||
|
||||
return BlogPost(slug=slug, title=title, url=url, image=image, date=date)
|
||||
|
||||
|
||||
def render_cards(posts: list[BlogPost]) -> str:
|
||||
if not posts:
|
||||
return "No relevant LMSYS blog posts matched the current sync keywords."
|
||||
|
||||
lines = [
|
||||
'<div className="not-prose">',
|
||||
" <div",
|
||||
" style={{",
|
||||
' display: "grid",',
|
||||
' gridTemplateColumns: "repeat(auto-fit, minmax(300px, 1fr))",',
|
||||
' gap: "1rem",',
|
||||
' alignItems: "stretch",',
|
||||
" }}",
|
||||
" >",
|
||||
]
|
||||
for post in posts:
|
||||
safe_title = json.dumps(post.title)
|
||||
safe_url = json.dumps(post.url)
|
||||
safe_image = json.dumps(post.image)
|
||||
lines.extend(
|
||||
[
|
||||
" <a",
|
||||
f" href={safe_url}",
|
||||
' target="_blank"',
|
||||
' rel="noopener noreferrer"',
|
||||
" style={{",
|
||||
' display: "block",',
|
||||
' border: "1px solid rgba(128, 128, 128, 0.3)",',
|
||||
' borderRadius: "0.75rem",',
|
||||
' overflow: "hidden",',
|
||||
' textDecoration: "none",',
|
||||
' color: "inherit",',
|
||||
' height: "100%",',
|
||||
" }}",
|
||||
" >",
|
||||
" <div",
|
||||
" style={{",
|
||||
' aspectRatio: "16 / 9",',
|
||||
' overflow: "hidden",',
|
||||
' background: "rgba(128, 128, 128, 0.15)",',
|
||||
" }}",
|
||||
" >",
|
||||
" <img",
|
||||
f" src={safe_image}",
|
||||
f" alt={safe_title}",
|
||||
" style={{",
|
||||
' width: "100%",',
|
||||
' height: "100%",',
|
||||
' objectFit: "cover",',
|
||||
' objectPosition: "center",',
|
||||
' display: "block",',
|
||||
" }}",
|
||||
" />",
|
||||
" </div>",
|
||||
' <div style={{ padding: "0.9rem 1rem 1rem" }}>',
|
||||
" <p",
|
||||
" style={{",
|
||||
" margin: 0,",
|
||||
" fontWeight: 600,",
|
||||
" lineHeight: 1.35,",
|
||||
' fontSize: "0.98rem",',
|
||||
" }}",
|
||||
" >",
|
||||
f" {{{safe_title}}}",
|
||||
" </p>",
|
||||
" <p",
|
||||
" style={{",
|
||||
' margin: "0.55rem 0 0",',
|
||||
' fontSize: "0.85rem",',
|
||||
" opacity: 0.75,",
|
||||
" }}",
|
||||
" >",
|
||||
f" {{{json.dumps(post.date)}}}",
|
||||
" </p>",
|
||||
" </div>",
|
||||
" </a>",
|
||||
]
|
||||
)
|
||||
lines.extend([" </div>", "</div>"])
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def replace_generated_block(index_text: str, generated_cards: str) -> str:
|
||||
pattern = re.compile(
|
||||
rf"{re.escape(START_MARKER)}.*?{re.escape(END_MARKER)}",
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
replacement = f"{START_MARKER}\n{generated_cards}\n{END_MARKER}"
|
||||
updated_text, replacements = pattern.subn(
|
||||
lambda _match: replacement, index_text, count=1
|
||||
)
|
||||
if replacements != 1:
|
||||
raise RuntimeError(
|
||||
f"Could not find exactly one marker block in {INDEX_PATH.name}. "
|
||||
f"Expected markers: {START_MARKER} ... {END_MARKER}"
|
||||
)
|
||||
return updated_text
|
||||
|
||||
|
||||
def main() -> None:
|
||||
sources = download_blog_sources()
|
||||
relevant_posts: list[BlogPost] = []
|
||||
|
||||
for filename, content in sources:
|
||||
post = parse_blog_post(filename=filename, content=content)
|
||||
if post is not None:
|
||||
relevant_posts.append(post)
|
||||
|
||||
relevant_posts.sort(key=lambda post: post.slug, reverse=True)
|
||||
selected_posts = relevant_posts[:MAX_CARDS]
|
||||
|
||||
generated_cards = render_cards(selected_posts)
|
||||
current_index = INDEX_PATH.read_text(encoding="utf-8")
|
||||
updated_index = replace_generated_block(
|
||||
index_text=current_index, generated_cards=generated_cards
|
||||
)
|
||||
|
||||
if updated_index != current_index:
|
||||
INDEX_PATH.write_text(updated_index, encoding="utf-8")
|
||||
|
||||
print(
|
||||
"Scanned "
|
||||
f"{len(sources)} blog files, matched {len(relevant_posts)} posts, "
|
||||
f"published {len(selected_posts)} cards."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user