#!/usr/bin/env python3 """Sync SGLang-related LMSYS blog cards into index.mdx.""" from __future__ import annotations import json import os import re import urllib.request from dataclasses import dataclass from pathlib import Path ROOT = Path(__file__).resolve().parents[1] INDEX_PATH = ROOT / "index.mdx" START_MARKER = "{/* BEGIN_LMSYS_SGLANG_BLOG_CARDS */}" END_MARKER = "{/* END_LMSYS_SGLANG_BLOG_CARDS */}" LMSYS_BLOG_API_URL = ( "https://api.github.com/repos/lm-sys/lm-sys.github.io/contents/blog" ) LMSYS_BLOG_BASE_URL = "https://lmsys.org/blog" LMSYS_BASE_URL = "https://lmsys.org" DEFAULT_IMAGE_URL = "https://lmsys.org/social.png" MAX_CARDS = int(os.getenv("LMSYS_SGLANG_MAX_CARDS", "6")) KEYWORDS = [ "sglang", "sgl-project/sglang", "sgl-kernel", "sglang-jax", "sgl diffusion", "sglang diffusion", ] FRONTMATTER_RE = re.compile(r"\A---\s*\n(.*?)\n---\s*\n?", flags=re.DOTALL) HTML_IMG_RE = re.compile(r"]*\ssrc=[\"']([^\"']+)[\"']", flags=re.IGNORECASE) MD_IMG_RE = re.compile(r"!\[[^\]]*]\(([^)]+)\)") @dataclass class BlogPost: slug: str title: str url: str image: str date: str def build_headers() -> dict[str, str]: headers = { "Accept": "application/vnd.github+json", "User-Agent": "sgl-docs-lmsys-blog-sync", } token = os.getenv("GITHUB_TOKEN") if token: headers["Authorization"] = f"Bearer {token}" return headers def download_blog_sources() -> list[tuple[str, str]]: # Fetch the directory listing for /blog only — no need to download the whole repo. request = urllib.request.Request(LMSYS_BLOG_API_URL, headers=build_headers()) with urllib.request.urlopen(request, timeout=60) as response: items: list[dict] = json.loads(response.read()) sources: list[tuple[str, str]] = [] for item in items: if item.get("type") != "file" or not item.get("name", "").endswith(".md"): continue download_url = item.get("download_url") if not download_url: continue raw_request = urllib.request.Request(download_url, headers=build_headers()) with urllib.request.urlopen(raw_request, timeout=30) as raw_response: content = raw_response.read().decode("utf-8", errors="replace") sources.append((item["name"], content)) return sources def split_frontmatter(content: str) -> tuple[dict[str, str], str]: match = FRONTMATTER_RE.match(content) if not match: return {}, content frontmatter: dict[str, str] = {} for raw_line in match.group(1).splitlines(): line = raw_line.strip() if not line or ":" not in line: continue key, value = line.split(":", 1) cleaned = value.strip() if ( (cleaned.startswith('"') and cleaned.endswith('"')) or (cleaned.startswith("'") and cleaned.endswith("'")) ) and len(cleaned) >= 2: cleaned = cleaned[1:-1] frontmatter[key.strip()] = cleaned return frontmatter, content[match.end() :] def first_image_from_body(body: str) -> str | None: markdown_match = MD_IMG_RE.search(body) if markdown_match: candidate = markdown_match.group(1).strip() if candidate.startswith("<") and candidate.endswith(">"): candidate = candidate[1:-1] if " " in candidate: candidate = candidate.split(" ", 1)[0] return candidate html_match = HTML_IMG_RE.search(body) if html_match: return html_match.group(1).strip() return None def to_absolute_url(url_or_path: str | None) -> str: if not url_or_path: return DEFAULT_IMAGE_URL value = url_or_path.strip() if value.startswith(("http://", "https://")): return value if value.startswith("//"): return f"https:{value}" return f"{LMSYS_BASE_URL}/{value.lstrip('/')}" def is_relevant(slug: str, title: str, body: str) -> bool: searchable = f"{slug}\n{title}\n{body}".lower() return any(keyword in searchable for keyword in KEYWORDS) def parse_blog_post(filename: str, content: str) -> BlogPost | None: if not filename.endswith(".md"): return None slug = filename[:-3] frontmatter, body = split_frontmatter(content) title = frontmatter.get("title", "").strip() or slug.replace("-", " ").title() preview_img = frontmatter.get("previewImg") or first_image_from_body(body) image = to_absolute_url(preview_img) url = f"{LMSYS_BLOG_BASE_URL}/{slug}/" date = frontmatter.get("date", "").strip() or slug[:10] if not is_relevant(slug=slug, title=title, body=body): return None return BlogPost(slug=slug, title=title, url=url, image=image, date=date) def render_cards(posts: list[BlogPost]) -> str: if not posts: return "No relevant LMSYS blog posts matched the current sync keywords." lines = [ '
', " ", ] for post in posts: safe_title = json.dumps(post.title) safe_url = json.dumps(post.url) safe_image = json.dumps(post.image) lines.extend( [ " ", " ", " ", "
", '
', " ", f" {{{safe_title}}}", "

", " ", f" {{{json.dumps(post.date)}}}", "

", "
", " ", ] ) lines.extend([" ", ""]) return "\n".join(lines) def replace_generated_block(index_text: str, generated_cards: str) -> str: pattern = re.compile( rf"{re.escape(START_MARKER)}.*?{re.escape(END_MARKER)}", flags=re.DOTALL, ) replacement = f"{START_MARKER}\n{generated_cards}\n{END_MARKER}" updated_text, replacements = pattern.subn( lambda _match: replacement, index_text, count=1 ) if replacements != 1: raise RuntimeError( f"Could not find exactly one marker block in {INDEX_PATH.name}. " f"Expected markers: {START_MARKER} ... {END_MARKER}" ) return updated_text def main() -> None: sources = download_blog_sources() relevant_posts: list[BlogPost] = [] for filename, content in sources: post = parse_blog_post(filename=filename, content=content) if post is not None: relevant_posts.append(post) relevant_posts.sort(key=lambda post: post.slug, reverse=True) selected_posts = relevant_posts[:MAX_CARDS] generated_cards = render_cards(selected_posts) current_index = INDEX_PATH.read_text(encoding="utf-8") updated_index = replace_generated_block( index_text=current_index, generated_cards=generated_cards ) if updated_index != current_index: INDEX_PATH.write_text(updated_index, encoding="utf-8") print( "Scanned " f"{len(sources)} blog files, matched {len(relevant_posts)} posts, " f"published {len(selected_posts)} cards." ) if __name__ == "__main__": main()