[Auto Sync] Update grpc_request_manager.py, tokenizer_manag... (20260214) (#18838)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Lianmin Zheng
2026-02-14 18:12:32 -08:00
committed by GitHub
co-authored by github-actions[bot]
parent 190fa8246f
commit b33769786f
7 changed files with 544 additions and 191 deletions
@@ -983,8 +983,8 @@ class GrpcRequestManager:
self.event_loop = loop self.event_loop = loop
# We cannot add signal handler when the grpc manager is not in # We only add signal handler when the tokenizer manager is in the main thread
# the main thread due to the CPython limitation. # due to the CPython limitation.
if threading.current_thread() is threading.main_thread(): if threading.current_thread() is threading.main_thread():
signal_handler = GrpcSignalHandler(self) signal_handler = GrpcSignalHandler(self)
loop.add_signal_handler(signal.SIGTERM, signal_handler.sigterm_handler) loop.add_signal_handler(signal.SIGTERM, signal_handler.sigterm_handler)
@@ -992,12 +992,7 @@ class GrpcRequestManager:
loop.add_signal_handler( loop.add_signal_handler(
signal.SIGQUIT, signal_handler.running_phase_sigquit_handler signal.SIGQUIT, signal_handler.running_phase_sigquit_handler
) )
else:
logger.warning(
"Signal handler is not added because the grpc request manager is "
"not in the main thread. This disables graceful shutdown of the "
"grpc request manager when SIGTERM is received."
)
self.asyncio_tasks.add( self.asyncio_tasks.add(
loop.create_task(print_exception_wrapper(self.sigterm_watchdog)) loop.create_task(print_exception_wrapper(self.sigterm_watchdog))
) )
@@ -1451,6 +1451,8 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
) )
self.event_loop = loop self.event_loop = loop
# We only add signal handler when the tokenizer manager is in the main thread
# due to the CPython limitation.
if threading.current_thread() is threading.main_thread(): if threading.current_thread() is threading.main_thread():
signal_handler = self.signal_handler_class(self) signal_handler = self.signal_handler_class(self)
loop.add_signal_handler(signal.SIGTERM, signal_handler.sigterm_handler) loop.add_signal_handler(signal.SIGTERM, signal_handler.sigterm_handler)
@@ -1458,14 +1460,6 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
loop.add_signal_handler( loop.add_signal_handler(
signal.SIGQUIT, signal_handler.running_phase_sigquit_handler signal.SIGQUIT, signal_handler.running_phase_sigquit_handler
) )
else:
# We cannot add signal handler when the tokenizer manager is not in
# the main thread due to the CPython limitation.
logger.warning(
"Signal handler is not added because the tokenizer manager is "
"not in the main thread. This disables graceful shutdown of the "
"tokenizer manager when SIGTERM is received."
)
self.asyncio_tasks.add( self.asyncio_tasks.add(
loop.create_task(print_exception_wrapper(self.sigterm_watchdog)) loop.create_task(print_exception_wrapper(self.sigterm_watchdog))
-4
View File
@@ -1127,10 +1127,6 @@ def check_pkg_version_at_least(pkg: str, min_version: str) -> bool:
def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = None): def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = None):
"""Kill the process and all its child processes.""" """Kill the process and all its child processes."""
# Remove sigchld handler to avoid spammy logs.
if threading.current_thread() is threading.main_thread():
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
if parent_pid is None: if parent_pid is None:
parent_pid = os.getpid() parent_pid = os.getpid()
include_parent = False include_parent = False
+268 -67
View File
@@ -2,13 +2,15 @@
List commits in the private repo that need to be synced to the OSS repo. List commits in the private repo that need to be synced to the OSS repo.
NOTE: NOTE:
1. You need to execute this script in the git root folder. 1. This script resolves the git root automatically and can be run anywhere
inside the repo.
This script will: This script will:
1. Find the most recent sync commit (message starts with 1. Find the most recent sync commit (message starts with
"[Automated PR] Copy OSS code from commit"). "[Automated PR] Copy OSS code from commit").
2. Scan commits after that point and keep those that touch the configured paths. 2. Scan commits after that point and keep those that touch the configured paths.
3. Print a markdown summary with commit links and write it to GitHub Step Summary. 3. Compare added diff lines in relevant files against OSS main.
4. Print a markdown summary with commit links and write it to GitHub Step Summary.
Usage: Usage:
python3 scripts/code_sync/check_commits.py python3 scripts/code_sync/check_commits.py
@@ -18,41 +20,35 @@ import argparse
import os import os
import shutil import shutil
import subprocess import subprocess
from typing import List, Optional, Tuple import sys
from dataclasses import dataclass
from typing import Dict, List, Optional, Set, Tuple
# Allow sibling imports regardless of the working directory.
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from utils import ( # noqa: E402
FOLDER_NAMES,
get_last_sync_commit,
write_github_step_summary,
)
# --- Configuration Begin --- # --- Configuration Begin ---
# List of folders and files to copy to the OSS repo.
# Changes outside these paths will be ignored.
folder_names = [
"3rdparty",
"assets",
"benchmark",
"docker",
"docs",
"examples",
"python/sglang/lang",
"python/sglang/jit_kernel",
"python/sglang/srt",
"python/sglang/test",
"python/sglang/utils.py",
"python/sglang/README.md",
"sgl-kernel",
"test/manual",
"test/registered",
"test/srt",
"test/README.md",
"test/run_suite.py",
"README.md",
]
private_repo = "your-org/sglang-private-repo" private_repo = "your-org/sglang-private-repo"
sync_commit_prefix = r"\[Automated PR\] Copy OSS code from commit" oss_repo_url = "https://github.com/sgl-project/sglang.git"
oss_repo_branch = "main"
default_oss_repo_dir = ".oss_repo"
# --- Configuration End --- # --- Configuration End ---
def write_github_step_summary(content: str) -> None: @dataclass
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: class CommitInfo:
f.write(content) commit_hash: str
subject: str
commit_date: str
relevant_files: List[str]
synced_lines: int
total_added_lines: int
def check_dependencies() -> None: def check_dependencies() -> None:
@@ -61,7 +57,23 @@ def check_dependencies() -> None:
raise EnvironmentError("git is not installed or not in PATH.") raise EnvironmentError("git is not installed or not in PATH.")
def get_repo_from_origin() -> str: def get_repo_root() -> str:
try:
output = subprocess.run(
["git", "rev-parse", "--show-toplevel"],
capture_output=True,
text=True,
check=True,
).stdout.strip()
except subprocess.CalledProcessError as e:
raise RuntimeError(f"Unable to determine git repo root: {e.stderr or e}") from e
if not output:
raise RuntimeError("Unable to determine git repo root.")
return os.path.abspath(output)
def get_repo_from_origin(repo_root: str) -> str:
"""Try to infer the repo slug (owner/name) from git remote.origin.url.""" """Try to infer the repo slug (owner/name) from git remote.origin.url."""
try: try:
url = subprocess.run( url = subprocess.run(
@@ -69,6 +81,7 @@ def get_repo_from_origin() -> str:
capture_output=True, capture_output=True,
text=True, text=True,
check=True, check=True,
cwd=repo_root,
).stdout.strip() ).stdout.strip()
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
return private_repo return private_repo
@@ -85,29 +98,48 @@ def get_repo_from_origin() -> str:
return repo or private_repo return repo or private_repo
def get_last_sync_commit() -> Optional[str]: def get_default_oss_repo_path(repo_root: str) -> str:
"""Find the most recent sync commit that copied from OSS.""" env_path = os.environ.get("OSS_REPO_PATH")
try: if env_path:
result = subprocess.run( return os.path.abspath(env_path)
[ return os.path.abspath(os.path.join(repo_root, default_oss_repo_dir))
"git",
"log",
"-1", def ensure_oss_repo(oss_repo_path: str, repo_url: str, branch: str) -> str:
"--grep", oss_repo_path = os.path.abspath(oss_repo_path)
sync_commit_prefix, if os.path.exists(oss_repo_path) and not os.path.isdir(oss_repo_path):
"--format=%H", raise RuntimeError(f"OSS repo path is not a directory: {oss_repo_path}")
],
capture_output=True, if os.path.isdir(os.path.join(oss_repo_path, ".git")):
text=True, try:
subprocess.run(
["git", "-C", oss_repo_path, "rev-parse", "--is-inside-work-tree"],
capture_output=True,
text=True,
check=True,
)
except subprocess.CalledProcessError as e:
raise RuntimeError(
f"OSS repo path exists but is not a git repo: {oss_repo_path}"
) from e
subprocess.run(
["git", "-C", oss_repo_path, "fetch", "origin", branch, "--depth", "1"],
check=True, check=True,
).stdout.strip() )
return result or None return oss_repo_path
except subprocess.CalledProcessError as e:
print(f"Error finding last sync commit: {e.stderr}") parent_dir = os.path.dirname(oss_repo_path)
return None if parent_dir and not os.path.isdir(parent_dir):
os.makedirs(parent_dir, exist_ok=True)
subprocess.run(
["git", "clone", "--depth", "1", "--branch", branch, repo_url, oss_repo_path],
check=True,
)
return oss_repo_path
def get_commits_since(last_sync_hash: Optional[str]) -> List[str]: def get_commits_since(repo_root: str, last_sync_hash: Optional[str]) -> List[str]:
"""Get commit hashes from last sync commit (exclusive) to HEAD.""" """Get commit hashes from last sync commit (exclusive) to HEAD."""
try: try:
if last_sync_hash: if last_sync_hash:
@@ -115,7 +147,7 @@ def get_commits_since(last_sync_hash: Optional[str]) -> List[str]:
else: else:
command = ["git", "rev-list", "HEAD"] command = ["git", "rev-list", "HEAD"]
result = subprocess.run( result = subprocess.run(
command, capture_output=True, text=True, check=True command, capture_output=True, text=True, check=True, cwd=repo_root
).stdout.strip() ).stdout.strip()
return [line for line in result.split("\n") if line] return [line for line in result.split("\n") if line]
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
@@ -123,13 +155,14 @@ def get_commits_since(last_sync_hash: Optional[str]) -> List[str]:
return [] return []
def get_changed_files(commit_hash: str) -> List[str]: def get_changed_files(repo_root: str, commit_hash: str) -> List[str]:
try: try:
output = subprocess.run( output = subprocess.run(
["git", "diff-tree", "--no-commit-id", "--name-only", "-r", commit_hash], ["git", "diff-tree", "--no-commit-id", "--name-only", "-r", commit_hash],
capture_output=True, capture_output=True,
text=True, text=True,
check=True, check=True,
cwd=repo_root,
).stdout.strip() ).stdout.strip()
return [line for line in output.split("\n") if line] return [line for line in output.split("\n") if line]
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
@@ -147,11 +180,115 @@ def get_relevant_files(changed_files: List[str]) -> List[str]:
return [ return [
changed_file changed_file
for changed_file in changed_files for changed_file in changed_files
if any(is_relevant_path(changed_file, path) for path in folder_names) if any(is_relevant_path(changed_file, path) for path in FOLDER_NAMES)
] ]
def get_commit_summary(commit_hash: str) -> Tuple[str, str]: def get_added_lines_by_file(
repo_root: str, commit_hash: str, relevant_files: List[str]
) -> Dict[str, List[str]]:
if not relevant_files:
return {}
command = [
"git",
"show",
"--no-color",
"--unified=0",
"--format=",
commit_hash,
"--",
] + relevant_files
try:
output = subprocess.run(
command, capture_output=True, text=True, check=True, cwd=repo_root
).stdout
except subprocess.CalledProcessError as e:
print(f"Error getting diff for {commit_hash}: {e.stderr}")
return {}
added_lines: Dict[str, List[str]] = {path: [] for path in relevant_files}
relevant_set = set(relevant_files)
current_file: Optional[str] = None
for line in output.splitlines():
if line.startswith("diff --git "):
current_file = None
continue
if line.startswith("+++ "):
file_path = None
if line.startswith("+++ b/"):
file_path = line[6:]
else:
candidate = line[4:]
if candidate == "/dev/null":
file_path = None
elif candidate.startswith("b/") or candidate.startswith("a/"):
file_path = candidate[2:]
else:
file_path = candidate
if file_path in relevant_set:
current_file = file_path
else:
current_file = None
continue
if current_file and line.startswith("+") and not line.startswith("+++ "):
added_lines[current_file].append(line[1:])
return added_lines
def get_oss_file_lines(
oss_repo_path: str,
oss_ref: str,
file_path: str,
cache: Dict[str, Optional[Set[str]]],
) -> Optional[Set[str]]:
if file_path in cache:
return cache[file_path]
try:
output = subprocess.run(
["git", "-C", oss_repo_path, "show", f"{oss_ref}:{file_path}"],
capture_output=True,
text=True,
errors="replace",
check=True,
).stdout
except subprocess.CalledProcessError:
cache[file_path] = None
return None
lines = output.splitlines()
line_set = set(lines)
cache[file_path] = line_set
return line_set
def count_synced_lines(
added_lines_by_file: Dict[str, List[str]],
oss_repo_path: str,
oss_ref: str,
oss_file_cache: Dict[str, Optional[Set[str]]],
) -> Tuple[int, int]:
total_added_lines = 0
synced_lines = 0
for file_path, lines in added_lines_by_file.items():
total_added_lines += len(lines)
if not lines:
continue
oss_lines = get_oss_file_lines(
oss_repo_path, oss_ref, file_path, oss_file_cache
)
if not oss_lines:
continue
for line in lines:
if line in oss_lines:
synced_lines += 1
return synced_lines, total_added_lines
def get_commit_summary(repo_root: str, commit_hash: str) -> Tuple[str, str]:
"""Return (subject, date) for a commit.""" """Return (subject, date) for a commit."""
try: try:
output = subprocess.run( output = subprocess.run(
@@ -159,6 +296,7 @@ def get_commit_summary(commit_hash: str) -> Tuple[str, str]:
capture_output=True, capture_output=True,
text=True, text=True,
check=True, check=True,
cwd=repo_root,
).stdout.strip() ).stdout.strip()
subject, commit_date = output.split("\x00", 1) subject, commit_date = output.split("\x00", 1)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
@@ -195,13 +333,20 @@ def format_commit_block(
commit_hash: str, commit_hash: str,
commit_date: str, commit_date: str,
relevant_files: List[str], relevant_files: List[str],
synced_lines: int,
total_added_lines: int,
) -> str: ) -> str:
short_hash = commit_hash[:9] short_hash = commit_hash[:9]
commit_url = f"https://github.com/{repo}/commit/{commit_hash}" commit_url = f"https://github.com/{repo}/commit/{commit_hash}"
files_str = format_files_list(relevant_files) if relevant_files else "- None" files_str = format_files_list(relevant_files) if relevant_files else "- None"
status_icon = "✅" if synced_lines == total_added_lines else "❌"
status_line = (
f"status: {status_icon} {synced_lines}/{total_added_lines} lines synced"
)
return "\n".join( return "\n".join(
[ [
f"#### {subject}", f"#### {subject}",
status_line,
f"date: {commit_date}", f"date: {commit_date}",
"files to sync:", "files to sync:",
files_str, files_str,
@@ -215,7 +360,7 @@ def format_commit_block(
def format_output( def format_output(
repo: str, repo: str,
last_sync: Optional[Tuple[str, str, str]], last_sync: Optional[Tuple[str, str, str]],
commits: List[Tuple[str, str, str, List[str]]], commits: List[CommitInfo],
) -> str: ) -> str:
lines: List[str] = [] lines: List[str] = []
if last_sync: if last_sync:
@@ -229,9 +374,17 @@ def format_output(
lines.append("No commits need to be synced.") lines.append("No commits need to be synced.")
return "\n".join(lines) + "\n" return "\n".join(lines) + "\n"
for commit_hash, subject, commit_date, relevant_files in commits: for commit in commits:
lines.append( lines.append(
format_commit_block(repo, subject, commit_hash, commit_date, relevant_files) format_commit_block(
repo,
commit.subject,
commit.commit_hash,
commit.commit_date,
commit.relevant_files,
commit.synced_lines,
commit.total_added_lines,
)
) )
return "\n".join(lines) return "\n".join(lines)
@@ -247,30 +400,78 @@ def main() -> None:
default=0, default=0,
help="Limit number of commits printed (0 means no limit).", help="Limit number of commits printed (0 means no limit).",
) )
parser.add_argument(
"--oss-repo-path",
default=None,
help="Path to OSS repo clone (default: $OSS_REPO_PATH or .oss_repo).",
)
parser.add_argument(
"--oss-repo-url",
default=oss_repo_url,
help="OSS repo URL (default: https://github.com/sgl-project/sglang.git).",
)
parser.add_argument(
"--oss-branch",
default=oss_repo_branch,
help="OSS repo branch to check (default: main).",
)
args = parser.parse_args() args = parser.parse_args()
check_dependencies() check_dependencies()
repo_root = get_repo_root()
oss_repo_path = (
os.path.abspath(args.oss_repo_path)
if args.oss_repo_path
else get_default_oss_repo_path(repo_root)
)
repo = get_repo_from_origin() repo = get_repo_from_origin(repo_root)
last_sync_hash = get_last_sync_commit() last_sync_hash = get_last_sync_commit(repo_root)
last_sync_block = None last_sync_block = None
if last_sync_hash: if last_sync_hash:
last_sync_subject, last_sync_date = get_commit_summary(last_sync_hash) last_sync_subject, last_sync_date = get_commit_summary(
repo_root, last_sync_hash
)
last_sync_block = (last_sync_subject, last_sync_hash, last_sync_date) last_sync_block = (last_sync_subject, last_sync_hash, last_sync_date)
commits = get_commits_since(last_sync_hash) commits = get_commits_since(repo_root, last_sync_hash)
if args.limit > 0: if args.limit > 0:
commits = commits[: args.limit] commits = commits[: args.limit]
relevant_commits: List[Tuple[str, str, str, List[str]]] = [] relevant_commit_inputs: List[Tuple[str, List[str]]] = []
for commit_hash in commits: for commit_hash in commits:
changed_files = get_changed_files(commit_hash) changed_files = get_changed_files(repo_root, commit_hash)
if not changed_files: if not changed_files:
continue continue
relevant_files = get_relevant_files(changed_files) relevant_files = get_relevant_files(changed_files)
if relevant_files: if relevant_files:
subject, commit_date = get_commit_summary(commit_hash) relevant_commit_inputs.append((commit_hash, relevant_files))
relevant_commits.append((commit_hash, subject, commit_date, relevant_files))
relevant_commits: List[CommitInfo] = []
if relevant_commit_inputs:
oss_repo_path = ensure_oss_repo(
oss_repo_path, args.oss_repo_url, args.oss_branch
)
oss_ref = f"origin/{args.oss_branch}"
oss_file_cache: Dict[str, Optional[Set[str]]] = {}
for commit_hash, relevant_files in relevant_commit_inputs:
subject, commit_date = get_commit_summary(repo_root, commit_hash)
added_lines_by_file = get_added_lines_by_file(
repo_root, commit_hash, relevant_files
)
synced_lines, total_added_lines = count_synced_lines(
added_lines_by_file, oss_repo_path, oss_ref, oss_file_cache
)
relevant_commits.append(
CommitInfo(
commit_hash=commit_hash,
subject=subject,
commit_date=commit_date,
relevant_files=relevant_files,
synced_lines=synced_lines,
total_added_lines=total_added_lines,
)
)
output = format_output(repo, last_sync_block, relevant_commits) output = format_output(repo, last_sync_block, relevant_commits)
print(output) print(output)
+9 -35
View File
@@ -31,45 +31,19 @@ import datetime
import os import os
import shutil import shutil
import subprocess import subprocess
import sys
import tempfile import tempfile
# --- Configuration Begin --- # Allow sibling imports regardless of the working directory.
# List of folders and files to copy from the OSS repo. sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
# Changes outside these paths will be ignored.
folder_names = [
"3rdparty",
"assets",
"benchmark",
"docker",
"docs",
"examples",
"python/sglang/lang",
"python/sglang/jit_kernel",
"python/sglang/srt",
"python/sglang/test",
"python/sglang/utils.py",
"python/sglang/README.md",
"sgl-kernel",
"test/manual",
"test/registered",
"test/srt",
"test/README.md",
"test/run_suite.py",
"README.md",
]
from utils import FOLDER_NAMES, write_github_step_summary # noqa: E402
# --- Configuration Begin ---
private_repo = "your-org/sglang-private-repo" private_repo = "your-org/sglang-private-repo"
# --- Configuration End --- # --- Configuration End ---
def write_github_step_summary(content):
if not os.environ.get("GITHUB_STEP_SUMMARY"):
return
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
f.write(content)
def check_dependencies(): def check_dependencies():
"""Check for required command-line tools.""" """Check for required command-line tools."""
if not shutil.which("git"): if not shutil.which("git"):
@@ -145,10 +119,10 @@ def get_source_folder(args):
return oss_root, temp_dir, commit_hash return oss_root, temp_dir, commit_hash
def sync_directories(oss_root, folder_names, dry_run): def sync_directories(oss_root, sync_paths, dry_run):
"""Sync specified directories from oss_root to current working directory.""" """Sync specified directories from oss_root to current working directory."""
rsync_commands = [] rsync_commands = []
for folder_name in folder_names: for folder_name in sync_paths:
target_name = f"{oss_root}/{folder_name}" target_name = f"{oss_root}/{folder_name}"
src_name = "./" + "/".join(folder_name.split("/")[:-1]) src_name = "./" + "/".join(folder_name.split("/")[:-1])
cmd = f"rsync -r --delete {target_name} {src_name}" cmd = f"rsync -r --delete {target_name} {src_name}"
@@ -259,7 +233,7 @@ def main():
try: try:
# Sync directories # Sync directories
sync_directories(oss_root, folder_names, args.dry_run) sync_directories(oss_root, FOLDER_NAMES, args.dry_run)
# Check for changes and create PR if necessary # Check for changes and create PR if necessary
if not check_for_changes(): if not check_for_changes():
+126 -69
View File
@@ -29,44 +29,20 @@ python3 scripts/copy_to_oss.py --commit <commit_hash> --dry-run
import argparse import argparse
import datetime import datetime
import os import os
import re
import shutil import shutil
import subprocess import subprocess
import sys
import tempfile import tempfile
# --- Configuration Begin --- # Allow sibling imports regardless of the working directory.
# List of folders and files to copy to the OSS repo. sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
# Changes outside these paths will be ignored.
folder_names = [
"3rdparty",
"assets",
"benchmark",
"docker",
"docs",
"examples",
"python/sglang/lang",
"python/sglang/jit_kernel",
"python/sglang/srt",
"python/sglang/test",
"python/sglang/utils.py",
"python/sglang/README.md",
"sgl-kernel",
"test/manual",
"test/registered",
"test/srt",
"test/README.md",
"test/run_suite.py",
"README.md",
]
# --- Configuration End --- from utils import ( # noqa: E402
FOLDER_NAMES,
find_latest_oss_sync_commit,
def write_github_step_summary(content): write_github_step_summary,
if not os.environ.get("GITHUB_STEP_SUMMARY"): )
return
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
f.write(content)
def get_commit_info(commit_ref): def get_commit_info(commit_ref):
@@ -131,7 +107,7 @@ def create_filtered_patch(commit_hash, dry_run):
# Filter the list of files # Filter the list of files
relevant_files = [ relevant_files = [
f for f in changed_files if any(f.startswith(path) for path in folder_names) f for f in changed_files if any(f.startswith(path) for path in FOLDER_NAMES)
] ]
if not relevant_files: if not relevant_files:
@@ -190,7 +166,7 @@ def get_oss_repo(dry_run):
print(f"\nCreated temporary directory for OSS repo: {temp_dir}") print(f"\nCreated temporary directory for OSS repo: {temp_dir}")
repo_url = f"https://{gh_token}@github.com/sgl-project/sglang.git" repo_url = f"https://{gh_token}@github.com/sgl-project/sglang.git"
command = ["git", "clone", "--branch", "main", repo_url, oss_root] command = ["git", "clone", repo_url, oss_root]
print(f"Run: {' '.join(command)}") print(f"Run: {' '.join(command)}")
if not dry_run: if not dry_run:
@@ -209,8 +185,9 @@ def _apply_patch(patch_file, dry_run):
""" """
Try to apply a patch, falling back to --3way merge if a clean apply fails. Try to apply a patch, falling back to --3way merge if a clean apply fails.
Returns True if the patch was applied successfully (clean or via --3way), Returns True if the patch was applied cleanly.
or raises an exception with full diagnostics if both methods fail. Returns False if conflicts were encountered (changes are still staged
with conflict markers so a PR can be created for manual resolution).
""" """
# --- Attempt 1: clean git apply --- # --- Attempt 1: clean git apply ---
apply_cmd = ["git", "apply", patch_file] apply_cmd = ["git", "apply", patch_file]
@@ -235,8 +212,8 @@ def _apply_patch(patch_file, dry_run):
print("✅ Patch applied via --3way merge (no conflicts).") print("✅ Patch applied via --3way merge (no conflicts).")
return True return True
# --- Both attempts failed — gather diagnostics --- # --- --3way left conflict markers in the working tree ---
print(f"❌ --3way merge also failed:\n{result_3way.stderr.strip()}\n") print(f"⚠️ --3way merge had conflicts:\n{result_3way.stderr.strip()}\n")
# Show which hunks conflict # Show which hunks conflict
check_cmd = ["git", "apply", "--check", "--verbose", patch_file] check_cmd = ["git", "apply", "--check", "--verbose", patch_file]
@@ -272,7 +249,7 @@ def _apply_patch(patch_file, dry_run):
# Write a rich summary to the GitHub Actions step summary # Write a rich summary to the GitHub Actions step summary
summary_lines = [ summary_lines = [
"\n## ❌ Patch could not be applied automatically\n", "\n## ⚠️ Patch had conflicts — PR created for manual resolution\n",
"### Conflict details\n", "### Conflict details\n",
f"```\n{conflict_details}\n```\n", f"```\n{conflict_details}\n```\n",
] ]
@@ -287,18 +264,22 @@ def _apply_patch(patch_file, dry_run):
) )
write_github_step_summary("".join(summary_lines)) write_github_step_summary("".join(summary_lines))
# Reset the working tree so the finally-block cleanup is safe return False
subprocess.run(["git", "checkout", "."], capture_output=True, text=True)
raise RuntimeError(
"Patch could not be applied. See the CI log and GitHub Actions "
"step summary for the full patch and conflict details."
)
def apply_patch_and_push(oss_root, patch_file, branch_name, commit_message, dry_run): def apply_patch_and_push(
oss_root, patch_file, branch_name, commit_message, base_oss_commit, dry_run
):
""" """
In the OSS repo, create a branch, apply the patch, commit, and push. In the OSS repo, create a branch from base_oss_commit, apply the patch,
commit, and push.
Args:
base_oss_commit: The OSS commit hash to branch from (the last sync
point). If None, the current HEAD (main) is used.
Returns True if the patch applied cleanly, False if there were conflicts
(the conflicted state is still committed and pushed so a PR can be opened).
""" """
print("\nApplying patch and pushing to OSS repo...") print("\nApplying patch and pushing to OSS repo...")
@@ -306,15 +287,19 @@ def apply_patch_and_push(oss_root, patch_file, branch_name, commit_message, dry_
if not dry_run: if not dry_run:
os.chdir(oss_root) os.chdir(oss_root)
applied_cleanly = True
try: try:
# Create a new branch # Check out a new branch from the base OSS commit
checkout_cmd = ["git", "checkout", "-b", branch_name] if base_oss_commit:
checkout_cmd = ["git", "checkout", "-b", branch_name, base_oss_commit]
else:
checkout_cmd = ["git", "checkout", "-b", branch_name]
print(f"Run: {' '.join(checkout_cmd)}") print(f"Run: {' '.join(checkout_cmd)}")
if not dry_run: if not dry_run:
subprocess.run(checkout_cmd, check=True, capture_output=True, text=True) subprocess.run(checkout_cmd, check=True, capture_output=True, text=True)
# Apply the patch (with --3way fallback and diagnostics) # Apply the patch (with --3way fallback and diagnostics)
_apply_patch(patch_file, dry_run) applied_cleanly = _apply_patch(patch_file, dry_run)
# Configure git user and stage changes # Configure git user and stage changes
post_apply_commands = [ post_apply_commands = [
@@ -359,7 +344,15 @@ def apply_patch_and_push(oss_root, patch_file, branch_name, commit_message, dry_
if not dry_run: if not dry_run:
os.chdir(original_cwd) os.chdir(original_cwd)
print("✅ Branch created, patch applied, and pushed successfully.") if applied_cleanly:
print("✅ Branch created, patch applied cleanly, and pushed successfully.")
else:
print(
"⚠️ Branch created and pushed with conflict markers. "
"A PR will be opened for manual resolution."
)
return applied_cleanly
def create_pull_request(oss_root, branch_name, title, body, dry_run): def create_pull_request(oss_root, branch_name, title, body, dry_run):
@@ -433,6 +426,36 @@ def get_commit_author(commit_hash):
raise raise
def get_all_co_author_lines(commit_hash, commit_message):
"""
Build a deduplicated list of Co-authored-by lines that includes both
the primary commit author and any Co-authored-by trailers already
present in the commit message.
Returns a list of unique "Co-authored-by: Name <email>" strings.
"""
seen = set()
co_author_lines = []
def _add(name, email):
key = (name.strip(), email.strip().lower())
if key not in seen:
seen.add(key)
co_author_lines.append(f"Co-authored-by: {name.strip()} <{email.strip()}>")
# 1. Primary author of the commit
author_name, author_email = get_commit_author(commit_hash)
_add(author_name, author_email)
# 2. Existing Co-authored-by trailers in the commit message
for line in commit_message.splitlines():
m = re.match(r"^\s*Co-authored-by:\s*(.+?)\s*<([^>]+)>", line, re.IGNORECASE)
if m:
_add(m.group(1), m.group(2))
return co_author_lines
def main(): def main():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Copy a commit from the private repo to OSS and open a PR." description="Copy a commit from the private repo to OSS and open a PR."
@@ -487,33 +510,67 @@ def main():
# 2. Get the OSS repo # 2. Get the OSS repo
oss_root, temp_dir = get_oss_repo(args.dry_run) oss_root, temp_dir = get_oss_repo(args.dry_run)
# 3. Get original commit author for the co-author line # 3. Find the latest OSS commit that was synced into sglang-private.
author_name, author_email = get_commit_author(commit_hash) # This is the correct base for our patch, since the private repo's
# code is based on this sync point.
base_oss_commit = find_latest_oss_sync_commit()
if base_oss_commit:
print(f"ℹ️ Will branch from OSS commit {base_oss_commit}")
else:
print(
"⚠️ Could not determine latest OSS sync commit. "
"Falling back to OSS main HEAD."
)
# 4. Prepare content for the commit and PR based on changed files # 4. Get all co-author lines (primary author + trailers from commit message)
co_author_lines = get_all_co_author_lines(commit_hash, original_commit_message)
authors_display = "\n".join(co_author_lines)
# 5. Prepare content for the commit and PR based on changed files
file_list_str = "\n".join([f"- {f}" for f in relevant_files]) file_list_str = "\n".join([f"- {f}" for f in relevant_files])
filename_list_str = ", ".join([f.split("/")[-1] for f in relevant_files]) filename_list_str = ", ".join([f.split("/")[-1] for f in relevant_files])
if len(filename_list_str) > 40: if len(filename_list_str) > 40:
filename_list_str = filename_list_str[:40] + "..." filename_list_str = filename_list_str[:40] + "..."
current_date = datetime.datetime.now().strftime("%Y%m%d") current_date = datetime.datetime.now().strftime("%Y%m%d")
pr_title = f"[Auto Sync] Update {filename_list_str} ({current_date})" pr_title = f"[Auto Sync] Update {filename_list_str} ({current_date})"
pr_body = (
f"Sync changes from commit `{short_hash}`.\n\n"
f"**Files Changed:**\n{file_list_str}\n\n"
f"Author: {author_name} <{author_email}>"
f"\n\n---\n\n"
f"*This is an automated PR created by scripts/copy_from_oss.py.*"
)
# 5. Create branch, apply patch, and push # 6. Create branch from the last synced OSS commit, apply patch, and push
branch_name = f"sync-{short_hash}-{current_date}" branch_name = f"sync-{short_hash}-{current_date}"
co_author_line = f"Co-authored-by: {author_name} <{author_email}>" co_authors_block = "\n".join(co_author_lines)
commit_message = f"{pr_title}\n\n{co_author_line}" commit_message = f"{pr_title}\n\n{co_authors_block}"
apply_patch_and_push( applied_cleanly = apply_patch_and_push(
oss_root, patch_file, branch_name, commit_message, args.dry_run oss_root,
patch_file,
branch_name,
commit_message,
base_oss_commit,
args.dry_run,
) )
# 6. Create Pull Request # 7. Adjust PR title and body when there are conflicts
if not applied_cleanly:
pr_title = (
f"[Auto Sync][⚠️ Conflicts] Update {filename_list_str} ({current_date})"
)
pr_body_parts = [
f"Sync changes from commit `{short_hash}`.\n",
f"**Files Changed:**\n{file_list_str}\n",
f"**Authors:**\n{authors_display}",
]
if not applied_cleanly:
pr_body_parts.append(
"\n\n⚠️ **This patch had merge conflicts.** "
"The branch contains conflict markers that must be resolved manually. "
"Please check the CI logs for the full patch and conflict details."
)
pr_body_parts.append(
f"\n\n---\n\n"
f"*This is an automated PR created by scripts/copy_to_oss.py.*"
)
pr_body = "\n".join(pr_body_parts)
# 8. Create Pull Request
create_pull_request(oss_root, branch_name, pr_title, pr_body, args.dry_run) create_pull_request(oss_root, branch_name, pr_title, pr_body, args.dry_run)
finally: finally:
+136
View File
@@ -0,0 +1,136 @@
"""
Shared constants and helpers for code-sync scripts.
"""
import os
import re
import subprocess
from typing import Optional
# --- Configuration Begin ---
# List of folders and files to copy to / from the OSS repo.
# Changes outside these paths will be ignored.
FOLDER_NAMES = [
"3rdparty",
"assets",
"benchmark",
"docker",
"docs",
"examples",
"python/sglang/lang",
"python/sglang/jit_kernel",
"python/sglang/srt",
"python/sglang/test",
"python/sglang/utils.py",
"python/sglang/README.md",
"sgl-kernel",
"test/manual",
"test/registered",
"test/srt",
"test/README.md",
"test/run_suite.py",
"README.md",
]
SYNC_COMMIT_PREFIX = r"\[Automated PR\] Copy OSS code from commit"
# --- Configuration End ---
def write_github_step_summary(content: str) -> None:
"""Append *content* to the GitHub Actions step summary (no-op outside CI)."""
summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
if not summary_path:
return
with open(summary_path, "a") as f:
f.write(content)
def get_last_sync_commit(repo_root: Optional[str] = None) -> Optional[str]:
"""
Find the most recent sync commit that copied from OSS.
Returns the full private-repo commit hash, or None if not found.
The match is restricted to commits whose **subject** starts with the
sync prefix so that unrelated commits mentioning the phrase in their
body are ignored.
"""
subject_pattern = re.compile("^" + SYNC_COMMIT_PREFIX)
try:
cmd = [
"git",
"log",
"--all",
"--grep",
SYNC_COMMIT_PREFIX,
"--format=%H %s",
]
result = subprocess.run(
cmd,
capture_output=True,
text=True,
check=True,
cwd=repo_root,
).stdout.strip()
for line in result.splitlines():
# Format: "<full_hash> <subject>"
parts = line.split(" ", 1)
if len(parts) != 2:
continue
commit_hash, subject = parts
if subject_pattern.search(subject):
return commit_hash
return None
except subprocess.CalledProcessError as e:
print(f"Error finding last sync commit: {e.stderr}")
return None
def find_latest_oss_sync_commit(repo_root: Optional[str] = None) -> Optional[str]:
"""
Search the private repo history for the latest commit whose **subject**
matches "[Automated PR] Copy OSS code from commit {commit_id} on {date}"
and return the embedded **OSS** commit hash.
Returns the short OSS commit hash string, or None if not found.
"""
oss_hash_pattern = re.compile("^" + SYNC_COMMIT_PREFIX + r" ([0-9a-f]+)")
try:
# --grep filters on the full message body, so we request subject-only
# output and validate the pattern against the subject ourselves.
result = subprocess.run(
[
"git",
"log",
"--all",
"--grep",
SYNC_COMMIT_PREFIX,
"--pretty=%s",
],
capture_output=True,
text=True,
check=True,
cwd=repo_root,
)
for subject in result.stdout.strip().splitlines():
m = oss_hash_pattern.search(subject)
if m:
oss_commit = m.group(1)
print(
f"✅ Latest OSS sync commit found: {oss_commit} "
f"(from: {subject})"
)
return oss_commit
print(
"⚠️ No '[Automated PR] Copy OSS code from commit ...' " "found in history."
)
return None
except subprocess.CalledProcessError as e:
print(f"Error searching for OSS sync commits: {e.stderr.strip()}")
return None