[Auto Sync] Update grpc_request_manager.py, tokenizer_manag... (20260214) (#18838)
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
github-actions[bot]
parent
190fa8246f
commit
b33769786f
@@ -983,8 +983,8 @@ class GrpcRequestManager:
|
|||||||
|
|
||||||
self.event_loop = loop
|
self.event_loop = loop
|
||||||
|
|
||||||
# We cannot add signal handler when the grpc manager is not in
|
# We only add signal handler when the tokenizer manager is in the main thread
|
||||||
# the main thread due to the CPython limitation.
|
# due to the CPython limitation.
|
||||||
if threading.current_thread() is threading.main_thread():
|
if threading.current_thread() is threading.main_thread():
|
||||||
signal_handler = GrpcSignalHandler(self)
|
signal_handler = GrpcSignalHandler(self)
|
||||||
loop.add_signal_handler(signal.SIGTERM, signal_handler.sigterm_handler)
|
loop.add_signal_handler(signal.SIGTERM, signal_handler.sigterm_handler)
|
||||||
@@ -992,12 +992,7 @@ class GrpcRequestManager:
|
|||||||
loop.add_signal_handler(
|
loop.add_signal_handler(
|
||||||
signal.SIGQUIT, signal_handler.running_phase_sigquit_handler
|
signal.SIGQUIT, signal_handler.running_phase_sigquit_handler
|
||||||
)
|
)
|
||||||
else:
|
|
||||||
logger.warning(
|
|
||||||
"Signal handler is not added because the grpc request manager is "
|
|
||||||
"not in the main thread. This disables graceful shutdown of the "
|
|
||||||
"grpc request manager when SIGTERM is received."
|
|
||||||
)
|
|
||||||
self.asyncio_tasks.add(
|
self.asyncio_tasks.add(
|
||||||
loop.create_task(print_exception_wrapper(self.sigterm_watchdog))
|
loop.create_task(print_exception_wrapper(self.sigterm_watchdog))
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1451,6 +1451,8 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
|
|||||||
)
|
)
|
||||||
self.event_loop = loop
|
self.event_loop = loop
|
||||||
|
|
||||||
|
# We only add signal handler when the tokenizer manager is in the main thread
|
||||||
|
# due to the CPython limitation.
|
||||||
if threading.current_thread() is threading.main_thread():
|
if threading.current_thread() is threading.main_thread():
|
||||||
signal_handler = self.signal_handler_class(self)
|
signal_handler = self.signal_handler_class(self)
|
||||||
loop.add_signal_handler(signal.SIGTERM, signal_handler.sigterm_handler)
|
loop.add_signal_handler(signal.SIGTERM, signal_handler.sigterm_handler)
|
||||||
@@ -1458,14 +1460,6 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
|
|||||||
loop.add_signal_handler(
|
loop.add_signal_handler(
|
||||||
signal.SIGQUIT, signal_handler.running_phase_sigquit_handler
|
signal.SIGQUIT, signal_handler.running_phase_sigquit_handler
|
||||||
)
|
)
|
||||||
else:
|
|
||||||
# We cannot add signal handler when the tokenizer manager is not in
|
|
||||||
# the main thread due to the CPython limitation.
|
|
||||||
logger.warning(
|
|
||||||
"Signal handler is not added because the tokenizer manager is "
|
|
||||||
"not in the main thread. This disables graceful shutdown of the "
|
|
||||||
"tokenizer manager when SIGTERM is received."
|
|
||||||
)
|
|
||||||
|
|
||||||
self.asyncio_tasks.add(
|
self.asyncio_tasks.add(
|
||||||
loop.create_task(print_exception_wrapper(self.sigterm_watchdog))
|
loop.create_task(print_exception_wrapper(self.sigterm_watchdog))
|
||||||
|
|||||||
@@ -1127,10 +1127,6 @@ def check_pkg_version_at_least(pkg: str, min_version: str) -> bool:
|
|||||||
|
|
||||||
def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = None):
|
def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = None):
|
||||||
"""Kill the process and all its child processes."""
|
"""Kill the process and all its child processes."""
|
||||||
# Remove sigchld handler to avoid spammy logs.
|
|
||||||
if threading.current_thread() is threading.main_thread():
|
|
||||||
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
|
|
||||||
|
|
||||||
if parent_pid is None:
|
if parent_pid is None:
|
||||||
parent_pid = os.getpid()
|
parent_pid = os.getpid()
|
||||||
include_parent = False
|
include_parent = False
|
||||||
|
|||||||
@@ -2,13 +2,15 @@
|
|||||||
List commits in the private repo that need to be synced to the OSS repo.
|
List commits in the private repo that need to be synced to the OSS repo.
|
||||||
|
|
||||||
NOTE:
|
NOTE:
|
||||||
1. You need to execute this script in the git root folder.
|
1. This script resolves the git root automatically and can be run anywhere
|
||||||
|
inside the repo.
|
||||||
|
|
||||||
This script will:
|
This script will:
|
||||||
1. Find the most recent sync commit (message starts with
|
1. Find the most recent sync commit (message starts with
|
||||||
"[Automated PR] Copy OSS code from commit").
|
"[Automated PR] Copy OSS code from commit").
|
||||||
2. Scan commits after that point and keep those that touch the configured paths.
|
2. Scan commits after that point and keep those that touch the configured paths.
|
||||||
3. Print a markdown summary with commit links and write it to GitHub Step Summary.
|
3. Compare added diff lines in relevant files against OSS main.
|
||||||
|
4. Print a markdown summary with commit links and write it to GitHub Step Summary.
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
python3 scripts/code_sync/check_commits.py
|
python3 scripts/code_sync/check_commits.py
|
||||||
@@ -18,41 +20,35 @@ import argparse
|
|||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
from typing import List, Optional, Tuple
|
import sys
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Dict, List, Optional, Set, Tuple
|
||||||
|
|
||||||
|
# Allow sibling imports regardless of the working directory.
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
|
||||||
|
from utils import ( # noqa: E402
|
||||||
|
FOLDER_NAMES,
|
||||||
|
get_last_sync_commit,
|
||||||
|
write_github_step_summary,
|
||||||
|
)
|
||||||
|
|
||||||
# --- Configuration Begin ---
|
# --- Configuration Begin ---
|
||||||
# List of folders and files to copy to the OSS repo.
|
|
||||||
# Changes outside these paths will be ignored.
|
|
||||||
folder_names = [
|
|
||||||
"3rdparty",
|
|
||||||
"assets",
|
|
||||||
"benchmark",
|
|
||||||
"docker",
|
|
||||||
"docs",
|
|
||||||
"examples",
|
|
||||||
"python/sglang/lang",
|
|
||||||
"python/sglang/jit_kernel",
|
|
||||||
"python/sglang/srt",
|
|
||||||
"python/sglang/test",
|
|
||||||
"python/sglang/utils.py",
|
|
||||||
"python/sglang/README.md",
|
|
||||||
"sgl-kernel",
|
|
||||||
"test/manual",
|
|
||||||
"test/registered",
|
|
||||||
"test/srt",
|
|
||||||
"test/README.md",
|
|
||||||
"test/run_suite.py",
|
|
||||||
"README.md",
|
|
||||||
]
|
|
||||||
|
|
||||||
private_repo = "your-org/sglang-private-repo"
|
private_repo = "your-org/sglang-private-repo"
|
||||||
sync_commit_prefix = r"\[Automated PR\] Copy OSS code from commit"
|
oss_repo_url = "https://github.com/sgl-project/sglang.git"
|
||||||
|
oss_repo_branch = "main"
|
||||||
|
default_oss_repo_dir = ".oss_repo"
|
||||||
# --- Configuration End ---
|
# --- Configuration End ---
|
||||||
|
|
||||||
|
|
||||||
def write_github_step_summary(content: str) -> None:
|
@dataclass
|
||||||
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
|
class CommitInfo:
|
||||||
f.write(content)
|
commit_hash: str
|
||||||
|
subject: str
|
||||||
|
commit_date: str
|
||||||
|
relevant_files: List[str]
|
||||||
|
synced_lines: int
|
||||||
|
total_added_lines: int
|
||||||
|
|
||||||
|
|
||||||
def check_dependencies() -> None:
|
def check_dependencies() -> None:
|
||||||
@@ -61,7 +57,23 @@ def check_dependencies() -> None:
|
|||||||
raise EnvironmentError("git is not installed or not in PATH.")
|
raise EnvironmentError("git is not installed or not in PATH.")
|
||||||
|
|
||||||
|
|
||||||
def get_repo_from_origin() -> str:
|
def get_repo_root() -> str:
|
||||||
|
try:
|
||||||
|
output = subprocess.run(
|
||||||
|
["git", "rev-parse", "--show-toplevel"],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=True,
|
||||||
|
).stdout.strip()
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
raise RuntimeError(f"Unable to determine git repo root: {e.stderr or e}") from e
|
||||||
|
|
||||||
|
if not output:
|
||||||
|
raise RuntimeError("Unable to determine git repo root.")
|
||||||
|
return os.path.abspath(output)
|
||||||
|
|
||||||
|
|
||||||
|
def get_repo_from_origin(repo_root: str) -> str:
|
||||||
"""Try to infer the repo slug (owner/name) from git remote.origin.url."""
|
"""Try to infer the repo slug (owner/name) from git remote.origin.url."""
|
||||||
try:
|
try:
|
||||||
url = subprocess.run(
|
url = subprocess.run(
|
||||||
@@ -69,6 +81,7 @@ def get_repo_from_origin() -> str:
|
|||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
check=True,
|
check=True,
|
||||||
|
cwd=repo_root,
|
||||||
).stdout.strip()
|
).stdout.strip()
|
||||||
except subprocess.CalledProcessError:
|
except subprocess.CalledProcessError:
|
||||||
return private_repo
|
return private_repo
|
||||||
@@ -85,29 +98,48 @@ def get_repo_from_origin() -> str:
|
|||||||
return repo or private_repo
|
return repo or private_repo
|
||||||
|
|
||||||
|
|
||||||
def get_last_sync_commit() -> Optional[str]:
|
def get_default_oss_repo_path(repo_root: str) -> str:
|
||||||
"""Find the most recent sync commit that copied from OSS."""
|
env_path = os.environ.get("OSS_REPO_PATH")
|
||||||
try:
|
if env_path:
|
||||||
result = subprocess.run(
|
return os.path.abspath(env_path)
|
||||||
[
|
return os.path.abspath(os.path.join(repo_root, default_oss_repo_dir))
|
||||||
"git",
|
|
||||||
"log",
|
|
||||||
"-1",
|
def ensure_oss_repo(oss_repo_path: str, repo_url: str, branch: str) -> str:
|
||||||
"--grep",
|
oss_repo_path = os.path.abspath(oss_repo_path)
|
||||||
sync_commit_prefix,
|
if os.path.exists(oss_repo_path) and not os.path.isdir(oss_repo_path):
|
||||||
"--format=%H",
|
raise RuntimeError(f"OSS repo path is not a directory: {oss_repo_path}")
|
||||||
],
|
|
||||||
capture_output=True,
|
if os.path.isdir(os.path.join(oss_repo_path, ".git")):
|
||||||
text=True,
|
try:
|
||||||
|
subprocess.run(
|
||||||
|
["git", "-C", oss_repo_path, "rev-parse", "--is-inside-work-tree"],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"OSS repo path exists but is not a git repo: {oss_repo_path}"
|
||||||
|
) from e
|
||||||
|
|
||||||
|
subprocess.run(
|
||||||
|
["git", "-C", oss_repo_path, "fetch", "origin", branch, "--depth", "1"],
|
||||||
check=True,
|
check=True,
|
||||||
).stdout.strip()
|
)
|
||||||
return result or None
|
return oss_repo_path
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
print(f"Error finding last sync commit: {e.stderr}")
|
parent_dir = os.path.dirname(oss_repo_path)
|
||||||
return None
|
if parent_dir and not os.path.isdir(parent_dir):
|
||||||
|
os.makedirs(parent_dir, exist_ok=True)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "clone", "--depth", "1", "--branch", branch, repo_url, oss_repo_path],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
return oss_repo_path
|
||||||
|
|
||||||
|
|
||||||
def get_commits_since(last_sync_hash: Optional[str]) -> List[str]:
|
def get_commits_since(repo_root: str, last_sync_hash: Optional[str]) -> List[str]:
|
||||||
"""Get commit hashes from last sync commit (exclusive) to HEAD."""
|
"""Get commit hashes from last sync commit (exclusive) to HEAD."""
|
||||||
try:
|
try:
|
||||||
if last_sync_hash:
|
if last_sync_hash:
|
||||||
@@ -115,7 +147,7 @@ def get_commits_since(last_sync_hash: Optional[str]) -> List[str]:
|
|||||||
else:
|
else:
|
||||||
command = ["git", "rev-list", "HEAD"]
|
command = ["git", "rev-list", "HEAD"]
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
command, capture_output=True, text=True, check=True
|
command, capture_output=True, text=True, check=True, cwd=repo_root
|
||||||
).stdout.strip()
|
).stdout.strip()
|
||||||
return [line for line in result.split("\n") if line]
|
return [line for line in result.split("\n") if line]
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
@@ -123,13 +155,14 @@ def get_commits_since(last_sync_hash: Optional[str]) -> List[str]:
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
def get_changed_files(commit_hash: str) -> List[str]:
|
def get_changed_files(repo_root: str, commit_hash: str) -> List[str]:
|
||||||
try:
|
try:
|
||||||
output = subprocess.run(
|
output = subprocess.run(
|
||||||
["git", "diff-tree", "--no-commit-id", "--name-only", "-r", commit_hash],
|
["git", "diff-tree", "--no-commit-id", "--name-only", "-r", commit_hash],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
check=True,
|
check=True,
|
||||||
|
cwd=repo_root,
|
||||||
).stdout.strip()
|
).stdout.strip()
|
||||||
return [line for line in output.split("\n") if line]
|
return [line for line in output.split("\n") if line]
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
@@ -147,11 +180,115 @@ def get_relevant_files(changed_files: List[str]) -> List[str]:
|
|||||||
return [
|
return [
|
||||||
changed_file
|
changed_file
|
||||||
for changed_file in changed_files
|
for changed_file in changed_files
|
||||||
if any(is_relevant_path(changed_file, path) for path in folder_names)
|
if any(is_relevant_path(changed_file, path) for path in FOLDER_NAMES)
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def get_commit_summary(commit_hash: str) -> Tuple[str, str]:
|
def get_added_lines_by_file(
|
||||||
|
repo_root: str, commit_hash: str, relevant_files: List[str]
|
||||||
|
) -> Dict[str, List[str]]:
|
||||||
|
if not relevant_files:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
command = [
|
||||||
|
"git",
|
||||||
|
"show",
|
||||||
|
"--no-color",
|
||||||
|
"--unified=0",
|
||||||
|
"--format=",
|
||||||
|
commit_hash,
|
||||||
|
"--",
|
||||||
|
] + relevant_files
|
||||||
|
try:
|
||||||
|
output = subprocess.run(
|
||||||
|
command, capture_output=True, text=True, check=True, cwd=repo_root
|
||||||
|
).stdout
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
print(f"Error getting diff for {commit_hash}: {e.stderr}")
|
||||||
|
return {}
|
||||||
|
|
||||||
|
added_lines: Dict[str, List[str]] = {path: [] for path in relevant_files}
|
||||||
|
relevant_set = set(relevant_files)
|
||||||
|
current_file: Optional[str] = None
|
||||||
|
for line in output.splitlines():
|
||||||
|
if line.startswith("diff --git "):
|
||||||
|
current_file = None
|
||||||
|
continue
|
||||||
|
if line.startswith("+++ "):
|
||||||
|
file_path = None
|
||||||
|
if line.startswith("+++ b/"):
|
||||||
|
file_path = line[6:]
|
||||||
|
else:
|
||||||
|
candidate = line[4:]
|
||||||
|
if candidate == "/dev/null":
|
||||||
|
file_path = None
|
||||||
|
elif candidate.startswith("b/") or candidate.startswith("a/"):
|
||||||
|
file_path = candidate[2:]
|
||||||
|
else:
|
||||||
|
file_path = candidate
|
||||||
|
|
||||||
|
if file_path in relevant_set:
|
||||||
|
current_file = file_path
|
||||||
|
else:
|
||||||
|
current_file = None
|
||||||
|
continue
|
||||||
|
|
||||||
|
if current_file and line.startswith("+") and not line.startswith("+++ "):
|
||||||
|
added_lines[current_file].append(line[1:])
|
||||||
|
|
||||||
|
return added_lines
|
||||||
|
|
||||||
|
|
||||||
|
def get_oss_file_lines(
|
||||||
|
oss_repo_path: str,
|
||||||
|
oss_ref: str,
|
||||||
|
file_path: str,
|
||||||
|
cache: Dict[str, Optional[Set[str]]],
|
||||||
|
) -> Optional[Set[str]]:
|
||||||
|
if file_path in cache:
|
||||||
|
return cache[file_path]
|
||||||
|
try:
|
||||||
|
output = subprocess.run(
|
||||||
|
["git", "-C", oss_repo_path, "show", f"{oss_ref}:{file_path}"],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
errors="replace",
|
||||||
|
check=True,
|
||||||
|
).stdout
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
cache[file_path] = None
|
||||||
|
return None
|
||||||
|
|
||||||
|
lines = output.splitlines()
|
||||||
|
line_set = set(lines)
|
||||||
|
cache[file_path] = line_set
|
||||||
|
return line_set
|
||||||
|
|
||||||
|
|
||||||
|
def count_synced_lines(
|
||||||
|
added_lines_by_file: Dict[str, List[str]],
|
||||||
|
oss_repo_path: str,
|
||||||
|
oss_ref: str,
|
||||||
|
oss_file_cache: Dict[str, Optional[Set[str]]],
|
||||||
|
) -> Tuple[int, int]:
|
||||||
|
total_added_lines = 0
|
||||||
|
synced_lines = 0
|
||||||
|
for file_path, lines in added_lines_by_file.items():
|
||||||
|
total_added_lines += len(lines)
|
||||||
|
if not lines:
|
||||||
|
continue
|
||||||
|
oss_lines = get_oss_file_lines(
|
||||||
|
oss_repo_path, oss_ref, file_path, oss_file_cache
|
||||||
|
)
|
||||||
|
if not oss_lines:
|
||||||
|
continue
|
||||||
|
for line in lines:
|
||||||
|
if line in oss_lines:
|
||||||
|
synced_lines += 1
|
||||||
|
return synced_lines, total_added_lines
|
||||||
|
|
||||||
|
|
||||||
|
def get_commit_summary(repo_root: str, commit_hash: str) -> Tuple[str, str]:
|
||||||
"""Return (subject, date) for a commit."""
|
"""Return (subject, date) for a commit."""
|
||||||
try:
|
try:
|
||||||
output = subprocess.run(
|
output = subprocess.run(
|
||||||
@@ -159,6 +296,7 @@ def get_commit_summary(commit_hash: str) -> Tuple[str, str]:
|
|||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
check=True,
|
check=True,
|
||||||
|
cwd=repo_root,
|
||||||
).stdout.strip()
|
).stdout.strip()
|
||||||
subject, commit_date = output.split("\x00", 1)
|
subject, commit_date = output.split("\x00", 1)
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
@@ -195,13 +333,20 @@ def format_commit_block(
|
|||||||
commit_hash: str,
|
commit_hash: str,
|
||||||
commit_date: str,
|
commit_date: str,
|
||||||
relevant_files: List[str],
|
relevant_files: List[str],
|
||||||
|
synced_lines: int,
|
||||||
|
total_added_lines: int,
|
||||||
) -> str:
|
) -> str:
|
||||||
short_hash = commit_hash[:9]
|
short_hash = commit_hash[:9]
|
||||||
commit_url = f"https://github.com/{repo}/commit/{commit_hash}"
|
commit_url = f"https://github.com/{repo}/commit/{commit_hash}"
|
||||||
files_str = format_files_list(relevant_files) if relevant_files else "- None"
|
files_str = format_files_list(relevant_files) if relevant_files else "- None"
|
||||||
|
status_icon = "✅" if synced_lines == total_added_lines else "❌"
|
||||||
|
status_line = (
|
||||||
|
f"status: {status_icon} {synced_lines}/{total_added_lines} lines synced"
|
||||||
|
)
|
||||||
return "\n".join(
|
return "\n".join(
|
||||||
[
|
[
|
||||||
f"#### {subject}",
|
f"#### {subject}",
|
||||||
|
status_line,
|
||||||
f"date: {commit_date}",
|
f"date: {commit_date}",
|
||||||
"files to sync:",
|
"files to sync:",
|
||||||
files_str,
|
files_str,
|
||||||
@@ -215,7 +360,7 @@ def format_commit_block(
|
|||||||
def format_output(
|
def format_output(
|
||||||
repo: str,
|
repo: str,
|
||||||
last_sync: Optional[Tuple[str, str, str]],
|
last_sync: Optional[Tuple[str, str, str]],
|
||||||
commits: List[Tuple[str, str, str, List[str]]],
|
commits: List[CommitInfo],
|
||||||
) -> str:
|
) -> str:
|
||||||
lines: List[str] = []
|
lines: List[str] = []
|
||||||
if last_sync:
|
if last_sync:
|
||||||
@@ -229,9 +374,17 @@ def format_output(
|
|||||||
lines.append("No commits need to be synced.")
|
lines.append("No commits need to be synced.")
|
||||||
return "\n".join(lines) + "\n"
|
return "\n".join(lines) + "\n"
|
||||||
|
|
||||||
for commit_hash, subject, commit_date, relevant_files in commits:
|
for commit in commits:
|
||||||
lines.append(
|
lines.append(
|
||||||
format_commit_block(repo, subject, commit_hash, commit_date, relevant_files)
|
format_commit_block(
|
||||||
|
repo,
|
||||||
|
commit.subject,
|
||||||
|
commit.commit_hash,
|
||||||
|
commit.commit_date,
|
||||||
|
commit.relevant_files,
|
||||||
|
commit.synced_lines,
|
||||||
|
commit.total_added_lines,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
@@ -247,30 +400,78 @@ def main() -> None:
|
|||||||
default=0,
|
default=0,
|
||||||
help="Limit number of commits printed (0 means no limit).",
|
help="Limit number of commits printed (0 means no limit).",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--oss-repo-path",
|
||||||
|
default=None,
|
||||||
|
help="Path to OSS repo clone (default: $OSS_REPO_PATH or .oss_repo).",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--oss-repo-url",
|
||||||
|
default=oss_repo_url,
|
||||||
|
help="OSS repo URL (default: https://github.com/sgl-project/sglang.git).",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--oss-branch",
|
||||||
|
default=oss_repo_branch,
|
||||||
|
help="OSS repo branch to check (default: main).",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
check_dependencies()
|
check_dependencies()
|
||||||
|
repo_root = get_repo_root()
|
||||||
|
oss_repo_path = (
|
||||||
|
os.path.abspath(args.oss_repo_path)
|
||||||
|
if args.oss_repo_path
|
||||||
|
else get_default_oss_repo_path(repo_root)
|
||||||
|
)
|
||||||
|
|
||||||
repo = get_repo_from_origin()
|
repo = get_repo_from_origin(repo_root)
|
||||||
last_sync_hash = get_last_sync_commit()
|
last_sync_hash = get_last_sync_commit(repo_root)
|
||||||
last_sync_block = None
|
last_sync_block = None
|
||||||
if last_sync_hash:
|
if last_sync_hash:
|
||||||
last_sync_subject, last_sync_date = get_commit_summary(last_sync_hash)
|
last_sync_subject, last_sync_date = get_commit_summary(
|
||||||
|
repo_root, last_sync_hash
|
||||||
|
)
|
||||||
last_sync_block = (last_sync_subject, last_sync_hash, last_sync_date)
|
last_sync_block = (last_sync_subject, last_sync_hash, last_sync_date)
|
||||||
|
|
||||||
commits = get_commits_since(last_sync_hash)
|
commits = get_commits_since(repo_root, last_sync_hash)
|
||||||
if args.limit > 0:
|
if args.limit > 0:
|
||||||
commits = commits[: args.limit]
|
commits = commits[: args.limit]
|
||||||
|
|
||||||
relevant_commits: List[Tuple[str, str, str, List[str]]] = []
|
relevant_commit_inputs: List[Tuple[str, List[str]]] = []
|
||||||
for commit_hash in commits:
|
for commit_hash in commits:
|
||||||
changed_files = get_changed_files(commit_hash)
|
changed_files = get_changed_files(repo_root, commit_hash)
|
||||||
if not changed_files:
|
if not changed_files:
|
||||||
continue
|
continue
|
||||||
relevant_files = get_relevant_files(changed_files)
|
relevant_files = get_relevant_files(changed_files)
|
||||||
if relevant_files:
|
if relevant_files:
|
||||||
subject, commit_date = get_commit_summary(commit_hash)
|
relevant_commit_inputs.append((commit_hash, relevant_files))
|
||||||
relevant_commits.append((commit_hash, subject, commit_date, relevant_files))
|
|
||||||
|
relevant_commits: List[CommitInfo] = []
|
||||||
|
if relevant_commit_inputs:
|
||||||
|
oss_repo_path = ensure_oss_repo(
|
||||||
|
oss_repo_path, args.oss_repo_url, args.oss_branch
|
||||||
|
)
|
||||||
|
oss_ref = f"origin/{args.oss_branch}"
|
||||||
|
oss_file_cache: Dict[str, Optional[Set[str]]] = {}
|
||||||
|
for commit_hash, relevant_files in relevant_commit_inputs:
|
||||||
|
subject, commit_date = get_commit_summary(repo_root, commit_hash)
|
||||||
|
added_lines_by_file = get_added_lines_by_file(
|
||||||
|
repo_root, commit_hash, relevant_files
|
||||||
|
)
|
||||||
|
synced_lines, total_added_lines = count_synced_lines(
|
||||||
|
added_lines_by_file, oss_repo_path, oss_ref, oss_file_cache
|
||||||
|
)
|
||||||
|
relevant_commits.append(
|
||||||
|
CommitInfo(
|
||||||
|
commit_hash=commit_hash,
|
||||||
|
subject=subject,
|
||||||
|
commit_date=commit_date,
|
||||||
|
relevant_files=relevant_files,
|
||||||
|
synced_lines=synced_lines,
|
||||||
|
total_added_lines=total_added_lines,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
output = format_output(repo, last_sync_block, relevant_commits)
|
output = format_output(repo, last_sync_block, relevant_commits)
|
||||||
print(output)
|
print(output)
|
||||||
|
|||||||
@@ -31,45 +31,19 @@ import datetime
|
|||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
||||||
# --- Configuration Begin ---
|
# Allow sibling imports regardless of the working directory.
|
||||||
# List of folders and files to copy from the OSS repo.
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||||
# Changes outside these paths will be ignored.
|
|
||||||
folder_names = [
|
|
||||||
"3rdparty",
|
|
||||||
"assets",
|
|
||||||
"benchmark",
|
|
||||||
"docker",
|
|
||||||
"docs",
|
|
||||||
"examples",
|
|
||||||
"python/sglang/lang",
|
|
||||||
"python/sglang/jit_kernel",
|
|
||||||
"python/sglang/srt",
|
|
||||||
"python/sglang/test",
|
|
||||||
"python/sglang/utils.py",
|
|
||||||
"python/sglang/README.md",
|
|
||||||
"sgl-kernel",
|
|
||||||
"test/manual",
|
|
||||||
"test/registered",
|
|
||||||
"test/srt",
|
|
||||||
"test/README.md",
|
|
||||||
"test/run_suite.py",
|
|
||||||
"README.md",
|
|
||||||
]
|
|
||||||
|
|
||||||
|
from utils import FOLDER_NAMES, write_github_step_summary # noqa: E402
|
||||||
|
|
||||||
|
# --- Configuration Begin ---
|
||||||
private_repo = "your-org/sglang-private-repo"
|
private_repo = "your-org/sglang-private-repo"
|
||||||
# --- Configuration End ---
|
# --- Configuration End ---
|
||||||
|
|
||||||
|
|
||||||
def write_github_step_summary(content):
|
|
||||||
if not os.environ.get("GITHUB_STEP_SUMMARY"):
|
|
||||||
return
|
|
||||||
|
|
||||||
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
|
|
||||||
f.write(content)
|
|
||||||
|
|
||||||
|
|
||||||
def check_dependencies():
|
def check_dependencies():
|
||||||
"""Check for required command-line tools."""
|
"""Check for required command-line tools."""
|
||||||
if not shutil.which("git"):
|
if not shutil.which("git"):
|
||||||
@@ -145,10 +119,10 @@ def get_source_folder(args):
|
|||||||
return oss_root, temp_dir, commit_hash
|
return oss_root, temp_dir, commit_hash
|
||||||
|
|
||||||
|
|
||||||
def sync_directories(oss_root, folder_names, dry_run):
|
def sync_directories(oss_root, sync_paths, dry_run):
|
||||||
"""Sync specified directories from oss_root to current working directory."""
|
"""Sync specified directories from oss_root to current working directory."""
|
||||||
rsync_commands = []
|
rsync_commands = []
|
||||||
for folder_name in folder_names:
|
for folder_name in sync_paths:
|
||||||
target_name = f"{oss_root}/{folder_name}"
|
target_name = f"{oss_root}/{folder_name}"
|
||||||
src_name = "./" + "/".join(folder_name.split("/")[:-1])
|
src_name = "./" + "/".join(folder_name.split("/")[:-1])
|
||||||
cmd = f"rsync -r --delete {target_name} {src_name}"
|
cmd = f"rsync -r --delete {target_name} {src_name}"
|
||||||
@@ -259,7 +233,7 @@ def main():
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
# Sync directories
|
# Sync directories
|
||||||
sync_directories(oss_root, folder_names, args.dry_run)
|
sync_directories(oss_root, FOLDER_NAMES, args.dry_run)
|
||||||
|
|
||||||
# Check for changes and create PR if necessary
|
# Check for changes and create PR if necessary
|
||||||
if not check_for_changes():
|
if not check_for_changes():
|
||||||
|
|||||||
@@ -29,44 +29,20 @@ python3 scripts/copy_to_oss.py --commit <commit_hash> --dry-run
|
|||||||
import argparse
|
import argparse
|
||||||
import datetime
|
import datetime
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
||||||
# --- Configuration Begin ---
|
# Allow sibling imports regardless of the working directory.
|
||||||
# List of folders and files to copy to the OSS repo.
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||||
# Changes outside these paths will be ignored.
|
|
||||||
folder_names = [
|
|
||||||
"3rdparty",
|
|
||||||
"assets",
|
|
||||||
"benchmark",
|
|
||||||
"docker",
|
|
||||||
"docs",
|
|
||||||
"examples",
|
|
||||||
"python/sglang/lang",
|
|
||||||
"python/sglang/jit_kernel",
|
|
||||||
"python/sglang/srt",
|
|
||||||
"python/sglang/test",
|
|
||||||
"python/sglang/utils.py",
|
|
||||||
"python/sglang/README.md",
|
|
||||||
"sgl-kernel",
|
|
||||||
"test/manual",
|
|
||||||
"test/registered",
|
|
||||||
"test/srt",
|
|
||||||
"test/README.md",
|
|
||||||
"test/run_suite.py",
|
|
||||||
"README.md",
|
|
||||||
]
|
|
||||||
|
|
||||||
# --- Configuration End ---
|
from utils import ( # noqa: E402
|
||||||
|
FOLDER_NAMES,
|
||||||
|
find_latest_oss_sync_commit,
|
||||||
def write_github_step_summary(content):
|
write_github_step_summary,
|
||||||
if not os.environ.get("GITHUB_STEP_SUMMARY"):
|
)
|
||||||
return
|
|
||||||
|
|
||||||
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
|
|
||||||
f.write(content)
|
|
||||||
|
|
||||||
|
|
||||||
def get_commit_info(commit_ref):
|
def get_commit_info(commit_ref):
|
||||||
@@ -131,7 +107,7 @@ def create_filtered_patch(commit_hash, dry_run):
|
|||||||
|
|
||||||
# Filter the list of files
|
# Filter the list of files
|
||||||
relevant_files = [
|
relevant_files = [
|
||||||
f for f in changed_files if any(f.startswith(path) for path in folder_names)
|
f for f in changed_files if any(f.startswith(path) for path in FOLDER_NAMES)
|
||||||
]
|
]
|
||||||
|
|
||||||
if not relevant_files:
|
if not relevant_files:
|
||||||
@@ -190,7 +166,7 @@ def get_oss_repo(dry_run):
|
|||||||
print(f"\nCreated temporary directory for OSS repo: {temp_dir}")
|
print(f"\nCreated temporary directory for OSS repo: {temp_dir}")
|
||||||
|
|
||||||
repo_url = f"https://{gh_token}@github.com/sgl-project/sglang.git"
|
repo_url = f"https://{gh_token}@github.com/sgl-project/sglang.git"
|
||||||
command = ["git", "clone", "--branch", "main", repo_url, oss_root]
|
command = ["git", "clone", repo_url, oss_root]
|
||||||
|
|
||||||
print(f"Run: {' '.join(command)}")
|
print(f"Run: {' '.join(command)}")
|
||||||
if not dry_run:
|
if not dry_run:
|
||||||
@@ -209,8 +185,9 @@ def _apply_patch(patch_file, dry_run):
|
|||||||
"""
|
"""
|
||||||
Try to apply a patch, falling back to --3way merge if a clean apply fails.
|
Try to apply a patch, falling back to --3way merge if a clean apply fails.
|
||||||
|
|
||||||
Returns True if the patch was applied successfully (clean or via --3way),
|
Returns True if the patch was applied cleanly.
|
||||||
or raises an exception with full diagnostics if both methods fail.
|
Returns False if conflicts were encountered (changes are still staged
|
||||||
|
with conflict markers so a PR can be created for manual resolution).
|
||||||
"""
|
"""
|
||||||
# --- Attempt 1: clean git apply ---
|
# --- Attempt 1: clean git apply ---
|
||||||
apply_cmd = ["git", "apply", patch_file]
|
apply_cmd = ["git", "apply", patch_file]
|
||||||
@@ -235,8 +212,8 @@ def _apply_patch(patch_file, dry_run):
|
|||||||
print("✅ Patch applied via --3way merge (no conflicts).")
|
print("✅ Patch applied via --3way merge (no conflicts).")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
# --- Both attempts failed — gather diagnostics ---
|
# --- --3way left conflict markers in the working tree ---
|
||||||
print(f"❌ --3way merge also failed:\n{result_3way.stderr.strip()}\n")
|
print(f"⚠️ --3way merge had conflicts:\n{result_3way.stderr.strip()}\n")
|
||||||
|
|
||||||
# Show which hunks conflict
|
# Show which hunks conflict
|
||||||
check_cmd = ["git", "apply", "--check", "--verbose", patch_file]
|
check_cmd = ["git", "apply", "--check", "--verbose", patch_file]
|
||||||
@@ -272,7 +249,7 @@ def _apply_patch(patch_file, dry_run):
|
|||||||
|
|
||||||
# Write a rich summary to the GitHub Actions step summary
|
# Write a rich summary to the GitHub Actions step summary
|
||||||
summary_lines = [
|
summary_lines = [
|
||||||
"\n## ❌ Patch could not be applied automatically\n",
|
"\n## ⚠️ Patch had conflicts — PR created for manual resolution\n",
|
||||||
"### Conflict details\n",
|
"### Conflict details\n",
|
||||||
f"```\n{conflict_details}\n```\n",
|
f"```\n{conflict_details}\n```\n",
|
||||||
]
|
]
|
||||||
@@ -287,18 +264,22 @@ def _apply_patch(patch_file, dry_run):
|
|||||||
)
|
)
|
||||||
write_github_step_summary("".join(summary_lines))
|
write_github_step_summary("".join(summary_lines))
|
||||||
|
|
||||||
# Reset the working tree so the finally-block cleanup is safe
|
return False
|
||||||
subprocess.run(["git", "checkout", "."], capture_output=True, text=True)
|
|
||||||
|
|
||||||
raise RuntimeError(
|
|
||||||
"Patch could not be applied. See the CI log and GitHub Actions "
|
|
||||||
"step summary for the full patch and conflict details."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def apply_patch_and_push(oss_root, patch_file, branch_name, commit_message, dry_run):
|
def apply_patch_and_push(
|
||||||
|
oss_root, patch_file, branch_name, commit_message, base_oss_commit, dry_run
|
||||||
|
):
|
||||||
"""
|
"""
|
||||||
In the OSS repo, create a branch, apply the patch, commit, and push.
|
In the OSS repo, create a branch from base_oss_commit, apply the patch,
|
||||||
|
commit, and push.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
base_oss_commit: The OSS commit hash to branch from (the last sync
|
||||||
|
point). If None, the current HEAD (main) is used.
|
||||||
|
|
||||||
|
Returns True if the patch applied cleanly, False if there were conflicts
|
||||||
|
(the conflicted state is still committed and pushed so a PR can be opened).
|
||||||
"""
|
"""
|
||||||
print("\nApplying patch and pushing to OSS repo...")
|
print("\nApplying patch and pushing to OSS repo...")
|
||||||
|
|
||||||
@@ -306,15 +287,19 @@ def apply_patch_and_push(oss_root, patch_file, branch_name, commit_message, dry_
|
|||||||
if not dry_run:
|
if not dry_run:
|
||||||
os.chdir(oss_root)
|
os.chdir(oss_root)
|
||||||
|
|
||||||
|
applied_cleanly = True
|
||||||
try:
|
try:
|
||||||
# Create a new branch
|
# Check out a new branch from the base OSS commit
|
||||||
checkout_cmd = ["git", "checkout", "-b", branch_name]
|
if base_oss_commit:
|
||||||
|
checkout_cmd = ["git", "checkout", "-b", branch_name, base_oss_commit]
|
||||||
|
else:
|
||||||
|
checkout_cmd = ["git", "checkout", "-b", branch_name]
|
||||||
print(f"Run: {' '.join(checkout_cmd)}")
|
print(f"Run: {' '.join(checkout_cmd)}")
|
||||||
if not dry_run:
|
if not dry_run:
|
||||||
subprocess.run(checkout_cmd, check=True, capture_output=True, text=True)
|
subprocess.run(checkout_cmd, check=True, capture_output=True, text=True)
|
||||||
|
|
||||||
# Apply the patch (with --3way fallback and diagnostics)
|
# Apply the patch (with --3way fallback and diagnostics)
|
||||||
_apply_patch(patch_file, dry_run)
|
applied_cleanly = _apply_patch(patch_file, dry_run)
|
||||||
|
|
||||||
# Configure git user and stage changes
|
# Configure git user and stage changes
|
||||||
post_apply_commands = [
|
post_apply_commands = [
|
||||||
@@ -359,7 +344,15 @@ def apply_patch_and_push(oss_root, patch_file, branch_name, commit_message, dry_
|
|||||||
if not dry_run:
|
if not dry_run:
|
||||||
os.chdir(original_cwd)
|
os.chdir(original_cwd)
|
||||||
|
|
||||||
print("✅ Branch created, patch applied, and pushed successfully.")
|
if applied_cleanly:
|
||||||
|
print("✅ Branch created, patch applied cleanly, and pushed successfully.")
|
||||||
|
else:
|
||||||
|
print(
|
||||||
|
"⚠️ Branch created and pushed with conflict markers. "
|
||||||
|
"A PR will be opened for manual resolution."
|
||||||
|
)
|
||||||
|
|
||||||
|
return applied_cleanly
|
||||||
|
|
||||||
|
|
||||||
def create_pull_request(oss_root, branch_name, title, body, dry_run):
|
def create_pull_request(oss_root, branch_name, title, body, dry_run):
|
||||||
@@ -433,6 +426,36 @@ def get_commit_author(commit_hash):
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def get_all_co_author_lines(commit_hash, commit_message):
|
||||||
|
"""
|
||||||
|
Build a deduplicated list of Co-authored-by lines that includes both
|
||||||
|
the primary commit author and any Co-authored-by trailers already
|
||||||
|
present in the commit message.
|
||||||
|
|
||||||
|
Returns a list of unique "Co-authored-by: Name <email>" strings.
|
||||||
|
"""
|
||||||
|
seen = set()
|
||||||
|
co_author_lines = []
|
||||||
|
|
||||||
|
def _add(name, email):
|
||||||
|
key = (name.strip(), email.strip().lower())
|
||||||
|
if key not in seen:
|
||||||
|
seen.add(key)
|
||||||
|
co_author_lines.append(f"Co-authored-by: {name.strip()} <{email.strip()}>")
|
||||||
|
|
||||||
|
# 1. Primary author of the commit
|
||||||
|
author_name, author_email = get_commit_author(commit_hash)
|
||||||
|
_add(author_name, author_email)
|
||||||
|
|
||||||
|
# 2. Existing Co-authored-by trailers in the commit message
|
||||||
|
for line in commit_message.splitlines():
|
||||||
|
m = re.match(r"^\s*Co-authored-by:\s*(.+?)\s*<([^>]+)>", line, re.IGNORECASE)
|
||||||
|
if m:
|
||||||
|
_add(m.group(1), m.group(2))
|
||||||
|
|
||||||
|
return co_author_lines
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description="Copy a commit from the private repo to OSS and open a PR."
|
description="Copy a commit from the private repo to OSS and open a PR."
|
||||||
@@ -487,33 +510,67 @@ def main():
|
|||||||
# 2. Get the OSS repo
|
# 2. Get the OSS repo
|
||||||
oss_root, temp_dir = get_oss_repo(args.dry_run)
|
oss_root, temp_dir = get_oss_repo(args.dry_run)
|
||||||
|
|
||||||
# 3. Get original commit author for the co-author line
|
# 3. Find the latest OSS commit that was synced into sglang-private.
|
||||||
author_name, author_email = get_commit_author(commit_hash)
|
# This is the correct base for our patch, since the private repo's
|
||||||
|
# code is based on this sync point.
|
||||||
|
base_oss_commit = find_latest_oss_sync_commit()
|
||||||
|
if base_oss_commit:
|
||||||
|
print(f"ℹ️ Will branch from OSS commit {base_oss_commit}")
|
||||||
|
else:
|
||||||
|
print(
|
||||||
|
"⚠️ Could not determine latest OSS sync commit. "
|
||||||
|
"Falling back to OSS main HEAD."
|
||||||
|
)
|
||||||
|
|
||||||
# 4. Prepare content for the commit and PR based on changed files
|
# 4. Get all co-author lines (primary author + trailers from commit message)
|
||||||
|
co_author_lines = get_all_co_author_lines(commit_hash, original_commit_message)
|
||||||
|
authors_display = "\n".join(co_author_lines)
|
||||||
|
|
||||||
|
# 5. Prepare content for the commit and PR based on changed files
|
||||||
file_list_str = "\n".join([f"- {f}" for f in relevant_files])
|
file_list_str = "\n".join([f"- {f}" for f in relevant_files])
|
||||||
filename_list_str = ", ".join([f.split("/")[-1] for f in relevant_files])
|
filename_list_str = ", ".join([f.split("/")[-1] for f in relevant_files])
|
||||||
if len(filename_list_str) > 40:
|
if len(filename_list_str) > 40:
|
||||||
filename_list_str = filename_list_str[:40] + "..."
|
filename_list_str = filename_list_str[:40] + "..."
|
||||||
current_date = datetime.datetime.now().strftime("%Y%m%d")
|
current_date = datetime.datetime.now().strftime("%Y%m%d")
|
||||||
pr_title = f"[Auto Sync] Update {filename_list_str} ({current_date})"
|
pr_title = f"[Auto Sync] Update {filename_list_str} ({current_date})"
|
||||||
pr_body = (
|
|
||||||
f"Sync changes from commit `{short_hash}`.\n\n"
|
|
||||||
f"**Files Changed:**\n{file_list_str}\n\n"
|
|
||||||
f"Author: {author_name} <{author_email}>"
|
|
||||||
f"\n\n---\n\n"
|
|
||||||
f"*This is an automated PR created by scripts/copy_from_oss.py.*"
|
|
||||||
)
|
|
||||||
|
|
||||||
# 5. Create branch, apply patch, and push
|
# 6. Create branch from the last synced OSS commit, apply patch, and push
|
||||||
branch_name = f"sync-{short_hash}-{current_date}"
|
branch_name = f"sync-{short_hash}-{current_date}"
|
||||||
co_author_line = f"Co-authored-by: {author_name} <{author_email}>"
|
co_authors_block = "\n".join(co_author_lines)
|
||||||
commit_message = f"{pr_title}\n\n{co_author_line}"
|
commit_message = f"{pr_title}\n\n{co_authors_block}"
|
||||||
apply_patch_and_push(
|
applied_cleanly = apply_patch_and_push(
|
||||||
oss_root, patch_file, branch_name, commit_message, args.dry_run
|
oss_root,
|
||||||
|
patch_file,
|
||||||
|
branch_name,
|
||||||
|
commit_message,
|
||||||
|
base_oss_commit,
|
||||||
|
args.dry_run,
|
||||||
)
|
)
|
||||||
|
|
||||||
# 6. Create Pull Request
|
# 7. Adjust PR title and body when there are conflicts
|
||||||
|
if not applied_cleanly:
|
||||||
|
pr_title = (
|
||||||
|
f"[Auto Sync][⚠️ Conflicts] Update {filename_list_str} ({current_date})"
|
||||||
|
)
|
||||||
|
|
||||||
|
pr_body_parts = [
|
||||||
|
f"Sync changes from commit `{short_hash}`.\n",
|
||||||
|
f"**Files Changed:**\n{file_list_str}\n",
|
||||||
|
f"**Authors:**\n{authors_display}",
|
||||||
|
]
|
||||||
|
if not applied_cleanly:
|
||||||
|
pr_body_parts.append(
|
||||||
|
"\n\n⚠️ **This patch had merge conflicts.** "
|
||||||
|
"The branch contains conflict markers that must be resolved manually. "
|
||||||
|
"Please check the CI logs for the full patch and conflict details."
|
||||||
|
)
|
||||||
|
pr_body_parts.append(
|
||||||
|
f"\n\n---\n\n"
|
||||||
|
f"*This is an automated PR created by scripts/copy_to_oss.py.*"
|
||||||
|
)
|
||||||
|
pr_body = "\n".join(pr_body_parts)
|
||||||
|
|
||||||
|
# 8. Create Pull Request
|
||||||
create_pull_request(oss_root, branch_name, pr_title, pr_body, args.dry_run)
|
create_pull_request(oss_root, branch_name, pr_title, pr_body, args.dry_run)
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
|
|||||||
@@ -0,0 +1,136 @@
|
|||||||
|
"""
|
||||||
|
Shared constants and helpers for code-sync scripts.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
# --- Configuration Begin ---
|
||||||
|
# List of folders and files to copy to / from the OSS repo.
|
||||||
|
# Changes outside these paths will be ignored.
|
||||||
|
FOLDER_NAMES = [
|
||||||
|
"3rdparty",
|
||||||
|
"assets",
|
||||||
|
"benchmark",
|
||||||
|
"docker",
|
||||||
|
"docs",
|
||||||
|
"examples",
|
||||||
|
"python/sglang/lang",
|
||||||
|
"python/sglang/jit_kernel",
|
||||||
|
"python/sglang/srt",
|
||||||
|
"python/sglang/test",
|
||||||
|
"python/sglang/utils.py",
|
||||||
|
"python/sglang/README.md",
|
||||||
|
"sgl-kernel",
|
||||||
|
"test/manual",
|
||||||
|
"test/registered",
|
||||||
|
"test/srt",
|
||||||
|
"test/README.md",
|
||||||
|
"test/run_suite.py",
|
||||||
|
"README.md",
|
||||||
|
]
|
||||||
|
|
||||||
|
SYNC_COMMIT_PREFIX = r"\[Automated PR\] Copy OSS code from commit"
|
||||||
|
# --- Configuration End ---
|
||||||
|
|
||||||
|
|
||||||
|
def write_github_step_summary(content: str) -> None:
|
||||||
|
"""Append *content* to the GitHub Actions step summary (no-op outside CI)."""
|
||||||
|
summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
|
||||||
|
if not summary_path:
|
||||||
|
return
|
||||||
|
with open(summary_path, "a") as f:
|
||||||
|
f.write(content)
|
||||||
|
|
||||||
|
|
||||||
|
def get_last_sync_commit(repo_root: Optional[str] = None) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
Find the most recent sync commit that copied from OSS.
|
||||||
|
|
||||||
|
Returns the full private-repo commit hash, or None if not found.
|
||||||
|
The match is restricted to commits whose **subject** starts with the
|
||||||
|
sync prefix so that unrelated commits mentioning the phrase in their
|
||||||
|
body are ignored.
|
||||||
|
"""
|
||||||
|
subject_pattern = re.compile("^" + SYNC_COMMIT_PREFIX)
|
||||||
|
|
||||||
|
try:
|
||||||
|
cmd = [
|
||||||
|
"git",
|
||||||
|
"log",
|
||||||
|
"--all",
|
||||||
|
"--grep",
|
||||||
|
SYNC_COMMIT_PREFIX,
|
||||||
|
"--format=%H %s",
|
||||||
|
]
|
||||||
|
result = subprocess.run(
|
||||||
|
cmd,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=True,
|
||||||
|
cwd=repo_root,
|
||||||
|
).stdout.strip()
|
||||||
|
|
||||||
|
for line in result.splitlines():
|
||||||
|
# Format: "<full_hash> <subject>"
|
||||||
|
parts = line.split(" ", 1)
|
||||||
|
if len(parts) != 2:
|
||||||
|
continue
|
||||||
|
commit_hash, subject = parts
|
||||||
|
if subject_pattern.search(subject):
|
||||||
|
return commit_hash
|
||||||
|
|
||||||
|
return None
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
print(f"Error finding last sync commit: {e.stderr}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def find_latest_oss_sync_commit(repo_root: Optional[str] = None) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
Search the private repo history for the latest commit whose **subject**
|
||||||
|
matches "[Automated PR] Copy OSS code from commit {commit_id} on {date}"
|
||||||
|
and return the embedded **OSS** commit hash.
|
||||||
|
|
||||||
|
Returns the short OSS commit hash string, or None if not found.
|
||||||
|
"""
|
||||||
|
oss_hash_pattern = re.compile("^" + SYNC_COMMIT_PREFIX + r" ([0-9a-f]+)")
|
||||||
|
|
||||||
|
try:
|
||||||
|
# --grep filters on the full message body, so we request subject-only
|
||||||
|
# output and validate the pattern against the subject ourselves.
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
"git",
|
||||||
|
"log",
|
||||||
|
"--all",
|
||||||
|
"--grep",
|
||||||
|
SYNC_COMMIT_PREFIX,
|
||||||
|
"--pretty=%s",
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=True,
|
||||||
|
cwd=repo_root,
|
||||||
|
)
|
||||||
|
|
||||||
|
for subject in result.stdout.strip().splitlines():
|
||||||
|
m = oss_hash_pattern.search(subject)
|
||||||
|
if m:
|
||||||
|
oss_commit = m.group(1)
|
||||||
|
print(
|
||||||
|
f"✅ Latest OSS sync commit found: {oss_commit} "
|
||||||
|
f"(from: {subject})"
|
||||||
|
)
|
||||||
|
return oss_commit
|
||||||
|
|
||||||
|
print(
|
||||||
|
"⚠️ No '[Automated PR] Copy OSS code from commit ...' " "found in history."
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
print(f"Error searching for OSS sync commits: {e.stderr.strip()}")
|
||||||
|
return None
|
||||||
Reference in New Issue
Block a user