[SMG-GO] implement a Go SGLang Model Gateway - OpenAI Compatible API Server (#14770)

2025-12-10 06:03:28 -08:00
parent 12b7a4fab0
commit 766476f52a
38 changed files with 9115 additions and 271 deletions
@@ -0,0 +1,111 @@
+#!/bin/bash
+
+# OpenAI-compatible server runner
+# Usage: ./run.sh [tokenizer_path] [endpoint] [port] [--profile] [--pprof-port PORT]
+#
+# Options:
+#   --profile          Enable pprof profiling (default port: 6060)
+#   --pprof-port PORT  Set pprof port (default: 6060, requires --profile)
+
+# Set library path for Rust FFI library
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BINDINGS_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
+LIB_DIR="${BINDINGS_DIR}/lib"
+
+if [ ! -d "$LIB_DIR" ]; then
+    echo "Error: Library directory not found at $LIB_DIR"
+    echo "Please run 'make lib' first to build and export the library"
+    exit 1
+fi
+
+# Get Python LDFLAGS (needed for Rust FFI that depends on Python)
+PYTHON_LDFLAGS=$(python3-config --ldflags --embed 2>/dev/null || python3-config --ldflags 2>/dev/null || echo "")
+
+# Set CGO_LDFLAGS to link with the Rust library
+# Note: -lsgl_model_gateway_go and -ldl are already in the #cgo directive in internal/ffi/client.go
+# We only need to add the library path (-L) and Python flags
+export CGO_LDFLAGS="-L${LIB_DIR} ${PYTHON_LDFLAGS}"
+
+# macOS uses DYLD_LIBRARY_PATH, Linux uses LD_LIBRARY_PATH
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    export DYLD_LIBRARY_PATH="${LIB_DIR}:${DYLD_LIBRARY_PATH}"
+else
+    export LD_LIBRARY_PATH="${LIB_DIR}:${LD_LIBRARY_PATH}"
+fi
+
+# Parse arguments
+ENABLE_PROFILE=false
+PPROF_PORT="6060"
+TOKENIZER_PATH=""
+ENDPOINT=""
+PORT=""
+
+while [[ $# -gt 0 ]]; do
+	case $1 in
+		--profile)
+			ENABLE_PROFILE=true
+			shift
+			;;
+		--pprof-port)
+			ENABLE_PROFILE=true
+			PPROF_PORT="$2"
+			shift 2
+			;;
+		*)
+			if [[ -z "$TOKENIZER_PATH" ]]; then
+				TOKENIZER_PATH="$1"
+			elif [[ -z "$ENDPOINT" ]]; then
+				ENDPOINT="$1"
+			elif [[ -z "$PORT" ]]; then
+				PORT="$1"
+			fi
+			shift
+			;;
+	esac
+done
+
+# Default configuration
+DEFAULT_TOKENIZER_PATH="${SGL_TOKENIZER_PATH:-../tokenizer}"
+DEFAULT_ENDPOINT="${SGL_GRPC_ENDPOINT:-grpc://localhost:20000}"
+DEFAULT_PORT="${PORT:-8080}"
+
+TOKENIZER_PATH="${TOKENIZER_PATH:-${DEFAULT_TOKENIZER_PATH}}"
+ENDPOINT="${ENDPOINT:-${DEFAULT_ENDPOINT}}"
+PORT="${PORT:-${DEFAULT_PORT}}"
+
+echo "Running OpenAI-compatible server..."
+echo "Library path: ${LIB_DIR}"
+echo "Tokenizer: $TOKENIZER_PATH"
+echo "Endpoint: $ENDPOINT"
+echo "Port: $PORT"
+echo "Client Mode: gRPC (default)"
+echo "FFI Postprocessing: ENABLED (normal mode)"
+echo "FFI Preprocessing: ENABLED (normal mode)"
+if [[ "$ENABLE_PROFILE" == "true" ]]; then
+	echo "Profiling: enabled (port: $PPROF_PORT)"
+	echo "  pprof endpoint: http://localhost:$PPROF_PORT/debug/pprof/"
+	export PPROF_ENABLED=true
+	export PPROF_PORT="$PPROF_PORT"
+else
+	echo "Profiling: disabled"
+fi
+echo ""
+
+# Change to script directory
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+# Ensure Go module is properly initialized
+if [ ! -f "go.mod" ]; then
+    echo "Error: go.mod not found in $(pwd)"
+    exit 1
+fi
+
+# Ensure Go modules are enabled
+export GO111MODULE=on
+
+# Sync Go module dependencies
+echo "Syncing Go module dependencies..."
+go mod tidy
+
+# Run the server (use ./main.go to ensure module context is correct)
+SGL_TOKENIZER_PATH="$TOKENIZER_PATH" SGL_GRPC_ENDPOINT="$ENDPOINT" PORT="$PORT" go run ./main.go