[model-gateway] change sgl-router to sgl-model-gateway (#14312)

sgl-model-gateway/bindings/golang/.gitignore

@@ -0,0 +1,24 @@
+# Build artifacts
+target/
+lib/
+
+# Compiled binaries
+examples/simple/simple
+examples/streaming/streaming
+
+# Go build artifacts
+*.o
+*.a
+*.so
+*.dylib
+
+# IDE and editor files
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Environment files
+.env
+.env.local

sgl-model-gateway/bindings/golang/Cargo.toml

@@ -0,0 +1,47 @@
+[package]
+name = "sgl-model-gateway-golang"
+version = "0.2.3"
+edition = "2021"
+
+[lib]
+name = "sgl_model_gateway_go"
+crate-type = ["cdylib"]
+
+[dependencies]
+tokio = { version = "1.42.0", features = ["full"] }
+serde_json = { version = "1.0", default-features = false, features = [
+    "std",
+    "preserve_order",
+] }
+uuid = { version = "1.10", features = ["v4", "serde"] }
+once_cell = "1.21.3"
+futures-util = "0.3"
+tracing = "0.1"
+
+[dependencies.sgl-model-gateway]
+path = "../.."
+default-features = true
+
+[features]
+default = []
+vendored-openssl = ["sgl-model-gateway/vendored-openssl"]
+
+[profile.release]
+opt-level = "z"     # Optimize for size
+lto = "fat"         # Full LTO for smaller binaries
+codegen-units = 1   # Better optimization, slower compile
+strip = true        # Strip debug symbols
+
+[profile.ci]
+inherits = "release"
+opt-level = 2       # Lighter optimization (still fast runtime, much faster compile)
+lto = "thin"        # Thin LTO - good balance
+codegen-units = 16  # More parallelization for faster builds
+strip = true
+
+[profile.dev]
+opt-level = 0
+debug = 1
+split-debuginfo = "unpacked"
+incremental = true
+codegen-units = 256

sgl-model-gateway/bindings/golang/Makefile

@@ -0,0 +1,103 @@
+# Makefile for sglang-router golang bindings
+# This builds the Rust FFI library and provides convenience targets for Go development
+
+# Configuration
+CARGO_BUILD_DIR ?= $(shell pwd)/target
+BUILD_MODE ?= release
+LIB_NAME = libsglang_router_rs
+
+# Detect OS
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Linux)
+    LIB_EXT = .so
+    LD_LIBRARY_PATH_VAR = LD_LIBRARY_PATH
+endif
+ifeq ($(UNAME_S),Darwin)
+    LIB_EXT = .dylib
+    LD_LIBRARY_PATH_VAR = DYLD_LIBRARY_PATH
+endif
+
+# Paths
+ROOT_DIR := $(shell pwd)
+RUST_SRC_DIR := $(ROOT_DIR)/src
+LIB_BUILD_DIR := $(CARGO_BUILD_DIR)/$(BUILD_MODE)
+LIB_BUILD_PATH := $(LIB_BUILD_DIR)/$(LIB_NAME)$(LIB_EXT)
+LIB_EXPORT_DIR := $(ROOT_DIR)/lib
+LIB_EXPORT_PATH := $(LIB_EXPORT_DIR)/$(LIB_NAME)$(LIB_EXT)
+
+# Python LDFLAGS (needed for Rust FFI that depends on Python)
+PYTHON_LDFLAGS := $(shell python3-config --ldflags --embed 2>/dev/null || python3-config --ldflags 2>/dev/null || echo "")
+
+# CGO flags - use exported lib directory if available, otherwise build directory
+LIB_DIR := $(if $(wildcard $(LIB_EXPORT_PATH)),$(LIB_EXPORT_DIR),$(LIB_BUILD_DIR))
+export CGO_LDFLAGS = -L$(LIB_DIR) -lsglang_router_rs $(PYTHON_LDFLAGS) -ldl
+export $(LD_LIBRARY_PATH_VAR) := $(LIB_DIR):$($(LD_LIBRARY_PATH_VAR))
+
+.PHONY: all build build-dev lib lib-clean clean test examples help run-simple run-streaming check-lib
+
+help:
+	@echo "Available targets:"
+	@echo "  build           - Build release version of Rust FFI library"
+	@echo "  build-dev       - Build debug version of Rust FFI library"
+	@echo "  lib             - Copy built library to ./lib directory"
+	@echo "  lib-clean       - Clean ./lib directory"
+	@echo "  clean           - Clean build artifacts"
+	@echo "  test            - Run Go tests"
+	@echo "  examples        - Build example programs"
+	@echo "  run-simple      - Run simple example"
+	@echo "  run-streaming   - Run streaming example"
+
+all: build
+
+build:
+	@echo "Building Rust FFI library (release mode)..."
+	@CARGO_TARGET_DIR=$(CARGO_BUILD_DIR) cargo build --release --manifest-path Cargo.toml
+	@echo "Library built at: $(LIB_BUILD_PATH)"
+
+build-dev:
+	@echo "Building Rust FFI library (debug mode)..."
+	@CARGO_TARGET_DIR=$(CARGO_BUILD_DIR) cargo build --manifest-path Cargo.toml
+	@echo "Library built at: $(LIB_BUILD_DIR)/debug/$(LIB_NAME)$(LIB_EXT)"
+
+lib: build
+	@echo "Copying library to ./lib directory..."
+	@mkdir -p $(LIB_EXPORT_DIR)
+	@cp $(LIB_BUILD_PATH) $(LIB_EXPORT_PATH)
+	@echo "Library exported at: $(LIB_EXPORT_PATH)"
+
+lib-clean:
+	@echo "Cleaning ./lib directory..."
+	@rm -rf $(LIB_EXPORT_DIR)
+	@echo "Lib directory cleaned"
+
+clean: lib-clean
+	@echo "Cleaning build artifacts..."
+	@CARGO_TARGET_DIR=$(CARGO_BUILD_DIR) cargo clean --manifest-path Cargo.toml
+	@echo "Clean complete"
+
+test: build
+	@echo "Running Go tests..."
+	@go test ./...
+
+examples: build
+	@echo "Building example programs..."
+	@cd examples/simple && go build -o simple main.go
+	@cd examples/streaming && go build -o streaming main.go
+	@echo "Examples built"
+
+run-simple: build
+	@echo "Running simple example..."
+	@cd examples/simple && bash run.sh
+
+run-streaming: build
+	@echo "Running streaming example..."
+	@cd examples/streaming && bash run.sh
+
+# Check if library exists (either in lib dir or build dir)
+check-lib:
+	@if [ ! -f "$(LIB_EXPORT_PATH)" ] && [ ! -f "$(LIB_BUILD_PATH)" ]; then \
+		echo "Error: Library not found at $(LIB_EXPORT_PATH) or $(LIB_BUILD_PATH)"; \
+		echo "Run 'make build' or 'make lib' first"; \
+		exit 1; \
+	fi
+	@echo "Library found at: $(LIB_DIR)/$(LIB_NAME)$(LIB_EXT)"

sgl-model-gateway/bindings/golang/README.md

@@ -0,0 +1,552 @@
+# SGLang Go gRPC SDK
+
+A high-level Go SDK for interacting with SGLang gRPC API, designed with an OpenAI-style API for familiarity and ease of use.
+
+**Location**: `sgl-model-gateway/bindings/golang/`
+
+## Table of Contents
+
+- [Features](#features)
+- [Installation](#installation)
+- [Quick Start](#quick-start)
+  - [Basic Usage](#basic-usage)
+  - [Streaming Usage](#streaming-usage)
+- [Examples](#examples)
+- [Configuration](#configuration)
+- [API Reference](#api-reference)
+- [Testing](#testing)
+  - [Unit Tests](#unit-tests)
+  - [Integration Tests](#integration-tests)
+  - [Benchmarks](#benchmarks)
+- [Documentation](#documentation)
+- [Development](#development)
+- [Troubleshooting](#troubleshooting)
+- [License](#license)
+
+## Features
+
+- **OpenAI-style API**: Familiar interface similar to OpenAI Go SDK
+- **Streaming Support**: Real-time streaming chat completions
+- **Non-streaming Support**: Simple request/response API
+- **Tool Calling**: Support for function calling and tool use
+- **Type-safe**: Full Go type definitions for requests and responses
+- **Comprehensive Testing**: 18+ unit and integration tests
+- **Thread-safe**: All public methods are safe for concurrent use
+- **Well-documented**: Full API documentation with examples
+
+## Installation
+
+```bash
+go get github.com/sglang/sglang-go-grpc-sdk
+```
+
+### Build Requirements
+
+- Go 1.21 or later
+- Rust toolchain (for building the FFI library)
+- Python 3.x (for Python bindings in Rust FFI)
+- Tokio runtime for async operations
+
+## Quick Start
+
+### Basic Usage (Non-streaming)
+
+```go
+package main
+
+import (
+    "context"
+    "fmt"
+    "log"
+
+    "github.com/sglang/sglang-go-grpc-sdk"
+)
+
+func main() {
+    // Create client
+    client, err := sglang.NewClient(sglang.ClientConfig{
+        Endpoint:      "grpc://localhost:20000",
+        TokenizerPath: "/path/to/tokenizer",
+    })
+    if err != nil {
+        log.Fatal(err)
+    }
+    defer client.Close()
+
+    // Create completion
+    resp, err := client.CreateChatCompletion(context.Background(), sglang.ChatCompletionRequest{
+        Model: "default",
+        Messages: []sglang.ChatMessage{
+            {Role: "user", Content: "Hello!"},
+        },
+        Stream: false,
+    })
+    if err != nil {
+        log.Fatal(err)
+    }
+
+    fmt.Println(resp.Choices[0].Message.Content)
+    fmt.Printf("Usage: Prompt=%d, Completion=%d, Total=%d\n",
+        resp.Usage.PromptTokens,
+        resp.Usage.CompletionTokens,
+        resp.Usage.TotalTokens)
+}
+```
+
+### Streaming Usage
+
+```go
+package main
+
+import (
+    "context"
+    "fmt"
+    "io"
+    "log"
+
+    "github.com/sglang/sglang-go-grpc-sdk"
+)
+
+func main() {
+    // Create client
+    client, err := sglang.NewClient(sglang.ClientConfig{
+        Endpoint:      "grpc://localhost:20000",
+        TokenizerPath: "/path/to/tokenizer",
+    })
+    if err != nil {
+        log.Fatal(err)
+    }
+    defer client.Close()
+
+    // Create streaming completion
+    ctx := context.Background()
+    stream, err := client.CreateChatCompletionStream(ctx, sglang.ChatCompletionRequest{
+        Model: "default",
+        Messages: []sglang.ChatMessage{
+            {Role: "user", Content: "Tell me a story"},
+        },
+        Stream:              true,
+        MaxCompletionTokens: intPtr(500),
+    })
+    if err != nil {
+        log.Fatal(err)
+    }
+    defer stream.Close()
+
+    // Read streaming response
+    for {
+        chunk, err := stream.Recv()
+        if err == io.EOF {
+            break
+        }
+        if err != nil {
+            log.Fatal(err)
+        }
+
+        for _, choice := range chunk.Choices {
+            if choice.Delta.Content != "" {
+                fmt.Print(choice.Delta.Content)
+            }
+        }
+    }
+    fmt.Println() // newline
+}
+
+// Helper functions for optional pointer fields
+func intPtr(i int) *int {
+    return &i
+}
+
+func float32Ptr(f float32) *float32 {
+    return &f
+}
+```
+
+## Examples
+
+The SDK includes several examples in the `examples/` directory:
+
+- **simple**: Basic non-streaming chat completion example
+- **streaming**: Real-time streaming with performance metrics
+
+### Running Examples
+
+```bash
+# Run simple example
+cd bindings/golang/examples/simple
+bash run.sh
+
+# Run streaming example
+cd bindings/golang/examples/streaming
+bash run.sh
+
+# Or use Makefile from bindings/golang directory
+cd bindings/golang
+make run-simple
+make run-streaming
+```
+
+Examples automatically detect the server endpoint and tokenizer path via environment variables or defaults.
+
+## Configuration
+
+### Environment Variables
+
+- `SGL_GRPC_ENDPOINT`: gRPC server endpoint (default: `grpc://localhost:20000`)
+- `SGL_TOKENIZER_PATH`: Path to tokenizer directory (required)
+- `CARGO_BUILD_DIR`: Rust build output directory (auto-detected if not set)
+
+### ClientConfig
+
+```go
+type ClientConfig struct {
+    // Endpoint is the gRPC endpoint URL (e.g., "grpc://localhost:20000")
+    // Required field. Must include the scheme (grpc://) and port number.
+    Endpoint string
+
+    // TokenizerPath is the path to the tokenizer directory containing
+    // tokenizer configuration files (e.g., tokenizer.json, vocab.json)
+    // Required field.
+    TokenizerPath string
+}
+```
+
+## API Reference
+
+### Client Methods
+
+```go
+type Client struct {
+    // Thread-safe client for SGLang gRPC API
+}
+
+// Creates a new client with the given configuration
+func NewClient(config ClientConfig) (*Client, error)
+
+// Closes the client and releases all resources
+func (c *Client) Close() error
+
+// Creates a non-streaming chat completion
+func (c *Client) CreateChatCompletion(ctx context.Context, req ChatCompletionRequest) (*ChatCompletionResponse, error)
+
+// Creates a streaming chat completion
+func (c *Client) CreateChatCompletionStream(ctx context.Context, req ChatCompletionRequest) (*ChatCompletionStream, error)
+```
+
+### Request Types
+
+- `ChatCompletionRequest`: Main request type for chat completions
+  - Model, Messages, Stream, Temperature, TopP, MaxCompletionTokens, Tools, etc.
+- `ChatMessage`: Individual message in a conversation
+  - Role, Content
+- `Tool`: Tool/function definition for function calling
+  - Type, Function (name, description, parameters)
+
+### Response Types
+
+- `ChatCompletionResponse`: Non-streaming response
+  - ID, Model, Created, Choices, Usage
+- `ChatCompletionStreamResponse`: Streaming response chunk
+  - Same structure as above but for incremental updates
+- `Message`: Complete message with content and tool calls
+- `ToolCall`: Tool call information with function and arguments
+- `Usage`: Token usage statistics
+  - PromptTokens, CompletionTokens, TotalTokens
+
+## Testing
+
+The SDK includes comprehensive testing infrastructure with both unit and integration tests.
+
+### Unit Tests
+
+Unit tests are located in `client_test.go` and test individual components without requiring a server.
+
+#### Running Unit Tests
+
+```bash
+# Run all unit tests
+go test ./...
+
+# Run with verbose output
+go test -v ./...
+
+# Run specific test
+go test -run TestClientConfig
+
+# Run tests with race detector (detects concurrency issues)
+go test -race ./...
+
+# Run with coverage analysis
+go test -cover ./...
+
+# Generate detailed coverage report
+go test -coverprofile=coverage.out ./...
+go tool cover -html=coverage.out -o coverage.html
+```
+
+#### Unit Test Coverage
+
+- **Configuration validation** (`TestClientConfig`) - Validates ClientConfig requirements
+- **Type structures** - Verifying all struct types work correctly
+- **Response handling** - Testing response parsing and validation
+- **Concurrent operations** (`TestConcurrentClientOperations`) - Thread-safety verification
+- **Benchmarks** (`BenchmarkChatCompletionRequest`) - Performance measurement
+
+**Test Files**:
+- `client_test.go` - 10 unit tests covering core functionality
+- Tests cover: config validation, message types, request validation, close operations, response types, streaming, tools, concurrency, and context cancellation
+
+### Integration Tests
+
+Integration tests require a running SGLang server and test the full client-server interaction.
+
+#### Prerequisites
+
+1. Start an SGLang server:
+
+```bash
+# Using Python (requires sglang package installed)
+python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-hf
+
+# Or using pre-built Docker image
+docker run -p 20000:20000 lmsys/sglang:latest
+
+# Or build your own
+sglang launch_server --model-path <model_path>
+```
+
+2. Set required environment variables:
+
+```bash
+# Set the gRPC endpoint (default: grpc://localhost:20000)
+export SGL_GRPC_ENDPOINT=grpc://localhost:20000
+
+# Set the tokenizer path (required)
+export SGL_TOKENIZER_PATH=/path/to/tokenizer
+```
+
+#### Running Integration Tests
+
+```bash
+# Run all integration tests
+go test -tags=integration ./...
+
+# Run specific integration test
+go test -tags=integration -run TestIntegrationNonStreamingCompletion
+
+# Run with verbose output
+go test -tags=integration -v ./...
+
+# Run with race detector
+go test -tags=integration -race ./...
+```
+
+#### Integration Test Coverage
+
+**Test File**: `integration_test.go` - 4 integration tests
+
+- `TestIntegrationNonStreamingCompletion` - Basic non-streaming request/response
+- `TestIntegrationStreamingCompletion` - Streaming response handling
+- `TestIntegrationConcurrentRequests` - Multiple simultaneous requests
+- `TestIntegrationContextCancellation` - Context timeout and cancellation
+
+### Benchmarks
+
+Measure performance of SDK operations:
+
+```bash
+# Run all benchmarks
+go test -bench=. -benchmem ./...
+
+# Run specific benchmark
+go test -bench=BenchmarkChatCompletionRequest -benchmem
+
+# Run for longer duration
+go test -bench=. -benchtime=10s ./...
+```
+
+Current benchmarks:
+- `BenchmarkChatCompletionRequest` - Measures request creation performance
+
+### CI/CD Integration
+
+Add to your GitHub Actions workflow:
+
+```yaml
+- name: Run Go tests
+  run: |
+    go test -race -cover ./...
+
+- name: Run integration tests (on main branch)
+  if: github.ref == 'refs/heads/main'
+  env:
+    SGL_GRPC_ENDPOINT: grpc://localhost:20000
+    SGL_TOKENIZER_PATH: /path/to/tokenizer
+  run: go test -tags=integration ./...
+```
+
+## Documentation
+
+### Code Documentation
+
+All public types and functions include comprehensive documentation:
+
+1. **Package-level documentation** in `client.go` with usage examples
+2. **Type documentation** for all structs with field descriptions
+3. **Function documentation** with:
+   - Purpose and behavior description
+   - Parameter documentation with types and constraints
+   - Return value documentation
+   - Error cases and handling
+   - Safety notes (for FFI functions)
+   - Usage examples
+
+### Key Documented Components
+
+- `Client` - Main client with thread-safety notes
+- `ClientConfig` - Configuration requirements and validation rules
+- `ChatCompletionRequest` - Request structure with field descriptions
+- `ChatCompletionResponse` - Response structure and usage
+- `ChatCompletionStreamResponse` - Streaming response format
+- `Usage` - Token usage information structure
+- `Tool`, `Function`, `ToolCall` - Tool call structures
+
+### Viewing Documentation
+
+Generate and view HTML documentation:
+
+```bash
+# Install godoc (if not already installed)
+go install golang.org/x/tools/cmd/godoc@latest
+
+# Generate and serve documentation
+godoc -http=:6060
+
+# Visit: http://localhost:6060/pkg/github.com/sglang/sglang-go-grpc-sdk/
+```
+
+## Development
+
+### Building
+
+```bash
+cd bindings/golang
+
+# Build the Go bindings (compiles Rust FFI library)
+make build
+
+# Clean build
+make clean && make build
+```
+
+### Code Quality
+
+Ensure code quality before committing:
+
+```bash
+# Run Go vet (check for potential bugs)
+go vet ./...
+
+# Format code
+go fmt ./...
+
+# Run all tests with race detection
+go test -race ./...
+```
+
+### Project Structure
+
+```
+bindings/golang/
+├── client.go                 # Main client implementation
+├── client_test.go            # Unit tests
+├── integration_test.go       # Integration tests
+├── README.md                 # This file
+├── Makefile                  # Build automation
+├── Cargo.toml               # Rust FFI dependencies
+├── examples/                # Example programs
+│   ├── simple/             # Non-streaming example
+│   └── streaming/          # Streaming example
+├── src/                    # Rust FFI source
+│   ├── client.rs          # Client FFI
+│   ├── stream.rs          # Stream handling
+│   ├── grpc_converter.rs  # Response conversion
+│   └── ...
+└── internal/               # Internal packages
+    └── ffi/               # FFI bindings
+```
+
+## Troubleshooting
+
+### Connection Errors
+
+**Error**: `connection refused` or `failed to dial`
+
+**Solution**:
+1. Ensure SGLang server is running: `python -m sglang.launch_server`
+2. Check endpoint: `echo $SGL_GRPC_ENDPOINT`
+3. Verify port is not blocked: `nc -zv localhost 20000`
+
+### Tokenizer Not Found
+
+**Error**: `tokenizer path not found` or `tokenizer configuration missing`
+
+**Solution**:
+1. Set `SGL_TOKENIZER_PATH` environment variable
+2. Verify path contains required files: `ls $SGL_TOKENIZER_PATH`
+3. Files should include: `tokenizer.json`, `vocab.json`, `config.json`
+
+### Build Failures
+
+**Error**: `library 'sglang_router_rs' not found`
+
+**Solution**:
+1. Rebuild Rust library: `cd sgl-model-gateway/bindings/golang && make build`
+2. Or manually with cargo: `cd sgl-model-gateway/bindings/golang && cargo build --release`
+3. Set `CARGO_BUILD_DIR` if using non-standard build location
+4. Ensure Rust toolchain is installed: `rustup toolchain list`
+
+### Tests Hanging
+
+**Error**: Tests seem to hang indefinitely
+
+**Solution**:
+1. Use timeout for hanging tests: `timeout 30s go test ./...`
+2. Run with verbose output to see which test hangs: `go test -v ./...`
+3. Ensure server is responsive: `grpcurl -plaintext localhost:20000 list`
+
+### Memory Issues
+
+**Error**: Out of memory during tests
+
+**Solution**:
+```bash
+# Run with memory limit for long-running tests
+GODEBUG=madvdontneed=1 go test -timeout 5m ./...
+
+# Monitor memory during tests
+watch -n1 'ps aux | grep test'
+```
+
+## Contributing
+
+When adding new features:
+
+1. Add comprehensive documentation to public types/functions
+2. Include usage examples for complex APIs
+3. Add unit tests covering happy path and error cases
+4. Add integration tests if server interaction required
+5. Ensure code passes `go vet` and `go test -race`
+6. Update this README if adding new features
+
+## License
+
+See LICENSE file for details.
+
+---
+
+**Need Help?**
+- Check examples in `examples/` directory
+- Run tests to see working code: `go test -v ./...`
+- Review function documentation: `godoc` or inline comments
+- Check troubleshooting section above

sgl-model-gateway/bindings/golang/client.go

@@ -0,0 +1,510 @@
+// Package sglang provides a Go SDK for SGLang gRPC API.
+//
+// SGLang is a fast language model serving framework. This package provides a Go client
+// library for interacting with SGLang's gRPC API, following the style of OpenAI's Go SDK.
+//
+// Basic usage:
+//
+//	client, err := sglang.NewClient(sglang.ClientConfig{
+//		Endpoint:      "grpc://localhost:20000",
+//		TokenizerPath: "/path/to/tokenizer",
+//	})
+//	if err != nil {
+//		log.Fatal(err)
+//	}
+//	defer client.Close()
+//
+//	resp, err := client.CreateChatCompletion(ctx, sglang.ChatCompletionRequest{
+//		Model: "default",
+//		Messages: []sglang.ChatMessage{
+//			{Role: "user", Content: "Hello"},
+//		},
+//	})
+//
+// For streaming responses, use CreateChatCompletionStream instead.
+package sglang
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+
+	"github.com/sglang/sglang-go-grpc-sdk/internal/ffi"
+)
+
+// Client is the main client for interacting with SGLang gRPC API.
+// It manages the connection to the SGLang server and handles both streaming
+// and non-streaming chat completions.
+//
+// Thread-safe: All public methods are safe for concurrent use.
+type Client struct {
+	endpoint      string
+	tokenizerPath string
+	clientHandle  *ffi.SglangClientHandle
+	mu            sync.RWMutex
+}
+
+// ClientConfig holds configuration for creating a new client.
+type ClientConfig struct {
+	// Endpoint is the gRPC endpoint URL (e.g., "grpc://localhost:20000").
+	// Required field. Must include the scheme (grpc://) and port number.
+	Endpoint string
+
+	// TokenizerPath is the path to the tokenizer directory containing
+	// tokenizer configuration files (e.g., tokenizer.json, vocab.json).
+	// Required field.
+	TokenizerPath string
+}
+
+// NewClient creates a new SGLang client with the given configuration.
+//
+// The client maintains a long-lived connection to the SGLang server and should
+// be reused for multiple requests. Call Close() to release resources.
+//
+// Returns an error if:
+// - Endpoint is empty
+// - TokenizerPath is empty
+// - Connection to the server fails
+func NewClient(config ClientConfig) (*Client, error) {
+	if config.Endpoint == "" {
+		return nil, errors.New("endpoint is required")
+	}
+	if config.TokenizerPath == "" {
+		return nil, errors.New("tokenizer path is required")
+	}
+
+	clientHandle, err := ffi.NewClient(config.Endpoint, config.TokenizerPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create client: %w", err)
+	}
+
+	return &Client{
+		endpoint:      config.Endpoint,
+		tokenizerPath: config.TokenizerPath,
+		clientHandle:  clientHandle,
+	}, nil
+}
+
+// Close closes the client and releases all resources.
+//
+// After Close() is called, the client cannot be used for further requests.
+// Calling Close() multiple times is safe and idempotent.
+func (c *Client) Close() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.clientHandle != nil {
+		c.clientHandle.Free()
+		c.clientHandle = nil
+	}
+	return nil
+}
+
+// ChatCompletionRequest represents a request for chat completion.
+// It follows the OpenAI API style for familiar usage.
+type ChatCompletionRequest struct {
+	// Model specifies the model to use for completion (e.g., "default")
+	Model string `json:"model"`
+	// Messages is the list of messages in the conversation
+	Messages            []ChatMessage   `json:"messages"`
+	Temperature         *float32        `json:"temperature,omitempty"`
+	TopP                *float32        `json:"top_p,omitempty"`
+	TopK                *int            `json:"top_k,omitempty"`
+	MaxCompletionTokens *int            `json:"max_completion_tokens,omitempty"`
+	Stream              bool            `json:"stream"`
+	Tools               []Tool          `json:"tools,omitempty"`
+	ToolChoice          interface{}     `json:"tool_choice,omitempty"`
+	Stop                interface{}     `json:"stop,omitempty"`
+	StopTokenIDs        []int           `json:"stop_token_ids,omitempty"`
+	SkipSpecialTokens   bool            `json:"skip_special_tokens,omitempty"`
+	FrequencyPenalty    *float32        `json:"frequency_penalty,omitempty"`
+	PresencePenalty     *float32        `json:"presence_penalty,omitempty"`
+	ResponseFormat      *ResponseFormat `json:"response_format,omitempty"`
+	Seed                *int            `json:"seed,omitempty"`
+	Logprobs            bool            `json:"logprobs,omitempty"`
+	TopLogprobs         *int            `json:"top_logprobs,omitempty"`
+	User                string          `json:"user,omitempty"`
+}
+
+// ChatMessage represents a single message in a chat conversation
+type ChatMessage struct {
+	Role    string      `json:"role"`
+	Content interface{} `json:"content"`
+	Name    string      `json:"name,omitempty"`
+}
+
+// Tool represents a tool/function that can be called
+type Tool struct {
+	Type     string   `json:"type"`
+	Function Function `json:"function"`
+}
+
+// Function represents a function definition
+type Function struct {
+	Name        string                 `json:"name"`
+	Description string                 `json:"description,omitempty"`
+	Parameters  map[string]interface{} `json:"parameters"`
+}
+
+// ResponseFormat represents the response format
+type ResponseFormat struct {
+	Type string `json:"type"`
+}
+
+// ChatCompletionResponse represents a non-streaming chat completion response
+type ChatCompletionResponse struct {
+	ID                string   `json:"id"`
+	Object            string   `json:"object"`
+	Created           int64    `json:"created"`
+	Model             string   `json:"model"`
+	SystemFingerprint string   `json:"system_fingerprint,omitempty"`
+	Choices           []Choice `json:"choices"`
+	Usage             Usage    `json:"usage"`
+}
+
+// Choice represents a choice in the completion response
+type Choice struct {
+	Index        int     `json:"index"`
+	Message      Message `json:"message"`
+	FinishReason string  `json:"finish_reason"`
+}
+
+// Message represents a message in the response
+type Message struct {
+	Role      string     `json:"role"`
+	Content   string     `json:"content"`
+	ToolCalls []ToolCall `json:"tool_calls,omitempty"`
+}
+
+// ToolCall represents a tool call in the response
+type ToolCall struct {
+	ID       string       `json:"id"`
+	Type     string       `json:"type"`
+	Function FunctionCall `json:"function"`
+}
+
+// FunctionCall represents a function call
+type FunctionCall struct {
+	Name      string `json:"name"`
+	Arguments string `json:"arguments"`
+}
+
+// Usage represents token usage information
+type Usage struct {
+	PromptTokens     int `json:"prompt_tokens"`
+	CompletionTokens int `json:"completion_tokens"`
+	TotalTokens      int `json:"total_tokens"`
+}
+
+// ChatCompletionStreamResponse represents a streaming chat completion response
+type ChatCompletionStreamResponse struct {
+	ID                string         `json:"id"`
+	Object            string         `json:"object"`
+	Created           int64          `json:"created"`
+	Model             string         `json:"model"`
+	SystemFingerprint string         `json:"system_fingerprint,omitempty"`
+	Choices           []StreamChoice `json:"choices"`
+	Usage             *Usage         `json:"usage,omitempty"`
+}
+
+// StreamChoice represents a choice in a streaming response
+type StreamChoice struct {
+	Index        int          `json:"index"`
+	Delta        MessageDelta `json:"delta"`
+	FinishReason string       `json:"finish_reason,omitempty"`
+}
+
+// MessageDelta represents incremental message updates
+type MessageDelta struct {
+	Role      string     `json:"role,omitempty"`
+	Content   string     `json:"content,omitempty"`
+	ToolCalls []ToolCall `json:"tool_calls,omitempty"`
+}
+
+// CreateChatCompletion creates a non-streaming chat completion with context support.
+//
+// Context Support:
+// The ctx parameter is fully supported for cancellation and timeouts:
+// - If ctx is cancelled, the request will be interrupted on the next stream.Recv() call
+// - If ctx times out, the request will return context.DeadlineExceeded
+//
+// Example with timeout:
+//
+//	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+//	defer cancel()
+//	resp, err := client.CreateChatCompletion(ctx, req)
+//
+// Note: Internally, this creates a stream and collects all chunks,
+// so context monitoring happens at the chunk level.
+func (c *Client) CreateChatCompletion(ctx context.Context, req ChatCompletionRequest) (*ChatCompletionResponse, error) {
+	// For non-streaming, we'll collect all chunks and return the final response
+	req.Stream = true // We still use streaming internally, but collect all chunks
+
+	// Prepare request: if Tools is empty, set to nil for proper JSON serialization
+	if len(req.Tools) == 0 {
+		req.Tools = nil
+	}
+
+	stream, err := c.CreateChatCompletionStream(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	defer stream.Close()
+
+	var fullContent strings.Builder
+	var fullToolCalls []ToolCall
+	var finishReason string
+	var usage Usage
+	var responseID string
+	var created int64
+	var model string
+	var systemFingerprint string
+
+	for {
+		chunk, err := stream.Recv()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		if chunk.ID != "" {
+			responseID = chunk.ID
+		}
+		if chunk.Created > 0 {
+			created = chunk.Created
+		}
+		if chunk.Model != "" {
+			model = chunk.Model
+		}
+		if chunk.SystemFingerprint != "" {
+			systemFingerprint = chunk.SystemFingerprint
+		}
+
+		for _, choice := range chunk.Choices {
+			if choice.Delta.Content != "" {
+				fullContent.WriteString(choice.Delta.Content)
+			}
+			if len(choice.Delta.ToolCalls) > 0 {
+				fullToolCalls = append(fullToolCalls, choice.Delta.ToolCalls...)
+			}
+			// Always update finish_reason if present (even if empty string, but should not be empty)
+			// The last chunk (Complete message) should have finish_reason set
+			if choice.FinishReason != "" {
+				finishReason = choice.FinishReason
+			}
+		}
+
+		// Extract usage from chunk if available (usually in the last chunk)
+		// Always update usage if present, as the last chunk should have the final usage
+		if chunk.Usage != nil {
+			usage = *chunk.Usage
+		}
+	}
+
+	// Build final response
+	message := Message{
+		Role:    "assistant",
+		Content: fullContent.String(),
+	}
+	if len(fullToolCalls) > 0 {
+		message.ToolCalls = fullToolCalls
+	}
+
+	// Ensure finish_reason is set (defensive check)
+	// If finish_reason is still empty, default to "stop"
+	if finishReason == "" {
+		finishReason = "stop"
+	}
+
+	return &ChatCompletionResponse{
+		ID:                responseID,
+		Object:            "chat.completion",
+		Created:           created,
+		Model:             model,
+		SystemFingerprint: systemFingerprint,
+		Choices: []Choice{
+			{
+				Index:        0,
+				Message:      message,
+				FinishReason: finishReason,
+			},
+		},
+		Usage: usage,
+	}, nil
+}
+
+// ChatCompletionStream represents a streaming chat completion
+type ChatCompletionStream struct {
+	stream *ffi.SglangStreamHandle
+	mu     sync.Mutex
+	done   bool               // Track if stream has been marked as done
+	ctx    context.Context    // Context for cancellation support
+	cancel context.CancelFunc // Cancel function to stop monitoring goroutine
+	closed chan struct{}      // Signal when stream is closed
+}
+
+// Recv receives the next chunk from the stream.
+//
+// Supports context cancellation: if the context passed to CreateChatCompletionStream
+// is cancelled, Recv will return context.Canceled error on the next call.
+func (s *ChatCompletionStream) Recv() (*ChatCompletionStreamResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Check if context was cancelled
+	select {
+	case <-s.ctx.Done():
+		return nil, s.ctx.Err() // Returns context.Canceled or context.DeadlineExceeded
+	default:
+	}
+
+	if s.stream == nil {
+		return nil, io.EOF
+	}
+
+	// If stream was already marked as done, immediately return EOF
+	// This prevents calling ReadNext() again after isDone=1
+	if s.done {
+		return nil, io.EOF
+	}
+
+	// Loop to handle empty responses (Ok(None) from Rust)
+	// Keep reading until we get actual data or stream ends
+	for {
+		responseJSON, isDone, err := s.stream.ReadNext()
+		if err != nil {
+			return nil, err
+		}
+
+		// Mark stream as done if ReadNext indicates completion
+		if isDone {
+			s.done = true
+		}
+
+		// If we have a response, parse and return it
+		if responseJSON != "" {
+			var response ChatCompletionStreamResponse
+			if err := json.Unmarshal([]byte(responseJSON), &response); err != nil {
+				return nil, fmt.Errorf("failed to parse response: %w", err)
+			}
+			return &response, nil
+		}
+
+		// If stream is done but no response, return EOF
+		if isDone {
+			return nil, io.EOF
+		}
+
+		// Empty response and stream not done - loop to read next chunk
+		// This handles Ok(None) cases where Rust returns no data but stream continues
+	}
+}
+
+// Close closes the stream and cancels any pending operations.
+func (s *ChatCompletionStream) Close() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Cancel the context to signal the monitoring goroutine to stop
+	if s.cancel != nil {
+		s.cancel()
+	}
+
+	// Signal that stream is closed
+	select {
+	case <-s.closed:
+		// Already closed
+	default:
+		close(s.closed)
+	}
+
+	// Free the stream to mark it as completed
+	// This prevents AbortOnDropStream from sending abort when dropped
+	if s.stream != nil {
+		s.stream.Free()
+		s.stream = nil
+	}
+	return nil
+}
+
+// CreateChatCompletionStream creates a streaming chat completion with context cancellation support.
+//
+// Context Support:
+// The ctx parameter is now fully supported for cancellation and timeouts:
+// - If ctx is cancelled, stream.Recv() will return context.Canceled on the next call
+// - If ctx times out (WithTimeout), stream.Recv() will return context.DeadlineExceeded
+// - Calling stream.Close() also cancels the context
+//
+// Example with timeout:
+//
+//	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+//	defer cancel()
+//	stream, err := client.CreateChatCompletionStream(ctx, req)
+//	// Stream will auto-close if 30 seconds elapse
+//
+// Example with cancellation:
+//
+//	ctx, cancel := context.WithCancel(context.Background())
+//	stream, err := client.CreateChatCompletionStream(ctx, req)
+//	go func() {
+//	    time.Sleep(5*time.Second)
+//	    cancel()  // Cancel after 5 seconds
+//	}()
+func (c *Client) CreateChatCompletionStream(ctx context.Context, req ChatCompletionRequest) (*ChatCompletionStream, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.clientHandle == nil {
+		return nil, errors.New("client is closed")
+	}
+
+	// Marshal request to JSON, then ensure tools field is always present.
+	// Due to omitempty tag, empty Tools slice will be omitted from JSON.
+	// We need to ensure tools field is always present as [] when empty (not omitted),
+	// matching the behavior of complete_sdk example.
+	reqJSON, err := json.Marshal(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal request: %w", err)
+	}
+
+	// Unmarshal into map and ensure tools field is present
+	var reqMap map[string]interface{}
+	if err := json.Unmarshal(reqJSON, &reqMap); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal request to map: %w", err)
+	}
+
+	// Add empty tools array if not present
+	if _, exists := reqMap["tools"]; !exists {
+		reqMap["tools"] = []interface{}{}
+	}
+
+	// Marshal back to JSON
+	reqJSON, err = json.Marshal(reqMap)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal request map to JSON: %w", err)
+	}
+
+	// Create stream
+	streamHandle, err := c.clientHandle.ChatCompletionStream(string(reqJSON))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create stream: %w", err)
+	}
+
+	// Create a child context from the provided context for cancellation support
+	streamCtx, cancel := context.WithCancel(ctx)
+
+	stream := &ChatCompletionStream{
+		stream: streamHandle,
+		ctx:    streamCtx,
+		cancel: cancel,
+		closed: make(chan struct{}),
+	}
+
+	return stream, nil
+}

sgl-model-gateway/bindings/golang/client_test.go

@@ -0,0 +1,325 @@
+package sglang
+
+import (
+	"context"
+	"testing"
+)
+
+// TestClientConfig tests ClientConfig validation
+func TestClientConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		config  ClientConfig
+		wantErr bool
+	}{
+		{
+			name: "valid config",
+			config: ClientConfig{
+				Endpoint:      "grpc://localhost:20000",
+				TokenizerPath: "/path/to/tokenizer",
+			},
+			wantErr: false,
+		},
+		{
+			name: "missing endpoint",
+			config: ClientConfig{
+				Endpoint:      "",
+				TokenizerPath: "/path/to/tokenizer",
+			},
+			wantErr: true,
+		},
+		{
+			name: "missing tokenizer path",
+			config: ClientConfig{
+				Endpoint:      "grpc://localhost:20000",
+				TokenizerPath: "",
+			},
+			wantErr: true,
+		},
+		{
+			name: "both missing",
+			config: ClientConfig{
+				Endpoint:      "",
+				TokenizerPath: "",
+			},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := NewClient(tt.config)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("NewClient() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+// TestChatMessageTypes tests ChatMessage struct and its variants
+func TestChatMessageTypes(t *testing.T) {
+	msg := ChatMessage{
+		Role:    "user",
+		Content: "Hello",
+	}
+
+	if msg.Role != "user" {
+		t.Errorf("Expected role 'user', got '%s'", msg.Role)
+	}
+	if msg.Content != "Hello" {
+		t.Errorf("Expected content 'Hello', got '%s'", msg.Content)
+	}
+}
+
+// TestChatCompletionRequestValidation tests ChatCompletionRequest validation
+func TestChatCompletionRequestValidation(t *testing.T) {
+	// Test valid request
+	req := ChatCompletionRequest{
+		Model: "default",
+		Messages: []ChatMessage{
+			{Role: "user", Content: "test"},
+		},
+		Stream: false,
+	}
+
+	if req.Model == "" {
+		t.Error("Expected model to be set")
+	}
+
+	if len(req.Messages) == 0 {
+		t.Error("Expected messages to be non-empty")
+	}
+
+	if req.Messages[0].Role != "user" {
+		t.Errorf("Expected first message role 'user', got '%s'", req.Messages[0].Role)
+	}
+}
+
+// TestClientClose tests that Close can be called multiple times safely
+func TestClientClose(t *testing.T) {
+	// Create a mock client (note: in real tests, you might want to skip this
+	// if it requires actual server connection)
+	config := ClientConfig{
+		Endpoint:      "grpc://localhost:20000",
+		TokenizerPath: "/path/to/tokenizer",
+	}
+
+	// Skip if connection fails (expected in unit test environment)
+	client, err := NewClient(config)
+	if err != nil {
+		t.Skip("Skipping client close test: server not available")
+	}
+
+	// First close should succeed
+	if err := client.Close(); err != nil {
+		t.Errorf("First Close() failed: %v", err)
+	}
+
+	// Second close should also succeed (idempotent)
+	if err := client.Close(); err != nil {
+		t.Errorf("Second Close() failed: %v", err)
+	}
+}
+
+// TestChatCompletionResponseTypes tests response type structures
+func TestChatCompletionResponseTypes(t *testing.T) {
+	resp := ChatCompletionResponse{
+		ID:      "test-id",
+		Model:   "default",
+		Created: 1234567890,
+		Choices: []Choice{
+			{
+				Message: Message{
+					Role:    "assistant",
+					Content: "Hello",
+				},
+				FinishReason: "stop",
+			},
+		},
+		Usage: Usage{
+			PromptTokens:     10,
+			CompletionTokens: 20,
+			TotalTokens:      30,
+		},
+	}
+
+	if resp.ID != "test-id" {
+		t.Errorf("Expected ID 'test-id', got '%s'", resp.ID)
+	}
+
+	if len(resp.Choices) != 1 {
+		t.Errorf("Expected 1 choice, got %d", len(resp.Choices))
+	}
+
+	if resp.Choices[0].Message.Content != "Hello" {
+		t.Errorf("Expected content 'Hello', got '%s'", resp.Choices[0].Message.Content)
+	}
+
+	if resp.Usage.TotalTokens != 30 {
+		t.Errorf("Expected total tokens 30, got %d", resp.Usage.TotalTokens)
+	}
+}
+
+// TestStreamingResponseTypes tests streaming response structures
+func TestStreamingResponseTypes(t *testing.T) {
+	chunk := ChatCompletionStreamResponse{
+		ID:      "stream-id",
+		Created: 1234567890,
+		Choices: []StreamChoice{
+			{
+				Index: 0,
+				Delta: MessageDelta{
+					Content: "Hello",
+				},
+				FinishReason: "",
+			},
+		},
+	}
+
+	if chunk.ID != "stream-id" {
+		t.Errorf("Expected ID 'stream-id', got '%s'", chunk.ID)
+	}
+
+	if len(chunk.Choices) == 0 {
+		t.Error("Expected at least one choice")
+	}
+
+	if chunk.Choices[0].Delta.Content != "Hello" {
+		t.Errorf("Expected delta content 'Hello', got '%s'", chunk.Choices[0].Delta.Content)
+	}
+}
+
+// TestToolCallStructure tests Tool and ToolCall structures
+func TestToolCallStructure(t *testing.T) {
+	tool := Tool{
+		Type: "function",
+		Function: Function{
+			Name:        "get_weather",
+			Description: "Get the weather",
+			Parameters: map[string]interface{}{
+				"location": "string",
+			},
+		},
+	}
+
+	if tool.Type != "function" {
+		t.Errorf("Expected tool type 'function', got '%s'", tool.Type)
+	}
+
+	if tool.Function.Name != "get_weather" {
+		t.Errorf("Expected function name 'get_weather', got '%s'", tool.Function.Name)
+	}
+
+	toolCall := ToolCall{
+		ID:   "call-123",
+		Type: "function",
+		Function: FunctionCall{
+			Name:      "get_weather",
+			Arguments: `{"location": "San Francisco"}`,
+		},
+	}
+
+	if toolCall.ID != "call-123" {
+		t.Errorf("Expected tool call ID 'call-123', got '%s'", toolCall.ID)
+	}
+}
+
+// TestConcurrentClientOperations tests thread safety
+// This is a basic test that just verifies concurrent calls don't panic
+func TestConcurrentClientOperations(t *testing.T) {
+	config := ClientConfig{
+		Endpoint:      "grpc://localhost:20000",
+		TokenizerPath: "/path/to/tokenizer",
+	}
+
+	client, err := NewClient(config)
+	if err != nil {
+		t.Skip("Skipping concurrent operations test: server not available")
+	}
+	defer client.Close()
+
+	// Try concurrent Close calls (should not panic or race)
+	done := make(chan bool, 2)
+
+	go func() {
+		client.Close()
+		done <- true
+	}()
+
+	go func() {
+		client.Close()
+		done <- true
+	}()
+
+	<-done
+	<-done
+}
+
+// BenchmarkChatCompletionRequest benchmarks request creation
+func BenchmarkChatCompletionRequest(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_ = ChatCompletionRequest{
+			Model: "default",
+			Messages: []ChatMessage{
+				{Role: "user", Content: "test message"},
+			},
+			Stream:              false,
+			Temperature:         floatPtr(0.7),
+			MaxCompletionTokens: intPtr(100),
+		}
+	}
+}
+
+// Helper functions for benchmarks
+func floatPtr(f float32) *float32 {
+	return &f
+}
+
+func intPtr(i int) *int {
+	return &i
+}
+
+// TestContextCancellation tests that cancelled context is handled gracefully.
+//
+// NOTE: Currently, the FFI layer is blocking and doesn't actively monitor context cancellation.
+// This test verifies that the client at least returns an error rather than panicking or
+// hanging indefinitely when a pre-cancelled context is passed.
+//
+// Future: When FFI supports context cancellation (via signals or async operations),
+// this test should be updated to assert that the error is context.Canceled or wrapped
+// context cancellation error.
+func TestContextCancellation(t *testing.T) {
+	config := ClientConfig{
+		Endpoint:      "grpc://localhost:20000",
+		TokenizerPath: "/path/to/tokenizer",
+	}
+
+	client, err := NewClient(config)
+	if err != nil {
+		t.Skip("Skipping context cancellation test: server not available")
+	}
+	defer client.Close()
+
+	// Create a pre-cancelled context
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	req := ChatCompletionRequest{
+		Model: "default",
+		Messages: []ChatMessage{
+			{Role: "user", Content: "test"},
+		},
+	}
+
+	// Attempt request with cancelled context
+	// Since FFI is blocking, we expect either:
+	// 1. An error from the server/network
+	// 2. The call to complete normally (FFI doesn't check context)
+	// What we DON'T expect is a panic or indefinite hang
+	_, err = client.CreateChatCompletion(ctx, req)
+	if err != nil {
+		t.Logf("Request with cancelled context returned error: %v", err)
+	} else {
+		t.Logf("Request with cancelled context completed (FFI may not support context cancellation)")
+	}
+}

sgl-model-gateway/bindings/golang/examples/simple/main.go

@@ -0,0 +1,85 @@
+// Simple example demonstrating basic usage of SGLang Go SDK
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/sglang/sglang-go-grpc-sdk"
+)
+
+func main() {
+	// Get configuration from environment or command line
+	endpoint := os.Getenv("SGL_GRPC_ENDPOINT")
+	if endpoint == "" {
+		endpoint = "grpc://localhost:20000"
+	}
+
+	tokenizerPath := os.Getenv("SGL_TOKENIZER_PATH")
+	if tokenizerPath == "" {
+		tokenizerPath = "./examples/tokenizer"
+	}
+
+	// Create client
+	client, err := sglang.NewClient(sglang.ClientConfig{
+		Endpoint:      endpoint,
+		TokenizerPath: tokenizerPath,
+	})
+	if err != nil {
+		log.Fatalf("Failed to create client: %v", err)
+	}
+	defer client.Close()
+
+	// Create chat completion request
+	req := sglang.ChatCompletionRequest{
+		Model: "default",
+		Messages: []sglang.ChatMessage{
+			{
+				Role:    "system",
+				Content: "You are a helpful assistant.",
+			},
+			{
+				Role:    "user",
+				Content: "写一首歌关于夏天",
+			},
+		},
+		Stream:              false,
+		Temperature:         float32Ptr(0.7),
+		MaxCompletionTokens: intPtr(200),
+		SkipSpecialTokens:   true,
+		Tools:               nil, // Use nil instead of empty slice to avoid template errors
+	}
+
+	// Create completion
+	ctx := context.Background()
+	resp, err := client.CreateChatCompletion(ctx, req)
+	if err != nil {
+		log.Fatalf("Failed to create completion: %v", err)
+	}
+
+	// Print response
+	fmt.Println("=== Response ===")
+	fmt.Printf("ID: %s\n", resp.ID)
+	fmt.Printf("Model: %s\n", resp.Model)
+	fmt.Printf("Created: %d\n", resp.Created)
+	fmt.Println("\nContent:")
+	for _, choice := range resp.Choices {
+		fmt.Println(choice.Message.Content)
+	}
+	fmt.Printf("\nFinish Reason: %s\n", resp.Choices[0].FinishReason)
+	fmt.Printf("\nUsage: Prompt=%d, Completion=%d, Total=%d\n",
+		resp.Usage.PromptTokens,
+		resp.Usage.CompletionTokens,
+		resp.Usage.TotalTokens,
+	)
+}
+
+func float32Ptr(f float32) *float32 {
+	return &f
+}
+
+func intPtr(i int) *int {
+	return &i
+}

sgl-model-gateway/bindings/golang/examples/simple/run.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Simple example runner
+# Usage: ./run.sh [tokenizer_path] [endpoint]
+
+# Set library path for Rust FFI library
+# The library should be in ./lib directory (created by 'make lib')
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+LIB_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)/lib"
+
+# Check if lib directory exists
+if [ ! -d "$LIB_DIR" ]; then
+    echo "Error: Library directory not found at $LIB_DIR"
+    echo "Please run 'make lib' first to build and export the library"
+    exit 1
+fi
+
+# Get Python LDFLAGS (needed for Rust FFI that depends on Python)
+PYTHON_LDFLAGS=$(python3-config --ldflags --embed 2>/dev/null || python3-config --ldflags 2>/dev/null || echo "")
+
+# Set CGO_LDFLAGS to link with the Rust library
+export CGO_LDFLAGS="-L${LIB_DIR} -lsglang_router_rs ${PYTHON_LDFLAGS} -ldl"
+
+# macOS uses DYLD_LIBRARY_PATH, Linux uses LD_LIBRARY_PATH
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    export DYLD_LIBRARY_PATH="${LIB_DIR}:${DYLD_LIBRARY_PATH}"
+else
+    export LD_LIBRARY_PATH="${LIB_DIR}:${LD_LIBRARY_PATH}"
+fi
+
+# Default configuration (can be overridden by environment variables or command line arguments)
+# Tokenizer path: ../tokenizer (relative to this script)
+DEFAULT_TOKENIZER_PATH="${SGL_TOKENIZER_PATH:-../tokenizer}"
+DEFAULT_ENDPOINT="${SGL_GRPC_ENDPOINT:-grpc://localhost:20000}"
+
+TOKENIZER_PATH="${1:-${DEFAULT_TOKENIZER_PATH}}"
+ENDPOINT="${2:-${DEFAULT_ENDPOINT}}"
+
+echo "Running simple example..."
+echo "Library path: ${LIB_DIR}"
+echo "Tokenizer: $TOKENIZER_PATH"
+echo "Endpoint: $ENDPOINT"
+echo ""
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+SGL_TOKENIZER_PATH="$TOKENIZER_PATH" SGL_GRPC_ENDPOINT="$ENDPOINT" go run main.go

sgl-model-gateway/bindings/golang/examples/streaming/main.go

@@ -0,0 +1,125 @@
+// Streaming example demonstrating real-time streaming with SGLang Go SDK
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/sglang/sglang-go-grpc-sdk"
+)
+
+func main() {
+	// Get configuration from environment or command line
+	endpoint := os.Getenv("SGL_GRPC_ENDPOINT")
+	if endpoint == "" {
+		endpoint = "grpc://localhost:20000"
+	}
+
+	tokenizerPath := os.Getenv("SGL_TOKENIZER_PATH")
+	if tokenizerPath == "" {
+		tokenizerPath = "./examples/tokenizer"
+	}
+
+	// Create client
+	client, err := sglang.NewClient(sglang.ClientConfig{
+		Endpoint:      endpoint,
+		TokenizerPath: tokenizerPath,
+	})
+	if err != nil {
+		log.Fatalf("Failed to create client: %v", err)
+	}
+	defer client.Close()
+
+	// Create streaming chat completion request
+	req := sglang.ChatCompletionRequest{
+		Model: "default",
+		Messages: []sglang.ChatMessage{
+			{
+				Role:    "system",
+				Content: "You are a helpful assistant.",
+			},
+			{
+				Role:    "user",
+				Content: "写一首春天的诗歌",
+			},
+		},
+		Stream:              true,
+		Temperature:         float32Ptr(0.7),
+		MaxCompletionTokens: intPtr(500),
+		SkipSpecialTokens:   true,
+		Tools:               nil, // Use nil instead of empty slice to avoid template errors
+	}
+
+	// Create streaming completion
+	ctx := context.Background()
+	stream, err := client.CreateChatCompletionStream(ctx, req)
+	if err != nil {
+		log.Fatalf("Failed to create stream: %v", err)
+	}
+	defer stream.Close()
+
+	fmt.Println("=== Streaming Response ===")
+	fmt.Println()
+
+	var fullContent strings.Builder
+	chunkCount := 0
+	startTime := time.Now()
+	var firstTokenTime time.Time
+	firstTokenReceived := false
+
+	for {
+		chunk, err := stream.Recv()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatalf("Stream error: %v", err)
+		}
+
+		chunkCount++
+
+		// Extract content from delta
+		for _, choice := range chunk.Choices {
+			if choice.Delta.Content != "" {
+				fmt.Print(choice.Delta.Content)
+				fullContent.WriteString(choice.Delta.Content)
+
+				// Track first token time (TTFT)
+				if !firstTokenReceived {
+					firstTokenTime = time.Now()
+					firstTokenReceived = true
+					ttft := firstTokenTime.Sub(startTime)
+					fmt.Printf("\n[TTFT: %v]\n", ttft)
+				}
+			}
+
+			if choice.FinishReason != "" {
+				fmt.Printf("\n\n[Finished: %s]\n", choice.FinishReason)
+			}
+		}
+	}
+
+	// Calculate metrics
+	if firstTokenReceived {
+		elapsed := time.Since(startTime)
+		tokensPerSecond := float64(fullContent.Len()) / elapsed.Seconds()
+		fmt.Printf("\n=== Metrics ===\n")
+		fmt.Printf("Total chunks: %d\n", chunkCount)
+		fmt.Printf("Total content length: %d characters\n", fullContent.Len())
+		fmt.Printf("Time elapsed: %v\n", elapsed)
+		fmt.Printf("Tokens per second: %.2f\n", tokensPerSecond)
+	}
+}
+
+func float32Ptr(f float32) *float32 {
+	return &f
+}
+
+func intPtr(i int) *int {
+	return &i
+}

sgl-model-gateway/bindings/golang/examples/streaming/run.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Streaming example runner
+# Usage: ./run.sh [tokenizer_path] [endpoint]
+
+# Set library path for Rust FFI library
+# The library should be in ./lib directory (created by 'make lib')
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+LIB_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)/lib"
+
+# Check if lib directory exists
+if [ ! -d "$LIB_DIR" ]; then
+    echo "Error: Library directory not found at $LIB_DIR"
+    echo "Please run 'make lib' first to build and export the library"
+    exit 1
+fi
+
+# Get Python LDFLAGS (needed for Rust FFI that depends on Python)
+PYTHON_LDFLAGS=$(python3-config --ldflags --embed 2>/dev/null || python3-config --ldflags 2>/dev/null || echo "")
+
+# Set CGO_LDFLAGS to link with the Rust library
+export CGO_LDFLAGS="-L${LIB_DIR} -lsglang_router_rs ${PYTHON_LDFLAGS} -ldl"
+
+# macOS uses DYLD_LIBRARY_PATH, Linux uses LD_LIBRARY_PATH
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    export DYLD_LIBRARY_PATH="${LIB_DIR}:${DYLD_LIBRARY_PATH}"
+else
+    export LD_LIBRARY_PATH="${LIB_DIR}:${LD_LIBRARY_PATH}"
+fi
+
+# Default configuration (can be overridden by environment variables or command line arguments)
+# Tokenizer path: ../tokenizer (relative to this script)
+DEFAULT_TOKENIZER_PATH="${SGL_TOKENIZER_PATH:-../tokenizer}"
+DEFAULT_ENDPOINT="${SGL_GRPC_ENDPOINT:-grpc://localhost:20000}"
+
+TOKENIZER_PATH="${1:-${DEFAULT_TOKENIZER_PATH}}"
+ENDPOINT="${2:-${DEFAULT_ENDPOINT}}"
+
+echo "Running streaming example..."
+echo "Library path: ${LIB_DIR}"
+echo "Tokenizer: $TOKENIZER_PATH"
+echo "Endpoint: $ENDPOINT"
+echo ""
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+SGL_TOKENIZER_PATH="$TOKENIZER_PATH" SGL_GRPC_ENDPOINT="$ENDPOINT" go run main.go

sgl-model-gateway/bindings/golang/integration_test.go

@@ -0,0 +1,228 @@
+//go:build integration
+// +build integration
+
+// integration_test.go contains integration tests that require a running SGLang server
+//
+// To run these tests:
+// 1. Start an SGLang server: python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-hf
+// 2. Run: go test -tags=integration -run TestIntegration
+
+package sglang
+
+import (
+	"context"
+	"io"
+	"os"
+	"testing"
+	"time"
+)
+
+// getTestConfig returns test configuration from environment or defaults
+func getTestConfig(t *testing.T) ClientConfig {
+	endpoint := os.Getenv("SGL_GRPC_ENDPOINT")
+	if endpoint == "" {
+		endpoint = "grpc://localhost:20000"
+	}
+
+	tokenizerPath := os.Getenv("SGL_TOKENIZER_PATH")
+	if tokenizerPath == "" {
+		t.Skip("SGL_TOKENIZER_PATH not set")
+	}
+
+	return ClientConfig{
+		Endpoint:      endpoint,
+		TokenizerPath: tokenizerPath,
+	}
+}
+
+// TestIntegrationNonStreamingCompletion tests non-streaming chat completion
+func TestIntegrationNonStreamingCompletion(t *testing.T) {
+	config := getTestConfig(t)
+
+	client, err := NewClient(config)
+	if err != nil {
+		t.Fatalf("Failed to create client: %v", err)
+	}
+	defer client.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	req := ChatCompletionRequest{
+		Model: "default",
+		Messages: []ChatMessage{
+			{Role: "user", Content: "Say 'Hello, World!' only"},
+		},
+		Stream:              false,
+		Temperature:         float32Ptr(0.0),
+		MaxCompletionTokens: intPtr(50),
+	}
+
+	resp, err := client.CreateChatCompletion(ctx, req)
+	if err != nil {
+		t.Fatalf("CreateChatCompletion failed: %v", err)
+	}
+
+	if resp.ID == "" {
+		t.Error("Response ID is empty")
+	}
+
+	if len(resp.Choices) == 0 {
+		t.Error("Response has no choices")
+	}
+
+	if resp.Choices[0].Message.Content == "" {
+		t.Error("Response content is empty")
+	}
+
+	if resp.Usage == nil || resp.Usage.TotalTokens == 0 {
+		t.Error("Usage information is missing or invalid")
+	}
+
+	t.Logf("Response: %s", resp.Choices[0].Message.Content)
+	t.Logf("Usage: %+v", resp.Usage)
+}
+
+// TestIntegrationStreamingCompletion tests streaming chat completion
+func TestIntegrationStreamingCompletion(t *testing.T) {
+	config := getTestConfig(t)
+
+	client, err := NewClient(config)
+	if err != nil {
+		t.Fatalf("Failed to create client: %v", err)
+	}
+	defer client.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	req := ChatCompletionRequest{
+		Model: "default",
+		Messages: []ChatMessage{
+			{Role: "user", Content: "Count from 1 to 5"},
+		},
+		Stream:              true,
+		Temperature:         float32Ptr(0.0),
+		MaxCompletionTokens: intPtr(100),
+	}
+
+	stream, err := client.CreateChatCompletionStream(ctx, req)
+	if err != nil {
+		t.Fatalf("CreateChatCompletionStream failed: %v", err)
+	}
+	defer stream.Close()
+
+	chunkCount := 0
+	totalContent := ""
+
+	for {
+		chunk, err := stream.Recv()
+		if err == io.EOF {
+			// io.EOF is expected at end of stream
+			break
+		}
+		if err != nil {
+			t.Fatalf("Stream error: %v", err)
+		}
+
+		chunkCount++
+
+		for _, choice := range chunk.Choices {
+			if choice.Delta.Content != "" {
+				totalContent += choice.Delta.Content
+			}
+		}
+	}
+
+	if chunkCount == 0 {
+		t.Error("Received no chunks from stream")
+	}
+
+	if totalContent == "" {
+		t.Error("Received no content from stream")
+	}
+
+	t.Logf("Received %d chunks with content: %s", chunkCount, totalContent)
+}
+
+// TestIntegrationConcurrentRequests tests multiple concurrent requests
+func TestIntegrationConcurrentRequests(t *testing.T) {
+	config := getTestConfig(t)
+
+	client, err := NewClient(config)
+	if err != nil {
+		t.Fatalf("Failed to create client: %v", err)
+	}
+	defer client.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	numRequests := 3
+	done := make(chan error, numRequests)
+
+	for i := 0; i < numRequests; i++ {
+		go func(idx int) {
+			req := ChatCompletionRequest{
+				Model: "default",
+				Messages: []ChatMessage{
+					{Role: "user", Content: "Say 'test'"},
+				},
+				Stream:              false,
+				MaxCompletionTokens: intPtr(50),
+			}
+
+			_, err := client.CreateChatCompletion(ctx, req)
+			done <- err
+		}(i)
+	}
+
+	// Collect results
+	for i := 0; i < numRequests; i++ {
+		if err := <-done; err != nil {
+			t.Errorf("Request %d failed: %v", i, err)
+		}
+	}
+
+	t.Logf("All %d concurrent requests completed successfully", numRequests)
+}
+
+// TestIntegrationContextCancellation tests that context cancellation is handled
+func TestIntegrationContextCancellation(t *testing.T) {
+	config := getTestConfig(t)
+
+	client, err := NewClient(config)
+	if err != nil {
+		t.Fatalf("Failed to create client: %v", err)
+	}
+	defer client.Close()
+
+	// Create a context that cancels immediately
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	req := ChatCompletionRequest{
+		Model: "default",
+		Messages: []ChatMessage{
+			{Role: "user", Content: "test"},
+		},
+		Stream: false,
+	}
+
+	// Should handle cancelled context gracefully
+	_, err = client.CreateChatCompletion(ctx, req)
+	if err == nil {
+		t.Error("Expected error from cancelled context")
+	}
+
+	t.Logf("Cancelled context handled: %v", err)
+}
+
+// Helper functions
+func float32Ptr(f float32) *float32 {
+	return &f
+}
+
+func intPtr(i int) *int {
+	return &i
+}

sgl-model-gateway/bindings/golang/internal/ffi/client.go

@@ -0,0 +1,228 @@
+// Package ffi provides Go bindings for SGLang's Rust FFI (Foreign Function Interface).
+//
+// This package wraps the Rust FFI layer of SGLang, providing low-level access to:
+// - Client creation and connection management
+// - Chat completion streaming
+// - Stream reading and response conversion
+// - Memory management for C strings
+//
+// Internal use only: This package is intended for internal use by the sglang package.
+// End users should use the public sglang package instead.
+package ffi
+
+/*
+#cgo LDFLAGS: -lsglang_router_rs -ldl
+#include <stdlib.h>
+#include <stdint.h>
+
+// Error codes
+typedef enum {
+    SGL_ERROR_SUCCESS = 0,
+    SGL_ERROR_INVALID_ARGUMENT = 1,
+    SGL_ERROR_TOKENIZATION_ERROR = 2,
+    SGL_ERROR_PARSING_ERROR = 3,
+    SGL_ERROR_MEMORY_ERROR = 4,
+    SGL_ERROR_UNKNOWN = 99
+} SglErrorCode;
+
+// Opaque handles
+typedef void* SglangClientHandle;
+typedef void* SglangStreamHandle;
+
+// Client SDK functions
+SglangClientHandle* sgl_client_create(const char* endpoint, const char* tokenizer_path, char** error_out);
+void sgl_client_free(SglangClientHandle* handle);
+SglErrorCode sgl_client_chat_completion_stream(SglangClientHandle* client_handle, const char* request_json, SglangStreamHandle** stream_handle_out, char** error_out);
+SglErrorCode sgl_stream_read_next(SglangStreamHandle* stream_handle, char** response_json_out, int* is_done_out, char** error_out);
+void sgl_stream_free(SglangStreamHandle* handle);
+void sgl_free_string(char* s);
+*/
+import "C"
+
+import (
+	"fmt"
+	"unsafe"
+)
+
+// ErrorCode represents FFI error codes returned by Rust functions.
+//
+// These codes indicate the result of FFI operations. Use Error() to get a human-readable
+// error message.
+type ErrorCode int
+
+const (
+	// ErrorSuccess indicates the operation completed successfully
+	ErrorSuccess ErrorCode = 0
+	// ErrorInvalidArgument indicates invalid arguments were passed to the FFI function
+	ErrorInvalidArgument ErrorCode = 1
+	// ErrorTokenizationError indicates an error during tokenization
+	ErrorTokenizationError ErrorCode = 2
+	// ErrorParsingError indicates an error parsing the response or request
+	ErrorParsingError ErrorCode = 3
+	// ErrorMemoryError indicates a memory allocation error
+	ErrorMemoryError ErrorCode = 4
+	// ErrorUnknown indicates an unclassified error
+	ErrorUnknown ErrorCode = 99
+)
+
+// Error implements the error interface for ErrorCode.
+func (e ErrorCode) Error() string {
+	switch e {
+	case ErrorSuccess:
+		return "success"
+	case ErrorInvalidArgument:
+		return "invalid argument"
+	case ErrorTokenizationError:
+		return "tokenization error"
+	case ErrorParsingError:
+		return "parsing error"
+	case ErrorMemoryError:
+		return "memory error"
+	case ErrorUnknown:
+		return "unknown error"
+	default:
+		return fmt.Sprintf("unknown error code: %d", e)
+	}
+}
+
+// SglangClientHandle wraps the Rust client SDK FFI handle.
+//
+// This struct maintains a connection to the SGLang gRPC server and is used
+// to create streams and manage the underlying Rust client resources.
+type SglangClientHandle struct {
+	handle *C.SglangClientHandle
+}
+
+// NewClient creates a new SGLang client handle via FFI.
+//
+// This function initializes the Rust client with the given endpoint and tokenizer path.
+//
+// Parameters:
+// - endpoint: gRPC endpoint URL (e.g., "grpc://localhost:20000")
+// - tokenizerPath: Path to tokenizer directory
+//
+// Returns:
+// - *SglangClientHandle: A new client handle
+// - error: An error if client creation failed
+func NewClient(endpoint, tokenizerPath string) (*SglangClientHandle, error) {
+	cEndpoint := C.CString(endpoint)
+	defer C.free(unsafe.Pointer(cEndpoint))
+
+	cTokenizerPath := C.CString(tokenizerPath)
+	defer C.free(unsafe.Pointer(cTokenizerPath))
+
+	var errorPtr *C.char
+	handle := C.sgl_client_create(cEndpoint, cTokenizerPath, &errorPtr)
+
+	if handle == nil {
+		errorMsg := ""
+		if errorPtr != nil {
+			errorMsg = C.GoString(errorPtr)
+			C.sgl_free_string(errorPtr)
+		}
+		if errorMsg == "" {
+			errorMsg = "failed to create client"
+		}
+		return nil, fmt.Errorf("%s", errorMsg)
+	}
+
+	return &SglangClientHandle{handle: handle}, nil
+}
+
+// Free releases the client handle
+func (h *SglangClientHandle) Free() {
+	if h.handle != nil {
+		C.sgl_client_free(h.handle)
+		h.handle = nil
+	}
+}
+
+// ChatCompletionStream creates a streaming chat completion request
+func (h *SglangClientHandle) ChatCompletionStream(requestJSON string) (*SglangStreamHandle, error) {
+	if h.handle == nil {
+		return nil, fmt.Errorf("client handle is nil")
+	}
+
+	cRequestJSON := C.CString(requestJSON)
+	defer C.free(unsafe.Pointer(cRequestJSON))
+
+	var streamHandle *C.SglangStreamHandle
+	var errorPtr *C.char
+
+	result := C.sgl_client_chat_completion_stream(
+		h.handle,
+		cRequestJSON,
+		&streamHandle,
+		&errorPtr,
+	)
+
+	if ErrorCode(result) != ErrorSuccess {
+		errorMsg := ""
+		if errorPtr != nil {
+			errorMsg = C.GoString(errorPtr)
+			C.sgl_free_string(errorPtr)
+		}
+		if errorMsg == "" {
+			errorMsg = fmt.Sprintf("error code %d", result)
+		}
+		return nil, fmt.Errorf("%s", errorMsg)
+	}
+
+	if streamHandle == nil {
+		return nil, fmt.Errorf("stream handle is nil")
+	}
+
+	return &SglangStreamHandle{handle: streamHandle}, nil
+}
+
+// SglangStreamHandle wraps the Rust stream FFI handle
+type SglangStreamHandle struct {
+	handle *C.SglangStreamHandle
+}
+
+// ReadNext reads the next chunk from the stream
+// Returns: (responseJSON, isDone, error)
+func (h *SglangStreamHandle) ReadNext() (string, bool, error) {
+	if h.handle == nil {
+		return "", true, fmt.Errorf("stream handle is nil")
+	}
+
+	var responseJSON *C.char
+	var isDone C.int
+	var errorPtr *C.char
+
+	result := C.sgl_stream_read_next(
+		h.handle,
+		&responseJSON,
+		&isDone,
+		&errorPtr,
+	)
+
+	if ErrorCode(result) != ErrorSuccess {
+		errorMsg := ""
+		if errorPtr != nil {
+			errorMsg = C.GoString(errorPtr)
+			C.sgl_free_string(errorPtr)
+		}
+		if errorMsg == "" {
+			errorMsg = fmt.Sprintf("error code %d", result)
+		}
+		return "", isDone == 1, fmt.Errorf("%s", errorMsg)
+	}
+
+	responseStr := ""
+	if responseJSON != nil {
+		responseStr = C.GoString(responseJSON)
+		C.sgl_free_string(responseJSON)
+	}
+
+	return responseStr, isDone == 1, nil
+}
+
+// Free releases the stream handle
+func (h *SglangStreamHandle) Free() {
+	if h.handle != nil {
+		C.sgl_stream_free(h.handle)
+		h.handle = nil
+	}
+}

sgl-model-gateway/bindings/golang/src/client.rs

@@ -0,0 +1,279 @@
+//! Client SDK FFI functions
+
+use std::ffi::{CStr, CString};
+use std::os::raw::{c_char};
+use std::ptr;
+use std::sync::Arc;
+use tokio::runtime::Runtime;
+use once_cell::sync::Lazy;
+use uuid::Uuid;
+
+use sgl_model_gateway::tokenizer::create_tokenizer_from_file;
+use sgl_model_gateway::tokenizer::traits::Tokenizer;
+use sgl_model_gateway::grpc_client::sglang_scheduler::SglangSchedulerClient;
+use sgl_model_gateway::protocols::chat::ChatCompletionRequest;
+use sgl_model_gateway::routers::grpc::utils::{process_chat_messages, generate_tool_constraints};
+
+use super::error::{SglErrorCode, set_error_message};
+use super::grpc_converter::sgl_grpc_response_converter_create;
+use super::tokenizer::TokenizerHandle;
+use super::stream::SglangStreamHandle;
+
+/// Global tokio runtime for async operations
+static RUNTIME: Lazy<Runtime> = Lazy::new(|| {
+    Runtime::new().expect("Failed to create tokio runtime for client FFI")
+});
+
+/// Handle for complete client SDK (gRPC client + tokenizer)
+/// This handle manages the connection to sglang and provides a complete SDK interface
+pub struct SglangClientHandle {
+    pub(crate) client: Arc<SglangSchedulerClient>,
+    pub(crate) tokenizer: Arc<dyn Tokenizer>,
+}
+
+/// Handle for streaming request (includes prompt token count)
+#[allow(dead_code)]
+pub struct StreamRequestState {
+    pub(crate) prompt_tokens: i32, // Number of prompt tokens for this request
+}
+
+/// Create a new SGLang client handle
+///
+/// # Arguments
+/// * `endpoint` - gRPC endpoint (e.g., "grpc://localhost:20000")
+/// * `tokenizer_path` - Path to tokenizer directory
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * Pointer to SglangClientHandle on success, null on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_client_create(
+    endpoint: *const c_char,
+    tokenizer_path: *const c_char,
+    error_out: *mut *mut c_char,
+) -> *mut SglangClientHandle {
+    if endpoint.is_null() || tokenizer_path.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return ptr::null_mut();
+    }
+
+    let endpoint_str = match CStr::from_ptr(endpoint).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in endpoint");
+            return ptr::null_mut();
+        }
+    };
+
+    let tokenizer_path_str = match CStr::from_ptr(tokenizer_path).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in tokenizer_path");
+            return ptr::null_mut();
+        }
+    };
+
+    // Create tokenizer
+    let tokenizer = match create_tokenizer_from_file(tokenizer_path_str) {
+        Ok(t) => t,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to create tokenizer: {}", e));
+            return ptr::null_mut();
+        }
+    };
+
+    // Create gRPC client
+    let client = match RUNTIME.block_on(async {
+        SglangSchedulerClient::connect(endpoint_str).await
+    }) {
+        Ok(c) => Arc::new(c),
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to connect to endpoint: {}", e));
+            return ptr::null_mut();
+        }
+    };
+
+    Box::into_raw(Box::new(SglangClientHandle {
+        client,
+        tokenizer,
+    }))
+}
+
+/// Free a client handle
+#[no_mangle]
+pub unsafe extern "C" fn sgl_client_free(handle: *mut SglangClientHandle) {
+    if !handle.is_null() {
+        let _ = Box::from_raw(handle);
+    }
+}
+
+/// Send a chat completion request and start streaming
+///
+/// # Arguments
+/// * `client_handle` - Client handle
+/// * `request_json` - OpenAI ChatCompletionRequest as JSON string
+/// * `stream_handle_out` - Pointer to receive stream handle
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_client_chat_completion_stream(
+    client_handle: *mut SglangClientHandle,
+    request_json: *const c_char,
+    stream_handle_out: *mut *mut SglangStreamHandle,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if client_handle.is_null() || request_json.is_null() || stream_handle_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let request_str = match CStr::from_ptr(request_json).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in request_json");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    let client_ref = &*client_handle;
+    let client = Arc::clone(&client_ref.client);
+    let tokenizer = Arc::clone(&client_ref.tokenizer);
+
+    // Parse OpenAI ChatCompletionRequest
+    let chat_request: ChatCompletionRequest = match serde_json::from_str(request_str) {
+        Ok(req) => req,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to parse request JSON: {}", e));
+            return SglErrorCode::ParsingError;
+        }
+    };
+
+    // Process messages and apply chat template
+    let processed_messages = match process_chat_messages(&chat_request, tokenizer.as_ref()) {
+        Ok(msgs) => msgs,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to process messages: {}", e));
+            return SglErrorCode::TokenizationError;
+        }
+    };
+
+    // Tokenize
+    let token_ids = match tokenizer.encode(&processed_messages.text) {
+        Ok(encoding) => encoding.token_ids().to_vec(),
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to tokenize: {}", e));
+            return SglErrorCode::TokenizationError;
+        }
+    };
+    let prompt_tokens = token_ids.len() as i32; // Save prompt token count
+
+    // Generate tool constraints if needed
+    let tool_constraint = if let Some(tools) = chat_request.tools.as_ref() {
+        match generate_tool_constraints(tools, &chat_request.tool_choice, &chat_request.model) {
+            Ok(Some((constraint_type, constraint_value))) => Some((constraint_type, constraint_value)),
+            Ok(None) => None,
+            Err(e) => {
+                set_error_message(error_out, &format!("Failed to generate tool constraints: {}", e));
+                return SglErrorCode::ParsingError;
+            }
+        }
+    } else {
+        None
+    };
+
+    // Build GenerateRequest
+    let request_id = format!("chatcmpl-{}", Uuid::new_v4());
+    let proto_request = match client.build_generate_request_from_chat(
+        request_id.clone(),
+        &chat_request,
+        processed_messages.text,
+        token_ids,
+        processed_messages.multimodal_inputs,
+        tool_constraint,
+    ) {
+        Ok(req) => req,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to build generate request: {}", e));
+            return SglErrorCode::ParsingError;
+        }
+    };
+
+    // Send request and get stream
+    let stream = match RUNTIME.block_on(async {
+        client.generate(proto_request).await
+    }) {
+        Ok(s) => s,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to send request: {}", e));
+            return SglErrorCode::UnknownError;
+        }
+    };
+
+    // Create response converter
+    let tools_json = chat_request.tools.as_ref()
+        .and_then(|t| serde_json::to_string(t).ok())
+        .map(|s| CString::new(s).unwrap().into_raw());
+    let tool_choice_json = chat_request.tool_choice.as_ref()
+        .and_then(|tc| serde_json::to_string(tc).ok())
+        .map(|s| CString::new(s).unwrap().into_raw());
+    let stop_json = chat_request.stop.as_ref()
+        .and_then(|s| serde_json::to_string(s).ok())
+        .map(|s| CString::new(s).unwrap().into_raw());
+    let stop_token_ids_json = chat_request.stop_token_ids.as_ref()
+        .and_then(|ids| serde_json::to_string(ids).ok())
+        .map(|s| CString::new(s).unwrap().into_raw());
+
+    // Create tokenizer handle for converter (we'll create a temporary one)
+    let tokenizer_handle = Box::into_raw(Box::new(TokenizerHandle {
+        tokenizer: Arc::clone(&tokenizer),
+    }));
+
+    let converter = sgl_grpc_response_converter_create(
+        tokenizer_handle,
+        CString::new(chat_request.model.clone()).unwrap().as_ptr(),
+        CString::new(request_id.clone()).unwrap().as_ptr(),
+        tools_json.unwrap_or(ptr::null_mut()),
+        tool_choice_json.unwrap_or(ptr::null_mut()),
+        stop_json.unwrap_or(ptr::null_mut()),
+        stop_token_ids_json.unwrap_or(ptr::null_mut()),
+        if chat_request.skip_special_tokens { 1 } else { 0 },
+        error_out,
+    );
+
+    // Free temporary tokenizer handle (converter now owns the tokenizer)
+    let _ = Box::from_raw(tokenizer_handle);
+
+    if converter.is_null() {
+        return SglErrorCode::MemoryError;
+    }
+
+    // Clean up temporary CStrings
+    if let Some(ptr) = tools_json {
+        let _ = CString::from_raw(ptr);
+    }
+    if let Some(ptr) = tool_choice_json {
+        let _ = CString::from_raw(ptr);
+    }
+    if let Some(ptr) = stop_json {
+        let _ = CString::from_raw(ptr);
+    }
+    if let Some(ptr) = stop_token_ids_json {
+        let _ = CString::from_raw(ptr);
+    }
+
+    // Create converter handle and set initial_prompt_tokens immediately
+    let mut converter_handle = *Box::from_raw(converter);
+    converter_handle.initial_prompt_tokens = Some(prompt_tokens);
+
+    // Create stream handle with prompt_tokens
+    *stream_handle_out = Box::into_raw(Box::new(SglangStreamHandle {
+        stream: Arc::new(tokio::sync::Mutex::new(stream)),
+        converter: Arc::new(tokio::sync::Mutex::new(converter_handle)),
+        client: Arc::clone(&client),
+        prompt_tokens,
+    }));
+
+    SglErrorCode::Success
+}

sgl-model-gateway/bindings/golang/src/error.rs

@@ -0,0 +1,50 @@
+//! Error handling for FFI functions
+
+use std::ffi::CString;
+use std::os::raw::c_char;
+use std::ptr;
+
+/// Error codes returned by FFI functions
+#[repr(C)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SglErrorCode {
+    Success = 0,
+    InvalidArgument = 1,
+    TokenizationError = 2,
+    ParsingError = 3,
+    MemoryError = 4,
+    UnknownError = 99,
+}
+
+/// Helper to set error message in FFI output parameter
+pub fn set_error_message(error_out: *mut *mut c_char, message: &str) {
+    unsafe {
+        if !error_out.is_null() {
+            if let Ok(cstr) = CString::new(message) {
+                *error_out = cstr.into_raw();
+            } else {
+                *error_out = ptr::null_mut();
+            }
+        }
+    }
+}
+
+/// Helper to set error message from format string
+pub fn set_error_message_fmt(error_out: *mut *mut c_char, fmt: std::fmt::Arguments) {
+    if !error_out.is_null() {
+        let msg = format!("{}", fmt);
+        set_error_message(error_out, &msg);
+    }
+}
+
+/// Helper to clear error message
+pub fn clear_error_message(error_out: *mut *mut c_char) {
+    unsafe {
+        if !error_out.is_null() {
+            *error_out = ptr::null_mut();
+        }
+    }
+}
+
+// Helper functions for error handling
+// Note: Some helper functions are kept for potential future use

sgl-model-gateway/bindings/golang/src/grpc_converter.rs

@@ -0,0 +1,758 @@
+//! gRPC response converter FFI functions
+
+use std::ffi::{CStr, CString};
+use std::os::raw::{c_char, c_int};
+use std::ptr;
+use std::sync::Arc;
+use std::collections::HashMap;
+use serde_json::Value;
+use tokio::runtime::Runtime;
+use once_cell::sync::Lazy;
+
+use sgl_model_gateway::tokenizer::traits::Tokenizer;
+use sgl_model_gateway::tokenizer::stream::DecodeStream;
+use sgl_model_gateway::tool_parser::ToolParser;
+use sgl_model_gateway::protocols::common::{Tool, ToolChoice, ToolChoiceValue, ToolCallDelta, FunctionCallDelta, Usage, StringOrArray};
+use sgl_model_gateway::tokenizer::stop::StopSequenceDecoder;
+use sgl_model_gateway::grpc_client::sglang_proto as proto;
+
+use super::error::{SglErrorCode, set_error_message, clear_error_message};
+use super::tokenizer::TokenizerHandle;
+use super::utils::generate_tool_call_id;
+
+/// Global parser factory (initialized once)
+// Use the re-exported ParserFactory from tool_parser module
+static PARSER_FACTORY: Lazy<sgl_model_gateway::tool_parser::ParserFactory> = Lazy::new(|| {
+    // ParserFactory is re-exported from tool_parser::factory, so we can use it directly
+    sgl_model_gateway::tool_parser::ParserFactory::default()
+});
+
+/// Global tokio runtime for async operations
+static RUNTIME: Lazy<Runtime> = Lazy::new(|| {
+    Runtime::new().expect("Failed to create tokio runtime for gRPC converter FFI")
+});
+
+/// Handle for gRPC response converter (maintains state for streaming)
+#[repr(C)]
+pub struct GrpcResponseConverterHandle {
+    pub(crate) tokenizer: Arc<dyn Tokenizer>,
+    pub(crate) tool_parser: Option<Arc<tokio::sync::Mutex<Box<dyn ToolParser>>>>,
+    pub(crate) stop_decoder: Option<Arc<tokio::sync::Mutex<StopSequenceDecoder>>>,
+    pub(crate) model: String,
+    pub(crate) request_id: String,
+    pub(crate) created: u64,
+    pub(crate) system_fingerprint: Option<String>,
+    pub(crate) tools: Option<Vec<Tool>>,
+    pub(crate) tool_choice: Option<ToolChoice>,
+    pub(crate) history_tool_calls_count: usize,
+    pub(crate) stream_buffers: HashMap<u32, String>, // Per-index text buffers
+    pub(crate) decode_streams: HashMap<u32, DecodeStream>, // Per-index incremental decoders
+    pub(crate) has_tool_calls: HashMap<u32, bool>, // Track if tool calls were emitted
+    pub(crate) is_first_chunk: HashMap<u32, bool>, // Track first chunk per index
+    pub(crate) prompt_tokens: HashMap<u32, i32>, // Track prompt tokens per index (from chunks)
+    pub(crate) completion_tokens: HashMap<u32, i32>, // Track completion tokens per index (cumulative)
+    pub(crate) initial_prompt_tokens: Option<i32>, // Initial prompt tokens from request (if available)
+    pub(crate) skip_special_tokens: bool, // Whether to skip special tokens when decoding
+}
+
+/// Create a gRPC response converter handle
+///
+/// # Arguments
+/// * `tokenizer_handle` - Tokenizer handle (must be valid)
+/// * `model` - Model name
+/// * `request_id` - Request ID
+/// * `tools_json` - Optional JSON array of tools
+/// * `tool_choice_json` - Optional JSON object for tool_choice
+/// * `stop` - Optional stop sequences (JSON array)
+/// * `stop_token_ids` - Optional stop token IDs (JSON array)
+/// * `skip_special_tokens` - Whether to skip special tokens
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * Pointer to GrpcResponseConverterHandle on success, null on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_grpc_response_converter_create(
+    tokenizer_handle: *mut TokenizerHandle,
+    model: *const c_char,
+    request_id: *const c_char,
+    tools_json: *const c_char,
+    tool_choice_json: *const c_char,
+    stop: *const c_char,
+    stop_token_ids: *const c_char,
+    skip_special_tokens: c_int,
+    error_out: *mut *mut c_char,
+) -> *mut GrpcResponseConverterHandle {
+    if tokenizer_handle.is_null() || model.is_null() || request_id.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return ptr::null_mut();
+    }
+
+    let model_str = match CStr::from_ptr(model).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in model");
+            return ptr::null_mut();
+        }
+    };
+
+    let request_id_str = match CStr::from_ptr(request_id).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in request_id");
+            return ptr::null_mut();
+        }
+    };
+
+    let handle_ref = &*tokenizer_handle;
+    let tokenizer = Arc::clone(&handle_ref.tokenizer);
+
+    // Parse tools if provided
+    let tools: Option<Vec<Tool>> = if !tools_json.is_null() {
+        match CStr::from_ptr(tools_json).to_str() {
+            Ok(s) => serde_json::from_str::<Vec<Tool>>(s).ok(),
+            Err(_) => None,
+        }
+    } else {
+        None
+    };
+
+    // Parse tool_choice if provided
+    let tool_choice: Option<ToolChoice> = if !tool_choice_json.is_null() {
+        match CStr::from_ptr(tool_choice_json).to_str() {
+            Ok(s) => serde_json::from_str::<ToolChoice>(s).ok(),
+            Err(_) => None,
+        }
+    } else {
+        None
+    };
+
+    // Parse stop sequences
+    let stop: Option<StringOrArray> = if !stop.is_null() {
+        let stop_str = match CStr::from_ptr(stop).to_str() {
+            Ok(s) => s,
+            Err(_) => return ptr::null_mut(),
+        };
+        serde_json::from_str::<StringOrArray>(stop_str).ok()
+    } else {
+        None
+    };
+
+    // Parse stop token IDs
+    let stop_token_ids: Option<Vec<u32>> = if !stop_token_ids.is_null() {
+        let ids_str = match CStr::from_ptr(stop_token_ids).to_str() {
+            Ok(s) => s,
+            Err(_) => return ptr::null_mut(),
+        };
+        serde_json::from_str::<Vec<u32>>(ids_str).ok()
+    } else {
+        None
+    };
+
+    // Create stop decoder if needed
+    let stop_decoder = if stop.is_some() || stop_token_ids.is_some() {
+        Some(Arc::new(tokio::sync::Mutex::new(
+            sgl_model_gateway::routers::grpc::utils::create_stop_decoder(
+                &tokenizer,
+                stop.as_ref(),
+                stop_token_ids.as_ref(),
+                skip_special_tokens != 0,
+                false, // no_stop_trim
+            ),
+        )))
+    } else {
+        None
+    };
+
+    // Create tool parser if tools are provided
+    let tool_parser = if tools.is_some() {
+        PARSER_FACTORY.registry().create_for_model(model_str)
+            .map(|p| Arc::new(tokio::sync::Mutex::new(p)))
+    } else {
+        None
+    };
+
+    // Get system fingerprint from model (simplified)
+    let system_fingerprint = Some("fp_placeholder".to_string()); // TODO: Get actual fingerprint
+
+    Box::into_raw(Box::new(GrpcResponseConverterHandle {
+        tokenizer,
+        tool_parser,
+        stop_decoder,
+        model: model_str.to_string(),
+        request_id: request_id_str.to_string(),
+        created: std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_secs(),
+        system_fingerprint,
+        tools,
+        tool_choice,
+        history_tool_calls_count: 0,
+        stream_buffers: HashMap::new(),
+        decode_streams: HashMap::new(),
+        has_tool_calls: HashMap::new(),
+        is_first_chunk: HashMap::new(),
+        prompt_tokens: HashMap::new(),
+        completion_tokens: HashMap::new(),
+        initial_prompt_tokens: None, // Will be set from stream handle
+        skip_special_tokens: skip_special_tokens != 0,
+    }))
+}
+
+/// Convert a gRPC GenerateResponse chunk to OpenAI format
+///
+/// # Arguments
+/// * `handle` - Converter handle
+/// * `response_json` - JSON string of proto.GenerateResponse
+/// * `result_json_out` - Pointer to receive OpenAI format JSON (must be freed with sgl_free_string)
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_grpc_response_converter_convert_chunk(
+    handle: *mut GrpcResponseConverterHandle,
+    response_json: *const c_char,
+    result_json_out: *mut *mut c_char,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if handle.is_null() || response_json.is_null() || result_json_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let response_str = match CStr::from_ptr(response_json).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in response_json");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Parse proto.GenerateResponse from JSON
+    let json_value: Value = match serde_json::from_str(response_str) {
+        Ok(v) => v,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to parse response JSON: {}", e));
+            return SglErrorCode::ParsingError;
+        }
+    };
+
+    // Build proto::GenerateResponse from JSON value
+    let mut proto_response = proto::GenerateResponse {
+        request_id: json_value.get("request_id")
+            .and_then(|v| v.as_str())
+            .unwrap_or("")
+            .to_string(),
+        response: None,
+    };
+
+    // Parse the response oneof field
+    if let Some(chunk_json) = json_value.get("chunk") {
+        let chunk = proto::GenerateStreamChunk {
+            token_ids: chunk_json.get("token_ids")
+                .and_then(|v| v.as_array())
+                .map(|arr| arr.iter().filter_map(|v| v.as_u64().map(|n| n as u32)).collect())
+                .unwrap_or_default(),
+            prompt_tokens: chunk_json.get("prompt_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            completion_tokens: chunk_json.get("completion_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            cached_tokens: chunk_json.get("cached_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            output_logprobs: None,
+            hidden_states: vec![],
+            input_logprobs: None,
+            index: 0,
+        };
+        proto_response.response = Some(proto::generate_response::Response::Chunk(chunk));
+    } else if let Some(complete_json) = json_value.get("complete") {
+        let complete = proto::GenerateComplete {
+            output_ids: complete_json.get("output_ids")
+                .and_then(|v| v.as_array())
+                .map(|arr| arr.iter().filter_map(|v| v.as_u64().map(|n| n as u32)).collect())
+                .unwrap_or_default(),
+            finish_reason: complete_json.get("finish_reason")
+                .and_then(|v| v.as_str())
+                .unwrap_or("")
+                .to_string(),
+            prompt_tokens: complete_json.get("prompt_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            completion_tokens: complete_json.get("completion_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            cached_tokens: complete_json.get("cached_tokens")
+                .and_then(|v| v.as_i64())
+                .map(|n| n as i32)
+                .unwrap_or(0),
+            output_logprobs: None,
+            all_hidden_states: vec![],
+            input_logprobs: None,
+            matched_stop: None,
+            index: 0,
+        };
+        proto_response.response = Some(proto::generate_response::Response::Complete(complete));
+    } else if let Some(error_json) = json_value.get("error") {
+        let error = proto::GenerateError {
+            message: error_json.get("message")
+                .and_then(|v| v.as_str())
+                .unwrap_or("")
+                .to_string(),
+            http_status_code: error_json.get("http_status_code")
+                .and_then(|v| v.as_str())
+                .unwrap_or("500")
+                .to_string(),
+            details: error_json.get("details")
+                .and_then(|v| v.as_str())
+                .unwrap_or("")
+                .to_string(),
+        };
+        proto_response.response = Some(proto::generate_response::Response::Error(error));
+    } else {
+        set_error_message(error_out, "Response JSON must contain 'chunk', 'complete', or 'error' field");
+        return SglErrorCode::ParsingError;
+    }
+
+    let handle_ref = &mut *handle;
+    let tokenizer = Arc::clone(&handle_ref.tokenizer);
+    let model = handle_ref.model.clone();
+    let request_id = handle_ref.request_id.clone();
+    let created = handle_ref.created;
+    let system_fingerprint = handle_ref.system_fingerprint.clone();
+
+    // Use tokio runtime to run async code
+    let result = RUNTIME.block_on(async {
+        convert_proto_chunk_to_openai(
+            proto_response,
+            handle_ref,
+            &tokenizer,
+            &model,
+            &request_id,
+            created,
+            system_fingerprint.as_deref(),
+        )
+        .await
+    });
+
+    match result {
+        Ok(Some(openai_response)) => {
+            // Serialize to JSON
+            let result_str = match serde_json::to_string(&openai_response) {
+                Ok(s) => s,
+                Err(e) => {
+                    set_error_message(error_out, &format!("Failed to serialize response: {}", e));
+                    return SglErrorCode::ParsingError;
+                }
+            };
+
+            let result_cstr = match CString::new(result_str) {
+                Ok(s) => s,
+                Err(e) => {
+                    set_error_message(error_out, &format!("Failed to create result string: {}", e));
+                    return SglErrorCode::MemoryError;
+                }
+            };
+
+            *result_json_out = result_cstr.into_raw();
+            clear_error_message(error_out);
+            SglErrorCode::Success
+        }
+        Ok(None) => {
+            // No response to send (e.g., empty chunk)
+            let empty = CString::new("").unwrap();
+            *result_json_out = empty.into_raw();
+            clear_error_message(error_out);
+            SglErrorCode::Success
+        }
+        Err(e) => {
+            set_error_message(error_out, &format!("Conversion error: {}", e));
+            SglErrorCode::ParsingError
+        }
+    }
+}
+
+/// Helper function to convert proto chunk to OpenAI format
+pub(crate) async fn convert_proto_chunk_to_openai(
+    proto_response: proto::GenerateResponse,
+    handle: &mut GrpcResponseConverterHandle,
+    tokenizer: &Arc<dyn Tokenizer>,
+    model: &str,
+    request_id: &str,
+    created: u64,
+    system_fingerprint: Option<&str>,
+) -> Result<Option<sgl_model_gateway::protocols::chat::ChatCompletionStreamResponse>, String> {
+    use sgl_model_gateway::grpc_client::sglang_proto::generate_response::Response::*;
+    use sgl_model_gateway::protocols::chat::{ChatCompletionStreamResponse, ChatMessageDelta, ChatStreamChoice};
+
+    match proto_response.response {
+        Some(Chunk(chunk)) => {
+            let index = chunk.index;
+
+            // Mark as not first chunk if we've seen this index before
+            let is_first = handle.is_first_chunk.entry(index).or_insert(true);
+            let first_chunk = *is_first;
+            *is_first = false;
+
+            // Track token counts from chunks (cumulative values from proto)
+            // These are cumulative values, so we always use the latest value
+            // For prompt_tokens, if chunk value is 0, preserve existing value or use initial_prompt_tokens
+            // This prevents overwriting valid prompt_tokens with 0
+            if chunk.prompt_tokens > 0 {
+                handle.prompt_tokens.insert(index, chunk.prompt_tokens);
+            } else {
+                // If chunk.prompt_tokens is 0, try to preserve existing value or use initial_prompt_tokens
+                if !handle.prompt_tokens.contains_key(&index) {
+                    // No existing value, try to use initial_prompt_tokens
+                    if let Some(initial_prompt) = handle.initial_prompt_tokens {
+                        handle.prompt_tokens.insert(index, initial_prompt);
+                    }
+                }
+                // If existing value exists, keep it (don't overwrite with 0)
+            }
+            // For completion_tokens, always update (even if 0) as it's cumulative
+            handle.completion_tokens.insert(index, chunk.completion_tokens);
+
+            // Process tokens through stop decoder if available, otherwise use incremental decoder
+            let chunk_text = if let Some(ref stop_decoder) = handle.stop_decoder {
+                let mut decoder_guard = stop_decoder.lock().await;
+                let mut text = String::new();
+                for &token_id in &chunk.token_ids {
+                    match decoder_guard.process_token(token_id).unwrap_or_else(|_| {
+                        sgl_model_gateway::tokenizer::stop::SequenceDecoderOutput::Held
+                    }) {
+                        sgl_model_gateway::tokenizer::stop::SequenceDecoderOutput::Text(t) => {
+                            text.push_str(&t);
+                        }
+                        sgl_model_gateway::tokenizer::stop::SequenceDecoderOutput::StoppedWithText(t) => {
+                            text.push_str(&t);
+                            break;
+                        }
+                        sgl_model_gateway::tokenizer::stop::SequenceDecoderOutput::Stopped => {
+                            break;
+                        }
+                        sgl_model_gateway::tokenizer::stop::SequenceDecoderOutput::Held => {}
+                    }
+                }
+                text
+            } else {
+                // Use incremental decoder to handle multi-byte character boundaries
+                let decode_stream = handle.decode_streams.entry(index).or_insert_with(|| {
+                    DecodeStream::new(
+                        Arc::clone(&tokenizer),
+                        &[], // No prompt tokens for completion
+                        handle.skip_special_tokens,
+                    )
+                });
+
+                // Process tokens incrementally
+                let mut text_parts = Vec::new();
+                for &token_id in &chunk.token_ids {
+                    if let Ok(Some(text)) = decode_stream.step(token_id) {
+                        text_parts.push(text);
+                    }
+                }
+                text_parts.join("")
+            };
+
+            if chunk_text.is_empty() {
+                return Ok(None);
+            }
+
+            // Send first chunk with role
+            if first_chunk {
+                let first_response = ChatCompletionStreamResponse {
+                    id: request_id.to_string(),
+                    object: "chat.completion.chunk".to_string(),
+                    created,
+                    model: model.to_string(),
+                    system_fingerprint: system_fingerprint.map(|s| s.to_string()),
+                    choices: vec![ChatStreamChoice {
+                        index,
+                        delta: ChatMessageDelta {
+                            role: Some("assistant".to_string()),
+                            content: None,
+                            tool_calls: None,
+                            reasoning_content: None,
+                        },
+                        logprobs: None,
+                        finish_reason: None,
+                        matched_stop: None,
+                    }],
+                    usage: None,
+                };
+                return Ok(Some(first_response));
+            }
+
+            // Update stream buffer
+            let stream_buffer = handle.stream_buffers.entry(index).or_default();
+            stream_buffer.push_str(&chunk_text);
+
+            // Handle tool calls if tools are provided
+            if let (Some(ref tools), Some(ref tool_parser)) = (handle.tools.as_ref(), handle.tool_parser.as_ref()) {
+                let tool_choice_enabled = !matches!(
+                    handle.tool_choice,
+                    Some(ToolChoice::Value(ToolChoiceValue::None))
+                );
+
+                if tool_choice_enabled {
+                    let mut parser_guard = tool_parser.lock().await;
+                    match parser_guard.parse_incremental(&chunk_text, tools).await {
+                        Ok(streaming_result) => {
+                            if !streaming_result.calls.is_empty() {
+                                handle.has_tool_calls.insert(index, true);
+                                // Convert tool call items to OpenAI format
+                                let tool_call_deltas: Vec<_> = streaming_result
+                                    .calls
+                                    .into_iter()
+                                    .map(|item| {
+                                        let id = if let Some(ref name) = item.name {
+                                            generate_tool_call_id(
+                                                model,
+                                                name,
+                                                item.tool_index,
+                                                handle.history_tool_calls_count,
+                                            )
+                                        } else {
+                                            format!("call_{}", item.tool_index)
+                                        };
+
+                                        ToolCallDelta {
+                                            index: item.tool_index as u32,
+                                            id: Some(id),
+                                            tool_type: if item.name.is_some() {
+                                                Some("function".to_string())
+                                            } else {
+                                                None
+                                            },
+                                            function: Some(FunctionCallDelta {
+                                                name: item.name,
+                                                arguments: if !item.parameters.is_empty() {
+                                                    Some(item.parameters)
+                                                } else {
+                                                    None
+                                                },
+                                            }),
+                                        }
+                                    })
+                                    .collect();
+
+                                let tool_response = ChatCompletionStreamResponse {
+                                    id: request_id.to_string(),
+                                    object: "chat.completion.chunk".to_string(),
+                                    created,
+                                    model: model.to_string(),
+                                    system_fingerprint: system_fingerprint.map(|s| s.to_string()),
+                                    choices: vec![ChatStreamChoice {
+                                        index,
+                                        delta: ChatMessageDelta {
+                                            role: Some("assistant".to_string()),
+                                            content: None,
+                                            tool_calls: Some(tool_call_deltas),
+                                            reasoning_content: None,
+                                        },
+                                        logprobs: None,
+                                        finish_reason: None,
+                                        matched_stop: None,
+                                    }],
+                                    usage: None,
+                                };
+                                return Ok(Some(tool_response));
+                            }
+                        }
+                        Err(e) => {
+                            // Log error but continue with regular content
+                            tracing::warn!("Tool parser error: {}", e);
+                        }
+                    }
+                }
+            }
+
+            // Regular content emission
+            let content_response = ChatCompletionStreamResponse {
+                id: request_id.to_string(),
+                object: "chat.completion.chunk".to_string(),
+                created,
+                model: model.to_string(),
+                system_fingerprint: system_fingerprint.map(|s| s.to_string()),
+                choices: vec![ChatStreamChoice {
+                    index,
+                    delta: ChatMessageDelta {
+                        role: Some("assistant".to_string()),
+                        content: Some(chunk_text),
+                        tool_calls: None,
+                        reasoning_content: None,
+                    },
+                    logprobs: None,
+                    finish_reason: None,
+                    matched_stop: None,
+                }],
+                usage: None,
+            };
+
+            Ok(Some(content_response))
+        }
+        Some(Complete(complete)) => {
+            let index = complete.index;
+
+            // Flush any remaining text
+            // Flush any remaining text from decode stream
+            let mut final_text = handle.stream_buffers.remove(&index).unwrap_or_default();
+            if let Some(ref mut decode_stream) = handle.decode_streams.get_mut(&index) {
+                if let Ok(Some(remaining)) = decode_stream.flush() {
+                    final_text.push_str(&remaining);
+                }
+            }
+            handle.decode_streams.remove(&index);
+
+            // Determine finish reason - ensure it's never empty
+            // If finish_reason is empty, try to infer from other fields or use default
+            let finish_reason = if handle.has_tool_calls.get(&index).copied().unwrap_or(false)
+                && (complete.finish_reason == "stop" || complete.finish_reason.is_empty())
+            {
+                "tool_calls".to_string()
+            } else if complete.finish_reason.is_empty() || complete.finish_reason.trim().is_empty() {
+                // If finish_reason is empty, try to infer from completion_tokens or use default
+                if complete.completion_tokens > 0 {
+                    // If we have completion tokens, likely stopped normally
+                    "stop".to_string()
+                } else if !complete.output_ids.is_empty() {
+                    // If we have output_ids, likely stopped normally
+                    "stop".to_string()
+                } else {
+                    // Default fallback - always ensure we have a value
+                    "stop".to_string()
+                }
+            } else {
+                complete.finish_reason.clone()
+            };
+
+            // Ensure finish_reason is never empty (defensive check)
+            let finish_reason = if finish_reason.is_empty() || finish_reason.trim().is_empty() {
+                "stop".to_string()
+            } else {
+                finish_reason
+            };
+
+            // Extract matched_stop
+            let matched_stop = match &complete.matched_stop {
+                Some(proto::generate_complete::MatchedStop::MatchedTokenId(token_id)) => {
+                    Some(Value::Number(serde_json::Number::from(*token_id)))
+                }
+                Some(proto::generate_complete::MatchedStop::MatchedStopStr(stop_str)) => {
+                    Some(Value::String(stop_str.clone()))
+                }
+                None => None,
+            };
+
+            // Build usage - prefer values from complete message, but fallback to accumulated values from chunks
+            // Complete message should have the final values, but sometimes they might be 0 or missing
+            // Always use the latest cumulative value from chunks if available, otherwise use complete message value
+            let mut prompt_tokens = handle.prompt_tokens.get(&index)
+                .copied()
+                .filter(|&v| v > 0)
+                .unwrap_or(complete.prompt_tokens);
+            let mut completion_tokens = handle.completion_tokens.get(&index)
+                .copied()
+                .filter(|&v| v > 0)
+                .unwrap_or(complete.completion_tokens);
+
+            // Always try to use initial_prompt_tokens if prompt_tokens is 0 or missing
+            // This is the most reliable source for prompt tokens since we calculate it from the request
+            if prompt_tokens == 0 {
+                if let Some(initial_prompt) = handle.initial_prompt_tokens {
+                    prompt_tokens = initial_prompt;
+                }
+            }
+
+            // If completion_tokens is 0, try to infer from output_ids or accumulated chunks
+            if completion_tokens == 0 {
+                // Try to use completion_tokens from complete message even if 0
+                // Or calculate from output_ids
+                if complete.completion_tokens > 0 {
+                    completion_tokens = complete.completion_tokens;
+                } else if !complete.output_ids.is_empty() {
+                    completion_tokens = complete.output_ids.len() as i32;
+                } else if let Some(&last_completion) = handle.completion_tokens.get(&index) {
+                    completion_tokens = last_completion;
+                }
+            }
+
+            // Final fallback: if both are still 0, try to use initial_prompt_tokens for prompt
+            // and calculate completion from output_ids
+            if prompt_tokens == 0 && completion_tokens == 0 {
+                // Try to infer from output_ids if available
+                let output_ids_len = complete.output_ids.len() as i32;
+                if output_ids_len > 0 {
+                    completion_tokens = output_ids_len;
+                    // Always try to use initial_prompt_tokens for prompt
+                    if let Some(initial_prompt) = handle.initial_prompt_tokens {
+                        prompt_tokens = initial_prompt;
+                    }
+                }
+            }
+
+            // Final defensive check: ensure prompt_tokens is set if we have initial_prompt_tokens
+            if prompt_tokens == 0 {
+                if let Some(initial_prompt) = handle.initial_prompt_tokens {
+                    prompt_tokens = initial_prompt;
+                }
+            }
+
+            // Always create usage, even if values are 0 (defensive)
+            let usage = Some(Usage {
+                prompt_tokens: prompt_tokens.max(0) as u32,
+                completion_tokens: completion_tokens.max(0) as u32,
+                total_tokens: (prompt_tokens.max(0) + completion_tokens.max(0)) as u32,
+                completion_tokens_details: None,
+            });
+
+            let finish_response = ChatCompletionStreamResponse {
+                id: request_id.to_string(),
+                object: "chat.completion.chunk".to_string(),
+                created,
+                model: model.to_string(),
+                system_fingerprint: system_fingerprint.map(|s| s.to_string()),
+                choices: vec![ChatStreamChoice {
+                    index,
+                    delta: ChatMessageDelta {
+                        role: Some("assistant".to_string()),
+                        content: if !final_text.is_empty() {
+                            Some(final_text)
+                        } else {
+                            None
+                        },
+                        tool_calls: None,
+                        reasoning_content: None,
+                    },
+                    logprobs: None,
+                    finish_reason: Some(finish_reason),
+                    matched_stop,
+                }],
+                usage,
+            };
+
+            Ok(Some(finish_response))
+        }
+        Some(Error(error)) => {
+            Err(format!("Server error: {} (status: {})", error.message, error.http_status_code))
+        }
+        None => Ok(None),
+    }
+}
+
+/// Free a gRPC response converter handle
+#[no_mangle]
+pub unsafe extern "C" fn sgl_grpc_response_converter_free(handle: *mut GrpcResponseConverterHandle) {
+    if !handle.is_null() {
+        let _ = Box::from_raw(handle);
+    }
+}

sgl-model-gateway/bindings/golang/src/lib.rs

@@ -0,0 +1,88 @@
+//! FFI module for exposing sgl-model-gateway preprocessing and postprocessing functions
+//! to C-compatible languages (e.g., Golang via cgo)
+//!
+//! This module provides C-compatible function signatures for:
+//! - Tokenizer operations (encode, decode, chat template)
+//! - Tool parser operations (parse tool calls)
+//! - Tool constraint generation
+//! - gRPC client SDK (complete request-response flow)
+//!
+//! # Safety
+//! All functions marked with `#[no_mangle]` and `extern "C"` must be called
+//! with valid pointers and follow the documented memory management rules.
+
+// Re-export error types
+pub use error::{SglErrorCode, set_error_message, set_error_message_fmt, clear_error_message};
+
+// Re-export memory management functions
+pub use memory::{sgl_free_string, sgl_free_token_ids};
+
+// Re-export tokenizer functions
+pub use tokenizer::{
+    TokenizerHandle,
+    sgl_tokenizer_create_from_file,
+    sgl_tokenizer_encode,
+    sgl_tokenizer_apply_chat_template,
+    sgl_tokenizer_apply_chat_template_with_tools,
+    sgl_tokenizer_decode,
+    sgl_tokenizer_free,
+};
+
+// Re-export tool parser functions
+pub use tool_parser::{
+    ToolParserHandle,
+    sgl_tool_parser_create,
+    sgl_tool_parser_parse_complete,
+    sgl_tool_parser_parse_incremental,
+    sgl_tool_parser_reset,
+    sgl_tool_parser_free,
+};
+
+// Re-export gRPC converter functions
+pub use grpc_converter::{
+    GrpcResponseConverterHandle,
+    sgl_grpc_response_converter_create,
+    sgl_grpc_response_converter_convert_chunk,
+    sgl_grpc_response_converter_free,
+};
+
+// Re-export client SDK functions
+pub use client::{
+    SglangClientHandle,
+    sgl_client_create,
+    sgl_client_free,
+};
+
+// Re-export stream functions
+pub use stream::{
+    SglangStreamHandle,
+    sgl_stream_read_next,
+    sgl_stream_free,
+};
+
+// Re-export client stream function (defined in client.rs but used by stream)
+pub use client::sgl_client_chat_completion_stream;
+
+// Re-export utility functions
+pub use utils::sgl_generate_tool_constraints;
+
+// Sub-modules
+mod error;
+mod memory;
+mod tokenizer;
+mod tool_parser;
+mod grpc_converter;
+mod client;
+mod stream;
+mod utils;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_error_codes() {
+        assert_eq!(SglErrorCode::Success as i32, 0);
+        assert_eq!(SglErrorCode::InvalidArgument as i32, 1);
+    }
+}

sgl-model-gateway/bindings/golang/src/memory.rs

@@ -0,0 +1,28 @@
+//! Memory management for FFI functions
+
+use std::ffi::CString;
+use std::os::raw::c_char;
+
+/// Free a C string allocated by Rust
+///
+/// # Safety
+/// This function must only be called with pointers returned by other FFI functions.
+/// Calling with arbitrary pointers or multiple times on the same pointer is undefined behavior.
+#[no_mangle]
+pub unsafe extern "C" fn sgl_free_string(s: *mut c_char) {
+    if !s.is_null() {
+        let _ = CString::from_raw(s);
+    }
+}
+
+/// Free token IDs array allocated by Rust
+///
+/// # Safety
+/// This function must only be called with pointers returned by `sgl_tokenizer_encode`.
+/// The `count` parameter must match the length of the array.
+#[no_mangle]
+pub unsafe extern "C" fn sgl_free_token_ids(ptr: *mut u32, count: usize) {
+    if !ptr.is_null() && count > 0 {
+        let _ = Vec::from_raw_parts(ptr, count, count);
+    }
+}

sgl-model-gateway/bindings/golang/src/stream.rs

@@ -0,0 +1,288 @@
+//! Stream handling FFI functions
+//!
+//! This module provides FFI (Foreign Function Interface) functions for managing
+//! streaming responses from the SGLang gRPC API. It handles:
+//!
+//! - Creating and managing stream handles
+//! - Reading chunks from streams and converting them to OpenAI format
+//! - Managing automatic abort on stream drop (via AbortOnDropStream)
+//! - Thread-safe access to streams and response converters
+//!
+//! # Safety
+//!
+//! All FFI functions are marked `unsafe` as per Rust FFI conventions. Callers must:
+//! - Pass valid pointers
+//! - Ensure proper pointer lifetime management
+//! - Call corresponding free functions for cleanup
+
+use std::ffi::CString;
+use std::os::raw::{c_char, c_int};
+use std::ptr;
+use std::sync::Arc;
+use tokio::runtime::Runtime;
+use once_cell::sync::Lazy;
+use futures_util::StreamExt;
+
+use sgl_model_gateway::grpc_client::{sglang_proto as proto, sglang_scheduler::{SglangSchedulerClient, AbortOnDropStream}};
+
+use super::error::{SglErrorCode, set_error_message};
+use super::grpc_converter::{GrpcResponseConverterHandle, convert_proto_chunk_to_openai};
+
+/// Global tokio runtime for async operations
+static RUNTIME: Lazy<Runtime> = Lazy::new(|| {
+    Runtime::new().expect("Failed to create tokio runtime for stream FFI")
+});
+
+/// Handle for an active streaming request.
+///
+/// This struct manages the stream and response converter for a single request.
+/// It is wrapped in Arc and Mutex for thread-safe concurrent access.
+///
+/// # Fields
+///
+/// * `stream` - The gRPC stream wrapped in AbortOnDropStream for automatic cleanup
+/// * `converter` - Response converter that transforms proto messages to OpenAI format
+/// * `client` - The underlying gRPC client connection
+/// * `prompt_tokens` - Number of prompt tokens from the original request
+pub struct SglangStreamHandle {
+    pub(crate) stream: Arc<tokio::sync::Mutex<AbortOnDropStream>>,
+    pub(crate) converter: Arc<tokio::sync::Mutex<GrpcResponseConverterHandle>>,
+    #[allow(dead_code)]
+    pub(crate) client: Arc<SglangSchedulerClient>,
+    #[allow(dead_code)]
+    pub(crate) prompt_tokens: i32, // Number of prompt tokens for this request
+}
+
+/// Read next chunk from stream and convert to OpenAI format.
+///
+/// This function reads the next chunk from the gRPC stream, converts it from the
+/// internal protocol format to OpenAI-compatible JSON format, and returns it via
+/// the output parameters.
+///
+/// # Arguments
+///
+/// * `stream_handle` - Mutable pointer to the stream handle
+/// * `response_json_out` - Pointer to receive OpenAI format JSON string
+///   - Caller must free this with `sgl_free_string`
+///   - May be NULL if no data available
+/// * `is_done_out` - Pointer to receive completion status
+///   - 0 = stream has more data
+///   - 1 = stream is complete
+/// * `error_out` - Optional pointer to receive error message
+///   - Only set if function returns an error code
+///   - Must be freed with `sgl_free_string` if not NULL
+///
+/// # Returns
+///
+/// * `SglErrorCode::Success` - Successfully read a chunk or reached end of stream
+/// * Other error codes - See `SglErrorCode` for details
+///
+/// # Safety
+///
+/// - All pointers must be valid and properly aligned
+/// - `stream_handle` must point to a valid `SglangStreamHandle`
+/// - Output pointers must be writable
+///
+/// # Notes
+///
+/// - Complete messages are identified by the presence of `proto::GenerateResponse::Complete`
+/// - When is_done=1, this may be the last readable chunk or the stream may be ending
+/// - Subsequent calls after is_done=1 will mark the stream as complete internally
+#[no_mangle]
+pub unsafe extern "C" fn sgl_stream_read_next(
+    stream_handle: *mut SglangStreamHandle,
+    response_json_out: *mut *mut c_char,
+    is_done_out: *mut c_int,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if stream_handle.is_null() || response_json_out.is_null() || is_done_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let handle_ref = &*stream_handle;
+    let stream = Arc::clone(&handle_ref.stream);
+    let converter = Arc::clone(&handle_ref.converter);
+
+    // Read next chunk from stream
+    let chunk_result = RUNTIME.block_on(async {
+        let mut stream_guard = stream.lock().await;
+        stream_guard.next().await
+    });
+
+    match chunk_result {
+        Some(Ok(proto_response)) => {
+            // Convert proto response to OpenAI format
+            // We need to get the converter lock first
+            let conversion_result = RUNTIME.block_on(async {
+                let mut converter_guard = converter.lock().await;
+
+                // Clone necessary fields for conversion
+                let tokenizer = Arc::clone(&converter_guard.tokenizer);
+                let model = converter_guard.model.clone();
+                let request_id = converter_guard.request_id.clone();
+                let created = converter_guard.created;
+                let system_fingerprint = converter_guard.system_fingerprint.clone();
+
+                // Call the conversion function
+                convert_proto_chunk_to_openai(
+                    proto_response.clone(),
+                    &mut *converter_guard,
+                    &tokenizer,
+                    &model,
+                    &request_id,
+                    created,
+                    system_fingerprint.as_deref(),
+                )
+                .await
+            });
+
+            match conversion_result {
+                Ok(Some(openai_response)) => {
+                    // Serialize to JSON
+                    let result_str = match serde_json::to_string(&openai_response) {
+                        Ok(s) => s,
+                        Err(e) => {
+                            set_error_message(error_out, &format!("Failed to serialize response: {}", e));
+                            return SglErrorCode::ParsingError;
+                        }
+                    };
+
+                    let result_cstr = match CString::new(result_str) {
+                        Ok(s) => s,
+                        Err(e) => {
+                            set_error_message(error_out, &format!("Failed to create result string: {}", e));
+                            return SglErrorCode::MemoryError;
+                        }
+                    };
+
+                    // Check if this is a complete response (stream done)
+                    let is_complete = matches!(proto_response.response, Some(proto::generate_response::Response::Complete(_)) | Some(proto::generate_response::Response::Error(_)));
+
+                    *response_json_out = result_cstr.into_raw();
+                    *is_done_out = if is_complete { 1 } else { 0 };
+
+                    if is_complete {
+                        // Mark stream as completed
+                        // Ensure mark_completed() completes and is visible before returning
+                        // Use yield_now to ensure Release ordering is fully propagated
+                        RUNTIME.block_on(async {
+                            let stream_guard = stream.lock().await;
+                            stream_guard.mark_completed();
+                            // Keep the guard until mark_completed() is fully executed
+                            drop(stream_guard);
+                            // Yield to ensure Release ordering is propagated before returning
+                            // This prevents race condition where Free() is called immediately
+                            // and Drop might not see the mark_completed() write
+                            tokio::task::yield_now().await;
+                        });
+                    }
+
+                    SglErrorCode::Success
+                }
+                Ok(None) => {
+                    // No response to send (e.g., empty chunk)
+                    // Don't mark as completed - stream might continue
+                    // Just return null and let caller read more
+                    *response_json_out = ptr::null_mut();
+                    *is_done_out = 0; // Keep stream open, not done yet
+                    SglErrorCode::Success
+                }
+                Err(e) => {
+                    // Conversion error - don't mark as completed
+                    // Let the stream end naturally or return error without stopping stream
+                    set_error_message(error_out, &format!("Conversion error: {}", e));
+                    *response_json_out = ptr::null_mut();
+                    *is_done_out = 0; // Don't mark as done - let caller decide
+                    SglErrorCode::ParsingError
+                }
+            }
+        }
+        Some(Err(e)) => {
+            // Stream error - mark as completed to prevent abort
+            RUNTIME.block_on(async {
+                let stream_guard = stream.lock().await;
+                stream_guard.mark_completed();
+                drop(stream_guard);
+                // Yield to ensure Release ordering is propagated
+                tokio::task::yield_now().await;
+            });
+
+            set_error_message(error_out, &format!("Stream error: {}", e));
+            *is_done_out = 1;
+            SglErrorCode::UnknownError
+        }
+        None => {
+            // Stream ended naturally (no more chunks)
+            // Mark stream as completed before returning to prevent abort
+            RUNTIME.block_on(async {
+                let stream_guard = stream.lock().await;
+                stream_guard.mark_completed();
+                drop(stream_guard);
+                // Yield to ensure Release ordering is propagated
+                tokio::task::yield_now().await;
+            });
+
+            *response_json_out = ptr::null_mut();
+            *is_done_out = 1;
+            SglErrorCode::Success
+        }
+    }
+}
+
+/// Free a stream handle and release all associated resources.
+///
+/// This function must be called exactly once for each stream handle returned by
+/// `sgl_client_chat_completion_stream`. It marks the stream as completed internally
+/// to prevent abort signals from being sent when resources are cleaned up.
+///
+/// # Arguments
+///
+/// * `handle` - Mutable pointer to the stream handle to free
+///   - If NULL, this function does nothing
+///
+/// # Safety
+///
+/// - Must be called only once per handle
+/// - Handle must not be used after calling this function
+/// - After this call, the stream is no longer valid
+///
+/// # Notes
+///
+/// - This function internally calls `mark_completed()` before freeing to ensure
+///   the stream cleanup doesn't trigger an abort RPC to the server
+/// - Memory fences are used to ensure visibility across threads
+#[no_mangle]
+pub unsafe extern "C" fn sgl_stream_free(handle: *mut SglangStreamHandle) {
+    if !handle.is_null() {
+        let handle_ref = Box::from_raw(handle);
+
+        // Mark stream as completed to prevent abort on drop
+        // By this point, the stream should already be completed by ReadNext()
+        // but we call it again to be safe
+        RUNTIME.block_on(async {
+            let stream_guard = handle_ref.stream.lock().await;
+            stream_guard.mark_completed();
+            // Keep guard alive to ensure mark_completed() write completes
+            drop(stream_guard);
+            // Yield to ensure the atomic write is visible
+            tokio::task::yield_now().await;
+        });
+
+        // Use a strong memory fence to ensure mark_completed()'s Release write
+        // is visible before we drop the last Arc reference
+        std::sync::atomic::fence(std::sync::atomic::Ordering::SeqCst);
+
+        // Now drop all references - if mark_completed() was called successfully,
+        // the drop won't send an abort
+        drop(handle_ref.stream);
+
+        // Free converter
+        let converter = Arc::try_unwrap(handle_ref.converter)
+            .ok()
+            .map(|m| m.into_inner());
+        if let Some(conv) = converter {
+            super::grpc_converter::sgl_grpc_response_converter_free(Box::into_raw(Box::new(conv)));
+        }
+    }
+}

sgl-model-gateway/bindings/golang/src/tokenizer.rs

@@ -0,0 +1,379 @@
+//! Tokenizer FFI functions
+
+use std::ffi::{CStr, CString};
+use std::os::raw::{c_char, c_int};
+use std::ptr;
+use std::sync::Arc;
+use serde_json::Value;
+
+use sgl_model_gateway::tokenizer::{
+    create_tokenizer_from_file,
+    traits::Tokenizer as TokenizerTrait,
+    chat_template::ChatTemplateParams,
+    huggingface::HuggingFaceTokenizer,
+};
+
+use super::error::{SglErrorCode, set_error_message, clear_error_message};
+
+/// Opaque handle for a tokenizer instance
+#[repr(C)]
+pub struct TokenizerHandle {
+    pub(crate) tokenizer: Arc<dyn TokenizerTrait>,
+}
+
+/// Create a tokenizer from a file path
+///
+/// # Arguments
+/// * `path` - Path to tokenizer.json file (null-terminated C string)
+/// * `error_out` - Optional pointer to receive error message (must be freed with sgl_free_string)
+///
+/// # Returns
+/// * Pointer to TokenizerHandle on success, null on failure
+///
+/// # Safety
+/// The returned handle must be freed with `sgl_tokenizer_free`.
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tokenizer_create_from_file(
+    path: *const c_char,
+    error_out: *mut *mut c_char,
+) -> *mut TokenizerHandle {
+    if path.is_null() {
+        set_error_message(error_out, "path cannot be null");
+        return ptr::null_mut();
+    }
+
+    let path_str = match CStr::from_ptr(path).to_str() {
+        Ok(s) => s,
+        Err(e) => {
+            set_error_message(error_out, &format!("Invalid UTF-8 in path: {}", e));
+            return ptr::null_mut();
+        }
+    };
+
+    match create_tokenizer_from_file(path_str) {
+        Ok(tokenizer) => {
+            clear_error_message(error_out);
+            Box::into_raw(Box::new(TokenizerHandle {
+                tokenizer,
+            }))
+        }
+        Err(e) => {
+            set_error_message(error_out, &e.to_string());
+            ptr::null_mut()
+        }
+    }
+}
+
+/// Encode text to token IDs
+///
+/// # Arguments
+/// * `handle` - Tokenizer handle (must not be null)
+/// * `text` - Input text (null-terminated C string)
+/// * `token_ids_out` - Pointer to receive array of token IDs (must be freed with sgl_free_token_ids)
+/// * `token_count_out` - Pointer to receive token count
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+///
+/// # Safety
+/// The token_ids_out array must be freed with sgl_free_token_ids() after use.
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tokenizer_encode(
+    handle: *mut TokenizerHandle,
+    text: *const c_char,
+    token_ids_out: *mut *mut u32,
+    token_count_out: *mut usize,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if handle.is_null() || text.is_null() || token_ids_out.is_null() || token_count_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let text_str = match CStr::from_ptr(text).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in text");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    let tokenizer = &(*handle).tokenizer;
+    match tokenizer.encode(text_str) {
+        Ok(encoding) => {
+            let token_ids = encoding.token_ids();
+            let count = token_ids.len();
+
+            // Allocate memory for token IDs using Vec, then leak to give ownership to C
+            let vec = token_ids.to_vec();
+            let ptr = vec.as_ptr() as *mut u32;
+            let _ = std::mem::ManuallyDrop::new(vec);
+
+            *token_ids_out = ptr;
+            *token_count_out = count;
+            clear_error_message(error_out);
+            SglErrorCode::Success
+        }
+        Err(e) => {
+            set_error_message(error_out, &e.to_string());
+            SglErrorCode::TokenizationError
+        }
+    }
+}
+
+/// Apply chat template to messages with tools support
+///
+/// # Arguments
+/// * `handle` - Tokenizer handle
+/// * `messages_json` - JSON string of messages array
+/// * `tools_json` - Optional JSON string of tools array (null or empty string for no tools)
+/// * `result_out` - Pointer to receive result string (must be freed with sgl_free_string)
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tokenizer_apply_chat_template_with_tools(
+    handle: *mut TokenizerHandle,
+    messages_json: *const c_char,
+    tools_json: *const c_char,
+    result_out: *mut *mut c_char,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if handle.is_null() || messages_json.is_null() || result_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let messages_str = match CStr::from_ptr(messages_json).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in messages_json");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Parse JSON messages
+    let messages: Vec<Value> = match serde_json::from_str(messages_str) {
+        Ok(msgs) => msgs,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to parse messages JSON: {}", e));
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Parse tools JSON if provided
+    let tools: Option<Vec<Value>> = if tools_json.is_null() {
+        None
+    } else {
+        let tools_str = match CStr::from_ptr(tools_json).to_str() {
+            Ok(s) => {
+                if s.is_empty() {
+                    None
+                } else {
+                    match serde_json::from_str::<Vec<Value>>(s) {
+                        Ok(t) => Some(t),
+                        Err(e) => {
+                            set_error_message(error_out, &format!("Failed to parse tools JSON: {}", e));
+                            return SglErrorCode::InvalidArgument;
+                        }
+                    }
+                }
+            }
+            Err(_) => {
+                set_error_message(error_out, "Invalid UTF-8 in tools_json");
+                return SglErrorCode::InvalidArgument;
+            }
+        };
+        tools_str
+    };
+
+    // Get the tokenizer from handle
+    let handle_ref = &*handle;
+    let tokenizer = &handle_ref.tokenizer;
+
+    // Try to downcast to HuggingFaceTokenizer
+    if let Some(hf_tokenizer) = tokenizer.as_any().downcast_ref::<HuggingFaceTokenizer>() {
+        // Apply chat template with tools
+        let empty_docs: [Value; 0] = [];
+        let tools_slice = tools.as_ref().map(|t| t.as_slice());
+        let params = ChatTemplateParams {
+            add_generation_prompt: true,
+            tools: tools_slice,
+            documents: Some(&empty_docs),
+            template_kwargs: None,
+        };
+
+        match hf_tokenizer.apply_chat_template(&messages, params) {
+            Ok(result) => {
+                let result_cstr = match CString::new(result) {
+                    Ok(s) => s,
+                    Err(e) => {
+                        set_error_message(error_out, &format!("Failed to create result string: {}", e));
+                        return SglErrorCode::MemoryError;
+                    }
+                };
+                *result_out = result_cstr.into_raw();
+                clear_error_message(error_out);
+                SglErrorCode::Success
+            }
+            Err(e) => {
+                set_error_message(error_out, &format!("Failed to apply chat template: {}", e));
+                SglErrorCode::TokenizationError
+            }
+        }
+    } else {
+        set_error_message(error_out, "Chat template is only supported for HuggingFace tokenizers");
+        SglErrorCode::TokenizationError
+    }
+}
+
+/// Apply chat template to messages
+///
+/// # Arguments
+/// * `handle` - Tokenizer handle
+/// * `messages_json` - JSON string of messages array
+/// * `result_out` - Pointer to receive result string (must be freed with sgl_free_string)
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tokenizer_apply_chat_template(
+    handle: *mut TokenizerHandle,
+    messages_json: *const c_char,
+    result_out: *mut *mut c_char,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if handle.is_null() || messages_json.is_null() || result_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let messages_str = match CStr::from_ptr(messages_json).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in messages_json");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Parse JSON messages
+    let messages: Vec<Value> = match serde_json::from_str(messages_str) {
+        Ok(msgs) => msgs,
+        Err(e) => {
+            set_error_message(error_out, &format!("Failed to parse messages JSON: {}", e));
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Get the tokenizer from handle
+    let handle_ref = &*handle;
+    let tokenizer = &handle_ref.tokenizer;
+
+    // Try to downcast to HuggingFaceTokenizer
+    if let Some(hf_tokenizer) = tokenizer.as_any().downcast_ref::<HuggingFaceTokenizer>() {
+        // Apply chat template with default parameters
+        // Use empty arrays instead of None to avoid template errors
+        // Set add_generation_prompt to true so the model knows to start generating
+        let empty_tools: [Value; 0] = [];
+        let empty_docs: [Value; 0] = [];
+        let params = ChatTemplateParams {
+            add_generation_prompt: true,  // Important: tells the model to start generating
+            tools: Some(&empty_tools),
+            documents: Some(&empty_docs),
+            template_kwargs: None,
+        };
+
+        match hf_tokenizer.apply_chat_template(&messages, params) {
+            Ok(result) => {
+                let result_cstr = match CString::new(result) {
+                    Ok(s) => s,
+                    Err(e) => {
+                        set_error_message(error_out, &format!("Failed to create result string: {}", e));
+                        return SglErrorCode::MemoryError;
+                    }
+                };
+                *result_out = result_cstr.into_raw();
+                clear_error_message(error_out);
+                SglErrorCode::Success
+            }
+            Err(e) => {
+                set_error_message(error_out, &format!("Failed to apply chat template: {}", e));
+                SglErrorCode::TokenizationError
+            }
+        }
+    } else {
+        set_error_message(error_out, "Chat template is only supported for HuggingFace tokenizers");
+        SglErrorCode::TokenizationError
+    }
+}
+
+/// Decode token IDs to text
+///
+/// # Arguments
+/// * `handle` - Tokenizer handle
+/// * `token_ids` - Array of token IDs
+/// * `token_count` - Number of tokens
+/// * `skip_special_tokens` - Whether to skip special tokens
+/// * `result_out` - Pointer to receive result string (must be freed with sgl_free_string)
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tokenizer_decode(
+    handle: *mut TokenizerHandle,
+    token_ids: *const u32,
+    token_count: usize,
+    skip_special_tokens: c_int,
+    result_out: *mut *mut c_char,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if handle.is_null() || token_ids.is_null() || result_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    if token_count == 0 {
+        let empty = CString::new("").unwrap();
+        *result_out = empty.into_raw();
+        clear_error_message(error_out);
+        return SglErrorCode::Success;
+    }
+
+    // Convert C array to Rust slice
+    let token_slice = std::slice::from_raw_parts(token_ids, token_count);
+
+    let tokenizer = &(*handle).tokenizer;
+    match tokenizer.decode(token_slice, skip_special_tokens != 0) {
+        Ok(text) => {
+            let result_cstr = match CString::new(text) {
+                Ok(s) => s,
+                Err(e) => {
+                    set_error_message(error_out, &format!("Failed to create result string: {}", e));
+                    return SglErrorCode::MemoryError;
+                }
+            };
+            *result_out = result_cstr.into_raw();
+            clear_error_message(error_out);
+            SglErrorCode::Success
+        }
+        Err(e) => {
+            set_error_message(error_out, &e.to_string());
+            SglErrorCode::TokenizationError
+        }
+    }
+}
+
+/// Free a tokenizer handle
+///
+/// # Safety
+/// This function must only be called once per handle, and the handle must not be used after calling.
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tokenizer_free(handle: *mut TokenizerHandle) {
+    if !handle.is_null() {
+        let _ = Box::from_raw(handle);
+    }
+}

sgl-model-gateway/bindings/golang/src/tool_parser.rs

@@ -0,0 +1,329 @@
+//! Tool parser FFI functions
+
+use std::ffi::{CStr, CString};
+use std::os::raw::{c_char};
+use std::ptr;
+use std::sync::Arc;
+use std::collections::HashMap;
+use serde_json::{json, Value};
+use tokio::runtime::Runtime;
+use once_cell::sync::Lazy;
+
+use sgl_model_gateway::tool_parser::{ParserFactory, ToolParser};
+use sgl_model_gateway::protocols::common::Tool;
+
+use super::error::{SglErrorCode, set_error_message, clear_error_message};
+use super::utils::generate_tool_call_id;
+
+/// Global parser factory (initialized once)
+static PARSER_FACTORY: Lazy<ParserFactory> = Lazy::new(|| ParserFactory::new());
+
+/// Global tokio runtime for async operations
+static RUNTIME: Lazy<Runtime> = Lazy::new(|| {
+    Runtime::new().expect("Failed to create tokio runtime for tool parser FFI")
+});
+
+/// Opaque handle for a tool parser instance
+/// Note: For streaming, we need mutable access, so we use Arc<Mutex<>> internally
+/// Note: This is an opaque handle, C code doesn't access fields directly
+pub struct ToolParserHandle {
+    parser: Arc<tokio::sync::Mutex<Box<dyn ToolParser>>>,
+    model: String, // Store model name for ID generation
+    history_tool_calls_count: usize, // Track tool call count for ID generation
+    tool_index_to_id: HashMap<usize, String>, // Map tool_index to ID for incremental updates
+}
+
+/// Create a tool parser
+///
+/// # Arguments
+/// * `parser_type` - Parser type name (e.g., "json", "llama", "mistral") or model name (e.g., "gpt-4")
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * Pointer to ToolParserHandle on success, null on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tool_parser_create(
+    parser_type: *const c_char,
+    error_out: *mut *mut c_char,
+) -> *mut ToolParserHandle {
+    if parser_type.is_null() {
+        set_error_message(error_out, "parser_type cannot be null");
+        return ptr::null_mut();
+    }
+
+    let type_str = match CStr::from_ptr(parser_type).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in parser_type");
+            return ptr::null_mut();
+        }
+    };
+
+    // Create parser using factory
+    // The factory will determine the parser type based on model name or use the provided type
+    let parser = if let Some(parser_box) = PARSER_FACTORY.registry().create_for_model(type_str) {
+        parser_box
+    } else if let Some(parser_box) = PARSER_FACTORY.registry().create_parser(type_str) {
+        parser_box
+    } else {
+        set_error_message(error_out, &format!("Unknown parser type: {}", type_str));
+        return ptr::null_mut();
+    };
+
+    Box::into_raw(Box::new(ToolParserHandle {
+        parser: Arc::new(tokio::sync::Mutex::new(parser)),
+        model: type_str.to_string(),
+        history_tool_calls_count: 0,
+        tool_index_to_id: HashMap::new(),
+    }))
+}
+
+/// Parse complete tool calls from text
+///
+/// # Arguments
+/// * `handle` - Tool parser handle
+/// * `text` - Input text to parse
+/// * `result_json_out` - Pointer to receive JSON result (must be freed with sgl_free_string)
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tool_parser_parse_complete(
+    handle: *mut ToolParserHandle,
+    text: *const c_char,
+    result_json_out: *mut *mut c_char,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if handle.is_null() || text.is_null() || result_json_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let text_str = match CStr::from_ptr(text).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in text");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    let handle_ref = &*handle;
+    let parser = Arc::clone(&handle_ref.parser);
+    let model = handle_ref.model.clone();
+    let history_count = handle_ref.history_tool_calls_count;
+
+    // Use tokio runtime to run async code
+    let result = RUNTIME.block_on(async {
+        let parser_guard = parser.lock().await;
+        parser_guard.parse_complete(text_str).await
+    });
+
+    match result {
+        Ok((normal_text, tool_calls)) => {
+            // Convert Rust ToolCall to OpenAI format
+            let openai_tool_calls: Vec<Value> = tool_calls
+                .into_iter()
+                .enumerate()
+                .map(|(index, tc)| {
+                    // Generate ID for this tool call
+                    let id = generate_tool_call_id(&model, &tc.function.name, index, history_count);
+                    json!({
+                        "id": id,
+                        "type": "function",
+                        "function": {
+                            "name": tc.function.name,
+                            "arguments": tc.function.arguments
+                        }
+                    })
+                })
+                .collect();
+
+            // Build result JSON
+            let result_json = json!({
+                "normal_text": normal_text,
+                "tool_calls": openai_tool_calls
+            });
+
+            let result_str = match serde_json::to_string(&result_json) {
+                Ok(s) => s,
+                Err(e) => {
+                    set_error_message(error_out, &format!("Failed to serialize JSON: {}", e));
+                    return SglErrorCode::ParsingError;
+                }
+            };
+
+            let result_cstr = match CString::new(result_str) {
+                Ok(s) => s,
+                Err(e) => {
+                    set_error_message(error_out, &format!("Failed to create result string: {}", e));
+                    return SglErrorCode::MemoryError;
+                }
+            };
+
+            *result_json_out = result_cstr.into_raw();
+            clear_error_message(error_out);
+            SglErrorCode::Success
+        }
+        Err(e) => {
+            set_error_message(error_out, &format!("Parse error: {}", e));
+            SglErrorCode::ParsingError
+        }
+    }
+}
+
+/// Parse tool calls incrementally from streaming chunks
+///
+/// # Arguments
+/// * `handle` - Tool parser handle
+/// * `chunk` - New text chunk from stream
+/// * `tools_json` - JSON array of available tools (for validation, can be null/empty)
+/// * `result_json_out` - Pointer to receive JSON result (must be freed with sgl_free_string)
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tool_parser_parse_incremental(
+    handle: *mut ToolParserHandle,
+    chunk: *const c_char,
+    tools_json: *const c_char,
+    result_json_out: *mut *mut c_char,
+    error_out: *mut *mut c_char,
+) -> SglErrorCode {
+    if handle.is_null() || chunk.is_null() || result_json_out.is_null() {
+        set_error_message(error_out, "Invalid arguments: null pointer");
+        return SglErrorCode::InvalidArgument;
+    }
+
+    let chunk_str = match CStr::from_ptr(chunk).to_str() {
+        Ok(s) => s,
+        Err(_) => {
+            set_error_message(error_out, "Invalid UTF-8 in chunk");
+            return SglErrorCode::InvalidArgument;
+        }
+    };
+
+    // Parse tools JSON if provided
+    let tools: Vec<Tool> = if !tools_json.is_null() {
+        let tools_str = match CStr::from_ptr(tools_json).to_str() {
+            Ok(s) => s,
+            Err(_) => {
+                set_error_message(error_out, "Invalid UTF-8 in tools_json");
+                return SglErrorCode::InvalidArgument;
+            }
+        };
+        match serde_json::from_str::<Vec<Tool>>(tools_str) {
+            Ok(t) => t,
+            Err(_) => vec![], // If parsing fails, use empty tools
+        }
+    } else {
+        vec![]
+    };
+
+    let handle_ref = &*handle;
+    let parser = Arc::clone(&handle_ref.parser);
+    let model = handle_ref.model.clone();
+    let history_count = handle_ref.history_tool_calls_count;
+
+    // Use tokio runtime to run async code
+    let result = RUNTIME.block_on(async {
+        let mut parser_guard = parser.lock().await;
+        parser_guard.parse_incremental(chunk_str, &tools).await
+    });
+
+    match result {
+        Ok(streaming_result) => {
+            // Convert StreamingParseResult to OpenAI format
+            let handle_mut = &mut *handle;
+            let openai_tool_calls: Vec<Value> = streaming_result
+                .calls
+                .into_iter()
+                .map(|item| {
+                    // For incremental parsing, we may not have complete tool calls yet
+                    // Generate or reuse ID based on tool_index
+                    let id = if let Some(ref name) = item.name {
+                        // New tool call with name - generate ID and store it
+                        let id = generate_tool_call_id(&model, name, item.tool_index, history_count);
+                        handle_mut.tool_index_to_id.insert(item.tool_index, id.clone());
+                        id
+                    } else {
+                        // Parameter update - reuse existing ID for this tool_index
+                        handle_mut.tool_index_to_id
+                            .get(&item.tool_index)
+                            .cloned()
+                            .unwrap_or_else(|| format!("call_{}", item.tool_index))
+                    };
+
+                    json!({
+                        "id": id,
+                        "type": "function",
+                        "function": {
+                            "name": item.name.unwrap_or_default(),
+                            "arguments": item.parameters
+                        }
+                    })
+                })
+                .collect();
+
+            // Build result JSON
+            let result_json = json!({
+                "normal_text": streaming_result.normal_text,
+                "tool_calls": openai_tool_calls
+            });
+
+            let result_str = match serde_json::to_string(&result_json) {
+                Ok(s) => s,
+                Err(e) => {
+                    set_error_message(error_out, &format!("Failed to serialize JSON: {}", e));
+                    return SglErrorCode::ParsingError;
+                }
+            };
+
+            let result_cstr = match CString::new(result_str) {
+                Ok(s) => s,
+                Err(e) => {
+                    set_error_message(error_out, &format!("Failed to create result string: {}", e));
+                    return SglErrorCode::MemoryError;
+                }
+            };
+
+            *result_json_out = result_cstr.into_raw();
+            clear_error_message(error_out);
+            SglErrorCode::Success
+        }
+        Err(e) => {
+            set_error_message(error_out, &format!("Parse incremental error: {}", e));
+            SglErrorCode::ParsingError
+        }
+    }
+}
+
+/// Reset the parser state for reuse
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tool_parser_reset(handle: *mut ToolParserHandle) {
+    if handle.is_null() {
+        return;
+    }
+
+    let handle_ref = &mut *handle;
+    let parser = Arc::clone(&handle_ref.parser);
+
+    // Reset parser state
+    RUNTIME.block_on(async {
+        let mut parser_guard = parser.lock().await;
+        parser_guard.reset();
+    });
+
+    // Reset history count and tool index mapping
+    handle_ref.history_tool_calls_count = 0;
+    handle_ref.tool_index_to_id.clear();
+}
+
+/// Free a tool parser handle
+#[no_mangle]
+pub unsafe extern "C" fn sgl_tool_parser_free(handle: *mut ToolParserHandle) {
+    if !handle.is_null() {
+        let _ = Box::from_raw(handle);
+    }
+}

sgl-model-gateway/bindings/golang/src/utils.rs

@@ -0,0 +1,44 @@
+//! Utility functions for FFI
+
+use uuid::Uuid;
+
+/// Helper function to generate tool call ID (matches router implementation)
+pub fn generate_tool_call_id(
+    model: &str,
+    function_name: &str,
+    index: usize,
+    history_tool_calls_count: usize,
+) -> String {
+    if model.to_lowercase().contains("kimi") {
+        // KimiK2 format: functions.{name}:{global_index}
+        format!("functions.{}:{}", function_name, history_tool_calls_count + index)
+    } else {
+        // Standard OpenAI format: call_{24-char-uuid}
+        format!("call_{}", &Uuid::new_v4().simple().to_string()[..24])
+    }
+}
+
+/// Generate tool constraints (placeholder implementation)
+///
+/// # Arguments
+/// * `tools_json` - JSON array of tools
+/// * `tool_choice_json` - JSON object representing tool_choice
+/// * `constraint_type_out` - Pointer to receive constraint type (e.g., "json_schema")
+/// * `constraint_schema_out` - Pointer to receive constraint schema JSON
+/// * `error_out` - Optional pointer to receive error message
+///
+/// # Returns
+/// * SglErrorCode::Success on success, error code on failure
+#[no_mangle]
+pub unsafe extern "C" fn sgl_generate_tool_constraints(
+    _tools_json: *const std::os::raw::c_char,
+    _tool_choice_json: *const std::os::raw::c_char,
+    _constraint_type_out: *mut *mut std::os::raw::c_char,
+    _constraint_schema_out: *mut *mut std::os::raw::c_char,
+    error_out: *mut *mut std::os::raw::c_char,
+) -> super::error::SglErrorCode {
+    // Implementation would parse JSON and call generate_tool_constraints
+    // This is a placeholder
+    super::error::set_error_message(error_out, "Tool constraint generation not yet implemented in FFI");
+    super::error::SglErrorCode::UnknownError
+}

sgl-model-gateway/bindings/python/.coveragerc

@@ -0,0 +1,9 @@
+[run]
+source = sglang_router
+omit =
+    */mini_lb.py
+    */cli.py
+    */__main__.py
+
+[report]
+fail_under = 80

sgl-model-gateway/bindings/python/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "sgl-model-gateway-python"
+version = "0.2.3"
+edition = "2021"
+
+[lib]
+name = "sglang_router_rs"
+crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = { version = "0.27.1", features = ["extension-module", "abi3-py38"] }
+tokio = { version = "1.42.0", features = ["full"] }
+
+[dependencies.sgl-model-gateway]
+path = "../.."
+default-features = true
+
+[features]
+default = ["pyo3/extension-module"]
+vendored-openssl = ["sgl-model-gateway/vendored-openssl"]
+
+[profile.ci]
+inherits = "release"
+opt-level = 2       # Lighter optimization (still fast runtime, much faster compile)
+lto = "thin"        # Thin LTO - good balance
+codegen-units = 16  # More parallelization for faster builds
+strip = true

sgl-model-gateway/bindings/python/MANIFEST.in

@@ -0,0 +1,9 @@
+# Must include:
+include Cargo.toml              # Python bindings Cargo configuration
+include ../../Cargo.toml        # Main Rust project configuration
+include ../../build.rs          # Build script for protobuf generation
+include ../../LICENSE
+recursive-include src *.rs      # Python bindings wrapper
+recursive-include ../../src *.rs  # Main Rust source files
+recursive-include ../../src/proto *.proto  # Protobuf definitions
+recursive-include sglang_router *.py  # Python source files

sgl-model-gateway/bindings/python/README.md

@@ -0,0 +1,71 @@
+# SGLang Model Gateway Python Bindings
+
+This directory contains the Python bindings for the SGLang Router, built using [maturin](https://github.com/PyO3/maturin) and [PyO3](https://github.com/PyO3/pyo3).
+
+## Directory Structure
+
+```
+bindings/python/
+├── src/                    # Rust source code for Python bindings
+│   └── lib.rs              # PyO3 bindings implementation
+├── sglang_router/          # Python source code
+│   ├── __init__.py
+│   ├── version.py
+│   ├── launch_server.py
+│   ├── launch_router.py
+│   ├── router.py
+│   ├── router_args.py
+│   └── mini_lb.py
+├── Cargo.toml              # Rust package configuration for bindings
+├── pyproject.toml          # Python package configuration
+├── setup.py                # Setup configuration
+├── MANIFEST.in             # Package manifest
+├── .coveragerc             # Test coverage configuration
+└── README.md               # This file
+```
+
+## Building
+
+### Development Build
+
+```bash
+# Install maturin
+pip install maturin
+
+# Build and install in development mode
+cd sgl-model-gateway/bindings/python
+maturin develop --features vendored-openssl
+```
+
+### Production Build
+
+```bash
+# Build wheel
+cd sgl-model-gateway/bindings/python
+maturin build --release --out dist --features vendored-openssl
+
+# Install the built wheel
+pip install dist/sglang_router-*.whl
+```
+
+## Testing
+
+```bash
+# Run Python tests
+cd sgl-model-gateway
+pytest py_test/
+```
+
+## Configuration
+
+- **pyproject.toml**: Defines package metadata, dependencies, and build configuration
+- **python-source**: Set to "." to indicate Python source is in the same directory as pyproject.toml
+- **module-name**: `sglang_router.sglang_router_rs` - the Rust extension module name
+
+## Notes
+
+- The Rust bindings source code is located in `src/lib.rs`
+- The bindings have their own `Cargo.toml` in this directory
+- The main sglang-router library is located in `../../` and is used as a dependency
+- The package includes both Python code and Rust extensions built with PyO3
+- PyO3 types are prefixed with `Py` in Rust but exposed to Python without the prefix using the `name` attribute

sgl-model-gateway/bindings/python/pyproject.toml

@@ -0,0 +1,54 @@
+[build-system]
+requires = ["maturin>=1.0,<2.0"]
+build-backend = "maturin"
+
+[project]
+name = "sglang-router"
+version = "0.2.3"
+description = "High-performance Rust-based load balancer for SGLang with multiple routing algorithms and prefill-decode disaggregation support"
+authors = [
+    {name = "Simo Lin", email = "linsimo.mark@gmail.com"},
+    {name = "Chang Su", email = "mckvtl@gmail.com"},
+    {name = "Keyang Ru", email = "rukeyang@gmail.com"},
+    {name = "Byron Hsu", email = "byronhsu1230@gmail.com"}
+]
+requires-python = ">=3.8"
+readme = "../../README.md"
+license = { text = "Apache-2.0" }
+classifiers = [
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+]
+
+dependencies = [
+    "setproctitle",
+    "aiohttp",
+    "orjson",
+    "uvicorn",
+    "fastapi",
+]
+
+[project.optional-dependencies]
+dev = [
+    "requests>=2.25.0",
+]
+
+[project.scripts]
+smg = "sglang_router.cli:main"
+amg = "sglang_router.cli:main"
+sglang-router = "sglang_router.cli:main"
+
+
+[tool.maturin]
+python-source = "."
+module-name = "sglang_router.sglang_router_rs"
+# Exclude bindings/python/README.md to use root README only
+exclude = ["README.md"]

sgl-model-gateway/bindings/python/setup.py

@@ -0,0 +1,28 @@
+import os
+import warnings
+
+from setuptools import setup
+
+with_rust = os.environ.get("SGLANG_ROUTER_BUILD_WITH_RUST", None)
+with_rust = with_rust is None or (not with_rust.lower() in ["0", "false", "no"])
+
+rust_extensions = []
+if with_rust:
+    from setuptools_rust import Binding, RustExtension
+
+    rust_extensions.append(
+        RustExtension(
+            target="sglang_router_rs",
+            path="Cargo.toml",
+            binding=Binding.PyO3,
+        )
+    )
+else:
+    warnings.warn(
+        "Building 'sglang-router' without Rust support. Performance may be degraded."
+    )
+
+setup(
+    rust_extensions=rust_extensions,
+    zip_safe=False,
+)

sgl-model-gateway/bindings/python/sglang_router/__init__.py

@@ -0,0 +1,3 @@
+from sglang_router.version import __version__
+
+__all__ = ["__version__"]

sgl-model-gateway/bindings/python/sglang_router/__main__.py

@@ -0,0 +1,8 @@
+"""
+Allow running the CLI via: python -m sglang_router
+"""
+
+from sglang_router.cli import main
+
+if __name__ == "__main__":
+    main()

sgl-model-gateway/bindings/python/sglang_router/cli.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""
+SGLang Model Gateway CLI
+
+Provides convenient command-line interface for launching the router and server.
+
+Usage:
+    smg launch [args]          # Launch router only
+    smg server [args]          # Launch router + server
+    smg --help                 # Show help
+"""
+
+import argparse
+import os
+import sys
+from typing import List, Optional
+
+from sglang_router.sglang_router_rs import (
+    get_verbose_version_string,
+    get_version_string,
+)
+
+
+def create_parser() -> argparse.ArgumentParser:
+    """Create the main CLI parser with subcommands."""
+    prog_name = os.path.basename(sys.argv[0]) if sys.argv else "smg"
+    parser = argparse.ArgumentParser(
+        prog=prog_name,
+        description="SGLang Model Gateway - High-performance inference router",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # Launch router subcommand
+    launch_parser = subparsers.add_parser(
+        "launch",
+        help="Launch router only (requires existing worker URLs)",
+        description="Launch the SGLang router with existing worker instances",
+        add_help=False,  # Let router handle --help
+    )
+
+    # Launch server + router subcommand
+    server_parser = subparsers.add_parser(
+        "server",
+        help="Launch router and server processes together",
+        description="Launch both SGLang router and server processes",
+        add_help=False,  # Let server handle --help
+    )
+
+    return parser
+
+
+def main(argv: Optional[List[str]] = None) -> None:
+    """Main CLI entry point."""
+    if argv is None:
+        argv = sys.argv[1:]
+
+    # Handle version flags before parsing
+    if argv and argv[0] in ["--version", "-V", "--version-verbose"]:
+        if argv[0] == "--version-verbose":
+            print(get_verbose_version_string())
+        else:
+            print(get_version_string())
+        sys.exit(0)
+
+    # Handle empty command - show help
+    if not argv or argv[0] not in ["launch", "server", "-h", "--help"]:
+        parser = create_parser()
+        parser.print_help()
+        sys.exit(1)
+
+    parser = create_parser()
+    args, unknown = parser.parse_known_args(argv)
+
+    if args.command == "launch":
+        # Import and call launch_router functions directly
+        from sglang_router.launch_router import launch_router, parse_router_args
+
+        # All router args are in unknown
+        router_args = parse_router_args(unknown)
+        launch_router(router_args)
+
+    elif args.command == "server":
+        # Import and call launch_server main with proper argv
+        # Note: launch_server.main() uses argparse internally which reads sys.argv
+        # We need to temporarily set sys.argv for compatibility
+        import sglang_router.launch_server as launch_server_module
+
+        # Preserve original sys.argv
+        original_argv = sys.argv
+        try:
+            # All server args are in unknown
+            prog_name = os.path.basename(sys.argv[0]) if sys.argv else "smg"
+            sys.argv = [f"{prog_name} server"] + unknown
+            launch_server_module.main()
+        finally:
+            # Restore original sys.argv
+            sys.argv = original_argv
+
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

sgl-model-gateway/bindings/python/sglang_router/launch_router.py

@@ -0,0 +1,113 @@
+import argparse
+import logging
+import sys
+from typing import List, Optional
+
+import setproctitle
+from sglang_router.mini_lb import MiniLoadBalancer
+from sglang_router.router_args import RouterArgs
+
+logger = logging.getLogger("router")
+
+try:
+    from sglang_router.router import Router
+except ImportError:
+    Router = None
+    logger.warning(
+        "Rust Router is not installed, only python MiniLB (debugging only) is available"
+    )
+
+
+def launch_router(args: argparse.Namespace) -> Optional[Router]:
+    """
+    Launch the SGLang router with the configuration from parsed arguments.
+
+    Args:
+        args: Namespace object containing router configuration
+            Can be either raw argparse.Namespace or converted RouterArgs
+
+    Returns:
+        Router instance if successful, None if failed
+    """
+    setproctitle.setproctitle("sglang::router")
+    try:
+        # Convert to RouterArgs if needed
+        if not isinstance(args, RouterArgs):
+            router_args = RouterArgs.from_cli_args(args)
+        else:
+            router_args = args
+
+        if router_args.mini_lb:
+            mini_lb = MiniLoadBalancer(router_args)
+            mini_lb.start()
+        else:
+            # TODO: support tracing for router(Rust).
+            del router_args.enable_trace
+            del router_args.otlp_traces_endpoint
+
+            if Router is None:
+                raise RuntimeError("Rust Router is not installed")
+            router_args._validate_router_args()
+            router = Router.from_args(router_args)
+            router.start()
+
+    except Exception as e:
+        logger.error(f"Error starting router: {e}")
+        raise e
+
+
+class CustomHelpFormatter(
+    argparse.RawDescriptionHelpFormatter, argparse.ArgumentDefaultsHelpFormatter
+):
+    """Custom formatter that preserves both description formatting and shows defaults"""
+
+    pass
+
+
+def parse_router_args(args: List[str]) -> RouterArgs:
+    """Parse command line arguments and return RouterArgs instance."""
+    parser = argparse.ArgumentParser(
+        description="""SGLang Router - High-performance request distribution across worker nodes
+
+Usage:
+This launcher enables starting a router with individual worker instances. It is useful for
+multi-node setups or when you want to start workers and router separately.
+
+Examples:
+  # Regular mode
+  python -m sglang_router.launch_router --worker-urls http://worker1:8000 http://worker2:8000
+
+  # PD disaggregated mode with same policy for both
+  python -m sglang_router.launch_router --pd-disaggregation \\
+    --prefill http://prefill1:8000 9000 --prefill http://prefill2:8000 \\
+    --decode http://decode1:8001 --decode http://decode2:8001 \\
+    --policy cache_aware
+
+  # PD mode with optional bootstrap ports
+  python -m sglang_router.launch_router --pd-disaggregation \\
+    --prefill http://prefill1:8000 9000 \\    # With bootstrap port
+    --prefill http://prefill2:8000 none \\    # Explicitly no bootstrap port
+    --prefill http://prefill3:8000 \\         # Defaults to no bootstrap port
+    --decode http://decode1:8001 --decode http://decode2:8001
+
+  # PD mode with different policies for prefill and decode
+  python -m sglang_router.launch_router --pd-disaggregation \\
+    --prefill http://prefill1:8000 --prefill http://prefill2:8000 \\
+    --decode http://decode1:8001 --decode http://decode2:8001 \\
+    --prefill-policy cache_aware --decode-policy power_of_two
+
+    """,
+        formatter_class=CustomHelpFormatter,
+    )
+
+    RouterArgs.add_cli_args(parser, use_router_prefix=False)
+    return RouterArgs.from_cli_args(parser.parse_args(args), use_router_prefix=False)
+
+
+def main() -> None:
+    router_args = parse_router_args(sys.argv[1:])
+    launch_router(router_args)
+
+
+if __name__ == "__main__":
+    main()

sgl-model-gateway/bindings/python/sglang_router/launch_server.py

@@ -0,0 +1,213 @@
+import argparse
+import asyncio
+import copy
+import logging
+import multiprocessing as mp
+import os
+import random
+import signal
+import sys
+import time
+from typing import List
+
+import requests
+from setproctitle import setproctitle
+from sglang_router.launch_router import RouterArgs, launch_router
+
+from sglang.srt.server_args import ServerArgs
+from sglang.srt.utils import is_port_available
+
+
+def setup_logger():
+    logger = logging.getLogger("router")
+    logger.setLevel(logging.INFO)
+
+    formatter = logging.Formatter(
+        "[Router (Python)] %(asctime)s - %(levelname)s - %(message)s - %(filename)s:%(lineno)d",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+    return logger
+
+
+logger = setup_logger()
+
+
+# Create new process group
+def run_server(server_args, dp_rank):
+    """
+    Note:
+
+    1. Without os.setpgrp(), all processes share the same PGID. When you press Ctrl+C, the terminal sends SIGINT to all processes in the group simultaneously.
+    This can cause leaf processes to terminate first, which messes up the cleaning order and produces orphaned processes.
+
+    Terminal (PGID=100)
+    └── Main Python Process (PGID=100)
+        └── Server Process 1 (PGID=100)
+            └── Scheduler 1
+            └── Detokenizer 1
+        └── Server Process 2 (PGID=100)
+            └── Scheduler 2
+            └── Detokenizer 2
+
+    2. With os.setpgrp(), the main Python process and its children are in a separate group. Now:
+
+    Terminal (PGID=100)
+    └── Main Python Process (PGID=200)
+        └── Server Process 1 (PGID=300)
+            └── Scheduler 1
+            └── Detokenizer 1
+        └── Server Process 2 (PGID=400)
+            └── Scheduler 2
+            └── Detokenizer 2
+    """
+    # create new process group
+    os.setpgrp()
+
+    setproctitle("sglang::server")
+    # Set SGLANG_DP_RANK environment variable
+    os.environ["SGLANG_DP_RANK"] = str(dp_rank)
+
+    # Launch server in appropriate mode (HTTP or gRPC)
+    if server_args.grpc_mode:
+        from sglang.srt.entrypoints.grpc_server import serve_grpc
+
+        asyncio.run(serve_grpc(server_args))
+    else:
+        from sglang.srt.entrypoints.http_server import launch_server
+
+        launch_server(server_args)
+
+
+def launch_server_process(
+    server_args: ServerArgs, worker_port: int, dp_id: int
+) -> mp.Process:
+    """Launch a single server process with the given args and port."""
+    server_args = copy.deepcopy(server_args)
+    server_args.port = worker_port
+    server_args.base_gpu_id = dp_id * server_args.tp_size
+    server_args.dp_size = 1
+
+    proc = mp.Process(target=run_server, args=(server_args, dp_id))
+    proc.start()
+    return proc
+
+
+def wait_for_server_health(host: str, port: int, timeout: int = 300) -> bool:
+    """Wait for server to be healthy by checking /health endpoint."""
+    start_time = time.perf_counter()
+    url = f"http://{host}:{port}/health"
+
+    while time.perf_counter() - start_time < timeout:
+        try:
+            response = requests.get(url, timeout=5)
+            if response.status_code == 200:
+                return True
+        except requests.exceptions.RequestException:
+            pass
+        time.sleep(1)
+    return False
+
+
+def find_available_ports(base_port: int, count: int) -> List[int]:
+    """Find consecutive available ports starting from base_port."""
+    available_ports = []
+    current_port = base_port
+
+    while len(available_ports) < count:
+        if is_port_available(current_port):
+            available_ports.append(current_port)
+        current_port += random.randint(100, 1000)
+
+    return available_ports
+
+
+def cleanup_processes(processes: List[mp.Process]):
+    for process in processes:
+        logger.info(f"Terminating process group {process.pid}")
+        try:
+            os.killpg(process.pid, signal.SIGTERM)
+        except ProcessLookupError:
+            # Process group may already be terminated
+            pass
+
+    # Wait for processes to terminate
+    for process in processes:
+        process.join(timeout=5)
+        if process.is_alive():
+            logger.warning(
+                f"Process {process.pid} did not terminate gracefully, forcing kill"
+            )
+            try:
+                os.killpg(process.pid, signal.SIGKILL)
+            except ProcessLookupError:
+                pass
+
+    logger.info("All process groups terminated")
+
+
+def main():
+    # CUDA runtime isn't fork-safe, which can lead to subtle bugs or crashes
+    mp.set_start_method("spawn")
+
+    parser = argparse.ArgumentParser(
+        description="Launch SGLang router and server processes"
+    )
+
+    ServerArgs.add_cli_args(parser)
+    RouterArgs.add_cli_args(parser, use_router_prefix=True, exclude_host_port=True)
+    parser.add_argument(
+        "--router-dp-worker-base-port",
+        type=int,
+        default=31000,
+        help="Base port number for data parallel workers",
+    )
+    # No extra retry/CB flags here; RouterArgs.add_cli_args already defines them with router- prefix
+
+    args = parser.parse_args()
+    server_args = ServerArgs.from_cli_args(args)
+    router_args = RouterArgs.from_cli_args(args, use_router_prefix=True)
+
+    # Find available ports for workers
+    worker_ports = find_available_ports(
+        args.router_dp_worker_base_port, server_args.dp_size
+    )
+
+    # Start server processes
+    server_processes = []
+
+    for i, worker_port in enumerate(worker_ports):
+        logger.info(f"Launching DP server process {i} on port {worker_port}")
+        proc = launch_server_process(server_args, worker_port, i)
+        server_processes.append(proc)
+
+    signal.signal(signal.SIGINT, lambda sig, frame: cleanup_processes(server_processes))
+    signal.signal(
+        signal.SIGTERM, lambda sig, frame: cleanup_processes(server_processes)
+    )
+    signal.signal(
+        signal.SIGQUIT, lambda sig, frame: cleanup_processes(server_processes)
+    )
+
+    # Update router args with worker URLs
+    # Use grpc:// protocol if server is in gRPC mode, otherwise http://
+    protocol = "grpc" if server_args.grpc_mode else "http"
+    router_args.worker_urls = [
+        f"{protocol}://{server_args.host}:{port}" for port in worker_ports
+    ]
+
+    # Start the router
+    try:
+        launch_router(router_args)
+    except Exception as e:
+        logger.error(f"Failed to start router: {e}")
+        cleanup_processes(server_processes)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

sgl-model-gateway/bindings/python/sglang_router/mini_lb.py

@@ -0,0 +1,488 @@
+"""
+Minimal HTTP load balancer for prefill and decode servers for testing.
+"""
+
+import asyncio
+import ipaddress
+import logging
+import random
+import urllib
+from http import HTTPStatus
+from itertools import chain
+from typing import Optional
+
+import aiohttp
+import orjson
+import uvicorn
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import ORJSONResponse, Response, StreamingResponse
+from sglang_router.router_args import RouterArgs
+
+try:
+    from sglang.srt.tracing.trace import (
+        process_tracing_init,
+        trace_get_remote_propagate_context,
+        trace_req_finish,
+        trace_req_start,
+        trace_set_thread_info,
+        trace_slice_end,
+        trace_slice_start,
+    )
+
+    trace_package_imported = True
+except ImportError:
+    trace_package_imported = False
+
+logger = logging.getLogger(__name__)
+
+AIOHTTP_STREAM_READ_CHUNK_SIZE = (
+    1024 * 64
+)  # 64KB, to prevent aiohttp's "Chunk too big" error
+
+
+def maybe_wrap_ipv6_address(address: str) -> str:
+    try:
+        ipaddress.IPv6Address(address)
+        return f"[{address}]"
+    except ValueError:
+        return address
+
+
+class MiniLoadBalancer:
+    def __init__(
+        self,
+        router_args: RouterArgs,
+    ):
+        self._validate_router_args(router_args)
+
+        self.host = router_args.host
+        self.port = router_args.port
+        self.timeout = router_args.request_timeout_secs
+        self.prefill_urls = [url[0] for url in router_args.prefill_urls]
+        self.prefill_bootstrap_ports = [url[1] for url in router_args.prefill_urls]
+        self.decode_urls = router_args.decode_urls
+        self.otlp_traces_endpoint = router_args.otlp_traces_endpoint
+        self.enable_trace = router_args.enable_trace
+        if self.enable_trace and not trace_package_imported:
+            logger.warning(
+                "Tracing is not supported in this environment. Please install sglang."
+            )
+            self.enable_trace = False
+
+    def _validate_router_args(self, router_args: RouterArgs):
+        logger.warning(
+            "\x1b[33mMiniLB is only for debugging purposes, it only supports random policy!\033[0m"
+        )
+
+        # NOTE: too many arguments unsupported, just validate some important ones
+        if router_args.policy != "random":
+            logger.warning("[MiniLB] Overriding policy to random")
+            router_args.policy = "random"
+
+        if not router_args.pd_disaggregation:
+            raise ValueError("MiniLB only supports PD disaggregation mode")
+
+        if len(router_args.prefill_urls) == 0 or len(router_args.decode_urls) == 0:
+            raise ValueError(
+                "MiniLB requires at least one prefill and one decode server"
+            )
+
+    def start(self):
+        global lb
+        lb = self
+        if self.enable_trace:
+            process_tracing_init(self.otlp_traces_endpoint, "sglang")
+            trace_set_thread_info("Mini lb")
+        uvicorn.run(app, host=self.host, port=self.port)
+
+    def select_pair(self):
+        assert len(self.prefill_urls) > 0, "No prefill servers available"
+        assert len(self.decode_urls) > 0, "No decode servers available"
+        pidx = random.randint(0, len(self.prefill_urls) - 1)
+        didx = random.randint(0, len(self.decode_urls) - 1)
+        return (
+            self.prefill_urls[pidx],
+            self.prefill_bootstrap_ports[pidx],
+            self.decode_urls[didx],
+        )
+
+    async def generate(
+        self, modified_request, prefill_server, decode_server, endpoint
+    ) -> ORJSONResponse:
+        assert endpoint[0] != "/", f"Endpoint should not start with '/': {endpoint}"
+
+        async with aiohttp.ClientSession(
+            timeout=aiohttp.ClientTimeout(
+                total=self.timeout
+            )  # Add timeout for request reliability
+        ) as session:
+            headers = {}
+            bootstrap_room_list = []
+            if self.enable_trace:
+                bootstrap_room_list = (
+                    modified_request["bootstrap_room"]
+                    if isinstance(modified_request["bootstrap_room"], list)
+                    else [modified_request["bootstrap_room"]]
+                )
+                trace_context = trace_get_remote_propagate_context(bootstrap_room_list)
+                headers = {"trace_context": trace_context}
+
+            tasks = [
+                session.post(
+                    f"{prefill_server}/{endpoint}",
+                    json=modified_request,
+                    headers=headers,
+                ),
+                session.post(
+                    f"{decode_server}/{endpoint}",
+                    json=modified_request,
+                    headers=headers,
+                ),
+            ]
+
+            for bootstrap_room in bootstrap_room_list:
+                trace_slice_end("mini_lb_launch", bootstrap_room, auto_next_anon=True)
+
+            # Wait for both responses to complete. Prefill should end first.
+            prefill_response, decode_response = await asyncio.gather(*tasks)
+
+            if "return_logprob" in modified_request:
+
+                prefill_json = await prefill_response.json()
+                ret_json = await decode_response.json()
+
+                # merge `meta_info.input_token_logprobs` from prefill to decode
+                if "meta_info" in ret_json:
+                    if "input_token_logprobs" in ret_json["meta_info"]:
+                        ret_json["meta_info"]["input_token_logprobs"] = (
+                            prefill_json["meta_info"]["input_token_logprobs"]
+                            + ret_json["meta_info"]["input_token_logprobs"]
+                        )
+            else:
+                ret_json = await decode_response.json()
+
+            for bootstrap_room in bootstrap_room_list:
+                trace_slice_end(
+                    "wait_PD_finish",
+                    bootstrap_room,
+                    thread_finish_flag=True,
+                )
+                trace_req_finish(bootstrap_room)
+
+            return ORJSONResponse(
+                content=ret_json,
+                status_code=decode_response.status,
+            )
+
+    async def generate_stream(
+        self, modified_request, prefill_server, decode_server, endpoint="generate"
+    ):
+        assert endpoint[0] != "/", f"Endpoint should not start with '/': {endpoint}"
+
+        async def stream_results():
+            async with aiohttp.ClientSession(
+                timeout=aiohttp.ClientTimeout(
+                    total=self.timeout
+                )  # Add timeout for request reliability
+            ) as session:
+                # Create the tasks for both prefill and decode requests
+                headers = {}
+                bootstrap_room_list = []
+                if self.enable_trace:
+                    bootstrap_room_list = (
+                        modified_request["bootstrap_room"]
+                        if isinstance(modified_request["bootstrap_room"], list)
+                        else [modified_request["bootstrap_room"]]
+                    )
+                    trace_context = trace_get_remote_propagate_context(
+                        bootstrap_room_list
+                    )
+                    headers = {"trace_context": trace_context}
+
+                tasks = [
+                    session.post(
+                        f"{prefill_server}/{endpoint}",
+                        json=modified_request,
+                        headers=headers,
+                    ),
+                    session.post(
+                        f"{decode_server}/{endpoint}",
+                        json=modified_request,
+                        headers=headers,
+                    ),
+                ]
+
+                for bootstrap_room in bootstrap_room_list:
+                    trace_slice_end(
+                        "mini_lb_launch", bootstrap_room, auto_next_anon=True
+                    )
+                # Wait for both responses to complete. Since this is streaming, they return immediately.
+                prefill_response, decode_response = await asyncio.gather(*tasks)
+
+                if modified_request.get("return_logprob", False):
+                    prefill_chunks = []
+                    async for chunk in prefill_response.content:
+                        prefill_chunks.append(chunk)
+
+                    first_prefill_chunk = (
+                        prefill_chunks[0].decode("utf-8")[5:].strip("\n")
+                    )
+                    first_prefill_chunk_json = orjson.loads(first_prefill_chunk)
+
+                    async for chunk in decode_response.content:
+                        # Note: This is inefficient
+                        # merge prefill input_token_logprobs, output_token_logprobs to decode
+                        decoded_chunk = chunk.decode("utf-8")
+                        if (
+                            decoded_chunk
+                            and decoded_chunk.startswith("data:")
+                            and "[DONE]" not in decoded_chunk
+                        ):
+                            ret_json = orjson.loads(decoded_chunk[5:].strip("\n"))
+                            ret_json["meta_info"]["input_token_logprobs"] = (
+                                first_prefill_chunk_json["meta_info"][
+                                    "input_token_logprobs"
+                                ]
+                                + ret_json["meta_info"]["input_token_logprobs"]
+                            )
+
+                            yield b"data: " + orjson.dumps(ret_json) + b"\n\n"
+                        else:
+                            yield chunk
+                else:
+                    async for chunk in decode_response.content.iter_chunked(
+                        AIOHTTP_STREAM_READ_CHUNK_SIZE
+                    ):
+                        yield chunk
+
+            for bootstrap_room in bootstrap_room_list:
+                trace_slice_end(
+                    "wait_PD_finish",
+                    bootstrap_room,
+                    thread_finish_flag=True,
+                )
+                trace_req_finish(bootstrap_room)
+
+        return StreamingResponse(
+            stream_results(),
+            media_type="text/event-stream",
+        )
+
+
+app = FastAPI()
+lb: Optional[MiniLoadBalancer] = None
+
+
+@app.get("/health")
+async def health_check():
+    return Response(status_code=200)
+
+
+@app.get("/health_generate")
+async def health_generate():
+    async with aiohttp.ClientSession() as session:
+        # Create the tasks
+        tasks = []
+        for server in chain(lb.prefill_urls, lb.decode_urls):
+            tasks.append(session.get(f"{server}/health_generate"))
+        for i, response in enumerate(asyncio.as_completed(tasks)):
+            await response
+    return Response(status_code=200)
+
+
+@app.post("/flush_cache")
+async def flush_cache():
+    async with aiohttp.ClientSession() as session:
+        # Create the tasks
+        tasks = []
+        for server in chain(lb.prefill_urls, lb.decode_urls):
+            tasks.append(session.post(f"{server}/flush_cache"))
+        for i, response in enumerate(asyncio.as_completed(tasks)):
+            await response
+    return Response(status_code=200)
+
+
+@app.get("/get_server_info")
+async def get_server_info():
+    prefill_infos = []
+    decode_infos = []
+    all_internal_states = []
+
+    async with aiohttp.ClientSession() as session:
+        for server in lb.prefill_urls:
+            server_info = await session.get(f"{server}/get_server_info")
+            prefill_infos.append(await server_info.json())
+        for server in lb.decode_urls:
+            server_info = await session.get(f"{server}/get_server_info")
+            info_json = await server_info.json()
+            decode_infos.append(info_json)
+            # Extract internal_states from decode servers
+            if "internal_states" in info_json:
+                all_internal_states.extend(info_json["internal_states"])
+
+    # Return format expected by bench_one_batch_server.py
+    if all_internal_states:
+        return {
+            "internal_states": all_internal_states,
+            "prefill": prefill_infos,
+            "decode": decode_infos,
+        }
+    else:
+        # Fallback with dummy data if no internal states found
+        return {
+            "internal_states": [
+                {
+                    "last_gen_throughput": 0.0,
+                    "avg_spec_accept_length": None,
+                }
+            ],
+            "prefill": prefill_infos,
+            "decode": decode_infos,
+        }
+
+
+@app.get("/get_model_info")
+async def get_model_info():
+    if not lb or not lb.prefill_urls:
+        raise HTTPException(
+            status_code=HTTPStatus.SERVICE_UNAVAILABLE,
+            detail="There is no server registered",
+        )
+
+    target_server_url = lb.prefill_urls[0]
+    endpoint_url = f"{target_server_url}/get_model_info"
+
+    async with aiohttp.ClientSession() as session:
+        try:
+            async with session.get(endpoint_url) as response:
+                if response.status != 200:
+                    error_text = await response.text()
+                    raise HTTPException(
+                        status_code=HTTPStatus.BAD_GATEWAY,
+                        detail=(
+                            f"Failed to get model info from {target_server_url}"
+                            f"Status: {response.status}, Response: {error_text}"
+                        ),
+                    )
+
+                model_info_json = await response.json()
+                return ORJSONResponse(content=model_info_json)
+
+        except aiohttp.ClientError as e:
+            raise HTTPException(
+                status_code=HTTPStatus.SERVICE_UNAVAILABLE,
+                detail=f"Failed to get model info from backend",
+            )
+
+
+@app.post("/generate")
+async def handle_generate_request(request_data: dict):
+    prefill_server, bootstrap_port, decode_server = lb.select_pair()
+
+    # Parse and transform prefill_server for bootstrap data
+    parsed_url = urllib.parse.urlparse(prefill_server)
+    hostname = maybe_wrap_ipv6_address(parsed_url.hostname)
+    modified_request = request_data.copy()
+
+    batch_size = _get_request_batch_size(modified_request)
+    if batch_size is not None:
+        modified_request.update(
+            {
+                "bootstrap_host": [hostname] * batch_size,
+                "bootstrap_port": [bootstrap_port] * batch_size,
+                "bootstrap_room": [
+                    _generate_bootstrap_room() for _ in range(batch_size)
+                ],
+            }
+        )
+    else:
+        modified_request.update(
+            {
+                "bootstrap_host": hostname,
+                "bootstrap_port": bootstrap_port,
+                "bootstrap_room": _generate_bootstrap_room(),
+            }
+        )
+
+    if request_data.get("stream", False):
+        return await lb.generate_stream(
+            modified_request, prefill_server, decode_server, "generate"
+        )
+    else:
+        return await lb.generate(
+            modified_request, prefill_server, decode_server, "generate"
+        )
+
+
+async def _forward_to_backend(request_data: dict, endpoint_name: str):
+    prefill_server, bootstrap_port, decode_server = lb.select_pair()
+
+    # Parse and transform prefill_server for bootstrap data
+    parsed_url = urllib.parse.urlparse(prefill_server)
+    hostname = maybe_wrap_ipv6_address(parsed_url.hostname)
+    modified_request = request_data.copy()
+    modified_request.update(
+        {
+            "bootstrap_host": hostname,
+            "bootstrap_port": bootstrap_port,
+            "bootstrap_room": _generate_bootstrap_room(),
+        }
+    )
+
+    if request_data.get("stream", False):
+        return await lb.generate_stream(
+            modified_request,
+            prefill_server,
+            decode_server,
+            endpoint=endpoint_name,
+        )
+    else:
+        return await lb.generate(
+            modified_request,
+            prefill_server,
+            decode_server,
+            endpoint=endpoint_name,
+        )
+
+
+@app.post("/v1/chat/completions")
+async def handle_chat_completion_request(request_data: dict):
+    return await _forward_to_backend(request_data, "v1/chat/completions")
+
+
+@app.post("/v1/completions")
+async def handle_completion_request(request_data: dict):
+    return await _forward_to_backend(request_data, "v1/completions")
+
+
+def _generate_bootstrap_room():
+    bootstrap_room = random.randint(0, 2**63 - 1)
+    if lb.enable_trace:
+        trace_req_start(bootstrap_room, bootstrap_room, role="router")
+        trace_slice_start("mini_lb_launch", bootstrap_room)
+    return bootstrap_room
+
+
+# We may utilize `GenerateReqInput`'s logic later
+def _get_request_batch_size(request):
+    if (text := request.get("text")) is not None:
+        return None if isinstance(text, str) else len(text)
+    if (input_ids := request.get("input_ids")) is not None:
+        return None if isinstance(input_ids[0], int) else len(input_ids)
+    return None
+
+
+@app.get("/v1/models")
+async def get_models():
+    prefill_server = lb.prefill_urls[0]  # Get the first prefill server
+    async with aiohttp.ClientSession() as session:
+        try:
+            response = await session.get(f"{prefill_server}/v1/models")
+            if response.status != 200:
+                raise HTTPException(
+                    status_code=response.status,
+                    detail=f"Prefill server error: Status {response.status}",
+                )
+            return ORJSONResponse(content=await response.json())
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=str(e))

sgl-model-gateway/bindings/python/sglang_router/router.py

@@ -0,0 +1,226 @@
+from typing import Optional
+
+from sglang_router.router_args import RouterArgs
+from sglang_router.sglang_router_rs import (
+    BackendType,
+    HistoryBackendType,
+    PolicyType,
+    PyOracleConfig,
+    PyPostgresConfig,
+)
+from sglang_router.sglang_router_rs import Router as _Router
+
+
+def policy_from_str(policy_str: Optional[str]) -> PolicyType:
+    """Convert policy string to PolicyType enum."""
+    if policy_str is None:
+        return None
+    policy_map = {
+        "random": PolicyType.Random,
+        "round_robin": PolicyType.RoundRobin,
+        "cache_aware": PolicyType.CacheAware,
+        "power_of_two": PolicyType.PowerOfTwo,
+        "bucket": PolicyType.Bucket,
+    }
+    return policy_map[policy_str]
+
+
+def backend_from_str(backend_str: Optional[str]) -> BackendType:
+    """Convert backend string to BackendType enum."""
+    if isinstance(backend_str, BackendType):
+        return backend_str
+    if backend_str is None:
+        return BackendType.Sglang
+    backend_map = {"sglang": BackendType.Sglang, "openai": BackendType.Openai}
+    backend_lower = backend_str.lower()
+    if backend_lower not in backend_map:
+        raise ValueError(
+            f"Unknown backend: {backend_str}. Valid options: {', '.join(backend_map.keys())}"
+        )
+    return backend_map[backend_lower]
+
+
+def history_backend_from_str(backend_str: Optional[str]) -> HistoryBackendType:
+    """Convert history backend string to HistoryBackendType enum."""
+    if isinstance(backend_str, HistoryBackendType):
+        return backend_str
+    if backend_str is None:
+        return HistoryBackendType.Memory
+    backend_lower = backend_str.lower()
+    if backend_lower == "memory":
+        return HistoryBackendType.Memory
+    elif backend_lower == "none":
+        # Use getattr to access 'None' which is a Python keyword
+        return getattr(HistoryBackendType, "None")
+    elif backend_lower == "oracle":
+        return HistoryBackendType.Oracle
+    elif backend_lower == "postgres":
+        return HistoryBackendType.Postgres
+    else:
+        raise ValueError(f"Unknown history backend: {backend_str}")
+
+
+class Router:
+    """
+    A high-performance router for distributing requests across worker nodes.
+
+    Args:
+        worker_urls: List of URLs for worker nodes that will handle requests. Each URL should include
+            the protocol, host, and port (e.g., ['http://worker1:8000', 'http://worker2:8000'])
+        policy: Load balancing policy to use. Options:
+            - PolicyType.Random: Randomly select workers
+            - PolicyType.RoundRobin: Distribute requests in round-robin fashion
+            - PolicyType.CacheAware: Distribute requests based on cache state and load balance
+            - PolicyType.PowerOfTwo: Select best of two random workers based on load (PD mode only)
+        host: Host address to bind the router server. Supports IPv4, IPv6 (e.g., ::, ::1), or 0.0.0.0 for all interfaces. Default: '0.0.0.0'
+        port: Port number to bind the router server. Default: 3001
+        worker_startup_timeout_secs: Timeout in seconds for worker startup and registration. Large models can take significant time to load into GPU memory. Default: 1800 (30 minutes)
+        worker_startup_check_interval: Interval in seconds between checks for worker initialization. Default: 10
+        cache_threshold: Cache threshold (0.0-1.0) for cache-aware routing. Routes to cached worker
+            if the match rate exceeds threshold, otherwise routes to the worker with the smallest
+            tree. Default: 0.5
+        balance_abs_threshold: Load balancing is triggered when (max_load - min_load) > abs_threshold
+            AND max_load > min_load * rel_threshold. Otherwise, use cache aware. Default: 32
+        balance_rel_threshold: Load balancing is triggered when (max_load - min_load) > abs_threshold
+            AND max_load > min_load * rel_threshold. Otherwise, use cache aware. Default: 1.0001
+        eviction_interval_secs: Interval in seconds between cache eviction operations in cache-aware
+            routing. Default: 60
+        max_payload_size: Maximum payload size in bytes. Default: 256MB
+        max_tree_size: Maximum size of the approximation tree for cache-aware routing. Default: 2^24
+        dp_aware: Enable data parallelism aware schedule. Default: False
+        enable_igw: Enable IGW (Inference-Gateway) mode for multi-model support. When enabled,
+            the router can manage multiple models simultaneously with per-model load balancing
+            policies. Default: False
+        api_key: The api key used for the authorization with the worker.
+            Useful when the dp aware scheduling strategy is enabled.
+            Default: None
+        log_dir: Directory to store log files. If None, logs are only output to console. Default: None
+        log_level: Logging level. Options: 'debug', 'info', 'warn', 'error'.
+        service_discovery: Enable Kubernetes service discovery. When enabled, the router will
+            automatically discover worker pods based on the selector. Default: False
+        selector: Dictionary mapping of label keys to values for Kubernetes pod selection.
+            Example: {"app": "sglang-worker"}. Default: {}
+        service_discovery_port: Port to use for service discovery. The router will generate
+            worker URLs using this port. Default: 80
+        service_discovery_namespace: Kubernetes namespace to watch for pods. If not provided,
+            watches pods across all namespaces (requires cluster-wide permissions). Default: None
+        prefill_selector: Dictionary mapping of label keys to values for Kubernetes pod selection
+            for prefill servers (PD mode only). Default: {}
+        decode_selector: Dictionary mapping of label keys to values for Kubernetes pod selection
+            for decode servers (PD mode only). Default: {}
+        prometheus_port: Port to expose Prometheus metrics. Default: None
+        prometheus_host: Host address to bind the Prometheus metrics server. Default: None
+        pd_disaggregation: Enable PD (Prefill-Decode) disaggregated mode. Default: False
+        prefill_urls: List of (url, bootstrap_port) tuples for prefill servers (PD mode only)
+        decode_urls: List of URLs for decode servers (PD mode only)
+        prefill_policy: Specific load balancing policy for prefill nodes (PD mode only).
+            If not specified, uses the main policy. Default: None
+        decode_policy: Specific load balancing policy for decode nodes (PD mode only).
+            If not specified, uses the main policy. Default: None
+        request_id_headers: List of HTTP headers to check for request IDs. If not specified,
+            uses common defaults: ['x-request-id', 'x-correlation-id', 'x-trace-id', 'request-id'].
+            Example: ['x-my-request-id', 'x-custom-trace-id']. Default: None
+        bootstrap_port_annotation: Kubernetes annotation name for bootstrap port (PD mode).
+            Default: 'sglang.ai/bootstrap-port'
+        request_timeout_secs: Request timeout in seconds. Default: 600
+        max_concurrent_requests: Maximum number of concurrent requests allowed for rate limiting. Default: 256
+        queue_size: Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately). Default: 100
+        queue_timeout_secs: Maximum time (in seconds) a request can wait in queue before timing out. Default: 60
+        rate_limit_tokens_per_second: Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests. Default: None
+        cors_allowed_origins: List of allowed origins for CORS. Empty list allows all origins. Default: []
+        health_failure_threshold: Number of consecutive health check failures before marking worker unhealthy. Default: 3
+        health_success_threshold: Number of consecutive health check successes before marking worker healthy. Default: 2
+        health_check_timeout_secs: Timeout in seconds for health check requests. Default: 5
+        health_check_interval_secs: Interval in seconds between runtime health checks. Default: 60
+        health_check_endpoint: Health check endpoint path. Default: '/health'
+        model_path: Model path for loading tokenizer (HuggingFace model ID or local path). Default: None
+        tokenizer_path: Explicit tokenizer path (overrides model_path tokenizer if provided). Default: None
+    """
+
+    def __init__(self, router: _Router):
+        self._router = router
+
+    @staticmethod
+    def from_args(args: RouterArgs) -> "Router":
+        """Create a router from a RouterArgs instance."""
+
+        args_dict = vars(args)
+        # Convert RouterArgs to _Router parameters
+        args_dict["worker_urls"] = (
+            []
+            if args_dict["service_discovery"] or args_dict["pd_disaggregation"]
+            else args_dict["worker_urls"]
+        )
+        args_dict["policy"] = policy_from_str(args_dict["policy"])
+        args_dict["prefill_urls"] = (
+            args_dict["prefill_urls"] if args_dict["pd_disaggregation"] else None
+        )
+        args_dict["decode_urls"] = (
+            args_dict["decode_urls"] if args_dict["pd_disaggregation"] else None
+        )
+        args_dict["prefill_policy"] = policy_from_str(args_dict["prefill_policy"])
+        args_dict["decode_policy"] = policy_from_str(args_dict["decode_policy"])
+
+        # Convert backend
+        args_dict["backend"] = backend_from_str(args_dict.get("backend"))
+
+        # Convert history_backend to enum first
+        history_backend_raw = args_dict.get("history_backend", "memory")
+        history_backend = history_backend_from_str(history_backend_raw)
+
+        # Convert Oracle config if needed
+        oracle_config = None
+        if history_backend == HistoryBackendType.Oracle:
+            # Prioritize TNS alias over connect descriptor
+            tns_alias = args_dict.get("oracle_tns_alias")
+            connect_descriptor = args_dict.get("oracle_connect_descriptor")
+
+            # Use TNS alias if provided, otherwise use connect descriptor
+            final_descriptor = tns_alias if tns_alias else connect_descriptor
+
+            oracle_config = PyOracleConfig(
+                password=args_dict.get("oracle_password"),
+                username=args_dict.get("oracle_username"),
+                connect_descriptor=final_descriptor,
+                wallet_path=args_dict.get("oracle_wallet_path"),
+                pool_min=args_dict.get("oracle_pool_min", 1),
+                pool_max=args_dict.get("oracle_pool_max", 16),
+                pool_timeout_secs=args_dict.get("oracle_pool_timeout_secs", 30),
+            )
+        args_dict["oracle_config"] = oracle_config
+        args_dict["history_backend"] = history_backend
+
+        # Convert Postgres config if needed
+        postgres_config = None
+        if history_backend == HistoryBackendType.Postgres:
+            postgres_config = PyPostgresConfig(
+                db_url=args_dict.get("postgres_db_url"),
+                pool_max=args_dict.get("postgres_pool_max", 16),
+            )
+        args_dict["postgres_config"] = postgres_config
+
+        # Remove fields that shouldn't be passed to Rust Router constructor
+        fields_to_remove = [
+            "mini_lb",
+            "oracle_wallet_path",
+            "oracle_tns_alias",
+            "oracle_connect_descriptor",
+            "oracle_username",
+            "oracle_password",
+            "oracle_pool_min",
+            "oracle_pool_max",
+            "oracle_pool_timeout_secs",
+            "postgres_db_url",
+            "postgres_pool_max",
+        ]
+        for field in fields_to_remove:
+            args_dict.pop(field, None)
+
+        return Router(_Router(**args_dict))
+
+    def start(self) -> None:
+        """Start the router server.
+
+        This method blocks until the server is shut down.
+        """
+        self._router.start()

sgl-model-gateway/bindings/python/sglang_router/router_args.py

@@ -0,0 +1,782 @@
+import argparse
+import dataclasses
+import logging
+import os
+from typing import Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass
+class RouterArgs:
+    # Worker configuration
+    worker_urls: List[str] = dataclasses.field(default_factory=list)
+    host: str = "0.0.0.0"
+    port: int = 30000
+
+    # PD-specific configuration
+    mini_lb: bool = False
+    pd_disaggregation: bool = False  # Enable PD disaggregated mode
+    prefill_urls: List[tuple] = dataclasses.field(
+        default_factory=list
+    )  # List of (url, bootstrap_port)
+    decode_urls: List[str] = dataclasses.field(default_factory=list)
+
+    # Routing policy
+    policy: str = "cache_aware"
+    prefill_policy: Optional[str] = None  # Specific policy for prefill nodes in PD mode
+    decode_policy: Optional[str] = None  # Specific policy for decode nodes in PD mode
+    worker_startup_timeout_secs: int = 1800
+    worker_startup_check_interval: int = 30
+    cache_threshold: float = 0.3
+    balance_abs_threshold: int = 64
+    balance_rel_threshold: float = 1.5
+    eviction_interval_secs: int = 120
+    max_tree_size: int = 2**26
+    max_payload_size: int = 512 * 1024 * 1024  # 512MB default for large batches
+    bucket_adjust_interval_secs: int = 5
+    dp_aware: bool = False
+    enable_igw: bool = False  # Enable IGW (Inter-Gateway) mode for multi-model support
+    api_key: Optional[str] = None
+    log_dir: Optional[str] = None
+    log_level: Optional[str] = None
+    # Service discovery configuration
+    service_discovery: bool = False
+    selector: Dict[str, str] = dataclasses.field(default_factory=dict)
+    service_discovery_port: int = 80
+    service_discovery_namespace: Optional[str] = None
+    # PD service discovery configuration
+    prefill_selector: Dict[str, str] = dataclasses.field(default_factory=dict)
+    decode_selector: Dict[str, str] = dataclasses.field(default_factory=dict)
+    bootstrap_port_annotation: str = "sglang.ai/bootstrap-port"
+    # Prometheus configuration
+    prometheus_port: Optional[int] = None
+    prometheus_host: Optional[str] = None
+    # Request ID headers configuration
+    request_id_headers: Optional[List[str]] = None
+    # Request timeout in seconds
+    request_timeout_secs: int = 1800
+    # Max concurrent requests for rate limiting (-1 to disable)
+    max_concurrent_requests: int = -1
+    # Queue size for pending requests when max concurrent limit reached
+    queue_size: int = 100
+    # Maximum time (in seconds) a request can wait in queue before timing out
+    queue_timeout_secs: int = 60
+    # Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests
+    rate_limit_tokens_per_second: Optional[int] = None
+    # CORS allowed origins
+    cors_allowed_origins: List[str] = dataclasses.field(default_factory=list)
+    # Retry configuration
+    retry_max_retries: int = 5
+    retry_initial_backoff_ms: int = 50
+    retry_max_backoff_ms: int = 30_000
+    retry_backoff_multiplier: float = 1.5
+    retry_jitter_factor: float = 0.2
+    disable_retries: bool = False
+    # Health check configuration
+    health_failure_threshold: int = 3
+    health_success_threshold: int = 2
+    health_check_timeout_secs: int = 5
+    health_check_interval_secs: int = 60
+    health_check_endpoint: str = "/health"
+    # Circuit breaker configuration
+    cb_failure_threshold: int = 10
+    cb_success_threshold: int = 3
+    cb_timeout_duration_secs: int = 60
+    cb_window_duration_secs: int = 120
+    disable_circuit_breaker: bool = False
+    model_path: Optional[str] = None
+    tokenizer_path: Optional[str] = None
+    chat_template: Optional[str] = None
+    # Tokenizer cache configuration
+    tokenizer_cache_enable_l0: bool = False
+    tokenizer_cache_l0_max_entries: int = 10000
+    tokenizer_cache_enable_l1: bool = False
+    tokenizer_cache_l1_max_memory: int = 50 * 1024 * 1024  # 50MB
+    reasoning_parser: Optional[str] = None
+    tool_call_parser: Optional[str] = None
+    # MCP server configuration
+    mcp_config_path: Optional[str] = None
+    # Backend selection
+    backend: str = "sglang"
+    # History backend configuration
+    history_backend: str = "memory"
+    oracle_wallet_path: Optional[str] = None
+    oracle_tns_alias: Optional[str] = None
+    oracle_connect_descriptor: Optional[str] = None
+    oracle_username: Optional[str] = None
+    oracle_password: Optional[str] = None
+    oracle_pool_min: int = 1
+    oracle_pool_max: int = 16
+    oracle_pool_timeout_secs: int = 30
+    postgres_db_url: Optional[str] = None
+    postgres_pool_max: int = 16
+    # mTLS configuration for worker communication
+    client_cert_path: Optional[str] = None
+    client_key_path: Optional[str] = None
+    ca_cert_paths: List[str] = dataclasses.field(default_factory=list)
+    # Trace
+    enable_trace: bool = False
+    otlp_traces_endpoint: str = "localhost:4317"
+
+    @staticmethod
+    def add_cli_args(
+        parser: argparse.ArgumentParser,
+        use_router_prefix: bool = False,
+        exclude_host_port: bool = False,
+    ):
+        """
+        Add router-specific arguments to an argument parser.
+
+        Args:
+            parser: The argument parser to add arguments to
+            use_router_prefix: If True, prefix all arguments with 'router-' to avoid conflicts
+            exclude_host_port: If True, don't add host and port arguments (used when inheriting from server)
+        """
+        prefix = "router-" if use_router_prefix else ""
+
+        # Worker configuration
+        if not exclude_host_port:
+            parser.add_argument(
+                "--host",
+                type=str,
+                default=RouterArgs.host,
+                help="Host address to bind the router server. Supports IPv4, IPv6 (e.g., ::, ::1), or 0.0.0.0 for all interfaces",
+            )
+            parser.add_argument(
+                "--port",
+                type=int,
+                default=RouterArgs.port,
+                help="Port number to bind the router server",
+            )
+
+        parser.add_argument(
+            "--worker-urls",
+            type=str,
+            nargs="*",
+            default=[],
+            help="List of worker URLs. Supports IPv4 and IPv6 addresses (use brackets for IPv6, e.g., http://[::1]:8000 http://192.168.1.1:8000)",
+        )
+
+        # Routing policy configuration
+        parser.add_argument(
+            f"--{prefix}policy",
+            type=str,
+            default=RouterArgs.policy,
+            choices=["random", "round_robin", "cache_aware", "power_of_two"],
+            help="Load balancing policy to use. In PD mode, this is used for both prefill and decode unless overridden",
+        )
+        parser.add_argument(
+            f"--{prefix}prefill-policy",
+            type=str,
+            default=None,
+            choices=["random", "round_robin", "cache_aware", "power_of_two", "bucket"],
+            help="Specific policy for prefill nodes in PD mode. If not specified, uses the main policy",
+        )
+        parser.add_argument(
+            f"--{prefix}decode-policy",
+            type=str,
+            default=None,
+            choices=["random", "round_robin", "cache_aware", "power_of_two"],
+            help="Specific policy for decode nodes in PD mode. If not specified, uses the main policy",
+        )
+
+        # PD-specific arguments
+        parser.add_argument(
+            f"--{prefix}mini-lb",
+            action="store_true",
+            help="Enable MiniLB",
+        )
+        parser.add_argument(
+            f"--{prefix}pd-disaggregation",
+            action="store_true",
+            help="Enable PD (Prefill-Decode) disaggregated mode",
+        )
+        parser.add_argument(
+            f"--{prefix}prefill",
+            nargs="+",
+            action="append",
+            help="Prefill server URL and optional bootstrap port. Can be specified multiple times. "
+            "Format: --prefill URL [BOOTSTRAP_PORT]. "
+            "BOOTSTRAP_PORT can be a port number, 'none', or omitted (defaults to none).",
+        )
+        parser.add_argument(
+            f"--{prefix}decode",
+            nargs=1,
+            action="append",
+            metavar=("URL",),
+            help="Decode server URL. Can be specified multiple times.",
+        )
+        parser.add_argument(
+            f"--{prefix}worker-startup-timeout-secs",
+            type=int,
+            default=RouterArgs.worker_startup_timeout_secs,
+            help="Timeout in seconds for worker startup and registration (default: 1800 / 30 minutes). Large models can take significant time to load into GPU memory.",
+        )
+        parser.add_argument(
+            f"--{prefix}worker-startup-check-interval",
+            type=int,
+            default=RouterArgs.worker_startup_check_interval,
+            help="Interval in seconds between checks for worker startup",
+        )
+        parser.add_argument(
+            f"--{prefix}cache-threshold",
+            type=float,
+            default=RouterArgs.cache_threshold,
+            help="Cache threshold (0.0-1.0) for cache-aware routing",
+        )
+        parser.add_argument(
+            f"--{prefix}balance-abs-threshold",
+            type=int,
+            default=RouterArgs.balance_abs_threshold,
+            help="Load balancing is triggered when (max_load - min_load) > abs_threshold AND max_load > min_load * rel_threshold. Otherwise, use cache aware",
+        )
+        parser.add_argument(
+            f"--{prefix}balance-rel-threshold",
+            type=float,
+            default=RouterArgs.balance_rel_threshold,
+            help="Load balancing is triggered when (max_load - min_load) > abs_threshold AND max_load > min_load * rel_threshold. Otherwise, use cache aware",
+        )
+        parser.add_argument(
+            f"--{prefix}bucket-adjust-interval-secs",
+            type=int,
+            default=RouterArgs.bucket_adjust_interval_secs,
+            help="Interval in seconds between bucket boundary adjustment operations",
+        )
+        parser.add_argument(
+            f"--{prefix}eviction-interval-secs",
+            type=int,
+            default=RouterArgs.eviction_interval_secs,
+            help="Interval in seconds between cache eviction operations",
+        )
+        parser.add_argument(
+            f"--{prefix}max-tree-size",
+            type=int,
+            default=RouterArgs.max_tree_size,
+            help="Maximum size of the approximation tree for cache-aware routing",
+        )
+        parser.add_argument(
+            f"--{prefix}max-payload-size",
+            type=int,
+            default=RouterArgs.max_payload_size,
+            help="Maximum payload size in bytes",
+        )
+        parser.add_argument(
+            f"--{prefix}dp-aware",
+            action="store_true",
+            help="Enable data parallelism aware schedule",
+        )
+        parser.add_argument(
+            f"--{prefix}enable-igw",
+            action="store_true",
+            help="Enable IGW (Inference-Gateway) mode for multi-model support",
+        )
+        parser.add_argument(
+            f"--{prefix}api-key",
+            type=str,
+            default=None,
+            help="The api key used for the authorization with the worker.  Useful when the dp aware scheduling strategy is enaled.",
+        )
+        parser.add_argument(
+            f"--{prefix}log-dir",
+            type=str,
+            default=None,
+            help="Directory to store log files. If not specified, logs are only output to console.",
+        )
+        parser.add_argument(
+            f"--{prefix}log-level",
+            type=str,
+            default="info",
+            choices=["debug", "info", "warn", "error"],
+            help="Set the logging level. If not specified, defaults to INFO.",
+        )
+        parser.add_argument(
+            f"--{prefix}service-discovery",
+            action="store_true",
+            help="Enable Kubernetes service discovery",
+        )
+        parser.add_argument(
+            f"--{prefix}selector",
+            type=str,
+            nargs="+",
+            default={},
+            help="Label selector for Kubernetes service discovery (format: key1=value1 key2=value2)",
+        )
+        parser.add_argument(
+            f"--{prefix}service-discovery-port",
+            type=int,
+            default=RouterArgs.service_discovery_port,
+            help="Port to use for discovered worker pods",
+        )
+        parser.add_argument(
+            f"--{prefix}service-discovery-namespace",
+            type=str,
+            help="Kubernetes namespace to watch for pods. If not provided, watches all namespaces (requires cluster-wide permissions)",
+        )
+        parser.add_argument(
+            f"--{prefix}prefill-selector",
+            type=str,
+            nargs="+",
+            default={},
+            help="Label selector for prefill server pods in PD mode (format: key1=value1 key2=value2)",
+        )
+        parser.add_argument(
+            f"--{prefix}decode-selector",
+            type=str,
+            nargs="+",
+            default={},
+            help="Label selector for decode server pods in PD mode (format: key1=value1 key2=value2)",
+        )
+        # Prometheus configuration
+        parser.add_argument(
+            f"--{prefix}prometheus-port",
+            type=int,
+            default=29000,
+            help="Port to expose Prometheus metrics. If not specified, Prometheus metrics are disabled",
+        )
+        parser.add_argument(
+            f"--{prefix}prometheus-host",
+            type=str,
+            default="0.0.0.0",
+            help="Host address to bind the Prometheus metrics server. Supports IPv4, IPv6 (e.g., ::, ::1), or 0.0.0.0 for all interfaces",
+        )
+        parser.add_argument(
+            f"--{prefix}request-id-headers",
+            type=str,
+            nargs="*",
+            help="Custom HTTP headers to check for request IDs (e.g., x-request-id x-trace-id). If not specified, uses common defaults.",
+        )
+        parser.add_argument(
+            f"--{prefix}request-timeout-secs",
+            type=int,
+            default=RouterArgs.request_timeout_secs,
+            help="Request timeout in seconds",
+        )
+        # Retry configuration
+        parser.add_argument(
+            f"--{prefix}retry-max-retries",
+            type=int,
+            default=RouterArgs.retry_max_retries,
+        )
+        parser.add_argument(
+            f"--{prefix}retry-initial-backoff-ms",
+            type=int,
+            default=RouterArgs.retry_initial_backoff_ms,
+        )
+        parser.add_argument(
+            f"--{prefix}retry-max-backoff-ms",
+            type=int,
+            default=RouterArgs.retry_max_backoff_ms,
+        )
+        parser.add_argument(
+            f"--{prefix}retry-backoff-multiplier",
+            type=float,
+            default=RouterArgs.retry_backoff_multiplier,
+        )
+        parser.add_argument(
+            f"--{prefix}retry-jitter-factor",
+            type=float,
+            default=RouterArgs.retry_jitter_factor,
+        )
+        parser.add_argument(
+            f"--{prefix}disable-retries",
+            action="store_true",
+            help="Disable retries (equivalent to setting retry_max_retries=1)",
+        )
+        # Circuit breaker configuration
+        parser.add_argument(
+            f"--{prefix}cb-failure-threshold",
+            type=int,
+            default=RouterArgs.cb_failure_threshold,
+        )
+        parser.add_argument(
+            f"--{prefix}cb-success-threshold",
+            type=int,
+            default=RouterArgs.cb_success_threshold,
+        )
+        parser.add_argument(
+            f"--{prefix}cb-timeout-duration-secs",
+            type=int,
+            default=RouterArgs.cb_timeout_duration_secs,
+        )
+        parser.add_argument(
+            f"--{prefix}cb-window-duration-secs",
+            type=int,
+            default=RouterArgs.cb_window_duration_secs,
+        )
+        parser.add_argument(
+            f"--{prefix}disable-circuit-breaker",
+            action="store_true",
+            help="Disable circuit breaker (equivalent to setting cb_failure_threshold to u32::MAX)",
+        )
+        # Health check configuration
+        parser.add_argument(
+            f"--{prefix}health-failure-threshold",
+            type=int,
+            default=RouterArgs.health_failure_threshold,
+            help="Number of consecutive health check failures before marking worker unhealthy",
+        )
+        parser.add_argument(
+            f"--{prefix}health-success-threshold",
+            type=int,
+            default=RouterArgs.health_success_threshold,
+            help="Number of consecutive health check successes before marking worker healthy",
+        )
+        parser.add_argument(
+            f"--{prefix}health-check-timeout-secs",
+            type=int,
+            default=RouterArgs.health_check_timeout_secs,
+            help="Timeout in seconds for health check requests",
+        )
+        parser.add_argument(
+            f"--{prefix}health-check-interval-secs",
+            type=int,
+            default=RouterArgs.health_check_interval_secs,
+            help="Interval in seconds between runtime health checks",
+        )
+        parser.add_argument(
+            f"--{prefix}health-check-endpoint",
+            type=str,
+            default=RouterArgs.health_check_endpoint,
+            help="Health check endpoint path",
+        )
+        parser.add_argument(
+            f"--{prefix}max-concurrent-requests",
+            type=int,
+            default=RouterArgs.max_concurrent_requests,
+            help="Maximum number of concurrent requests allowed (for rate limiting). Set to -1 to disable rate limiting.",
+        )
+        parser.add_argument(
+            f"--{prefix}queue-size",
+            type=int,
+            default=RouterArgs.queue_size,
+            help="Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately)",
+        )
+        parser.add_argument(
+            f"--{prefix}queue-timeout-secs",
+            type=int,
+            default=RouterArgs.queue_timeout_secs,
+            help="Maximum time (in seconds) a request can wait in queue before timing out",
+        )
+        parser.add_argument(
+            f"--{prefix}rate-limit-tokens-per-second",
+            type=int,
+            default=RouterArgs.rate_limit_tokens_per_second,
+            help="Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests",
+        )
+        parser.add_argument(
+            f"--{prefix}cors-allowed-origins",
+            type=str,
+            nargs="*",
+            default=[],
+            help="CORS allowed origins (e.g., http://localhost:3000 https://example.com)",
+        )
+        # Tokenizer configuration
+        parser.add_argument(
+            f"--{prefix}model-path",
+            type=str,
+            default=None,
+            help="Model path for loading tokenizer (HuggingFace model ID or local path)",
+        )
+        parser.add_argument(
+            f"--{prefix}tokenizer-path",
+            type=str,
+            default=None,
+            help="Explicit tokenizer path (overrides model_path tokenizer if provided)",
+        )
+        parser.add_argument(
+            f"--{prefix}chat-template",
+            type=str,
+            default=None,
+            help="Chat template path (optional)",
+        )
+        parser.add_argument(
+            f"--{prefix}tokenizer-cache-enable-l0",
+            action="store_true",
+            default=RouterArgs.tokenizer_cache_enable_l0,
+            help="Enable L0 (whole-string exact match) tokenizer cache (default: False)",
+        )
+        parser.add_argument(
+            f"--{prefix}tokenizer-cache-l0-max-entries",
+            type=int,
+            default=RouterArgs.tokenizer_cache_l0_max_entries,
+            help="Maximum number of entries in L0 tokenizer cache (default: 10000)",
+        )
+        parser.add_argument(
+            f"--{prefix}tokenizer-cache-enable-l1",
+            action="store_true",
+            default=RouterArgs.tokenizer_cache_enable_l1,
+            help="Enable L1 (prefix matching) tokenizer cache (default: False)",
+        )
+        parser.add_argument(
+            f"--{prefix}tokenizer-cache-l1-max-memory",
+            type=int,
+            default=RouterArgs.tokenizer_cache_l1_max_memory,
+            help="Maximum memory for L1 tokenizer cache in bytes (default: 50MB)",
+        )
+        parser.add_argument(
+            f"--{prefix}reasoning-parser",
+            type=str,
+            default=None,
+            help="Specify the parser for reasoning models (e.g., deepseek-r1, qwen3)",
+        )
+        parser.add_argument(
+            f"--{prefix}tool-call-parser",
+            type=str,
+            default=None,
+            help="Specify the parser for handling tool-call interactions",
+        )
+        # MCP server configuration
+        parser.add_argument(
+            f"--{prefix}mcp-config-path",
+            type=str,
+            default=None,
+            help="Path to MCP (Model Context Protocol) server configuration file",
+        )
+        # Backend selection
+        parser.add_argument(
+            f"--{prefix}backend",
+            type=str,
+            default=RouterArgs.backend,
+            choices=["sglang", "openai"],
+            help="Backend runtime to use (default: sglang)",
+        )
+        # History backend configuration
+        parser.add_argument(
+            f"--{prefix}history-backend",
+            type=str,
+            default=RouterArgs.history_backend,
+            choices=["memory", "none", "oracle", "postgres"],
+            help="History storage backend for conversations and responses (default: memory)",
+        )
+        # Oracle configuration
+        parser.add_argument(
+            f"--{prefix}oracle-wallet-path",
+            type=str,
+            default=os.getenv("ATP_WALLET_PATH"),
+            help="Path to Oracle ATP wallet directory (env: ATP_WALLET_PATH)",
+        )
+        parser.add_argument(
+            f"--{prefix}oracle-tns-alias",
+            type=str,
+            default=os.getenv("ATP_TNS_ALIAS"),
+            help="Oracle TNS alias from tnsnames.ora (env: ATP_TNS_ALIAS).",
+        )
+        parser.add_argument(
+            f"--{prefix}oracle-connect-descriptor",
+            type=str,
+            default=os.getenv("ATP_DSN"),
+            help="Oracle connection descriptor/DSN (full connection string) (env: ATP_DSN)",
+        )
+        parser.add_argument(
+            f"--{prefix}oracle-username",
+            type=str,
+            default=os.getenv("ATP_USER"),
+            help="Oracle database username (env: ATP_USER)",
+        )
+        parser.add_argument(
+            f"--{prefix}oracle-password",
+            type=str,
+            default=os.getenv("ATP_PASSWORD"),
+            help="Oracle database password (env: ATP_PASSWORD)",
+        )
+        parser.add_argument(
+            f"--{prefix}oracle-pool-min",
+            type=int,
+            default=int(os.getenv("ATP_POOL_MIN", RouterArgs.oracle_pool_min)),
+            help="Minimum Oracle connection pool size (default: 1, env: ATP_POOL_MIN)",
+        )
+        parser.add_argument(
+            f"--{prefix}oracle-pool-max",
+            type=int,
+            default=int(os.getenv("ATP_POOL_MAX", RouterArgs.oracle_pool_max)),
+            help="Maximum Oracle connection pool size (default: 16, env: ATP_POOL_MAX)",
+        )
+        parser.add_argument(
+            f"--{prefix}oracle-pool-timeout-secs",
+            type=int,
+            default=int(
+                os.getenv("ATP_POOL_TIMEOUT_SECS", RouterArgs.oracle_pool_timeout_secs)
+            ),
+            help="Oracle connection pool timeout in seconds (default: 30, env: ATP_POOL_TIMEOUT_SECS)",
+        )
+        # Postgres configuration
+        parser.add_argument(
+            f"--{prefix}postgres-db-url",
+            type=str,
+            default=os.getenv("POSTGRES_DB_URL"),
+            help="PostgreSQL database connection URL (env: POSTGRES_DB_URL)",
+        )
+        parser.add_argument(
+            f"--{prefix}postgres-pool-max",
+            type=int,
+            default=int(os.getenv("POSTGRES_POOL_MAX", RouterArgs.postgres_pool_max)),
+            help="Maximum PostgreSQL connection pool size (default: 16, env: POSTGRES_POOL_MAX)",
+        )
+        # mTLS configuration
+        parser.add_argument(
+            f"--{prefix}client-cert-path",
+            type=str,
+            default=None,
+            help="Path to client certificate for mTLS authentication with workers",
+        )
+        parser.add_argument(
+            f"--{prefix}client-key-path",
+            type=str,
+            default=None,
+            help="Path to client private key for mTLS authentication with workers",
+        )
+        parser.add_argument(
+            f"--{prefix}ca-cert-paths",
+            type=str,
+            nargs="*",
+            default=[],
+            help="Path(s) to CA certificate(s) for verifying worker TLS certificates. Can specify multiple CAs.",
+        )
+        parser.add_argument(
+            f"--{prefix}enable-trace",
+            action="store_true",
+            help="Enable opentelemetry trace",
+        )
+        parser.add_argument(
+            f"--{prefix}otlp-traces-endpoint",
+            type=str,
+            default="localhost:4317",
+            help="Config opentelemetry collector endpoint if --enable-trace is set. format: <ip>:<port>",
+        )
+
+    @classmethod
+    def from_cli_args(
+        cls, args: argparse.Namespace, use_router_prefix: bool = False
+    ) -> "RouterArgs":
+        """
+        Create RouterArgs instance from parsed command line arguments.
+
+        Args:
+            args: Parsed command line arguments
+            use_router_prefix: If True, look for arguments with 'router-' prefix
+        """
+        prefix = "router_" if use_router_prefix else ""
+        cli_args_dict = vars(args)
+        args_dict = {}
+
+        for attr in dataclasses.fields(cls):
+            # Auto strip prefix from args
+            if f"{prefix}{attr.name}" in cli_args_dict:
+                args_dict[attr.name] = cli_args_dict[f"{prefix}{attr.name}"]
+            elif attr.name in cli_args_dict:
+                args_dict[attr.name] = cli_args_dict[attr.name]
+
+        # parse special arguments and remove "--prefill" and "--decode" from cli_args_dict
+        args_dict["prefill_urls"] = cls._parse_prefill_urls(
+            cli_args_dict.get(f"{prefix}prefill", None)
+        )
+        args_dict["decode_urls"] = cls._parse_decode_urls(
+            cli_args_dict.get(f"{prefix}decode", None)
+        )
+        args_dict["selector"] = cls._parse_selector(
+            cli_args_dict.get(f"{prefix}selector", None)
+        )
+        args_dict["prefill_selector"] = cls._parse_selector(
+            cli_args_dict.get(f"{prefix}prefill_selector", None)
+        )
+        args_dict["decode_selector"] = cls._parse_selector(
+            cli_args_dict.get(f"{prefix}decode_selector", None)
+        )
+
+        # Mooncake-specific annotation
+        args_dict["bootstrap_port_annotation"] = "sglang.ai/bootstrap-port"
+
+        return cls(**args_dict)
+
+    def _validate_router_args(self):
+        # Validate configuration based on mode
+        if self.pd_disaggregation:
+            # Allow empty URLs even without service discovery to support dynamic worker addition
+            # URLs will be validated separately if provided
+            pass
+
+            # Warn about policy usage in PD mode
+            if self.prefill_policy and self.decode_policy and self.policy:
+                logger.warning(
+                    "Both --prefill-policy and --decode-policy are specified. "
+                    "The main --policy flag will be ignored for PD mode."
+                )
+            elif self.prefill_policy and not self.decode_policy and self.policy:
+                logger.info(
+                    f"Using --prefill-policy '{self.prefill_policy}' for prefill nodes "
+                    f"and --policy '{self.policy}' for decode nodes."
+                )
+            elif self.decode_policy and not self.prefill_policy and self.policy:
+                logger.info(
+                    f"Using --policy '{self.policy}' for prefill nodes "
+                    f"and --decode-policy '{self.decode_policy}' for decode nodes."
+                )
+
+    @staticmethod
+    def _parse_selector(selector_list):
+        if not selector_list:
+            return {}
+
+        # Support `- --selector\n- a=b c=d` case
+        if len(selector_list) == 1 and (" " in selector_list[0]):
+            selector_list = selector_list[0].split(" ")
+
+        selector = {}
+        for item in selector_list:
+            if "=" in item:
+                key, value = item.split("=", 1)
+                selector[key] = value
+        return selector
+
+    @staticmethod
+    def _parse_prefill_urls(prefill_list):
+        """Parse prefill URLs from --prefill arguments.
+
+        Format: --prefill URL [BOOTSTRAP_PORT]
+        Example:
+            --prefill http://prefill1:8080 9000  # With bootstrap port
+            --prefill http://prefill2:8080 none  # Explicitly no bootstrap port
+            --prefill http://prefill3:8080       # Defaults to no bootstrap port
+        """
+        if not prefill_list:
+            return []
+
+        prefill_urls = []
+        for prefill_args in prefill_list:
+
+            url = prefill_args[0]
+
+            # Handle optional bootstrap port
+            if len(prefill_args) >= 2:
+                bootstrap_port_str = prefill_args[1]
+                # Handle 'none' as None
+                if bootstrap_port_str.lower() == "none":
+                    bootstrap_port = None
+                else:
+                    try:
+                        bootstrap_port = int(bootstrap_port_str)
+                    except ValueError:
+                        raise ValueError(
+                            f"Invalid bootstrap port: {bootstrap_port_str}. Must be a number or 'none'"
+                        )
+            else:
+                # No bootstrap port specified, default to None
+                bootstrap_port = None
+
+            prefill_urls.append((url, bootstrap_port))
+
+        return prefill_urls
+
+    @staticmethod
+    def _parse_decode_urls(decode_list):
+        """Parse decode URLs from --decode arguments.
+
+        Format: --decode URL
+        Example: --decode http://decode1:8081 --decode http://decode2:8081
+        """
+        if not decode_list:
+            return []
+
+        # decode_list is a list of single-element lists due to nargs=1
+        return [url[0] for url in decode_list]

sgl-model-gateway/bindings/python/sglang_router/version.py

@@ -0,0 +1 @@
+__version__ = "0.2.3"

sgl-model-gateway/bindings/python/src/lib.rs

@@ -0,0 +1,730 @@
+use pyo3::prelude::*;
+use sgl_model_gateway::*;
+use std::collections::HashMap;
+
+// Define the enums with PyO3 bindings
+#[pyclass(eq)]
+#[derive(Clone, PartialEq, Debug)]
+pub enum PolicyType {
+    Random,
+    RoundRobin,
+    CacheAware,
+    PowerOfTwo,
+    Bucket,
+}
+
+#[pyclass(eq)]
+#[derive(Clone, PartialEq, Debug)]
+pub enum BackendType {
+    Sglang,
+    Openai,
+}
+
+#[pyclass(eq)]
+#[derive(Clone, PartialEq, Debug)]
+pub enum HistoryBackendType {
+    Memory,
+    None,
+    Oracle,
+    Postgres,
+}
+
+#[pyclass]
+#[derive(Clone, PartialEq)]
+pub struct PyOracleConfig {
+    #[pyo3(get, set)]
+    pub wallet_path: Option<String>,
+    #[pyo3(get, set)]
+    pub connect_descriptor: Option<String>,
+    #[pyo3(get, set)]
+    pub username: Option<String>,
+    #[pyo3(get, set)]
+    pub password: Option<String>,
+    #[pyo3(get, set)]
+    pub pool_min: usize,
+    #[pyo3(get, set)]
+    pub pool_max: usize,
+    #[pyo3(get, set)]
+    pub pool_timeout_secs: u64,
+}
+
+impl std::fmt::Debug for PyOracleConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("PyOracleConfig")
+            .field("wallet_path", &self.wallet_path)
+            .field("connect_descriptor", &"<redacted>")
+            .field("username", &self.username)
+            .field("password", &"<redacted>")
+            .field("pool_min", &self.pool_min)
+            .field("pool_max", &self.pool_max)
+            .field("pool_timeout_secs", &self.pool_timeout_secs)
+            .finish()
+    }
+}
+
+#[pymethods]
+impl PyOracleConfig {
+    #[new]
+    #[pyo3(signature = (
+        password = None,
+        username = None,
+        connect_descriptor = None,
+        wallet_path = None,
+        pool_min = 1,
+        pool_max = 16,
+        pool_timeout_secs = 30,
+    ))]
+    fn new(
+        password: Option<String>,
+        username: Option<String>,
+        connect_descriptor: Option<String>,
+        wallet_path: Option<String>,
+        pool_min: usize,
+        pool_max: usize,
+        pool_timeout_secs: u64,
+    ) -> PyResult<Self> {
+        if pool_min == 0 {
+            return Err(pyo3::exceptions::PyValueError::new_err(
+                "pool_min must be at least 1",
+            ));
+        }
+        if pool_max < pool_min {
+            return Err(pyo3::exceptions::PyValueError::new_err(
+                "pool_max must be >= pool_min",
+            ));
+        }
+
+        Ok(PyOracleConfig {
+            wallet_path,
+            connect_descriptor,
+            username,
+            password,
+            pool_min,
+            pool_max,
+            pool_timeout_secs,
+        })
+    }
+}
+
+impl PyOracleConfig {
+    pub fn to_config_oracle(&self) -> config::OracleConfig {
+        config::OracleConfig {
+            wallet_path: self.wallet_path.clone(),
+            connect_descriptor: self.connect_descriptor.clone().unwrap_or_default(),
+            username: self.username.clone().unwrap_or_default(),
+            password: self.password.clone().unwrap_or_default(),
+            pool_min: self.pool_min,
+            pool_max: self.pool_max,
+            pool_timeout_secs: self.pool_timeout_secs,
+        }
+    }
+}
+
+#[pyclass]
+#[derive(Debug, Clone, PartialEq)]
+pub struct PyPostgresConfig {
+    #[pyo3(get, set)]
+    pub db_url: Option<String>,
+
+    #[pyo3(get, set)]
+    pub pool_max: usize,
+}
+
+#[pymethods]
+impl PyPostgresConfig {
+    #[new]
+    #[pyo3(signature = (db_url = None,pool_max = 16,))]
+    fn new(db_url: Option<String>, pool_max: usize) -> PyResult<Self> {
+        Ok(PyPostgresConfig { db_url, pool_max })
+    }
+}
+
+impl PyPostgresConfig {
+    pub fn to_config_postgres(&self) -> config::PostgresConfig {
+        config::PostgresConfig {
+            db_url: self.db_url.clone().unwrap_or_default(),
+            pool_max: self.pool_max,
+        }
+    }
+}
+
+#[pyclass]
+#[derive(Debug, Clone, PartialEq)]
+struct Router {
+    host: String,
+    port: u16,
+    worker_urls: Vec<String>,
+    policy: PolicyType,
+    worker_startup_timeout_secs: u64,
+    worker_startup_check_interval: u64,
+    cache_threshold: f32,
+    balance_abs_threshold: usize,
+    balance_rel_threshold: f32,
+    eviction_interval_secs: u64,
+    max_tree_size: usize,
+    max_payload_size: usize,
+    dp_aware: bool,
+    api_key: Option<String>,
+    log_dir: Option<String>,
+    log_level: Option<String>,
+    service_discovery: bool,
+    selector: HashMap<String, String>,
+    service_discovery_port: u16,
+    service_discovery_namespace: Option<String>,
+    prefill_selector: HashMap<String, String>,
+    decode_selector: HashMap<String, String>,
+    bootstrap_port_annotation: String,
+    prometheus_port: Option<u16>,
+    prometheus_host: Option<String>,
+    request_timeout_secs: u64,
+    request_id_headers: Option<Vec<String>>,
+    pd_disaggregation: bool,
+    bucket_adjust_interval_secs: usize,
+    prefill_urls: Option<Vec<(String, Option<u16>)>>,
+    decode_urls: Option<Vec<String>>,
+    prefill_policy: Option<PolicyType>,
+    decode_policy: Option<PolicyType>,
+    max_concurrent_requests: i32,
+    cors_allowed_origins: Vec<String>,
+    retry_max_retries: u32,
+    retry_initial_backoff_ms: u64,
+    retry_max_backoff_ms: u64,
+    retry_backoff_multiplier: f32,
+    retry_jitter_factor: f32,
+    disable_retries: bool,
+    cb_failure_threshold: u32,
+    cb_success_threshold: u32,
+    cb_timeout_duration_secs: u64,
+    cb_window_duration_secs: u64,
+    disable_circuit_breaker: bool,
+    health_failure_threshold: u32,
+    health_success_threshold: u32,
+    health_check_timeout_secs: u64,
+    health_check_interval_secs: u64,
+    health_check_endpoint: String,
+    enable_igw: bool,
+    queue_size: usize,
+    queue_timeout_secs: u64,
+    rate_limit_tokens_per_second: Option<i32>,
+    connection_mode: core::ConnectionMode,
+    model_path: Option<String>,
+    tokenizer_path: Option<String>,
+    chat_template: Option<String>,
+    tokenizer_cache_enable_l0: bool,
+    tokenizer_cache_l0_max_entries: usize,
+    tokenizer_cache_enable_l1: bool,
+    tokenizer_cache_l1_max_memory: usize,
+    reasoning_parser: Option<String>,
+    tool_call_parser: Option<String>,
+    mcp_config_path: Option<String>,
+    backend: BackendType,
+    history_backend: HistoryBackendType,
+    oracle_config: Option<PyOracleConfig>,
+    postgres_config: Option<PyPostgresConfig>,
+    client_cert_path: Option<String>,
+    client_key_path: Option<String>,
+    ca_cert_paths: Vec<String>,
+}
+
+impl Router {
+    fn determine_connection_mode(worker_urls: &[String]) -> core::ConnectionMode {
+        for url in worker_urls {
+            if url.starts_with("grpc://") || url.starts_with("grpcs://") {
+                return core::ConnectionMode::Grpc { port: None };
+            }
+        }
+        core::ConnectionMode::Http
+    }
+
+    pub fn to_router_config(&self) -> config::ConfigResult<config::RouterConfig> {
+        use config::{
+            DiscoveryConfig, MetricsConfig, PolicyConfig as ConfigPolicyConfig, RoutingMode,
+        };
+
+        let convert_policy = |policy: &PolicyType| -> ConfigPolicyConfig {
+            match policy {
+                PolicyType::Random => ConfigPolicyConfig::Random,
+                PolicyType::RoundRobin => ConfigPolicyConfig::RoundRobin,
+                PolicyType::CacheAware => ConfigPolicyConfig::CacheAware {
+                    cache_threshold: self.cache_threshold,
+                    balance_abs_threshold: self.balance_abs_threshold,
+                    balance_rel_threshold: self.balance_rel_threshold,
+                    eviction_interval_secs: self.eviction_interval_secs,
+                    max_tree_size: self.max_tree_size,
+                },
+                PolicyType::PowerOfTwo => ConfigPolicyConfig::PowerOfTwo {
+                    load_check_interval_secs: 5,
+                },
+                PolicyType::Bucket => ConfigPolicyConfig::Bucket {
+                    balance_abs_threshold: self.balance_abs_threshold,
+                    balance_rel_threshold: self.balance_rel_threshold,
+                    bucket_adjust_interval_secs: self.bucket_adjust_interval_secs,
+                },
+            }
+        };
+
+        let mode = if self.enable_igw {
+            RoutingMode::Regular {
+                worker_urls: vec![],
+            }
+        } else if matches!(self.backend, BackendType::Openai) {
+            RoutingMode::OpenAI {
+                worker_urls: self.worker_urls.clone(),
+            }
+        } else if self.pd_disaggregation {
+            RoutingMode::PrefillDecode {
+                prefill_urls: self.prefill_urls.clone().unwrap_or_default(),
+                decode_urls: self.decode_urls.clone().unwrap_or_default(),
+                prefill_policy: self.prefill_policy.as_ref().map(convert_policy),
+                decode_policy: self.decode_policy.as_ref().map(convert_policy),
+            }
+        } else {
+            RoutingMode::Regular {
+                worker_urls: self.worker_urls.clone(),
+            }
+        };
+
+        let policy = convert_policy(&self.policy);
+
+        let discovery = if self.service_discovery {
+            Some(DiscoveryConfig {
+                enabled: true,
+                namespace: self.service_discovery_namespace.clone(),
+                port: self.service_discovery_port,
+                check_interval_secs: 60,
+                selector: self.selector.clone(),
+                prefill_selector: self.prefill_selector.clone(),
+                decode_selector: self.decode_selector.clone(),
+                bootstrap_port_annotation: self.bootstrap_port_annotation.clone(),
+            })
+        } else {
+            None
+        };
+
+        let metrics = match (self.prometheus_port, self.prometheus_host.as_ref()) {
+            (Some(port), Some(host)) => Some(MetricsConfig {
+                port,
+                host: host.clone(),
+            }),
+            _ => None,
+        };
+
+        let history_backend = match self.history_backend {
+            HistoryBackendType::Memory => config::HistoryBackend::Memory,
+            HistoryBackendType::None => config::HistoryBackend::None,
+            HistoryBackendType::Oracle => config::HistoryBackend::Oracle,
+            HistoryBackendType::Postgres => config::HistoryBackend::Postgres,
+        };
+
+        let oracle = if matches!(self.history_backend, HistoryBackendType::Oracle) {
+            self.oracle_config
+                .as_ref()
+                .map(|cfg| cfg.to_config_oracle())
+        } else {
+            None
+        };
+
+        let postgres_config = if matches!(self.history_backend, HistoryBackendType::Postgres) {
+            self.postgres_config
+                .as_ref()
+                .map(|cfg| cfg.to_config_postgres())
+        } else {
+            None
+        };
+
+        config::RouterConfig::builder()
+            .mode(mode)
+            .policy(policy)
+            .host(&self.host)
+            .port(self.port)
+            .connection_mode(self.connection_mode.clone())
+            .max_payload_size(self.max_payload_size)
+            .request_timeout_secs(self.request_timeout_secs)
+            .worker_startup_timeout_secs(self.worker_startup_timeout_secs)
+            .worker_startup_check_interval_secs(self.worker_startup_check_interval)
+            .max_concurrent_requests(self.max_concurrent_requests)
+            .queue_size(self.queue_size)
+            .queue_timeout_secs(self.queue_timeout_secs)
+            .cors_allowed_origins(self.cors_allowed_origins.clone())
+            .retry_config(config::RetryConfig {
+                max_retries: self.retry_max_retries,
+                initial_backoff_ms: self.retry_initial_backoff_ms,
+                max_backoff_ms: self.retry_max_backoff_ms,
+                backoff_multiplier: self.retry_backoff_multiplier,
+                jitter_factor: self.retry_jitter_factor,
+            })
+            .circuit_breaker_config(config::CircuitBreakerConfig {
+                failure_threshold: self.cb_failure_threshold,
+                success_threshold: self.cb_success_threshold,
+                timeout_duration_secs: self.cb_timeout_duration_secs,
+                window_duration_secs: self.cb_window_duration_secs,
+            })
+            .health_check_config(config::HealthCheckConfig {
+                failure_threshold: self.health_failure_threshold,
+                success_threshold: self.health_success_threshold,
+                timeout_secs: self.health_check_timeout_secs,
+                check_interval_secs: self.health_check_interval_secs,
+                endpoint: self.health_check_endpoint.clone(),
+            })
+            .tokenizer_cache(config::TokenizerCacheConfig {
+                enable_l0: self.tokenizer_cache_enable_l0,
+                l0_max_entries: self.tokenizer_cache_l0_max_entries,
+                enable_l1: self.tokenizer_cache_enable_l1,
+                l1_max_memory: self.tokenizer_cache_l1_max_memory,
+            })
+            .history_backend(history_backend)
+            .maybe_api_key(self.api_key.as_ref())
+            .maybe_discovery(discovery)
+            .maybe_metrics(metrics)
+            .maybe_log_dir(self.log_dir.as_ref())
+            .maybe_log_level(self.log_level.as_ref())
+            .maybe_request_id_headers(self.request_id_headers.clone())
+            .maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second)
+            .maybe_model_path(self.model_path.as_ref())
+            .maybe_tokenizer_path(self.tokenizer_path.as_ref())
+            .maybe_chat_template(self.chat_template.as_ref())
+            .maybe_oracle(oracle)
+            .maybe_postgres(postgres_config)
+            .maybe_reasoning_parser(self.reasoning_parser.as_ref())
+            .maybe_tool_call_parser(self.tool_call_parser.as_ref())
+            .maybe_mcp_config_path(self.mcp_config_path.as_ref())
+            .dp_aware(self.dp_aware)
+            .retries(!self.disable_retries)
+            .circuit_breaker(!self.disable_circuit_breaker)
+            .igw(self.enable_igw)
+            .maybe_client_cert_and_key(
+                self.client_cert_path.as_ref(),
+                self.client_key_path.as_ref(),
+            )
+            .add_ca_certificates(self.ca_cert_paths.clone())
+            .build()
+    }
+}
+
+#[pymethods]
+impl Router {
+    #[new]
+    #[pyo3(signature = (
+        worker_urls,
+        policy = PolicyType::RoundRobin,
+        host = String::from("0.0.0.0"),
+        port = 3001,
+        worker_startup_timeout_secs = 600,
+        worker_startup_check_interval = 30,
+        cache_threshold = 0.3,
+        balance_abs_threshold = 64,
+        balance_rel_threshold = 1.5,
+        eviction_interval_secs = 120,
+        max_tree_size = 2usize.pow(26),
+        max_payload_size = 512 * 1024 * 1024,
+        dp_aware = false,
+        api_key = None,
+        log_dir = None,
+        log_level = None,
+        service_discovery = false,
+        selector = HashMap::new(),
+        service_discovery_port = 80,
+        service_discovery_namespace = None,
+        prefill_selector = HashMap::new(),
+        decode_selector = HashMap::new(),
+        bootstrap_port_annotation = String::from("sglang.ai/bootstrap-port"),
+        prometheus_port = None,
+        prometheus_host = None,
+        request_timeout_secs = 1800,
+        request_id_headers = None,
+        pd_disaggregation = false,
+        bucket_adjust_interval_secs = 5,
+        prefill_urls = None,
+        decode_urls = None,
+        prefill_policy = None,
+        decode_policy = None,
+        max_concurrent_requests = -1,
+        cors_allowed_origins = vec![],
+        retry_max_retries = 5,
+        retry_initial_backoff_ms = 50,
+        retry_max_backoff_ms = 30_000,
+        retry_backoff_multiplier = 1.5,
+        retry_jitter_factor = 0.2,
+        disable_retries = false,
+        cb_failure_threshold = 10,
+        cb_success_threshold = 3,
+        cb_timeout_duration_secs = 60,
+        cb_window_duration_secs = 120,
+        disable_circuit_breaker = false,
+        health_failure_threshold = 3,
+        health_success_threshold = 2,
+        health_check_timeout_secs = 5,
+        health_check_interval_secs = 60,
+        health_check_endpoint = String::from("/health"),
+        enable_igw = false,
+        queue_size = 100,
+        queue_timeout_secs = 60,
+        rate_limit_tokens_per_second = None,
+        model_path = None,
+        tokenizer_path = None,
+        chat_template = None,
+        tokenizer_cache_enable_l0 = false,
+        tokenizer_cache_l0_max_entries = 10000,
+        tokenizer_cache_enable_l1 = false,
+        tokenizer_cache_l1_max_memory = 52428800,
+        reasoning_parser = None,
+        tool_call_parser = None,
+        mcp_config_path = None,
+        backend = BackendType::Sglang,
+        history_backend = HistoryBackendType::Memory,
+        oracle_config = None,
+        postgres_config = None,
+        client_cert_path = None,
+        client_key_path = None,
+        ca_cert_paths = vec![],
+    ))]
+    #[allow(clippy::too_many_arguments)]
+    fn new(
+        worker_urls: Vec<String>,
+        policy: PolicyType,
+        host: String,
+        port: u16,
+        worker_startup_timeout_secs: u64,
+        worker_startup_check_interval: u64,
+        cache_threshold: f32,
+        balance_abs_threshold: usize,
+        balance_rel_threshold: f32,
+        eviction_interval_secs: u64,
+        max_tree_size: usize,
+        max_payload_size: usize,
+        dp_aware: bool,
+        api_key: Option<String>,
+        log_dir: Option<String>,
+        log_level: Option<String>,
+        service_discovery: bool,
+        selector: HashMap<String, String>,
+        service_discovery_port: u16,
+        service_discovery_namespace: Option<String>,
+        prefill_selector: HashMap<String, String>,
+        decode_selector: HashMap<String, String>,
+        bootstrap_port_annotation: String,
+        prometheus_port: Option<u16>,
+        prometheus_host: Option<String>,
+        request_timeout_secs: u64,
+        request_id_headers: Option<Vec<String>>,
+        pd_disaggregation: bool,
+        bucket_adjust_interval_secs: usize,
+        prefill_urls: Option<Vec<(String, Option<u16>)>>,
+        decode_urls: Option<Vec<String>>,
+        prefill_policy: Option<PolicyType>,
+        decode_policy: Option<PolicyType>,
+        max_concurrent_requests: i32,
+        cors_allowed_origins: Vec<String>,
+        retry_max_retries: u32,
+        retry_initial_backoff_ms: u64,
+        retry_max_backoff_ms: u64,
+        retry_backoff_multiplier: f32,
+        retry_jitter_factor: f32,
+        disable_retries: bool,
+        cb_failure_threshold: u32,
+        cb_success_threshold: u32,
+        cb_timeout_duration_secs: u64,
+        cb_window_duration_secs: u64,
+        disable_circuit_breaker: bool,
+        health_failure_threshold: u32,
+        health_success_threshold: u32,
+        health_check_timeout_secs: u64,
+        health_check_interval_secs: u64,
+        health_check_endpoint: String,
+        enable_igw: bool,
+        queue_size: usize,
+        queue_timeout_secs: u64,
+        rate_limit_tokens_per_second: Option<i32>,
+        model_path: Option<String>,
+        tokenizer_path: Option<String>,
+        chat_template: Option<String>,
+        tokenizer_cache_enable_l0: bool,
+        tokenizer_cache_l0_max_entries: usize,
+        tokenizer_cache_enable_l1: bool,
+        tokenizer_cache_l1_max_memory: usize,
+        reasoning_parser: Option<String>,
+        tool_call_parser: Option<String>,
+        mcp_config_path: Option<String>,
+        backend: BackendType,
+        history_backend: HistoryBackendType,
+        oracle_config: Option<PyOracleConfig>,
+        postgres_config: Option<PyPostgresConfig>,
+        client_cert_path: Option<String>,
+        client_key_path: Option<String>,
+        ca_cert_paths: Vec<String>,
+    ) -> PyResult<Self> {
+        let mut all_urls = worker_urls.clone();
+
+        if let Some(ref prefill_urls) = prefill_urls {
+            for (url, _) in prefill_urls {
+                all_urls.push(url.clone());
+            }
+        }
+
+        if let Some(ref decode_urls) = decode_urls {
+            all_urls.extend(decode_urls.clone());
+        }
+
+        let connection_mode = Self::determine_connection_mode(&all_urls);
+
+        Ok(Router {
+            host,
+            port,
+            worker_urls,
+            policy,
+            worker_startup_timeout_secs,
+            worker_startup_check_interval,
+            cache_threshold,
+            balance_abs_threshold,
+            balance_rel_threshold,
+            eviction_interval_secs,
+            max_tree_size,
+            max_payload_size,
+            dp_aware,
+            api_key,
+            log_dir,
+            log_level,
+            service_discovery,
+            selector,
+            service_discovery_port,
+            service_discovery_namespace,
+            prefill_selector,
+            decode_selector,
+            bootstrap_port_annotation,
+            prometheus_port,
+            prometheus_host,
+            request_timeout_secs,
+            request_id_headers,
+            pd_disaggregation,
+            bucket_adjust_interval_secs,
+            prefill_urls,
+            decode_urls,
+            prefill_policy,
+            decode_policy,
+            max_concurrent_requests,
+            cors_allowed_origins,
+            retry_max_retries,
+            retry_initial_backoff_ms,
+            retry_max_backoff_ms,
+            retry_backoff_multiplier,
+            retry_jitter_factor,
+            disable_retries,
+            cb_failure_threshold,
+            cb_success_threshold,
+            cb_timeout_duration_secs,
+            cb_window_duration_secs,
+            disable_circuit_breaker,
+            health_failure_threshold,
+            health_success_threshold,
+            health_check_timeout_secs,
+            health_check_interval_secs,
+            health_check_endpoint,
+            enable_igw,
+            queue_size,
+            queue_timeout_secs,
+            rate_limit_tokens_per_second,
+            connection_mode,
+            model_path,
+            tokenizer_path,
+            chat_template,
+            tokenizer_cache_enable_l0,
+            tokenizer_cache_l0_max_entries,
+            tokenizer_cache_enable_l1,
+            tokenizer_cache_l1_max_memory,
+            reasoning_parser,
+            tool_call_parser,
+            mcp_config_path,
+            backend,
+            history_backend,
+            oracle_config,
+            postgres_config,
+            client_cert_path,
+            client_key_path,
+            ca_cert_paths,
+        })
+    }
+
+    fn start(&self) -> PyResult<()> {
+        use metrics::PrometheusConfig;
+
+        let router_config = self.to_router_config().map_err(|e| {
+            pyo3::exceptions::PyValueError::new_err(format!("Configuration error: {}", e))
+        })?;
+
+        router_config.validate().map_err(|e| {
+            pyo3::exceptions::PyValueError::new_err(format!(
+                "Configuration validation failed: {}",
+                e
+            ))
+        })?;
+
+        let service_discovery_config = if self.service_discovery {
+            Some(service_discovery::ServiceDiscoveryConfig {
+                enabled: true,
+                selector: self.selector.clone(),
+                check_interval: std::time::Duration::from_secs(60),
+                port: self.service_discovery_port,
+                namespace: self.service_discovery_namespace.clone(),
+                pd_mode: self.pd_disaggregation,
+                prefill_selector: self.prefill_selector.clone(),
+                decode_selector: self.decode_selector.clone(),
+                bootstrap_port_annotation: self.bootstrap_port_annotation.clone(),
+            })
+        } else {
+            None
+        };
+
+        let prometheus_config = Some(PrometheusConfig {
+            port: self.prometheus_port.unwrap_or(29000),
+            host: self
+                .prometheus_host
+                .clone()
+                .unwrap_or_else(|| "127.0.0.1".to_string()),
+        });
+
+        let runtime = tokio::runtime::Runtime::new()
+            .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?;
+
+        runtime.block_on(async move {
+            server::startup(server::ServerConfig {
+                host: self.host.clone(),
+                port: self.port,
+                router_config,
+                max_payload_size: self.max_payload_size,
+                log_dir: self.log_dir.clone(),
+                log_level: self.log_level.clone(),
+                service_discovery_config,
+                prometheus_config,
+                request_timeout_secs: self.request_timeout_secs,
+                request_id_headers: self.request_id_headers.clone(),
+            })
+            .await
+            .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))
+        })
+    }
+}
+
+/// Get simple version string (default for --version)
+#[pyfunction]
+fn get_version_string() -> String {
+    version::get_version_string()
+}
+
+/// Get verbose version information string with full build details (for --version-verbose)
+#[pyfunction]
+fn get_verbose_version_string() -> String {
+    version::get_verbose_version_string()
+}
+
+#[pymodule]
+fn sglang_router_rs(m: &Bound<'_, PyModule>) -> PyResult<()> {
+    m.add_class::<PolicyType>()?;
+    m.add_class::<BackendType>()?;
+    m.add_class::<HistoryBackendType>()?;
+    m.add_class::<PyOracleConfig>()?;
+    m.add_class::<PyPostgresConfig>()?;
+    m.add_class::<Router>()?;
+    m.add_function(wrap_pyfunction!(get_version_string, m)?)?;
+    m.add_function(wrap_pyfunction!(get_verbose_version_string, m)?)?;
+    Ok(())
+}