[SMG-GO] implement a Go SGLang Model Gateway - OpenAI Compatible API Server (#14770)

2025-12-10 06:03:28 -08:00
parent 12b7a4fab0
commit 766476f52a
38 changed files with 9115 additions and 271 deletions
@@ -0,0 +1,126 @@
+// Package ffi provides Go bindings for SGLang's Rust FFI (Foreign Function Interface).
+package ffi
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// BatchPostprocessor handles batch postprocessing of stream chunks to reduce FFI overhead
+type BatchPostprocessor struct {
+	converter     *GrpcResponseConverterHandle
+	buffer        []string
+	batchSize     int
+	flushInterval time.Duration
+	lastFlush     time.Time
+	timer         *time.Timer
+}
+
+// NewBatchPostprocessor creates a new batch postprocessor
+func NewBatchPostprocessor(converter *GrpcResponseConverterHandle, batchSize int, flushInterval time.Duration) *BatchPostprocessor {
+	if batchSize <= 0 {
+		batchSize = 1
+	}
+	if flushInterval < 0 {
+		flushInterval = 0
+	}
+
+	return &BatchPostprocessor{
+		converter:     converter,
+		buffer:        make([]string, 0, batchSize),
+		batchSize:     batchSize,
+		flushInterval: flushInterval,
+		lastFlush:     time.Now(),
+	}
+}
+
+// AddChunk adds a chunk to the buffer and processes if batch is full
+func (b *BatchPostprocessor) AddChunk(chunkJSON string) (results []string, shouldFlush bool, err error) {
+	if b.batchSize == 1 {
+		openaiJSON, _, err := PostprocessStreamChunk(b.converter, chunkJSON)
+		if err != nil {
+			return nil, false, err
+		}
+		return []string{openaiJSON}, false, nil
+	}
+
+	b.buffer = append(b.buffer, chunkJSON)
+	shouldProcess := len(b.buffer) >= b.batchSize
+	shouldFlushTimeout := b.flushInterval > 0 && time.Since(b.lastFlush) >= b.flushInterval
+
+	if shouldProcess || shouldFlushTimeout {
+		return b.processBatch()
+	}
+
+	return nil, false, nil
+}
+
+// Flush processes any remaining chunks in the buffer
+func (b *BatchPostprocessor) Flush() (results []string, err error) {
+	if len(b.buffer) == 0 {
+		return nil, nil
+	}
+
+	res, _, err := b.processBatch()
+	return res, err
+}
+
+// processBatch processes the current buffer and returns results
+func (b *BatchPostprocessor) processBatch() (results []string, shouldFlush bool, err error) {
+	if len(b.buffer) == 0 {
+		return nil, false, nil
+	}
+
+	var sb strings.Builder
+	sb.Grow(len(b.buffer) * 200)
+	sb.WriteString(`[`)
+	for i, chunkJSONStr := range b.buffer {
+		if i > 0 {
+			sb.WriteString(`,`)
+		}
+		sb.WriteString(chunkJSONStr)
+	}
+	sb.WriteString(`]`)
+	bufferJSON := sb.String()
+
+	resultJSON, _, err := PostprocessStreamChunksBatch(
+		b.converter,
+		bufferJSON,
+		b.batchSize*2,
+	)
+	if err != nil {
+		return nil, false, fmt.Errorf("batch postprocessing failed: %w", err)
+	}
+
+	var resultArray []json.RawMessage
+	if err := json.Unmarshal([]byte(resultJSON), &resultArray); err != nil {
+		return nil, false, fmt.Errorf("failed to unmarshal results array: %w", err)
+	}
+
+	resultStrings := make([]string, 0, len(resultArray))
+	for _, rawMsg := range resultArray {
+		resultStrings = append(resultStrings, string(rawMsg))
+	}
+
+	b.buffer = b.buffer[:0]
+	b.lastFlush = time.Now()
+
+	if b.timer != nil {
+		b.timer.Stop()
+		b.timer = nil
+	}
+
+	return resultStrings, false, nil
+}
+
+// Reset clears the buffer and resets the postprocessor state
+func (b *BatchPostprocessor) Reset() {
+	b.buffer = b.buffer[:0]
+	b.lastFlush = time.Now()
+	if b.timer != nil {
+		b.timer.Stop()
+		b.timer = nil
+	}
+}
@@ -11,7 +11,7 @@
 package ffi

 /*
-#cgo LDFLAGS: -lsglang_router_rs -ldl
+#cgo LDFLAGS: -lsgl_model_gateway_go -ldl
 #include <stdlib.h>
 #include <stdint.h>

@@ -0,0 +1,275 @@
+package ffi
+
+/*
+#cgo LDFLAGS: -lsgl_model_gateway_go -ldl
+#include <stdlib.h>
+#include <stdint.h>
+
+// Error codes (must match client.go)
+typedef enum {
+    SGL_ERROR_SUCCESS = 0,
+    SGL_ERROR_INVALID_ARGUMENT = 1,
+    SGL_ERROR_TOKENIZATION_ERROR = 2,
+    SGL_ERROR_PARSING_ERROR = 3,
+    SGL_ERROR_MEMORY_ERROR = 4,
+    SGL_ERROR_UNKNOWN = 99
+} SglErrorCode;
+
+// Opaque handles
+typedef void* TokenizerHandle;
+typedef void* GrpcResponseConverterHandle;
+
+// Converter functions
+GrpcResponseConverterHandle* sgl_grpc_response_converter_create(
+    TokenizerHandle* tokenizer_handle,
+    const char* model,
+    const char* request_id,
+    const char* tools_json,
+    const char* tool_choice_json,
+    const char* stop,
+    const char* stop_token_ids,
+    int skip_special_tokens,
+    int initial_prompt_tokens,
+    char** error_out
+);
+
+void sgl_grpc_response_converter_free(GrpcResponseConverterHandle* handle);
+
+// Tokenizer functions
+TokenizerHandle* sgl_tokenizer_create_from_file(const char* tokenizer_path, char** error_out);
+void sgl_tokenizer_free(TokenizerHandle* handle);
+
+// Memory management
+void sgl_free_string(char* s);
+*/
+import "C"
+
+import (
+	"fmt"
+	"unsafe"
+)
+
+// CreateGrpcResponseConverter creates a gRPC response converter handle
+// This function creates a new tokenizer handle each time (for backward compatibility)
+// For better performance, use CreateGrpcResponseConverterWithTokenizer with a cached tokenizer
+func CreateGrpcResponseConverter(
+	tokenizerPath string,
+	model string,
+	requestID string,
+	toolsJSON string,
+	toolChoiceJSON string,
+	stopJSON string,
+	stopTokenIDs []uint32,
+	skipSpecialTokens bool,
+	initialPromptTokens int32,
+) (*GrpcResponseConverterHandle, error) {
+	// Create tokenizer handle
+	tokenizerHandle, err := createTokenizerHandle(tokenizerPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create tokenizer handle: %w", err)
+	}
+	defer C.sgl_tokenizer_free(tokenizerHandle)
+
+	return createGrpcResponseConverterWithTokenizerHandle(
+		tokenizerHandle,
+		model,
+		requestID,
+		toolsJSON,
+		toolChoiceJSON,
+		stopJSON,
+		stopTokenIDs,
+		skipSpecialTokens,
+		initialPromptTokens,
+	)
+}
+
+// CreateGrpcResponseConverterWithTokenizer creates a gRPC response converter handle using a cached tokenizer
+// This is more efficient as it reuses the tokenizer instead of creating a new one each time
+func CreateGrpcResponseConverterWithTokenizer(
+	tokenizerHandle *TokenizerHandle,
+	model string,
+	requestID string,
+	toolsJSON string,
+	toolChoiceJSON string,
+	stopJSON string,
+	stopTokenIDs []uint32,
+	skipSpecialTokens bool,
+	initialPromptTokens int32,
+) (*GrpcResponseConverterHandle, error) {
+	if tokenizerHandle == nil || tokenizerHandle.handle == nil {
+		return nil, fmt.Errorf("invalid tokenizer handle")
+	}
+
+	return createGrpcResponseConverterWithTokenizerHandle(
+		tokenizerHandle.handle,
+		model,
+		requestID,
+		toolsJSON,
+		toolChoiceJSON,
+		stopJSON,
+		stopTokenIDs,
+		skipSpecialTokens,
+		initialPromptTokens,
+	)
+}
+
+// createGrpcResponseConverterWithTokenizerHandle is the internal implementation
+func createGrpcResponseConverterWithTokenizerHandle(
+	tokenizerHandle *C.TokenizerHandle,
+	model string,
+	requestID string,
+	toolsJSON string,
+	toolChoiceJSON string,
+	stopJSON string,
+	stopTokenIDs []uint32,
+	skipSpecialTokens bool,
+	initialPromptTokens int32,
+) (*GrpcResponseConverterHandle, error) {
+
+	// Convert strings to C strings
+	modelC := C.CString(model)
+	defer C.free(unsafe.Pointer(modelC))
+
+	requestIDC := C.CString(requestID)
+	defer C.free(unsafe.Pointer(requestIDC))
+
+	var toolsJSONC *C.char
+	if toolsJSON != "" {
+		toolsJSONC = C.CString(toolsJSON)
+		defer C.free(unsafe.Pointer(toolsJSONC))
+	}
+
+	var toolChoiceJSONC *C.char
+	if toolChoiceJSON != "" {
+		toolChoiceJSONC = C.CString(toolChoiceJSON)
+		defer C.free(unsafe.Pointer(toolChoiceJSONC))
+	}
+
+	var stopJSONC *C.char
+	if stopJSON != "" {
+		stopJSONC = C.CString(stopJSON)
+		defer C.free(unsafe.Pointer(stopJSONC))
+	}
+
+	// Convert stop_token_ids to JSON string
+	stopTokenIDsJSON := ""
+	if len(stopTokenIDs) > 0 {
+		stopTokenIDsJSON = fmt.Sprintf("[%d", stopTokenIDs[0])
+		for i := 1; i < len(stopTokenIDs); i++ {
+			stopTokenIDsJSON += fmt.Sprintf(",%d", stopTokenIDs[i])
+		}
+		stopTokenIDsJSON += "]"
+	}
+
+	var stopTokenIDsJSONC *C.char
+	if stopTokenIDsJSON != "" {
+		stopTokenIDsJSONC = C.CString(stopTokenIDsJSON)
+		defer C.free(unsafe.Pointer(stopTokenIDsJSONC))
+	}
+
+	var errorOut *C.char
+	skipSpecialTokensC := C.int(0)
+	if skipSpecialTokens {
+		skipSpecialTokensC = C.int(1)
+	}
+
+	initialPromptTokensC := C.int(initialPromptTokens)
+
+	converterHandle := C.sgl_grpc_response_converter_create(
+		tokenizerHandle,
+		modelC,
+		requestIDC,
+		toolsJSONC,
+		toolChoiceJSONC,
+		stopJSONC,
+		stopTokenIDsJSONC,
+		skipSpecialTokensC,
+		initialPromptTokensC,
+		&errorOut,
+	)
+
+	if converterHandle == nil {
+		errorMsg := ""
+		if errorOut != nil {
+			errorMsg = C.GoString(errorOut)
+			C.sgl_free_string(errorOut)
+		}
+		if errorMsg == "" {
+			errorMsg = "failed to create converter handle"
+		}
+		return nil, fmt.Errorf("%s", errorMsg)
+	}
+
+	return &GrpcResponseConverterHandle{
+		handle: converterHandle,
+	}, nil
+}
+
+// FreeGrpcResponseConverter frees a gRPC response converter handle
+func FreeGrpcResponseConverter(handle *GrpcResponseConverterHandle) {
+	if handle != nil && handle.handle != nil {
+		C.sgl_grpc_response_converter_free(handle.handle)
+		handle.handle = nil
+	}
+}
+
+// TokenizerHandle wraps the Rust tokenizer FFI handle
+type TokenizerHandle struct {
+	handle *C.TokenizerHandle
+}
+
+// CreateTokenizerHandle creates a tokenizer handle (exported for caching)
+func CreateTokenizerHandle(tokenizerPath string) (*TokenizerHandle, error) {
+	tokenizerPathC := C.CString(tokenizerPath)
+	defer C.free(unsafe.Pointer(tokenizerPathC))
+
+	var errorOut *C.char
+	tokenizerHandle := C.sgl_tokenizer_create_from_file(tokenizerPathC, &errorOut)
+
+	if tokenizerHandle == nil {
+		errorMsg := ""
+		if errorOut != nil {
+			errorMsg = C.GoString(errorOut)
+			C.sgl_free_string(errorOut)
+		}
+		if errorMsg == "" {
+			errorMsg = "failed to create tokenizer handle"
+		}
+		return nil, fmt.Errorf("%s", errorMsg)
+	}
+
+	return &TokenizerHandle{
+		handle: tokenizerHandle,
+	}, nil
+}
+
+// FreeTokenizerHandle frees a tokenizer handle
+func FreeTokenizerHandle(handle *TokenizerHandle) {
+	if handle != nil && handle.handle != nil {
+		C.sgl_tokenizer_free(handle.handle)
+		handle.handle = nil
+	}
+}
+
+// createTokenizerHandle creates a tokenizer handle (helper function, internal use)
+func createTokenizerHandle(tokenizerPath string) (*C.TokenizerHandle, error) {
+	tokenizerPathC := C.CString(tokenizerPath)
+	defer C.free(unsafe.Pointer(tokenizerPathC))
+
+	var errorOut *C.char
+	tokenizerHandle := C.sgl_tokenizer_create_from_file(tokenizerPathC, &errorOut)
+
+	if tokenizerHandle == nil {
+		errorMsg := ""
+		if errorOut != nil {
+			errorMsg = C.GoString(errorOut)
+			C.sgl_free_string(errorOut)
+		}
+		if errorMsg == "" {
+			errorMsg = "failed to create tokenizer handle"
+		}
+		return nil, fmt.Errorf("%s", errorMsg)
+	}
+
+	return tokenizerHandle, nil
+}
@@ -0,0 +1,156 @@
+// Package ffi provides Go bindings for SGLang's Rust FFI (Foreign Function Interface).
+package ffi
+
+/*
+#cgo LDFLAGS: -lsgl_model_gateway_go -ldl
+#include <stdlib.h>
+#include <stdint.h>
+
+// Error codes (must match client.go)
+typedef enum {
+    SGL_ERROR_SUCCESS = 0,
+    SGL_ERROR_INVALID_ARGUMENT = 1,
+    SGL_ERROR_TOKENIZATION_ERROR = 2,
+    SGL_ERROR_PARSING_ERROR = 3,
+    SGL_ERROR_MEMORY_ERROR = 4,
+    SGL_ERROR_UNKNOWN = 99
+} SglErrorCode;
+
+// Opaque handle (must match grpc_converter.go)
+typedef void* GrpcResponseConverterHandle;
+
+// Postprocessor functions
+SglErrorCode sgl_postprocess_stream_chunk(
+    GrpcResponseConverterHandle* converter_handle,
+    const char* proto_chunk_json,
+    char** openai_json_out,
+    int* is_done_out,
+    char** error_out
+);
+
+SglErrorCode sgl_postprocess_stream_chunks_batch(
+    GrpcResponseConverterHandle* converter_handle,
+    const char* proto_chunks_json_array,
+    int max_chunks,
+    char** openai_chunks_json_array_out,
+    int* chunks_count_out,
+    char** error_out
+);
+
+// Memory management
+void sgl_free_string(char* s);
+*/
+import "C"
+
+import (
+	"fmt"
+	"unsafe"
+)
+
+// GrpcResponseConverterHandle wraps the Rust gRPC response converter FFI handle
+type GrpcResponseConverterHandle struct {
+	handle *C.GrpcResponseConverterHandle
+}
+
+// PostprocessStreamChunk postprocesses a gRPC stream chunk to OpenAI format
+//
+// This function:
+// 1. Parses the proto chunk from JSON
+// 2. Converts it to OpenAI format using the converter handle
+// 3. Returns the OpenAI format JSON
+//
+// Returns the OpenAI format JSON, is_done flag, and any error.
+func PostprocessStreamChunk(converterHandle *GrpcResponseConverterHandle, protoChunkJSON string) (openaiJSON string, isDone bool, err error) {
+	if converterHandle == nil || converterHandle.handle == nil {
+		return "", false, fmt.Errorf("invalid converter handle")
+	}
+
+	protoChunkJSONC := C.CString(protoChunkJSON)
+	defer C.free(unsafe.Pointer(protoChunkJSONC))
+
+	var openaiJSONOut *C.char
+	var isDoneOut C.int
+	var errorOut *C.char
+
+	errorCode := C.sgl_postprocess_stream_chunk(
+		converterHandle.handle,
+		protoChunkJSONC,
+		&openaiJSONOut,
+		&isDoneOut,
+		&errorOut,
+	)
+
+	if errorCode != C.SGL_ERROR_SUCCESS {
+		errorMsg := ""
+		if errorOut != nil {
+			errorMsg = C.GoString(errorOut)
+			C.sgl_free_string(errorOut)
+		}
+		return "", false, fmt.Errorf("postprocessing failed: %s", errorMsg)
+	}
+
+	openaiJSON = C.GoString(openaiJSONOut)
+	isDone = isDoneOut != 0
+
+	// Free the C string allocated by Rust
+	if openaiJSONOut != nil {
+		C.sgl_free_string(openaiJSONOut)
+	}
+
+	return openaiJSON, isDone, nil
+}
+
+// PostprocessStreamChunksBatch postprocesses multiple gRPC stream chunks in batch
+//
+// This function processes multiple chunks in a single FFI call, significantly reducing
+// FFI overhead in streaming scenarios.
+//
+// Arguments:
+// - converterHandle: Converter handle
+// - protoChunksJSONArray: JSON array string of proto chunks
+// - maxChunks: Maximum number of chunks to process (for safety, typically 10-20)
+//
+// Returns:
+// - openaiChunksJSONArray: JSON array of OpenAI format chunks
+// - chunksCount: Number of processed chunks
+// - error: Any error that occurred
+func PostprocessStreamChunksBatch(converterHandle *GrpcResponseConverterHandle, protoChunksJSONArray string, maxChunks int) (openaiChunksJSONArray string, chunksCount int, err error) {
+	if converterHandle == nil || converterHandle.handle == nil {
+		return "", 0, fmt.Errorf("invalid converter handle")
+	}
+
+	protoChunksJSONArrayC := C.CString(protoChunksJSONArray)
+	defer C.free(unsafe.Pointer(protoChunksJSONArrayC))
+
+	var openaiChunksJSONArrayOut *C.char
+	var chunksCountOut C.int
+	var errorOut *C.char
+
+	errorCode := C.sgl_postprocess_stream_chunks_batch(
+		converterHandle.handle,
+		protoChunksJSONArrayC,
+		C.int(maxChunks),
+		&openaiChunksJSONArrayOut,
+		&chunksCountOut,
+		&errorOut,
+	)
+
+	if errorCode != C.SGL_ERROR_SUCCESS {
+		errorMsg := ""
+		if errorOut != nil {
+			errorMsg = C.GoString(errorOut)
+			C.sgl_free_string(errorOut)
+		}
+		return "", 0, fmt.Errorf("batch postprocessing failed: %s", errorMsg)
+	}
+
+	openaiChunksJSONArray = C.GoString(openaiChunksJSONArrayOut)
+	chunksCount = int(chunksCountOut)
+
+	// Free the C string allocated by Rust
+	if openaiChunksJSONArrayOut != nil {
+		C.sgl_free_string(openaiChunksJSONArrayOut)
+	}
+
+	return openaiChunksJSONArray, chunksCount, nil
+}
@@ -0,0 +1,246 @@
+// Package ffi provides Go bindings for SGLang's Rust FFI (Foreign Function Interface).
+package ffi
+
+/*
+#cgo LDFLAGS: -lsgl_model_gateway_go -ldl
+#include <stdlib.h>
+#include <stdint.h>
+
+// Error codes (must match client.go)
+typedef enum {
+    SGL_ERROR_SUCCESS = 0,
+    SGL_ERROR_INVALID_ARGUMENT = 1,
+    SGL_ERROR_TOKENIZATION_ERROR = 2,
+    SGL_ERROR_PARSING_ERROR = 3,
+    SGL_ERROR_MEMORY_ERROR = 4,
+    SGL_ERROR_UNKNOWN = 99
+} SglErrorCode;
+
+// Preprocessor functions
+SglErrorCode sgl_preprocess_chat_request(
+    const char* request_json,
+    const char* tokenizer_path,
+    char** prompt_text_out,
+    uint32_t** token_ids_out,
+    size_t* token_ids_len_out,
+    char** tool_constraints_json_out,
+    int32_t* prompt_tokens_out,
+    char** error_out
+);
+
+// Opaque handle (must match grpc_converter.go)
+typedef void* TokenizerHandle;
+
+SglErrorCode sgl_preprocess_chat_request_with_tokenizer(
+    const char* request_json,
+    void* tokenizer_handle,
+    char** prompt_text_out,
+    uint32_t** token_ids_out,
+    size_t* token_ids_len_out,
+    char** tool_constraints_json_out,
+    int32_t* prompt_tokens_out,
+    char** error_out
+);
+
+void sgl_preprocessed_request_free(
+    char* prompt_text,
+    uint32_t* token_ids,
+    size_t token_ids_len,
+    char* tool_constraints_json
+);
+
+// Memory management
+void sgl_free_string(char* s);
+void sgl_free_token_ids(uint32_t* ptr, size_t count);
+*/
+import "C"
+
+import (
+	"fmt"
+	"unsafe"
+)
+
+// PreprocessedRequest represents a preprocessed chat request
+type PreprocessedRequest struct {
+	PromptText          string
+	TokenIDs            []uint32
+	ToolConstraintsJSON string
+	PromptTokens        int32
+	// Internal pointers for memory management
+	promptTextPtr          *C.char
+	tokenIDsPtr            *C.uint32_t
+	tokenIDsLen            uintptr
+	toolConstraintsJSONPtr *C.char
+}
+
+// PreprocessChatRequest preprocesses a chat completion request
+//
+// This function:
+// 1. Applies chat_template to messages
+// 2. Tokenizes the processed text
+// 3. Generates tool constraints (if tools are present)
+//
+// Returns the preprocessed request data and any error.
+func PreprocessChatRequest(requestJSON, tokenizerPath string) (*PreprocessedRequest, error) {
+	requestJSONC := C.CString(requestJSON)
+	defer C.free(unsafe.Pointer(requestJSONC))
+
+	tokenizerPathC := C.CString(tokenizerPath)
+	defer C.free(unsafe.Pointer(tokenizerPathC))
+
+	var promptTextOut *C.char
+	var tokenIDsOut *C.uint32_t
+	var tokenIDsLenOut C.size_t
+	var toolConstraintsJSONOut *C.char
+	var promptTokensOut C.int32_t
+	var errorOut *C.char
+
+	errorCode := C.sgl_preprocess_chat_request(
+		requestJSONC,
+		tokenizerPathC,
+		&promptTextOut,
+		&tokenIDsOut,
+		&tokenIDsLenOut,
+		&toolConstraintsJSONOut,
+		&promptTokensOut,
+		&errorOut,
+	)
+
+	if errorCode != C.SGL_ERROR_SUCCESS {
+		errorMsg := ""
+		if errorOut != nil {
+			errorMsg = C.GoString(errorOut)
+			C.sgl_free_string(errorOut)
+		}
+		return nil, fmt.Errorf("preprocessing failed: %s", errorMsg)
+	}
+
+	result := &PreprocessedRequest{
+		PromptText:          C.GoString(promptTextOut),
+		TokenIDs:            make([]uint32, tokenIDsLenOut),
+		ToolConstraintsJSON: "",
+		PromptTokens:        int32(promptTokensOut),
+	}
+
+	// Copy token IDs
+	if tokenIDsOut != nil && tokenIDsLenOut > 0 {
+		tokenIDsSlice := (*[1 << 30]C.uint32_t)(unsafe.Pointer(tokenIDsOut))[:tokenIDsLenOut:tokenIDsLenOut]
+		for i := range result.TokenIDs {
+			result.TokenIDs[i] = uint32(tokenIDsSlice[i])
+		}
+	}
+
+	// Copy tool constraints JSON if present
+	if toolConstraintsJSONOut != nil {
+		result.ToolConstraintsJSON = C.GoString(toolConstraintsJSONOut)
+	}
+
+	// Store pointers for later cleanup
+	result.promptTextPtr = promptTextOut
+	result.tokenIDsPtr = tokenIDsOut
+	result.tokenIDsLen = uintptr(tokenIDsLenOut)
+	result.toolConstraintsJSONPtr = toolConstraintsJSONOut
+
+	return result, nil
+}
+
+// PreprocessChatRequestWithTokenizer preprocesses a chat completion request using an existing tokenizer handle
+//
+// This function is similar to PreprocessChatRequest, but accepts a TokenizerHandle
+// instead of creating a new tokenizer. This allows reusing a cached tokenizer instance,
+// significantly reducing initialization overhead in concurrent scenarios.
+//
+// Returns the preprocessed request data and any error.
+func PreprocessChatRequestWithTokenizer(requestJSON string, tokenizerHandle *TokenizerHandle) (*PreprocessedRequest, error) {
+	requestJSONC := C.CString(requestJSON)
+	defer C.free(unsafe.Pointer(requestJSONC))
+
+	if tokenizerHandle == nil || tokenizerHandle.handle == nil {
+		return nil, fmt.Errorf("invalid tokenizer handle")
+	}
+
+	var promptTextOut *C.char
+	var tokenIDsOut *C.uint32_t
+	var tokenIDsLenOut C.size_t
+	var toolConstraintsJSONOut *C.char
+	var promptTokensOut C.int32_t
+	var errorOut *C.char
+
+	errorCode := C.sgl_preprocess_chat_request_with_tokenizer(
+		requestJSONC,
+		unsafe.Pointer(tokenizerHandle.handle), // Convert *C.TokenizerHandle to void*
+		&promptTextOut,
+		&tokenIDsOut,
+		&tokenIDsLenOut,
+		&toolConstraintsJSONOut,
+		&promptTokensOut,
+		&errorOut,
+	)
+
+	if errorCode != C.SGL_ERROR_SUCCESS {
+		errorMsg := ""
+		if errorOut != nil {
+			errorMsg = C.GoString(errorOut)
+			C.sgl_free_string(errorOut)
+		}
+		return nil, fmt.Errorf("preprocessing failed: %s", errorMsg)
+	}
+
+	result := &PreprocessedRequest{
+		PromptText:          C.GoString(promptTextOut),
+		TokenIDs:            make([]uint32, tokenIDsLenOut),
+		ToolConstraintsJSON: "",
+		PromptTokens:        int32(promptTokensOut),
+	}
+
+	// Copy token IDs
+	if tokenIDsOut != nil && tokenIDsLenOut > 0 {
+		tokenIDsSlice := (*[1 << 30]C.uint32_t)(unsafe.Pointer(tokenIDsOut))[:tokenIDsLenOut:tokenIDsLenOut]
+		for i := range result.TokenIDs {
+			result.TokenIDs[i] = uint32(tokenIDsSlice[i])
+		}
+	}
+
+	// Copy tool constraints JSON if present
+	if toolConstraintsJSONOut != nil {
+		result.ToolConstraintsJSON = C.GoString(toolConstraintsJSONOut)
+	}
+
+	// Store pointers for later cleanup
+	result.promptTextPtr = promptTextOut
+	result.tokenIDsPtr = tokenIDsOut
+	result.tokenIDsLen = uintptr(tokenIDsLenOut)
+	result.toolConstraintsJSONPtr = toolConstraintsJSONOut
+
+	return result, nil
+}
+
+// Free frees the memory allocated for a preprocessed request
+func (p *PreprocessedRequest) Free() {
+	if p.promptTextPtr != nil || p.tokenIDsPtr != nil || p.toolConstraintsJSONPtr != nil {
+		C.sgl_preprocessed_request_free(
+			p.promptTextPtr,
+			p.tokenIDsPtr,
+			C.size_t(p.tokenIDsLen),
+			p.toolConstraintsJSONPtr,
+		)
+		// Clear pointers to prevent double-free
+		p.promptTextPtr = nil
+		p.tokenIDsPtr = nil
+		p.tokenIDsLen = 0
+		p.toolConstraintsJSONPtr = nil
+	}
+}
+
+// FreePreprocessedRequest frees the memory allocated for a preprocessed request
+// This is a convenience function for direct pointer management
+func FreePreprocessedRequest(promptTextPtr *C.char, tokenIDsPtr *C.uint32_t, tokenIDsLen uintptr, toolConstraintsJSONPtr *C.char) {
+	if promptTextPtr != nil || tokenIDsPtr != nil || toolConstraintsJSONPtr != nil {
+		C.sgl_preprocessed_request_free(
+			promptTextPtr,
+			tokenIDsPtr,
+			C.size_t(tokenIDsLen),
+			toolConstraintsJSONPtr,
+		)
+	}
+}