[SMG-GO] implement a Go SGLang Model Gateway - OpenAI Compatible API Server (#14770)
This commit is contained in:
ybyang
+554
@@ -0,0 +1,554 @@
|
||||
#!/bin/bash
|
||||
|
||||
# TPOT performance bottleneck analysis script
|
||||
# Specifically designed to analyze why Go Router is twice as slow as Rust Router
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/analyze_tpot.sh [options]
|
||||
#
|
||||
# Options:
|
||||
# --duration SECONDS CPU profile duration (default: 60)
|
||||
# --requests NUM Number of requests (default: 100)
|
||||
# --concurrency NUM Concurrency level (default: 20)
|
||||
# --pprof-port PORT pprof port (default: 6060)
|
||||
# --server-url URL Server URL (default: http://localhost:8080)
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
PROFILE_DIR="${PROJECT_ROOT}/profiles"
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
OUTPUT_DIR="${PROFILE_DIR}/tpot_analysis_${TIMESTAMP}"
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
# Default values
|
||||
DURATION=${DURATION:-60}
|
||||
NUM_REQUESTS=${NUM_REQUESTS:-100}
|
||||
CONCURRENCY=${CONCURRENCY:-20}
|
||||
PPROF_PORT=${PPROF_PORT:-6060}
|
||||
SERVER_URL=${SERVER_URL:-http://localhost:8080}
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--duration)
|
||||
DURATION="$2"
|
||||
shift 2
|
||||
;;
|
||||
--requests)
|
||||
NUM_REQUESTS="$2"
|
||||
shift 2
|
||||
;;
|
||||
--concurrency)
|
||||
CONCURRENCY="$2"
|
||||
shift 2
|
||||
;;
|
||||
--pprof-port)
|
||||
PPROF_PORT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--server-url)
|
||||
SERVER_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# Check for graphviz (optional, needed for some pprof visualizations)
|
||||
HAS_GRAPHVIZ=false
|
||||
if command -v dot >/dev/null 2>&1; then
|
||||
HAS_GRAPHVIZ=true
|
||||
fi
|
||||
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}TPOT Performance Bottleneck Analysis${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
echo "Configuration:"
|
||||
echo " Duration: ${DURATION}s"
|
||||
echo " Requests: $NUM_REQUESTS"
|
||||
echo " Concurrency: $CONCURRENCY"
|
||||
echo " Server URL: $SERVER_URL"
|
||||
echo " pprof Port: $PPROF_PORT"
|
||||
echo " Output Dir: $OUTPUT_DIR"
|
||||
if [ "$HAS_GRAPHVIZ" = "false" ]; then
|
||||
echo ""
|
||||
echo -e "${YELLOW}Note: graphviz not found. Some pprof visualizations may not work.${NC}"
|
||||
echo -e "${YELLOW}To install graphviz:${NC}"
|
||||
echo -e "${YELLOW} macOS: brew install graphviz${NC}"
|
||||
echo -e "${YELLOW} Ubuntu: sudo apt-get install graphviz${NC}"
|
||||
echo -e "${YELLOW} CentOS: sudo yum install graphviz${NC}"
|
||||
echo -e "${YELLOW}Text reports will still be generated without graphviz.${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Check if server is running
|
||||
echo -e "${YELLOW}[Check] Verifying server is running...${NC}"
|
||||
if ! curl -s "${SERVER_URL}/health" > /dev/null 2>&1; then
|
||||
echo -e "${RED}Error: Server not responding at ${SERVER_URL}${NC}"
|
||||
echo ""
|
||||
echo "Please start the server first with profiling enabled:"
|
||||
echo " ./run.sh --profile --pprof-port $PPROF_PORT"
|
||||
echo " or"
|
||||
echo " PPROF_ENABLED=true PPROF_PORT=$PPROF_PORT make run"
|
||||
exit 1
|
||||
fi
|
||||
echo -e "${GREEN}✓ Server is running${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if pprof is enabled
|
||||
echo -e "${YELLOW}[Check] Verifying pprof is enabled...${NC}"
|
||||
if ! curl -s "http://localhost:${PPROF_PORT}/debug/pprof/" > /dev/null 2>&1; then
|
||||
echo -e "${RED}Error: pprof not accessible at http://localhost:${PPROF_PORT}/debug/pprof/${NC}"
|
||||
echo ""
|
||||
echo "Please start the server with profiling enabled:"
|
||||
echo " ./run.sh --profile --pprof-port $PPROF_PORT"
|
||||
exit 1
|
||||
fi
|
||||
echo -e "${GREEN}✓ pprof is enabled${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 1: Collect baseline profiles
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 1/8] Collecting baseline profiles...${NC}"
|
||||
|
||||
# Baseline memory
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/heap_before.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/heap" > /dev/null 2>&1 || true
|
||||
|
||||
# Baseline goroutine
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/goroutine_before.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/goroutine" > /dev/null 2>&1 || true
|
||||
|
||||
echo -e "${GREEN}✓ Baseline profiles collected${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 2: Start CPU profile collection
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 2/8] Starting CPU profile collection (${DURATION}s)...${NC}"
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=${DURATION}" &
|
||||
CPU_PID=$!
|
||||
sleep 2
|
||||
echo -e "${GREEN}✓ CPU profile collection started${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 3: Run load test with streaming requests
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 3/8] Running load test ($NUM_REQUESTS streaming requests, concurrency=$CONCURRENCY)...${NC}"
|
||||
|
||||
# Function to run a single streaming request
|
||||
run_streaming_request() {
|
||||
local request_id=$1
|
||||
local start_time=$(date +%s)
|
||||
local start_nanos=$(date +%N 2>/dev/null || echo "000000000")
|
||||
|
||||
curl -N -s -X POST "${SERVER_URL}/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"model\": \"default\",
|
||||
\"messages\": [{\"role\": \"user\", \"content\": \"Write a 500-word story with character dialogue and scene descriptions\"}],
|
||||
\"stream\": true,
|
||||
\"max_tokens\": 300,
|
||||
\"temperature\": 0.7
|
||||
}" > /dev/null
|
||||
|
||||
local end_time=$(date +%s)
|
||||
local end_nanos=$(date +%N 2>/dev/null || echo "000000000")
|
||||
local duration=$((end_time - start_time))
|
||||
echo "$duration" >> "${OUTPUT_DIR}/request_times.txt"
|
||||
}
|
||||
|
||||
# Run requests with controlled concurrency
|
||||
# Use a temporary file to track job PIDs to avoid conflicts with CPU_PID
|
||||
JOB_PIDS_FILE="${OUTPUT_DIR}/.job_pids_$$"
|
||||
> "$JOB_PIDS_FILE"
|
||||
|
||||
for i in $(seq 1 $NUM_REQUESTS); do
|
||||
# Wait if we've reached concurrency limit
|
||||
while [ $(wc -l < "$JOB_PIDS_FILE" 2>/dev/null || echo 0) -ge $CONCURRENCY ]; do
|
||||
# Check and remove completed jobs
|
||||
while IFS= read -r pid; do
|
||||
if [ -n "$pid" ] && ! kill -0 "$pid" 2>/dev/null; then
|
||||
# Process completed, remove from file
|
||||
grep -v "^${pid}$" "$JOB_PIDS_FILE" > "${JOB_PIDS_FILE}.tmp" && \
|
||||
mv "${JOB_PIDS_FILE}.tmp" "$JOB_PIDS_FILE" || true
|
||||
fi
|
||||
done < "$JOB_PIDS_FILE"
|
||||
sleep 0.1
|
||||
done
|
||||
|
||||
# Start new request
|
||||
run_streaming_request $i &
|
||||
echo $! >> "$JOB_PIDS_FILE"
|
||||
|
||||
# Progress indicator
|
||||
if [ $((i % 10)) -eq 0 ]; then
|
||||
echo " Progress: $i/$NUM_REQUESTS requests sent..."
|
||||
fi
|
||||
done
|
||||
|
||||
# Wait for all remaining jobs (excluding CPU_PID)
|
||||
while IFS= read -r pid; do
|
||||
if [ -n "$pid" ] && [ "$pid" != "$CPU_PID" ]; then
|
||||
wait "$pid" 2>/dev/null || true
|
||||
fi
|
||||
done < "$JOB_PIDS_FILE"
|
||||
|
||||
# Clean up
|
||||
rm -f "$JOB_PIDS_FILE" "${JOB_PIDS_FILE}.tmp" 2>/dev/null || true
|
||||
|
||||
echo -e "${GREEN}✓ Load test completed${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 4: Wait for CPU profile to complete
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 4/8] Waiting for CPU profile to complete...${NC}"
|
||||
# Wait for the process, but handle the case where it might have already completed
|
||||
if kill -0 $CPU_PID 2>/dev/null; then
|
||||
wait $CPU_PID 2>/dev/null || true
|
||||
else
|
||||
# Process already completed, just wait a bit to ensure file is written
|
||||
sleep 1
|
||||
fi
|
||||
echo -e "${GREEN}✓ CPU profile collection completed${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 5: Collect final profiles
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 5/8] Collecting final profiles...${NC}"
|
||||
|
||||
# Final memory
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/heap_after.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/heap" > /dev/null 2>&1 || true
|
||||
|
||||
# Final goroutine
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/goroutine_after.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/goroutine" > /dev/null 2>&1 || true
|
||||
|
||||
# Mutex profile
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/mutex.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/mutex" > /dev/null 2>&1 || true
|
||||
|
||||
# Block profile
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/block.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/block" > /dev/null 2>&1 || true
|
||||
|
||||
echo -e "${GREEN}✓ Final profiles collected${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 6: Generate analysis reports
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 6/8] Generating analysis reports...${NC}"
|
||||
|
||||
# CPU analysis
|
||||
echo " Generating CPU reports..."
|
||||
go tool pprof -top -cum "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" > "${OUTPUT_DIR}/01_cpu_top_cum.txt" 2>&1 || true
|
||||
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" > "${OUTPUT_DIR}/02_cpu_top_flat.txt" 2>&1 || true
|
||||
|
||||
# Memory analysis
|
||||
echo " Generating memory reports..."
|
||||
if [ -f "${OUTPUT_DIR}/heap_after.pb.gz" ]; then
|
||||
go tool pprof -top -alloc_space "${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/03_memory_alloc_space.txt" 2>&1 || true
|
||||
go tool pprof -top -alloc_objects "${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/04_memory_alloc_objects.txt" 2>&1 || true
|
||||
go tool pprof -top -inuse_space "${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/05_memory_inuse_space.txt" 2>&1 || true
|
||||
fi
|
||||
|
||||
# Memory growth
|
||||
if [ -f "${OUTPUT_DIR}/heap_before.pb.gz" ] && [ -f "${OUTPUT_DIR}/heap_after.pb.gz" ]; then
|
||||
go tool pprof -top -base="${OUTPUT_DIR}/heap_before.pb.gz" \
|
||||
"${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/06_memory_growth.txt" 2>&1 || true
|
||||
fi
|
||||
|
||||
# FFI/CGO analysis
|
||||
echo " Analyzing FFI/CGO calls..."
|
||||
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" 2>&1 | \
|
||||
grep -iE "(block_on|CGO|FFI|ffi|runtime\.cgo|_Cfunc)" > "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt" || \
|
||||
echo "No FFI/CGO related functions found" > "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt"
|
||||
|
||||
# JSON serialization analysis
|
||||
echo " Analyzing JSON serialization..."
|
||||
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" 2>&1 | \
|
||||
grep -iE "(json|Marshal|Unmarshal|Encode|Decode|sonic|jsoniter)" > "${OUTPUT_DIR}/08_json_analysis.txt" || \
|
||||
echo "No JSON related functions found" > "${OUTPUT_DIR}/08_json_analysis.txt"
|
||||
|
||||
# Goroutine analysis
|
||||
if [ -f "${OUTPUT_DIR}/goroutine_after.pb.gz" ]; then
|
||||
echo " Analyzing goroutines..."
|
||||
go tool pprof -top "${OUTPUT_DIR}/goroutine_after.pb.gz" > "${OUTPUT_DIR}/09_goroutine_analysis.txt" 2>&1 || true
|
||||
fi
|
||||
|
||||
# Mutex analysis
|
||||
if [ -f "${OUTPUT_DIR}/mutex.pb.gz" ]; then
|
||||
echo " Analyzing mutex contention..."
|
||||
go tool pprof -top "${OUTPUT_DIR}/mutex.pb.gz" > "${OUTPUT_DIR}/10_mutex_analysis.txt" 2>&1 || true
|
||||
fi
|
||||
|
||||
# Block analysis
|
||||
if [ -f "${OUTPUT_DIR}/block.pb.gz" ]; then
|
||||
echo " Analyzing blocking operations..."
|
||||
go tool pprof -top "${OUTPUT_DIR}/block.pb.gz" > "${OUTPUT_DIR}/11_block_analysis.txt" 2>&1 || true
|
||||
fi
|
||||
|
||||
# Request timing statistics
|
||||
if [ -f "${OUTPUT_DIR}/request_times.txt" ] && [ -s "${OUTPUT_DIR}/request_times.txt" ]; then
|
||||
echo " Calculating request timing statistics..."
|
||||
{
|
||||
echo "Request Timing Statistics"
|
||||
echo "========================"
|
||||
echo ""
|
||||
echo "Total requests: $(wc -l < "${OUTPUT_DIR}/request_times.txt" | tr -d ' ')"
|
||||
echo ""
|
||||
awk '{
|
||||
sum+=$1
|
||||
sumsq+=$1*$1
|
||||
if(NR==1 || $1<min) min=$1
|
||||
if(NR==1 || $1>max) max=$1
|
||||
} END {
|
||||
if(NR > 0) {
|
||||
mean=sum/NR
|
||||
variance=(sumsq/NR - mean*mean)
|
||||
stddev=sqrt(variance)
|
||||
print "Min: " min "s"
|
||||
print "Max: " max "s"
|
||||
print "Mean: " mean "s"
|
||||
print "StdDev: " stddev "s"
|
||||
}
|
||||
}' "${OUTPUT_DIR}/request_times.txt"
|
||||
} > "${OUTPUT_DIR}/12_request_timing.txt"
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}✓ Analysis reports generated${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 7: Generate summary report
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 7/8] Generating summary report...${NC}"
|
||||
|
||||
SUMMARY_FILE="${OUTPUT_DIR}/00_SUMMARY.md"
|
||||
cat > "$SUMMARY_FILE" <<EOF
|
||||
# TPOT Performance Analysis Summary
|
||||
|
||||
**Analysis Date:** $(date)
|
||||
**Duration:** ${DURATION}s
|
||||
**Requests:** $NUM_REQUESTS
|
||||
**Concurrency:** $CONCURRENCY
|
||||
|
||||
## Key Findings
|
||||
|
||||
### 1. CPU Hotspots (Top 10 Cumulative Time)
|
||||
|
||||
\`\`\`
|
||||
$(head -15 "${OUTPUT_DIR}/01_cpu_top_cum.txt" | tail -10)
|
||||
\`\`\`
|
||||
|
||||
### 2. CPU Hotspots (Top 10 Flat Time)
|
||||
|
||||
\`\`\`
|
||||
$(head -15 "${OUTPUT_DIR}/02_cpu_top_flat.txt" | tail -10)
|
||||
\`\`\`
|
||||
|
||||
### 3. FFI/CGO Overhead
|
||||
|
||||
\`\`\`
|
||||
$(cat "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt")
|
||||
\`\`\`
|
||||
|
||||
### 4. JSON Serialization Overhead
|
||||
|
||||
\`\`\`
|
||||
$(cat "${OUTPUT_DIR}/08_json_analysis.txt")
|
||||
\`\`\`
|
||||
|
||||
### 5. Memory Allocation (Top 10 by Space)
|
||||
|
||||
\`\`\`
|
||||
$(head -15 "${OUTPUT_DIR}/03_memory_alloc_space.txt" | tail -10)
|
||||
\`\`\`
|
||||
|
||||
### 6. Memory Allocation (Top 10 by Objects)
|
||||
|
||||
\`\`\`
|
||||
$(head -15 "${OUTPUT_DIR}/04_memory_alloc_objects.txt" | tail -10)
|
||||
\`\`\`
|
||||
|
||||
### 7. Mutex Contention
|
||||
|
||||
\`\`\`
|
||||
$(head -15 "${OUTPUT_DIR}/10_mutex_analysis.txt" | tail -10 2>/dev/null || echo "No significant mutex contention detected")
|
||||
\`\`\`
|
||||
|
||||
### 8. Blocking Operations
|
||||
|
||||
\`\`\`
|
||||
$(head -15 "${OUTPUT_DIR}/11_block_analysis.txt" | tail -10 2>/dev/null || echo "No significant blocking detected")
|
||||
\`\`\`
|
||||
|
||||
## Performance Bottlenecks Identified
|
||||
|
||||
### High Priority Issues
|
||||
|
||||
1. **FFI/CGO Overhead**
|
||||
- Check: \`cat ${OUTPUT_DIR}/07_ffi_cgo_analysis.txt\`
|
||||
- Impact: FFI calls add overhead compared to native Rust code
|
||||
- Recommendation: Minimize FFI calls, batch operations
|
||||
|
||||
2. **JSON Serialization**
|
||||
- Check: \`cat ${OUTPUT_DIR}/08_json_analysis.txt\`
|
||||
- Impact: JSON marshaling/unmarshaling can be expensive
|
||||
- Recommendation: Use faster JSON library (jsoniter), reduce serialization frequency
|
||||
|
||||
3. **Memory Allocations**
|
||||
- Check: \`cat ${OUTPUT_DIR}/03_memory_alloc_space.txt\`
|
||||
- Impact: Frequent allocations cause GC pressure
|
||||
- Recommendation: Use object pools, pre-allocate buffers
|
||||
|
||||
### Medium Priority Issues
|
||||
|
||||
4. **Goroutine Overhead**
|
||||
- Check: \`cat ${OUTPUT_DIR}/09_goroutine_analysis.txt\`
|
||||
- Impact: Too many goroutines can cause scheduling overhead
|
||||
- Recommendation: Limit goroutine count, use worker pools
|
||||
|
||||
5. **Lock Contention**
|
||||
- Check: \`cat ${OUTPUT_DIR}/10_mutex_analysis.txt\`
|
||||
- Impact: Lock contention reduces parallelism
|
||||
- Recommendation: Reduce lock granularity, use lock-free structures
|
||||
|
||||
## Comparison with Rust Router
|
||||
|
||||
### Expected Differences
|
||||
|
||||
1. **FFI Overhead**: Go → Rust FFI calls add ~100-500ns per call
|
||||
2. **GC Overhead**: Go's GC can cause pauses (usually <1ms)
|
||||
3. **JSON Library**: Go's standard library is slower than Rust's serde
|
||||
4. **Memory Layout**: Go's GC affects cache locality
|
||||
|
||||
### Optimization Opportunities
|
||||
|
||||
1. **Reduce FFI Calls**
|
||||
- Batch token processing
|
||||
- Use async FFI (if possible)
|
||||
- Cache frequently used FFI results
|
||||
|
||||
2. **Optimize JSON**
|
||||
- Use jsoniter (already implemented)
|
||||
- Pre-allocate JSON buffers
|
||||
- Reduce serialization frequency
|
||||
|
||||
3. **Memory Management**
|
||||
- Use sync.Pool for frequently allocated objects
|
||||
- Pre-allocate slices with known capacity
|
||||
- Avoid unnecessary string copies
|
||||
|
||||
4. **Concurrency**
|
||||
- Use worker pools instead of spawning goroutines per request
|
||||
- Limit concurrent FFI calls
|
||||
- Use channels efficiently
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Review detailed reports in this directory
|
||||
2. Use interactive pprof: \`go tool pprof -http=:8081 ${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz\`
|
||||
3. Compare with Rust router profiles (if available)
|
||||
4. Implement optimizations based on findings
|
||||
5. Re-run analysis to measure improvements
|
||||
|
||||
## Files Generated
|
||||
|
||||
- \`00_SUMMARY.md\` - This summary
|
||||
- \`01_cpu_top_cum.txt\` - CPU top functions (cumulative)
|
||||
- \`02_cpu_top_flat.txt\` - CPU top functions (flat)
|
||||
- \`03_memory_alloc_space.txt\` - Memory allocation by space
|
||||
- \`04_memory_alloc_objects.txt\` - Memory allocation by objects
|
||||
- \`05_memory_inuse_space.txt\` - Memory in use by space
|
||||
- \`06_memory_growth.txt\` - Memory growth during test
|
||||
- \`07_ffi_cgo_analysis.txt\` - FFI/CGO overhead analysis
|
||||
- \`08_json_analysis.txt\` - JSON serialization analysis
|
||||
- \`09_goroutine_analysis.txt\` - Goroutine analysis
|
||||
- \`10_mutex_analysis.txt\` - Mutex contention analysis
|
||||
- \`11_block_analysis.txt\` - Blocking operations analysis
|
||||
- \`12_request_timing.txt\` - Request timing statistics
|
||||
- \`*.pb.gz\` - Raw profile files for interactive analysis
|
||||
|
||||
EOF
|
||||
|
||||
echo -e "${GREEN}✓ Summary report generated${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 8: Display summary
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 8/8] Analysis Complete!${NC}"
|
||||
echo ""
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}Summary${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Top CPU Hotspots (Cumulative):${NC}"
|
||||
head -12 "${OUTPUT_DIR}/01_cpu_top_cum.txt" | tail -10
|
||||
echo ""
|
||||
echo -e "${YELLOW}FFI/CGO Overhead:${NC}"
|
||||
cat "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt"
|
||||
echo ""
|
||||
echo -e "${YELLOW}JSON Serialization Overhead:${NC}"
|
||||
cat "${OUTPUT_DIR}/08_json_analysis.txt"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Top Memory Allocations:${NC}"
|
||||
head -12 "${OUTPUT_DIR}/03_memory_alloc_space.txt" | tail -10
|
||||
echo ""
|
||||
if [ -f "${OUTPUT_DIR}/12_request_timing.txt" ]; then
|
||||
echo -e "${YELLOW}Request Timing:${NC}"
|
||||
cat "${OUTPUT_DIR}/12_request_timing.txt"
|
||||
echo ""
|
||||
fi
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
echo -e "${BLUE}Detailed Reports:${NC}"
|
||||
echo " Summary: cat ${OUTPUT_DIR}/00_SUMMARY.md"
|
||||
echo " CPU (cum): cat ${OUTPUT_DIR}/01_cpu_top_cum.txt"
|
||||
echo " CPU (flat): cat ${OUTPUT_DIR}/02_cpu_top_flat.txt"
|
||||
echo " FFI/CGO: cat ${OUTPUT_DIR}/07_ffi_cgo_analysis.txt"
|
||||
echo " JSON: cat ${OUTPUT_DIR}/08_json_analysis.txt"
|
||||
echo " Memory: cat ${OUTPUT_DIR}/03_memory_alloc_space.txt"
|
||||
echo ""
|
||||
echo -e "${BLUE}Interactive Analysis:${NC}"
|
||||
echo " Run: go tool pprof -http=:8081 ${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz"
|
||||
echo " Then visit:"
|
||||
echo " - http://localhost:8081/ui/flamegraph (Flame Graph - no graphviz needed)"
|
||||
echo " - http://localhost:8081/ui/top (Top Functions - no graphviz needed)"
|
||||
if [ "$HAS_GRAPHVIZ" = "true" ]; then
|
||||
echo " - http://localhost:8081/ui/graph (Call Graph - requires graphviz)"
|
||||
else
|
||||
echo " - http://localhost:8081/ui/graph (Call Graph - requires graphviz, not available)"
|
||||
fi
|
||||
echo ""
|
||||
if [ "$HAS_GRAPHVIZ" = "false" ]; then
|
||||
echo -e "${YELLOW}Note: Install graphviz to enable call graph visualization:${NC}"
|
||||
echo -e "${YELLOW} macOS: brew install graphviz${NC}"
|
||||
echo -e "${YELLOW} Ubuntu: sudo apt-get install graphviz${NC}"
|
||||
echo -e "${YELLOW} CentOS: sudo yum install graphviz${NC}"
|
||||
echo ""
|
||||
fi
|
||||
echo -e "${GREEN}All files saved to: ${OUTPUT_DIR}${NC}"
|
||||
echo ""
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
#!/bin/bash
|
||||
|
||||
# pprof performance analysis script
|
||||
# Used to analyze performance bottlenecks of Go OpenAI server
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
# Configuration
|
||||
PPROF_PORT=${PPROF_PORT:-6060}
|
||||
SERVER_PORT=${SERVER_PORT:-8080}
|
||||
DURATION=${DURATION:-60} # Performance test duration (seconds)
|
||||
OUTPUT_DIR="./pprof_results"
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
|
||||
# Create output directory
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
echo "=========================================="
|
||||
echo "pprof Performance Analysis Tool"
|
||||
echo "=========================================="
|
||||
echo "PPROF_PORT: $PPROF_PORT"
|
||||
echo "SERVER_PORT: $SERVER_PORT"
|
||||
echo "DURATION: ${DURATION}s"
|
||||
echo "OUTPUT_DIR: $OUTPUT_DIR"
|
||||
echo ""
|
||||
|
||||
# Check if go tool pprof is available
|
||||
if ! command -v go &> /dev/null; then
|
||||
echo "Error: go command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if server is running
|
||||
check_server() {
|
||||
if curl -s "http://localhost:${SERVER_PORT}/health" > /dev/null 2>&1; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check if pprof is available
|
||||
check_pprof() {
|
||||
if curl -s "http://localhost:${PPROF_PORT}/debug/pprof/" > /dev/null 2>&1; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Start server (if not running)
|
||||
if ! check_server; then
|
||||
echo "Server not running, please start the server first:"
|
||||
echo " export PPROF_ENABLED=true"
|
||||
echo " export PPROF_PORT=$PPROF_PORT"
|
||||
echo " ./oai_server"
|
||||
echo ""
|
||||
echo "Or use the following command to start:"
|
||||
echo " PPROF_ENABLED=true PPROF_PORT=$PPROF_PORT ./oai_server"
|
||||
echo ""
|
||||
read -p "Start server now? (y/n) " -n 1 -r
|
||||
echo
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "Starting server..."
|
||||
PPROF_ENABLED=true PPROF_PORT=$PPROF_PORT ./oai_server &
|
||||
SERVER_PID=$!
|
||||
echo "Server PID: $SERVER_PID"
|
||||
|
||||
# Wait for server to start
|
||||
echo "Waiting for server to start..."
|
||||
for i in {1..30}; do
|
||||
if check_server; then
|
||||
echo "Server started"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if ! check_server; then
|
||||
echo "Error: Server failed to start"
|
||||
kill $SERVER_PID 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check if pprof is available
|
||||
if ! check_pprof; then
|
||||
echo "Error: pprof not enabled. Please set environment variables:"
|
||||
echo " export PPROF_ENABLED=true"
|
||||
echo " export PPROF_PORT=$PPROF_PORT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Starting to collect performance data..."
|
||||
echo ""
|
||||
|
||||
# 1. CPU Profile (30 seconds)
|
||||
echo "[1/6] Collecting CPU Profile (30 seconds)..."
|
||||
go tool pprof -proto -output="$OUTPUT_DIR/cpu_${TIMESTAMP}.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=30" &
|
||||
CPU_PID=$!
|
||||
|
||||
# 2. Collect Heap Profile simultaneously
|
||||
echo "[2/6] Collecting Heap Profile..."
|
||||
go tool pprof -proto -output="$OUTPUT_DIR/heap_${TIMESTAMP}.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/heap" &
|
||||
HEAP_PID=$!
|
||||
|
||||
# 3. Collect Goroutine Profile
|
||||
echo "[3/6] Collecting Goroutine Profile..."
|
||||
go tool pprof -proto -output="$OUTPUT_DIR/goroutine_${TIMESTAMP}.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/goroutine" &
|
||||
GOROUTINE_PID=$!
|
||||
|
||||
# 4. Collect Mutex Profile
|
||||
echo "[4/6] Collecting Mutex Profile..."
|
||||
go tool pprof -proto -output="$OUTPUT_DIR/mutex_${TIMESTAMP}.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/mutex" &
|
||||
MUTEX_PID=$!
|
||||
|
||||
# 5. Collect Block Profile
|
||||
echo "[5/6] Collecting Block Profile..."
|
||||
go tool pprof -proto -output="$OUTPUT_DIR/block_${TIMESTAMP}.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/block" &
|
||||
BLOCK_PID=$!
|
||||
|
||||
# 6. Run performance test (during CPU profile collection)
|
||||
echo "[6/6] Running performance test..."
|
||||
echo "Tip: Please use your performance testing tool (curl, ab, wrk, etc.) to send requests to the server"
|
||||
echo " CPU profile will collect 30 seconds of performance data"
|
||||
echo ""
|
||||
|
||||
# Wait for CPU profile to complete
|
||||
wait $CPU_PID
|
||||
echo "CPU Profile collection completed"
|
||||
|
||||
# Wait for other profiles
|
||||
wait $HEAP_PID
|
||||
wait $GOROUTINE_PID
|
||||
wait $MUTEX_PID
|
||||
wait $BLOCK_PID
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Performance data collection completed!"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "Generated analysis files:"
|
||||
ls -lh "$OUTPUT_DIR"/*_${TIMESTAMP}.* 2>/dev/null || true
|
||||
echo ""
|
||||
|
||||
# Generate analysis report
|
||||
echo "Generating analysis report..."
|
||||
echo ""
|
||||
|
||||
# CPU Top 20
|
||||
echo "=== CPU Top 20 (sorted by flat time) ===" > "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
go tool pprof -top -cum "$OUTPUT_DIR/cpu_${TIMESTAMP}.pb.gz" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt" 2>&1 || true
|
||||
echo "" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
|
||||
# Heap Top 20
|
||||
echo "=== Heap Top 20 (sorted by allocation size) ===" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
go tool pprof -top "$OUTPUT_DIR/heap_${TIMESTAMP}.pb.gz" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt" 2>&1 || true
|
||||
echo "" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
|
||||
# Goroutine statistics
|
||||
echo "=== Goroutine Statistics ===" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
go tool pprof -top "$OUTPUT_DIR/goroutine_${TIMESTAMP}.pb.gz" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt" 2>&1 || true
|
||||
echo "" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
|
||||
# Mutex statistics
|
||||
echo "=== Mutex Wait Time ===" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
go tool pprof -top "$OUTPUT_DIR/mutex_${TIMESTAMP}.pb.gz" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt" 2>&1 || true
|
||||
echo "" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
|
||||
# Block statistics
|
||||
echo "=== Block Wait Time ===" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
go tool pprof -top "$OUTPUT_DIR/block_${TIMESTAMP}.pb.gz" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt" 2>&1 || true
|
||||
|
||||
echo "Analysis report saved to: $OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
echo ""
|
||||
|
||||
# Display key information
|
||||
echo "=========================================="
|
||||
echo "Key Performance Metrics Summary"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "View detailed report:"
|
||||
echo " cat $OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
echo ""
|
||||
echo "Interactive CPU Profile view:"
|
||||
echo " go tool pprof $OUTPUT_DIR/cpu_${TIMESTAMP}.pb.gz"
|
||||
echo ""
|
||||
echo "Interactive Heap Profile view:"
|
||||
echo " go tool pprof $OUTPUT_DIR/heap_${TIMESTAMP}.pb.gz"
|
||||
echo ""
|
||||
echo "Generate flame graph (requires go-torch or pprof):"
|
||||
echo " go tool pprof -http=:8080 $OUTPUT_DIR/cpu_${TIMESTAMP}.pb.gz"
|
||||
echo ""
|
||||
|
||||
# If server was started, ask if it should be closed
|
||||
if [ -n "$SERVER_PID" ]; then
|
||||
read -p "Close server? (y/n) " -n 1 -r
|
||||
echo
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
||||
kill $SERVER_PID 2>/dev/null || true
|
||||
echo "Server closed"
|
||||
fi
|
||||
fi
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Quick pprof analysis script
|
||||
# Collects 30-second CPU profile and immediately displays top results
|
||||
|
||||
set -e
|
||||
|
||||
PPROF_PORT=${PPROF_PORT:-6060}
|
||||
DURATION=${DURATION:-30}
|
||||
|
||||
echo "=========================================="
|
||||
echo "Quick pprof Analysis"
|
||||
echo "=========================================="
|
||||
echo "PPROF_PORT: $PPROF_PORT"
|
||||
echo "DURATION: ${DURATION}s"
|
||||
echo ""
|
||||
echo "Tip: During data collection, please send requests to the server"
|
||||
echo " You can use: ./pprof_test.sh"
|
||||
echo ""
|
||||
|
||||
# Check if pprof is available
|
||||
if ! curl -s "http://localhost:${PPROF_PORT}/debug/pprof/" > /dev/null 2>&1; then
|
||||
echo "Error: pprof not enabled. Please set environment variables:"
|
||||
echo " export PPROF_ENABLED=true"
|
||||
echo " export PPROF_PORT=$PPROF_PORT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Starting to collect CPU Profile (${DURATION} seconds)..."
|
||||
echo ""
|
||||
|
||||
# Collect CPU profile and directly display top results
|
||||
go tool pprof -top -cum "http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=${DURATION}"
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Analysis Complete"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "More analysis options:"
|
||||
echo " # Interactive view"
|
||||
echo " go tool pprof http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=30"
|
||||
echo ""
|
||||
echo " # View heap memory"
|
||||
echo " go tool pprof http://localhost:${PPROF_PORT}/debug/pprof/heap"
|
||||
echo ""
|
||||
echo " # View goroutines"
|
||||
echo " go tool pprof http://localhost:${PPROF_PORT}/debug/pprof/goroutine"
|
||||
echo ""
|
||||
echo " # Generate Web UI"
|
||||
echo " go tool pprof -http=:8080 http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=30"
|
||||
echo ""
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Simple performance test script for sending requests while collecting pprof data
|
||||
|
||||
set -e
|
||||
|
||||
SERVER_URL=${SERVER_URL:-"http://localhost:8080"}
|
||||
DURATION=${DURATION:-30} # Test duration (seconds)
|
||||
CONCURRENT=${CONCURRENT:-1} # Number of concurrent requests
|
||||
|
||||
echo "=========================================="
|
||||
echo "Performance Test Script"
|
||||
echo "=========================================="
|
||||
echo "SERVER_URL: $SERVER_URL"
|
||||
echo "DURATION: ${DURATION}s"
|
||||
echo "CONCURRENT: $CONCURRENT"
|
||||
echo ""
|
||||
|
||||
# Test request JSON
|
||||
TEST_REQUEST='{
|
||||
"model": "default",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello, how are you?"}
|
||||
],
|
||||
"stream": true,
|
||||
"max_tokens": 100
|
||||
}'
|
||||
|
||||
# Check if server is available
|
||||
if ! curl -s "${SERVER_URL}/health" > /dev/null 2>&1; then
|
||||
echo "Error: Server not available (${SERVER_URL}/health)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Starting to send test requests..."
|
||||
echo ""
|
||||
|
||||
# Function to send streaming request
|
||||
send_stream_request() {
|
||||
local request_num=$1
|
||||
local start_time=$(date +%s.%N)
|
||||
|
||||
curl -s -N -X POST "${SERVER_URL}/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$TEST_REQUEST" \
|
||||
> /dev/null 2>&1
|
||||
|
||||
local end_time=$(date +%s.%N)
|
||||
local duration=$(echo "$end_time - $start_time" | bc)
|
||||
echo "Request $request_num completed, duration: ${duration}s"
|
||||
}
|
||||
|
||||
# Send requests concurrently
|
||||
if [ "$CONCURRENT" -eq 1 ]; then
|
||||
# Single-threaded mode: continuously send requests
|
||||
end_time=$(($(date +%s) + DURATION))
|
||||
request_count=0
|
||||
|
||||
while [ $(date +%s) -lt $end_time ]; do
|
||||
request_count=$((request_count + 1))
|
||||
send_stream_request $request_count
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Test completed, sent $request_count requests"
|
||||
else
|
||||
# Multi-threaded mode: send requests concurrently
|
||||
end_time=$(($(date +%s) + DURATION))
|
||||
request_count=0
|
||||
|
||||
while [ $(date +%s) -lt $end_time ]; do
|
||||
# Start concurrent requests
|
||||
for i in $(seq 1 $CONCURRENT); do
|
||||
request_count=$((request_count + 1))
|
||||
send_stream_request $request_count &
|
||||
done
|
||||
|
||||
# Wait for all requests to complete
|
||||
wait
|
||||
|
||||
# Brief rest to avoid overload
|
||||
sleep 0.1
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Test completed, sent $request_count requests"
|
||||
fi
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
#!/bin/bash
|
||||
|
||||
# TPOT performance analysis script
|
||||
# Quickly collect and analyze TPOT-related performance data
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
PROFILE_DIR="${PROJECT_ROOT}/profiles"
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
OUTPUT_DIR="${PROFILE_DIR}/${TIMESTAMP}"
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
# Default values
|
||||
PPROF_PORT=${PPROF_PORT:-6060}
|
||||
SERVER_URL=${SERVER_URL:-http://localhost:8080}
|
||||
DURATION=${DURATION:-30}
|
||||
NUM_REQUESTS=${NUM_REQUESTS:-20}
|
||||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
echo -e "${GREEN}TPOT Performance Analysis${NC}"
|
||||
echo "=========================="
|
||||
echo "Profile directory: $OUTPUT_DIR"
|
||||
echo "Duration: ${DURATION}s"
|
||||
echo "Requests: $NUM_REQUESTS"
|
||||
echo ""
|
||||
|
||||
# Check if server is running
|
||||
if ! curl -s "${SERVER_URL}/health" > /dev/null 2>&1; then
|
||||
echo -e "${YELLOW}Warning: Server not responding at ${SERVER_URL}${NC}"
|
||||
echo "Please start the server first with profiling enabled:"
|
||||
echo " PPROF_ENABLED=true PPROF_PORT=$PPROF_PORT make run"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Collect baseline memory
|
||||
echo -e "${GREEN}[1/5] Collecting baseline memory profile...${NC}"
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/heap_before.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/heap" > /dev/null 2>&1 || true
|
||||
|
||||
# Start CPU profile collection in background
|
||||
echo -e "${GREEN}[2/5] Starting CPU profile collection (${DURATION}s)...${NC}"
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=${DURATION}" &
|
||||
CPU_PID=$!
|
||||
|
||||
# Wait a bit for profile to start
|
||||
sleep 2
|
||||
|
||||
# Run load test
|
||||
echo -e "${GREEN}[3/5] Running load test ($NUM_REQUESTS requests)...${NC}"
|
||||
for i in $(seq 1 $NUM_REQUESTS); do
|
||||
curl -N -s -X POST "${SERVER_URL}/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"model\": \"default\",
|
||||
\"messages\": [{\"role\": \"user\", \"content\": \"Write a story\"}],
|
||||
\"stream\": true,
|
||||
\"max_tokens\": 200
|
||||
}" > /dev/null &
|
||||
|
||||
# Limit concurrency
|
||||
if [ $((i % 5)) -eq 0 ]; then
|
||||
wait
|
||||
fi
|
||||
done
|
||||
wait
|
||||
|
||||
# Wait for CPU profile to complete
|
||||
echo -e "${GREEN}[4/5] Waiting for CPU profile to complete...${NC}"
|
||||
# Wait for the CPU profile process, but handle the case where it's not a child process
|
||||
if kill -0 $CPU_PID 2>/dev/null; then
|
||||
# Process is still running, wait for it
|
||||
while kill -0 $CPU_PID 2>/dev/null; do
|
||||
sleep 1
|
||||
done
|
||||
else
|
||||
# Process already completed or not found, just wait a bit
|
||||
sleep 2
|
||||
fi
|
||||
|
||||
# Collect final memory
|
||||
echo -e "${GREEN}[5/5] Collecting final memory profile...${NC}"
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/heap_after.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/heap" > /dev/null 2>&1 || true
|
||||
|
||||
# Generate reports
|
||||
echo ""
|
||||
echo -e "${GREEN}Generating reports...${NC}"
|
||||
|
||||
# CPU top (cumulative)
|
||||
go tool pprof -top -cum "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" > "${OUTPUT_DIR}/cpu_top_cum.txt" 2>&1 || true
|
||||
|
||||
# CPU top (flat)
|
||||
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" > "${OUTPUT_DIR}/cpu_top_flat.txt" 2>&1 || true
|
||||
|
||||
# Memory growth
|
||||
if [ -f "${OUTPUT_DIR}/heap_before.pb.gz" ] && [ -f "${OUTPUT_DIR}/heap_after.pb.gz" ]; then
|
||||
go tool pprof -top -base="${OUTPUT_DIR}/heap_before.pb.gz" \
|
||||
"${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/heap_growth.txt" 2>&1 || true
|
||||
fi
|
||||
|
||||
# FFI/CGO related
|
||||
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" 2>&1 | \
|
||||
grep -E "(block_on|CGO|FFI|json|Marshal|Unmarshal)" > "${OUTPUT_DIR}/ffi_related.txt" || \
|
||||
echo "No FFI/CGO related functions found" > "${OUTPUT_DIR}/ffi_related.txt"
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
echo -e "${GREEN}=== Analysis Summary ===${NC}"
|
||||
echo ""
|
||||
echo -e "${YELLOW}CPU Top (Cumulative) - Top 10:${NC}"
|
||||
head -12 "${OUTPUT_DIR}/cpu_top_cum.txt" | tail -10 || true
|
||||
|
||||
echo ""
|
||||
echo -e "${YELLOW}CPU Top (Flat) - Top 10:${NC}"
|
||||
head -12 "${OUTPUT_DIR}/cpu_top_flat.txt" | tail -10 || true
|
||||
|
||||
echo ""
|
||||
echo -e "${YELLOW}FFI/CGO Related Functions:${NC}"
|
||||
cat "${OUTPUT_DIR}/ffi_related.txt" || true
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}=== Detailed Reports ===${NC}"
|
||||
echo "CPU (cumulative): cat ${OUTPUT_DIR}/cpu_top_cum.txt"
|
||||
echo "CPU (flat): cat ${OUTPUT_DIR}/cpu_top_flat.txt"
|
||||
echo "Memory growth: cat ${OUTPUT_DIR}/heap_growth.txt"
|
||||
echo "FFI related: cat ${OUTPUT_DIR}/ffi_related.txt"
|
||||
echo ""
|
||||
echo -e "${GREEN}=== Interactive Analysis ===${NC}"
|
||||
echo "Run: go tool pprof -http=:8081 ${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz"
|
||||
echo "Then visit: http://localhost:8081/ui/flamegraph"
|
||||
echo ""
|
||||
echo "Profile files saved to: ${OUTPUT_DIR}"
|
||||
Reference in New Issue
Block a user