[model-gateway] Add model scope support and LRU eviction for GPU-constrained environments (#16525)

sgl-model-gateway/e2e_test/router/test_mmlu.py

@@ -35,7 +35,7 @@ class TestMMLU:
 
         Note: setup_backend fixture already waits for workers to be ready.
         """
-        backend, model, client = setup_backend
+        backend, model, client, *_ = setup_backend
         base_url = str(client.base_url).rstrip("/v1")
 
         args = SimpleNamespace(
@@ -59,7 +59,7 @@ class TestMMLU:
         Runs MMLU with 128 examples for more statistically
         significant results.
         """
-        backend, model, client = setup_backend
+        backend, model, client, *_ = setup_backend
         base_url = str(client.base_url).rstrip("/v1")
 
         args = SimpleNamespace(

sgl-model-gateway/e2e_test/router/test_pd_mmlu.py

@@ -10,7 +10,8 @@ Requirements:
 
 Configuration via markers:
     @pytest.mark.model("model-id")  # Override default model
-    @pytest.mark.pd(num_prefill=2, num_decode=2)  # Custom worker counts
+    @pytest.mark.workers(prefill=2, decode=2)  # Custom worker counts
+    @pytest.mark.gateway(policy="round_robin")  # Gateway configuration
 
 Usage:
     # Basic (1 prefill + 1 decode)
@@ -42,7 +43,7 @@ class TestPDMMLU:
         Runs MMLU with 1 prefill + 1 decode worker and validates
         accuracy meets threshold (>= 0.65).
         """
-        backend, model, client = setup_backend
+        backend, model, client, *_ = setup_backend
         base_url = str(client.base_url).rstrip("/v1")
 
         args = SimpleNamespace(

sgl-model-gateway/e2e_test/router/test_worker_api.py

@@ -0,0 +1,156 @@
+"""Tests for gateway worker management APIs.
+
+Tests the gateway's worker management endpoints:
+- GET /workers - List all workers
+- POST /add_worker - Add a worker dynamically
+- POST /remove_worker - Remove a worker dynamically
+- GET /v1/models - List available models
+
+Usage:
+    pytest e2e_test/router/test_worker_api.py -v
+"""
+
+from __future__ import annotations
+
+import logging
+
+import pytest
+from infra import ConnectionMode, Gateway, ModelPool
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.mark.e2e
+@pytest.mark.parametrize("setup_backend", ["grpc", "http"], indirect=True)
+class TestWorkerAPI:
+    """Tests for worker management APIs using setup_backend fixture."""
+
+    def test_list_workers(self, setup_backend):
+        """Test listing workers via /workers endpoint."""
+        backend, model, client, gateway = setup_backend
+
+        workers = gateway.list_workers()
+        assert len(workers) >= 1, "Expected at least one worker"
+        logger.info("Found %d workers", len(workers))
+
+        for worker in workers:
+            logger.info(
+                "Worker: id=%s, url=%s, status=%s",
+                worker.id,
+                worker.url,
+                worker.status,
+            )
+            assert worker.url, "Worker should have a URL"
+
+    def test_list_models(self, setup_backend):
+        """Test listing models via /v1/models endpoint."""
+        backend, model, client, gateway = setup_backend
+
+        models = gateway.list_models()
+        assert len(models) >= 1, "Expected at least one model"
+        logger.info("Found %d models", len(models))
+
+        for m in models:
+            logger.info("Model: %s", m.get("id", "unknown"))
+            assert "id" in m, "Model should have an id"
+
+    def test_health_endpoint(self, setup_backend):
+        """Test health check endpoint."""
+        backend, model, client, gateway = setup_backend
+
+        assert gateway.health(), "Gateway should be healthy"
+        logger.info("Gateway health check passed")
+
+
+@pytest.mark.e2e
+class TestIGWMode:
+    """Tests for IGW mode - start gateway empty, add workers via API."""
+
+    def test_igw_start_empty(self, model_pool: ModelPool):
+        """Test starting gateway in IGW mode with no workers."""
+        gateway = Gateway()
+        gateway.start(igw_mode=True)
+
+        try:
+            assert gateway.health(), "Gateway should be healthy"
+            assert gateway.igw_mode, "Gateway should be in IGW mode"
+
+            workers = gateway.list_workers()
+            logger.info("IGW gateway started with %d workers", len(workers))
+        finally:
+            gateway.shutdown()
+
+    def test_igw_add_worker(self, model_pool: ModelPool):
+        """Test adding a worker to IGW gateway."""
+        http_instance = model_pool.get("llama-8b", ConnectionMode.HTTP)
+
+        gateway = Gateway()
+        gateway.start(igw_mode=True)
+
+        try:
+            # Add worker
+            success, result = gateway.add_worker(http_instance.worker_url)
+            assert success, f"Failed to add worker: {result}"
+            logger.info("Added worker: %s", result)
+
+            # Verify worker was added
+            workers = gateway.list_workers()
+            assert len(workers) >= 1, "Expected at least one worker"
+            logger.info("Worker count: %d", len(workers))
+
+            # Verify models are available
+            models = gateway.list_models()
+            logger.info("Models available: %d", len(models))
+        finally:
+            gateway.shutdown()
+
+    def test_igw_add_and_remove_worker(self, model_pool: ModelPool):
+        """Test adding and removing workers dynamically."""
+        http_instance = model_pool.get("llama-8b", ConnectionMode.HTTP)
+
+        gateway = Gateway()
+        gateway.start(igw_mode=True)
+
+        try:
+            # Add worker
+            success, _ = gateway.add_worker(http_instance.worker_url)
+            assert success, "Failed to add worker"
+
+            initial_count = len(gateway.list_workers())
+            logger.info("Worker count after add: %d", initial_count)
+
+            # Remove worker
+            success, msg = gateway.remove_worker(http_instance.worker_url)
+            if success:
+                logger.info("Removed worker: %s", msg)
+                final_count = len(gateway.list_workers())
+                logger.info("Worker count after remove: %d", final_count)
+            else:
+                logger.warning("Remove worker not supported: %s", msg)
+        finally:
+            gateway.shutdown()
+
+    def test_igw_multiple_workers(self, model_pool: ModelPool):
+        """Test adding multiple workers to IGW gateway."""
+        http_instance = model_pool.get("llama-8b", ConnectionMode.HTTP)
+        grpc_instance = model_pool.get("llama-8b", ConnectionMode.GRPC)
+
+        gateway = Gateway()
+        gateway.start(igw_mode=True)
+
+        try:
+            # Add both workers
+            success1, _ = gateway.add_worker(http_instance.worker_url)
+            success2, _ = gateway.add_worker(grpc_instance.worker_url)
+
+            if not success1 or not success2:
+                pytest.skip("Dynamic worker management not fully supported")
+
+            workers = gateway.list_workers()
+            logger.info("Worker count: %d", len(workers))
+            assert len(workers) >= 2, "Expected at least 2 workers"
+
+            for w in workers:
+                logger.info("Worker: id=%s, url=%s", w.id, w.url)
+        finally:
+            gateway.shutdown()