Open AI API hidden states (#6716)

2025-06-10 14:37:29 -07:00
parent ce5ee3bdf0
commit b56de8f943
17 changed files with 606 additions and 44 deletions
@@ -22,6 +22,7 @@ def main():
    # Create an LLM.
    llm = sgl.Engine(
        model_path="Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+        enable_return_hidden_states=True,
    )

    sampling_params = {