sglang

Request 1: [SYSTEM_PROMPT] + "What's the weather?"
→ Computes full prompt (1000 tokens)

Request 2: [SAME_SYSTEM_PROMPT] + "Book a flight"
→ Reuses system prompt KV cache (998 tokens)
→ Only computes 2 new tokens
→ 5× faster!

@sgl.function
def structured_extraction(s, article):
 s += f"Article: {article}\n\n"
 s += "Extract key information as JSON:\n"

 # JSON schema constraint
 schema = {
 "type": "object",
 "properties": {
 "title": {"type": "string"},
 "author": {"type": "string"},
 "summary": {"type": "string"},
 "sentiment": {"type": "string", "enum": ["positive", "negative", "neutral"]}
 },
 "required": ["title", "author", "summary", "sentiment"]
 }

 s += sgl.gen("info", max_tokens=300, json_schema=schema)

state = structured_extraction.run(article="...")
print(state["info"])
# Output: Valid JSON matching schema

@sgl.function
def extract_email(s, text):
 s += f"Extract email from: {text}\n"
 s += "Email: "

 # Email regex pattern
 s += sgl.gen(
 "email",
 max_tokens=50,
 regex=r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
 )

state = extract_email.run(text="Contact [email protected] for details")
print(state["email"])
# Output: "[email protected]"

@sgl.function
def generate_code(s, description):
 s += f"Generate Python code for: {description}\n"
 s += "```python\n"

 # EBNF grammar for Python
 python_grammar = """
 ?start: function_def
 function_def: "def" NAME "(" [parameters] "):" suite
 parameters: parameter ("," parameter)*
 parameter: NAME
 suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
 """

 s += sgl.gen("code", max_tokens=200, grammar=python_grammar)
 s += "\n```"

import sglang as sgl

# Define tools
tools = [
 {
 "name": "get_weather",
 "description": "Get weather for a location",
 "parameters": {
 "type": "object",
 "properties": {
 "location": {"type": "string"}
 }
 }
 },
 {
 "name": "book_flight",
 "description": "Book a flight",
 "parameters": {
 "type": "object",
 "properties": {
 "from": {"type": "string"},
 "to": {"type": "string"},
 "date": {"type": "string"}
 }
 }
 }
]

@sgl.function
def agent_workflow(s, user_query, tools):
 # System prompt (cached with RadixAttention)
 s += "You are a helpful assistant with access to tools.\n"
 s += f"Available tools: {tools}\n\n"

 # User query
 s += f"User: {user_query}\n"
 s += "Assistant: "

 # Generate with function calling
 s += sgl.gen(
 "response",
 max_tokens=200,
 tools=tools, # SGLang handles tool call format
 stop=["User:", "\n\n"]
 )

# Multiple queries reuse system prompt
state1 = agent_workflow.run(
 user_query="What's the weather in NYC?",
 tools=tools
)
# First call: Computes full system prompt

state2 = agent_workflow.run(
 user_query="Book a flight to LA",
 tools=tools
)
# Second call: Reuses system prompt (5× faster)

@sgl.function
def multi_turn_chat(s, history, new_message):
 # System prompt (always cached)
 s += "You are a helpful AI assistant.\n\n"

 # Conversation history (cached as it grows)
 for msg in history:
 s += f"{msg['role']}: {msg['content']}\n"

 # New user message (only new part)
 s += f"User: {new_message}\n"
 s += "Assistant: "
 s += sgl.gen("response", max_tokens=200)

# Turn 1
history = []
state = multi_turn_chat.run(history=history, new_message="Hi there!")
history.append({"role": "User", "content": "Hi there!"})
history.append({"role": "Assistant", "content": state["response"]})

# Turn 2 (reuses Turn 1 KV cache)
state = multi_turn_chat.run(history=history, new_message="What's 2+2?")
# Only computes new message (much faster!)

# Turn 3 (reuses Turn 1 + Turn 2 KV cache)
state = multi_turn_chat.run(history=history, new_message="Tell me a joke")
# Progressively faster as history grows

# Launch with draft model (2-3× faster)
python -m sglang.launch_server \
 --model-path meta-llama/Meta-Llama-3-70B-Instruct \
 --speculative-model meta-llama/Meta-Llama-3-8B-Instruct \
 --speculative-num-steps 5

@sgl.function
def describe_image(s, image_path):
 s += sgl.image(image_path)
 s += "Describe this image in detail: "
 s += sgl.gen("description", max_tokens=200)

state = describe_image.run(image_path="photo.jpg")
print(state["description"])

# Automatic batching (continuous batching)
states = sgl.run_batch(
 [
 simple_gen.bind(question="What is AI?"),
 simple_gen.bind(question="What is ML?"),
 simple_gen.bind(question="What is DL?"),
 ]
)

# All 3 processed in single batch (efficient)

# Start server with OpenAI API
python -m sglang.launch_server \
 --model-path meta-llama/Meta-Llama-3-8B-Instruct \
 --port 30000

# Use with OpenAI client
curl http://localhost:30000/v1/chat/completions \
 -H "Content-Type: application/json" \
 -d '{
 "model": "default",
 "messages": [
 {"role": "system", "content": "You are helpful"},
 {"role": "user", "content": "Hello"}
 ],
 "temperature": 0.7,
 "max_tokens": 100
 }'

# Works with OpenAI Python SDK
from openai import OpenAI
client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY")

response = client.chat.completions.create(
 model="default",
 messages=[{"role": "user", "content": "Hello"}]
)