llmproxy

Install

go get github.com/agentuity/llmproxy

Quick Start

Simple Proxy

package main
import (
 "context"
 "io"
 "net/http"
 "github.com/agentuity/llmproxy"
 "github.com/agentuity/llmproxy/interceptors"
 "github.com/agentuity/llmproxy/providers/openai"
)
func main() {
 ctx := context.Background()
 provider, _ := openai.New("sk-your-key")
 proxy := llmproxy.NewProxy(provider,
 llmproxy.WithInterceptor(interceptors.NewLogging(nil)),
 )
 http.HandleFunc("/v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
 resp, meta, err := proxy.Forward(ctx, r)
 if err != nil {
 http.Error(w, err.Error(), 500)
 return
 }
 defer resp.Body.Close()
 // Response includes token usage
 _ = meta.Usage.PromptTokens
 _ = meta.Usage.CompletionTokens
 io.Copy(w, resp.Body)
 })
 http.ListenAndServe(":8080", nil)
}

AutoRouter (Recommended)

package main
import (
 "net/http"
 "github.com/agentuity/llmproxy"
 "github.com/agentuity/llmproxy/providers/openai"
 "github.com/agentuity/llmproxy/providers/anthropic"
)
func main() {
 openaiProvider, _ := openai.New("sk-openai-key")
 anthropicProvider, _ := anthropic.New("sk-ant-key")
 router := llmproxy.NewAutoRouter(
 llmproxy.WithAutoRouterFallbackProvider(openaiProvider),
 )
 router.RegisterProvider(openaiProvider)
 router.RegisterProvider(anthropicProvider)
 // Single endpoint handles all providers and APIs
 http.Handle("/", router)
 http.ListenAndServe(":8080", nil)
}

# Auto-detect OpenAI from gpt-4 model name
curl -X POST http://localhost:8080/ \
 -H 'Content-Type: application/json' \
 -d '{"model":"gpt-4","messages":[{"role":"user","content":"Hello"}]}'
# Auto-detect Anthropic from claude model name 
curl -X POST http://localhost:8080/ \
 -H 'Content-Type: application/json' \
 -d '{"model":"claude-3-opus","max_tokens":1024,"messages":[{"role":"user","content":"Hello"}]}'
# Auto-detect Responses API from input field
curl -X POST http://localhost:8080/ \
 -H 'Content-Type: application/json' \
 -d '{"model":"gpt-4o","input":"Hello"}'

Features

AutoRouter

Detection Order

Provider Detection

Examples

# Explicit provider via header
curl -X POST http://localhost:8080/ \
 -H 'Content-Type: application/json' \
 -H 'X-Provider: anthropic' \
 -d '{"model":"claude-3-opus","max_tokens":1024,"messages":[{"role":"user","content":"Hello"}]}'
# Provider prefix in model (gets stripped)
curl -X POST http://localhost:8080/ \
 -H 'Content-Type: application/json' \
 -d '{"model":"anthropic/claude-3-opus","max_tokens":1024,"messages":[{"role":"user","content":"Hello"}]}'
# Traditional path still works
curl -X POST http://localhost:8080/v1/chat/completions \
 -H 'Content-Type: application/json' \
 -d '{"model":"gpt-4","messages":[{"role":"user","content":"Hello"}]}'

Streaming

# Streaming with automatic usage extraction
curl -X POST http://localhost:8080/ \
 -H 'Content-Type: application/json' \
 -d '{"model":"gpt-4","stream":true,"messages":[{"role":"user","content":"Hello"}]}'

adapter, _ := modelsdev.LoadFromURL()
billingCallback := func(r llmproxy.BillingResult) {
 log.Printf("Cost: $%.6f (tokens: %d/%d)", r.TotalCost, r.PromptTokens, r.CompletionTokens)
}
router := llmproxy.NewAutoRouter(
 llmproxy.WithAutoRouterBillingCalculator(llmproxy.NewBillingCalculator(adapter.GetCostLookup(), billingCallback)),
)

WebSocket Mode

gorilla/websocket Example

package main
import (
 "context"
 "log"
 "net/http"
 "github.com/agentuity/llmproxy"
 "github.com/agentuity/llmproxy/providers/openai"
 "github.com/gorilla/websocket"
)
// Configure allowed origins for WebSocket upgrades.
var trustedOrigins = []string{"https://myapp.example.com"}
// Thin adapters — gorilla's *Conn already satisfies llmproxy.WSConn
type gorillaUpgrader struct{ websocket.Upgrader }
func (u *gorillaUpgrader) Upgrade(w http.ResponseWriter, r *http.Request, h http.Header) (llmproxy.WSConn, error) {
 conn, err := u.Upgrader.Upgrade(w, r, h)
 return conn, err
}
type gorillaDialer struct{ websocket.Dialer }
func (d *gorillaDialer) DialContext(ctx context.Context, urlStr string, h http.Header) (llmproxy.WSConn, *http.Response, error) {
 conn, resp, err := d.Dialer.DialContext(ctx, urlStr, h)
 return conn, resp, err
}
func main() {
 // In production, validate the Origin header against trusted origins.
 // This example allows all origins for brevity.
 upgrader := websocket.Upgrader{
 CheckOrigin: func(r *http.Request) bool {
 origin := r.Header.Get("Origin")
 for _, allowed := range trustedOrigins {
 if origin == allowed {
 return true
 }
 }
 return false
 },
 }
 provider, _ := openai.New("sk-your-key")
 router := llmproxy.NewAutoRouter(
 llmproxy.WithAutoRouterFallbackProvider(provider),
 llmproxy.WithAutoRouterWebSocket(
 &gorillaUpgrader{upgrader},
 &gorillaDialer{websocket.Dialer{}},
 ),
 llmproxy.WithAutoRouterWSBillingCallback(func(turn int, meta llmproxy.ResponseMetadata, billing *llmproxy.BillingResult) {
 log.Printf("Turn %d: %d prompt + %d completion tokens",
 turn, meta.Usage.PromptTokens, meta.Usage.CompletionTokens)
 }),
 )
 router.RegisterProvider(provider)
 http.Handle("/", router)
 log.Fatal(http.ListenAndServe(":8080", nil))
}

from websocket import create_connection
import json
ws = create_connection("ws://localhost:8080/v1/responses",
 header=["Authorization: Bearer sk-your-key"])
ws.send(json.dumps({
 "type": "response.create",
 "model": "gpt-4o",
 "input": [{"type": "message", "role": "user",
 "content": [{"type": "input_text", "text": "Hello!"}]}],
}))
for msg in ws:
 event = json.loads(msg)
 print(event["type"], event.get("delta", ""))
 if event["type"] == "response.completed":
 break

Providers

Interceptors

// Logging
llmproxy.WithInterceptor(interceptors.NewLogging(logger))
// Metrics (thread-safe)
metrics := &interceptors.Metrics{}
llmproxy.WithInterceptor(interceptors.NewMetrics(metrics))
// Retry on 429/5xx
llmproxy.WithInterceptor(interceptors.NewRetry(3, time.Second))
// Billing with models.dev pricing
adapter, _ := modelsdev.LoadFromURL()
llmproxy.WithInterceptor(interceptors.NewBilling(adapter.GetCostLookup(), func(r llmproxy.BillingResult) {
 log.Printf("Cost: $%.6f", r.TotalCost)
}))
// OTel tracing
llmproxy.WithInterceptor(interceptors.NewTracing(otelExtractor))
// Strip sensitive headers
llmproxy.WithInterceptor(interceptors.NewResponseHeaderBan("Openai-Organization"))
// Add custom headers
llmproxy.WithInterceptor(interceptors.NewAddResponseHeader(
 interceptors.NewHeader("X-Gateway", "llmproxy"),
))
// Anthropic prompt caching (default 5 min, free)
llmproxy.WithInterceptor(interceptors.NewAnthropicPromptCaching(interceptors.CacheRetentionDefault))
// Anthropic prompt caching with 1h retention (costs more)
llmproxy.WithInterceptor(interceptors.NewAnthropicPromptCaching(interceptors.CacheRetention1h))
// OpenAI prompt caching with explicit cache key
llmproxy.WithInterceptor(interceptors.NewOpenAIPromptCaching(interceptors.CacheRetention24h, "my-cache-key"))
// OpenAI prompt caching with auto-derived key and tenant namespace
llmproxy.WithInterceptor(interceptors.NewOpenAIPromptCachingAuto("tenant-123", interceptors.CacheRetentionDefault))
// xAI/Grok prompt caching (uses x-grok-conv-id header)
llmproxy.WithInterceptor(interceptors.NewXAIPromptCaching("conv-abc123"))
// Fireworks prompt caching (uses x-session-affinity and x-prompt-cache-isolation-key headers)
llmproxy.WithInterceptor(interceptors.NewFireworksPromptCaching("session-123"))

Architecture

Parse → Enrich → Resolve → Forward → Extract

Example

cd examples/basic
go run main.go

License