LLM Router Server is a high-performance routing service designed for multi-model deployment scenarios, used to uniformly manage and orchestrate multiple local Large Language Model (LLM) services, Embedding models, Re-ranking models, and other inference services.

English | 中文

Key Features

Use Cases

Features

Independent Multi-Model Execution

Intelligent Load Balancing

Embedding and Reranker Integration

Fully Compatible with OpenAI SDK

Workflow

Directory Structure

LLM-Router-Server/
├── configs/ # Configuration directory
│ ├── config.yaml # Main configuration file (models, server settings)
│ └── gunicorn.conf.py # Gunicorn configuration
├── docker/ # Docker related files
│ ├── Dockerfile # Docker image build file
│ └── docker-compose.yaml # Docker Compose configuration
├── logs/ # Log directory
├── scripts/ # Startup scripts directory
│ ├── start_all_models.py # Python script to start all models
│ └── start_all.sh # One-click startup script (models + router)
├── src/ # Main source code directory
│ ├── embedding_reranker/ # Embedding and Reranker module
│ │ ├── __init__.py
│ │ ├── embedding_reranker_launcher.py # Launcher
│ │ ├── schema.py # Data structure definitions
│ │ └── embedding_engine/ # Inference engine
│ │ ├── baseinferencer.py # Base inference class
│ │ ├── embed_rerank.py # Embedding/Rerank implementation
│ │ ├── generator.py # Generator
│ │ └── optimize.py # Optimization tools
│ ├── llm_router/ # LLM routing module
│ │ ├── __init__.py
│ │ ├── config_loader.py # Configuration loader
│ │ ├── env.py # Environment variable management
│ │ ├── main.py # FastAPI application entry point
│ │ ├── router.py # Routing logic
│ │ └── vllm_launcher.py # vLLM launcher
│ └── metrics/ # Monitoring and metrics
│ └── basic_metrics.py # Basic metrics collection
├── test/ # Test files directory
│ └── test_router_server.py # Router server tests
├── requirements.txt # Python dependencies list
└── README.md # Project documentation

Installation Guide

Install Dependencies

pip install -r requirements.txt

Configuration Guide

1. Edit Configuration File

LLM Engine Configuration

LLM_engines:
 # Model with multiple instances
 Qwen3-0.6B:
 instances:
 # First instance
 - id: "qwen3-1" # Instance ID
 host: "localhost" # Service host
 port: 8002 # Service port
 cuda_device: 0 # CUDA device number
 
 # Second instance
 - id: "qwen3-2" # Instance ID
 host: "localhost" # Service host
 port: 8004 # Service port
 cuda_device: 0 # CUDA device number
 
 # Model configuration (shared by all instances)
 model_config:
 model_tag: "Qwen/Qwen3-0.6B" # Model path or HuggingFace ID
 dtype: "float16" # Data type
 max_model_len: 500 # Maximum sequence length
 gpu_memory_utilization: 0.35 # GPU memory utilization
 tensor_parallel_size: 1 # Tensor parallel size
# Embedding and Reranking server configuration
embedding_server:
 host: "localhost"
 port: 8005
 cuda_device: 1
 # Embedding model list
 embedding_models:
 m3e-base:
 model_name: "moka-ai/m3e-base"
 model_path: "./models/embedding_engine/model/embedding_model/m3e-base-model"
 tokenizer_path: "./models/embedding_engine/model/embedding_model/m3e-base-tokenizer"
 max_length: 512
 use_gpu: true
 use_float16: true
 
 bge-m3:
 model_name: "BAAI/bge-m3"
 model_path: "./models/embedding_engine/model/embedding_model/bge-m3-model"
 tokenizer_path: "./models/embedding_engine/model/embedding_model/bge-m3-tokenizer"
 max_length: 512
 use_gpu: true
 use_float16: true
 # Reranking model list
 reranking_models:
 bge-reranker-large:
 model_name: "BAAI/bge-reranker-large"
 model_path: "./models/embedding_engine/model/reranking_model/bge-reranker-large-model"
 tokenizer_path: "./models/embedding_engine/model/reranking_model/bge-reranker-large-tokenizer"
 max_length: 512
 use_gpu: true
 use_float16: true

Configuration Parameters

2. Configure Gunicorn

# gunicorn.conf.py
import os
# Bind address and port
bind = "0.0.0.0:8947"
# Number of workers (recommended: CPU core count)
workers = 4
# Worker class (using Uvicorn Worker for ASGI support)
worker_class = "uvicorn.workers.UvicornWorker"
# Timeout (0 means unlimited)
timeout = 0
# Log level
loglevel = "info"
# Access log output to stdout
accesslog = "-"
# Error log output to stdout
errorlog = "-"
# Whether to preload the application
preload_app = False

Usage Guide

1. Start All Services

sh scripts/start_all.sh ./configs/config.yaml ./configs/gunicorn.conf.py

3. Verify Service Status

curl http://localhost:8947/v1/models

4. Using OpenAI SDK

Chat Completions

from openai import OpenAI
client = OpenAI(
 api_key="EMPTY",
 base_url="http://localhost:8947/v1"
)
# Non-streaming request
response = client.chat.completions.create(
 model="Qwen2.5-14B-Instruct",
 messages=[
 {"role": "system", "content": "You are a helpful assistant."},
 {"role": "user", "content": "Please introduce the advantages of Python."}
 ],
 temperature=0.7,
 max_tokens=500
)
print(response.choices[0].message.content)
# Streaming request
stream = client.chat.completions.create(
 model="Qwen2.5-14B-Instruct",
 messages=[
 {"role": "user", "content": "Write a poem about spring."}
 ],
 temperature=0.8,
 stream=True
)
for chunk in stream:
 if chunk.choices[0].delta.content:
 print(chunk.choices[0].delta.content, end="", flush=True)

Embeddings

response = client.embeddings.create(
 model="m3e-base",
 input=["This is the first text", "This is the second text"]
)
# Get embedding vectors
embedding_1 = response.data[0].embedding
embedding_2 = response.data[1].embedding
print(f"Embedding dimension: {len(embedding_1)}")

Reranking

documents = [
 "Machine learning is best learned through projects.",
 "Theory is essential for understanding machine learning.",
 "Practical tutorials are the best way to learn machine learning."
]
response = client.embeddings.create(
 model="bge-reranker-large",
 input=documents,
 extra_body={"query": "How to learn machine learning?"}
)
# Get reranking scores
for idx, item in enumerate(response.data):
 print(f"Document {idx}: Score {item.embedding}")

API Documentation

Endpoint List

Internal Project Documentation

License