LLM Proxy to Train Data

🚀 核心特性

🔄 动态代理服务器

📊 数据收集与管理

🎯 训练数据导出

📦 安装依赖

pip install -r requirements.txt

🎯 快速开始

1. 启动代理服务器

# 使用启动脚本(推荐)
./start.sh
# 或者直接运行
python proxy_dynamic.py --port 8080

http://localhost:8080/{target_domain}/{api_path}

2. 启动Web管理界面(可选)

# 使用启动脚本
./start_web.sh
# 或者直接运行
python app.py

3. 导出训练数据(可选)

# 使用导出脚本(如存在)
./export_data.sh
# 或直接运行导出器(默认导出 ShareGPT 扩展格式)
python process_conversations.py \
 --db interactions.db \
 --table interactions \
 --output conversations_sharegpt.jsonl \
 --invalid invalid_conversations.jsonl \
 --format sharegpt

# 1) 混合数据(既有普通对话也有含工具的对话),自动附 tools
python process_conversations.py \
 --format openai \
 --tools-schema auto \
 --max-records 1000 \
 --output conversations_openai_mixed.jsonl \
 --invalid invalid_openai_mixed.jsonl
# 2) 仅导出 function_call-only(以工具调用结束,无后续助手文本)
python process_conversations.py \
 --only-function-call-only \
 --format openai \
 --tools-schema auto \
 --max-records 100 \
 --output conversations_openai_fc_only.jsonl \
 --invalid invalid_openai_fc_only.jsonl

# 取首条样本并打印顶层键(应仅有 messages 和可能的 tools)
python3 -c "import json; d=json.loads(open('conversations_openai_fc_only.jsonl','r',encoding='utf-8').readline()); print(sorted(d.keys()))"
# 抽一条包含 role:tool 的样本,检查 tool_calls.id 与 tool.tool_call_id 配对
python3 - <<'PY'
import json
path='conversations_openai_mixed.jsonl'
with open(path,'r',encoding='utf-8') as f:
 for line in f:
 d=json.loads(line)
 msgs=d.get('messages',[])
 t=next((m for m in msgs if isinstance(m,dict) and m.get('role')=='tool'),None)
 if not t: 
 continue
 tc=next((m.get('tool_calls') for m in msgs if isinstance(m,dict) and m.get('role')=='assistant' and m.get('tool_calls')),[])
 print(json.dumps({
 'assistant_tool_calls': tc,
 'first_tool_message': {'tool_call_id': t.get('tool_call_id'), 'content': (t.get('content')[:200]+'...') if isinstance(t.get('content'),str) and len(t.get('content'))>200 else t.get('content')},
 'tools': d.get('tools',[])
 }, ensure_ascii=False, indent=2))
 break
PY

python process_conversations.py \
 --format sharegpt \
 --max-records 1000 \
 --output conversations_sharegpt.jsonl \
 --invalid invalid_sharegpt.jsonl

🔧 动态代理原理

URL格式说明

POST http://localhost:8080/{target_domain}/{api_path}

认证类型自动识别

# Anthropic类型识别
if "/anthropic/" in path or "/v1/messages" in path:
 auth_type = "anthropic"
# OpenAI类型识别
elif "/v1/chat/completions" in path or "/chat/completions" in path:
 auth_type = "openai"
# 其他路径默认使用OpenAI格式
else:
 auth_type = "openai"

支持的域名

📝 API调用示例

OpenAI风格API调用

curl -X POST "http://localhost:8080/api.deepseek.com/v1/chat/completions" \
 -H "Authorization: Bearer your-api-key" \
 -H "Content-Type: application/json" \
 -d '{
 "model": "deepseek-chat",
 "messages": [
 {"role": "user", "content": "你好"}
 ]
 }'

Anthropic风格API调用

curl -X POST "http://localhost:8080/api.moonshot.cn/anthropic/v1/messages" \
 -H "Authorization: Bearer your-api-key" \
 -H "Content-Type: application/json" \
 -d '{
 "model": "claude-3-5-haiku-20241022",
 "messages": [
 {"role": "user", "content": "你好"}
 ],
 "max_tokens": 100
 }'

Google AI(Gemini)接口

curl -X POST "http://localhost:8080/v1/chat/completions" \
 -H "Authorization: Bearer ${GOOGLE_API_KEY}" \
 -H "Content-Type: application/json" \
 -d '{
 "model": "gemini-2.0-flash-exp",
 "messages": [
 {"role": "user", "content": "用三句话介绍量子计算"}
 ]
 }'

curl -X POST "http://localhost:8080/generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-exp:generateContent" \
 -H "Authorization: Bearer ${GOOGLE_API_KEY}" \
 -H "Content-Type: application/json" \
 -d '{
 "contents": [
 {"role": "user", "parts": [{"text": "用三句话介绍量子计算"}]}
 ]
 }'

curl -N -X POST "http://localhost:8080/generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-exp:streamGenerateContent" \
 -H "Authorization: Bearer ${GOOGLE_API_KEY}" \
 -H "Content-Type: application/json" \
 -d '{
 "contents": [
 {"role": "user", "parts": [{"text": "给我一个Python异步示例"}]}
 ]
 }'

文本嵌入API

curl -X POST "http://localhost:8080/api.siliconflow.cn/v1/embeddings" \
 -H "Authorization: Bearer your-api-key" \
 -H "Content-Type: application/json" \
 -d '{
 "model": "BAAI/bge-large-zh-v1.5",
 "input": "人工智能是计算机科学的一个分支"
 }'

文档重排API

curl -X POST "http://localhost:8080/deepsearch.jina.ai/v1/rerank" \
 -H "Authorization: Bearer your-api-key" \
 -H "Content-Type: application/json" \
 -d '{
 "model": "jina-reranker-v1-base-en",
 "query": "What is artificial intelligence?",
 "documents": [
 "Artificial intelligence is a branch of computer science.",
 "Machine learning is a subset of AI.",
 "Deep learning uses neural networks."
 ],
 "top_k": 3
 }'

流式请求

curl -X POST "http://localhost:8080/api.siliconflow.cn/v1/chat/completions" \
 -H "Authorization: Bearer your-api-key" \
 -H "Content-Type: application/json" \
 -d '{
 "model": "Qwen/Qwen2.5-7B-Instruct",
 "messages": [
 {"role": "user", "content": "介绍一下Python"}
 ],
 "stream": true
 }'

📁 项目结构

llm_proxy_sharegpt/
|-- proxy_dynamic.py # 主服务器文件
|-- app.py # Web管理界面
|-- process_conversations.py # 数据处理和导出
|-- utils.py # 工具函数
|-- test_dynamic_proxy.py # 基础功能测试
|-- test_embedding_rerank.py # 嵌入和重排测试
|-- requirements.txt # 项目依赖
|-- start.sh # 代理服务器启动脚本
|-- start_web.sh # Web界面启动脚本
|-- export_data.sh # 数据导出脚本
|-- config.example.json # 配置文件示例
|-- README.md # 项目说明(本文件)
|-- templates/ # Web界面模板
| `-- index.html
`-- static/ # 静态资源
 |-- css/
 `-- js/

🧪 测试

# 测试基础功能
python test_dynamic_proxy.py
# 测试嵌入和重排功能
python test_embedding_rerank.py

🔄 完整工作流程

🛡️ 安全特性

域名白名单

请求大小限制

探针请求过滤

📊 性能优化

连接池管理

批量数据保存

内存优化

🔧 配置选项

命令行参数

config.json 配置

{
 "allowed_domains": {
 "generativelanguage.googleapis.com": { "auth_type": "google", "https": true },
 "api.openai.com": { "auth_type": "openai", "https": true },
 "api.deepseek.com": { "auth_type": "openai", "https": true }
 },
 "probe_request": {
 "path_blocklist": ["/", "/favicon.ico"],
 "path_prefix_blocklist": ["/.well-known/", "/locales/"],
 "user_agent_substrings": ["CensysInspect", "Go-http-client"],
 "allowed_methods": ["GET","POST","PUT","DELETE","PATCH","OPTIONS"],
 "ip_blocklist": ["193.34.212.110","185.191.127.222"]
 },
 "probe_filter": {
 "patterns": [
 "GET / HTTP",
 "GET /favicon.ico",
 "GET \\/\\.well-known\\/",
 "Go-http-client",
 "BadHttpMessage"
 ],
 "ip_patterns": [
 "193\\.34\\.212\\.\\d+",
 "185\\.191\\.127\\.\\d+"
 ]
 }
}

扩展白名单

{
 "allowed_domains": {
 "generativelanguage.googleapis.com": { "auth_type": "google", "https": true },
 "api.openai.com": { "auth_type": "openai", "https": true },
 "new-api.example.com": { "auth_type": "openai", "https": true }
 }
}

🔍 监控和日志

健康检查

GET http://localhost:8080/health

日志级别

关键指标

🚨 故障排除

常见问题

调试方法

🆚 优势对比

动态代理方式

POST /api.example.com/v1/chat/completions

🤝 贡献

📄 许可证

📖 总结