Siumai — Unified LLM Interface for Rust

What It Provides

Install

[dependencies]
siumai = "0.11.0-beta.8"
tokio = { version = "1", features = ["rt-multi-thread", "macros"] }

Migration

# One provider
siumai = { version = "0.11.0-beta.8", features = ["openai"] }
# Multiple providers
siumai = { version = "0.11.0-beta.8", features = ["openai", "anthropic", "google"] }
# All
siumai = { version = "0.11.0-beta.8", features = ["all-providers"] }

[dependencies]
siumai = "0.11.0-beta.8"
siumai-extras = { version = "0.11.0-beta.8", features = ["schema", "telemetry", "opentelemetry", "server", "mcp"] }

Usage

Construction order

Public surface map

Registry (recommended)

use siumai::prelude::unified::*;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
 let reg = registry::global();
 let model = reg.language_model("openai:gpt-4o-mini")?;
 let resp = text::generate(
 &model,
 ChatRequest::new(vec![user!("Hello")]),
 text::GenerateOptions::default(),
 )
 .await?;
 println!("{}", resp.content_text().unwrap_or_default());
 Ok(())
}

use siumai::prelude::unified::*;
use siumai::providers::openai::{OpenAiChatRequestExt, OpenAiOptions, ResponsesApiConfig};
let req = ChatRequest::new(vec![user!("Hello")]).with_openai_options(
 OpenAiOptions::new().with_responses_api(ResponsesApiConfig {
 enabled: false,
 ..Default::default()
 }),
);

OpenAI-compatible vendors (config-first)

use siumai::prelude::unified::*;
use siumai::providers::openai_compatible::OpenAiCompatibleClient;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
 // Reads `DEEPSEEK_API_KEY` by default.
 let client = OpenAiCompatibleClient::from_builtin_env("deepseek", Some("deepseek-chat")).await?;
 let resp = text::generate(
 &client,
 ChatRequest::new(vec![user!("hi")]),
 text::GenerateOptions::default(),
 )
 .await?;
 println!("{}", resp.content_text().unwrap_or_default());
 Ok(())
}

Provider clients (config-first)

use siumai::prelude::unified::*;
use siumai::providers::openai::{OpenAiClient, OpenAiConfig};
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
 let cfg = OpenAiConfig::new(std::env::var("OPENAI_API_KEY")?)
 .with_model("gpt-4o")
 .with_temperature(0.7);
 let client = OpenAiClient::from_config(cfg)?;
 let resp = text::generate(
 &client,
 ChatRequest::new(vec![user!("Hi")]),
 text::GenerateOptions::default(),
 )
 .await?;
 println!("{}", resp.content_text().unwrap_or_default());
 Ok(())
}

use siumai::models;
use siumai::prelude::unified::*;
use siumai::providers::minimaxi::{MinimaxiClient, MinimaxiConfig};
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
 let cfg = MinimaxiConfig::new(std::env::var("MINIMAXI_API_KEY")?)
 .with_model(models::minimaxi::MINIMAX_M2);
 let client = MinimaxiClient::from_config(cfg)?;
 let resp = text::generate(
 &client,
 ChatRequest::new(vec![user!("Hello MiniMaxi!")]),
 text::GenerateOptions::default(),
 )
 .await?;
 println!("{}", resp.content_text().unwrap_or_default());
 Ok(())
}

use siumai::prelude::unified::*;
use siumai::providers::openai::{OpenAiClient, OpenAiConfig};
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
 // For OpenAI-compatible local endpoints, you can use any non-empty API key.
 let cfg = OpenAiConfig::new("dummy")
 .with_base_url("http://localhost:8000/v1")
 .with_model("meta-llama/Llama-3.1-8B-Instruct");
 let vllm = OpenAiClient::from_config(cfg)?;
 let resp = text::generate(
 &vllm,
 ChatRequest::new(vec![user!("Hello from vLLM")]),
 text::GenerateOptions::default(),
 )
 .await?;
 println!("{}", resp.content_text().unwrap_or_default());
 Ok(())
}

Rerank (registry-first)

use siumai::prelude::unified::*;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
 let registry_id = "cohere:rerank-english-v3.0";
 let model = registry::global().reranking_model(registry_id)?;
 let response = rerank::rerank(
 &model,
 RerankRequest::new(
 "rerank-english-v3.0".to_string(),
 "Which document is about Rust SDK architecture?".to_string(),
 vec![
 "A Python crawler tutorial".to_string(),
 "A Rust SDK architecture guide".to_string(),
 "A dumpling recipe".to_string(),
 ],
 )
 .with_top_n(2),
 rerank::RerankOptions::default(),
 )
 .await?;
 println!("{:?}", response.sorted_indices());
 Ok(())
}

OpenAI endpoint routing (Responses vs Chat Completions)

Builder convenience (compat)

use siumai::compat::Siumai;
let client = Siumai::builder()
 .openai()
 .api_key(std::env::var("OPENAI_API_KEY")?)
 .model("gpt-4o-mini")
 .build()
 .await?;

Streaming

use futures::StreamExt;
use siumai::prelude::unified::*;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
 let reg = registry::global();
 let model = reg.language_model("openai:gpt-4o-mini")?;
 let mut stream = text::stream(
 &model,
 ChatRequest::new(vec![user!("Stream a long answer")]),
 text::StreamOptions::default(),
 )
 .await?;
 while let Some(ev) = stream.next().await {
 let ev = ev?;
 if let Some(delta) = ev.text_delta() { print!("{delta}"); }
 }
 Ok(())
}

Streaming cancellation

use futures::StreamExt;
use siumai::prelude::unified::*;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
 let reg = registry::global();
 let model = reg.language_model("openai:gpt-4o-mini")?;
 let handle = text::stream_with_cancel(
 &model,
 ChatRequest::new(vec![user!("Stream...")]),
 text::StreamOptions::default(),
 )
 .await?;
 let ChatStreamHandle { mut stream, cancel } = handle;
 let reader = tokio::spawn(async move { while stream.next().await.is_some() {} });
 cancel.cancel();
 reader.await?;
 Ok(())
}

OpenAI WebSocket streaming (Responses API)

# Cargo.toml
siumai = { version = "0.11.0-beta.8", features = ["openai-websocket"] }

	use futures::StreamExt;
	use siumai::prelude::unified::*;
	use siumai::providers::openai::{OpenAiClient, OpenAiConfig, OpenAiWebSocketTransport};
	use std::sync::Arc;
	#[tokio::main]
	async fn main() -> Result<(), Box<dyn std::error::Error>> {
	 // OpenAI WebSocket connections are time-limited; by default we avoid reusing connections
	 // older than ~55 minutes. Customize or disable if needed.
	 let ws = OpenAiWebSocketTransport::default()
	 // Keep up to N idle connections for concurrent tool loops.
	 .with_max_idle_connections(2);
	 // let ws = ws.with_max_connection_age(std::time::Duration::from_secs(55 * 60));
	 // let ws = ws.without_max_connection_age();
 // Optional: connection-local incremental continuation (`previous_response_id`).
 // Note: OpenAI caches the most recent response per WebSocket connection, so this is
 // only unambiguous when `max_idle_connections == 1`.
 // let ws = ws.with_stateful_previous_response_id(true);
 let cfg = OpenAiConfig::new(std::env::var("OPENAI_API_KEY")?)
 .with_model("gpt-4o-mini")
 .with_http_transport(Arc::new(ws.clone()));
 let client = OpenAiClient::from_config(cfg)?;
 // Streaming `/responses` requests are routed through WebSocket; everything else uses HTTP.
 let mut stream = text::stream(
 &client,
 ChatRequest::new(vec![user!("Hello!")]),
 text::StreamOptions::default(),
 )
 .await?;
 while let Some(ev) = stream.next().await {
 let ev = ev?;
 if let Some(delta) = ev.text_delta() {
 print!("{delta}");
 }
 }
 ws.close().await; // optional: close the cached connection
 Ok(())
}

OpenAI WebSocket session (warm-up + single connection)

use futures::StreamExt;
use siumai::prelude::unified::*;
use siumai::providers::openai::{OpenAiConfig, OpenAiWebSocketSession};
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
 let cfg = OpenAiConfig::new(std::env::var("OPENAI_API_KEY")?)
 .with_model("gpt-4o-mini");
 let session = OpenAiWebSocketSession::from_config_default_http(cfg)?;
 session.warm_up_messages(vec![user!("Warm up with my toolset")], None).await?;
 let mut stream = text::stream(
 &session,
 ChatRequest::new(vec![user!("Hello!")]),
 text::StreamOptions::default(),
 )
 .await?;
 while let Some(ev) = stream.next().await {
 let ev = ev?;
 if let Some(delta) = ev.text_delta() {
 print!("{delta}");
 }
 }
 session.close().await;
 Ok(())
}

Structured output

1) Provider‐agnostic decoding (recommended for cross‐provider flows)

use serde::Deserialize;
use siumai::prelude::unified::*;
use siumai_extras::highlevel::object::generate_object;
#[derive(Deserialize, Debug)]
struct Post { title: String }
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
 let cfg = siumai::providers::openai::OpenAiConfig::new(std::env::var("OPENAI_API_KEY")?)
 .with_model("gpt-4o-mini");
 let client = siumai::providers::openai::OpenAiClient::from_config(cfg)?;
 let (post, _resp) = generate_object::<Post>(
 &client,
 vec![user!("Return JSON: {\"title\":\"hi\"}")],
 None,
 Default::default(),
 ).await?;
 println!("{}", post.title);
 Ok(())
}

2) Provider‐native structured outputs (example: OpenAI Responses API)

use siumai::prelude::unified::*;
use siumai::provider_ext::openai::options::{OpenAiChatRequestExt, OpenAiOptions, ResponsesApiConfig};
use serde_json::json;
let schema = json!({"type":"object","properties":{"title":{"type":"string"}},"required":["title"]});
let req = ChatRequestBuilder::new()
 .message(user!("Return an object with title"))
 .build()
 .with_openai_options(OpenAiOptions::new().with_responses_api(
 ResponsesApiConfig::new().with_response_format(json!({
 "type": "json_object",
 "json_schema": { "schema": schema, "strict": true }
 }))
 ));
let resp = text::generate(&client, req, text::GenerateOptions::default()).await?;
// Optionally: further validate/repair/deserialize using `siumai-extras` helpers.

Retries

use siumai::prelude::unified::*;
use siumai::retry_api::{retry_with, RetryOptions};
let text = client
 .ask_with_retry("Hello".to_string(), RetryOptions::backoff())
 .await?;

Customization

HTTP configuration

use siumai::prelude::unified::*;
use siumai::providers::openai::{OpenAiClient, OpenAiConfig};
let http_cfg = HttpConfig::builder()
 .timeout(Some(std::time::Duration::from_secs(30)))
 .connect_timeout(Some(std::time::Duration::from_secs(10)))
 .user_agent(Some("my-app/1.0"))
 .header("X-User-Project", "acme")
 .stream_disable_compression(true) // keep SSE stable; default can be controlled by env
 .build();
let cfg = OpenAiConfig::new(std::env::var("OPENAI_API_KEY")?)
 .with_model("gpt-4o-mini")
 .with_http_config(http_cfg);
let client = OpenAiClient::from_config(cfg)?;

use siumai::experimental::execution::http::client::build_http_client_from_config;
use siumai::prelude::unified::*;
use siumai::providers::openai::{OpenAiClient, OpenAiConfig};
// Construct a reusable HTTP config
let http_cfg = HttpConfig::builder()
 .timeout(Some(std::time::Duration::from_secs(30)))
 .connect_timeout(Some(std::time::Duration::from_secs(10)))
 .user_agent(Some("my-app/1.0"))
 .proxy(Some("http://proxy.example.com:8080"))
 .header("X-User-Project", "acme")
 .stream_disable_compression(true) // explicit SSE stability
 .build();
// Build reqwest client using the shared helper
let http = build_http_client_from_config(&http_cfg)?;
let cfg = OpenAiConfig::new(std::env::var("OPENAI_API_KEY")?)
 .with_model("gpt-4o-mini")
 .with_http_config(http_cfg);
let client = OpenAiClient::new(cfg, http);

use siumai::prelude::unified::*;
use siumai::providers::openai::{OpenAiClient, OpenAiConfig};
let http = reqwest::Client::builder()
 .timeout(std::time::Duration::from_secs(30))
 // .danger_accept_invalid_certs(true) // if needed for dev
 .build()?;
let cfg = OpenAiConfig::new(std::env::var("OPENAI_API_KEY")?)
 .with_model("gpt-4o-mini");
let client = OpenAiClient::new(cfg, http);

use siumai::prelude::unified::*;
use siumai::experimental::execution::middleware::samples::chain_default_and_clamp;
use siumai::experimental::execution::http::interceptor::LoggingInterceptor;
use siumai::prelude::unified::registry::{create_provider_registry, RegistryOptions};
use std::collections::HashMap;
use std::sync::Arc;
let reg = create_provider_registry(
 HashMap::new(),
 Some(RegistryOptions {
 separator: ':',
 language_model_middleware: chain_default_and_clamp(),
 http_interceptors: vec![Arc::new(LoggingInterceptor)],
 http_config: None,
 retry_options: None,
 max_cache_entries: Some(128),
 client_ttl: None,
 auto_middleware: true,
 })
);

Extras (siumai-extras)

Examples

cargo run --example basic-chat --features openai
cargo run --example streaming --features openai
cargo run -p siumai-extras --example basic-orchestrator --features openai
cargo run --example bedrock-chat --features bedrock

Status and notes

Acknowledgements

Changelog and license