Bidirectional streaming using the Gemini Live API

importFirebaseAI
// Initialize the Gemini Developer API backend service
// Create a `LiveModel` instance with the flash-live model (only model that supports the Live API)
letmodel=FirebaseAI.firebaseAI(backend:.googleAI()).liveModel(
modelName:"gemini-2.0-flash-live-preview-04-09",
// Configure the model to respond with text
generationConfig:LiveGenerationConfig(
responseModalities:[.text]
)
)
do{
letsession=tryawaitmodel.connect()
// Load the audio file, or tap a microphone
guardletaudioFile=NSDataAsset(name:"audio.pcm")else{
fatalError("Failed to load audio file")
}
// Provide the audio data
awaitsession.sendAudioRealtime(audioFile.data)
varoutputText=""
fortryawaitmessageinsession.responses{
ifcaselet.content(content)=message.payload{
content.modelTurn?.parts.forEach{partin
ifletpart=partas?TextPart{
outputText+=part.text
}
}
// Optional: if you don't require to send more requests.
ifcontent.isTurnComplete{
awaitsession.close()
}
}
}
// Output received from the server.
print(outputText)
}catch{
fatalError(error.localizedDescription)
}

// Initialize the Gemini Developer API backend service
// Create a `LiveModel` instance with the flash-live model (only model that supports the Live API)
valmodel=Firebase.ai(backend=GenerativeBackend.googleAI()).liveModel(
modelName="gemini-2.0-flash-live-preview-04-09",
// Configure the model to respond with text
generationConfig=liveGenerationConfig{
responseModality=ResponseModality.TEXT
}
)
valsession=model.connect()
// Provide a text prompt
valaudioContent=content("user"){audioData}
session.send(audioContent)
varoutputText=""
session.receive().collect{
if(it.status==Status.TURN_COMPLETE){
// Optional: if you don't require to send more requests.
session.stopReceiving();
}
outputText=outputText+it.text
}
// Output received from the server.
println(outputText)

ExecutorServiceexecutor=Executors.newFixedThreadPool(1);
// Initialize the Gemini Developer API backend service
// Create a `LiveModel` instance with the flash-live model (only model that supports the Live API)
LiveGenerativeModellm=FirebaseAI.getInstance(GenerativeBackend.googleAI()).liveModel(
"gemini-2.0-flash-live-preview-04-09",
// Configure the model to respond with text
newLiveGenerationConfig.Builder()
.setResponseModalities(ResponseModality.TEXT)
.build()
);
LiveModelFuturesmodel=LiveModelFutures.from(lm);
ListenableFuture<LiveSession>sessionFuture=model.connect();
class LiveContentResponseSubscriberimplementsSubscriber<LiveContentResponse>{
@Override
publicvoidonSubscribe(Subscriptions){
s.request(Long.MAX_VALUE);// Request an unlimited number of items
}
@Override
publicvoidonNext(LiveContentResponseliveContentResponse){
// Handle the response from the server.
System.out.println(liveContentResponse.getText());
}
@Override
publicvoidonError(Throwablet){
System.err.println("Error: "+t.getMessage());
}
@Override
publicvoidonComplete(){
System.out.println("Done receiving messages!");
}
}
Futures.addCallback(sessionFuture,newFutureCallback<LiveSession>(){
@Override
publicvoidonSuccess(LiveSessionses){
LiveSessionFuturessession=LiveSessionFutures.from(ses);
// Send Audio data
session.send(newContent.Builder().addInlineData(audioData,"audio/pcm").build());
session.send(text);
Publisher<LiveContentResponse>publisher=session.receive();
publisher.subscribe(newLiveContentResponseSubscriber());
}
@Override
publicvoidonFailure(Throwablet){
// Handle exceptions
}
},executor);

// Initialize the Gemini Developer API backend service
constai=getAI(firebaseApp,{backend:newGoogleAIBackend()});
// Create a `LiveGenerativeModel` instance with the flash-live model (only model that supports the Live API)
constmodel=getLiveGenerativeModel(ai,{
model:"gemini-2.0-flash-live-preview-04-09",
// Configure the model to respond with text
generationConfig:{
responseModalities:[ResponseModality.TEXT],
},
});
constsession=awaitmodel.connect();
// TODO(developer): Collect audio data (16-bit 16kHz PCM)
// const audioData = ...
// Send audio
constaudioPart={
inlineData:{data:audioData,mimeType:"audio/pcm"},
};
session.send([audioPart]);
// Collect text from model's turn
lettext="";
constmessages=session.receive();
forawait(constmessageofmessages){
switch(message.type){
case"serverContent":
if(message.turnComplete){
console.log(text);
}else{
constparts=message.modelTurn?.parts;
if(parts){
text+=parts.map((part)=>part.text).join("");
}
}
break;
case"toolCall":
// Ignore
case"toolCallCancellation":
// Ignore
}
}

import'package:firebase_ai/firebase_ai.dart';
import'package:firebase_core/firebase_core.dart';
import'firebase_options.dart';
import'package:your_audio_recorder_package/your_audio_recorder_package.dart';
import'dart:async';
lateLiveModelSession_session;
final_audioRecorder=YourAudioRecorder();
awaitFirebase.initializeApp(
options:DefaultFirebaseOptions.currentPlatform,
);
// Initialize the Gemini Developer API backend service
// Create a `LiveModel` instance with the flash-live model (only model that supports the Live API)
finalmodel=FirebaseAI.googleAI().liveGenerativeModel(
model:'gemini-2.0-flash-live-preview-04-09',
// Configure the model to respond with text
liveGenerationConfig:LiveGenerationConfig(responseModalities:ResponseModalities.text),
);
_session=awaitmodel.connect();
finalaudioRecordStream=_audioRecorder.startRecordingStream();
finalmediaChunkStream=audioRecordStream.map((data){
returnInlineDataPart('audio/pcm',data);
});
await_session.startMediaStream(mediaChunkStream);
finalresponseStream=_session.receive();
returnresponseStream.asyncMap((response)async{
if(response.parts.isNotEmpty&&response.parts.first.text!=null){
returnresponse.parts.first.text!;
}else{
throwException('Text response not found.');
}
});
Futuremain()async{
try{
finaltextStream=awaitaudioToText();
awaitfor(finaltextintextStream){
print('Received text: $text');
// Handle the text response
}
}catch(e){
print('Error: $e');
}
}

usingFirebase;
usingFirebase.AI;
asyncTaskSendAudioReceiveText(){
// Initialize the Gemini Developer API backend service
// Create a `LiveModel` instance with the flash-live model (only model that supports the Live API)
varmodel=FirebaseAI.GetInstance(FirebaseAI.Backend.GoogleAI()).GetLiveModel(
modelName:"gemini-2.0-flash-live-preview-04-09",
// Configure the model to respond with text
liveGenerationConfig:newLiveGenerationConfig(
responseModalities:new[]{ResponseModality.Text})
);
LiveSessionsession=awaitmodel.ConnectAsync();
// Start a coroutine to send audio from the Microphone
varrecordingCoroutine=StartCoroutine(SendAudio(session));
// Receive the response
awaitforeach(varmessageinsession.ReceiveAsync()){
// Process the received message
if(!string.IsNullOrEmpty(message.Text)){
UnityEngine.Debug.Log("Received message: "+message.Text);
}
}
StopCoroutine(recordingCoroutine);
}
IEnumeratorSendAudio(LiveSessionliveSession){
stringmicrophoneDeviceName=null;
intrecordingFrequency=16000;
intrecordingBufferSeconds=2;
varrecordingClip=Microphone.Start(microphoneDeviceName,true,
recordingBufferSeconds,recordingFrequency);
intlastSamplePosition=0;
while(true){
if(!Microphone.IsRecording(microphoneDeviceName)){
yieldbreak;
}
intcurrentSamplePosition=Microphone.GetPosition(microphoneDeviceName);
if(currentSamplePosition!=lastSamplePosition){
// The Microphone uses a circular buffer, so we need to check if the
// current position wrapped around to the beginning, and handle it
// accordingly.
intsampleCount;
if(currentSamplePosition>lastSamplePosition){
sampleCount=currentSamplePosition-lastSamplePosition;
}else{
sampleCount=recordingClip.samples-lastSamplePosition+currentSamplePosition;
}
if(sampleCount>0){
// Get the audio chunk
float[]samples=newfloat[sampleCount];
recordingClip.GetData(samples,lastSamplePosition);
// Send the data, discarding the resulting Task to avoid the warning
_=liveSession.SendAudioAsync(samples);
lastSamplePosition=currentSamplePosition;
}
}
// Wait for a short delay before reading the next sample from the Microphone
constfloatMicrophoneReadDelay=0.5f;
yieldreturnnewWaitForSeconds(MicrophoneReadDelay);
}
}

importFirebaseAI
// Initialize the Gemini Developer API backend service
// Create a `LiveModel` instance with the flash-live model (only model that supports the Live API)
letmodel=FirebaseAI.firebaseAI(backend:.googleAI()).liveModel(
modelName:"gemini-2.0-flash-live-preview-04-09",
// Configure the model to respond with audio
generationConfig:LiveGenerationConfig(
responseModalities:[.audio]
)
)
do{
letsession=tryawaitmodel.connect()
// Provide a text prompt
lettext="tell a short story"
awaitsession.sendTextRealtime(text)
varoutputText=""
fortryawaitmessageinsession.responses{
ifcaselet.content(content)=message.payload{
content.modelTurn?.parts.forEach{partin
ifletpart=partas?InlineDataPart,part.mimeType.starts(with:"audio/pcm"){
// Handle 16bit pcm audio data at 24khz
playAudio(part.data)
}
}
// Optional: if you don't require to send more requests.
ifcontent.isTurnComplete{
awaitsession.close()
}
}
}
}catch{
fatalError(error.localizedDescription)
}

// Initialize the Gemini Developer API backend service
// Create a `LiveModel` instance with the flash-live model (only model that supports the Live API)
valmodel=Firebase.ai(backend=GenerativeBackend.googleAI()).liveModel(
modelName="gemini-2.0-flash-live-preview-04-09",
// Configure the model to respond with audio
generationConfig=liveGenerationConfig{
responseModality=ResponseModality.AUDIO
}
)
valsession=model.connect()
// Provide a text prompt
valtext="tell a short story"
session.send(text)
session.receive().collect{
if(it.turnComplete){
// Optional: if you don't require to send more requests.
session.stopReceiving();
}
// Handle 16bit pcm audio data at 24khz
playAudio(it.data)
}

ExecutorServiceexecutor=Executors.newFixedThreadPool(1);
// Initialize the Gemini Developer API backend service
// Create a `LiveModel` instance with the flash-live model (only model that supports the Live API)
LiveGenerativeModellm=FirebaseAI.getInstance(GenerativeBackend.googleAI()).liveModel(
"gemini-2.0-flash-live-preview-04-09",
// Configure the model to respond with text
newLiveGenerationConfig.Builder()
.setResponseModalities(ResponseModality.AUDIO)
.build()
);
LiveModelFuturesmodel=LiveModelFutures.from(lm);
ListenableFuture<LiveSession>sessionFuture=model.connect();
class LiveContentResponseSubscriberimplementsSubscriber<LiveContentResponse>{
@Override
publicvoidonSubscribe(Subscriptions){
s.request(Long.MAX_VALUE);// Request an unlimited number of items
}
@Override
publicvoidonNext(LiveContentResponseliveContentResponse){
// Handle 16bit pcm audio data at 24khz
liveContentResponse.getData();
}
@Override
publicvoidonError(Throwablet){
System.err.println("Error: "+t.getMessage());
}
@Override
publicvoidonComplete(){
System.out.println("Done receiving messages!");
}
}
Futures.addCallback(sessionFuture,newFutureCallback<LiveSession>(){
@Override
publicvoidonSuccess(LiveSessionses){
LiveSessionFuturessession=LiveSessionFutures.from(ses);
// Provide a text prompt
Stringtext="tell me a short story?";
session.send(text);
Publisher<LiveContentResponse>publisher=session.receive();
publisher.subscribe(newLiveContentResponseSubscriber());
}
@Override
publicvoidonFailure(Throwablet){
// Handle exceptions
}
},executor);

// Initialize the Gemini Developer API backend service
constai=getAI(firebaseApp,{backend:newGoogleAIBackend()});
// Create a `LiveGenerativeModel` instance with the flash-live model (only model that supports the Live API)
constmodel=getLiveGenerativeModel(ai,{
model:"gemini-2.0-flash-live-preview-04-09",
// Configure the model to respond with audio
generationConfig:{
responseModalities:[ResponseModality.AUDIO],
},
});
constsession=awaitmodel.connect();
// Provide a text prompt
constprompt="tell a short story";
session.send(prompt);
// Handle the model's audio output
constmessages=session.receive();
forawait(constmessageofmessages){
switch(message.type){
case"serverContent":
if(message.turnComplete){
// TODO(developer): Handle turn completion
}elseif(message.interrupted){
// TODO(developer): Handle the interruption
break;
}elseif(message.modelTurn){
constparts=message.modelTurn?.parts;
parts?.forEach((part)=>{
if(part.inlineData){
// TODO(developer): Play the audio chunk
}
});
}
break;
case"toolCall":
// Ignore
case"toolCallCancellation":
// Ignore
}
}

import'package:firebase_ai/firebase_ai.dart';
import'package:firebase_core/firebase_core.dart';
import'firebase_options.dart';
import'dart:async';
import'dart:typed_data';
lateLiveModelSession_session;
Future<Stream<Uint8List>>textToAudio(StringtextPrompt)async{
WidgetsFlutterBinding.ensureInitialized();
awaitFirebase.initializeApp(
options:DefaultFirebaseOptions.currentPlatform,
);
// Initialize the Gemini Developer API backend service
// Create a `LiveModel` instance with the flash-live model (only model that supports the Live API)
finalmodel=FirebaseAI.googleAI().liveGenerativeModel(
model:'gemini-2.0-flash-live-preview-04-09',
// Configure the model to respond with audio
liveGenerationConfig:LiveGenerationConfig(responseModalities:ResponseModalities.audio),
);
_session=awaitmodel.connect();
finalprompt=Content.text(textPrompt);
await_session.send(input:prompt);
return_session.receive().asyncMap((response)async{
if(responseisLiveServerContent && response.modelTurn?.parts!=null){
for(finalpartinresponse.modelTurn!.parts){
if(partisInlineDataPart){
returnpart.bytes;
}
}
}
throwException('Audio data not found');
});
}
Future<void>main()async{
try{
finalaudioStream=awaittextToAudio('Convert this text to audio.');
awaitfor(finalaudioDatainaudioStream){
// Process the audio data (e.g., play it using an audio player package)
print('Received audio data: ${audioData.length} bytes');
// Example using flutter_sound (replace with your chosen package):
// await _flutterSoundPlayer.startPlayer(fromDataBuffer: audioData);
}
}catch(e){
print('Error: $e');
}
}

usingFirebase;
usingFirebase.AI;
asyncTaskSendTextReceiveAudio(){
// Initialize the Gemini Developer API backend service
// Create a `LiveModel` instance with the flash-live model (only model that supports the Live API)
varmodel=FirebaseAI.GetInstance(FirebaseAI.Backend.GoogleAI()).GetLiveModel(
modelName:"gemini-2.0-flash-live-preview-04-09",
// Configure the model to respond with audio
liveGenerationConfig:newLiveGenerationConfig(
responseModalities:new[]{ResponseModality.Audio})
);
LiveSessionsession=awaitmodel.ConnectAsync();
// Provide a text prompt
varprompt=ModelContent.Text("Convert this text to audio.");
awaitsession.SendAsync(content:prompt,turnComplete:true);
// Start receiving the response
awaitReceiveAudio(session);
}
Queue<float>audioBuffer=new();
asyncTaskReceiveAudio(LiveSessionsession){
intsampleRate=24000;
intchannelCount=1;
// Create a looping AudioClip to fill with the received audio data
intbufferSamples=(int)(sampleRate*channelCount);
AudioClipclip=AudioClip.Create("StreamingPCM",bufferSamples,channelCount,
sampleRate,true,OnAudioRead);
// Attach the clip to an AudioSource and start playing it
AudioSourceaudioSource=GetComponent<AudioSource>();
audioSource.clip=clip;
audioSource.loop=true;
audioSource.Play();
// Start receiving the response
awaitforeach(varmessageinsession.ReceiveAsync()){
// Process the received message
foreach(float[]pcmDatainmessage.AudioAsFloat){
lock(audioBuffer){
foreach(floatsampleinpcmData){
audioBuffer.Enqueue(sample);
}
}
}
}
}
// This method is called by the AudioClip to load audio data.
privatevoidOnAudioRead(float[]data){
intsamplesToProvide=data.Length;
intsamplesProvided=0;
lock(audioBuffer){
while(samplesProvided < samplesToProvide && audioBuffer.Count > 0){
data[samplesProvided]=audioBuffer.Dequeue();
samplesProvided++;
}
}
while(samplesProvided < samplesToProvide){
data[samplesProvided]=0.0f;
samplesProvided++;
}
}