Using speech-to-speech translation

# Set language_code to your desired language, in this case, Mandarin Chinese.
speech_config = SpeechConfig(language_code="cmn")
config = LiveConnectConfig(
 response_modalities=["AUDIO"],
 speech_config=speech_config,
 input_audio_transcription=input_transcription,
 output_audio_transcription=output_transcription,
)
audio_file = Part.from_uri(file_uri=audio_url, mime_type="audio/mpeg")
contents = [audio_file]
response = client.models.generate_content(model=MODEL_ID, contents=contents)
display(Markdown(response.text))

importasyncio
# Set model generation_config
CONFIG = {
 "response_modalities": ["AUDIO"],
 "speech_config": {
 "language_code": "cmn",
 },
}
headers = {
 "Content-Type": "application/json",
 "Authorization": f"Bearer {bearer_token[0]}",
}
# Connect to the server
async with connect(SERVICE_URL, additional_headers=headers) as ws:
 # Setup the session
 await ws.send(
 json.dumps(
 {
 "setup": {
 "model": MODEL,
 "generation_config": CONFIG,
 "input_audio_transcription": {},
 "output_audio_transcription": {},
 "enable_speech_to_speech_translation": True,
 }
 }
 )
 )
 # Receive setup response
 raw_response = await ws.recv(decode=False)
 setup_response = json.loads(raw_response.decode("ascii"))
 print(setup_response)
 msg = {
 "realtime_input": {
 "audio": {
 "mime_type": "audio/pcm",
 "data": base64.b64encode(wav_data).decode('utf-8'),
 }
 }
 }
 await ws.send(json.dumps(msg))
 overall_responses = []
 timeout_seconds = 10 # Set timeout to 3 seconds
 # Receive chucks of server response with a timeout
 try:
 while True:
 try:
 raw_response = await asyncio.wait_for(ws.recv(decode=False), timeout_seconds)
 response = json.loads(raw_response.decode())
 server_content = response.pop("serverContent", None)
 if server_content is None:
 break
 # Input Transcription.
 input_transcription = server_content.pop("inputTranscription", None)
 if input_transcription is not None:
 raw_text = input_transcription.pop("text", None)
 if raw_text is not None:
 display(Markdown(f"**Input>** {raw_text}"))
 # Output Transcription.
 output_transcription = server_content.pop("outputTranscription", None)
 if output_transcription is not None:
 raw_text = output_transcription.pop("text", None)
 if raw_text is not None:
 display(Markdown(f"**Response>** {raw_text}"))
 model_turn = server_content.pop("modelTurn", None)
 if model_turn is not None:
 parts = model_turn.pop("parts", None)
 if parts is not None:
 for part in parts:
 pcm_data = base64.b64decode(part["inlineData"]["data"])
 overall_responses.append(np.frombuffer(pcm_data, dtype=np.int16))
 # End of turn
 # turn_complete = server_content.pop("turnComplete", None)
 # if turn_complete:
 # break
 except asyncio.TimeoutError:
 print(f"Timeout: No response received from the websocket within {timeout_seconds} seconds.")
 if overall_responses:
 display(Audio(np.concatenate(overall_responses), rate=24000, autoplay=True))
 break # Exit the loop on timeout
 except websockets.exceptions.ConnectionClosed as e:
 print(f"Connection closed by exception, code: {e.code}, reason: {e.reason}")
 if overall_responses:
 display(Audio(np.concatenate(overall_responses), rate=24000, autoplay=True))
 break # Exit the loop on connection closed
 except Exception as e:
 print(f"An unexpected error occurred: {e}")
 if overall_responses:
 display(Audio(np.concatenate(overall_responses), rate=24000, autoplay=True))
 break # Exit the loop on other exceptions
 finally:
 try:
 await ws.close(code=1000, reason="Normal closure") #example close
 except websockets.exceptions.ConnectionClosed as e:
 print(f"Connection closed by exception, code: {e.code}, reason: {e.reason}")
 except Exception as e:
 print(f"An unexpected error occurred: {e}")