Multimodal embeddings API

importvertexai
fromvertexai.vision_modelsimport Image, MultiModalEmbeddingModel
# TODO(developer): Update & uncomment line below
# PROJECT_ID = "your-project-id"
vertexai.init(project=PROJECT_ID, location="us-central1")
model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding@001")
image = Image.load_from_file(
 "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"
)
embeddings = model.get_embeddings(
 image=image,
 contextual_text="Colosseum",
 dimension=1408,
)
print(f"Image Embedding: {embeddings.image_embedding}")
print(f"Text Embedding: {embeddings.text_embedding}")
# Example response:
# Image Embedding: [-0.0123147098, 0.0727171078, ...]
# Text Embedding: [0.00230263756, 0.0278981831, ...]

/**
 * TODO(developer): Uncomment these variables before running the sample.\
 * (Not necessary if passing values as arguments)
 */
// const project = 'YOUR_PROJECT_ID';
// const location = 'YOUR_PROJECT_LOCATION';
// const baseImagePath = 'YOUR_BASE_IMAGE_PATH';
// const textPrompt = 'YOUR_TEXT_PROMPT';
constaiplatform=require('@google-cloud/aiplatform');
// Imports the Google Cloud Prediction service client
const{PredictionServiceClient}=aiplatform.v1;
// Import the helper module for converting arbitrary protobuf.Value objects.
const{helpers}=aiplatform;
// Specifies the location of the api endpoint
constclientOptions={
apiEndpoint:'us-central1-aiplatform.googleapis.com',
};
constpublisher='google';
constmodel='multimodalembedding@001';
// Instantiates a client
constpredictionServiceClient=newPredictionServiceClient (clientOptions);
asyncfunctionpredictImageFromImageAndText(){
// Configure the parent resource
constendpoint=`projects/${project}/locations/${location}/publishers/${publisher}/models/${model}`;
constfs=require('fs');
constimageFile=fs.readFileSync(baseImagePath);
// Convert the image data to a Buffer and base64 encode it.
constencodedImage=Buffer.from(imageFile).toString('base64');
constprompt={
text:textPrompt,
image:{
bytesBase64Encoded:encodedImage,
},
};
constinstanceValue=helpers .toValue(prompt);
constinstances=[instanceValue];
constparameter={
sampleCount:1,
};
constparameters=helpers .toValue(parameter);
constrequest={
endpoint,
instances,
parameters,
};
// Predict request
const[response]=awaitpredictionServiceClient.predict(request);
console.log('Get image embedding response');
constpredictions=response.predictions;
console.log('\tPredictions :');
for(constpredictionofpredictions){
console.log(`\t\tPrediction : ${JSON.stringify(prediction)}`);
}
}
awaitpredictImageFromImageAndText();

importcom.google.cloud.aiplatform.v1beta1.EndpointName;
importcom.google.cloud.aiplatform.v1beta1.PredictResponse;
importcom.google.cloud.aiplatform.v1beta1.PredictionServiceClient;
importcom.google.cloud.aiplatform.v1beta1.PredictionServiceSettings;
importcom.google.gson.Gson;
importcom.google.gson.JsonObject;
importcom.google.protobuf.InvalidProtocolBufferException ;
importcom.google.protobuf.Value ;
importcom.google.protobuf.util.JsonFormat ;
importjava.io.IOException;
importjava.nio.charset.StandardCharsets;
importjava.nio.file.Files;
importjava.nio.file.Paths;
importjava.util.ArrayList;
importjava.util.Base64;
importjava.util.HashMap;
importjava.util.List;
importjava.util.Map;
publicclass PredictImageFromImageAndTextSample{
publicstaticvoidmain(String[]args)throwsIOException{
// TODO(developer): Replace this variable before running the sample.
Stringproject="YOUR_PROJECT_ID";
StringtextPrompt="YOUR_TEXT_PROMPT";
StringbaseImagePath="YOUR_BASE_IMAGE_PATH";
// Learn how to use text prompts to update an image:
// https://cloud.google.com/vertex-ai/docs/generative-ai/image/edit-images
Map<String,Object>parameters=newHashMap<String,Object>();
parameters.put("sampleCount",1);
Stringlocation="us-central1";
Stringpublisher="google";
Stringmodel="multimodalembedding@001";
predictImageFromImageAndText(
project,location,publisher,model,textPrompt,baseImagePath,parameters);
}
// Update images using text prompts
publicstaticvoidpredictImageFromImageAndText(
Stringproject,
Stringlocation,
Stringpublisher,
Stringmodel,
StringtextPrompt,
StringbaseImagePath,
Map<String,Object>parameters)
throwsIOException{
finalStringendpoint=String.format("%s-aiplatform.googleapis.com:443",location);
finalPredictionServiceSettingspredictionServiceSettings=
PredictionServiceSettings.newBuilder().setEndpoint(endpoint).build();
// Initialize client that will be used to send requests. This client only needs to be created
// once, and can be reused for multiple requests.
try(PredictionServiceClientpredictionServiceClient=
PredictionServiceClient.create(predictionServiceSettings)){
finalEndpointNameendpointName=
EndpointName.ofProjectLocationPublisherModelName(project,location,publisher,model);
// Convert the image to Base64
byte[]imageData=Base64.getEncoder().encode(Files.readAllBytes(Paths.get(baseImagePath)));
StringencodedImage=newString(imageData,StandardCharsets.UTF_8);
JsonObjectjsonInstance=newJsonObject();
jsonInstance.addProperty("text",textPrompt);
JsonObjectjsonImage=newJsonObject();
jsonImage.addProperty("bytesBase64Encoded",encodedImage);
jsonInstance.add("image",jsonImage);
Value instanceValue=stringToValue(jsonInstance.toString());
List<Value> instances=newArrayList<>();
instances.add(instanceValue);
Gsongson=newGson();
StringgsonString=gson.toJson(parameters);
Value parameterValue=stringToValue(gsonString);
PredictResponsepredictResponse=
predictionServiceClient.predict(endpointName,instances,parameterValue);
System.out.println("Predict Response");
System.out.println(predictResponse);
for(Value prediction:predictResponse.getPredictionsList()){
System.out.format("\tPrediction: %s\n",prediction);
}
}
}
// Convert a Json string to a protobuf.Value
staticValue stringToValue(Stringvalue)throwsInvalidProtocolBufferException {
Value .Builderbuilder=Value .newBuilder();
JsonFormat .parser().merge(value,builder);
returnbuilder .build();
}
}

import(
"context"
"encoding/json"
"fmt"
"io"
aiplatform"cloud.google.com/go/aiplatform/apiv1beta1"
aiplatformpb"cloud.google.com/go/aiplatform/apiv1beta1/aiplatformpb"
"google.golang.org/api/option"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/structpb"
)
// generateForTextAndImage shows how to use the multimodal model to generate embeddings for
// text and image inputs.
funcgenerateForTextAndImage(wio.Writer,project,locationstring)error{
// location = "us-central1"
ctx:=context.Background()
apiEndpoint:=fmt.Sprintf("%s-aiplatform.googleapis.com:443",location)
client,err:=aiplatform.NewPredictionClient (ctx,option.WithEndpoint(apiEndpoint))
iferr!=nil{
returnfmt.Errorf("failed to construct API client: %w",err)
}
deferclient.Close()
model:="multimodalembedding@001"
endpoint:=fmt.Sprintf("projects/%s/locations/%s/publishers/google/models/%s",project,location,model)
// This is the input to the model's prediction call. For schema, see:
// https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/multimodal-embeddings-api#request_body
instance,err:=structpb.NewValue(map[string]any{
"image":map[string]any{
// Image input can be provided either as a Google Cloud Storage URI or as
// base64-encoded bytes using the "bytesBase64Encoded" field.
"gcsUri":"gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png",
},
"text":"Colosseum",
})
iferr!=nil{
returnfmt.Errorf("failed to construct request payload: %w",err)
}
req:=&aiplatformpb.PredictRequest{
Endpoint:endpoint,
// The model supports only 1 instance per request.
Instances:[]*structpb.Value{instance},
}
resp,err:=client.Predict(ctx,req)
iferr!=nil{
returnfmt.Errorf("failed to generate embeddings: %w",err)
}
instanceEmbeddingsJson,err:=protojson.Marshal(resp.GetPredictions()[0])
iferr!=nil{
returnfmt.Errorf("failed to convert protobuf value to JSON: %w",err)
}
// For response schema, see:
// https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/multimodal-embeddings-api#response-body
varinstanceEmbeddingsstruct{
ImageEmbeddings[]float32`json:"imageEmbedding"`
TextEmbeddings[]float32`json:"textEmbedding"`
}
iferr:=json.Unmarshal(instanceEmbeddingsJson,&instanceEmbeddings);err!=nil{
returnfmt.Errorf("failed to unmarshal JSON: %w",err)
}
imageEmbedding:=instanceEmbeddings.ImageEmbeddings
textEmbedding:=instanceEmbeddings.TextEmbeddings
fmt.Fprintf(w,"Text embedding (length=%d): %v\n",len(textEmbedding),textEmbedding)
fmt.Fprintf(w,"Image embedding (length=%d): %v\n",len(imageEmbedding),imageEmbedding)
// Example response:
// Text embedding (length=1408): [0.0023026613 0.027898183 -0.011858357 ... ]
// Image embedding (length=1408): [-0.012314269 0.07271844 0.00020170923 ... ]
returnnil
}

importvertexai
fromvertexai.vision_modelsimport MultiModalEmbeddingModel, Video
fromvertexai.vision_modelsimport VideoSegmentConfig
# TODO(developer): Update & uncomment line below
# PROJECT_ID = "your-project-id"
vertexai.init(project=PROJECT_ID, location="us-central1")
model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding@001")
embeddings = model.get_embeddings(
 video=Video.load_from_file(
 "gs://cloud-samples-data/vertex-ai-vision/highway_vehicles.mp4"
 ),
 video_segment_config=VideoSegmentConfig(end_offset_sec=1),
)
# Video Embeddings are segmented based on the video_segment_config.
print("Video Embeddings:")
for video_embedding in embeddings.video_embeddings:
 print(
 f"Video Segment: {video_embedding.start_offset_sec} - {video_embedding.end_offset_sec}"
 )
 print(f"Embedding: {video_embedding.embedding}")
# Example response:
# Video Embeddings:
# Video Segment: 0.0 - 1.0
# Embedding: [-0.0206376351, 0.0123456789, ...]

import(
"context"
"encoding/json"
"fmt"
"io"
"time"
aiplatform"cloud.google.com/go/aiplatform/apiv1beta1"
aiplatformpb"cloud.google.com/go/aiplatform/apiv1beta1/aiplatformpb"
"google.golang.org/api/option"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/structpb"
)
// generateForVideo shows how to use the multimodal model to generate embeddings for video input.
funcgenerateForVideo(wio.Writer,project,locationstring)error{
// location = "us-central1"
// The default context timeout may be not enough to process a video input.
ctx,cancel:=context.WithTimeout(context.Background(),15*time.Second)
defercancel()
apiEndpoint:=fmt.Sprintf("%s-aiplatform.googleapis.com:443",location)
client,err:=aiplatform.NewPredictionClient (ctx,option.WithEndpoint(apiEndpoint))
iferr!=nil{
returnfmt.Errorf("failed to construct API client: %w",err)
}
deferclient.Close()
model:="multimodalembedding@001"
endpoint:=fmt.Sprintf("projects/%s/locations/%s/publishers/google/models/%s",project,location,model)
// This is the input to the model's prediction call. For schema, see:
// https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/multimodal-embeddings-api#request_body
instances,err:=structpb.NewValue(map[string]any{
"video":map[string]any{
// Video input can be provided either as a Google Cloud Storage URI or as base64-encoded
// bytes using the "bytesBase64Encoded" field.
"gcsUri":"gs://cloud-samples-data/vertex-ai-vision/highway_vehicles.mp4",
"videoSegmentConfig":map[string]any{
"startOffsetSec":1,
"endOffsetSec":5,
},
},
})
iferr!=nil{
returnfmt.Errorf("failed to construct request payload: %w",err)
}
req:=&aiplatformpb.PredictRequest{
Endpoint:endpoint,
// The model supports only 1 instance per request.
Instances:[]*structpb.Value{instances},
}
resp,err:=client.Predict(ctx,req)
iferr!=nil{
returnfmt.Errorf("failed to generate embeddings: %w",err)
}
instanceEmbeddingsJson,err:=protojson.Marshal(resp.GetPredictions()[0])
iferr!=nil{
returnfmt.Errorf("failed to convert protobuf value to JSON: %w",err)
}
// For response schema, see:
// https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/multimodal-embeddings-api#response-body
varinstanceEmbeddingsstruct{
VideoEmbeddings[]struct{
Embedding[]float32`json:"embedding"`
StartOffsetSecfloat64`json:"startOffsetSec"`
EndOffsetSecfloat64`json:"endOffsetSec"`
}`json:"videoEmbeddings"`
}
iferr:=json.Unmarshal(instanceEmbeddingsJson,&instanceEmbeddings);err!=nil{
returnfmt.Errorf("failed to unmarshal json: %w",err)
}
// Get the embedding for our single video segment (`.videoEmbeddings` object has one entry per
// each processed segment).
videoEmbedding:=instanceEmbeddings.VideoEmbeddings[0]
fmt.Fprintf(w,"Video embedding (seconds: %.f-%.f; length=%d): %v\n",
videoEmbedding.StartOffsetSec,
videoEmbedding.EndOffsetSec,
len(videoEmbedding.Embedding),
videoEmbedding.Embedding,
)
// Example response:
// Video embedding (seconds: 1-5; length=1408): [-0.016427778 0.032878537 -0.030755188 ... ]
returnnil
}

importvertexai
fromvertexai.vision_modelsimport Image, MultiModalEmbeddingModel, Video
fromvertexai.vision_modelsimport VideoSegmentConfig
# TODO(developer): Update & uncomment line below
# PROJECT_ID = "your-project-id"
vertexai.init(project=PROJECT_ID, location="us-central1")
model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding@001")
image = Image.load_from_file(
 "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"
)
video = Video.load_from_file(
 "gs://cloud-samples-data/vertex-ai-vision/highway_vehicles.mp4"
)
embeddings = model.get_embeddings(
 image=image,
 video=video,
 video_segment_config=VideoSegmentConfig(end_offset_sec=1),
 contextual_text="Cars on Highway",
)
print(f"Image Embedding: {embeddings.image_embedding}")
# Video Embeddings are segmented based on the video_segment_config.
print("Video Embeddings:")
for video_embedding in embeddings.video_embeddings:
 print(
 f"Video Segment: {video_embedding.start_offset_sec} - {video_embedding.end_offset_sec}"
 )
 print(f"Embedding: {video_embedding.embedding}")
print(f"Text Embedding: {embeddings.text_embedding}")
# Example response:
# Image Embedding: [-0.0123144267, 0.0727186054, 0.000201397663, ...]
# Video Embeddings:
# Video Segment: 0.0 - 1.0
# Embedding: [-0.0206376351, 0.0345234685, ...]
# Text Embedding: [-0.0207006838, -0.00251058186, ...]

import(
"context"
"encoding/json"
"fmt"
"io"
"time"
aiplatform"cloud.google.com/go/aiplatform/apiv1beta1"
aiplatformpb"cloud.google.com/go/aiplatform/apiv1beta1/aiplatformpb"
"google.golang.org/api/option"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/structpb"
)
// generateForImageTextAndVideo shows how to use the multimodal model to generate embeddings for
// image, text and video data.
funcgenerateForImageTextAndVideo(wio.Writer,project,locationstring)error{
// location = "us-central1"
// The default context timeout may be not enough to process a video input.
ctx,cancel:=context.WithTimeout(context.Background(),15*time.Second)
defercancel()
apiEndpoint:=fmt.Sprintf("%s-aiplatform.googleapis.com:443",location)
client,err:=aiplatform.NewPredictionClient (ctx,option.WithEndpoint(apiEndpoint))
iferr!=nil{
returnfmt.Errorf("failed to construct API client: %w",err)
}
deferclient.Close()
model:="multimodalembedding@001"
endpoint:=fmt.Sprintf("projects/%s/locations/%s/publishers/google/models/%s",project,location,model)
// This is the input to the model's prediction call. For schema, see:
// https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/multimodal-embeddings-api#request_body
instance,err:=structpb.NewValue(map[string]any{
"text":"Domestic cats in natural conditions",
"image":map[string]any{
// Image and video inputs can be provided either as a Google Cloud Storage URI or as
// base64-encoded bytes using the "bytesBase64Encoded" field.
"gcsUri":"gs://cloud-samples-data/generative-ai/image/320px-Felis_catus-cat_on_snow.jpg",
},
"video":map[string]any{
"gcsUri":"gs://cloud-samples-data/video/cat.mp4",
},
})
iferr!=nil{
returnfmt.Errorf("failed to construct request payload: %w",err)
}
req:=&aiplatformpb.PredictRequest{
Endpoint:endpoint,
// The model supports only 1 instance per request.
Instances:[]*structpb.Value{instance},
}
resp,err:=client.Predict(ctx,req)
iferr!=nil{
returnfmt.Errorf("failed to generate embeddings: %w",err)
}
instanceEmbeddingsJson,err:=protojson.Marshal(resp.GetPredictions()[0])
iferr!=nil{
returnfmt.Errorf("failed to convert protobuf value to JSON: %w",err)
}
// For response schema, see:
// https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/multimodal-embeddings-api#response-body
varinstanceEmbeddingsstruct{
ImageEmbeddings[]float32`json:"imageEmbedding"`
TextEmbeddings[]float32`json:"textEmbedding"`
VideoEmbeddings[]struct{
Embedding[]float32`json:"embedding"`
StartOffsetSecfloat64`json:"startOffsetSec"`
EndOffsetSecfloat64`json:"endOffsetSec"`
}`json:"videoEmbeddings"`
}
iferr:=json.Unmarshal(instanceEmbeddingsJson,&instanceEmbeddings);err!=nil{
returnfmt.Errorf("failed to unmarshal JSON: %w",err)
}
imageEmbedding:=instanceEmbeddings.ImageEmbeddings
textEmbedding:=instanceEmbeddings.TextEmbeddings
// Get the embedding for our single video segment (`.videoEmbeddings` object has one entry per
// each processed segment).
videoEmbedding:=instanceEmbeddings.VideoEmbeddings[0].Embedding
fmt.Fprintf(w,"Image embedding (length=%d): %v\n",len(imageEmbedding),imageEmbedding)
fmt.Fprintf(w,"Text embedding (length=%d): %v\n",len(textEmbedding),textEmbedding)
fmt.Fprintf(w,"Video embedding (length=%d): %v\n",len(videoEmbedding),videoEmbedding)
// Example response:
// Image embedding (length=1408): [-0.01558477 0.0258355 0.016342038 ... ]
// Text embedding (length=1408): [-0.005894961 0.008349559 0.015355394 ... ]
// Video embedding (length=1408): [-0.018867437 0.013997682 0.0012682161 ... ]
returnnil
}