Download objects into memory
This page shows you how to download objects from your Cloud Storage buckets into memory by using Cloud Storage client libraries. Downloading into memory is useful when you want to avoid unnecessary writes to persistent storage. For instructions on how to download objects directly to persistent memory, see Downloading objects to persistent memory. For a conceptual overview of how downloads work in Cloud Storage, see Uploads and downloads.
Required roles
In order to get the required permissions for downloading objects into memory,
ask your administrator to grant you the Storage Object Viewer
(roles/storage.objectViewer) role on the bucket.
This role contains the permission required to download objects. To see the exact permission that's required, expand the Required permissions section:
Required permissions
storage.objects.get
You might also be able to get this permission with other predefined roles or custom roles.
For instructions on granting roles on buckets, see Set and manage IAM policies on buckets.
Download an object into memory
Client libraries
For more information, see the
Cloud Storage C++ API
reference documentation.
To authenticate to Cloud Storage, set up Application Default Credentials.
For more information, see
Set up authentication for client libraries.
For more information, see the
Cloud Storage C# API
reference documentation.
To authenticate to Cloud Storage, set up Application Default Credentials.
For more information, see
Set up authentication for client libraries.
For more information, see the
Cloud Storage Go API
reference documentation.
To authenticate to Cloud Storage, set up Application Default Credentials.
For more information, see
Set up authentication for client libraries.
For more information, see the
Cloud Storage Java API
reference documentation.
To authenticate to Cloud Storage, set up Application Default Credentials.
For more information, see
Set up authentication for client libraries.
For more information, see the
Cloud Storage Node.js API
reference documentation.
To authenticate to Cloud Storage, set up Application Default Credentials.
For more information, see
Set up authentication for client libraries.
For more information, see the
Cloud Storage PHP API
reference documentation.
To authenticate to Cloud Storage, set up Application Default Credentials.
For more information, see
Set up authentication for client libraries.
For more information, see the
Cloud Storage Python API
reference documentation.
To authenticate to Cloud Storage, set up Application Default Credentials.
For more information, see
Set up authentication for client libraries.
For more information, see the
Cloud Storage Ruby API
reference documentation.
To authenticate to Cloud Storage, set up Application Default Credentials.
For more information, see
Set up authentication for client libraries.
C++
namespacegcs=::google::cloud::storage;
[](gcs::Clientclient,std::stringconst&bucket_name,
std::stringconst&object_name){
gcs::ObjectReadStreamstream=client.ReadObject(bucket_name,object_name);
std::stringbuffer{std::istreambuf_iterator<char>(stream),
std::istreambuf_iterator<char>()};
if(stream.bad())throwgoogle::cloud::Status(stream.status());
std::cout << "The object has " << buffer.size() << " characters\n";
}C#
usingGoogle.Cloud.Storage.V1 ;
usingSystem;
usingSystem.IO;
publicclassDownloadObjectIntoMemorySample
{
publicStreamDownloadObjectIntoMemory(
stringbucketName="unique-bucket-name",
stringobjectName="file-name")
{
varstorage=StorageClient .Create ();
Streamstream=newMemoryStream();
storage.DownloadObject(bucketName,objectName,stream);
Console.WriteLine($"The contents of {objectName} from bucket {bucketName} are downloaded");
returnstream;
}
}
Go
import(
"context"
"fmt"
"io"
"time"
"cloud.google.com/go/storage"
)
// downloadFileIntoMemory downloads an object.
funcdownloadFileIntoMemory(wio.Writer ,bucket,objectstring)([]byte,error){
// bucket := "bucket-name"
// object := "object-name"
ctx:=context.Background()
client,err:=storage.NewClient(ctx)
iferr!=nil{
returnnil,fmt.Errorf("storage.NewClient: %w",err)
}
deferclient.Close()
ctx,cancel:=context.WithTimeout(ctx,time.Second*50)
defercancel()
rc,err:=client.Bucket (bucket).Object (object).NewReader (ctx)
iferr!=nil{
returnnil,fmt.Errorf("Object(%q).NewReader: %w",object,err)
}
deferrc.Close()
data,err:=io.ReadAll(rc)
iferr!=nil{
returnnil,fmt.Errorf("io.ReadAll: %w",err)
}
fmt.Fprintf(w,"Blob %v downloaded.\n",object)
returndata,nil
}
Java
importcom.google.cloud.storage.Storage ;
importcom.google.cloud.storage.StorageOptions ;
importjava.nio.charset.StandardCharsets;
publicclass DownloadObjectIntoMemory{
publicstaticvoiddownloadObjectIntoMemory(
StringprojectId,StringbucketName,StringobjectName){
// The ID of your GCP project
// String projectId = "your-project-id";
// The ID of your GCS bucket
// String bucketName = "your-unique-bucket-name";
// The ID of your GCS object
// String objectName = "your-object-name";
Storage storage=StorageOptions .newBuilder().setProjectId(projectId).build().getService ();
byte[]content=storage.readAllBytes (bucketName,objectName);
System.out.println(
"The contents of "
+objectName
+" from bucket name "
+bucketName
+" are: "
+newString(content,StandardCharsets.UTF_8));
}
}Node.js
/**
* TODO(developer): Uncomment the following lines before running the sample.
*/
// The ID of your GCS bucket
// const bucketName = 'your-unique-bucket-name';
// The ID of your GCS file
// const fileName = 'your-file-name';
// Imports the Google Cloud client library
const{Storage}=require('@google-cloud/storage');
// Creates a client
conststorage=newStorage();
asyncfunctiondownloadIntoMemory(){
// Downloads the file into a buffer in memory.
constcontents=awaitstorage.bucket(bucketName).file(fileName).download ();
console.log(
`Contents of gs://${bucketName}/${fileName} are ${contents.toString()}.`
);
}
downloadIntoMemory().catch(console.error);PHP
use Google\Cloud\Storage\StorageClient;
/**
* Download an object from Cloud Storage and save into a buffer in memory.
*
* @param string $bucketName The name of your Cloud Storage bucket.
* (e.g. 'my-bucket')
* @param string $objectName The name of your Cloud Storage object.
* (e.g. 'my-object')
*/
function download_object_into_memory(
string $bucketName,
string $objectName
): void {
$storage = new StorageClient();
$bucket = $storage->bucket($bucketName);
$object = $bucket->object($objectName);
$contents = $object->downloadAsString();
printf(
'Downloaded %s from gs://%s/%s' . PHP_EOL,
$contents,
$bucketName,
$objectName
);
}Python
fromgoogle.cloudimport storage
defdownload_blob_into_memory(bucket_name, blob_name):
"""Downloads a blob into memory."""
# The ID of your GCS bucket
# bucket_name = "your-bucket-name"
# The ID of your GCS object
# blob_name = "storage-object-name"
storage_client = storage .Client ()
bucket = storage_client.bucket (bucket_name)
# Construct a client side representation of a blob.
# Note `Bucket.blob` differs from `Bucket.get_blob` as it doesn't retrieve
# any content from Google Cloud Storage. As we don't need additional data,
# using `Bucket.blob` is preferred here.
blob = bucket.blob(blob_name)
contents = blob.download_as_bytes ()
print(
"Downloaded storage object {} from bucket {} as the following bytes object: {}.".format(
blob_name, bucket_name, contents.decode("utf-8")
)
)
Ruby
# The name of the bucket to access
# bucket_name = "my-bucket"
# The name of the remote file to download
# file_name = "file.txt"
require"google/cloud/storage"
storage=Google::Cloud::Storage .new
bucket=storage.bucketbucket_name,skip_lookup:true
file=bucket.file file_name
downloaded=file.download
downloaded.rewind# Optional - not needed on first read
contents=downloaded.read
puts"Contents of storage object #{file.name} in bucket #{bucket_name} are: #{contents}"
What's next
- Transfer data from cloud providers or other online sources, such as URL lists.