Perl
Chilkat2 Perl
S3 Upload the Parts for a Multipart Upload
See more Amazon S3 (new) Examples
This example uploads a large file in parts. The multipart upload needs to have been first initiated prior to uploading the parts.See http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html for more information about uploading parts.
Chilkat Perl Downloads
use strict;
use warnings;
use Chilkat2;
# In the 1st step for uploading a large file, the multipart upload was initiated
# as shown here: Initiate Multipart Upload
# Other S3 Multipart Upload Examples:
# Complete Multipart Upload
# Abort Multipart Upload
# List Parts
# When we initiated the multipart upload, we saved the XML response to a file. This
# XML response contains the UploadId. We'll begin by loading that XML and getting
# the Upload ID.
my $xmlInit = Chilkat2::Xml->new;
my $success = $xmlInit->LoadXmlFile('s3_multipart_uploads/initiate.xml');
if (!$success) {
print "Did not find the initiate.xml XML file.\n";
exit;
}
my $uploadId = $xmlInit->GetChildContent('UploadId');
print 'UploadId = ', $uploadId, "\n";
# When uploading parts, we need to keep an XML record of each part number
# and its corresponding ETag, which is received in the response for each part.
# There can be up to 10000 parts, numbered 1 to 10000.
# After all parts have been uploaded, the final step will be to complete
# the multipart upload (see Complete Multipart Upload)
# In this example, the large file we want to upload is somethingBig.zip
my $fileToUploadPath = 's3_multipart_uploads/somethingBig.zip';
# The minimum allowed part size is 5MB (5242880 bytes). The last part can be smaller because
# it will contain the remainder of the file. (This minimum is enforced by the AWS service.)
# We'll use the minimum allowed part size for this example.
my $partSize = 5242880;
# Let's use Chilkat's FileAccess API to examine the file to be uploaded. We'll get the size
# of the file and find out how many parts will be needed, including the final "partial" part.
my $fac = Chilkat2::FileAccess->new;
$fac->OpenForRead($fileToUploadPath);
# How many parts will there be if each part is 5242880 bytes?
my $numParts = $fac->GetNumBlocks($partSize);
print 'numParts = ', $numParts, "\n";
$fac->FileClose();
# Imagine that we may be running this for the 1st time, or maybe we already
# attempted to upload parts, and something failed. Maybe there was a network problem
# the resulted in not all parts getting uploaded. We'll write this code so that if run again,
# it will upload whatever parts haven't yet been uploaded.
# We'll keep a partsList.xml file to record the parts that have already been successfully
# uploaded. If this file does not yet exist, we'll create it..
my $partsListFile = 's3_multipart_uploads/partsList.xml';
my $partsListXml = Chilkat2::Xml->new;
if ($fac->FileExists($partsListFile)) {
$partsListXml->LoadXmlFile($partsListFile);
}
# Make sure the top-level tag is "CompleteMultipartUpload"
$partsListXml->Tag('CompleteMultipartUpload');
# --------------------------------------
# Before entering the loop to upload parts,
# setup the REST object with AWS authentication,
# and make the initial connection.
my $rest = Chilkat2::Rest->new;
# Connect to the Amazon AWS REST server.
my $bTls = 1;
my $port = 443;
my $bAutoReconnect = 1;
$success = $rest->Connect('s3.amazonaws.com', $port, $bTls, $bAutoReconnect);
# ----------------------------------------------------------------------------
# Important: For buckets created in regions outside us-east-1,
# there are three important changes that need to be made.
# See Working with S3 Buckets in Non-us-east-1 Regions for the details.
# ----------------------------------------------------------------------------
# Provide AWS credentials for the REST call.
my $authAws = Chilkat2::AuthAws->new;
$authAws->AccessKey('AWS_ACCESS_KEY');
$authAws->SecretKey('AWS_SECRET_KEY');
$authAws->ServiceName('s3');
$success = $rest->SetAuthAws($authAws);
# Set the bucket name via the HOST header.
# In this case, the bucket name is "chilkat100".
$rest->Host('chilkat100.s3.amazonaws.com');
# --------------------------------------
my $partNumber = 1;
my $sbPartNumber = Chilkat2::StringBuilder->new;
while ($partNumber <= $numParts) {
print '---- ', $partNumber, " ----\n";
# This cumbersome way of converting an integer to a string is because
# Chilkat examples are written in a script that is converted to many programming languages.
# At this time, the translator does not have integer-to-string code generation capability..
$sbPartNumber->Clear();
$sbPartNumber->AppendInt($partNumber);
my $bPartAlreadyUploaded = 0;
# If there are no children, then the XML is empty and no parts have yet been uploaded.
my $numUploadedParts = $partsListXml->NumChildren;
if ($numUploadedParts > 0) {
# If some parts have been uploaded, check to see if this particular part was already upload.
# If so, then it can be skipped.
# Position ourselves at the 1st record.
my $xRec0 = $partsListXml->GetChild(0);
my $foundRec = $xRec0->FindNextRecord('PartNumber', $sbPartNumber->GetAsString());
if ($xRec0->LastMethodSuccess) {
$bPartAlreadyUploaded = 1;
print 'Part ', $partNumber, " was previously uploaded.\n";
print $foundRec->GetXml(), "\n";
}
}
# If this part was not already uploaded, we need to upload.
# Also update the partsListXml and save as each part is successfully uploaded.
if (!$bPartAlreadyUploaded) {
print 'Uploading part ', $partNumber, " ...\n";
# Setup the stream source for the large file to be uploaded..
my $fileStream = Chilkat2::Stream->new;
$fileStream->SourceFile($fileToUploadPath);
# The Chilkat Stream API has features to make uploading a parts
# of a file easy. Indicate the part size by setting the SourceFilePartSize
# property.
$fileStream->SourceFilePartSize($partSize);
# Our HTTP start line to upload a part will look like this:
# PUT /ObjectName?partNumber=PartNumber&uploadId=UploadId HTTP/1.1
# Set the query params. We'll need partNumber and uploadId.
# Make sure the query params from previous iterations are clear.
$rest->ClearAllQueryParams();
$rest->AddQueryParam('partNumber', $sbPartNumber->GetAsString());
$rest->AddQueryParam('uploadId', $uploadId);
# Upload this particular file part.
# Tell the fileStream which part is being uploaded.
# Our partNumber is 1-based (the 1st part is at index 1), but the fileStream's SourceFilePart
# property is 0-based. Therefore we use partNumber-1.
$fileStream->SourceFilePart($partNumber - 1);
# Because the SourceFilePart and SourceFilePartSize properties are set, the stream will
# will provide just that part of the file.
my $responseStr = $rest->FullRequestStream('PUT', '/somethingBig.zip', $fileStream);
if (!$rest->LastMethodSuccess) {
print $rest->LastErrorText, "\n";
exit;
}
if ($rest->ResponseStatusCode != 200) {
# Examine the request/response to see what happened.
print 'response status code = ', $rest->ResponseStatusCode, "\n";
print 'response status text = ', $rest->ResponseStatusText, "\n";
print 'response header: ', $rest->ResponseHeader, "\n";
print 'response body: ', $responseStr, "\n";
print "---\n";
print 'LastRequestStartLine: ', $rest->LastRequestStartLine, "\n";
print 'LastRequestHeader: ', $rest->LastRequestHeader, "\n";
exit;
}
# OK, this part was uploaded..
# The response will have a 0-length body. The only information we need is the
# ETag response header field.
my $etag = $rest->ResponseHdrByName('ETag');
# It should be present, but just in case there was no ETag header...
if (!$rest->LastMethodSuccess) {
print "No ETag response header found!\n";
print 'response header: ', $rest->ResponseHeader, "\n";
exit;
}
# We need to add record to the partsListXml.
# The record will look like this:
# <Part>
# <PartNumber>PartNumber</PartNumber>
# <ETag>ETag</ETag>
# </Part>
my $xPart = $partsListXml->NewChild('Part', '');
$xPart->NewChildInt2('PartNumber', $partNumber);
$xPart->NewChild2('ETag', $etag);
$success = $partsListXml->SaveXml($partsListFile);
if (!$success) {
print $partsListXml->LastErrorText, "\n";
exit;
}
print '-- Part ', $partNumber, " uploaded. ---------------------\n";
}
$partNumber = $partNumber + 1;
}
print "Finished. All parts uploaded.\n";