Dart

Amazon S3 List More than 1000 Objects in Bucket

See more Amazon S3 Examples

S3 limits the size of the "List Objects" response to 1000 objects. To retrieve a listing of all of the objects in a bucket containing more than 1000 objects, we'll need to send several requests using continuation tokens. Dart

import 'package:chilkat/chilkat.dart';

void main() {
  // This example assumes the Chilkat HTTP API to have been previously unlocked.
  // See Global Unlock Sample for sample code.

  final http = CkHttp();

  // Insert your access key here:
  http.awsAccessKey = 'AWS_ACCESS_KEY';

  // Insert your secret key here:
  http.awsSecretKey = 'AWS_SECRET_KEY';

  // For the example, we'll get a listing containing approx. 25 objects
  // using continuation tokens with an artificially low max-keys set to 5.
  // (You may omit the max-keys query parameter to get the default 1000 maximum
  // number of keys per request.)

  final sbContinuationToken = CkStringBuilder();
  var bGetMore = true;
  final sbUri = CkStringBuilder();
  final xml = CkXml();

  var itemKey = '';
  var itemSizeDecimalStr = '';
  var lastModTimestamp = '';
  final dt = CkDateTime();
  final dtObj = CkDtObj();

  while (bGetMore) {

    // Make sure to set the "list-type" query param equal to "2".
    // This chooses the GET Bucket (List Objects) Version 2 of the method.

    // IMPORTANT: You must include the max-keys param to get a response with a continuation token.
    // S3 limits the number of objects in a single response to 1000. Therefore, your max-keys
    // can be up to 1000.  If, for example, you have 2000 objects and do not specify
    // max-keys, then the response will contain 1000 objects with no continuation token. 
    sbUri.setString('chilkat100?list-type=2&max-keys=5');
    if (sbContinuationToken.length > 0) {
      sbUri.append('&continuation-token=');
      sbUri.append(sbContinuationToken.getEncoded('url', 'utf-8'));
    }

    final String strXml;
    try {
      strXml = http.s3ListBucketObjects(sbUri.getAsString());
    } on ChilkatException catch (e) {
      print(e.lastErrorText);
      return;
    }

    print('Response status code = ${http.lastStatus}');

    try {
      xml.loadXml(strXml);
    } on ChilkatException catch (e) {
      print(e.lastErrorText);
      return;
    }

    // If the response status code was not 200, then the XML response is not a 
    // listing of objects, but instead contains error information.
    if (http.lastStatus != 200) {
      print(xml.getXml());
      print('Failed.');
      return;
    }

    // If this is not the final response, then we'll get an XML response that begins
    // like this.  (The IsTruncated child will be "true", and the NextContinuationToken
    // will have a value to be used in the next request.)
    // 	<?xml version="1.0" encoding="UTF-8" ?>
    // 	<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
    // 	    <Name>chilkat100</Name>
    // 	    <Prefix />
    // 	    <NextContinuationToken>1Mlcx+W9OKsr8cxp3DP6r71wgsTUMj0vqlntWoaJKNbYdrauLdf40LsUdBeSYGFhzbGIHdcf5DSLcEBWbqG+1fW1UcQkUW1V4qgQONAKOwb8y8vOLJAQ8iQ==</NextContinuationToken>
    // 	    <KeyCount>5</KeyCount>
    // 	    <MaxKeys>5</MaxKeys>
    // 	    <IsTruncated>true</IsTruncated>
    // 	

    // Iterate over the bucket items in this chunk get information for each..
    final numItems = xml.numChildrenHavingTag('Contents');
    print('Number of bucket items = $numItems');

    var i = 0;
    while (i < numItems) {
      xml.i = i;

      itemKey = xml.getChildContent('Contents[i]|Key');
      itemSizeDecimalStr = xml.getChildContent('Contents[i]|Size');
      lastModTimestamp = xml.getChildContent('Contents[i]|LastModified');

      dt.setFromRfc822(lastModTimestamp);
      // Get a local date/time.
      final bLocal = true;
      dt.toDtObj(bLocal, dtObj);

      print('$i: $itemKey, $itemSizeDecimalStr, ${dtObj.day}-${dtObj.month}-${dtObj.year}:${dtObj.hour}:${dtObj.minute}');

      i++;
    }

    // Check IsTruncated.
    bGetMore = false;
    if (xml.childContentMatches('IsTruncated', 'true', true)) {
      sbContinuationToken.setString(xml.getChildContent('NextContinuationToken'));
      if (sbContinuationToken.length > 0) {
        bGetMore = true;
      }
    }
  }

  // The output of this example (when tested) was:

  // Response status code = 200
  // Number of bucket items = 5
  // 0: Abc.ics, 1833, 25-5-2011:9:53
  // 1: Corpse Bride film poster.jpg, 53481, 6-9-2016:13:32
  // 2: chiliPepper.gif, 7718, 12-3-2017:12:18
  // 3: chilkatdude.jpg, 35137, 20-5-2011:16:14
  // 4: cloud.search/dfe/indexer/pscc/2016/3/28/id,x-2-15-0-25-87-0.json, 1238, 2-4-2016:12:0
  // Response status code = 200
  // Number of bucket items = 5
  // 0: cloud.search/dfe/indexer/pscc/2016/3/28/idx-2-15-0-25-87-0.json, 1238, 2-4-2016:11:33
  // 1: dude.gif, 6373, 25-5-2011:17:29
  // 2: french.txt, 47, 12-3-2017:12:18
  // 3: hamlet.xml, 279658, 2-5-2016:12:21
  // 4: hamlet_play.xml, 279658, 20-3-2017:8:22
  // Response status code = 200
  // Number of bucket items = 5
  // 0: images/sea_creatures/starfish123.jpg, 6229, 19-1-2017:10:45
  // 1: images/sea_creatures/starfishåäö.jpg, 6229, 19-1-2017:12:7
  // 2: new folder/, 0, 26-11-2014:12:36
  // 3: new_starfish.jpg, 6229, 20-3-2017:8:22
  // 4: pigs.xml, 2804, 20-3-2017:8:22
  // Response status code = 200
  // Number of bucket items = 5
  // 0: somethingBig.zip, 13089458, 26-9-2016:9:29
  // 1: starfish.jpg, 6229, 12-3-2017:12:18
  // 2: starfish/, 0, 10-11-2014:10:7
  // 3: starfish/starfish.jpg, 6229, 10-11-2014:10:8
  // 4: starfish/starfish2.jpg, 6229, 19-11-2014:10:36
  // Response status code = 200
  // Number of bucket items = 5
  // 0: starfish/starfish3.jpg, 6229, 24-11-2014:14:33
  // 1: starfish2.jpg, 5987, 20-4-2012:12:6
  // 2: starfish3.jpg, 5987, 11-4-2012:7:10
  // 3: starfishA.jpg, 6229, 10-5-2016:8:44
  // 4: starfishCust.jpg, 6229, 12-11-2014:18:25
  // Response status code = 200
  // Number of bucket items = 1
  // 0: xyz.ics, 1833, 25-5-2011:8:52
  // 
  // 
}