Java

Amazon S3 List More than 1000 Objects in Bucket

See more Amazon S3 Examples

S3 limits the size of the "List Objects" response to 1000 objects. To retrieve a listing of all of the objects in a bucket containing more than 1000 objects, we'll need to send several requests using continuation tokens. Chilkat2 Java

import com.chilkatsoft.chilkat2.ChilkatException;
import com.chilkatsoft.chilkat2.DateTime;
import com.chilkatsoft.chilkat2.DtObj;
import com.chilkatsoft.chilkat2.Http;
import com.chilkatsoft.chilkat2.StringBuilder;
import com.chilkatsoft.chilkat2.Xml;

public class ChilkatExample {

    public static void main(String[] args) {
        // This example assumes the Chilkat HTTP API to have been previously unlocked.
        // See Global Unlock Sample for sample code.

        Http http = new Http();

        // Insert your access key here:
        http.setAwsAccessKey("AWS_ACCESS_KEY");

        // Insert your secret key here:
        http.setAwsSecretKey("AWS_SECRET_KEY");

        // For the example, we'll get a listing containing approx. 25 objects
        // using continuation tokens with an artificially low max-keys set to 5.
        // (You may omit the max-keys query parameter to get the default 1000 maximum
        // number of keys per request.)

        StringBuilder sbContinuationToken = new StringBuilder();
        boolean bGetMore = true;
        StringBuilder sbUri = new StringBuilder();
        Xml xml = new Xml();

        String itemKey = "";
        String itemSizeDecimalStr = "";
        String lastModTimestamp = "";
        DateTime dt = new DateTime();
        DtObj dtObj = new DtObj();

        while (bGetMore) {
            // Make sure to set the "list-type" query param equal to "2".
            // This chooses the GET Bucket (List Objects) Version 2 of the method.

            // IMPORTANT: You must include the max-keys param to get a response with a continuation token.
            // S3 limits the number of objects in a single response to 1000. Therefore, your max-keys
            // can be up to 1000.  If, for example, you have 2000 objects and do not specify
            // max-keys, then the response will contain 1000 objects with no continuation token.
            sbUri.setString("chilkat100?list-type=2&max-keys=5");
            if (sbContinuationToken.getLength() > 0) {
                sbUri.append("&continuation-token=");
                sbUri.append(sbContinuationToken.getEncoded("url", "utf-8"));
            }

            String strXml;
            try {
                strXml = http.s3ListBucketObjects(sbUri.getAsString());
            } catch (ChilkatException e) {
                System.out.println(e.getLastErrorText());
                return;
            }

            System.out.println("Response status code = " + http.getLastStatus());

            try {
                xml.loadXml(strXml);
            } catch (ChilkatException e) {
                System.out.println(e.getLastErrorText());
                return;
            }

            // If the response status code was not 200, then the XML response is not a
            // listing of objects, but instead contains error information.
            if (http.getLastStatus() != 200) {
                System.out.println(xml.getXml());
                System.out.println("Failed.");
                return;
            }

            // If this is not the final response, then we'll get an XML response that begins
            // like this.  (The IsTruncated child will be "true", and the NextContinuationToken
            // will have a value to be used in the next request.)
            //     <?xml version="1.0" encoding="UTF-8" ?>
            //     <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
            //         <Name>chilkat100</Name>
            //         <Prefix />
            //         <NextContinuationToken>1Mlcx+W9OKsr8cxp3DP6r71wgsTUMj0vqlntWoaJKNbYdrauLdf40LsUdBeSYGFhzbGIHdcf5DSLcEBWbqG+1fW1UcQkUW1V4qgQONAKOwb8y8vOLJAQ8iQ==</NextContinuationToken>
            //         <KeyCount>5</KeyCount>
            //         <MaxKeys>5</MaxKeys>
            //         <IsTruncated>true</IsTruncated>
            //

            // Iterate over the bucket items in this chunk get information for each..
            int numItems = xml.numChildrenHavingTag("Contents");
            System.out.println("Number of bucket items = " + numItems);

            int i = 0;
            while (i < numItems) {
                xml.setI(i);

                itemKey = xml.getChildContent("Contents[i]|Key");
                itemSizeDecimalStr = xml.getChildContent("Contents[i]|Size");
                lastModTimestamp = xml.getChildContent("Contents[i]|LastModified");

                dt.setFromRfc822(lastModTimestamp);
                // Get a local date/time.
                boolean bLocal = true;
                dt.toDtObj(bLocal, dtObj);

                System.out.println(i + ": " + itemKey + ", " + itemSizeDecimalStr + ", " + dtObj.getDay() + "-" + dtObj.getMonth() + "-" + dtObj.getYear() + ":" + dtObj.getHour() + ":" + dtObj.getMinute());

                i = i + 1;
            }

            // Check IsTruncated.
            bGetMore = false;
            if (xml.childContentMatches("IsTruncated", "true", true)) {
                sbContinuationToken.setString(xml.getChildContent("NextContinuationToken"));
                if (sbContinuationToken.getLength() > 0) {
                    bGetMore = true;
                }
            }
        }

        // The output of this example (when tested) was:

        // Response status code = 200
        // Number of bucket items = 5
        // 0: Abc.ics, 1833, 25-5-2011:9:53
        // 1: Corpse Bride film poster.jpg, 53481, 6-9-2016:13:32
        // 2: chiliPepper.gif, 7718, 12-3-2017:12:18
        // 3: chilkatdude.jpg, 35137, 20-5-2011:16:14
        // 4: cloud.search/dfe/indexer/pscc/2016/3/28/id,x-2-15-0-25-87-0.json, 1238, 2-4-2016:12:0
        // Response status code = 200
        // Number of bucket items = 5
        // 0: cloud.search/dfe/indexer/pscc/2016/3/28/idx-2-15-0-25-87-0.json, 1238, 2-4-2016:11:33
        // 1: dude.gif, 6373, 25-5-2011:17:29
        // 2: french.txt, 47, 12-3-2017:12:18
        // 3: hamlet.xml, 279658, 2-5-2016:12:21
        // 4: hamlet_play.xml, 279658, 20-3-2017:8:22
        // Response status code = 200
        // Number of bucket items = 5
        // 0: images/sea_creatures/starfish123.jpg, 6229, 19-1-2017:10:45
        // 1: images/sea_creatures/starfishåäö.jpg, 6229, 19-1-2017:12:7
        // 2: new folder/, 0, 26-11-2014:12:36
        // 3: new_starfish.jpg, 6229, 20-3-2017:8:22
        // 4: pigs.xml, 2804, 20-3-2017:8:22
        // Response status code = 200
        // Number of bucket items = 5
        // 0: somethingBig.zip, 13089458, 26-9-2016:9:29
        // 1: starfish.jpg, 6229, 12-3-2017:12:18
        // 2: starfish/, 0, 10-11-2014:10:7
        // 3: starfish/starfish.jpg, 6229, 10-11-2014:10:8
        // 4: starfish/starfish2.jpg, 6229, 19-11-2014:10:36
        // Response status code = 200
        // Number of bucket items = 5
        // 0: starfish/starfish3.jpg, 6229, 24-11-2014:14:33
        // 1: starfish2.jpg, 5987, 20-4-2012:12:6
        // 2: starfish3.jpg, 5987, 11-4-2012:7:10
        // 3: starfishA.jpg, 6229, 10-5-2016:8:44
        // 4: starfishCust.jpg, 6229, 12-11-2014:18:25
        // Response status code = 200
        // Number of bucket items = 1
        // 0: xyz.ics, 1833, 25-5-2011:8:52
        //
        //
    }
}