(JavaScript) A Simple Web Crawler

This demonstrates a very simple web crawler using the Chilkat Spider component.

Note

This example is intended for running within a Chilkat.Js embedded JavaScript engine. All Chilkat JavaScript examples require Chilkat v11.4.0 or greater.

var success = false;

var spider = new CkSpider();

var seenDomains = new CkStringArray();
var seedUrls = new CkStringArray();

seenDomains.Unique = true;
seedUrls.Unique = true;

// You will need to change the start URL to something else...
seedUrls.Append("http://something.whateverYouWant.com/");

// Set outbound URL exclude patterns
// URLs matching any of these patterns will not be added to the 
// collection of outbound links.
spider.AddAvoidOutboundLinkPattern("*?id=*");
spider.AddAvoidOutboundLinkPattern("*.mypages.*");
spider.AddAvoidOutboundLinkPattern("*.personal.*");
spider.AddAvoidOutboundLinkPattern("*.comcast.*");
spider.AddAvoidOutboundLinkPattern("*.aol.*");
spider.AddAvoidOutboundLinkPattern("*~*");

// Use a cache so we don't have to re-fetch URLs previously fetched.
spider.CacheDir = "c:/spiderCache/";
spider.FetchFromCache = true;
spider.UpdateCache = true;

while (seedUrls.Count > 0) {

    var url = seedUrls.Pop();
    spider.Initialize(url);

    // Spider 5 URLs of this domain.
    // but first, save the base domain in seenDomains
    var domain = spider.GetUrlDomain(url);
    seenDomains.Append(spider.GetBaseDomain(domain));

    var i;

    for (i = 0; i <= 4; i++) {
        success = spider.CrawlNext();
        if (success == true) {

            // Display the URL we just crawled.
            console.log(spider.LastUrl);

            // If the last URL was retrieved from cache,
            // we won't wait.  Otherwise we'll wait 1 second
            // before fetching the next URL.
            if (spider.LastFromCache !== true) {
                spider.SleepMs(1000);
            }

        }
        else {
            // cause the loop to exit..
            i = 999;
        }

    }

    // Add the outbound links to seedUrls, except
    // for the domains we've already seen.
    for (i = 0; i <= spider.NumOutboundLinks - 1; i++) {

        url = spider.GetOutboundLink(i);
        var domain = spider.GetUrlDomain(url);
        var baseDomain = spider.GetBaseDomain(domain);
        if (seenDomains.Contains(baseDomain) == false) {
            // Don't let our list of seedUrls grow too large.
            if (seedUrls.Count < 1000) {
                seedUrls.Append(url);
            }

        }

    }

}