Dart

A Simple Web Crawler

See more Spider Examples

This demonstrates a very simple web crawler using the Chilkat Spider component. Dart

import 'package:chilkat/chilkat.dart';

void main() {
  var success = false;

  final spider = CkSpider();

  final seenDomains = CkStringArray();
  final seedUrls = CkStringArray();

  seenDomains.unique = true;
  seedUrls.unique = true;

  // You will need to change the start URL to something else...
  seedUrls.append('http://something.whateverYouWant.com/');

  // Set outbound URL exclude patterns
  // URLs matching any of these patterns will not be added to the 
  // collection of outbound links.
  spider.addAvoidOutboundLinkPattern('*?id=*');
  spider.addAvoidOutboundLinkPattern('*.mypages.*');
  spider.addAvoidOutboundLinkPattern('*.personal.*');
  spider.addAvoidOutboundLinkPattern('*.comcast.*');
  spider.addAvoidOutboundLinkPattern('*.aol.*');
  spider.addAvoidOutboundLinkPattern('*~*');

  // Use a cache so we don't have to re-fetch URLs previously fetched.
  spider.cacheDir = 'c:/spiderCache/';
  spider.fetchFromCache = true;
  spider.updateCache = true;

  while (seedUrls.count > 0) {

    var url = seedUrls.pop();
    spider.initialize(url);

    // Spider 5 URLs of this domain.
    // but first, save the base domain in seenDomains
    var domain = spider.getUrlDomain(url);
    seenDomains.append(spider.getBaseDomain(domain));

    var numCrawled = 0;
    success = true;
    while ((success) && (numCrawled < 5)) {
      success = true;
      try {
        spider.crawlNext();
      } on ChilkatException {
        success = false;
      }
      if (success) {
        // Display the URL we just crawled.
        print(spider.lastUrl);

        // If the last URL was retrieved from cache,
        // we won't wait.  Otherwise we'll wait 1 second
        // before fetching the next URL.
        if (!spider.lastFromCache) {
          spider.sleepMs(1000);
        }

        numCrawled++;
      }

      // If CrawlNext fails (no more URLs to crawl in this domain), success is false and the loop exits.
    }

    // Add the outbound links to seedUrls, except
    // for the domains we've already seen.
    for (var i = 0; i < spider.numOutboundLinks; i++) {

      url = spider.getOutboundLink(i);
      domain = spider.getUrlDomain(url);
      final baseDomain = spider.getBaseDomain(domain);
      if (!seenDomains.contains(baseDomain)) {
        // Don't let our list of seedUrls grow too large.
        if (seedUrls.count < 1000) {
          seedUrls.append(url);
        }
      }
    }
  }
}