Rust

A Simple Web Crawler

See more Spider Examples

This demonstrates a very simple web crawler using the Chilkat Spider component. Rust

let mut success = false;

let spider = chilkat::Spider::new();

let seen_domains = chilkat::StringArray::new();
let seed_urls = chilkat::StringArray::new();

seen_domains.set_unique(true);
seed_urls.set_unique(true);

// You will need to change the start URL to something else...
let _ = seed_urls.append("http://something.whateverYouWant.com/");

// Set outbound URL exclude patterns
// URLs matching any of these patterns will not be added to the 
// collection of outbound links.
spider.add_avoid_outbound_link_pattern("*?id=*");
spider.add_avoid_outbound_link_pattern("*.mypages.*");
spider.add_avoid_outbound_link_pattern("*.personal.*");
spider.add_avoid_outbound_link_pattern("*.comcast.*");
spider.add_avoid_outbound_link_pattern("*.aol.*");
spider.add_avoid_outbound_link_pattern("*~*");

// Use a cache so we don't have to re-fetch URLs previously fetched.
spider.set_cache_dir("c:/spiderCache/");
spider.set_fetch_from_cache(true);
spider.set_update_cache(true);

while seed_urls.count() > 0 {

    let mut url = seed_urls.pop().unwrap_or_default();
    spider.initialize(&url);

    // Spider 5 URLs of this domain.
    // but first, save the base domain in seenDomains
    let mut domain = spider.get_url_domain(&url).unwrap_or_default();
    let _ = seen_domains.append(&spider.get_base_domain(&domain).unwrap_or_default());

    let mut num_crawled = 0;
    success = true;
    while (success) && (num_crawled < 5) {
        if spider.crawl_next().is_ok() {
            // Display the URL we just crawled.
            println!("{}", spider.last_url());

            // If the last URL was retrieved from cache,
            // we won't wait.  Otherwise we'll wait 1 second
            // before fetching the next URL.
            if !spider.last_from_cache() {
                spider.sleep_ms(1000);
            }

            num_crawled = num_crawled + 1;
        }

        // If CrawlNext fails (no more URLs to crawl in this domain), success is false and the loop exits.
    }

    // Add the outbound links to seedUrls, except
    // for the domains we've already seen.
    for i in 0..spider.num_outbound_links() {

        url = spider.get_outbound_link(i).unwrap_or_default();
        domain = spider.get_url_domain(&url).unwrap_or_default();
        let base_domain = spider.get_base_domain(&domain).unwrap_or_default();
        if !seen_domains.contains(&base_domain) {
            // Don't let our list of seedUrls grow too large.
            if seed_urls.count() < 1000 {
                let _ = seed_urls.append(&url);
            }

        }

    }

}