A Simple Web Crawler

This demonstrates a very simple web crawler using the Chilkat Spider component.

Downloads:

MS Windows Visual C/C++ Libraries

Linux/CentOS C/C++ Libraries

MAC OS X C/C++ Libraries

Solaris C/C++ Libraries

C++ Builder Libraries

#include <C_CkSpider.h>
#include <C_CkStringArray.h>

void ChilkatSample(void)
    {
    HCkSpider spider;
    HCkStringArray seenDomains;
    HCkStringArray seedUrls;
    const char * url;
    const char * domain;
    long i;
    BOOL success;
    const char * baseDomain;

    //  The Chilkat Spider component/library is free.
    spider = CkSpider_Create();

    seenDomains = CkStringArray_Create();
    seedUrls = CkStringArray_Create();

    CkStringArray_putUnique(seenDomains,TRUE);
    CkStringArray_putUnique(seedUrls,TRUE);

    CkStringArray_Append(seedUrls,"http://directory.google.com/Top/Recreation/Outdoors/Hiking/Backpacking/");

    //  Set our outbound URL exclude patterns
    CkSpider_AddAvoidOutboundLinkPattern(spider,"*?id=*");
    CkSpider_AddAvoidOutboundLinkPattern(spider,"*.mypages.*");
    CkSpider_AddAvoidOutboundLinkPattern(spider,"*.personal.*");
    CkSpider_AddAvoidOutboundLinkPattern(spider,"*.comcast.*");
    CkSpider_AddAvoidOutboundLinkPattern(spider,"*.aol.*");
    CkSpider_AddAvoidOutboundLinkPattern(spider,"*~*");

    //  Use a cache so we don't have to re-fetch URLs previously fetched.
    CkSpider_putCacheDir(spider,"c:/spiderCache/");
    CkSpider_putFetchFromCache(spider,TRUE);
    CkSpider_putUpdateCache(spider,TRUE);

    while (CkStringArray_getCount(seedUrls) > 0) {

        url = CkStringArray_pop(seedUrls);
        CkSpider_Initialize(spider,url);

        //  Spider 5 URLs of this domain.
        //  but first, save the base domain in seenDomains

        domain = CkSpider_getDomain(spider,url);
        CkStringArray_Append(seenDomains,CkSpider_getBaseDomain(spider,domain));

        for (i = 0; i <= 4; i++) {
            success = CkSpider_CrawlNext(spider);
            if (success != TRUE) {
                break;
            }

            //  Display the URL we just crawled.
            printf("%s\n",CkSpider_lastUrl(spider));

            //  If the last URL was retrieved from cache,
            //  we won't wait.  Otherwise we'll wait 1 second
            //  before fetching the next URL.
            if (CkSpider_getLastFromCache(spider) != TRUE) {
                CkSpider_SleepMs(spider,1000);
            }

        }

        //  Add the outbound links to seedUrls, except
        //  for the domains we've already seen.
        for (i = 0; i <= CkSpider_getNumOutboundLinks(spider) - 1; i++) {

            url = CkSpider_getOutboundLink(spider,i);

            domain = CkSpider_getDomain(spider,url);

            baseDomain = CkSpider_getBaseDomain(spider,domain);
            if (!CkStringArray_Contains(seenDomains,baseDomain)) {
                CkStringArray_Append(seedUrls,url);
            }

            //  Don't let our list of seedUrls grow too large.
            if (CkStringArray_getCount(seedUrls) > 1000) {
                break;
            }

        }

    }


    CkSpider_Dispose(spider);
    CkStringArray_Dispose(seenDomains);
    CkStringArray_Dispose(seedUrls);

    }