Get Base Domains

Demonstrates how to accumulate a list of unique domain names referenced from outbound URLs.

Downloads:

MS Windows Visual C/C++ Libraries

Linux/CentOS C/C++ Libraries

MAC OS X C/C++ Libraries

Solaris C/C++ Libraries

C++ Builder Libraries

#include <C_CkSpider.h>
#include <C_CkStringArray.h>

void ChilkatSample(void)
    {
    HCkSpider spider;
    HCkStringArray domainList;
    BOOL success;
    long i;
    const char * url;

    //  The Chilkat Spider component/library is free.
    spider = CkSpider_Create();

    domainList = CkStringArray_Create();

    //  Set the Unique property so that duplicates are not added.
    CkStringArray_putUnique(domainList,TRUE);

    //  Crawl the home page of joelonsoftware.com and get the outbound URLs
    CkSpider_Initialize(spider,"www.joelonsoftware.com");
    CkSpider_AddUnspidered(spider,"http://www.joelonsoftware.com/");

    success = CkSpider_CrawlNext(spider);

    //  Build a list of unique domains.

    for (i = 0; i <= CkSpider_getNumOutboundLinks(spider) - 1; i++) {
        url = CkSpider_getOutboundLink(spider,i);
        CkStringArray_Append(domainList,CkSpider_getDomain(spider,url));
    }

    //  Display the domains.
    for (i = 0; i <= CkStringArray_getCount(domainList) - 1; i++) {
        printf("%s\n",CkStringArray_getString(domainList,i));
        printf("%s\n\n",CkSpider_getBaseDomain(spider,CkStringArray_getString(domainList,i)));
    }


    CkSpider_Dispose(spider);
    CkStringArray_Dispose(domainList);

    }