Get Base Domains

Demonstrates how to accumulate a list of unique domain names referenced from outbound URLs.

 Chilkat Perl Module Downloads for Windows, Linux, and MAC OS X

use chilkat();

#  The Chilkat Spider component/library is free.
$spider = new chilkat::CkSpider();

$domainList = new chilkat::CkStringArray();

#  Set the Unique property so that duplicates are not added.
$domainList->put_Unique(1);

#  Crawl the home page of joelonsoftware.com and get the outbound URLs
$spider->Initialize("www.joelonsoftware.com");
$spider->AddUnspidered("http://www.joelonsoftware.com/");

$success = $spider->CrawlNext();

#  Build a list of unique domains.

for ($i = 0; $i <= $spider->get_NumOutboundLinks() - 1; $i++) {
    $url = $spider->getOutboundLink($i);
    $domainList->Append($spider->getUrlDomain($url));
}

#  Display the domains.
for ($i = 0; $i <= $domainList->get_Count() - 1; $i++) {
    print $domainList->getString($i) . "\r\n";
    print $spider->getBaseDomain($domainList->getString($i))
         . "\r\n" . "\r\n";
}