Get Base Domains

Demonstrates how to accumulate a list of unique domain names referenced from outbound URLs.

Downloads for Windows/Linux and Install Instructions

require 'chilkat'

#  The Chilkat Spider component/library is free.
spider = Chilkat::CkSpider.new()

domainList = Chilkat::CkStringArray.new()

#  Set the Unique property so that duplicates are not added.
domainList.put_Unique(true)

#  Crawl the home page of joelonsoftware.com and get the outbound URLs
spider.Initialize("www.joelonsoftware.com")
spider.AddUnspidered("http://www.joelonsoftware.com/")

success = spider.CrawlNext()

#  Build a list of unique domains.

for i in 0 .. spider.get_NumOutboundLinks() - 1
    url = spider.getOutboundLink(i)
    domainList.Append(spider.getUrlDomain(url))
end

#  Display the domains.
for i in 0 .. domainList.get_Count() - 1
    print domainList.getString(i) + "\n";
    print spider.getBaseDomain(domainList.getString(i)) + 
        "\r\n" + "\n";
end