Get Base Domains

Demonstrates how to accumulate a list of unique domain names referenced from outbound URLs.

Download Chilkat Ruby Library

require 'chilkat'

# The Chilkat Spider component/library is free.
spider = Chilkat::CkSpider.new()

domainList = Chilkat::CkStringArray.new()

# Set the Unique property so that duplicates are not added.
domainList.put_Unique(true)

# Crawl the home page of joelonsoftware.com and get the outbound URLs
spider.Initialize("www.joelonsoftware.com")
spider.AddUnspidered("http://www.joelonsoftware.com/")

success = spider.CrawlNext()

# Build a list of unique domains.

for i in 0 .. spider.get_NumOutboundLinks() - 1
    url = spider.getOutboundLink(i)
    domainList.Append(spider.getDomain(url))
end

# Display the domains.
for i in 0 .. domainList.get_Count() - 1
    print domainList.getString(i) + "\n";
    print spider.getBaseDomain(domainList.getString(i)) + 
        "\n" + "\n";
end