Get Base Domains

Demonstrates how to accumulate a list of unique domain names referenced from outbound URLs.

Download Chilkat Python Library

import chilkat

# The Chilkat Spider component/library is free.
spider = chilkat.CkSpider()

domainList = chilkat.CkStringArray()

# Set the Unique property so that duplicates are not added.
domainList.put_Unique(True)

# Crawl the home page of joelonsoftware.com and get the outbound URLs
spider.Initialize("www.joelonsoftware.com")
spider.AddUnspidered("http://www.joelonsoftware.com/")

success = spider.CrawlNext()

# Build a list of unique domains.

for i in range(0,spider.get_NumOutboundLinks()):
    url = spider.getOutboundLink(i)
    domainList.Append(spider.getDomain(url))

# Display the domains.
for i in range(0,domainList.get_Count()):
    print domainList.getString(i)
    print spider.getBaseDomain(domainList.getString(i))\
         + "\n"