Get Base Domains

Demonstrates how to accumulate a list of unique domain names referenced from outbound URLs.

 Chilkat Python Module Downloads for Windows, Linux, and MAC OS X

import chilkat

#  The Chilkat Spider component/library is free.
spider = chilkat.CkSpider()

domainList = chilkat.CkStringArray()

#  Set the Unique property so that duplicates are not added.
domainList.put_Unique(True)

#  Crawl the home page of joelonsoftware.com and get the outbound URLs
spider.Initialize("www.joelonsoftware.com")
spider.AddUnspidered("http://www.joelonsoftware.com/")

success = spider.CrawlNext()

#  Build a list of unique domains.

for i in range(0,spider.get_NumOutboundLinks()):
    url = spider.getOutboundLink(i)
    domainList.Append(spider.getUrlDomain(url))

#  Display the domains.
for i in range(0,domainList.get_Count()):
    print domainList.getString(i)
    print spider.getBaseDomain(domainList.getString(i))\
         + "\r\n"