Get Base Domains

Demonstrates how to accumulate a list of unique domain names referenced from outbound URLs.

Download Chilkat .NET for 2.0 Framework

Download Chilkat .NET for 1.0 / 1.1 Framework

' The Chilkat Spider component/library is free.
Dim spider As New Chilkat.Spider()

Dim domainList As New Chilkat.StringArray()

' Set the Unique property so that duplicates are not added.
domainList.Unique = true

' Crawl the home page of joelonsoftware.com and get the outbound URLs
spider.Initialize("www.joelonsoftware.com")
spider.AddUnspidered("http://www.joelonsoftware.com/")

Dim success As Boolean
success = spider.CrawlNext()

' Build a list of unique domains.
Dim i As Integer
Dim url As String
For i = 0 To spider.NumOutboundLinks - 1
    url = spider.GetOutboundLink(i)
    domainList.Append(spider.GetDomain(url))
Next

' Display the domains.
For i = 0 To domainList.Count - 1
    TextBox1.Text = TextBox1.Text & domainList.GetString(i) & vbCrLf
    TextBox1.Refresh()
    TextBox1.Text = TextBox1.Text & spider.GetBaseDomain(domainList.GetString(i)) _
         & vbCrLf & vbCrLf
    TextBox1.Refresh()
Next