Get Base Domains

Download: Chilkat .NET Assemblies

Demonstrates how to accumulate a list of unique domain names referenced from outbound URLs.

// The Chilkat Spider component/library is free.
Chilkat.Spider spider = new Chilkat.Spider();

Chilkat.StringArray domainList = new Chilkat.StringArray();

// Set the Unique property so that duplicates are not added.
domainList.Unique = true;

// Crawl the home page of joelonsoftware.com and get the outbound URLs
spider.Initialize("www.joelonsoftware.com");
spider.AddUnspidered("http://www.joelonsoftware.com/");

bool success;
success = spider.CrawlNext();

// Build a list of unique domains.
int i;
string url;
for (i = 0; i <= spider.NumOutboundLinks - 1; i++) {
    url = spider.GetOutboundLink(i);
    domainList.Append(spider.GetDomain(url));
}

// Display the domains.
for (i = 0; i <= domainList.Count - 1; i++) {
    textBox1.Text += domainList.GetString(i) + "\r\n";
    textBox1.Refresh();
    textBox1.Text += spider.GetBaseDomain(domainList.GetString(i))
         + "\r\n" + "\r\n";
    textBox1.Refresh();
}