Get Base Domains

Demonstrates how to accumulate a list of unique domain names referenced from outbound URLs.

Download Chilkat Spider ActiveX

uses
    Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms,
    Dialogs, StdCtrls,
    SPIDERXLib_TLB,
    CHILKATUTILLib_TLB,
    SPIDERXLib_TLB,
    OleCtrls;

...

procedure TForm1.Button1Click(Sender: TObject);
var
spider: TSpider;
domainList: CHILKATUTILLib_TLB.ICkStringArray;
success: Integer;
i: Integer;
url: String;

begin
//  The Chilkat Spider component/library is free.
spider := TSpider.Create(Self);

domainList := TCkStringArray.Create(Self).ControlInterface;

//  Set the Unique property so that duplicates are not added.
domainList.Unique := 1;

//  Crawl the home page of joelonsoftware.com and get the outbound URLs
spider.Initialize('www.joelonsoftware.com');
spider.AddUnspidered('http://www.joelonsoftware.com/');

success := spider.CrawlNext();

//  Build a list of unique domains.

for i := 0 to spider.NumOutboundLinks - 1 do
  begin
    url := spider.GetOutboundLink(i);
    domainList.Append(spider.GetDomain(url));
  end;

//  Display the domains.
for i := 0 to domainList.Count - 1 do
  begin
    Memo1.Lines.Add(domainList.GetString(i));
    Memo1.Lines.Add(spider.GetBaseDomain(domainList.GetString(i))
         + #13#10);
  end;

end;