A Simple Web Crawler

This demonstrates a very simple web crawler using the Chilkat Spider component.

Download Chilkat .NET for 2.0 Framework

Download Chilkat .NET for 1.0 / 1.1 Framework

'  The Chilkat Spider component/library is free.
Dim spider As New Chilkat.Spider()

Dim seenDomains As New Chilkat.StringArray()
Dim seedUrls As New Chilkat.StringArray()

seenDomains.Unique = true
seedUrls.Unique = true

seedUrls.Append("http://directory.google.com/Top/Recreation/Outdoors/Hiking/Backpacking/")

'  Set our outbound URL exclude patterns
spider.AddAvoidOutboundLinkPattern("*?id=*")
spider.AddAvoidOutboundLinkPattern("*.mypages.*")
spider.AddAvoidOutboundLinkPattern("*.personal.*")
spider.AddAvoidOutboundLinkPattern("*.comcast.*")
spider.AddAvoidOutboundLinkPattern("*.aol.*")
spider.AddAvoidOutboundLinkPattern("*~*")

'  Use a cache so we don't have to re-fetch URLs previously fetched.
spider.CacheDir = "c:/spiderCache/"
spider.FetchFromCache = true
spider.UpdateCache = true

While seedUrls.Count > 0

    Dim url As String
    url = seedUrls.Pop()
    spider.Initialize(url)

    '  Spider 5 URLs of this domain.
    '  but first, save the base domain in seenDomains
    Dim domain As String
    domain = spider.GetDomain(url)
    seenDomains.Append(spider.GetBaseDomain(domain))

    Dim i As Long
    Dim success As Boolean
    For i = 0 To 4
        success = spider.CrawlNext()
        If (success <> true) Then
            Exit For
        End If


        '  Display the URL we just crawled.
        TextBox1.Text = TextBox1.Text & spider.LastUrl & vbCrLf

        '  If the last URL was retrieved from cache,
        '  we won't wait.  Otherwise we'll wait 1 second
        '  before fetching the next URL.
        If (spider.LastFromCache <> true) Then
            spider.SleepMs(1000)
        End If

    Next

    '  Add the outbound links to seedUrls, except
    '  for the domains we've already seen.
    For i = 0 To spider.NumOutboundLinks - 1

        url = spider.GetOutboundLink(i)

        domain = spider.GetDomain(url)
        Dim baseDomain As String
        baseDomain = spider.GetBaseDomain(domain)
        If (Not seenDomains.Contains(baseDomain)) Then
            seedUrls.Append(url)
        End If


        '  Don't let our list of seedUrls grow too large.
        If (seedUrls.Count > 1000) Then
            Exit For
        End If

    Next

End While