Tcl

A Simple Web Crawler

See more Spider Examples

This demonstrates a very simple web crawler using the Chilkat Spider component. Tcl

load ./chilkat.dll

set success 0

set spider [new_CkSpider]

set seenDomains [new_CkStringArray]

set seedUrls [new_CkStringArray]

CkStringArray_put_Unique $seenDomains 1
CkStringArray_put_Unique $seedUrls 1

# You will need to change the start URL to something else...
CkStringArray_Append $seedUrls "http://something.whateverYouWant.com/"

# Set outbound URL exclude patterns
# URLs matching any of these patterns will not be added to the 
# collection of outbound links.
CkSpider_AddAvoidOutboundLinkPattern $spider "*?id=*"
CkSpider_AddAvoidOutboundLinkPattern $spider "*.mypages.*"
CkSpider_AddAvoidOutboundLinkPattern $spider "*.personal.*"
CkSpider_AddAvoidOutboundLinkPattern $spider "*.comcast.*"
CkSpider_AddAvoidOutboundLinkPattern $spider "*.aol.*"
CkSpider_AddAvoidOutboundLinkPattern $spider "*~*"

# Use a cache so we don't have to re-fetch URLs previously fetched.
CkSpider_put_CacheDir $spider "c:/spiderCache/"
CkSpider_put_FetchFromCache $spider 1
CkSpider_put_UpdateCache $spider 1

while {[CkStringArray_get_Count $seedUrls] > 0} {

    set url [CkStringArray_pop $seedUrls]
    CkSpider_Initialize $spider $url

    # Spider 5 URLs of this domain.
    # but first, save the base domain in seenDomains
    set domain [CkSpider_getUrlDomain $spider $url]
    CkStringArray_Append $seenDomains [CkSpider_getBaseDomain $spider $domain]

    for {set i 0} {$i <= 4} {incr i} {
        set success [CkSpider_CrawlNext $spider]
        if {$success == 1} then {

            # Display the URL we just crawled.
            puts [CkSpider_lastUrl $spider]

            # If the last URL was retrieved from cache,
            # we won't wait.  Otherwise we'll wait 1 second
            # before fetching the next URL.
            if {[CkSpider_get_LastFromCache $spider] != 1} then {
                CkSpider_SleepMs $spider 1000
            }

        }         else {
            # cause the loop to exit..
            set i 999
        }

    }

    # Add the outbound links to seedUrls, except
    # for the domains we've already seen.
    for {set i 0} {$i <= [expr [CkSpider_get_NumOutboundLinks $spider] - 1]} {incr i} {

        set url [CkSpider_getOutboundLink $spider $i]
        set domain [CkSpider_getUrlDomain $spider $url]
        set baseDomain [CkSpider_getBaseDomain $spider $domain]
        if {[CkStringArray_Contains $seenDomains $baseDomain] == 0} then {
            # Don't let our list of seedUrls grow too large.
            if {[CkStringArray_get_Count $seedUrls] < 1000} then {
                CkStringArray_Append $seedUrls $url
            }

        }

    }

}

delete_CkSpider $spider
delete_CkStringArray $seenDomains
delete_CkStringArray $seedUrls