Chilkat HOME ASP Visual Basic VB.NET C# Visual C++ C MFC Delphi FoxPro Java Perl PHP Python Ruby SQL Server VBScript
A Simple Web CrawlerThis demonstrates a very simple web crawler using the Chilkat Spider component.
import chilkat # The Chilkat Spider component/library is free. spider = chilkat.CkSpider() seenDomains = chilkat.CkStringArray() seedUrls = chilkat.CkStringArray() seenDomains.put_Unique(True) seedUrls.put_Unique(True) seedUrls.Append("http://directory.google.com/Top/Recreation/Outdoors/Hiking/Backpacking/") # Set our outbound URL exclude patterns spider.AddAvoidOutboundLinkPattern("*?id=*") spider.AddAvoidOutboundLinkPattern("*.mypages.*") spider.AddAvoidOutboundLinkPattern("*.personal.*") spider.AddAvoidOutboundLinkPattern("*.comcast.*") spider.AddAvoidOutboundLinkPattern("*.aol.*") spider.AddAvoidOutboundLinkPattern("*~*") # Use a cache so we don't have to re-fetch URLs previously fetched. spider.put_CacheDir("c:/spiderCache/") spider.put_FetchFromCache(True) spider.put_UpdateCache(True) while seedUrls.get_Count() > 0 : url = seedUrls.pop() spider.Initialize(url) # Spider 5 URLs of this domain. # but first, save the base domain in seenDomains domain = spider.getDomain(url) seenDomains.Append(spider.getBaseDomain(domain)) for i in range(0,5): success = spider.CrawlNext() if (success != True): break # Display the URL we just crawled. print spider.lastUrl() # If the last URL was retrieved from cache, # we won't wait. Otherwise we'll wait 1 second # before fetching the next URL. if (spider.get_LastFromCache() != True): spider.SleepMs(1000) # Add the outbound links to seedUrls, except # for the domains we've already seen. for i in range(0,spider.get_NumOutboundLinks()): url = spider.getOutboundLink(i) domain = spider.getDomain(url) baseDomain = spider.getBaseDomain(domain) if (not seenDomains.Contains(baseDomain)): seedUrls.Append(url) # Don't let our list of seedUrls grow too large. if (seedUrls.get_Count() > 1000): break |
Need a specific example? Send a request to support@chilkatsoft.com
© 2000-2007 Chilkat Software, Inc. All Rights Reserved.