Chilkat HOME ASP Visual Basic VB.NET C# Visual C++ C MFC Delphi FoxPro Java Perl PHP Python Ruby SQL Server VBScript
|
A Simple Web CrawlerThis demonstrates a very simple web crawler using the Chilkat Spider component.
// The Chilkat Spider component/library is free. Chilkat.Spider spider = new Chilkat.Spider(); Chilkat.StringArray seenDomains = new Chilkat.StringArray(); Chilkat.StringArray seedUrls = new Chilkat.StringArray(); seenDomains.Unique = true; seedUrls.Unique = true; seedUrls.Append("http://directory.google.com/Top/Recreation/Outdoors/Hiking/Backpacking/"); // Set our outbound URL exclude patterns spider.AddAvoidOutboundLinkPattern("*?id=*"); spider.AddAvoidOutboundLinkPattern("*.mypages.*"); spider.AddAvoidOutboundLinkPattern("*.personal.*"); spider.AddAvoidOutboundLinkPattern("*.comcast.*"); spider.AddAvoidOutboundLinkPattern("*.aol.*"); spider.AddAvoidOutboundLinkPattern("*~*"); // Use a cache so we don't have to re-fetch URLs previously fetched. spider.CacheDir = "c:/spiderCache/"; spider.FetchFromCache = true; spider.UpdateCache = true; while (seedUrls.Count > 0) { string url; url = seedUrls.Pop(); spider.Initialize(url); // Spider 5 URLs of this domain. // but first, save the base domain in seenDomains string domain; domain = spider.GetDomain(url); seenDomains.Append(spider.GetBaseDomain(domain)); int i; bool success; for (i = 0; i <= 4; i++) { success = spider.CrawlNext(); if (success != true) { break; } // Display the URL we just crawled. textBox1.Text += spider.LastUrl + "\r\n"; // If the last URL was retrieved from cache, // we won't wait. Otherwise we'll wait 1 second // before fetching the next URL. if (spider.LastFromCache != true) { spider.SleepMs(1000); } } // Add the outbound links to seedUrls, except // for the domains we've already seen. for (i = 0; i <= spider.NumOutboundLinks - 1; i++) { url = spider.GetOutboundLink(i); domain = spider.GetDomain(url); string baseDomain; baseDomain = spider.GetBaseDomain(domain); if (!seenDomains.Contains(baseDomain)) { seedUrls.Append(url); } // Don't let our list of seedUrls grow too large. if (seedUrls.Count > 1000) { break; } } } |
Need a specific example? Send a request to support@chilkatsoft.com
© 2000-2008 Chilkat Software, Inc. All Rights Reserved.