Chilkat HOME Android™ ASP Visual Basic VB.NET C# iOS (IPhone) Objective-C C++ C Unicode C++ Unicode C MFC Delphi DLL Delphi ActiveX FoxPro Java Perl PHP Extension PHP ActiveX Python PowerShell Ruby SQL Server VBScript
(C#) A Simple Web CrawlerThis demonstrates a very simple web crawler using the Chilkat Spider component. Download: Chilkat .NET Assemblies // The Chilkat Spider component/library is free. Chilkat.Spider spider = new Chilkat.Spider(); Chilkat.StringArray seenDomains = new Chilkat.StringArray(); Chilkat.StringArray seedUrls = new Chilkat.StringArray(); seenDomains.Unique = true; seedUrls.Unique = true; // You will need to change the start URL to something else... seedUrls.Append("http://something.whateverYouWant.com/"); // Set outbound URL exclude patterns // URLs matching any of these patterns will not be added to the // collection of outbound links. spider.AddAvoidOutboundLinkPattern("*?id=*"); spider.AddAvoidOutboundLinkPattern("*.mypages.*"); spider.AddAvoidOutboundLinkPattern("*.personal.*"); spider.AddAvoidOutboundLinkPattern("*.comcast.*"); spider.AddAvoidOutboundLinkPattern("*.aol.*"); spider.AddAvoidOutboundLinkPattern("*~*"); // Use a cache so we don't have to re-fetch URLs previously fetched. spider.CacheDir = "c:/spiderCache/"; spider.FetchFromCache = true; spider.UpdateCache = true; while (seedUrls.Count > 0) { string url; url = seedUrls.Pop(); spider.Initialize(url); // Spider 5 URLs of this domain. // but first, save the base domain in seenDomains string domain; domain = spider.GetDomain(url); seenDomains.Append(spider.GetBaseDomain(domain)); int i; bool success; for (i = 0; i <= 4; i++) { success = spider.CrawlNext(); if (success != true) { break; } // Display the URL we just crawled. textBox1.Text += spider.LastUrl + "\r\n"; // If the last URL was retrieved from cache, // we won't wait. Otherwise we'll wait 1 second // before fetching the next URL. if (spider.LastFromCache != true) { spider.SleepMs(1000); } } // Add the outbound links to seedUrls, except // for the domains we've already seen. for (i = 0; i <= spider.NumOutboundLinks - 1; i++) { url = spider.GetOutboundLink(i); domain = spider.GetUrlDomain(url); string baseDomain; baseDomain = spider.GetBaseDomain(domain); if (!seenDomains.Contains(baseDomain)) { seedUrls.Append(url); } // Don't let our list of seedUrls grow too large. if (seedUrls.Count > 1000) { break; } } } |
© 2000-2013 Chilkat Software, Inc. All Rights Reserved.