Chilkat HOME .NET Core C# Android™ AutoIt C C# C++ Chilkat2-Python CkPython Classic ASP DataFlex Delphi ActiveX Delphi DLL Go Java Lianja Mono C# Node.js Objective-C PHP ActiveX PHP Extension Perl PowerBuilder PowerShell PureBasic Ruby SQL Server Swift 2 Swift 3,4,5... Tcl Unicode C Unicode C++ VB.NET VBScript Visual Basic 6.0 Visual FoxPro Xojo Plugin
(.NET Core C#) A Simple Web CrawlerThis demonstrates a very simple web crawler using the Chilkat Spider component.
Chilkat.Spider spider = new Chilkat.Spider(); Chilkat.StringArray seenDomains = new Chilkat.StringArray(); Chilkat.StringArray seedUrls = new Chilkat.StringArray(); seenDomains.Unique = true; seedUrls.Unique = true; // You will need to change the start URL to something else... seedUrls.Append("http://something.whateverYouWant.com/"); // Set outbound URL exclude patterns // URLs matching any of these patterns will not be added to the // collection of outbound links. spider.AddAvoidOutboundLinkPattern("*?id=*"); spider.AddAvoidOutboundLinkPattern("*.mypages.*"); spider.AddAvoidOutboundLinkPattern("*.personal.*"); spider.AddAvoidOutboundLinkPattern("*.comcast.*"); spider.AddAvoidOutboundLinkPattern("*.aol.*"); spider.AddAvoidOutboundLinkPattern("*~*"); // Use a cache so we don't have to re-fetch URLs previously fetched. spider.CacheDir = "c:/spiderCache/"; spider.FetchFromCache = true; spider.UpdateCache = true; while (seedUrls.Count > 0) { string url = seedUrls.Pop(); spider.Initialize(url); // Spider 5 URLs of this domain. // but first, save the base domain in seenDomains string domain = spider.GetUrlDomain(url); seenDomains.Append(spider.GetBaseDomain(domain)); int i; bool success; for (i = 0; i <= 4; i++) { success = spider.CrawlNext(); if (success == true) { // Display the URL we just crawled. Debug.WriteLine(spider.LastUrl); // If the last URL was retrieved from cache, // we won't wait. Otherwise we'll wait 1 second // before fetching the next URL. if (spider.LastFromCache != true) { spider.SleepMs(1000); } } else { // cause the loop to exit.. i = 999; } } // Add the outbound links to seedUrls, except // for the domains we've already seen. for (i = 0; i <= spider.NumOutboundLinks - 1; i++) { url = spider.GetOutboundLink(i); string domain = spider.GetUrlDomain(url); string baseDomain = spider.GetBaseDomain(domain); if (seenDomains.Contains(baseDomain) == false) { // Don't let our list of seedUrls grow too large. if (seedUrls.Count < 1000) { seedUrls.Append(url); } } } } |
© 2000-2024 Chilkat Software, Inc. All Rights Reserved.