Pascal (Lazarus/Delphi)

Avoid URLs Matching Any of a Set of Patterns

See more Spider Examples

Demonstrates how to use "avoid patterns" to prevent spidering any URL that matches a wildcarded pattern. This example avoids URLs containing the substrings "java", "python", or "perl". Pascal (Lazarus/Delphi)

program ChilkatDemo;

// Demonstrates using the Chilkat Pascal wrapper via the C bridge DLL.
// Builds as a console application under Lazarus (FPC) or Delphi.

{$IFDEF FPC}
  {$MODE DELPHI}
{$ENDIF}
{$APPTYPE CONSOLE}

uses
  {$IFDEF UNIX}
  cthreads,
  {$ENDIF}
  SysUtils,
  CkDllLoader,
  Chilkat.Spider;

// ---------------------------------------------------------------------------

procedure RunDemo;
var
  success: Boolean;
  spider: TSpider;
  i: Integer;

begin
  success := False;

  spider := TSpider.Create;

  //  The spider object crawls a single web site at a time.  As you'll see
  //  in later examples, you can collect outbound links and use them to 
  //  crawl the web.  For now, we'll simply spider 10 pages of chilkatsoft.com
  spider.Initialize('www.chilkatsoft.com');

  //  Add the 1st URL:
  spider.AddUnspidered('http://www.chilkatsoft.com/');

  //  Avoid URLs matching these patterns:
  spider.AddAvoidPattern('*java*');
  spider.AddAvoidPattern('*python*');
  spider.AddAvoidPattern('*perl*');

  //  Begin crawling the site by calling CrawlNext repeatedly.

  for i := 0 to 9 do
    begin

      success := spider.CrawlNext();
      if (success = True) then
        begin
          //  Show the URL of the page just spidered.
          WriteLn(spider.LastUrl);
          //  The HTML is available in the LastHtml property
        end
      else
        begin
          //  Did we get an error or are there no more URLs to crawl?
          if (spider.NumUnspidered = 0) then
            begin
              WriteLn('No more URLs to spider');
            end
          else
            begin
              WriteLn(spider.LastErrorText);
            end;
        end;
      //  Sleep 1 second before spidering the next URL.
      spider.SleepMs(1000);
    end;



  spider.Free;

end;

// ---------------------------------------------------------------------------

begin

  try
    RunDemo;
  except
    on E: Exception do
      WriteLn('Unhandled exception: ', E.ClassName, ': ', E.Message);
  end;

  WriteLn;
  {$IFDEF MSWINDOWS}
  WriteLn('Press Enter to exit...');
  ReadLn;
  {$ENDIF}
end.