Send If-Modified-Since header on crawling and indexing

[phinde.git] / src / phinde / Crawler.php

diff --git a/src/phinde/Crawler.php b/src/phinde/Crawler.php
index ced40b83d4ca23bb8d39e74fb352a0ef31e676b9..43d9459328ce9cf8fa4bc998f7de4b96555a1a7f 100644 (file)
--- a/src/phinde/Crawler.php
+++ b/src/phinde/Crawler.php
@@ -6,6 +6,11 @@ class Crawler
     protected $es;
     protected $queue;
 
+    /**
+     * If the links only should be shown, not queued
+     */
+    protected $showLinksOnly = false;
+
     static $supportedIndexTypes = array(
         'application/atom+xml'  => '\\phinde\\LinkExtractor\\Atom',
         'application/xhtml+xml' => '\\phinde\\LinkExtractor\\Html',
@@ -20,20 +25,38 @@ class Crawler
 
     public function crawl($url)
     {
-        $res       = $this->fetch($url);
+        $res = $this->fetch($url);
+        if ($res === false) {
+            return;
+        }
+
         $linkInfos = $this->extractLinks($res);
-        $this->enqueue($linkInfos);
+        if ($this->showLinksOnly) {
+            $this->showLinks($linkInfos);
+        } else {
+            $this->enqueue($linkInfos);
+        }
     }
 
     protected function fetch($url)
     {
+        $existingDoc = $this->es->get($url);
+
         $req = new HttpRequest($url);
         $req->setHeader(
             'accept',
             implode(',', array_keys(static::$supportedIndexTypes))
         );
+        if ($existingDoc) {
+            $nMoDate = strtotime($existingDoc->modate);
+            $req->setHeader('If-Modified-Since: ' . date('r', $nMoDate));
+        }
+
         $res = $req->send();
-        if ($res->getStatus() !== 200) {
+        if ($res->getStatus() === 304) {
+            //not modified since last time, so don't crawl again
+            return false;
+        } else if ($res->getStatus() !== 200) {
             throw new \Exception(
                 "Response code is not 200 but "
                 . $res->getStatus() . ", stopping"
@@ -57,7 +80,6 @@ class Crawler
 
     protected function enqueue($linkInfos)
     {
-        var_dump($linkInfos);die();
         foreach ($linkInfos as $linkInfo) {
             if ($this->es->isKnown($linkInfo->url)) {
                 continue;
@@ -71,5 +93,21 @@ class Crawler
             }
         }
     }
+
+    protected function showLinks($linkInfos)
+    {
+        foreach ($linkInfos as $linkInfo) {
+            echo $linkInfo->url . "\n";
+            if ($linkInfo->title) {
+                echo '  title: ' . $linkInfo->title . "\n";
+                echo '  source: ' . $linkInfo->source . "\n";
+            }
+        }
+    }
+
+    public function setShowLinksOnly($showLinksOnly)
+    {
+        $this->showLinksOnly = $showLinksOnly;
+    }
 }
 ?>