From: Christian Weiske <cweiske@cweiske.de>
Date: Tue, 30 Aug 2016 06:13:33 +0000 (+0200)
Subject: Option to disable linked URL indexing
X-Git-Tag: v0.2.0~33
X-Git-Url: https://git.cweiske.de/phinde.git/commitdiff_plain/45638a5de3c8c05c1792f8a3ab93acb11a2c1a86

Option to disable linked URL indexing

Resolves: #2
---

diff --git a/data/config.php.dist b/data/config.php.dist
index b4d7d5c..dc1cff8 100644
--- a/data/config.php.dist
+++ b/data/config.php.dist
@@ -1,6 +1,7 @@
 <?php
 $GLOBALS['phinde'] = array(
     'elasticsearch' => 'http://127.0.0.1:9200/phinde/',
+    //whitelist of domains that shall be crawled
     'domains' => array(
         'www.example.org',
         'test.example.org'
@@ -15,5 +16,8 @@ $GLOBALS['phinde'] = array(
     ),
     //time in seconds after which URLs may be re-indexed
     'refreshtime' => 86400,
+    //if directly linked URLs shall be indexed, even if they are
+    // on a non-whitelisted domain
+    'indexNonAllowed' => true,
 );
 ?>
\ No newline at end of file
diff --git a/src/phinde/Crawler.php b/src/phinde/Crawler.php
index 43d9459..6459fb4 100644
--- a/src/phinde/Crawler.php
+++ b/src/phinde/Crawler.php
@@ -84,11 +84,19 @@ class Crawler
             if ($this->es->isKnown($linkInfo->url)) {
                 continue;
             }
-            $this->es->markQueued($linkInfo->url);
-            $this->queue->addToIndex(
-                $linkInfo->url, $linkInfo->title, $linkInfo->source
-            );
-            if (Helper::isUrlAllowed($linkInfo->url)) {
+            $allowed = Helper::isUrlAllowed($linkInfo->url);
+            $crawl   = $allowed;
+            $index   = $GLOBALS['phinde']['indexNonAllowed'] || $allowed;
+
+            if ($crawl || $index) {
+                $this->es->markQueued($linkInfo->url);
+            }
+            if ($index) {
+                $this->queue->addToIndex(
+                    $linkInfo->url, $linkInfo->title, $linkInfo->source
+                );
+            }
+            if ($allowed) {
                 $this->queue->addToCrawl($linkInfo->url);
             }
         }