From: Christian Weiske <cweiske@cweiske.de>
Date: Sat, 3 Dec 2016 13:28:21 +0000 (+0100)
Subject: Remove URL check from process.php. Checking is done in Crawler already
X-Git-Tag: v0.2.1~2
X-Git-Url: https://git.cweiske.de/phinde.git/commitdiff_plain/cbefad7ddd4927b1bbe225fec540a082c15f5403

Remove URL check from process.php. Checking is done in Crawler already
---

diff --git a/bin/process.php b/bin/process.php
index 1bae7c4..9fc4752 100755
--- a/bin/process.php
+++ b/bin/process.php
@@ -53,10 +53,6 @@ $url = $res->args['url'];
 $url = Helper::addSchema($url);
 $urlObj = new \Net_URL2($url);
 $url = $urlObj->getNormalizedURL();
-if (!Helper::isUrlAllowed($url)) {
-    Log::error("Domain is not allowed; not crawling");
-    exit(2);
-}
 
 try {
     $actions = array();