Remove URL check from process.php. Checking is done in Crawler already

[phinde.git] / bin / process.php

diff --git a/bin/process.php b/bin/process.php
index ababb036c063c67a2247796af59060ce11525dc3..9fc47527b9dd12ad43e8115fc038a9d5f5e765af 100755 (executable)
--- a/bin/process.php
+++ b/bin/process.php
@@ -53,10 +53,6 @@ $url = $res->args['url'];
 $url = Helper::addSchema($url);
 $urlObj = new \Net_URL2($url);
 $url = $urlObj->getNormalizedURL();
 $url = Helper::addSchema($url);
 $urlObj = new \Net_URL2($url);
 $url = $urlObj->getNormalizedURL();

-if (!Helper::isUrlAllowed($url)) {
-    echo "Domain is not allowed; not crawling\n";
-    exit(2);
-}

 
 try {
     $actions = array();
 
 try {
     $actions = array();


@@ -78,7 +74,7 @@ try {
 
     $update = false;
     foreach ($actions as $key => $action) {
 
     $update = false;
     foreach ($actions as $key => $action) {

-        echo "step: $key\n";
+        Log::info("step: $key");

         $update |= $action->run($retrieved);
     }
 
         $update |= $action->run($retrieved);
     }
 


@@ -86,10 +82,10 @@ try {
         //FIXME: update index if it exists already
         $fetcher->storeDoc($retrieved->url, $retrieved->esDoc);
     } else {
         //FIXME: update index if it exists already
         $fetcher->storeDoc($retrieved->url, $retrieved->esDoc);
     } else {

-        echo "Not updating\n";
+        Log::info("Not updating");

     }
 } catch (\Exception $e) {
     }
 } catch (\Exception $e) {

-    echo $e->getMessage() . "\n";
+    Log::error($e->getMessage());

     exit(10);
 }
 ?>
\ No newline at end of file
     exit(10);
 }
 ?>
\ No newline at end of file