Respect <meta name="robots" content="noindex"/>

[phinde.git] / bin / index.php

diff --git a/bin/index.php b/bin/index.php
index d110423b4c6b81a5065afec5e35d90ba0c1d6b70..5985a3e9bfb2e0a8e1d3103c6e351f8a1d6d6a66 100755 (executable)
--- a/bin/index.php
+++ b/bin/index.php
@@ -24,27 +24,38 @@ function removeTags($doc, $tag) {
     }
 }
 
     }
 }
 

-$es = new Elasticsearch($GLOBALS['phinde']['elasticsearch']);
-

 $url = $argv[1];
 $url = $argv[1];

-$existingDoc = $es->get($url);
-if ($existingDoc && $existingDoc->status == 'indexed') {
-    echo "URL already indexed: $url\n";
-    exit(0);
-}
-//FIXME: size limit
-//FIXME: sourcetitle, sourcelink

 
 $req = new \HTTP_Request2($url);
 $req->setConfig('follow_redirects', true);
 $req->setConfig('connect_timeout', 5);
 $req->setConfig('timeout', 10);
 $req->setConfig('ssl_verify_peer', false);
 
 $req = new \HTTP_Request2($url);
 $req->setConfig('follow_redirects', true);
 $req->setConfig('connect_timeout', 5);
 $req->setConfig('timeout', 10);
 $req->setConfig('ssl_verify_peer', false);

+//FIXME: size limit
+
+$es = new Elasticsearch($GLOBALS['phinde']['elasticsearch']);
+$existingDoc = $es->get($url);
+if ($existingDoc && $existingDoc->status == 'indexed') {
+    $nMoDate     = strtotime($existingDoc->modate);
+    $refreshtime = $GLOBALS['phinde']['refreshtime'];
+    if (time() - $nMoDate < $refreshtime) {
+        echo "URL already indexed less than $refreshtime seconds ago: $url\n";
+        exit(0);
+    }
+
+    $req->setHeader('If-Modified-Since: ' . date('r', $nMoDate));
+}
+//FIXME: sourcetitle, sourcelink
+

 $res = $req->send();
 //FIXME: try-catch
 
 $res = $req->send();
 //FIXME: try-catch
 

-//FIXME: delete if 401 gone or 404 when updating
-if ($res->getStatus() !== 200) {
+if ($res->getStatus() === 304) {
+    //not modified since last time
+    //FIXME: store "last try" time
+    exit(0);
+} else if ($res->getStatus() !== 200) {
+    //FIXME: delete if 401 gone or 404 when updating

     echo "Response code is not 200 but " . $res->getStatus() . ", stopping\n";
     //FIXME: update status
     exit(3);
     echo "Response code is not 200 but " . $res->getStatus() . ", stopping\n";
     //FIXME: update status
     exit(3);


@@ -79,6 +90,17 @@ if ($xbase) {
     );
 }
 
     );
 }
 

+$meta = $dx->evaluate('/html/head/meta[@name="robots" and @content]')
+    ->item(0);
+if ($meta) {
+    $robots = $meta->attributes->getNamedItem('content')->textContent;
+    foreach (explode(',', $robots) as $value) {
+        if (trim($value) == 'noindex') {
+            echo "URL does not want to be indexed: $url\n";
+            exit(0);
+        }
+    }
+}

 
 //remove script tags
 removeTags($doc, 'script');
 
 //remove script tags
 removeTags($doc, 'script');