From: Christian Weiske <cweiske@cweiske.de>
Date: Mon, 29 Aug 2016 20:59:16 +0000 (+0200)
Subject: Respect <meta name="robots" content="noindex"/>
X-Git-Tag: v0.2.0~36
X-Git-Url: https://git.cweiske.de/phinde.git/commitdiff_plain/686f1cec3fd35782c30d20f891fec2f434e5d02f

Respect <meta name="robots" content="noindex"/>

Fixes: #1
---

diff --git a/bin/index.php b/bin/index.php
index 5a55427..5985a3e 100755
--- a/bin/index.php
+++ b/bin/index.php
@@ -90,6 +90,17 @@ if ($xbase) {
     );
 }
 
+$meta = $dx->evaluate('/html/head/meta[@name="robots" and @content]')
+    ->item(0);
+if ($meta) {
+    $robots = $meta->attributes->getNamedItem('content')->textContent;
+    foreach (explode(',', $robots) as $value) {
+        if (trim($value) == 'noindex') {
+            echo "URL does not want to be indexed: $url\n";
+            exit(0);
+        }
+    }
+}
 
 //remove script tags
 removeTags($doc, 'script');