X-Git-Url: https://git.cweiske.de/phinde.git/blobdiff_plain/90d2e0f8c2a80f4d5db5ae20248337d3a0594411..686f1cec3fd35782c30d20f891fec2f434e5d02f:/bin/index.php

diff --git a/bin/index.php b/bin/index.php
index 5a55427..5985a3e 100755
--- a/bin/index.php
+++ b/bin/index.php
@@ -90,6 +90,17 @@ if ($xbase) {
     );
 }
 
+$meta = $dx->evaluate('/html/head/meta[@name="robots" and @content]')
+    ->item(0);
+if ($meta) {
+    $robots = $meta->attributes->getNamedItem('content')->textContent;
+    foreach (explode(',', $robots) as $value) {
+        if (trim($value) == 'noindex') {
+            echo "URL does not want to be indexed: $url\n";
+            exit(0);
+        }
+    }
+}
 
 //remove script tags
 removeTags($doc, 'script');