1 files changed, 11 insertions, 0 deletions
diff --git a/bin/index.php b/bin/index.php
index 5a55427..5985a3e 100755
--- a/bin/index.php
+++ b/bin/index.php
@@ -90,6 +90,17 @@ if ($xbase) {
     );
 }
 
+$meta = $dx->evaluate('/html/head/meta[@name="robots" and @content]')
+    ->item(0);
+if ($meta) {
+    $robots = $meta->attributes->getNamedItem('content')->textContent;
+    foreach (explode(',', $robots) as $value) {
+        if (trim($value) == 'noindex') {
+            echo "URL does not want to be indexed: $url\n";
+            exit(0);
+        }
+    }
+}
 
 //remove script tags
 removeTags($doc, 'script');