+ $dx = new \DOMXPath($doc);
+
+ $meta = $dx->evaluate('/html/head/meta[@name="robots" and @content]')
+ ->item(0);
+ if ($meta) {
+ $robots = $meta->attributes->getNamedItem('content')->textContent;
+ foreach (explode(',', $robots) as $value) {
+ if (trim($value) == 'nofollow') {
+ //we shall not follow the links
+ return array();
+ }
+ }
+ }
+
+ $links = $dx->evaluate('//a');