remove anchor from source URLs

[phinde.git] / src / phinde / LinkExtractor / Html.php

diff --git a/src/phinde/LinkExtractor/Html.php b/src/phinde/LinkExtractor/Html.php
index 538d6c4fd0ccefd7da222561eaefd3d7fa71d77e..b3a9ea65170f8f50bd5a09492c72eb7e028829f3 100644 (file)
--- a/src/phinde/LinkExtractor/Html.php
+++ b/src/phinde/LinkExtractor/Html.php
@@ -2,12 +2,13 @@
 namespace phinde\LinkExtractor;
 
 use phinde\LinkInfo;
+use phinde\Helper;
 
 class Html
 {
     public function extract(\HTTP_Request2_Response $res)
     {
-        $url = $res->getEffectiveUrl();
+        $url = Helper::removeAnchor($res->getEffectiveUrl());
 
         $linkInfos = array();
 
@@ -19,18 +20,45 @@ class Html
         //FIXME: extract base url from html
         $base = new \Net_URL2($url);
 
-        $xpath = new \DOMXPath($doc);
-        $links = $xpath->evaluate('//a');
+        $dx = new \DOMXPath($doc);
+
+        $xbase = $dx->evaluate('/html/head/base[@href]')->item(0);
+        if ($xbase) {
+            $base = $base->resolve(
+                $xbase->attributes->getNamedItem('href')->textContent
+            );
+        }
+
+        $meta = $dx->evaluate('/html/head/meta[@name="robots" and @content]')
+            ->item(0);
+        if ($meta) {
+            $robots = $meta->attributes->getNamedItem('content')->textContent;
+            foreach (explode(',', $robots) as $value) {
+                if (trim($value) == 'nofollow') {
+                    //we shall not follow the links
+                    return array();
+                }
+            }
+        }
+
+        $links = $dx->evaluate('//a');
         //FIXME: link rel, img, video
 
-        $alreadySeen = array();
+        $alreadySeen = array($url => true);
 
         foreach ($links as $link) {
-            $linkTitle = $link->textContent;
+            $linkTitle = Helper::sanitizeTitle($link->textContent);
             $href = '';
             foreach ($link->attributes as $attribute) {
                 if ($attribute->name == 'href') {
                     $href = $attribute->textContent;
+                } else if ($attribute->name == 'rel') {
+                    foreach (explode(',', $attribute->textContent) as $value) {
+                        if (trim($value) == 'nofollow') {
+                            //we shall not follow this link
+                            continue 3;
+                        }
+                    }
                 }
             }
             if ($href == '' || $href{0} == '#') {
@@ -54,7 +82,6 @@ class Html
             }
 
             //FIXME: check target type
-            //FIXME: check nofollow
             $linkInfos[] = new LinkInfo(
                $linkUrl, $linkTitle, $url
             );