namespace phinde\LinkExtractor;
use phinde\LinkInfo;
+use phinde\Helper;
class Html
{
//FIXME: extract base url from html
$base = new \Net_URL2($url);
- $xpath = new \DOMXPath($doc);
- $links = $xpath->evaluate('//a');
+ $dx = new \DOMXPath($doc);
+
+ $meta = $dx->evaluate('/html/head/meta[@name="robots" and @content]')
+ ->item(0);
+ if ($meta) {
+ $robots = $meta->attributes->getNamedItem('content')->textContent;
+ foreach (explode(',', $robots) as $value) {
+ if (trim($value) == 'nofollow') {
+ //we shall not follow the links
+ return array();
+ }
+ }
+ }
+
+ $links = $dx->evaluate('//a');
//FIXME: link rel, img, video
- $alreadySeen = array();
+ $alreadySeen = array($url => true);
foreach ($links as $link) {
- $linkTitle = $link->textContent;
+ $linkTitle = Helper::sanitizeTitle($link->textContent);
$href = '';
foreach ($link->attributes as $attribute) {
if ($attribute->name == 'href') {
$href = $attribute->textContent;
+ } else if ($attribute->name == 'rel') {
+ foreach (explode(',', $attribute->textContent) as $value) {
+ if (trim($value) == 'nofollow') {
+ //we shall not follow this link
+ continue 3;
+ }
+ }
}
}
if ($href == '' || $href{0} == '#') {
}
//FIXME: check target type
- //FIXME: check nofollow
$linkInfos[] = new LinkInfo(
$linkUrl, $linkTitle, $url
);