crawler supports "nofollow" now

[phinde.git] / src / phinde / Crawler.php

diff --git a/src/phinde/Crawler.php b/src/phinde/Crawler.php
index 53320ec6cb6101f3ce61b88bc0d28c283c4514dd..ced40b83d4ca23bb8d39e74fb352a0ef31e676b9 100644 (file)
--- a/src/phinde/Crawler.php
+++ b/src/phinde/Crawler.php
@@ -28,6 +28,10 @@ class Crawler
     protected function fetch($url)
     {
         $req = new HttpRequest($url);
+        $req->setHeader(
+            'accept',
+            implode(',', array_keys(static::$supportedIndexTypes))
+        );
         $res = $req->send();
         if ($res->getStatus() !== 200) {
             throw new \Exception(
@@ -53,6 +57,7 @@ class Crawler
 
     protected function enqueue($linkInfos)
     {
+        var_dump($linkInfos);die();
         foreach ($linkInfos as $linkInfo) {
             if ($this->es->isKnown($linkInfo->url)) {
                 continue;