Fix accept header in crawler

[phinde.git] / src / phinde / Fetcher.php

diff --git a/src/phinde/Fetcher.php b/src/phinde/Fetcher.php
index dccb118c0ac52f0bd47fd73029b65cb7b5780a69..666d7eda2449a6240a14d657bca6844922ecf3da 100644 (file)
--- a/src/phinde/Fetcher.php
+++ b/src/phinde/Fetcher.php
@@ -15,18 +15,21 @@ class Fetcher
      */
     public function fetch($url, $actions, $force = false)
     {
+        $url = Helper::rewriteUrl($url);
+
         $esDoc = $this->es->get($url);
         if (isset($esDoc->status->location)
             && $esDoc->status->location != ''
         ) {
             //TODO: what if location redirects change?
             $url = $esDoc->status->location;
+            $url = Helper::rewriteUrl($url);
             $esDoc = $this->es->get($url);
         }
 
         $types = array();
         foreach ($actions as $action) {
-            $types = array_merge($action::$supportedTypes);
+            $types = array_merge($types, array_keys($action::$supportedTypes));
         }
         $types = array_unique($types);
 
@@ -53,6 +56,7 @@ class Fetcher
         }
 
         $effUrl = Helper::removeAnchor($res->getEffectiveUrl());
+        $effUrl = Helper::rewriteUrl($effUrl);
         if ($effUrl != $url) {
             $this->storeRedirect($url, $effUrl);
             $url = $effUrl;