Add URL rewrites/replacements

[phinde.git] / src / phinde / Crawler.php

diff --git a/src/phinde/Crawler.php b/src/phinde/Crawler.php
index 38e3c3f907a84b40c9c25c8b85f4eb120ab48eea..4d596b40e0abc49031fa4127211defb78aed58c4 100644 (file)
--- a/src/phinde/Crawler.php
+++ b/src/phinde/Crawler.php
@@ -40,7 +40,7 @@ class Crawler
     {
         $mimetype = explode(';', $res->getHeader('content-type'))[0];
         if (!isset(static::$supportedTypes[$mimetype])) {
     {
         $mimetype = explode(';', $res->getHeader('content-type'))[0];
         if (!isset(static::$supportedTypes[$mimetype])) {

-            echo "MIME type not supported for indexing: $mimetype\n";
+            Log::info("MIME type not supported for crawling: $mimetype");

             return array();
         }
 
             return array();
         }
 


@@ -53,6 +53,7 @@ class Crawler
     {
         $filteredLinkInfos = array();
         foreach ($linkInfos as $linkInfo) {
     {
         $filteredLinkInfos = array();
         foreach ($linkInfos as $linkInfo) {

+            $linkInfo->url = Helper::rewriteUrl($linkInfo->url);

             $allowed = Helper::isUrlAllowed($linkInfo->url);
             $crawl   = $allowed;
             $index   = $GLOBALS['phinde']['indexNonAllowed'] || $allowed;
             $allowed = Helper::isUrlAllowed($linkInfo->url);
             $crawl   = $allowed;
             $index   = $GLOBALS['phinde']['indexNonAllowed'] || $allowed;


@@ -98,13 +99,15 @@ class Crawler
     protected function showLinks($linkInfos)
     {
         foreach ($linkInfos as $linkInfo) {
     protected function showLinks($linkInfos)
     {
         foreach ($linkInfos as $linkInfo) {

-            echo $linkInfo->url . "\n";
+            Log::msg($linkInfo->url);

             if ($linkInfo->title) {
             if ($linkInfo->title) {

-                echo '   title: ' . $linkInfo->title . "\n";
-                echo '  source: ' . $linkInfo->source . "\n";
-                echo '   known: ' . intval($linkInfo->known)
+                Log::msg('   title: ' . $linkInfo->title);
+                Log::msg('  source: ' . $linkInfo->source);
+                Log::msg(
+                    '   known: ' . intval($linkInfo->known)

                     . ', crawl: ' . intval($linkInfo->crawl)
                     . ', crawl: ' . intval($linkInfo->crawl)

-                    . ', index: ' . intval($linkInfo->index) . "\n";
+                    . ', index: ' . intval($linkInfo->index)
+                );

             }
         }
     }
             }
         }
     }