Add URL rewrites/replacements

diff --git a/data/config.php.dist b/data/config.php.dist
index cef499b39a0f815178d212747d0b29ff42e16258..38c04328e38659c816b63c64c72b0fc7ab136d57 100644 (file)
--- a/data/config.php.dist
+++ b/data/config.php.dist
@@ -13,6 +13,10 @@ $GLOBALS['phinde'] = array(
     //list of regexes for URLs that should not be crawled
     'crawlBlacklist' => array(
     ),
     //list of regexes for URLs that should not be crawled
     'crawlBlacklist' => array(
     ),

+    //modify URLs with regex
+    'urlRewrites' => array(
+        // '^http://example.org/' => 'https://example.org/',
+    ),

     //verbose output
     'debug' => true,
     //full path to log file
     //verbose output
     'debug' => true,
     //full path to log file

diff --git a/src/phinde/Crawler.php b/src/phinde/Crawler.php
index 1f63e60bb946b76df1d49187aeef92e972496214..4d596b40e0abc49031fa4127211defb78aed58c4 100644 (file)
--- a/src/phinde/Crawler.php
+++ b/src/phinde/Crawler.php
@@ -53,6 +53,7 @@ class Crawler
     {
         $filteredLinkInfos = array();
         foreach ($linkInfos as $linkInfo) {
     {
         $filteredLinkInfos = array();
         foreach ($linkInfos as $linkInfo) {

+            $linkInfo->url = Helper::rewriteUrl($linkInfo->url);

             $allowed = Helper::isUrlAllowed($linkInfo->url);
             $crawl   = $allowed;
             $index   = $GLOBALS['phinde']['indexNonAllowed'] || $allowed;
             $allowed = Helper::isUrlAllowed($linkInfo->url);
             $crawl   = $allowed;
             $index   = $GLOBALS['phinde']['indexNonAllowed'] || $allowed;

diff --git a/src/phinde/Fetcher.php b/src/phinde/Fetcher.php
index dccb118c0ac52f0bd47fd73029b65cb7b5780a69..7cf11b77ec92f1df0b8f67e38c3b78834fb00558 100644 (file)
--- a/src/phinde/Fetcher.php
+++ b/src/phinde/Fetcher.php
@@ -15,12 +15,15 @@ class Fetcher
      */
     public function fetch($url, $actions, $force = false)
     {
      */
     public function fetch($url, $actions, $force = false)
     {

+        $url = Helper::rewriteUrl($url);
+

         $esDoc = $this->es->get($url);
         if (isset($esDoc->status->location)
             && $esDoc->status->location != ''
         ) {
             //TODO: what if location redirects change?
             $url = $esDoc->status->location;
         $esDoc = $this->es->get($url);
         if (isset($esDoc->status->location)
             && $esDoc->status->location != ''
         ) {
             //TODO: what if location redirects change?
             $url = $esDoc->status->location;

+            $url = Helper::rewriteUrl($url);

             $esDoc = $this->es->get($url);
         }
 
             $esDoc = $this->es->get($url);
         }
 


@@ -53,6 +56,7 @@ class Fetcher
         }
 
         $effUrl = Helper::removeAnchor($res->getEffectiveUrl());
         }
 
         $effUrl = Helper::removeAnchor($res->getEffectiveUrl());

+        $effUrl = Helper::rewriteUrl($effUrl);

         if ($effUrl != $url) {
             $this->storeRedirect($url, $effUrl);
             $url = $effUrl;
         if ($effUrl != $url) {
             $this->storeRedirect($url, $effUrl);
             $url = $effUrl;

diff --git a/src/phinde/Helper.php b/src/phinde/Helper.php
index aeb8ba5d4e8c08874963fc5cf2b28843677d753f..d22b9c89f16a46e6597bd8586ed1b4189a140190 100644 (file)
--- a/src/phinde/Helper.php
+++ b/src/phinde/Helper.php
@@ -3,6 +3,20 @@ namespace phinde;
 
 class Helper
 {
 
 class Helper
 {

+    public static function rewriteUrl($url)
+    {
+        if (!isset($GLOBALS['phinde']['urlRewrites'])
+            || count($GLOBALS['phinde']['urlRewrites']) == 0
+        ) {
+            return $url;
+        }
+
+        foreach ($GLOBALS['phinde']['urlRewrites'] as $pattern => $replacement) {
+            $url = preg_replace('#' . $pattern . '#', $replacement, $url);
+        }
+        return $url;
+    }
+

     public static function isUrlAllowed($url)
     {
         $urlDomain = parse_url($url, PHP_URL_HOST);
     public static function isUrlAllowed($url)
     {
         $urlDomain = parse_url($url, PHP_URL_HOST);