follow redirect, do not verify ssl certificates, use final after-redirect url

[phinde.git] / bin / index.php

diff --git a/bin/index.php b/bin/index.php
index 6a13afd546c3293837654dd9507821ed59239b7b..dd32dea86148e5007a8aa133a8e2a21bfa579212 100755 (executable)
--- a/bin/index.php
+++ b/bin/index.php
@@ -2,12 +2,7 @@
 <?php
 namespace phinde;
 // index a given URL
 <?php
 namespace phinde;
 // index a given URL

-set_include_path(__DIR__ . '/../src/' . PATH_SEPARATOR . get_include_path());
-require_once __DIR__ . '/../data/config.php';
-require_once 'HTTP/Request2.php';
-require_once 'Elasticsearch.php';
-require_once 'Elasticsearch/Request.php';
-require_once 'functions.php';
+require_once __DIR__ . '/../src/init.php';

 
 $supportedIndexTypes = array(
     'application/xhtml+xml',
 
 $supportedIndexTypes = array(
     'application/xhtml+xml',


@@ -29,11 +24,11 @@ if ($existingDoc && $existingDoc->status == 'indexed') {
 }
 //FIXME: sourcetitle, sourcelink
 
 }
 //FIXME: sourcetitle, sourcelink
 

-//FIXME: enable redirects
-//FIXME: enable ssl 

 $req = new \HTTP_Request2($url);
 $req = new \HTTP_Request2($url);

+$req->setConfig('follow_redirects', true);

 $req->setConfig('connect_timeout', 5);
 $req->setConfig('timeout', 10);
 $req->setConfig('connect_timeout', 5);
 $req->setConfig('timeout', 10);

+$req->setConfig('ssl_verify_peer', false);

 $res = $req->send();
 //FIXME: try-catch
 
 $res = $req->send();
 //FIXME: try-catch
 


@@ -54,7 +49,7 @@ if (!in_array($mimetype, $supportedIndexTypes)) {
 
 //FIXME: update index only if changed since last index time
 //FIXME: extract base url from html
 
 //FIXME: update index only if changed since last index time
 //FIXME: extract base url from html

-//FIXME: use final URL after redirects
+$url = $res->getEffectiveUrl();

 $base = new \Net_URL2($url);
 
 $indexDoc = new \stdClass();
 $base = new \Net_URL2($url);
 
 $indexDoc = new \stdClass();


@@ -66,6 +61,7 @@ $doc = new \DOMDocument();
 $sx = simplexml_import_dom($doc);
 
 $indexDoc->url = $url;
 $sx = simplexml_import_dom($doc);
 
 $indexDoc->url = $url;

+$indexDoc->schemalessUrl = Helper::noSchema($url);

 $indexDoc->type = 'html';
 $indexDoc->subtype = '';
 $indexDoc->mimetype = $mimetype;
 $indexDoc->type = 'html';
 $indexDoc->subtype = '';
 $indexDoc->mimetype = $mimetype;


@@ -143,7 +139,7 @@ if (count($arSxdates)) {
 
 //language
 //there may be "en-US" and "de-DE"
 
 //language
 //there may be "en-US" and "de-DE"

-$indexDoc->language = substr((string) $sx['lang'], 0, 2);
+$indexDoc->language = strtolower(substr((string) $sx['lang'], 0, 2));

 //FIXME: fallback, autodetection
 //FIXME: check noindex
 
 //FIXME: fallback, autodetection
 //FIXME: check noindex