X-Git-Url: https://git.cweiske.de/phinde.git/blobdiff_plain/7b4425b096fa8c18d0db9fd9b1ae96d63ee8af55..18dc4a1560c24ffd70d659ef1a96caf90ec9a4c8:/bin/index.php

diff --git a/bin/index.php b/bin/index.php
index 6a13afd..dd32dea 100755
--- a/bin/index.php
+++ b/bin/index.php
@@ -2,12 +2,7 @@
 <?php
 namespace phinde;
 // index a given URL
-set_include_path(__DIR__ . '/../src/' . PATH_SEPARATOR . get_include_path());
-require_once __DIR__ . '/../data/config.php';
-require_once 'HTTP/Request2.php';
-require_once 'Elasticsearch.php';
-require_once 'Elasticsearch/Request.php';
-require_once 'functions.php';
+require_once __DIR__ . '/../src/init.php';
 
 $supportedIndexTypes = array(
     'application/xhtml+xml',
@@ -29,11 +24,11 @@ if ($existingDoc && $existingDoc->status == 'indexed') {
 }
 //FIXME: sourcetitle, sourcelink
 
-//FIXME: enable redirects
-//FIXME: enable ssl 
 $req = new \HTTP_Request2($url);
+$req->setConfig('follow_redirects', true);
 $req->setConfig('connect_timeout', 5);
 $req->setConfig('timeout', 10);
+$req->setConfig('ssl_verify_peer', false);
 $res = $req->send();
 //FIXME: try-catch
 
@@ -54,7 +49,7 @@ if (!in_array($mimetype, $supportedIndexTypes)) {
 
 //FIXME: update index only if changed since last index time
 //FIXME: extract base url from html
-//FIXME: use final URL after redirects
+$url = $res->getEffectiveUrl();
 $base = new \Net_URL2($url);
 
 $indexDoc = new \stdClass();
@@ -66,6 +61,7 @@ $doc = new \DOMDocument();
 $sx = simplexml_import_dom($doc);
 
 $indexDoc->url = $url;
+$indexDoc->schemalessUrl = Helper::noSchema($url);
 $indexDoc->type = 'html';
 $indexDoc->subtype = '';
 $indexDoc->mimetype = $mimetype;
@@ -143,7 +139,7 @@ if (count($arSxdates)) {
 
 //language
 //there may be "en-US" and "de-DE"
-$indexDoc->language = substr((string) $sx['lang'], 0, 2);
+$indexDoc->language = strtolower(substr((string) $sx['lang'], 0, 2));
 //FIXME: fallback, autodetection
 //FIXME: check noindex