}
}
-$es = new Elasticsearch($GLOBALS['phinde']['elasticsearch']);
-
$url = $argv[1];
-$existingDoc = $es->get($url);
-if ($existingDoc && $existingDoc->status == 'indexed') {
- echo "URL already indexed: $url\n";
- exit(0);
-}
-//FIXME: sourcetitle, sourcelink
$req = new \HTTP_Request2($url);
$req->setConfig('follow_redirects', true);
$req->setConfig('connect_timeout', 5);
$req->setConfig('timeout', 10);
$req->setConfig('ssl_verify_peer', false);
+//FIXME: size limit
+
+$es = new Elasticsearch($GLOBALS['phinde']['elasticsearch']);
+$existingDoc = $es->get($url);
+if ($existingDoc && $existingDoc->status == 'indexed') {
+ $nMoDate = strtotime($existingDoc->modate);
+ $refreshtime = $GLOBALS['phinde']['refreshtime'];
+ if (time() - $nMoDate < $refreshtime) {
+ echo "URL already indexed less than $refreshtime seconds ago: $url\n";
+ exit(0);
+ }
+
+ $req->setHeader('If-Modified-Since: ' . date('r', $nMoDate));
+}
+//FIXME: sourcetitle, sourcelink
+
$res = $req->send();
//FIXME: try-catch
-//FIXME: delete if 401 gone or 404 when updating
-if ($res->getStatus() !== 200) {
+if ($res->getStatus() === 304) {
+ //not modified since last time
+ //FIXME: store "last try" time
+ exit(0);
+} else if ($res->getStatus() !== 200) {
+ //FIXME: delete if 401 gone or 404 when updating
echo "Response code is not 200 but " . $res->getStatus() . ", stopping\n";
//FIXME: update status
exit(3);
//FIXME: update index only if changed since last index time
//FIXME: extract base url from html
+//FIXME: check if effective url needs updating
$url = $res->getEffectiveUrl();
$base = new \Net_URL2($url);
@$doc->loadHTML($res->getBody());
$dx = new \DOMXPath($doc);
+$xbase = $dx->evaluate('/html/head/base[@href]')->item(0);
+if ($xbase) {
+ $base = $base->resolve(
+ $xbase->attributes->getNamedItem('href')->textContent
+ );
+}
+
+$meta = $dx->evaluate('/html/head/meta[@name="robots" and @content]')
+ ->item(0);
+if ($meta) {
+ $robots = $meta->attributes->getNamedItem('content')->textContent;
+ foreach (explode(',', $robots) as $value) {
+ if (trim($value) == 'noindex') {
+ echo "URL does not want to be indexed: $url\n";
+ exit(0);
+ }
+ }
+}
+
//remove script tags
removeTags($doc, 'script');
removeTags($doc, 'style');
//default content: <body>
$xpContext = $doc->getElementsByTagName('body')->item(0);
+//FIXME: follow meta refresh, no body
+// example: https://www.gnu.org/software/coreutils/
//use microformats content if it exists
$xpElems = $dx->query(
$indexDoc->author = new \stdClass();
-$arXpElems = $dx->query('/html/head/meta[@name="author"]');
+$arXpElems = $dx->query('/html/head/meta[@name="author" and @content]');
if ($arXpElems->length) {
$indexDoc->author->name = trim(
$arXpElems->item(0)->attributes->getNamedItem('content')->textContent
);
}
-$arXpElems = $dx->query('/html/head/link[@rel="author"]');
+$arXpElems = $dx->query('/html/head/link[@rel="author" and @href]');
if ($arXpElems->length) {
$indexDoc->author->url = trim(
$base->resolve(
//tags
$tags = array();
-foreach ($dx->query('/html/head/meta[@name="keywords"]') as $xkeywords) {
+foreach ($dx->query('/html/head/meta[@name="keywords" and @content]') as $xkeywords) {
$keywords = $xkeywords->attributes->getNamedItem('content')->textContent;
foreach (explode(',', $keywords) as $keyword) {
$tags[trim($keyword)] = true;
$indexDoc->tags = array_keys($tags);
//dates
-$arXpdates = $dx->query('/html/head/meta[@name="DC.date.created"]');
+$arXpdates = $dx->query('/html/head/meta[@name="DC.date.created" and @content]');
if ($arXpdates->length) {
$indexDoc->crdate = date(
'c',
//FIXME: keep creation date from database, or use modified date if we
// do not have it there
-$arXpdates = $dx->query('/html/head/meta[@name="DC.date.modified"]');
+$arXpdates = $dx->query('/html/head/meta[@name="DC.date.modified" and @content]');
if ($arXpdates->length) {
$indexDoc->modate = date(
'c',