aboutsummaryrefslogtreecommitdiff
path: root/src/phinde/Indexer.php
diff options
context:
space:
mode:
authorChristian Weiske <cweiske@cweiske.de>2016-11-07 21:41:36 +0100
committerChristian Weiske <cweiske@cweiske.de>2016-11-07 21:41:36 +0100
commitd7651fd96dcfa2829519504e4c8ec1ce511cd57f (patch)
treee24d7a9f90060b0fee5a652de43bd0627f1c5bde /src/phinde/Indexer.php
parentf90790c6b2a54c9b1c8a0aeaf1f23e6aa67d7aca (diff)
downloadphinde-d7651fd96dcfa2829519504e4c8ec1ce511cd57f.tar.gz
phinde-d7651fd96dcfa2829519504e4c8ec1ce511cd57f.zip
Big patch merging crawling+indexing into one command, new json document structure
Diffstat (limited to 'src/phinde/Indexer.php')
-rw-r--r--src/phinde/Indexer.php205
1 files changed, 205 insertions, 0 deletions
diff --git a/src/phinde/Indexer.php b/src/phinde/Indexer.php
new file mode 100644
index 0000000..98b52c3
--- /dev/null
+++ b/src/phinde/Indexer.php
@@ -0,0 +1,205 @@
+<?php
+namespace phinde;
+
+class Indexer
+{
+ static $supportedTypes = array(
+ 'application/xhtml+xml',
+ 'text/html',
+ );
+
+ public function run(Retrieved $retrieved)
+ {
+ $res = $retrieved->httpRes;
+ $esDoc = $retrieved->esDoc;
+ $url = $retrieved->url;
+
+ $mimetype = explode(';', $res->getHeader('content-type'))[0];
+ if (!in_array($mimetype, static::$supportedTypes)) {
+ echo "MIME type not supported for indexing: $mimetype\n";
+ return false;
+ }
+
+ if ($esDoc === null) {
+ $esDoc = new \stdClass();
+ }
+ if (!isset($esDoc->status)) {
+ $esDoc->status = new \stdClass();
+ }
+
+ //FIXME: update index only if changed since last index time
+ //FIXME: extract base url from html
+ //FIXME: check if effective url needs updating
+
+ $base = new \Net_URL2($url);
+
+ //FIXME: MIME type switch
+ $doc = new \DOMDocument();
+ //@ to hide parse warning messages in invalid html
+ @$doc->loadHTML($res->getBody());
+ $dx = new \DOMXPath($doc);
+
+ $xbase = $dx->evaluate('/html/head/base[@href]')->item(0);
+ if ($xbase) {
+ $base = $base->resolve(
+ $xbase->attributes->getNamedItem('href')->textContent
+ );
+ }
+
+ $meta = $dx->evaluate('/html/head/meta[@name="robots" and @content]')
+ ->item(0);
+ if ($meta) {
+ $robots = $meta->attributes->getNamedItem('content')->textContent;
+ foreach (explode(',', $robots) as $value) {
+ if (trim($value) == 'noindex') {
+ echo "URL does not want to be indexed: $url\n";
+ exit(0);
+ }
+ }
+ }
+
+ //remove script tags
+ $this->removeTags($doc, 'script');
+ $this->removeTags($doc, 'style');
+ $this->removeTags($doc, 'nav');
+
+ //default content: <body>
+ $xpContext = $doc->getElementsByTagName('body')->item(0);
+ //FIXME: follow meta refresh, no body
+ // example: https://www.gnu.org/software/coreutils/
+
+ //use microformats content if it exists
+ $xpElems = $dx->query(
+ "//*[contains(concat(' ', normalize-space(@class), ' '), ' e-content ')]"
+ );
+ if ($xpElems->length) {
+ $xpContext = $xpElems->item(0);
+ } else if ($doc->getElementById('content')) {
+ //if there is an element with ID "content", we'll use this
+ $xpContext = $doc->getElementById('content');
+ }
+
+ $esDoc->url = $url;
+ $esDoc->schemalessUrl = Helper::noSchema($url);
+ $esDoc->type = 'html';
+ $esDoc->subtype = '';
+ $esDoc->mimetype = $mimetype;
+ $esDoc->domain = parse_url($url, PHP_URL_HOST);
+
+ //$esDoc->source = 'FIXME';
+ //$esDoc->sourcetitle = 'FIXME';
+
+ $esDoc->author = new \stdClass();
+
+ $arXpElems = $dx->query('/html/head/meta[@name="author" and @content]');
+ if ($arXpElems->length) {
+ $esDoc->author->name = trim(
+ $arXpElems->item(0)->attributes->getNamedItem('content')->textContent
+ );
+ }
+ $arXpElems = $dx->query('/html/head/link[@rel="author" and @href]');
+ if ($arXpElems->length) {
+ $esDoc->author->url = trim(
+ $base->resolve(
+ $arXpElems->item(0)->attributes->getNamedItem('href')->textContent
+ )
+ );
+ }
+
+
+ $arXpElems = $dx->query('/html/head/title');
+ if ($arXpElems->length) {
+ $esDoc->title = trim(
+ $arXpElems->item(0)->textContent
+ );
+ }
+
+ foreach (array('h1', 'h2', 'h3', 'h4', 'h5', 'h6') as $headlinetype) {
+ $esDoc->$headlinetype = array();
+ foreach ($xpContext->getElementsByTagName($headlinetype) as $xheadline) {
+ array_push(
+ $esDoc->$headlinetype,
+ trim($xheadline->textContent)
+ );
+ }
+ }
+
+ //FIXME: split paragraphs
+ //FIXME: insert space after br
+ $esDoc->text = array();
+ $esDoc->text[] = trim(
+ str_replace(
+ array("\r\n", "\n", "\r", ' '),
+ ' ',
+ $xpContext->textContent
+ )
+ );
+
+ //tags
+ $tags = array();
+ foreach ($dx->query('/html/head/meta[@name="keywords" and @content]') as $xkeywords) {
+ $keywords = $xkeywords->attributes->getNamedItem('content')->textContent;
+ foreach (explode(',', $keywords) as $keyword) {
+ $tags[trim($keyword)] = true;
+ }
+ }
+ $esDoc->tags = array_keys($tags);
+
+ //dates
+ $arXpdates = $dx->query('/html/head/meta[@name="DC.date.created" and @content]');
+ if ($arXpdates->length) {
+ $esDoc->status->crdate = gmdate(
+ 'c',
+ strtotime(
+ $arXpdates->item(0)->attributes->getNamedItem('content')->textContent
+ )
+ );
+ }
+ //FIXME: keep creation date from database, or use modified date if we
+ // do not have it there
+
+ $arXpdates = $dx->query('/html/head/meta[@name="DC.date.modified" and @content]');
+ if ($arXpdates->length) {
+ $esDoc->status->modate = gmdate(
+ 'c',
+ strtotime(
+ $arXpdates->item(0)->attributes->getNamedItem('content')->textContent
+ )
+ );
+ } else {
+ $lm = $res->getHeader('last-modified');
+ if ($lm !== null) {
+ $esDoc->status->modate = gmdate('c', strtotime($lm));
+ } else {
+ //use current time since we don't have any other data
+ $esDoc->status->modate = gmdate('c');
+ }
+ }
+ $esDoc->status->findable = true;
+
+ //language
+ //there may be "en-US" and "de-DE"
+ $xlang = $doc->documentElement->attributes->getNamedItem('lang');
+ if ($xlang) {
+ $esDoc->language = strtolower(substr($xlang->textContent, 0, 2));
+ }
+ //FIXME: fallback, autodetection
+ //FIXME: check noindex
+
+ //var_dump($esDoc);die();
+
+ $retrieved->esDoc = $esDoc;
+ return true;
+ }
+
+ function removeTags($doc, $tag) {
+ $elems = array();
+ foreach ($doc->getElementsbyTagName($tag) as $elem) {
+ $elems[] = $elem;
+ }
+ foreach ($elems as $elem) {
+ $elem->parentNode->removeChild($elem);
+ }
+ }
+}
+?>
generated by cgit v1.2.3 (git 2.46.0) at 2026-04-06 08:26:49 +0000