5 require_once __DIR__ . '/../src/init.php';
7 $supportedIndexTypes = array(
8 'application/xhtml+xml',
13 echo "No URL given\n";
17 function removeTags($doc, $tag) {
19 foreach ($doc->getElementsbyTagName($tag) as $elem) {
22 foreach ($elems as $elem) {
23 $elem->parentNode->removeChild($elem);
27 $es = new Elasticsearch($GLOBALS['phinde']['elasticsearch']);
30 $existingDoc = $es->get($url);
31 if ($existingDoc && $existingDoc->status == 'indexed') {
32 echo "URL already indexed: $url\n";
36 //FIXME: sourcetitle, sourcelink
38 $req = new \HTTP_Request2($url);
39 $req->setConfig('follow_redirects', true);
40 $req->setConfig('connect_timeout', 5);
41 $req->setConfig('timeout', 10);
42 $req->setConfig('ssl_verify_peer', false);
46 //FIXME: delete if 401 gone or 404 when updating
47 if ($res->getStatus() !== 200) {
48 echo "Response code is not 200 but " . $res->getStatus() . ", stopping\n";
49 //FIXME: update status
53 $mimetype = explode(';', $res->getHeader('content-type'))[0];
54 if (!in_array($mimetype, $supportedIndexTypes)) {
55 echo "MIME type not supported for indexing: $mimetype\n";
56 //FIXME: update status
61 //FIXME: update index only if changed since last index time
62 //FIXME: extract base url from html
63 //FIXME: check if effective url needs updating
64 $url = $res->getEffectiveUrl();
65 $base = new \Net_URL2($url);
67 $indexDoc = new \stdClass();
69 //FIXME: MIME type switch
70 $doc = new \DOMDocument();
71 //@ to hide parse warning messages in invalid html
72 @$doc->loadHTML($res->getBody());
73 $dx = new \DOMXPath($doc);
76 removeTags($doc, 'script');
77 removeTags($doc, 'style');
78 removeTags($doc, 'nav');
80 //default content: <body>
81 $xpContext = $doc->getElementsByTagName('body')->item(0);
82 //FIXME: follow meta refresh, no body
83 // example: https://www.gnu.org/software/coreutils/
85 //use microformats content if it exists
86 $xpElems = $dx->query(
87 "//*[contains(concat(' ', normalize-space(@class), ' '), ' e-content ')]"
89 if ($xpElems->length) {
90 $xpContext = $xpElems->item(0);
91 } else if ($doc->getElementById('content')) {
92 //if there is an element with ID "content", we'll use this
93 $xpContext = $doc->getElementById('content');
96 $indexDoc->url = $url;
97 $indexDoc->schemalessUrl = Helper::noSchema($url);
98 $indexDoc->type = 'html';
99 $indexDoc->subtype = '';
100 $indexDoc->mimetype = $mimetype;
101 $indexDoc->domain = parse_url($url, PHP_URL_HOST);
103 //$indexDoc->source = 'FIXME';
104 //$indexDoc->sourcetitle = 'FIXME';
106 $indexDoc->author = new \stdClass();
108 $arXpElems = $dx->query('/html/head/meta[@name="author" and @content]');
109 if ($arXpElems->length) {
110 $indexDoc->author->name = trim(
111 $arXpElems->item(0)->attributes->getNamedItem('content')->textContent
114 $arXpElems = $dx->query('/html/head/link[@rel="author" and @href]');
115 if ($arXpElems->length) {
116 $indexDoc->author->url = trim(
118 $arXpElems->item(0)->attributes->getNamedItem('href')->textContent
124 $arXpElems = $dx->query('/html/head/title');
125 if ($arXpElems->length) {
126 $indexDoc->title = trim(
127 $arXpElems->item(0)->textContent
131 foreach (array('h1', 'h2', 'h3', 'h4', 'h5', 'h6') as $headlinetype) {
132 $indexDoc->$headlinetype = array();
133 foreach ($xpContext->getElementsByTagName($headlinetype) as $xheadline) {
135 $indexDoc->$headlinetype,
136 trim($xheadline->textContent)
141 //FIXME: split paragraphs
142 //FIXME: insert space after br
143 $indexDoc->text = array();
144 $indexDoc->text[] = trim(
146 array("\r\n", "\n", "\r", ' '),
148 $xpContext->textContent
154 foreach ($dx->query('/html/head/meta[@name="keywords" and @content]') as $xkeywords) {
155 $keywords = $xkeywords->attributes->getNamedItem('content')->textContent;
156 foreach (explode(',', $keywords) as $keyword) {
157 $tags[trim($keyword)] = true;
160 $indexDoc->tags = array_keys($tags);
163 $arXpdates = $dx->query('/html/head/meta[@name="DC.date.created" and @content]');
164 if ($arXpdates->length) {
165 $indexDoc->crdate = date(
168 $arXpdates->item(0)->attributes->getNamedItem('content')->textContent
172 //FIXME: keep creation date from database, or use modified date if we
173 // do not have it there
175 $arXpdates = $dx->query('/html/head/meta[@name="DC.date.modified" and @content]');
176 if ($arXpdates->length) {
177 $indexDoc->modate = date(
180 $arXpdates->item(0)->attributes->getNamedItem('content')->textContent
184 $lm = $res->getHeader('last-modified');
186 $indexDoc->modate = date('c', strtotime($lm));
188 //use current time since we don't have any other data
189 $indexDoc->modate = date('c');
194 //there may be "en-US" and "de-DE"
195 $xlang = $doc->documentElement->attributes->getNamedItem('lang');
197 $indexDoc->language = strtolower(substr($xlang->textContent, 0, 2));
199 //FIXME: fallback, autodetection
200 //FIXME: check noindex
202 //var_dump($indexDoc);die();
204 $indexDoc->status = 'indexed';
206 //FIXME: update index if it exists already
207 $r = new Elasticsearch_Request(
208 $GLOBALS['phinde']['elasticsearch'] . 'document/' . rawurlencode($url),
209 \HTTP_Request2::METHOD_PUT
211 $r->setBody(json_encode($indexDoc));