git.cweiske.de
/
phinde.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Send If-Modified-Since header on crawling and indexing
[phinde.git]
/
src
/
phinde
/
Crawler.php
diff --git
a/src/phinde/Crawler.php
b/src/phinde/Crawler.php
index 9b148789437bac97dbdf897048487e48b2ca0121..43d9459328ce9cf8fa4bc998f7de4b96555a1a7f 100644
(file)
--- a/
src/phinde/Crawler.php
+++ b/
src/phinde/Crawler.php
@@
-25,7
+25,11
@@
class Crawler
public function crawl($url)
{
public function crawl($url)
{
- $res = $this->fetch($url);
+ $res = $this->fetch($url);
+ if ($res === false) {
+ return;
+ }
+
$linkInfos = $this->extractLinks($res);
if ($this->showLinksOnly) {
$this->showLinks($linkInfos);
$linkInfos = $this->extractLinks($res);
if ($this->showLinksOnly) {
$this->showLinks($linkInfos);
@@
-36,13
+40,23
@@
class Crawler
protected function fetch($url)
{
protected function fetch($url)
{
+ $existingDoc = $this->es->get($url);
+
$req = new HttpRequest($url);
$req->setHeader(
'accept',
implode(',', array_keys(static::$supportedIndexTypes))
);
$req = new HttpRequest($url);
$req->setHeader(
'accept',
implode(',', array_keys(static::$supportedIndexTypes))
);
+ if ($existingDoc) {
+ $nMoDate = strtotime($existingDoc->modate);
+ $req->setHeader('If-Modified-Since: ' . date('r', $nMoDate));
+ }
+
$res = $req->send();
$res = $req->send();
- if ($res->getStatus() !== 200) {
+ if ($res->getStatus() === 304) {
+ //not modified since last time, so don't crawl again
+ return false;
+ } else if ($res->getStatus() !== 200) {
throw new \Exception(
"Response code is not 200 but "
. $res->getStatus() . ", stopping"
throw new \Exception(
"Response code is not 200 but "
. $res->getStatus() . ", stopping"