- protected function fetch($url)
- {
- $existingDoc = $this->es->get($url);
-
- $req = new HttpRequest($url);
- $req->setHeader(
- 'accept',
- implode(',', array_keys(static::$supportedIndexTypes))
- );
- if ($existingDoc && isset($existingDoc->modate)) {
- $nMoDate = strtotime($existingDoc->modate);
- $req->setHeader('If-Modified-Since: ' . date('r', $nMoDate));
- }
-
- $res = $req->send();
- if ($res->getStatus() === 304) {
- //not modified since last time, so don't crawl again
- $this->log('Not modified since last fetch');
- return false;
- } else if ($res->getStatus() !== 200) {
- throw new \Exception(
- "Response code is not 200 but "
- . $res->getStatus() . ", stopping"
- );
- }
- return $res;
- }
-