9 static $supportedIndexTypes = array(
10 'application/atom+xml' => '\\phinde\\LinkExtractor\\Atom',
11 'application/xhtml+xml' => '\\phinde\\LinkExtractor\\Html',
12 'text/html' => '\\phinde\\LinkExtractor\\Html',
15 public function __construct()
17 $this->es = new Elasticsearch($GLOBALS['phinde']['elasticsearch']);
18 $this->queue = new Queue();
21 public function crawl($url)
23 $res = $this->fetch($url);
24 $linkInfos = $this->extractLinks($res);
25 $this->enqueue($linkInfos);
28 protected function fetch($url)
30 $req = new HttpRequest($url);
32 if ($res->getStatus() !== 200) {
34 "Response code is not 200 but "
35 . $res->getStatus() . ", stopping"
41 protected function extractLinks(\HTTP_Request2_Response $res)
43 $mimetype = explode(';', $res->getHeader('content-type'))[0];
44 if (!isset(static::$supportedIndexTypes[$mimetype])) {
45 echo "MIME type not supported for indexing: $mimetype\n";
49 $class = static::$supportedIndexTypes[$mimetype];
50 $extractor = new $class();
51 return $extractor->extract($res);
54 protected function enqueue($linkInfos)
56 foreach ($linkInfos as $linkInfo) {
57 if ($this->es->isKnown($linkInfo->url)) {
60 $this->es->markQueued($linkInfo->url);
61 $this->queue->addToIndex(
62 $linkInfo->url, $linkInfo->title, $linkInfo->source
64 if (Helper::isUrlAllowed($linkInfo->url)) {
65 $this->queue->addToCrawl($linkInfo->url);