fix indexing, boost config

[phinde.git] / bin / crawl.php

diff --git a/bin/crawl.php b/bin/crawl.php
index 26cf9944e4aa67eb8691019d6ff92500f30e66fc..e39a62257254086f8faa110dc33670fbcb0ff725 100755 (executable)
--- a/bin/crawl.php
+++ b/bin/crawl.php
@@ -1,14 +1,7 @@
 #!/usr/bin/env php
 <?php
 namespace phinde;
-
-set_include_path(__DIR__ . '/../src/' . PATH_SEPARATOR . get_include_path());
-require_once __DIR__ . '/../data/config.php';
-require_once 'HTTP/Request2.php';
-require_once 'Elasticsearch.php';
-require_once 'Elasticsearch/Request.php';
-require_once 'Net/URL2.php';
-require_once 'functions.php';
+require_once __DIR__ . '/../src/init.php';
 
 $supportedCrawlTypes = array(
     'text/html', 'application/xhtml+xml'
@@ -23,7 +16,7 @@ if ($argc < 2) {
 $es = new Elasticsearch($GLOBALS['phinde']['elasticsearch']);
 
 $url = $argv[1];
-if (!isUrlAllowed($url)) {
+if (!Helper::isUrlAllowed($url)) {
     echo "Domain is not allowed; not crawling\n";
     exit(2);
 }
@@ -93,7 +86,7 @@ foreach ($links as $link) {
     //var_dump($linkTitle, $linkUrl);
     $es->markQueued($linkUrl);
     addToIndex($linkUrl, $linkTitle, $url);
-    if (isUrlAllowed($linkUrl)) {
+    if (Helper::isUrlAllowed($linkUrl)) {
         addToCrawl($linkUrl);
     }
     $alreadySeen[$linkUrl] = true;