namespace phinde;
require_once __DIR__ . '/../src/init.php';
-if ($argc < 2) {
- echo "No URL given\n";
- exit(1);
+$cc = new \Console_CommandLine();
+$cc->description = 'phinde URL crawler';
+$cc->version = '0.0.1';
+$cc->addOption(
+ 'showLinksOnly',
+ array(
+ 'short_name' => '-s',
+ 'long_name' => '--show-links',
+ 'description' => 'Only show which URLs were found',
+ 'action' => 'StoreTrue',
+ 'default' => false
+ )
+);
+$cc->addArgument(
+ 'url',
+ array(
+ 'description' => 'URL to crawl',
+ 'multiple' => false
+ )
+);
+try {
+ $res = $cc->parse();
+} catch (\Exception $e) {
+ $cc->displayError($e->getMessage());
}
-$url = $argv[1];
+$url = $res->args['url'];
$url = Helper::addSchema($url);
if (!Helper::isUrlAllowed($url)) {
echo "Domain is not allowed; not crawling\n";
try {
$crawler = new Crawler();
+ $crawler->setShowLinksOnly($res->options['showLinksOnly']);
$crawler->crawl($url);
} catch (\Exception $e) {
echo $e->getMessage() . "\n";
protected $es;
protected $queue;
+ /**
+ * If the links only should be shown, not queued
+ */
+ protected $showLinksOnly = false;
+
static $supportedIndexTypes = array(
'application/atom+xml' => '\\phinde\\LinkExtractor\\Atom',
'application/xhtml+xml' => '\\phinde\\LinkExtractor\\Html',
{
$res = $this->fetch($url);
$linkInfos = $this->extractLinks($res);
- $this->enqueue($linkInfos);
+ if ($this->showLinksOnly) {
+ $this->showLinks($linkInfos);
+ } else {
+ $this->enqueue($linkInfos);
+ }
}
protected function fetch($url)
}
}
}
+
+ protected function showLinks($linkInfos)
+ {
+ foreach ($linkInfos as $linkInfo) {
+ echo $linkInfo->url . "\n";
+ if ($linkInfo->title) {
+ echo ' title: ' . $linkInfo->title . "\n";
+ echo ' source: ' . $linkInfo->source . "\n";
+ }
+ }
+ }
+
+ public function setShowLinksOnly($showLinksOnly)
+ {
+ $this->showLinksOnly = $showLinksOnly;
+ }
}
?>