From: Christian Weiske Date: Fri, 14 Jun 2013 08:46:48 +0000 (+0200) Subject: command line interface to update and force since feeds, feed entries and ping urls X-Git-Url: https://git.cweiske.de/stapibas.git/commitdiff_plain/2a9a6c334ed1f091e9965f63392bc6733cae9b28 command line interface to update and force since feeds, feed entries and ping urls --- diff --git a/README.rst b/README.rst index a3e81d7..108c90c 100644 --- a/README.rst +++ b/README.rst @@ -13,11 +13,13 @@ Dependencies ============ - PHP 5.3+ - PDO +- `Console_CommandLine`__ - `Net_URL2`__ - `HTTP_Request2`__ - `PEAR2 Services_Pingback`__ - `SimplePie`__ +__ http://pear.php.net/package/Console_CommandLine __ http://pear.php.net/package/Net_URL2 __ http://pear.php.net/package/HTTP_Request2 __ https://github.com/pear2/Services_Pingback diff --git a/bin/stapibas b/bin/stapibas index cecbcb9..97028ec 100755 --- a/bin/stapibas +++ b/bin/stapibas @@ -4,22 +4,6 @@ namespace stapibas; require_once __DIR__ . '/../data/config.php'; require_once 'stapibas/autoloader.php'; -$db = new PDO($dbdsn, $dbuser, $dbpass); -$log = new Logger(); - -$uf = new Feed_UpdateFeeds(); -$uf->db = $db; -$uf->log = $log; -$uf->updateAll(); - -$uf = new Feed_UpdateEntries(); -$uf->db = $db; -$uf->log = $log; -$uf->updateAll(); - -$uf = new Feed_PingUrls(); -$uf->db = $db; -$uf->log = $log; -$uf->pingAll(); - +$cli = new Cli(); +$cli->run(); ?> diff --git a/src/stapibas/Cli.php b/src/stapibas/Cli.php new file mode 100644 index 0000000..683a726 --- /dev/null +++ b/src/stapibas/Cli.php @@ -0,0 +1,172 @@ +setupCli(); + } + + public function run() + { + try { + $result = $this->cliParser->parse(); + } catch (\Exception $exc) { + $this->cliParser->displayError($exc->getMessage()); + } + + $log = new Logger(); + if ($result->options['debug']) { + $log->debug = true; + } + + try { + $deps = new Dependencies(); + $deps->db = new PDO( + $GLOBALS['dbdsn'], $GLOBALS['dbuser'], $GLOBALS['dbpass'] + ); + $deps->log = $log; + $deps->options = $result->options; + + $tasks = array_flip(explode(',', $result->options['tasks'])); + + if (isset($tasks['feeds'])) { + $this->runUpdateFeeds($deps); + } + if (isset($tasks['entries'])) { + $this->runUpdateEntries($deps); + } + if (isset($tasks['urls'])) { + $this->runPingUrls($deps); + } + } catch (\Exception $e) { + $msg = 'stapibas exception!' . "\n" + . 'Code: ' . $e->getCode() . "\n" + . 'Message: ' . $e->getMessage() . "\n"; + file_put_contents('php://stderr', $msg); + exit(1); + } + } + + protected function runUpdateFeeds($deps) + { + $uf = new Feed_UpdateFeeds($deps); + if ($deps->options['feed'] === null) { + $uf->updateAll(); + } else { + $urlOrIds = explode(',', $deps->options['feed']); + $uf->updateSome($urlOrIds); + } + } + + protected function runUpdateEntries($deps) + { + $ue = new Feed_UpdateEntries($deps); + if ($deps->options['entry'] === null) { + $ue->updateAll(); + } else { + $urlOrIds = explode(',', $deps->options['entry']); + $ue->updateSome($urlOrIds); + } + } + + protected function runPingUrls($deps) + { + $uf = new Feed_PingUrls($deps); + if ($deps->options['entryurl'] === null) { + $uf->pingAll(); + } else { + $urls = explode(',', $deps->options['entryurl']); + $uf->pingSome($urls); + } + } + + + public function setupCli() + { + $p = new \Console_CommandLine(); + $p->description = 'Sends pingbacks to URLs linked in Atom feed entries'; + $p->version = '0.0.1'; + + $p->addOption( + 'feed', + array( + 'short_name' => '-i', + 'long_name' => '--feed', + 'description' => 'Update this feed URL or ID', + 'help_name' => 'URL|ID', + 'action' => 'StoreString' + ) + ); + + $p->addOption( + 'entry', + array( + 'short_name' => '-e', + 'long_name' => '--entry', + 'description' => 'Update this feed entry URL or ID', + 'help_name' => 'URL|ID', + 'action' => 'StoreString' + ) + ); + + $p->addOption( + 'tasks', + array( + 'short_name' => '-t', + 'long_name' => '--tasks', + 'description' => 'Execute the given tasks (comma-separated)', + 'help_name' => 'tasks', + 'action' => 'StoreString', + 'default' => 'feeds,entries,urls', + ) + ); + $p->addOption( + 'list_tasks', + array( + 'long_name' => '--list-tasks', + 'description' => 'Show all possible tasks', + 'action' => 'List', + 'list' => array('feeds', 'entries', 'urls') + ) + ); + + $p->addOption( + 'entryurl', + array( + 'short_name' => '-u', + 'long_name' => '--url', + 'description' => 'Ping this URL or ID', + 'help_name' => 'URL|ID', + 'action' => 'StoreString' + ) + ); + + + $p->addOption( + 'debug', + array( + 'short_name' => '-d', + 'long_name' => '--debug', + 'description' => "Output debug messages", + 'action' => 'StoreTrue' + ) + ); + $p->addOption( + 'force', + array( + 'short_name' => '-f', + 'long_name' => '--force', + 'description' => "Update even when resource did not change", + 'action' => 'StoreTrue' + ) + ); + + $this->cliParser = $p; + } + +} +?> diff --git a/src/stapibas/Dependencies.php b/src/stapibas/Dependencies.php new file mode 100644 index 0000000..6bb57a4 --- /dev/null +++ b/src/stapibas/Dependencies.php @@ -0,0 +1,13 @@ + diff --git a/src/stapibas/Feed/PingUrls.php b/src/stapibas/Feed/PingUrls.php index 17fff79..2299352 100644 --- a/src/stapibas/Feed/PingUrls.php +++ b/src/stapibas/Feed/PingUrls.php @@ -10,8 +10,12 @@ class Feed_PingUrls public $log; public $pbc; - public function __construct() + public function __construct(Dependencies $deps) { + $this->deps = $deps; + $this->db = $deps->db; + $this->log = $deps->log; + $this->pbc = new \PEAR2\Services\Pingback\Client(); $req = new \HTTP_Request2(); @@ -30,17 +34,46 @@ class Feed_PingUrls $this->log->info('Pinging all URLs..'); $res = $this->db->query( 'SELECT fe_url, feu_id, feu_url FROM feedentries, feedentryurls' - . ' WHERE fe_id = feu_fe_id AND feu_active = 1 AND feu_pinged = 0' + . ' WHERE fe_id = feu_fe_id' . $this->sqlNeedsUpdate() + ); + $items = 0; + while ($row = $res->fetch(\PDO::FETCH_OBJ)) { + $this->log->info( + 'Pinging URL #%d: %s', $row->feu_id, $row->feu_url + ); + $this->ping($row); + ++$items; + } + $this->log->info('Finished pinging %d URLs.', $items); + } + + public function pingSome($urlOrIds) + { + $options = array(); + foreach ($urlOrIds as $urlOrId) { + if (is_numeric($urlOrId)) { + $options[] = 'feu_id = ' . intval($urlOrId); + } else { + $options[] = 'feu_url = ' . $this->db->quote($urlOrId); + } + } + + $this->log->info('Pinging %d URLs..', count($options)); + $res = $this->db->query( + 'SELECT fe_url, feu_id, feu_url FROM feedentries, feedentryurls' + . ' WHERE fe_id = feu_fe_id' + . $this->sqlNeedsUpdate() + . ' AND (' . implode(' OR ', $options) . ')' ); $items = 0; while ($row = $res->fetch(\PDO::FETCH_OBJ)) { $this->log->info( - sprintf('Pinging URL #%d: %s', $row->feu_id, $row->feu_url) + 'Pinging URL #%d: %s', $row->feu_id, $row->feu_url ); $this->ping($row); ++$items; } - $this->log->info(sprintf('Finished pinging %d URLs.', $items)); + $this->log->info('Finished pinging %d URLs.', $items); } public function ping($row) @@ -75,11 +108,14 @@ class Feed_PingUrls } else { //error $this->log->err('Error: ' . $res->getCode() . ': ' . $res->getMessage()); - $this->log->info( - 'Pingback response: Status code ' . $res->getResponse()->getStatus() - . ', headers: ' . print_r($res->getResponse()->getHeader(), true) - . ', body: ' . $res->getResponse()->getBody() - ); + $httpRes = $res->getResponse(); + if ($httpRes) { + $this->log->info( + 'Pingback response: Status code ' . $httpRes->getStatus() + . ', headers: ' . print_r($httpRes->getHeader(), true) + . ', body: ' . $httpRes->getBody() + ); + } $this->db->exec( 'UPDATE feedentryurls SET' . ' feu_pinged = 1' @@ -90,5 +126,13 @@ class Feed_PingUrls ); } } + + protected function sqlNeedsUpdate() + { + if ($this->deps->options['force']) { + return ''; + } + return ' AND feu_active = 1 AND feu_pinged = 0'; + } } ?> diff --git a/src/stapibas/Feed/UpdateEntries.php b/src/stapibas/Feed/UpdateEntries.php index b0daf6f..119b207 100644 --- a/src/stapibas/Feed/UpdateEntries.php +++ b/src/stapibas/Feed/UpdateEntries.php @@ -9,27 +9,60 @@ class Feed_UpdateEntries public $db; public $log; + public function __construct(Dependencies $deps) + { + $this->deps = $deps; + $this->db = $deps->db; + $this->log = $deps->log; + } + public function updateAll() { $this->log->info('Updating feed entries..'); $res = $this->db->query( 'SELECT * FROM feedentries' - . ' WHERE fe_needs_update = 1 OR fe_updated = "0000-00-00 00:00:00"' + . ' WHERE ' . $this->sqlNeedsUpdate() ); + $items = 0; while ($entryRow = $res->fetch(\PDO::FETCH_OBJ)) { - $this->log->info( - sprintf( - 'Updating feed entry #%d: %s', - $entryRow->fe_id, $entryRow->fe_url - ) - ); + ++$items; $this->updateEntry($entryRow); } - $this->log->info('Finished updating entries.'); + $this->log->info('Finished updating %d entries.', $items); + } + + public function updateSome($urlOrIds) + { + $options = array(); + foreach ($urlOrIds as $urlOrId) { + if (is_numeric($urlOrId)) { + $options[] = 'fe_id = ' . intval($urlOrId); + } else { + $options[] = 'fe_url = ' . $this->db->quote($urlOrId); + } + } + + $this->log->info('Updating %d feed entries..', count($options)); + $res = $this->db->query( + 'SELECT * FROM feedentries' + . ' WHERE ' . $this->sqlNeedsUpdate() + . ' AND (' . implode(' OR ', $options) . ')' + ); + + $items = 0; + while ($entryRow = $res->fetch(\PDO::FETCH_OBJ)) { + ++$items; + $this->updateEntry($entryRow); + } + $this->log->info('Finished updating %d entries.', $items); } protected function updateEntry($entryRow) { + $this->log->info( + 'Updating feed entry #%d: %s', $entryRow->fe_id, $entryRow->fe_url + ); + $req = new \HTTP_Request2($entryRow->fe_url); $req->setHeader('User-Agent', 'stapibas'); $req->setHeader( @@ -122,10 +155,8 @@ class Feed_UpdateEntries ); } $this->log->info( - sprintf( - 'Feed entry #%d: %d new, %d updated, %d deleted of %d URLs', - $entryRow->fe_id, $new, $updated, $deleted, $items - ) + 'Feed entry #%d: %d new, %d updated, %d deleted of %d URLs', + $entryRow->fe_id, $new, $updated, $deleted, $items ); } @@ -149,7 +180,7 @@ class Feed_UpdateEntries . '//*[' . $this->xpc('e-content') . ' or ' . $this->xpc('entry-content') . ']' . '//*[(self::a or self::h:a) and @href and not(starts-with(@href, "#"))]'; $links = $xpath->query($query); - $this->log->info(sprintf('%d links found', $links->length)); + $this->log->info('%d links found', $links->length); $entryUrl = new \Net_URL2($entryRow->fe_url); //FIXME: base URL in html code @@ -193,5 +224,12 @@ class Feed_UpdateEntries ); } + protected function sqlNeedsUpdate() + { + if ($this->deps->options['force']) { + return ' 1'; + } + return ' (fe_needs_update = 1 OR fe_updated = "0000-00-00 00:00:00")'; + } } ?> diff --git a/src/stapibas/Feed/UpdateFeeds.php b/src/stapibas/Feed/UpdateFeeds.php index 77b5890..54e89ce 100644 --- a/src/stapibas/Feed/UpdateFeeds.php +++ b/src/stapibas/Feed/UpdateFeeds.php @@ -9,24 +9,59 @@ class Feed_UpdateFeeds public $db; public $log; + public function __construct(Dependencies $deps) + { + $this->deps = $deps; + $this->db = $deps->db; + $this->log = $deps->log; + } + public function updateAll() { $this->log->info('Updating feeds..'); $res = $this->db->query( 'SELECT * FROM feeds' - . ' WHERE f_needs_update = 1 OR f_updated = "0000-00-00 00:00:00"' + . ' WHERE ' . $this->sqlNeedsUpdate() ); while ($feedRow = $res->fetch(\PDO::FETCH_OBJ)) { - $this->log->info( - sprintf('Updating feed #%d: %s', $feedRow->f_id, $feedRow->f_url) - ); $this->updateFeed($feedRow); } $this->log->info('Finished updating feeds.'); } + public function updateSome($urlOrIds) + { + $options = array(); + foreach ($urlOrIds as $urlOrId) { + if (is_numeric($urlOrId)) { + $options[] = 'f_id = ' . intval($urlOrId); + } else { + $options[] = 'f_url = ' . $this->db->quote($urlOrId); + } + } + + $this->log->info('Updating %d feeds..', $options); + $res = $this->db->query( + 'SELECT * FROM feeds' + . ' WHERE' + . $this->sqlNeedsUpdate() + . ' AND (' . implode(' OR ', $options) . ')' + ); + + $items = 0; + while ($feedRow = $res->fetch(\PDO::FETCH_OBJ)) { + ++$items; + $this->updateFeed($feedRow); + } + $this->log->info('Finished updating %d feeds.', $items); + } + protected function updateFeed($feedRow) { + $this->log->info( + 'Updating feed #%d: %s', $feedRow->f_id, $feedRow->f_url + ); + $req = new \HTTP_Request2($feedRow->f_url); $req->setHeader('User-Agent', 'stapibas'); @@ -97,10 +132,8 @@ class Feed_UpdateFeeds } } $this->log->info( - sprintf( - 'Feed #%d: %d new, %d updated of %d entries', - $feedRow->f_id, $new, $updated, $items - ) + 'Feed #%d: %d new, %d updated of %d entries', + $feedRow->f_id, $new, $updated, $items ); $this->setUpdated($feedRow, $res); } @@ -125,6 +158,12 @@ class Feed_UpdateFeeds ); } + protected function sqlNeedsUpdate() + { + if ($this->deps->options['force']) { + return ' 1'; + } + return ' (f_needs_update = 1 OR f_updated = "0000-00-00 00:00:00")'; + } } - ?> \ No newline at end of file diff --git a/src/stapibas/Logger.php b/src/stapibas/Logger.php index 407ebee..abe4435 100644 --- a/src/stapibas/Logger.php +++ b/src/stapibas/Logger.php @@ -3,18 +3,31 @@ namespace stapibas; class Logger { + public $debug = false; + public function err($msg) { - $this->log($msg); + $args = func_get_args(); + if (count($args) > 1) { + $msg = call_user_func_array('sprintf', $args); + } + file_put_contents('php://stderr', $msg . "\n"); } public function info($msg) { - $this->log($msg); + if ($this->debug == 1) { + $args = func_get_args(); + call_user_func_array(array($this, 'log'), $args); + } } public function log($msg) { + $args = func_get_args(); + if (count($args) > 1) { + $msg = call_user_func_array('sprintf', $args); + } echo $msg . "\n"; } }