From 3c83676016bce1727f0046f4aad7865be8a71fd4 Mon Sep 17 00:00:00 2001 From: Christian Weiske Date: Mon, 7 May 2012 20:36:13 +0200 Subject: [PATCH] use httprequest wrapper to automatically check for errors --- scripts/index.php | 87 +++---------------- scripts/search.php | 2 +- src/phorkie/Database.php | 4 + .../Adapter/Elasticsearch/HTTPRequest.php | 25 ++++++ src/phorkie/Database/Setup/Elasticsearch.php | 22 ++++- src/phorkie/Indexer/Elasticsearch.php | 60 ++++++++++--- src/phorkie/Search/Elasticsearch.php | 2 +- 7 files changed, 110 insertions(+), 92 deletions(-) create mode 100644 src/phorkie/Database/Adapter/Elasticsearch/HTTPRequest.php diff --git a/scripts/index.php b/scripts/index.php index 0a2e26c..eb328b5 100644 --- a/scripts/index.php +++ b/scripts/index.php @@ -24,85 +24,24 @@ if ($GLOBALS['phorkie']['cfg']['setupcheck']) { SetupCheck::run(); } -//delete all repos -$r = new \HTTP_Request2( - $GLOBALS['phorkie']['cfg']['elasticsearch'] . 'repo/_query', - \HTTP_Request2::METHOD_DELETE -); -$r->setBody( - json_encode( - (object)array( - 'match_all' => (object)array() - ) - ) -); -$r->send(); -$r = new \HTTP_Request2( - $GLOBALS['phorkie']['cfg']['elasticsearch'] . 'file/_query', - \HTTP_Request2::METHOD_DELETE -); -$r->setBody( - json_encode( - (object)array( - 'match_all' => (object)array() - ) - ) -); -$r->send(); -//create mapping -$r = new \HTTP_Request2( - $GLOBALS['phorkie']['cfg']['elasticsearch'] . 'file/_mapping', - \HTTP_Request2::METHOD_PUT -); -$r->setBody( - json_encode( - (object)array( - 'file' => (object)array( - '_parent' => (object)array( - 'type' => 'repo' - ) - ) - ) - ) -); -$r->send(); +$db = new Database(); +$idx = $db->getIndexer(); + +//cleanup +echo "Deleting all index data\n"; +$idx->deleteAllRepos(); +//create mapping +echo "Index setup\n"; +$db->getSetup()->setup(); -//FIXME: define schema $rs = new Repositories(); -foreach ($rs->getList(0, 10000) as $repo) { +list($repos, $count) = $rs->getList(0, 10000); +$idx = new Indexer_Elasticsearch(); +foreach ($repos as $repo) { echo 'Indexing ' . $repo->id . "\n"; - $r = new \HTTP_Request2( - $GLOBALS['phorkie']['cfg']['elasticsearch'] . 'repo/' . $repo->id, - \HTTP_Request2::METHOD_PUT - ); - $r->setBody( - json_encode( - (object)array( - 'id' => $repo->id, - 'description' => $repo->getDescription(), - ) - ) - ); - $res = $r->send(); - - foreach ($repo->getFiles() as $file) { - $r = new \HTTP_Request2( - $GLOBALS['phorkie']['cfg']['elasticsearch'] . 'file/?parent=' . $repo->id, - \HTTP_Request2::METHOD_POST - ); - $r->setBody( - json_encode( - (object)array( - 'name' => $file->getFilename(), - 'extension' => $file->getExt(), - 'content' => $file->isText() ? $file->getContent() : '', - ) - ) - ); - $r->send(); - } + $idx->addRepo($repo, filectime($repo->gitDir)); } ?> diff --git a/scripts/search.php b/scripts/search.php index f9688bb..4de514f 100644 --- a/scripts/search.php +++ b/scripts/search.php @@ -24,7 +24,7 @@ if ($GLOBALS['phorkie']['cfg']['setupcheck']) { SetupCheck::run(); } -$r = new \HTTP_Request2( +$r = new Database_Adapter_Elasticsearch_HTTPRequest( $GLOBALS['phorkie']['cfg']['elasticsearch'] . 'repo/_search', \HTTP_Request2::METHOD_GET ); diff --git a/src/phorkie/Database.php b/src/phorkie/Database.php index 9df6d8c..e9a75a1 100644 --- a/src/phorkie/Database.php +++ b/src/phorkie/Database.php @@ -13,6 +13,10 @@ class Database return new Indexer_Elasticsearch(); } + public function getSetup() + { + return new Database_Setup_Elasticsearch(); + } } ?> diff --git a/src/phorkie/Database/Adapter/Elasticsearch/HTTPRequest.php b/src/phorkie/Database/Adapter/Elasticsearch/HTTPRequest.php new file mode 100644 index 0000000..2d74a4e --- /dev/null +++ b/src/phorkie/Database/Adapter/Elasticsearch/HTTPRequest.php @@ -0,0 +1,25 @@ +getStatus() / 100); + if ($mainCode != 2) { + $js = json_decode($res->getBody()); + if (isset($js->error)) { + $error = $js->error; + } else { + $error = $res->getBody(); + } + throw new Exception( + 'Error in elasticsearch communication: ' . $error + ); + } + return $res; + } +} + +?> diff --git a/src/phorkie/Database/Setup/Elasticsearch.php b/src/phorkie/Database/Setup/Elasticsearch.php index acf4bfc..b5e2005 100644 --- a/src/phorkie/Database/Setup/Elasticsearch.php +++ b/src/phorkie/Database/Setup/Elasticsearch.php @@ -10,9 +10,15 @@ class Database_Setup_Elasticsearch public function setup() { + $r = new Database_Adapter_Elasticsearch_HTTPRequest( + $this->searchInstance . 'repo/_mapping', + \HTTP_Request2::METHOD_DELETE + ); + $r->send(); + //create mapping //mapping for repositories - $r = new \HTTP_Request2( + $r = new Database_Adapter_Elasticsearch_HTTPRequest( $this->searchInstance . 'repo/_mapping', \HTTP_Request2::METHOD_PUT ); @@ -20,6 +26,10 @@ class Database_Setup_Elasticsearch json_encode( (object)array( 'repo' => (object)array( + '_timestamp' => (object)array( + 'enabled' => true, + 'path' => 'tstamp', + ), 'properties' => (object)array( 'id' => (object)array( 'type' => 'long' @@ -27,6 +37,14 @@ class Database_Setup_Elasticsearch 'description' => (object)array( 'type' => 'string', 'boost' => 2.0 + ), + 'crdate' => (object)array( + //creation date + 'type' => 'date', + ), + 'tstamp' => (object)array( + //modification date + 'type' => 'date', ) ) ) @@ -36,7 +54,7 @@ class Database_Setup_Elasticsearch $r->send(); //mapping for files - $r = new \HTTP_Request2( + $r = new Database_Adapter_Elasticsearch_HTTPRequest( $this->searchInstance . 'file/_mapping', \HTTP_Request2::METHOD_PUT ); diff --git a/src/phorkie/Indexer/Elasticsearch.php b/src/phorkie/Indexer/Elasticsearch.php index 2921239..077865e 100644 --- a/src/phorkie/Indexer/Elasticsearch.php +++ b/src/phorkie/Indexer/Elasticsearch.php @@ -9,26 +9,30 @@ class Indexer_Elasticsearch } - public function addRepo(Repository $repo) + public function addRepo(Repository $repo, $crdate = null) { - $this->updateRepo($repo); + if ($crdate == null) { + $crdate = time(); + } + $this->updateRepo($repo, $crdate); } - public function updateRepo(Repository $repo) + public function updateRepo(Repository $repo, $crdate = null) { //add repository - $r = new \HTTP_Request2( + $r = new Database_Adapter_Elasticsearch_HTTPRequest( $this->searchInstance . 'repo/' . $repo->id, \HTTP_Request2::METHOD_PUT ); - $r->setBody( - json_encode( - (object)array( - 'id' => $repo->id, - 'description' => $repo->getDescription(), - ) - ) + $repoData = array( + 'id' => $repo->id, + 'description' => $repo->getDescription(), + 'tstamp' => gmdate('c', time()), ); + if ($crdate !== null) { + $repoData['crdate'] = gmdate('c', $crdate); + } + $r->setBody(json_encode((object)$repoData)); $r->send(); //add files @@ -36,7 +40,7 @@ class Indexer_Elasticsearch $this->deleteRepoFiles($repo); foreach ($repo->getFiles() as $file) { - $r = new \HTTP_Request2( + $r = new Database_Adapter_Elasticsearch_HTTPRequest( $this->searchInstance . 'file/?parent=' . $repo->id, \HTTP_Request2::METHOD_POST ); @@ -53,10 +57,38 @@ class Indexer_Elasticsearch } } + public function deleteAllRepos() + { + $r = new Database_Adapter_Elasticsearch_HTTPRequest( + $this->searchInstance . 'repo/_query', + \HTTP_Request2::METHOD_DELETE + ); + $r->setBody( + json_encode( + (object)array( + 'match_all' => (object)array() + ) + ) + ); + $r->send(); + $r = new Database_Adapter_Elasticsearch_HTTPRequest( + $this->searchInstance . 'file/_query', + \HTTP_Request2::METHOD_DELETE + ); + $r->setBody( + json_encode( + (object)array( + 'match_all' => (object)array() + ) + ) + ); + $r->send(); + } + public function deleteRepo(Repository $repo) { //delete repository from index - $r = new \HTTP_Request2( + $r = new Database_Adapter_Elasticsearch_HTTPRequest( $this->searchInstance . 'repo/' . $repo->id, \HTTP_Request2::METHOD_DELETE ); @@ -68,7 +100,7 @@ class Indexer_Elasticsearch protected function deleteRepoFiles(Repository $repo) { //delete files of that repository - $r = new \HTTP_Request2( + $r = new Database_Adapter_Elasticsearch_HTTPRequest( $this->searchInstance . 'file/_query', \HTTP_Request2::METHOD_DELETE ); diff --git a/src/phorkie/Search/Elasticsearch.php b/src/phorkie/Search/Elasticsearch.php index 0ed766b..06a0cfe 100644 --- a/src/phorkie/Search/Elasticsearch.php +++ b/src/phorkie/Search/Elasticsearch.php @@ -20,7 +20,7 @@ class Search_Elasticsearch */ public function search($term, $page = 0, $perPage = 10) { - $r = new \HTTP_Request2( + $r = new Database_Adapter_Elasticsearch_HTTPRequest( $this->searchInstance . 'repo/_search', \HTTP_Request2::METHOD_GET ); -- 2.30.2