rework crawler; add atom link extraction

1 files changed, 16 insertions, 0 deletions

diff --git a/src/phinde/HttpRequest.php b/src/phinde/HttpRequest.php
new file mode 100644
index 0000000..e68bd84
--- /dev/null
+++ b/src/phinde/HttpRequest.php
@@ -0,0 +1,16 @@
+<?php
+namespace phinde;
+
+class HttpRequest extends \HTTP_Request2
+{
+    public function __construct($url)
+    {
+        parent::__construct($url);
+        $this->setConfig('follow_redirects', true);
+        $this->setConfig('connect_timeout', 5);
+        $this->setConfig('timeout', 10);
+        $this->setConfig('ssl_verify_peer', false);
+        $this->setHeader('user-agent', 'phinde/bot');
+    }
+}
+?>