change package pycurl to option.

pyspider/fetcher/tornado_fetcher.py

@@ -27,31 +27,41 @@
 from requests import cookies
 from six.moves.urllib.parse import urljoin, urlsplit
 from tornado import gen
-from tornado.curl_httpclient import CurlAsyncHTTPClient
-from tornado.simple_httpclient import SimpleAsyncHTTPClient
-
 from pyspider.libs import utils, dataurl, counter
 from pyspider.libs.url import quote_chinese
 from .cookie_utils import extract_cookies_to_jar
 logger = logging.getLogger('fetcher')
 
 
-class MyCurlAsyncHTTPClient(CurlAsyncHTTPClient):
+def make_http_client(client, async, ioloop, poolsize):
+    if client == 'pycurl':
+        from tornado.curl_httpclient import CurlAsyncHTTPClient
 
-    def free_size(self):
-        return len(self._free_list)
+        class MyAsyncHTTPClient(CurlAsyncHTTPClient):
+            def free_size(self):
+                return len(self._free_list)
 
-    def size(self):
-        return len(self._curls) - self.free_size()
+            def size(self):
+                return len(self._curls) - self.free_size()
 
+    elif client == 'simple':
+        from tornado.simple_httpclient import SimpleAsyncHTTPClient
 
-class MySimpleAsyncHTTPClient(SimpleAsyncHTTPClient):
+        class MyAsyncHTTPClient(SimpleAsyncHTTPClient):
+            def free_size(self):
+                return self.max_clients - self.size()
 
-    def free_size(self):
-        return self.max_clients - self.size()
+            def size(self):
+                return len(self.active)
+    else:
+        raise Exception('invalidate client  argument')
+
+    if async:
+        http_client = MyAsyncHTTPClient(max_clients=poolsize, io_loop=ioloop)
+    else:
+        http_client = tornado.httpclient.HTTPClient(MyAsyncHTTPClient)
+    return http_client
 
-    def size(self):
-        return len(self.active)
 
 fetcher_output = {
     "status_code": int,
@@ -78,7 +88,8 @@ class Fetcher(object):
     splash_lua_source = open(os.path.join(os.path.dirname(__file__), "splash_fetcher.lua")).read()
     robot_txt_age = 60*60  # 1h
 
-    def __init__(self, inqueue, outqueue, poolsize=100, proxy=None, async=True):
+    def __init__(self, inqueue, outqueue, poolsize=100, proxy=None, async=True,
+                       client="simple", ioloop=None):
         self.inqueue = inqueue
         self.outqueue = outqueue
 
@@ -87,16 +98,13 @@ def __init__(self, inqueue, outqueue, poolsize=100, proxy=None, async=True):
         self._quit = False
         self.proxy = proxy
         self.async = async
-        self.ioloop = tornado.ioloop.IOLoop()
+        if ioloop is None:
+            self.ioloop = tornado.ioloop.IOLoop()
 
         self.robots_txt_cache = {}
 
         # binding io_loop to http_client here
-        if self.async:
-            self.http_client = MyCurlAsyncHTTPClient(max_clients=self.poolsize,
-                                                     io_loop=self.ioloop)
-        else:
-            self.http_client = tornado.httpclient.HTTPClient(MyCurlAsyncHTTPClient, max_clients=self.poolsize)
+        self.http_client = make_http_client(client, async, ioloop, poolsize)
 
         self._cnt = {
             '5m': counter.CounterManager(

pyspider/run.py

@@ -225,9 +225,10 @@ def scheduler(ctx, xmlrpc, xmlrpc_host, xmlrpc_port,
 @click.option('--splash-endpoint', help="execute endpoint of splash: http://splash.readthedocs.io/en/stable/api.html#execute")
 @click.option('--fetcher-cls', default='pyspider.fetcher.Fetcher', callback=load_cls,
               help='Fetcher class to be used.')
+@click.option('--tornado-http-client', default='simple', type=click.Choice(['simple', 'pycurl']))
 @click.pass_context
 def fetcher(ctx, xmlrpc, xmlrpc_host, xmlrpc_port, poolsize, proxy, user_agent,
-            timeout, phantomjs_endpoint, splash_endpoint, fetcher_cls,
+            timeout, phantomjs_endpoint, splash_endpoint, fetcher_cls, tornado_http_client,
             async=True, get_object=False, no_input=False):
     """
     Run Fetcher.
@@ -241,8 +242,11 @@ def fetcher(ctx, xmlrpc, xmlrpc_host, xmlrpc_port, poolsize, proxy, user_agent,
     else:
         inqueue = g.scheduler2fetcher
         outqueue = g.fetcher2processor
+    kwargs = {}
+    if tornado_http_client:
+        kwargs['client'] = tornado_http_client
     fetcher = Fetcher(inqueue=inqueue, outqueue=outqueue,
-                      poolsize=poolsize, proxy=proxy, async=async)
+                      poolsize=poolsize, proxy=proxy, async=async, **kwargs)
     fetcher.phantomjs_proxy = phantomjs_endpoint or g.phantomjs_proxy
     fetcher.splash_endpoint = splash_endpoint
     if user_agent:

setup.py

@@ -23,7 +23,6 @@
     'chardet>=2.2',
     'cssselect>=0.9',
     'lxml',
-    'pycurl',
     'pyquery',
     'requests>=2.2',
     'tornado>=3.2',
@@ -54,6 +53,7 @@
     'redis',
     'psycopg2',
     'elasticsearch>=2.0.0,<2.4.0',
+    'pycurl',
 ]
 if sys.version_info < (2, 7):  # 2.6
     extras_require_all.extend([

tests/__init__.py

@@ -7,5 +7,12 @@
 
 import os
 import unittest2 as unittest
+path = os.path
+
+library_base_path = path.abspath(path.join(path.dirname(__file__), '..'))
+
+logger_config_path = path.join(library_base_path, "pyspider/logging.conf")
 
 all_suite = unittest.TestLoader().discover(os.path.dirname(__file__), "test_*.py")
+
+

tests/data_fetcher_processor_handler.py

@@ -47,6 +47,7 @@ def get_process_save(self, response):
     def set_process_save(self, response):
         self.save['roy'] = 'binux'
 
+
 class IgnoreHandler(BaseHandler):
     pass
 

tests/data_handler.py

@@ -6,11 +6,17 @@
 # Created on 2014-02-22 14:02:21
 
 import time
+import logging
 from pyspider.libs.base_handler import BaseHandler, catch_status_code_error, every
 
+
+logger = logging.getLogger(__name__)
+
+
 class IgnoreHandler(object):
     pass
 
+
 class TestHandler(BaseHandler):
     retry_delay = {
         1: 10,

tests/test_counter.py

@@ -11,6 +11,7 @@
 
 from pyspider.libs import counter
 
+
 class TestCounter(unittest.TestCase):
     def test_010_TimebaseAverageEventCounter(self):
         c = counter.TimebaseAverageEventCounter(2, 1)

tests/test_fetcher.py

@@ -13,10 +13,13 @@
 import umsgpack
 import subprocess
 import unittest2 as unittest
-
 import logging
 import logging.config
-logging.config.fileConfig("pyspider/logging.conf")
+
+from . import logger_config_path
+
+
+logging.config.fileConfig(logger_config_path)
 
 try:
     from six.moves import xmlrpc_client
@@ -61,7 +64,7 @@ def setUpClass(self):
 
         self.inqueue = Queue(10)
         self.outqueue = Queue(10)
-        self.fetcher = Fetcher(self.inqueue, self.outqueue)
+        self.fetcher = Fetcher(self.inqueue, self.outqueue, client='pycurl')
         self.fetcher.phantomjs_proxy = '127.0.0.1:25555'
         self.rpc = xmlrpc_client.ServerProxy('http://localhost:%d' % 24444)
         self.xmlrpc_thread = utils.run_in_thread(self.fetcher.xmlrpc_run, port=24444)

tests/test_fetcher_processor.py

@@ -23,7 +23,7 @@ class TestFetcherProcessor(unittest.TestCase):
     @classmethod
     def setUpClass(self):
         self.projectdb = ProjectDB([os.path.join(os.path.dirname(__file__), 'data_fetcher_processor_handler.py')])
-        self.fetcher = Fetcher(None, None, async=False)
+        self.fetcher = Fetcher(None, None, async=False, client='pycurl')
         self.status_queue = Queue()
         self.newtask_queue = Queue()
         self.result_queue = Queue()

tests/test_processor.py

@@ -11,7 +11,9 @@
 import time
 import unittest2 as unittest
 import logging.config
-logging.config.fileConfig("pyspider/logging.conf")
+from . import logger_config_path
+
+logging.config.fileConfig(logger_config_path)
 
 from pyspider.libs import utils
 from pyspider.processor.project_module import ProjectManager

tests/test_result_worker.py

@@ -9,7 +9,9 @@
 import time
 import unittest2 as unittest
 import logging.config
-logging.config.fileConfig("pyspider/logging.conf")
+
+from . import logger_config_path
+logging.config.fileConfig(logger_config_path)
 
 import shutil
 from pyspider.database.sqlite import resultdb

tests/test_scheduler.py

@@ -11,7 +11,9 @@
 import unittest2 as unittest
 import logging
 import logging.config
-logging.config.fileConfig("pyspider/logging.conf")
+
+from . import logger_config_path
+logging.config.fileConfig(logger_config_path)
 
 from pyspider.scheduler.task_queue import TaskQueue
 from pyspider.libs import utils