From aa354f23c93399e8e21e26da5563beb4c60d9631 Mon Sep 17 00:00:00 2001
From: ricterz <ricterzheng@gmail.com>
Date: Sun, 10 May 2015 18:04:07 +0800
Subject: [PATCH] use threadpool instead of threading

---
 nhentai/dojinshi.py   |  5 ++---
 nhentai/downloader.py | 45 +++++++++++++++----------------------------
 requirements.txt      |  3 ++-
 3 files changed, 20 insertions(+), 33 deletions(-)

diff --git a/nhentai/dojinshi.py b/nhentai/dojinshi.py
index baeb206..945aeea 100644
--- a/nhentai/dojinshi.py
+++ b/nhentai/dojinshi.py
@@ -1,4 +1,3 @@
-import Queue
 from constant import DETAIL_URL, IMAGE_URL
 from logger import logger
 
@@ -27,9 +26,9 @@ class Dojinshi(object):
     def download(self):
         logger.info('Start download dojinshi: %s' % self.name)
         if self.downloader:
-            download_queue = Queue.Queue()
+            download_queue = []
             for i in xrange(1, self.pages + 1):
-                download_queue.put('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext))
+                download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext))
             self.downloader.download(download_queue, self.id)
         else:
             logger.critical('Downloader has not be loaded')
diff --git a/nhentai/downloader.py b/nhentai/downloader.py
index 76dbb62..41f2678 100644
--- a/nhentai/downloader.py
+++ b/nhentai/downloader.py
@@ -5,6 +5,7 @@ import socket
 import threading
 import Queue
 import requests
+import threadpool
 from urlparse import urlparse
 from logger import logger
 
@@ -36,9 +37,10 @@ class Downloader(object):
             try:
                 os.mkdir(folder)
             except os.error, e:
-                logger.error('%s error %s' % (threading.currentThread().getName(), str(e)))
+                logger.critical('Error: %s' % str(e))
                 sys.exit()
 
+        logger.info('Start downloading: %s ...' % url)
         filename = filename if filename else os.path.basename(urlparse(url).path)
         try:
             with open(os.path.join(folder, filename), "wb") as f:
@@ -50,23 +52,14 @@ class Downloader(object):
                     for chunk in response.iter_content(2048):
                         f.write(chunk)
         except (os.error, IOError), e:
-            logger.error('%s error %s' % (threading.currentThread().getName(), str(e)))
+            logger.critical('Error: %s' % str(e))
             sys.exit()
         except Exception, e:
             raise e
-        logger.info('%s %s downloaded.' % (threading.currentThread().getName(), url))
+        return url
 
-    def _download_thread(self, queue, folder=''):
-        while not self.kill_received:
-            if queue.empty():
-                queue.task_done()
-                break
-            try:
-                url = queue.get(False)
-                logger.info('%s downloading: %s ...' % (threading.currentThread().getName(), url))
-                self._download(url, folder)
-            except Queue.Empty:
-                break
+    def _download_callback(self, request, result):
+        logger.log(15, '%s download successfully' % result)
 
     def download(self, queue, folder=''):
         if not isinstance(folder, (str, unicode)):
@@ -80,20 +73,14 @@ class Downloader(object):
         else:
             logger.warn('Path \'%s\' not exist' % folder)
 
-        for i in range(self.thread_count):
-            _ = threading.Thread(target=self._download_thread, args=(queue, folder, ))
-            _.setDaemon(True)
-            self.threads.append(_)
+        queue = [([url], {'folder': folder}) for url in queue]
 
-        for thread in self.threads:
-            thread.start()
+        self.thread_pool = threadpool.ThreadPool(self.thread_count)
+        requests_ = threadpool.makeRequests(self._download, queue, self._download_callback)
+        [self.thread_pool.putRequest(req) for req in requests_]
 
-        while len(self.threads) > 0:
-            try:
-                self.threads = [t.join(THREAD_TIMEOUT) for t in self.threads if t and t.isAlive()]
-            except KeyboardInterrupt:
-                logger.warning('Ctrl-C received, sending kill signal.')
-                self.kill_received = True
-
-        # clean threads list
-        self.threads = []
+        try:
+            self.thread_pool.wait()
+        except KeyboardInterrupt:
+            print
+            logger.error('Ctrl-C pressed, exiting threads ...')
diff --git a/requirements.txt b/requirements.txt
index eb03d15..ba0166e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,3 @@
 requests>=2.5.0
-BeautifulSoup4>=4.0.0
\ No newline at end of file
+BeautifulSoup4>=4.0.0
+threadpool>=1.2.7
\ No newline at end of file