fix bugs

nhentai/command.py

@@ -4,11 +4,12 @@ from __future__ import unicode_literals, print_function
 import signal
 import platform
 import time
+import multiprocessing
 
 from nhentai.cmdline import cmd_parser, banner
 from nhentai.parser import doujinshi_parser, search_parser, print_doujinshi, favorites_parser, tag_parser, login
 from nhentai.doujinshi import Doujinshi
-from nhentai.downloader import Downloader
+from nhentai.downloader import Downloader, init_worker
 from nhentai.logger import logger
 from nhentai.constant import BASE_URL
 from nhentai.utils import generate_html, generate_cbz, generate_main_html, check_cookie, signal_handler
@@ -55,25 +56,35 @@ def main():
         doujinshi_ids = options.id
 
     if doujinshi_ids:
-        for id_ in doujinshi_ids:
+        for i, id_ in enumerate(doujinshi_ids):
             if options.delay:
                 time.sleep(options.delay)
+
             doujinshi_info = doujinshi_parser(id_)
+
+            if doujinshi_info:
                 doujinshi_list.append(Doujinshi(name_format=options.name_format, **doujinshi_info))
 
+            if (i + 1) % 10 == 0:
+                logger.info('Progress: %d / %d' % (i + 1, len(doujinshi_ids)))
+
     if not options.is_show:
-        downloader = Downloader(path=options.output_dir,
-                                thread=options.threads, timeout=options.timeout, delay=options.delay)
+        downloader = Downloader(path=options.output_dir, size=options.threads,
+                                timeout=options.timeout, delay=options.delay)
 
         for doujinshi in doujinshi_list:
+
             doujinshi.downloader = downloader
             doujinshi.download()
+
             if not options.is_nohtml and not options.is_cbz:
                 generate_html(options.output_dir, doujinshi)
             elif options.is_cbz:
                 generate_cbz(options.output_dir, doujinshi, options.rm_origin_dir)
+
         if options.main_viewer:
             generate_main_html(options.output_dir)
+
         if not platform.system() == 'Windows':
             logger.log(15, '🍻 All done.')
         else:

nhentai/downloader.py

@@ -1,6 +1,7 @@
 # coding: utf-
 from __future__ import unicode_literals, print_function
 
+import multiprocessing
 import signal
 
 from future.builtins import str as text
@@ -27,26 +28,11 @@ class NHentaiImageNotExistException(Exception):
     pass
 
 
-class Pool(Singleton):
-    pool = None
-
-    def __init__(self, size, init):
-        if self.pool is None:
-            if os.getenv('DEBUG'):
-                logger.info('Process pool created')
-
-            self.pool = mp.Pool(size, initializer=init)
-
-
 class Downloader(Singleton):
 
-    def __init__(self, path='', thread=1, timeout=30, delay=0):
-        if not isinstance(thread, (int, )) or thread < 1 or thread > 15:
-            raise ValueError('Invalid threads count')
-
+    def __init__(self, path='', size=5, timeout=30, delay=0):
+        self.size = size
         self.path = str(path)
-        self.thread_count = thread
-        self.threads = []
         self.timeout = timeout
         self.delay = delay
 
@@ -140,13 +126,14 @@ class Downloader(Singleton):
                 os.makedirs(folder)
             except EnvironmentError as e:
                 logger.critical('{0}'.format(str(e)))
-                exit(1)
+
         else:
             logger.warn('Path \'{0}\' already exist.'.format(folder))
 
         queue = [(self, url, folder) for url in queue]
 
-        pool = Pool(self.thread_count, init_worker).pool
+        pool = multiprocessing.Pool(self.size, init_worker)
+
         for item in queue:
             pool.apply_async(download_wrapper, args=item, callback=self._download_callback)
 

nhentai/parser.py

@@ -121,8 +121,8 @@ def doujinshi_parser(id_):
             return doujinshi_parser(str(id_))
 
     except Exception as e:
-        logger.critical(str(e))
-        raise SystemExit
+        logger.warn('Error: {}, ignored'.format(str(e)))
+        return None
 
     html = BeautifulSoup(response, 'html.parser')
     doujinshi_info = html.find('div', attrs={'id': 'info'})
@@ -171,12 +171,7 @@ def doujinshi_parser(id_):
 
 def search_parser(keyword, sorting='date', page=1):
     logger.debug('Searching doujinshis of keyword {0}'.format(keyword))
-    try:
     response = request('get', url=constant.SEARCH_URL, params={'q': keyword, 'page': page, 'sort': sorting}).content
-    except requests.ConnectionError as e:
-        logger.critical(e)
-        logger.warn('If you are in China, please configure the proxy to fu*k GFW.')
-        raise SystemExit
 
     result = _get_title_and_id(response)
     if not result: