Merge pull request #112 from RicterZ/dev

0.3.8

nhentai/__init__.py

@@ -1,3 +1,3 @@
-__version__ = '0.3.7'
+__version__ = '0.3.8'
 __author__ = 'RicterZ'
 __email__ = 'ricterzheng@gmail.com'

nhentai/cmdline.py

@@ -10,7 +10,7 @@ except ImportError:
 
 import nhentai.constant as constant
 from nhentai import __version__
-from nhentai.utils import urlparse, generate_html, generate_main_html
+from nhentai.utils import urlparse, generate_html, generate_main_html, DB
 from nhentai.logger import logger
 
 try:
@@ -52,10 +52,12 @@ def cmd_parser():
                       help='search doujinshi by keyword')
     parser.add_option('--tag', type='string', dest='tag', action='store', help='download doujinshi by tag')
     parser.add_option('--artist', type='string', dest='artist', action='store', help='download doujinshi by artist')
-    parser.add_option('--character', type='string', dest='character', action='store', help='download doujinshi by character')
+    parser.add_option('--character', type='string', dest='character', action='store',
+                      help='download doujinshi by character')
     parser.add_option('--parody', type='string', dest='parody', action='store', help='download doujinshi by parody')
     parser.add_option('--group', type='string', dest='group', action='store', help='download doujinshi by group')
-    parser.add_option('--language', type='string', dest='language', action='store', help='download doujinshi by language')
+    parser.add_option('--language', type='string', dest='language', action='store',
+                      help='download doujinshi by language')
     parser.add_option('--favorites', '-F', action='store_true', dest='favorites',
                       help='list or download your favorites.')
 
@@ -99,6 +101,10 @@ def cmd_parser():
     # nhentai options
     parser.add_option('--cookie', type='str', dest='cookie', action='store',
                       help='set cookie of nhentai to bypass Google recaptcha')
+    parser.add_option('--save-download-history', dest='is_save_download_history', action='store_true',
+                      default=False, help='save downloaded doujinshis, whose will be skipped if you re-download them')
+    parser.add_option('--clean-download-history', action='store_true', default=False, dest='clean_download_history',
+                      help='clean download history')
 
     try:
         sys.argv = [unicode(i.decode(sys.stdin.encoding)) for i in sys.argv]
@@ -120,8 +126,15 @@ def cmd_parser():
         generate_main_html()
         exit(0)
 
-    if os.path.exists(os.path.join(constant.NHENTAI_HOME, 'cookie')):
+    if args.clean_download_history:
-        with open(os.path.join(constant.NHENTAI_HOME, 'cookie'), 'r') as f:
+        with DB() as db:
+            db.clean_all()
+
+        logger.info('Download history cleaned.')
+        exit(0)
+
+    if os.path.exists(constant.NHENTAI_COOKIE):
+        with open(constant.NHENTAI_COOKIE, 'r') as f:
             constant.COOKIE = f.read()
 
     if args.cookie:
@@ -129,7 +142,7 @@ def cmd_parser():
             if not os.path.exists(constant.NHENTAI_HOME):
                 os.mkdir(constant.NHENTAI_HOME)
 
-            with open(os.path.join(constant.NHENTAI_HOME, 'cookie'), 'w') as f:
+            with open(constant.NHENTAI_COOKIE, 'w') as f:
                 f.write(args.cookie)
         except Exception as e:
             logger.error('Cannot create NHENTAI_HOME: {}'.format(str(e)))
@@ -138,8 +151,8 @@ def cmd_parser():
         logger.info('Cookie saved.')
         exit(0)
 
-    if os.path.exists(os.path.join(constant.NHENTAI_HOME, 'proxy')):
+    if os.path.exists(constant.NHENTAI_PROXY):
-        with open(os.path.join(constant.NHENTAI_HOME, 'proxy'), 'r') as f:
+        with open(constant.NHENTAI_PROXY, 'r') as f:
             link = f.read()
             constant.PROXY = {'http': link, 'https': link}
 
@@ -152,8 +165,9 @@ def cmd_parser():
             if proxy_url.scheme not in ('http', 'https'):
                 logger.error('Invalid protocol \'{0}\' of proxy, ignored'.format(proxy_url.scheme))
             else:
-                with open(os.path.join(constant.NHENTAI_HOME, 'proxy'), 'w') as f:
+                with open(constant.NHENTAI_PROXY, 'w') as f:
                     f.write(args.proxy)
+
         except Exception as e:
             logger.error('Cannot create NHENTAI_HOME: {}'.format(str(e)))
             exit(1)

nhentai/command.py

@@ -4,15 +4,14 @@ from __future__ import unicode_literals, print_function
 import signal
 import platform
 import time
-import multiprocessing
 
 from nhentai.cmdline import cmd_parser, banner
-from nhentai.parser import doujinshi_parser, search_parser, print_doujinshi, favorites_parser, tag_parser, login
+from nhentai.parser import doujinshi_parser, search_parser, print_doujinshi, favorites_parser, tag_parser
 from nhentai.doujinshi import Doujinshi
-from nhentai.downloader import Downloader, init_worker
+from nhentai.downloader import Downloader
 from nhentai.logger import logger
 from nhentai.constant import BASE_URL
-from nhentai.utils import generate_html, generate_cbz, generate_main_html, check_cookie, signal_handler
+from nhentai.utils import generate_html, generate_cbz, generate_main_html, check_cookie, signal_handler, DB
 
 
 def main():
@@ -28,6 +27,8 @@ def main():
     # check your cookie
     check_cookie()
 
+    index = 0
+    doujinshis = []
     doujinshi_ids = []
     doujinshi_list = []
 
@@ -36,55 +37,44 @@ def main():
             logger.warning('You do not specify --download option')
 
         doujinshis = favorites_parser(options.page_range)
-        print_doujinshi(doujinshis)
-        if options.is_download and doujinshis:
-            doujinshi_ids = [i['id'] for i in doujinshis]
 
     elif options.tag:
         doujinshis = tag_parser(options.tag, sorting=options.sorting, max_page=options.max_page)
-        print_doujinshi(doujinshis)
-        if options.is_download and doujinshis:
-            doujinshi_ids = [i['id'] for i in doujinshis]
 
     elif options.artist:
-        doujinshis = tag_parser(options.artist, max_page=options.max_page, index=1)
+        index = 1
-        print_doujinshi(doujinshis)
-        if options.is_download and doujinshis:
-            doujinshi_ids = [i['id'] for i in doujinshis]
 
     elif options.character:
-        doujinshis = tag_parser(options.character, max_page=options.max_page, index=2)
+        index = 2
-        print_doujinshi(doujinshis)
-        if options.is_download and doujinshis:
-            doujinshi_ids = [i['id'] for i in doujinshis]
 
     elif options.parody:
-        doujinshis = tag_parser(options.parody, max_page=options.max_page, index=3)
+        index = 3
-        print_doujinshi(doujinshis)
-        if options.is_download and doujinshis:
-            doujinshi_ids = [i['id'] for i in doujinshis]
 
     elif options.group:
-        doujinshis = tag_parser(options.group, max_page=options.max_page, index=4)
+        index = 4
-        print_doujinshi(doujinshis)
-        if options.is_download and doujinshis:
-            doujinshi_ids = [i['id'] for i in doujinshis]
 
     elif options.language:
-        doujinshis = tag_parser(options.language, max_page=options.max_page, index=5)
+        index = 5
-        print_doujinshi(doujinshis)
-        if options.is_download and doujinshis:
-            doujinshi_ids = [i['id'] for i in doujinshis]
 
     elif options.keyword:
         doujinshis = search_parser(options.keyword, sorting=options.sorting, page=options.page)
-        print_doujinshi(doujinshis)
-        if options.is_download:
-            doujinshi_ids = [i['id'] for i in doujinshis]
 
     elif not doujinshi_ids:
         doujinshi_ids = options.id
 
+    if index:
+        doujinshis = tag_parser(options.language, max_page=options.max_page, index=index)
+
+    print_doujinshi(doujinshis)
+    if options.is_download and doujinshis:
+        doujinshi_ids = [i['id'] for i in doujinshis]
+
+        if options.is_save_download_history:
+            with DB() as db:
+                data = set(db.get_all())
+
+            doujinshi_ids = list(set(doujinshi_ids) - data)
+
     if doujinshi_ids:
         for i, id_ in enumerate(doujinshi_ids):
             if options.delay:
@@ -106,6 +96,9 @@ def main():
 
             doujinshi.downloader = downloader
             doujinshi.download()
+            if options.is_save_download_history:
+                with DB() as db:
+                    db.add_one(doujinshi.id)
 
             if not options.is_nohtml and not options.is_cbz:
                 generate_html(options.output_dir, doujinshi)

nhentai/constant.py

@@ -33,6 +33,9 @@ u = urlparse(BASE_URL)
 IMAGE_URL = '%s://i.%s/galleries' % (u.scheme, u.hostname)
 
 NHENTAI_HOME = os.path.join(os.getenv('HOME', tempfile.gettempdir()), '.nhentai')
+NHENTAI_PROXY = os.path.join(NHENTAI_HOME, 'proxy')
+NHENTAI_COOKIE = os.path.join(NHENTAI_HOME, 'cookie')
+NHENTAI_HISTORY = os.path.join(NHENTAI_HOME, 'history.sqlite3')
 
 PROXY = {}
 

nhentai/utils.py

@@ -8,6 +8,7 @@ import string
 import zipfile
 import shutil
 import requests
+import sqlite3
 
 from nhentai import constant
 from nhentai.logger import logger
@@ -214,3 +215,30 @@ an invalid filename.
 def signal_handler(signal, frame):
     logger.error('Ctrl-C signal received. Stopping...')
     exit(1)
+
+
+class DB(object):
+    conn = None
+    cur = None
+
+    def __enter__(self):
+        self.conn = sqlite3.connect(constant.NHENTAI_HISTORY)
+        self.cur = self.conn.cursor()
+        self.cur.execute('CREATE TABLE IF NOT EXISTS download_history (id text)')
+        self.conn.commit()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.conn.close()
+
+    def clean_all(self):
+        self.cur.execute('DELETE FROM download_history WHERE 1')
+        self.conn.commit()
+
+    def add_one(self, data):
+        self.cur.execute('INSERT INTO download_history VALUES (?)', [data])
+        self.conn.commit()
+
+    def get_all(self):
+        data = self.cur.execute('SELECT id FROM download_history')
+        return [i[0] for i in data]