From 5c7bdae0d7254aab6e81c83334153efc4ab5ec51 Mon Sep 17 00:00:00 2001 From: RicterZ Date: Thu, 9 Apr 2020 20:32:20 +0800 Subject: [PATCH 1/5] add a new option #111 --- nhentai/cmdline.py | 21 +++++++++++++-------- nhentai/constant.py | 2 ++ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/nhentai/cmdline.py b/nhentai/cmdline.py index f74a0e4..1a19de6 100644 --- a/nhentai/cmdline.py +++ b/nhentai/cmdline.py @@ -52,10 +52,12 @@ def cmd_parser(): help='search doujinshi by keyword') parser.add_option('--tag', type='string', dest='tag', action='store', help='download doujinshi by tag') parser.add_option('--artist', type='string', dest='artist', action='store', help='download doujinshi by artist') - parser.add_option('--character', type='string', dest='character', action='store', help='download doujinshi by character') + parser.add_option('--character', type='string', dest='character', action='store', + help='download doujinshi by character') parser.add_option('--parody', type='string', dest='parody', action='store', help='download doujinshi by parody') parser.add_option('--group', type='string', dest='group', action='store', help='download doujinshi by group') - parser.add_option('--language', type='string', dest='language', action='store', help='download doujinshi by language') + parser.add_option('--language', type='string', dest='language', action='store', + help='download doujinshi by language') parser.add_option('--favorites', '-F', action='store_true', dest='favorites', help='list or download your favorites.') @@ -99,6 +101,8 @@ def cmd_parser(): # nhentai options parser.add_option('--cookie', type='str', dest='cookie', action='store', help='set cookie of nhentai to bypass Google recaptcha') + parser.add_option('--save-download-states', dest='is_save_download_states', action='store_true', + default=False, help='save downloaded doujinshis, whose will be skipped if you re-download them') try: sys.argv = [unicode(i.decode(sys.stdin.encoding)) for i in sys.argv] @@ -120,8 +124,8 @@ def cmd_parser(): generate_main_html() exit(0) - if os.path.exists(os.path.join(constant.NHENTAI_HOME, 'cookie')): - with open(os.path.join(constant.NHENTAI_HOME, 'cookie'), 'r') as f: + if os.path.exists(constant.NHENTAI_COOKIE): + with open(constant.NHENTAI_COOKIE, 'r') as f: constant.COOKIE = f.read() if args.cookie: @@ -129,7 +133,7 @@ def cmd_parser(): if not os.path.exists(constant.NHENTAI_HOME): os.mkdir(constant.NHENTAI_HOME) - with open(os.path.join(constant.NHENTAI_HOME, 'cookie'), 'w') as f: + with open(constant.NHENTAI_COOKIE, 'w') as f: f.write(args.cookie) except Exception as e: logger.error('Cannot create NHENTAI_HOME: {}'.format(str(e))) @@ -138,8 +142,8 @@ def cmd_parser(): logger.info('Cookie saved.') exit(0) - if os.path.exists(os.path.join(constant.NHENTAI_HOME, 'proxy')): - with open(os.path.join(constant.NHENTAI_HOME, 'proxy'), 'r') as f: + if os.path.exists(constant.NHENTAI_PROXY): + with open(constant.NHENTAI_PROXY, 'r') as f: link = f.read() constant.PROXY = {'http': link, 'https': link} @@ -152,8 +156,9 @@ def cmd_parser(): if proxy_url.scheme not in ('http', 'https'): logger.error('Invalid protocol \'{0}\' of proxy, ignored'.format(proxy_url.scheme)) else: - with open(os.path.join(constant.NHENTAI_HOME, 'proxy'), 'w') as f: + with open(constant.NHENTAI_PROXY, 'w') as f: f.write(args.proxy) + except Exception as e: logger.error('Cannot create NHENTAI_HOME: {}'.format(str(e))) exit(1) diff --git a/nhentai/constant.py b/nhentai/constant.py index 5641618..4cead2e 100644 --- a/nhentai/constant.py +++ b/nhentai/constant.py @@ -33,6 +33,8 @@ u = urlparse(BASE_URL) IMAGE_URL = '%s://i.%s/galleries' % (u.scheme, u.hostname) NHENTAI_HOME = os.path.join(os.getenv('HOME', tempfile.gettempdir()), '.nhentai') +NHENTAI_PROXY = os.path.join(NHENTAI_HOME, 'proxy') +NHENTAI_COOKIE = os.path.join(NHENTAI_HOME, 'cookie') PROXY = {} From 49f07de95d70340e68c75341b9b6543ab6f493bc Mon Sep 17 00:00:00 2001 From: RicterZ Date: Thu, 9 Apr 2020 20:37:13 +0800 Subject: [PATCH 2/5] remove repeat code --- nhentai/command.py | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/nhentai/command.py b/nhentai/command.py index 40f924f..e8ee74e 100644 --- a/nhentai/command.py +++ b/nhentai/command.py @@ -28,6 +28,7 @@ def main(): # check your cookie check_cookie() + doujinshis = [] doujinshi_ids = [] doujinshi_list = [] @@ -36,55 +37,39 @@ def main(): logger.warning('You do not specify --download option') doujinshis = favorites_parser(options.page_range) - print_doujinshi(doujinshis) - if options.is_download and doujinshis: - doujinshi_ids = [i['id'] for i in doujinshis] elif options.tag: doujinshis = tag_parser(options.tag, sorting=options.sorting, max_page=options.max_page) - print_doujinshi(doujinshis) - if options.is_download and doujinshis: - doujinshi_ids = [i['id'] for i in doujinshis] elif options.artist: doujinshis = tag_parser(options.artist, max_page=options.max_page, index=1) - print_doujinshi(doujinshis) - if options.is_download and doujinshis: - doujinshi_ids = [i['id'] for i in doujinshis] elif options.character: doujinshis = tag_parser(options.character, max_page=options.max_page, index=2) - print_doujinshi(doujinshis) - if options.is_download and doujinshis: - doujinshi_ids = [i['id'] for i in doujinshis] elif options.parody: doujinshis = tag_parser(options.parody, max_page=options.max_page, index=3) - print_doujinshi(doujinshis) - if options.is_download and doujinshis: - doujinshi_ids = [i['id'] for i in doujinshis] elif options.group: doujinshis = tag_parser(options.group, max_page=options.max_page, index=4) - print_doujinshi(doujinshis) - if options.is_download and doujinshis: - doujinshi_ids = [i['id'] for i in doujinshis] elif options.language: doujinshis = tag_parser(options.language, max_page=options.max_page, index=5) - print_doujinshi(doujinshis) - if options.is_download and doujinshis: - doujinshi_ids = [i['id'] for i in doujinshis] elif options.keyword: doujinshis = search_parser(options.keyword, sorting=options.sorting, page=options.page) - print_doujinshi(doujinshis) - if options.is_download: - doujinshi_ids = [i['id'] for i in doujinshis] elif not doujinshi_ids: doujinshi_ids = options.id + if options.is_download and doujinshis: + print_doujinshi(doujinshis) + doujinshi_ids = [i['id'] for i in doujinshis] + + if options.is_save_download_states: + # TODO: + pass + if doujinshi_ids: for i, id_ in enumerate(doujinshi_ids): if options.delay: From ca713197cc6e9cafc052868424836aaffbd163fa Mon Sep 17 00:00:00 2001 From: RicterZ Date: Thu, 9 Apr 2020 21:07:20 +0800 Subject: [PATCH 3/5] add sqlite3 db to save download history --- nhentai/cmdline.py | 13 +++++++++++-- nhentai/command.py | 20 ++++++++++++-------- nhentai/constant.py | 1 + nhentai/utils.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 10 deletions(-) diff --git a/nhentai/cmdline.py b/nhentai/cmdline.py index 1a19de6..9decc88 100644 --- a/nhentai/cmdline.py +++ b/nhentai/cmdline.py @@ -10,7 +10,7 @@ except ImportError: import nhentai.constant as constant from nhentai import __version__ -from nhentai.utils import urlparse, generate_html, generate_main_html +from nhentai.utils import urlparse, generate_html, generate_main_html, DB from nhentai.logger import logger try: @@ -101,8 +101,10 @@ def cmd_parser(): # nhentai options parser.add_option('--cookie', type='str', dest='cookie', action='store', help='set cookie of nhentai to bypass Google recaptcha') - parser.add_option('--save-download-states', dest='is_save_download_states', action='store_true', + parser.add_option('--save-download-history', dest='is_save_download_history', action='store_true', default=False, help='save downloaded doujinshis, whose will be skipped if you re-download them') + parser.add_option('--clean-download-history', action='store_true', default=False, dest='clean_download_history', + help='clean download history') try: sys.argv = [unicode(i.decode(sys.stdin.encoding)) for i in sys.argv] @@ -124,6 +126,13 @@ def cmd_parser(): generate_main_html() exit(0) + if args.clean_download_history: + with DB() as db: + db.clean_all() + + logger.info('Download history cleaned.') + exit(0) + if os.path.exists(constant.NHENTAI_COOKIE): with open(constant.NHENTAI_COOKIE, 'r') as f: constant.COOKIE = f.read() diff --git a/nhentai/command.py b/nhentai/command.py index e8ee74e..8af01a6 100644 --- a/nhentai/command.py +++ b/nhentai/command.py @@ -4,15 +4,14 @@ from __future__ import unicode_literals, print_function import signal import platform import time -import multiprocessing from nhentai.cmdline import cmd_parser, banner -from nhentai.parser import doujinshi_parser, search_parser, print_doujinshi, favorites_parser, tag_parser, login +from nhentai.parser import doujinshi_parser, search_parser, print_doujinshi, favorites_parser, tag_parser from nhentai.doujinshi import Doujinshi -from nhentai.downloader import Downloader, init_worker +from nhentai.downloader import Downloader from nhentai.logger import logger from nhentai.constant import BASE_URL -from nhentai.utils import generate_html, generate_cbz, generate_main_html, check_cookie, signal_handler +from nhentai.utils import generate_html, generate_cbz, generate_main_html, check_cookie, signal_handler, DB def main(): @@ -62,13 +61,15 @@ def main(): elif not doujinshi_ids: doujinshi_ids = options.id + print_doujinshi(doujinshis) if options.is_download and doujinshis: - print_doujinshi(doujinshis) doujinshi_ids = [i['id'] for i in doujinshis] - if options.is_save_download_states: - # TODO: - pass + if options.is_save_download_history: + with DB() as db: + data = set(db.get_all()) + + doujinshi_ids = list(set(doujinshi_ids) - data) if doujinshi_ids: for i, id_ in enumerate(doujinshi_ids): @@ -91,6 +92,9 @@ def main(): doujinshi.downloader = downloader doujinshi.download() + if options.is_save_download_history: + with DB() as db: + db.add_one(doujinshi.id) if not options.is_nohtml and not options.is_cbz: generate_html(options.output_dir, doujinshi) diff --git a/nhentai/constant.py b/nhentai/constant.py index 4cead2e..c5c7c53 100644 --- a/nhentai/constant.py +++ b/nhentai/constant.py @@ -35,6 +35,7 @@ IMAGE_URL = '%s://i.%s/galleries' % (u.scheme, u.hostname) NHENTAI_HOME = os.path.join(os.getenv('HOME', tempfile.gettempdir()), '.nhentai') NHENTAI_PROXY = os.path.join(NHENTAI_HOME, 'proxy') NHENTAI_COOKIE = os.path.join(NHENTAI_HOME, 'cookie') +NHENTAI_HISTORY = os.path.join(NHENTAI_HOME, 'history.sqlite3') PROXY = {} diff --git a/nhentai/utils.py b/nhentai/utils.py index 3d7e3e7..f9f5946 100644 --- a/nhentai/utils.py +++ b/nhentai/utils.py @@ -8,6 +8,7 @@ import string import zipfile import shutil import requests +import sqlite3 from nhentai import constant from nhentai.logger import logger @@ -214,3 +215,30 @@ an invalid filename. def signal_handler(signal, frame): logger.error('Ctrl-C signal received. Stopping...') exit(1) + + +class DB(object): + conn = None + cur = None + + def __enter__(self): + self.conn = sqlite3.connect(constant.NHENTAI_HISTORY) + self.cur = self.conn.cursor() + self.cur.execute('CREATE TABLE IF NOT EXISTS download_history (id text)') + self.conn.commit() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.conn.close() + + def clean_all(self): + self.cur.execute('DELETE FROM download_history WHERE 1') + self.conn.commit() + + def add_one(self, data): + self.cur.execute('INSERT INTO download_history VALUES (?)', [data]) + self.conn.commit() + + def get_all(self): + data = self.cur.execute('SELECT id FROM download_history') + return [i[0] for i in data] From 9f747dad7e66396b9f4e621ad9b9fc9d73ca8200 Mon Sep 17 00:00:00 2001 From: RicterZ Date: Thu, 9 Apr 2020 21:12:24 +0800 Subject: [PATCH 4/5] 0.3.8 --- nhentai/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nhentai/__init__.py b/nhentai/__init__.py index f0d7796..772a1a5 100644 --- a/nhentai/__init__.py +++ b/nhentai/__init__.py @@ -1,3 +1,3 @@ -__version__ = '0.3.7' +__version__ = '0.3.8' __author__ = 'RicterZ' __email__ = 'ricterzheng@gmail.com' From 197b5e492329ac6960041ea793341d7eab12b32d Mon Sep 17 00:00:00 2001 From: RicterZ Date: Thu, 9 Apr 2020 22:04:45 +0800 Subject: [PATCH 5/5] update --- nhentai/command.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/nhentai/command.py b/nhentai/command.py index 8af01a6..4afabef 100644 --- a/nhentai/command.py +++ b/nhentai/command.py @@ -27,6 +27,7 @@ def main(): # check your cookie check_cookie() + index = 0 doujinshis = [] doujinshi_ids = [] doujinshi_list = [] @@ -41,19 +42,19 @@ def main(): doujinshis = tag_parser(options.tag, sorting=options.sorting, max_page=options.max_page) elif options.artist: - doujinshis = tag_parser(options.artist, max_page=options.max_page, index=1) + index = 1 elif options.character: - doujinshis = tag_parser(options.character, max_page=options.max_page, index=2) + index = 2 elif options.parody: - doujinshis = tag_parser(options.parody, max_page=options.max_page, index=3) + index = 3 elif options.group: - doujinshis = tag_parser(options.group, max_page=options.max_page, index=4) + index = 4 elif options.language: - doujinshis = tag_parser(options.language, max_page=options.max_page, index=5) + index = 5 elif options.keyword: doujinshis = search_parser(options.keyword, sorting=options.sorting, page=options.page) @@ -61,6 +62,9 @@ def main(): elif not doujinshi_ids: doujinshi_ids = options.id + if index: + doujinshis = tag_parser(options.language, max_page=options.max_page, index=index) + print_doujinshi(doujinshis) if options.is_download and doujinshis: doujinshi_ids = [i['id'] for i in doujinshis]