From be05b9c0eb54a8982bd443a5854af9db5046d364 Mon Sep 17 00:00:00 2001 From: Nontre12 <62849884+Nontre12@users.noreply.github.com> Date: Sat, 10 Oct 2020 18:29:00 +0200 Subject: [PATCH 1/5] Added --page-all option to download all search results --- nhentai/cmdline.py | 2 ++ nhentai/command.py | 9 ++++++++- nhentai/parser.py | 34 +++++++++++++++++++++++++++++++++- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/nhentai/cmdline.py b/nhentai/cmdline.py index 1e13215..76d2168 100644 --- a/nhentai/cmdline.py +++ b/nhentai/cmdline.py @@ -54,6 +54,8 @@ def cmd_parser(): help='list or download your favorites.') # page options + parser.add_option('--page-all', dest='page_all', action='store_true', default=False, + help='all search results') parser.add_option('--page', type='int', dest='page', action='store', default=1, help='page number of search results') parser.add_option('--page-range', type='string', dest='page_range', action='store', diff --git a/nhentai/command.py b/nhentai/command.py index 621fef2..0f3a776 100644 --- a/nhentai/command.py +++ b/nhentai/command.py @@ -6,7 +6,7 @@ import platform import time from nhentai.cmdline import cmd_parser, banner -from nhentai.parser import doujinshi_parser, search_parser, print_doujinshi, favorites_parser +from nhentai.parser import doujinshi_parser, search_parser, search_parser_all, print_doujinshi, favorites_parser from nhentai.doujinshi import Doujinshi from nhentai.downloader import Downloader from nhentai.logger import logger @@ -37,6 +37,13 @@ def main(): doujinshis = favorites_parser(options.page_range) + elif options.keyword and options.page_all: + from nhentai.constant import LANGUAGE + if LANGUAGE: + logger.info('Using default language: {0}'.format(LANGUAGE)) + options.keyword += ', language:{}'.format(LANGUAGE) + doujinshis = search_parser_all(options.keyword) + elif options.keyword: from nhentai.constant import LANGUAGE if LANGUAGE: diff --git a/nhentai/parser.py b/nhentai/parser.py index bece331..ae739da 100644 --- a/nhentai/parser.py +++ b/nhentai/parser.py @@ -215,7 +215,7 @@ def print_doujinshi(doujinshi_list): return doujinshi_list = [(i['id'], i['title']) for i in doujinshi_list] headers = ['id', 'doujinshi'] - logger.info('Search Result\n' + + logger.info('Search Result || Found %i doujinshis \n' % doujinshi_list.__len__() + tabulate(tabular_data=doujinshi_list, headers=headers, tablefmt='rst')) @@ -247,6 +247,38 @@ def search_parser(keyword, sorting, page): return result +def search_parser_all(keyword): + logger.debug('Searching doujinshis using keywords {0}'.format(keyword)) + + result = [] + + url = request('get', url=constant.SEARCH_URL, params={'query': keyword}).url + init_response = request('get', url.replace('%2B', '+')).json() + + for page in range(init_response['num_pages']): + try: + url = request('get', url=constant.SEARCH_URL, params={'query': keyword, 'page': page+1}).url + response = request('get', url.replace('%2B', '+')).json() + + print('Obtained %d / %d pages.' % (page+1, init_response['num_pages']), end='\r') + + except Exception as e: + logger.critical(str(e)) + + if 'result' not in response: + raise Exception('No result in response') + + for row in response['result']: + title = row['title']['english'] + title = title[:85] + '..' if len(title) > 85 else title + result.append({'id': row['id'], 'title': title}) + + if not result: + logger.warn('No results for keywords {}'.format(keyword)) + + return result + + def __api_suspended_doujinshi_parser(id_): if not isinstance(id_, (int,)) and (isinstance(id_, (str,)) and not id_.isdigit()): raise Exception('Doujinshi id({0}) is not valid'.format(id_)) From f157ac32469231a01823fd14b17870fb4e6b44e8 Mon Sep 17 00:00:00 2001 From: RicterZ Date: Sun, 11 Oct 2020 01:09:13 +0800 Subject: [PATCH 2/5] merge to functions --- nhentai/command.py | 15 ++++----------- nhentai/parser.py | 44 ++++++++++---------------------------------- 2 files changed, 14 insertions(+), 45 deletions(-) diff --git a/nhentai/command.py b/nhentai/command.py index f22e946..963cd84 100644 --- a/nhentai/command.py +++ b/nhentai/command.py @@ -6,11 +6,11 @@ import platform import time from nhentai.cmdline import cmd_parser, banner -from nhentai.parser import doujinshi_parser, search_parser, search_parser_all, print_doujinshi, favorites_parser +from nhentai.parser import doujinshi_parser, search_parser, print_doujinshi, favorites_parser from nhentai.doujinshi import Doujinshi from nhentai.downloader import Downloader from nhentai.logger import logger -from nhentai.constant import BASE_URL +from nhentai.constant import BASE_URL, LANGUAGE from nhentai.utils import generate_html, generate_cbz, generate_main_html, generate_pdf, \ paging, check_cookie, signal_handler, DB @@ -40,19 +40,12 @@ def main(): doujinshis = favorites_parser(page=page_list) - elif options.keyword and options.page_all: - from nhentai.constant import LANGUAGE - if LANGUAGE: - logger.info('Using default language: {0}'.format(LANGUAGE)) - options.keyword += ', language:{}'.format(LANGUAGE) - doujinshis = search_parser_all(options.keyword) - elif options.keyword: - from nhentai.constant import LANGUAGE if LANGUAGE: logger.info('Using default language: {0}'.format(LANGUAGE)) options.keyword += ', language:{}'.format(LANGUAGE) - doujinshis = search_parser(options.keyword, sorting=options.sorting, page=page_list) + doujinshis = search_parser(options.keyword, sorting=options.sorting, page=page_list, + is_page_all=options.page_all) elif not doujinshi_ids: doujinshi_ids = options.id diff --git a/nhentai/parser.py b/nhentai/parser.py index bd60e7a..c61f8f7 100644 --- a/nhentai/parser.py +++ b/nhentai/parser.py @@ -193,15 +193,23 @@ def print_doujinshi(doujinshi_list): tabulate(tabular_data=doujinshi_list, headers=headers, tablefmt='rst')) -def search_parser(keyword, sorting, page): +def search_parser(keyword, sorting, page, is_page_all=False): # keyword = '+'.join([i.strip().replace(' ', '-').lower() for i in keyword.split(',')]) result = [] if not page: page = [1] + if is_page_all: + url = request('get', url=constant.SEARCH_URL, params={'query': keyword}).url + init_response = request('get', url.replace('%2B', '+')).json() + page = range(1, init_response['num_pages']+1) + for p in page: i = 0 - logger.info('Searching doujinshis using keywords "{0}" on page {1}'.format(keyword, p)) + if is_page_all: + total = '/{0}'.format(page[-1]) + + logger.info('Searching doujinshis using keywords "{0}" on page {1}{2}'.format(keyword, p, total)) while i < 3: try: url = request('get', url=constant.SEARCH_URL, params={'query': keyword, @@ -227,38 +235,6 @@ def search_parser(keyword, sorting, page): return result -def search_parser_all(keyword): - logger.debug('Searching doujinshis using keywords {0}'.format(keyword)) - - result = [] - - url = request('get', url=constant.SEARCH_URL, params={'query': keyword}).url - init_response = request('get', url.replace('%2B', '+')).json() - - for page in range(init_response['num_pages']): - try: - url = request('get', url=constant.SEARCH_URL, params={'query': keyword, 'page': page+1}).url - response = request('get', url.replace('%2B', '+')).json() - - print('Obtained %d / %d pages.' % (page+1, init_response['num_pages']), end='\r') - - except Exception as e: - logger.critical(str(e)) - - if 'result' not in response: - raise Exception('No result in response') - - for row in response['result']: - title = row['title']['english'] - title = title[:85] + '..' if len(title) > 85 else title - result.append({'id': row['id'], 'title': title}) - - if not result: - logger.warn('No results for keywords {}'.format(keyword)) - - return result - - def __api_suspended_doujinshi_parser(id_): if not isinstance(id_, (int,)) and (isinstance(id_, (str,)) and not id_.isdigit()): raise Exception('Doujinshi id({0}) is not valid'.format(id_)) From b2fae226f9f80b5e25c3150fbc5d6ce7a68c94ce Mon Sep 17 00:00:00 2001 From: RicterZ Date: Sun, 11 Oct 2020 01:38:08 +0800 Subject: [PATCH 3/5] use config.json --- nhentai/cmdline.py | 106 +++++++++++++++++--------------------------- nhentai/command.py | 18 ++++---- nhentai/constant.py | 12 ++--- nhentai/parser.py | 4 +- nhentai/utils.py | 4 +- 5 files changed, 60 insertions(+), 84 deletions(-) diff --git a/nhentai/cmdline.py b/nhentai/cmdline.py index 39787cd..f1e8184 100644 --- a/nhentai/cmdline.py +++ b/nhentai/cmdline.py @@ -2,7 +2,9 @@ from __future__ import print_function import os import sys +import json from optparse import OptionParser + try: from itertools import ifilter as filter except ImportError: @@ -35,7 +37,29 @@ def banner(): ''' % __version__) +def load_config(): + if not os.path.exists(constant.NHENTAI_CONFIG_FILE): + return + + try: + with open(constant.NHENTAI_CONFIG_FILE, 'r') as f: + constant.CONFIG = json.load(f) + except json.JSONDecodeError: + logger.error('Failed to load config file.') + write_config() + + +def write_config(): + if not os.path.exists(constant.NHENTAI_HOME): + os.mkdir(constant.NHENTAI_HOME) + + with open(constant.NHENTAI_CONFIG_FILE, 'w') as f: + f.write(json.dumps(constant.CONFIG)) + + def cmd_parser(): + load_config() + parser = OptionParser('\n nhentai --search [keyword] --download' '\n NHENTAI=http://h.loli.club nhentai --id [ID ...]' '\n nhentai --file [filename]' @@ -128,83 +152,35 @@ def cmd_parser(): logger.info('Download history cleaned.') exit(0) - if os.path.exists(constant.NHENTAI_COOKIE): - with open(constant.NHENTAI_COOKIE, 'r') as f: - constant.COOKIE = f.read() - + # --- set config --- if args.cookie: - try: - if not os.path.exists(constant.NHENTAI_HOME): - os.mkdir(constant.NHENTAI_HOME) - - with open(constant.NHENTAI_COOKIE, 'w') as f: - f.write(args.cookie) - except Exception as e: - logger.error('Cannot create NHENTAI_HOME: {}'.format(str(e))) - exit(1) - + constant.CONFIG['cookie'] = args.cookie logger.info('Cookie saved.') + write_config() exit(0) - if os.path.exists(constant.NHENTAI_LANGUAGE) and not args.language: - with open(constant.NHENTAI_LANGUAGE, 'r') as f: - constant.LANGUAGE = f.read() - args.language = f.read() - if args.language: - try: - if not os.path.exists(constant.NHENTAI_HOME): - os.mkdir(constant.NHENTAI_HOME) - - with open(constant.NHENTAI_LANGUAGE, 'w') as f: - f.write(args.language) - except Exception as e: - logger.error('Cannot create NHENTAI_HOME: {}'.format(str(e))) - exit(1) - - logger.info('Default language now is {}.'.format(args.language)) + constant.CONFIG['language'] = args.language + logger.info('LANGUAGE now set to \'{0}\''.format(args.language)) + write_config() exit(0) - - if args.clean_language: - try: - if not os.path.exists(constant.NHENTAI_HOME): - os.mkdir(constant.NHENTAI_HOME) - - with open(constant.NHENTAI_LANGUAGE, 'w') as f: - f.close() - except Exception as e: - logger.error('Cannot create NHENTAI_HOME: {}'.format(str(e))) - exit(1) - - logger.info('Language now is DEFAULT') - exit(0) - - if os.path.exists(constant.NHENTAI_PROXY): - with open(constant.NHENTAI_PROXY, 'r') as f: - link = f.read() - constant.PROXY = {'http': link, 'https': link} + # TODO: search without language if args.proxy: - try: - if not os.path.exists(constant.NHENTAI_HOME): - os.mkdir(constant.NHENTAI_HOME) - - proxy_url = urlparse(args.proxy) - if proxy_url.scheme not in ('http', 'https'): - logger.error('Invalid protocol \'{0}\' of proxy, ignored'.format(proxy_url.scheme)) - else: - with open(constant.NHENTAI_PROXY, 'w') as f: - f.write(args.proxy) - - except Exception as e: - logger.error('Cannot create NHENTAI_HOME: {}'.format(str(e))) - exit(1) - + proxy_url = urlparse(args.proxy) + if proxy_url.scheme not in ('http', 'https'): + logger.error('Invalid protocol \'{0}\' of proxy, ignored'.format(proxy_url.scheme)) + constant.CONFIG['proxy'] = { + 'http': args.proxy, + 'https': args.proxy, + } logger.info('Proxy \'{0}\' saved.'.format(args.proxy)) + write_config() exit(0) + # --- end set config --- if args.favorites: - if not constant.COOKIE: + if not constant.CONFIG['cookie']: logger.warning('Cookie has not been set, please use `nhentai --cookie \'COOKIE\'` to set it.') exit(1) diff --git a/nhentai/command.py b/nhentai/command.py index 963cd84..9b0a99e 100644 --- a/nhentai/command.py +++ b/nhentai/command.py @@ -1,16 +1,19 @@ #!/usr/bin/env python2.7 # coding: utf-8 from __future__ import unicode_literals, print_function +import json +import os import signal import platform import time +from nhentai import constant from nhentai.cmdline import cmd_parser, banner from nhentai.parser import doujinshi_parser, search_parser, print_doujinshi, favorites_parser from nhentai.doujinshi import Doujinshi from nhentai.downloader import Downloader from nhentai.logger import logger -from nhentai.constant import BASE_URL, LANGUAGE +from nhentai.constant import NHENTAI_CONFIG_FILE, BASE_URL from nhentai.utils import generate_html, generate_cbz, generate_main_html, generate_pdf, \ paging, check_cookie, signal_handler, DB @@ -20,10 +23,9 @@ def main(): options = cmd_parser() logger.info('Using mirror: {0}'.format(BASE_URL)) - from nhentai.constant import PROXY - # constant.PROXY will be changed after cmd_parser() - if PROXY: - logger.info('Using proxy: {0}'.format(PROXY)) + # CONFIG['proxy'] will be changed after cmd_parser() + if constant.CONFIG['proxy']: + logger.info('Using proxy: {0}'.format(constant.CONFIG['proxy'])) # check your cookie check_cookie() @@ -41,9 +43,9 @@ def main(): doujinshis = favorites_parser(page=page_list) elif options.keyword: - if LANGUAGE: - logger.info('Using default language: {0}'.format(LANGUAGE)) - options.keyword += ', language:{}'.format(LANGUAGE) + if constant.CONFIG['language']: + logger.info('Using default language: {0}'.format(constant.CONFIG['language'])) + options.keyword += ' language:{}'.format(constant.CONFIG['language']) doujinshis = search_parser(options.keyword, sorting=options.sorting, page=page_list, is_page_all=options.page_all) diff --git a/nhentai/constant.py b/nhentai/constant.py index c6b40c4..8ec89f7 100644 --- a/nhentai/constant.py +++ b/nhentai/constant.py @@ -26,12 +26,12 @@ u = urlparse(BASE_URL) IMAGE_URL = '%s://i.%s/galleries' % (u.scheme, u.hostname) NHENTAI_HOME = os.path.join(os.getenv('HOME', tempfile.gettempdir()), '.nhentai') -NHENTAI_PROXY = os.path.join(NHENTAI_HOME, 'proxy') -NHENTAI_COOKIE = os.path.join(NHENTAI_HOME, 'cookie') -NHENTAI_LANGUAGE = os.path.join(NHENTAI_HOME, 'language') NHENTAI_HISTORY = os.path.join(NHENTAI_HOME, 'history.sqlite3') +NHENTAI_CONFIG_FILE = os.path.join(NHENTAI_HOME, 'config.json') -PROXY = {} +CONFIG = { + 'proxy': {}, + 'cookie': '', + 'language': '', +} -COOKIE = '' -LANGUAGE = '' diff --git a/nhentai/parser.py b/nhentai/parser.py index c61f8f7..484c0f6 100644 --- a/nhentai/parser.py +++ b/nhentai/parser.py @@ -1,7 +1,6 @@ # coding: utf-8 from __future__ import unicode_literals, print_function -import sys import os import re import time @@ -204,10 +203,9 @@ def search_parser(keyword, sorting, page, is_page_all=False): init_response = request('get', url.replace('%2B', '+')).json() page = range(1, init_response['num_pages']+1) + total = '/{0}'.format(page[-1]) if is_page_all else '' for p in page: i = 0 - if is_page_all: - total = '/{0}'.format(page[-1]) logger.info('Searching doujinshis using keywords "{0}" on page {1}{2}'.format(keyword, p, total)) while i < 3: diff --git a/nhentai/utils.py b/nhentai/utils.py index 2a0cb5c..fb73559 100644 --- a/nhentai/utils.py +++ b/nhentai/utils.py @@ -20,9 +20,9 @@ def request(method, url, **kwargs): session.headers.update({ 'Referer': constant.LOGIN_URL, 'User-Agent': 'nhentai command line client (https://github.com/RicterZ/nhentai)', - 'Cookie': constant.COOKIE + 'Cookie': constant.CONFIG['cookie'] }) - return getattr(session, method)(url, proxies=constant.PROXY, verify=False, **kwargs) + return getattr(session, method)(url, proxies=constant.CONFIG['proxy'], verify=False, **kwargs) def check_cookie(): From 641f8e4c51200544103301d3ddd2e85859c7ee2f Mon Sep 17 00:00:00 2001 From: RicterZ Date: Sun, 11 Oct 2020 01:42:02 +0800 Subject: [PATCH 4/5] 0.4.4 --- nhentai/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nhentai/__init__.py b/nhentai/__init__.py index 1aa66db..93e1977 100644 --- a/nhentai/__init__.py +++ b/nhentai/__init__.py @@ -1,3 +1,3 @@ -__version__ = '0.4.3' +__version__ = '0.4.4' __author__ = 'RicterZ' __email__ = 'ricterzheng@gmail.com' From f04359e4861d8b261cac10765656587f28c848c9 Mon Sep 17 00:00:00 2001 From: RicterZ Date: Sun, 11 Oct 2020 01:57:37 +0800 Subject: [PATCH 5/5] 0.4.5 --- nhentai/__init__.py | 2 +- nhentai/cmdline.py | 28 +++++++++++++++------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/nhentai/__init__.py b/nhentai/__init__.py index 93e1977..44f202e 100644 --- a/nhentai/__init__.py +++ b/nhentai/__init__.py @@ -1,3 +1,3 @@ -__version__ = '0.4.4' +__version__ = '0.4.5' __author__ = 'RicterZ' __email__ = 'ricterzheng@gmail.com' diff --git a/nhentai/cmdline.py b/nhentai/cmdline.py index f1e8184..3347ab0 100644 --- a/nhentai/cmdline.py +++ b/nhentai/cmdline.py @@ -95,7 +95,7 @@ def cmd_parser(): help='timeout for downloading doujinshi') parser.add_option('--delay', '-d', type='int', dest='delay', action='store', default=0, help='slow down between downloading every doujinshi') - parser.add_option('--proxy', '-p', type='string', dest='proxy', action='store', default='', + parser.add_option('--proxy', type='string', dest='proxy', action='store', default='', help='store a proxy, for example: -p \'http://127.0.0.1:1080\'') parser.add_option('--file', '-f', type='string', dest='file', action='store', help='read gallery IDs from file.') parser.add_option('--format', type='string', dest='name_format', action='store', @@ -153,30 +153,32 @@ def cmd_parser(): exit(0) # --- set config --- - if args.cookie: + if args.cookie is not None: constant.CONFIG['cookie'] = args.cookie logger.info('Cookie saved.') write_config() exit(0) - if args.language: + if args.language is not None: constant.CONFIG['language'] = args.language - logger.info('LANGUAGE now set to \'{0}\''.format(args.language)) + logger.info('Default language now set to \'{0}\''.format(args.language)) write_config() exit(0) # TODO: search without language - if args.proxy: + if args.proxy is not None: proxy_url = urlparse(args.proxy) - if proxy_url.scheme not in ('http', 'https'): + if not args.proxy == '' and proxy_url.scheme not in ('http', 'https'): logger.error('Invalid protocol \'{0}\' of proxy, ignored'.format(proxy_url.scheme)) - constant.CONFIG['proxy'] = { - 'http': args.proxy, - 'https': args.proxy, - } - logger.info('Proxy \'{0}\' saved.'.format(args.proxy)) - write_config() - exit(0) + exit(0) + else: + constant.CONFIG['proxy'] = { + 'http': args.proxy, + 'https': args.proxy, + } + logger.info('Proxy now set to \'{0}\'.'.format(args.proxy)) + write_config() + exit(0) # --- end set config --- if args.favorites: