From 967e0b4ff5cac6a3a166515af36be4d02934e107 Mon Sep 17 00:00:00 2001 From: Ricter Z Date: Thu, 19 Apr 2018 17:21:43 +0800 Subject: [PATCH] fix #18 #19 use nhentai api --- nhentai/constant.py | 4 +-- nhentai/doujinshi.py | 13 +++++++-- nhentai/downloader.py | 7 ++++- nhentai/parser.py | 67 +++++++++++++++---------------------------- 4 files changed, 41 insertions(+), 50 deletions(-) diff --git a/nhentai/constant.py b/nhentai/constant.py index c5e014c..55e2fa4 100644 --- a/nhentai/constant.py +++ b/nhentai/constant.py @@ -5,8 +5,8 @@ from nhentai.utils import urlparse BASE_URL = os.getenv('NHENTAI', 'https://nhentai.net') -DETAIL_URL = '%s/g' % BASE_URL -SEARCH_URL = '%s/search/' % BASE_URL +DETAIL_URL = '%s/api/gallery' % BASE_URL +SEARCH_URL = '%s/api/galleries/search' % BASE_URL LOGIN_URL = '%s/login/' % BASE_URL FAV_URL = '%s/favorites/' % BASE_URL diff --git a/nhentai/doujinshi.py b/nhentai/doujinshi.py index d7de7d3..b6c814d 100644 --- a/nhentai/doujinshi.py +++ b/nhentai/doujinshi.py @@ -8,6 +8,12 @@ from nhentai.logger import logger from nhentai.utils import format_filename +EXT_MAP = { + 'j': 'jpg', + 'p': 'png', +} + + class DoujinshiInfo(dict): def __init__(self, **kwargs): super(DoujinshiInfo, self).__init__(**kwargs) @@ -20,7 +26,7 @@ class DoujinshiInfo(dict): class Doujinshi(object): - def __init__(self, name=None, id=None, img_id=None, ext='jpg', pages=0, **kwargs): + def __init__(self, name=None, id=None, img_id=None, ext='', pages=0, **kwargs): self.name = name self.id = id self.img_id = img_id @@ -50,8 +56,9 @@ class Doujinshi(object): logger.info('Start download doujinshi: %s' % self.name) if self.downloader: download_queue = [] - for i in range(1, self.pages + 1): - download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext)) + for i in range(len(self.ext)): + download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i+1, EXT_MAP[self.ext[i]])) + self.downloader.download(download_queue, format_filename('%s-%s' % (self.id, self.name[:200]))) else: logger.critical('Downloader has not be loaded') diff --git a/nhentai/downloader.py b/nhentai/downloader.py index 1a6e61c..27b2769 100644 --- a/nhentai/downloader.py +++ b/nhentai/downloader.py @@ -36,6 +36,11 @@ class Downloader(Singleton): filename = filename if filename else os.path.basename(urlparse(url).path) base_filename, extension = os.path.splitext(filename) try: + if os.path.exists(os.path.join(folder, base_filename.zfill(3) + extension)): + logger.warning('File: {0} existed, ignore.'.format(os.path.join(folder, base_filename.zfill(3) + + extension))) + return 1, url + with open(os.path.join(folder, base_filename.zfill(3) + extension), "wb") as f: response = request('get', url, stream=True, timeout=self.timeout) if response.status_code != 200: @@ -75,7 +80,7 @@ class Downloader(Singleton): logger.log(15, '{0} download successfully'.format(data)) def download(self, queue, folder=''): - if not isinstance(folder, (text)): + if not isinstance(folder, text): folder = str(folder) if self.path: diff --git a/nhentai/parser.py b/nhentai/parser.py index e01c492..6e9e117 100644 --- a/nhentai/parser.py +++ b/nhentai/parser.py @@ -87,49 +87,29 @@ def doujinshi_parser(id_): logger.log(15, 'Fetching doujinshi information of id {0}'.format(id_)) doujinshi = dict() doujinshi['id'] = id_ - url = '{0}/{1}/'.format(constant.DETAIL_URL, id_) + url = '{0}/{1}'.format(constant.DETAIL_URL, id_) try: - response = request('get', url).content + response = request('get', url).json() except Exception as e: logger.critical(str(e)) exit(1) - html = BeautifulSoup(response, 'html.parser') - doujinshi_info = html.find('div', attrs={'id': 'info'}) - - title = doujinshi_info.find('h1').text - subtitle = doujinshi_info.find('h2') - - doujinshi['name'] = title - doujinshi['subtitle'] = subtitle.text if subtitle else '' - - doujinshi_cover = html.find('div', attrs={'id': 'cover'}) - img_id = re.search('/galleries/([\d]+)/cover\.(jpg|png)$', doujinshi_cover.a.img.attrs['data-src']) - if not img_id: - logger.critical('Tried yo get image id failed') - exit(1) - - doujinshi['img_id'] = img_id.group(1) - doujinshi['ext'] = img_id.group(2) - - pages = 0 - for _ in doujinshi_info.find_all('div', class_=''): - pages = re.search('([\d]+) pages', _.text) - if pages: - pages = pages.group(1) - break - doujinshi['pages'] = int(pages) + doujinshi['name'] = response['title']['english'] + doujinshi['subtitle'] = response['title']['japanese'] + doujinshi['img_id'] = response['media_id'] + doujinshi['ext'] = ''.join(map(lambda s: s['t'], response['images']['pages'])) + doujinshi['pages'] = len(response['images']['pages']) # gain information of the doujinshi - information_fields = doujinshi_info.find_all('div', attrs={'class': 'field-name'}) - needed_fields = ['Characters', 'Artists', 'Language', 'Tags'] - for field in information_fields: - field_name = field.contents[0].strip().strip(':') - if field_name in needed_fields: - data = [sub_field.contents[0].strip() for sub_field in - field.find_all('a', attrs={'class': 'tag'})] - doujinshi[field_name.lower()] = ', '.join(data) + needed_fields = ['character', 'artist', 'language'] + for tag in response['tags']: + tag_type = tag['type'] + if tag_type in needed_fields: + if tag_type not in doujinshi: + doujinshi[tag_type] = tag['name'] + else: + doujinshi[tag_type] += tag['name'] return doujinshi @@ -138,20 +118,19 @@ def search_parser(keyword, page): logger.debug('Searching doujinshis of keyword {0}'.format(keyword)) result = [] try: - response = request('get', url=constant.SEARCH_URL, params={'q': keyword, 'page': page}).content + response = request('get', url=constant.SEARCH_URL, params={'query': keyword, 'page': page}).json() + if 'result' not in response: + raise Exception('No result in response') except requests.ConnectionError as e: logger.critical(e) logger.warn('If you are in China, please configure the proxy to fu*k GFW.') exit(1) - html = BeautifulSoup(response, 'html.parser') - doujinshi_search_result = html.find_all('div', attrs={'class': 'gallery'}) - for doujinshi in doujinshi_search_result: - doujinshi_container = doujinshi.find('div', attrs={'class': 'caption'}) - title = doujinshi_container.text.strip() - title = (title[:85] + '..') if len(title) > 85 else title - id_ = re.search('/g/(\d+)/', doujinshi.a['href']).group(1) - result.append({'id': id_, 'title': title}) + for row in response['result']: + title = row['title']['english'] + title = title[:85] + '..' if len(title) > 85 else title + result.append({'id': row['id'], 'title': title}) + if not result: logger.warn('Not found anything of keyword {}'.format(keyword))