tag page download #40

nhentai/command.py

@@ -5,7 +5,7 @@ import signal
 import platform
 
 from nhentai.cmdline import cmd_parser, banner
-from nhentai.parser import doujinshi_parser, search_parser, print_doujinshi, login_parser, tag_guessing, tag_parser
+from nhentai.parser import doujinshi_parser, search_parser, print_doujinshi, login_parser, tag_parser
 from nhentai.doujinshi import Doujinshi
 from nhentai.downloader import Downloader
 from nhentai.logger import logger
@@ -28,12 +28,10 @@ def main():
             doujinshi_list.append(Doujinshi(**doujinshi_info))
 
     if options.tag:
-        tag_id = tag_guessing(options.tag)
+        doujinshis = tag_parser(options.tag, max_page=options.max_page)
-        if tag_id:
+        print_doujinshi(doujinshis)
-            doujinshis = tag_parser(tag_id, max_page=options.max_page)
+        if options.is_download:
-            print_doujinshi(doujinshis)
+            doujinshi_ids = map(lambda d: d['id'], doujinshis)
-            if options.is_download:
-                doujinshi_ids = map(lambda d: d['id'], doujinshis)
 
     if options.keyword:
         doujinshis = search_parser(options.keyword, options.page)

nhentai/constant.py

@@ -5,8 +5,12 @@ from nhentai.utils import urlparse
 
 BASE_URL = os.getenv('NHENTAI', 'https://nhentai.net')
 
-DETAIL_URL = '%s/api/gallery' % BASE_URL
+__api_suspended_DETAIL_URL = '%s/api/gallery' % BASE_URL
-SEARCH_URL = '%s/api/galleries/search' % BASE_URL
+__api_suspended_SEARCH_URL = '%s/api/galleries/search' % BASE_URL
+
+DETAIL_URL = '%s/g' % BASE_URL
+SEARCH_URL = '%s/search/' % BASE_URL
+
 TAG_URL = '%s/tag' % BASE_URL
 TAG_API_URL = '%s/api/galleries/tagged' % BASE_URL
 LOGIN_URL = '%s/login/' % BASE_URL

nhentai/doujinshi.py

@@ -58,9 +58,17 @@ class Doujinshi(object):
         logger.info('Starting to download doujinshi: %s' % self.name)
         if self.downloader:
             download_queue = []
+
+            for i in range(1, self.pages + 1):
+                download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext))
+
+            self.downloader.download(download_queue, self.filename)
+
+            '''
             for i in range(len(self.ext)):
                 download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i+1, EXT_MAP[self.ext[i]]))
-                self.downloader.download(download_queue, self.filename)
+            '''
+
         else:
             logger.critical('Downloader has not been loaded')
 

nhentai/parser.py

@@ -74,7 +74,7 @@ def login_parser(username, password):
 
     thread_pool = threadpool.ThreadPool(5)
 
-    for page in range(1, pages+1):
+    for page in range(1, pages + 1):
         try:
             logger.info('Getting doujinshi ids of page %d' % page)
             resp = s.get(constant.FAV_URL + '?page=%d' % page).text
@@ -92,18 +92,97 @@ def doujinshi_parser(id_):
     if not isinstance(id_, (int,)) and (isinstance(id_, (str,)) and not id_.isdigit()):
         raise Exception('Doujinshi id({0}) is not valid'.format(id_))
 
+    id_ = int(id_)
+    logger.log(15, 'Fetching doujinshi information of id {0}'.format(id_))
+    doujinshi = dict()
+    doujinshi['id'] = id_
+    url = '{0}/{1}/'.format(constant.DETAIL_URL, id_)
+
+    try:
+        response = request('get', url).content
+    except Exception as e:
+        logger.critical(str(e))
+        raise SystemExit
+
+    html = BeautifulSoup(response, 'html.parser')
+    doujinshi_info = html.find('div', attrs={'id': 'info'})
+
+    title = doujinshi_info.find('h1').text
+    subtitle = doujinshi_info.find('h2')
+
+    doujinshi['name'] = title
+    doujinshi['subtitle'] = subtitle.text if subtitle else ''
+
+    doujinshi_cover = html.find('div', attrs={'id': 'cover'})
+    img_id = re.search('/galleries/([\d]+)/cover\.(jpg|png)$', doujinshi_cover.a.img.attrs['data-src'])
+    if not img_id:
+        logger.critical('Tried yo get image id failed')
+        exit(1)
+
+    doujinshi['img_id'] = img_id.group(1)
+    doujinshi['ext'] = img_id.group(2)
+
+    pages = 0
+    for _ in doujinshi_info.find_all('div', class_=''):
+        pages = re.search('([\d]+) pages', _.text)
+        if pages:
+            pages = pages.group(1)
+            break
+    doujinshi['pages'] = int(pages)
+
+    # gain information of the doujinshi
+    information_fields = doujinshi_info.find_all('div', attrs={'class': 'field-name'})
+    needed_fields = ['Characters', 'Artists', 'Language', 'Tags']
+    for field in information_fields:
+        field_name = field.contents[0].strip().strip(':')
+        if field_name in needed_fields:
+            data = [sub_field.contents[0].strip() for sub_field in
+                    field.find_all('a', attrs={'class': 'tag'})]
+            doujinshi[field_name.lower()] = ', '.join(data)
+
+    return doujinshi
+
+
+def search_parser(keyword, page):
+    logger.debug('Searching doujinshis of keyword {0}'.format(keyword))
+    result = []
+    try:
+        response = request('get', url=constant.SEARCH_URL, params={'q': keyword, 'page': page}).content
+    except requests.ConnectionError as e:
+        logger.critical(e)
+        logger.warn('If you are in China, please configure the proxy to fu*k GFW.')
+        raise SystemExit
+
+    html = BeautifulSoup(response, 'html.parser')
+    doujinshi_search_result = html.find_all('div', attrs={'class': 'gallery'})
+    for doujinshi in doujinshi_search_result:
+        doujinshi_container = doujinshi.find('div', attrs={'class': 'caption'})
+        title = doujinshi_container.text.strip()
+        title = title if len(title) < 85 else title[:82] + '...'
+        id_ = re.search('/g/(\d+)/', doujinshi.a['href']).group(1)
+        result.append({'id': id_, 'title': title})
+    if not result:
+        logger.warn('Not found anything of keyword {}'.format(keyword))
+
+    return result
+
+
+def __api_suspended_doujinshi_parser(id_):
+    if not isinstance(id_, (int,)) and (isinstance(id_, (str,)) and not id_.isdigit()):
+        raise Exception('Doujinshi id({0}) is not valid'.format(id_))
+
     id_ = int(id_)
     logger.log(15, 'Fetching information of doujinshi id {0}'.format(id_))
     doujinshi = dict()
     doujinshi['id'] = id_
     url = '{0}/{1}'.format(constant.DETAIL_URL, id_)
-    i=0
+    i = 0
-    while i<5:
+    while 5 > i:
         try:
             response = request('get', url).json()
         except Exception as e:
-            i+=1
+            i += 1
-            if not i<5:
+            if not i < 5:
                 logger.critical(str(e))
                 exit(1)
             continue
@@ -135,16 +214,16 @@ def doujinshi_parser(id_):
     return doujinshi
 
 
-def search_parser(keyword, page):
+def __api_suspended_search_parser(keyword, page):
     logger.debug('Searching doujinshis using keywords {0}'.format(keyword))
     result = []
-    i=0
+    i = 0
-    while i<5:
+    while i < 5:
         try:
             response = request('get', url=constant.SEARCH_URL, params={'query': keyword, 'page': page}).json()
         except Exception as e:
-            i+=1
+            i += 1
-            if not i<5:
+            if not i < 5:
                 logger.critical(str(e))
                 logger.warn('If you are in China, please configure the proxy to fu*k GFW.')
                 exit(1)
@@ -174,37 +253,18 @@ def print_doujinshi(doujinshi_list):
                 tabulate(tabular_data=doujinshi_list, headers=headers, tablefmt='rst'))
 
 
-def tag_parser(tag_id, max_page=1):
+def __api_suspended_tag_parser(tag_id, max_page=1):
     logger.info('Searching for doujinshi with tag id {0}'.format(tag_id))
     result = []
-    i=0
+    response = request('get', url=constant.TAG_API_URL, params={'sort': 'popular', 'tag_id': tag_id}).json()
-    while i<5:
-        try:
-            response = request('get', url=constant.TAG_API_URL, params={'sort': 'popular', 'tag_id': tag_id}).json()
-        except Exception as e:
-            i+=1
-            if not i<5:
-                logger.critical(str(e))
-                exit(1)
-            continue
-        break
     page = max_page if max_page <= response['num_pages'] else int(response['num_pages'])
 
-    for i in range(1, page+1):
+    for i in range(1, page + 1):
         logger.info('Getting page {} ...'.format(i))
 
         if page != 1:
-            i=0
+            response = request('get', url=constant.TAG_API_URL,
-            while i<5:
+                               params={'sort': 'popular', 'tag_id': tag_id}).json()
-                try:
-                    response = request('get', url=constant.TAG_API_URL, params={'sort': 'popular', 'tag_id': tag_id}).json()
-                except Exception as e:
-                    i+=1
-                    if not i<5:
-                        logger.critical(str(e))
-                        exit(1)
-                    continue
-                break
     for row in response['result']:
         title = row['title']['english']
         title = title[:85] + '..' if len(title) > 85 else title
@@ -216,42 +276,31 @@ def tag_parser(tag_id, max_page=1):
     return result
 
 
-def tag_guessing(tag_name):
+def tag_parser(tag_name, max_page=1):
+    result = []
     tag_name = tag_name.lower()
     tag_name = tag_name.replace(' ', '-')
-    logger.info('Trying to get tag_id of tag \'{0}\''.format(tag_name))
-    i=0
-    while i<5:
-        try:
-            response = request('get', url='%s/%s' % (constant.TAG_URL, tag_name)).content
-        except Exception as e:
-            i+=1
-            if not i<5:
-                logger.critical(str(e))
-                exit(1)
-            continue
-        break
 
-    html = BeautifulSoup(response, 'html.parser')
+    for p in range(1, max_page + 1):
-    first_item = html.find('div', attrs={'class': 'gallery'})
+        logger.debug('Fetching page {0} for doujinshi with tag \'{1}\''.format(p, tag_name))
-    if not first_item:
+        response = request('get', url='%s/%s?page=%d' % (constant.TAG_URL, tag_name, p)).content
-        logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name))
-        return
 
-    doujinshi_id = re.findall('(\d+)', first_item.a.attrs['href'])
+        html = BeautifulSoup(response, 'html.parser')
-    if not doujinshi_id:
+        doujinshi_items = html.find_all('div', attrs={'class': 'gallery'})
-        logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name))
+        if not doujinshi_items:
-        return
+            logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name))
+            return
 
-    ret = doujinshi_parser(doujinshi_id[0])
+        for i in doujinshi_items:
-    if 'tag' in ret and tag_name in ret['tag']:
+            doujinshi_id = i.a.attrs['href'].strip('/g')
-        tag_id = ret['tag'][tag_name]
+            doujinshi_title = i.a.text.strip()
-        logger.info('Tag id of tag \'{0}\' is {1}'.format(tag_name, tag_id))
+            doujinshi_title = doujinshi_title if len(doujinshi_title) < 85 else doujinshi_title[:82] + '...'
-    else:
+            result.append({'title': doujinshi_title, 'id': doujinshi_id})
-        logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name))
-        return
 
-    return tag_id
+    if not result:
+        logger.warn('No results for tag \'{}\''.format(tag_name))
+
+    return result
 
 
 if __name__ == '__main__':

nhentai/utils.py

@@ -121,4 +121,7 @@ an invalid filename.
     filename = filename.replace(' ', '_')  # I don't like spaces in filenames.
     if len(filename) > 100:
         filename = filename[:100] + '...]'
+
+    # Remove [] from filename
+    filename = filename.replace('[]', '')
     return filename