tag page download #40

.travis.yml

@@ -14,6 +14,6 @@ install:
 script:
     - NHENTAI=https://nhentai.net nhentai --search umaru
     - NHENTAI=https://nhentai.net nhentai --id=152503,146134 -t 10 --output=/tmp/
-    - NHENTAI=https://nhentai.net nhentai -l nhentai_test:nhentai --output=/tmp/
+    - NHENTAI=https://nhentai.net nhentai -l nhentai_test:nhentai --download --output=/tmp/
     - NHENTAI=https://nhentai.net nhentai --tag lolicon
-    - NHENTAI=https://nhentai.net nhentai --id 92066 --output=/tmp/ --cbz
+    - NHENTAI=https://nhentai.net nhentai --id 92066 --output=/tmp/ --cbz

nhentai/__init__.py

@@ -1,3 +1,3 @@
-__version__ = '0.2.18'
+__version__ = '0.2.19'
 __author__ = 'RicterZ'
 __email__ = 'ricterzheng@gmail.com'

nhentai/command.py

@@ -5,7 +5,7 @@ import signal
 import platform
 
 from nhentai.cmdline import cmd_parser, banner
-from nhentai.parser import doujinshi_parser, search_parser, print_doujinshi, login_parser, tag_guessing, tag_parser
+from nhentai.parser import doujinshi_parser, search_parser, print_doujinshi, login_parser, tag_parser
 from nhentai.doujinshi import Doujinshi
 from nhentai.downloader import Downloader
 from nhentai.logger import logger
@@ -28,12 +28,10 @@ def main():
             doujinshi_list.append(Doujinshi(**doujinshi_info))
 
     if options.tag:
-        tag_id = tag_guessing(options.tag)
-        if tag_id:
-            doujinshis = tag_parser(tag_id, max_page=options.max_page)
-            print_doujinshi(doujinshis)
-            if options.is_download:
-                doujinshi_ids = map(lambda d: d['id'], doujinshis)
+        doujinshis = tag_parser(options.tag, max_page=options.max_page)
+        print_doujinshi(doujinshis)
+        if options.is_download:
+            doujinshi_ids = map(lambda d: d['id'], doujinshis)
 
     if options.keyword:
         doujinshis = search_parser(options.keyword, options.page)

nhentai/constant.py

@@ -5,8 +5,12 @@ from nhentai.utils import urlparse
 
 BASE_URL = os.getenv('NHENTAI', 'https://nhentai.net')
 
-DETAIL_URL = '%s/api/gallery' % BASE_URL
-SEARCH_URL = '%s/api/galleries/search' % BASE_URL
+__api_suspended_DETAIL_URL = '%s/api/gallery' % BASE_URL
+__api_suspended_SEARCH_URL = '%s/api/galleries/search' % BASE_URL
+
+DETAIL_URL = '%s/g' % BASE_URL
+SEARCH_URL = '%s/search/' % BASE_URL
+
 TAG_URL = '%s/tag' % BASE_URL
 TAG_API_URL = '%s/api/galleries/tagged' % BASE_URL
 LOGIN_URL = '%s/login/' % BASE_URL

nhentai/doujinshi.py

@@ -36,6 +36,7 @@ class Doujinshi(object):
         self.downloader = None
         self.url = '%s/%d' % (DETAIL_URL, self.id)
         self.info = DoujinshiInfo(**kwargs)
+        self.filename = format_filename('[%s][%s][%s]' % (self.id, self.info.artist, self.name))
 
     def __repr__(self):
         return '<Doujinshi: {0}>'.format(self.name)
@@ -44,10 +45,10 @@ class Doujinshi(object):
         table = [
             ["Doujinshi", self.name],
             ["Subtitle", self.info.subtitle],
-            ["Characters", self.info.characters],
-            ["Authors", self.info.artists],
+            ["Characters", self.info.character],
+            ["Authors", self.info.artist],
             ["Language", self.info.language],
-            ["Tags", self.info.tags],
+            ["Tags", ', '.join(self.info.tag.keys())],
             ["URL", self.url],
             ["Pages", self.pages],
         ]
@@ -57,10 +58,17 @@ class Doujinshi(object):
         logger.info('Starting to download doujinshi: %s' % self.name)
         if self.downloader:
             download_queue = []
+
+            for i in range(1, self.pages + 1):
+                download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext))
+
+            self.downloader.download(download_queue, self.filename)
+
+            '''
             for i in range(len(self.ext)):
                 download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i+1, EXT_MAP[self.ext[i]]))
+            '''
 
-            self.downloader.download(download_queue, format_filename('%s-%s' % (self.id, self.name[:200])))
         else:
             logger.critical('Downloader has not been loaded')
 

nhentai/downloader.py

@@ -29,6 +29,7 @@ class Downloader(Singleton):
         self.path = str(path)
         self.thread_count = thread
         self.threads = []
+        self.thread_pool = None
         self.timeout = timeout
 
     def _download(self, url, folder='', filename='', retried=0):

nhentai/parser.py

@@ -74,7 +74,7 @@ def login_parser(username, password):
 
     thread_pool = threadpool.ThreadPool(5)
 
-    for page in range(1, pages+1):
+    for page in range(1, pages + 1):
         try:
             logger.info('Getting doujinshi ids of page %d' % page)
             resp = s.get(constant.FAV_URL + '?page=%d' % page).text
@@ -92,18 +92,97 @@ def doujinshi_parser(id_):
     if not isinstance(id_, (int,)) and (isinstance(id_, (str,)) and not id_.isdigit()):
         raise Exception('Doujinshi id({0}) is not valid'.format(id_))
 
+    id_ = int(id_)
+    logger.log(15, 'Fetching doujinshi information of id {0}'.format(id_))
+    doujinshi = dict()
+    doujinshi['id'] = id_
+    url = '{0}/{1}/'.format(constant.DETAIL_URL, id_)
+
+    try:
+        response = request('get', url).content
+    except Exception as e:
+        logger.critical(str(e))
+        raise SystemExit
+
+    html = BeautifulSoup(response, 'html.parser')
+    doujinshi_info = html.find('div', attrs={'id': 'info'})
+
+    title = doujinshi_info.find('h1').text
+    subtitle = doujinshi_info.find('h2')
+
+    doujinshi['name'] = title
+    doujinshi['subtitle'] = subtitle.text if subtitle else ''
+
+    doujinshi_cover = html.find('div', attrs={'id': 'cover'})
+    img_id = re.search('/galleries/([\d]+)/cover\.(jpg|png)$', doujinshi_cover.a.img.attrs['data-src'])
+    if not img_id:
+        logger.critical('Tried yo get image id failed')
+        exit(1)
+
+    doujinshi['img_id'] = img_id.group(1)
+    doujinshi['ext'] = img_id.group(2)
+
+    pages = 0
+    for _ in doujinshi_info.find_all('div', class_=''):
+        pages = re.search('([\d]+) pages', _.text)
+        if pages:
+            pages = pages.group(1)
+            break
+    doujinshi['pages'] = int(pages)
+
+    # gain information of the doujinshi
+    information_fields = doujinshi_info.find_all('div', attrs={'class': 'field-name'})
+    needed_fields = ['Characters', 'Artists', 'Language', 'Tags']
+    for field in information_fields:
+        field_name = field.contents[0].strip().strip(':')
+        if field_name in needed_fields:
+            data = [sub_field.contents[0].strip() for sub_field in
+                    field.find_all('a', attrs={'class': 'tag'})]
+            doujinshi[field_name.lower()] = ', '.join(data)
+
+    return doujinshi
+
+
+def search_parser(keyword, page):
+    logger.debug('Searching doujinshis of keyword {0}'.format(keyword))
+    result = []
+    try:
+        response = request('get', url=constant.SEARCH_URL, params={'q': keyword, 'page': page}).content
+    except requests.ConnectionError as e:
+        logger.critical(e)
+        logger.warn('If you are in China, please configure the proxy to fu*k GFW.')
+        raise SystemExit
+
+    html = BeautifulSoup(response, 'html.parser')
+    doujinshi_search_result = html.find_all('div', attrs={'class': 'gallery'})
+    for doujinshi in doujinshi_search_result:
+        doujinshi_container = doujinshi.find('div', attrs={'class': 'caption'})
+        title = doujinshi_container.text.strip()
+        title = title if len(title) < 85 else title[:82] + '...'
+        id_ = re.search('/g/(\d+)/', doujinshi.a['href']).group(1)
+        result.append({'id': id_, 'title': title})
+    if not result:
+        logger.warn('Not found anything of keyword {}'.format(keyword))
+
+    return result
+
+
+def __api_suspended_doujinshi_parser(id_):
+    if not isinstance(id_, (int,)) and (isinstance(id_, (str,)) and not id_.isdigit()):
+        raise Exception('Doujinshi id({0}) is not valid'.format(id_))
+
     id_ = int(id_)
     logger.log(15, 'Fetching information of doujinshi id {0}'.format(id_))
     doujinshi = dict()
     doujinshi['id'] = id_
     url = '{0}/{1}'.format(constant.DETAIL_URL, id_)
-    i=0
-    while i<5:
+    i = 0
+    while 5 > i:
         try:
             response = request('get', url).json()
         except Exception as e:
-            i+=1
-            if not i<5:
+            i += 1
+            if not i < 5:
                 logger.critical(str(e))
                 exit(1)
             continue
@@ -130,21 +209,21 @@ def doujinshi_parser(id_):
             elif tag_type not in doujinshi:
                 doujinshi[tag_type] = tag['name']
             else:
-                doujinshi[tag_type] += tag['name']
+                doujinshi[tag_type] += ', ' + tag['name']
 
     return doujinshi
 
 
-def search_parser(keyword, page):
+def __api_suspended_search_parser(keyword, page):
     logger.debug('Searching doujinshis using keywords {0}'.format(keyword))
     result = []
-    i=0
-    while i<5:
+    i = 0
+    while i < 5:
         try:
             response = request('get', url=constant.SEARCH_URL, params={'query': keyword, 'page': page}).json()
         except Exception as e:
-            i+=1
-            if not i<5:
+            i += 1
+            if not i < 5:
                 logger.critical(str(e))
                 logger.warn('If you are in China, please configure the proxy to fu*k GFW.')
                 exit(1)
@@ -174,37 +253,18 @@ def print_doujinshi(doujinshi_list):
                 tabulate(tabular_data=doujinshi_list, headers=headers, tablefmt='rst'))
 
 
-def tag_parser(tag_id, max_page=1):
+def __api_suspended_tag_parser(tag_id, max_page=1):
     logger.info('Searching for doujinshi with tag id {0}'.format(tag_id))
     result = []
-    i=0
-    while i<5:
-        try:
-            response = request('get', url=constant.TAG_API_URL, params={'sort': 'popular', 'tag_id': tag_id}).json()
-        except Exception as e:
-            i+=1
-            if not i<5:
-                logger.critical(str(e))
-                exit(1)
-            continue
-        break
+    response = request('get', url=constant.TAG_API_URL, params={'sort': 'popular', 'tag_id': tag_id}).json()
     page = max_page if max_page <= response['num_pages'] else int(response['num_pages'])
 
-    for i in range(1, page+1):
+    for i in range(1, page + 1):
         logger.info('Getting page {} ...'.format(i))
 
         if page != 1:
-            i=0
-            while i<5:
-                try:
-                    response = request('get', url=constant.TAG_API_URL, params={'sort': 'popular', 'tag_id': tag_id}).json()
-                except Exception as e:
-                    i+=1
-                    if not i<5:
-                        logger.critical(str(e))
-                        exit(1)
-                    continue
-                break
+            response = request('get', url=constant.TAG_API_URL,
+                               params={'sort': 'popular', 'tag_id': tag_id}).json()
     for row in response['result']:
         title = row['title']['english']
         title = title[:85] + '..' if len(title) > 85 else title
@@ -212,46 +272,35 @@ def tag_parser(tag_id, max_page=1):
 
     if not result:
         logger.warn('No results for tag id {}'.format(tag_id))
-    
+
     return result
 
 
-def tag_guessing(tag_name):
+def tag_parser(tag_name, max_page=1):
+    result = []
     tag_name = tag_name.lower()
     tag_name = tag_name.replace(' ', '-')
-    logger.info('Trying to get tag_id of tag \'{0}\''.format(tag_name))
-    i=0
-    while i<5:
-        try:
-            response = request('get', url='%s/%s' % (constant.TAG_URL, tag_name)).content
-        except Exception as e:
-            i+=1
-            if not i<5:
-                logger.critical(str(e))
-                exit(1)
-            continue
-        break
 
-    html = BeautifulSoup(response, 'html.parser')
-    first_item = html.find('div', attrs={'class': 'gallery'})
-    if not first_item:
-        logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name))
-        return
+    for p in range(1, max_page + 1):
+        logger.debug('Fetching page {0} for doujinshi with tag \'{1}\''.format(p, tag_name))
+        response = request('get', url='%s/%s?page=%d' % (constant.TAG_URL, tag_name, p)).content
 
-    doujinshi_id = re.findall('(\d+)', first_item.a.attrs['href'])
-    if not doujinshi_id:
-        logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name))
-        return
+        html = BeautifulSoup(response, 'html.parser')
+        doujinshi_items = html.find_all('div', attrs={'class': 'gallery'})
+        if not doujinshi_items:
+            logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name))
+            return
 
-    ret = doujinshi_parser(doujinshi_id[0])
-    if 'tag' in ret and tag_name in ret['tag']:
-        tag_id = ret['tag'][tag_name]
-        logger.info('Tag id of tag \'{0}\' is {1}'.format(tag_name, tag_id))
-    else:
-        logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name))
-        return
+        for i in doujinshi_items:
+            doujinshi_id = i.a.attrs['href'].strip('/g')
+            doujinshi_title = i.a.text.strip()
+            doujinshi_title = doujinshi_title if len(doujinshi_title) < 85 else doujinshi_title[:82] + '...'
+            result.append({'title': doujinshi_title, 'id': doujinshi_id})
 
-    return tag_id
+    if not result:
+        logger.warn('No results for tag \'{}\''.format(tag_name))
+
+    return result
 
 
 if __name__ == '__main__':

nhentai/utils.py

@@ -43,8 +43,7 @@ def generate_html(output_dir='.', doujinshi_obj=None):
     image_html = ''
 
     if doujinshi_obj is not None:
-        doujinshi_dir = os.path.join(output_dir, format_filename('%s-%s' % (doujinshi_obj.id,
-                                                                            doujinshi_obj.name)))
+        doujinshi_dir = os.path.join(output_dir, doujinshi_obj.filename)
     else:
         doujinshi_dir = '.'
 
@@ -85,8 +84,7 @@ def generate_html(output_dir='.', doujinshi_obj=None):
 
 def generate_cbz(output_dir='.', doujinshi_obj=None, rm_origin_dir=False):
     if doujinshi_obj is not None:
-        doujinshi_dir = os.path.join(output_dir, format_filename('%s-%s' % (doujinshi_obj.id,
-                                                                            doujinshi_obj.name)))
+        doujinshi_dir = os.path.join(output_dir, doujinshi_obj.filename)
         cbz_filename = os.path.join(os.path.join(doujinshi_dir, '..'), '%s.cbz' % doujinshi_obj.id)
     else:
         cbz_filename = './doujinshi.cbz'
@@ -118,9 +116,12 @@ and append a file extension like '.txt', so I avoid the potential of using
 an invalid filename.
 
 """
-    valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
+    valid_chars = "-_.()[] %s%s" % (string.ascii_letters, string.digits)
     filename = ''.join(c for c in s if c in valid_chars)
     filename = filename.replace(' ', '_')  # I don't like spaces in filenames.
     if len(filename) > 100:
-        filename = filename[:100]
+        filename = filename[:100] + '...]'
+
+    # Remove [] from filename
+    filename = filename.replace('[]', '')
     return filename

requirements.txt

@@ -3,4 +3,3 @@ BeautifulSoup4>=4.0.0
 threadpool>=1.2.7
 tabulate>=0.7.5
 future>=0.15.2
-threadpool==1.3.2