NussNougate/nhentai
4
0
Fork 0
mirror of https://github.com/RicterZ/nhentai.git synced 2025-04-19 18:31:24 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #353 from hzxjy1/master

Browse Source 
Fix issue #7
This commit is contained in:
Ricter Zheng 2024-11-19 02:10:34 +08:00 committed by GitHub
commit c044b64beb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
nhentai/downloader.py
View File

@ -49,14 +49,10 @@ class Downloader(Singleton):
time.sleep(self.delay) time.sleep(self.delay)
logger.info(f'Starting to download {url} ...') logger.info(f'Starting to download {url} ...')
filename = filename if filename else os.path.basename(urlparse(url).path) filename = filename if filename else os.path.basename(urlparse(url).path)
base_filename, extension = os.path.splitext(filename)
save_file_path = os.path.join(folder, base_filename.zfill(3) + extension) save_file_path = os.path.join(self.folder, filename)
try: try:
if not os.path.exists(folder):
os.makedirs(folder, exist_ok=True)
if os.path.exists(save_file_path): if os.path.exists(save_file_path):
logger.warning(f'Skipped download: {save_file_path} already exists') logger.warning(f'Skipped download: {save_file_path} already exists')
return 1, url return 1, url
@ -66,20 +62,20 @@ class Downloader(Singleton):
if response.status_code != 200: if response.status_code != 200:
path = urlparse(url).path path = urlparse(url).path
for mirror in constant.IMAGE_URL_MIRRORS: for mirror in constant.IMAGE_URL_MIRRORS:
print(f'{mirror}{path}') logger.info(f"Try mirror: {mirror}{path}")
mirror_url = f'{mirror}{path}' mirror_url = f'{mirror}{path}'
response = await self.async_request(mirror_url, self.timeout) response = await self.async_request(mirror_url, self.timeout)
if response.status_code == 200: if response.status_code == 200:
break break
if not await self.save(save_file_path, response): if not await self.save(filename, response):
logger.error(f'Can not download image {url}') logger.error(f'Can not download image {url}')
return 1, None return 1, None
except (httpx.HTTPStatusError, httpx.TimeoutException) as e: except (httpx.HTTPStatusError, httpx.TimeoutException, httpx.ConnectError) as e:
if retried < 3: if retried < 3:
logger.warning(f'Warning: {e}, retrying({retried}) ...') logger.info(f'Download {filename} failed, retrying({retried + 1}) times...')
return 0, await self.download( return await self.download(
url=url, url=url,
folder=folder, folder=folder,
filename=filename, filename=filename,
@ -96,6 +92,7 @@ class Downloader(Singleton):
except Exception as e: except Exception as e:
import traceback import traceback
logger.error(f"Exception type: {type(e)}")
traceback.print_stack() traceback.print_stack()
logger.critical(str(e)) logger.critical(str(e))
return 0, None return 0, None
@ -109,7 +106,7 @@ class Downloader(Singleton):
if response is None: if response is None:
logger.error('Error: Response is None') logger.error('Error: Response is None')
return False return False
save_file_path = os.path.join(self.folder, save_file_path)
with open(save_file_path, 'wb') as f: with open(save_file_path, 'wb') as f:
if response is not None: if response is not None:
length = response.headers.get('content-length') length = response.headers.get('content-length')
@ -125,6 +122,7 @@ class Downloader(Singleton):
return await client.get(url, timeout=timeout) return await client.get(url, timeout=timeout)
def start_download(self, queue, folder='') -> bool: def start_download(self, queue, folder='') -> bool:
logger.warning("Proxy temporarily unavailable, it will be fixed later. ")
if not isinstance(folder, (str, )): if not isinstance(folder, (str, )):
folder = str(folder) folder = str(folder)
@ -137,12 +135,13 @@ class Downloader(Singleton):
os.makedirs(folder) os.makedirs(folder)
except EnvironmentError as e: except EnvironmentError as e:
logger.critical(str(e)) logger.critical(str(e))
self.folder = folder
if os.getenv('DEBUG', None) == 'NODOWNLOAD': if os.getenv('DEBUG', None) == 'NODOWNLOAD':
# Assuming we want to continue with rest of process. # Assuming we want to continue with rest of process.
return True return True
async def co_wrapper(tasks): async def fiber(tasks):
for completed_task in asyncio.as_completed(tasks): for completed_task in asyncio.as_completed(tasks):
try: try:
result = await completed_task result = await completed_task
@ -155,6 +154,6 @@ class Downloader(Singleton):
for url in queue for url in queue
] ]
# Prevent coroutines infection # Prevent coroutines infection
asyncio.run(co_wrapper(tasks)) asyncio.run(fiber(tasks))
return True return True

1
requirements.txt
View File

@ -1,3 +1,4 @@
httpx
requests requests
soupsieve soupsieve
setuptools setuptools