From e3410f5a9ad09f8eb236e3ab87202b444ec83bc1 Mon Sep 17 00:00:00 2001
From: normalizedwater546 <normalizedwater546@ak.gy>
Date: Sat, 23 Nov 2024 13:11:25 +0000
Subject: [PATCH] fix: add headers, proxy to async_request
---
nhentai/downloader.py | 15 +++++----------
nhentai/utils.py | 21 +++++++++++++++++++++
2 files changed, 26 insertions(+), 10 deletions(-)
diff --git a/nhentai/downloader.py b/nhentai/downloader.py
index 9e3b21d..9bcb6f2 100644
--- a/nhentai/downloader.py
+++ b/nhentai/downloader.py
@@ -1,19 +1,18 @@
# coding: utf-
import os
+import asyncio
+import httpx
import urllib3.exceptions
from urllib.parse import urlparse
from nhentai import constant
from nhentai.logger import logger
-from nhentai.utils import Singleton
+from nhentai.utils import Singleton, async_request
-import asyncio
-import httpx
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
class NHentaiImageNotExistException(Exception):
pass
@@ -68,14 +67,14 @@ class Downloader(Singleton):
logger.warning(f'Skipped download: {save_file_path} already exists')
return 1, url
- response = await self.async_request(url, self.timeout) # TODO: Add proxy
+ response = await async_request('GET', url, timeout=self.timeout, proxies=proxy)
if response.status_code != 200:
path = urlparse(url).path
for mirror in constant.IMAGE_URL_MIRRORS:
logger.info(f"Try mirror: {mirror}{path}")
mirror_url = f'{mirror}{path}'
- response = await self.async_request(mirror_url, self.timeout)
+ response = await async_request('GET', mirror_url, timeout=self.timeout, proxies=proxy)
if response.status_code == 200:
break
@@ -128,12 +127,8 @@ class Downloader(Singleton):
f.write(chunk)
return True
- async def async_request(self, url, timeout):
- async with httpx.AsyncClient() as client:
- return await client.get(url, timeout=timeout)
def start_download(self, queue, folder='') -> bool:
- logger.warning("Proxy temporarily unavailable, it will be fixed later. ")
if not isinstance(folder, (str,)):
folder = str(folder)
diff --git a/nhentai/utils.py b/nhentai/utils.py
index 6e50a7e..f445201 100644
--- a/nhentai/utils.py
+++ b/nhentai/utils.py
@@ -6,6 +6,7 @@ import os
import zipfile
import shutil
+import httpx
import requests
import sqlite3
import urllib.parse
@@ -32,8 +33,28 @@ def request(method, url, **kwargs):
return getattr(session, method)(url, verify=False, **kwargs)
+async def async_request(method, url, proxies = None, **kwargs):
+ headers = {
+ 'Referer': constant.LOGIN_URL,
+ 'User-Agent': constant.CONFIG['useragent'],
+ 'Cookie': constant.CONFIG['cookie'],
+ }
+
+ if proxies is None:
+ proxies = constant.CONFIG['proxy']
+
+ if proxies.get('http') == '' and proxies.get('https') == '':
+ proxies = None
+
+ async with httpx.AsyncClient(headers=headers, verify=False, proxies=proxies, **kwargs) as client:
+ response = await client.request(method, url, **kwargs)
+
+ return response
+
+
def check_cookie():
response = request('get', constant.BASE_URL)
+
if response.status_code == 403 and 'Just a moment...' in response.text:
logger.error('Blocked by Cloudflare captcha, please set your cookie and useragent')
sys.exit(1)