Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 3 additions & 21 deletions assets/docs/sources/tutorial/12_domain_strategy.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,28 +55,10 @@ op = create_option('option.yml')
cl = op.new_jm_client()
```

### 2.2 通过 GitHub 兜底获取域名
> `get_html_domain_all_via_github` 已废弃。原 GitHub 仓库不再提供禁漫域名,兼容入口目前会转发到
> `get_html_domain_all`,并将在未来版本移除。

如果连禁漫的发布页本身都被墙了无法访问,可以请求禁漫官方放在 GitHub 的仓库来解析最新域名。

```python
from jmcomic import *

# 该请求发往 github.com,在大多数常规网络中均能保持连通
domains = JmModuleConfig.get_html_domain_all_via_github()

op = JmOption.default()
# 可以结合重试机制,允许失败时轮换多次
op.client.retry_times = 3

# 应用域名池新建包含该域名的 Client (记得指定 impl='html')
# 将新建的 client 赋值回 op,使其在后续的下载中生效
op.client = op.new_jm_client(domain_list=domains, impl='html')

download_album('438696', op)
```

### 2.3 获取单个跳转域名
### 2.2 获取单个跳转域名

除了获取全部域名,也可以通过访问永久跳转页获取单个重定向用的新域名。

Expand Down
71 changes: 68 additions & 3 deletions assets/docs/sources/tutorial/14_async_usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@ import jmcomic

async def main():
# 异步下载单个本子
await jmcomic.download_album_async('438696')
album, downloader = await jmcomic.download_album_async('438696')

# 返回的 downloader 已释放网络连接和线程池,只用于读取下载结果
print(downloader.download_failed_image)

# 异步下载单章节
await jmcomic.download_photo_async('438696')
Expand Down Expand Up @@ -63,6 +66,16 @@ async with JmOption.default().new_jm_async_client() as cl:

无论哪种写法都只会初始化一次,你不需要自己去调用任何初始化代码,直接用就行。

如果不使用 `async with`,使用完成后需要显式关闭客户端:

```python
cl = JmOption.default().new_jm_async_client()
try:
album = await cl.get_album_detail(123)
finally:
await cl.close()
```

---

### 并发请求示例
Expand Down Expand Up @@ -169,7 +182,8 @@ asyncio.run(main())

## 7. 异步分类 / 排行榜

分类和排行榜本质上都是过滤请求,可以使用 `categories_filter` 异步方法获取分页。
分类和排行榜本质上都是过滤请求,可以使用 `categories_filter` 获取单页,或使用
`categories_filter_gen` 异步生成器自动翻页。

```python
import asyncio
Expand All @@ -188,10 +202,61 @@ async def main():
for aid, atitle in page:
print(aid, atitle)

async for page in cl.categories_filter_gen(
time='a',
category='all',
order_by='mv',
):
Comment thread
coderabbitai[bot] marked this conversation as resolved.
print(page.page)

asyncio.run(main())
```

## 8. 关于 `async_impl` 配置
## 8. 指定异步 API 域名

`new_jm_async_client` 支持通过 `domain_list` 显式覆盖 API 域名。参数可以是字符串序列或多行字符串:

```python
async with JmOption.default().new_jm_async_client(
domain_list=['domain-a.example', 'domain-b.example'],
) as cl:
album = await cl.get_album_detail(123)
```

异步客户端不支持同步客户端的 `domain_retry_strategy` 参数。异步请求使用客户端内置的域名遍历与重试机制。

## 9. 下载完成回调与 downloader 生命周期

异步下载 API 的 `callback` 同时支持同步函数和异步函数:

```python
def sync_callback(album, downloader):
print(album.id)

async def async_callback(album, downloader):
await notify_server(album.id)

await jmcomic.download_album_async(123, callback=sync_callback)
await jmcomic.download_album_async(456, callback=async_callback)
```

`download_album_async` 和 `download_photo_async` 是一次性便利 API。返回的 downloader 对象仍可用于读取
`download_success_dict`、`download_failed_image` 和 `download_failed_photo`,但它持有的客户端和线程池已经关闭,
不能继续下载。

如果需要使用同一个 downloader 连续下载多个本子或章节,请显式管理 downloader 生命周期:

```python
from jmcomic import JmOption, new_async_downloader

option = JmOption.default()
async with new_async_downloader(option) as downloader:
await downloader.download_album(123)
await downloader.download_album(456)
await downloader.download_photo(789)
```

## 10. 关于 `async_impl` 配置

注意:仅仅在 `option.yml` 中增加配置**并不能**让代码自动变成异步,你必须要在代码中改为调用 `_async` 相关方法(如上文所示)。

Expand Down
24 changes: 3 additions & 21 deletions assets/docs/sources/tutorial/8_pick_domain.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,8 @@ disable_jm_log()


def get_all_domain():
template = 'https://jmcmomic.github.io/go/{}.html'
url_ls = [
template.format(i)
for i in range(300, 309)
]
domain_set: Set[str] = set()

def fetch_domain(url):
from curl_cffi import requests as postman
text = postman.get(url, allow_redirects=False, **meta_data).text
for domain in JmcomicText.analyse_jm_pub_html(text):
if domain.startswith('jm365.work'):
continue
domain_set.add(domain)

multi_thread_launcher(
iter_objs=url_ls,
apply_each_obj_func=fetch_domain,
)
return domain_set
postman = JmModuleConfig.new_postman(**meta_data)
return set(JmModuleConfig.get_html_domain_all(postman))


domain_set = get_all_domain()
Expand Down Expand Up @@ -78,4 +60,4 @@ for domain, status in domain_status_dict.items():
jmcomic1.me: ok
jmcomic.me: ok
18comic-palworld.club: ok
```
```
2 changes: 1 addition & 1 deletion src/jmcomic/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader

__version__ = '2.7.1'
__version__ = '2.7.2'

from .api import *
from .jm_plugin import *
Expand Down
29 changes: 24 additions & 5 deletions src/jmcomic/api.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,28 @@
import asyncio
import inspect

from .jm_downloader import *

__DOWNLOAD_API_RET = DownloadResult


async def _invoke_async_callback(callback, entity, downloader):
if callback is None:
return None

is_async_callback = (
inspect.iscoroutinefunction(callback)
or inspect.iscoroutinefunction(getattr(callback, '__call__', None))
)
if is_async_callback:
return await callback(entity, downloader)

result = await asyncio.to_thread(callback, entity, downloader)
if inspect.isawaitable(result):
return await result
return result


def download_batch(download_api,
jm_id_iter: Union[Iterable, Generator],
option=None,
Expand Down Expand Up @@ -167,7 +185,8 @@ async def download_album_async(jm_album_id,
异步下载一个本子(album),包含其所有的章节(photo)。

- 支持批量下载(当 jm_album_id 为可迭代对象时)
- 返回 (album, downloader) 元组
- callback 支持同步函数和异步函数
- 返回 (album, downloader) 元组,其中 downloader 的网络和线程池资源已关闭,仅用于读取下载结果
"""
if not isinstance(jm_album_id, (str, int)):
return await download_batch_async(download_album_async,
Expand All @@ -181,8 +200,7 @@ async def download_album_async(jm_album_id,
dler.add_features(extra, 'download_album')
album = await dler.download_album(jm_album_id)

if callback is not None:
callback(album, dler)
await _invoke_async_callback(callback, album, dler)
if check_exception:
dler.raise_if_has_exception()

Expand All @@ -198,6 +216,8 @@ async def download_photo_async(jm_photo_id,
):
"""
异步下载一个章节(photo)。
callback 支持同步函数和异步函数。
返回的 downloader 已关闭网络和线程池资源,仅用于读取下载结果。
"""
if not isinstance(jm_photo_id, (str, int)):
return await download_batch_async(download_photo_async,
Expand All @@ -211,8 +231,7 @@ async def download_photo_async(jm_photo_id,
dler.add_features(extra, 'download_photo')
photo = await dler.download_photo(jm_photo_id)

if callback is not None:
callback(photo, dler)
await _invoke_async_callback(callback, photo, dler)
if check_exception:
dler.raise_if_has_exception()

Expand Down
68 changes: 48 additions & 20 deletions src/jmcomic/jm_async_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import asyncio
import json
from typing import Sequence
from urllib.parse import urlencode

from curl_cffi.requests import AsyncSession
Expand Down Expand Up @@ -51,12 +52,15 @@ class AsyncJmApiClient(AsyncJmcomicClient):
_SENTINEL = object()

# 类级别初始化标记与锁,防止并发更新域名
_has_setup_domain_and_cookies = False
_has_setup_domain = False
_setup_lock = asyncio.Lock()

def __init__(self, option: JmOption, max_clients=None, **kwargs):
def __init__(self, option: JmOption, max_clients=None, domain_list=None, **kwargs):
if 'domain_retry_strategy' in kwargs:
raise TypeError('Async client does not support domain_retry_strategy')

self.option = option
self._domain_list = self._resolve_domain_list()
self._domain_list = self._resolve_domain_list(domain_list)
retry_times = option.client.get('retry_times')
self._retry_times = retry_times if retry_times is not None else 5
self._timeout = option.client.get('timeout', 30) or 30
Expand Down Expand Up @@ -84,22 +88,44 @@ def __init__(self, option: JmOption, max_clients=None, **kwargs):
# 域名管理
# ======================================================================

def _resolve_domain_list(self) -> list[str]:
@staticmethod
def _normalize_domain_list(domain_list) -> list[str]:
if isinstance(domain_list, str):
return [domain.strip() for domain in domain_list.splitlines() if domain.strip()]

if isinstance(domain_list, Sequence):
result = []
for domain in domain_list:
if not isinstance(domain, str):
raise TypeError(f'domain must be str, got {type(domain)}')
domain = domain.strip()
if domain:
result.append(domain)
return result

raise TypeError('domain_list must be str, Sequence[str], or None')

def _resolve_domain_list(self, domain_list=None) -> list[str]:
"""解析并返回可用的 API 域名列表"""
if domain_list is not None:
resolved = self._normalize_domain_list(domain_list)
if resolved:
return resolved

updated = JmModuleConfig.DOMAIN_API_UPDATED_LIST
if updated:
return list(updated)
domain = self.option.client.domain
if hasattr(domain, 'get'):
domain_list = domain.get('api', [])
elif isinstance(domain, list):
domain_list = domain
elif isinstance(domain, str):
domain_list = [d.strip() for d in domain.split('\n') if d.strip()]
else:
domain_list = []
if domain_list:
return domain_list
domain_list = domain

if domain_list is not None:
resolved = self._normalize_domain_list(domain_list)
if resolved:
return resolved

return list(JmModuleConfig.DOMAIN_API_LIST)

def get_domain_list(self) -> list[str]:
Expand Down Expand Up @@ -708,20 +734,22 @@ async def setup(self):

cls = self.__class__
async with cls._setup_lock:
if not cls._has_setup_domain_and_cookies:
if self._has_setup:
return

if not cls._has_setup_domain:
await self.auto_update_domain()
if JmModuleConfig.FLAG_API_CLIENT_REQUIRE_COOKIES:
await self.ensure_have_cookies()
cls._has_setup_domain_and_cookies = True
cls._has_setup_domain = True
else:
# 即使已经初始化过域名和 cookie,也需要将已保存的全局 DOMAIN 和 COOKIES 赋值到当前 client
# 即使已经初始化过域名,也需要将已保存的全局 DOMAIN 赋值到当前 client
if JmModuleConfig.DOMAIN_API_UPDATED_LIST:
self._domain_list = list(JmModuleConfig.DOMAIN_API_UPDATED_LIST)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
if JmModuleConfig.FLAG_API_CLIENT_REQUIRE_COOKIES and JmModuleConfig.APP_COOKIES:
# noinspection PyUnresolvedReferences
self._session.cookies.update(JmModuleConfig.APP_COOKIES)

self._has_setup = True
# Cookie 属于 session 状态,每个 client 都需要独立确认。
if JmModuleConfig.FLAG_API_CLIENT_REQUIRE_COOKIES:
await self.ensure_have_cookies()

self._has_setup = True

# ======================================================================
# 生命周期
Expand Down
13 changes: 12 additions & 1 deletion src/jmcomic/jm_async_downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,18 @@ def shutdown(self):
async def __aenter__(self):
# 创建并独占一个 async client(含 AsyncSession)。
self.client = self.option.new_jm_async_client(max_clients=self._image_concurrency)
await self.client.setup()
try:
await self.client.setup()
except BaseException:
try:
await self.client.close()
except BaseException as cleanup_error:
jm_log('dler.cleanup.exception',
f'初始化失败后的资源清理也发生异常: {cleanup_error}', cleanup_error)
finally:
self.client = None
self.shutdown()
raise
return self

async def __aexit__(self, exc_type, exc_val, exc_tb):
Expand Down
Loading
Loading