junior.gj/AutoComicRss
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement RSS subscriptions, downloader routing and local media checks

Browse Source
This commit is contained in:
junior
2026-10-17 22:49:11 +08:00
parent 645ea6fe2d
commit 68c972dadd
28 changed files with 9226 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+26
View File
@@ -0,0 +1,26 @@
# Copy values into your shell, or source a private .env file before cargo run.
# This application does not automatically load .env files.
APP_PORT=8080
DATABASE_URL=sqlite://data/ani.db
POLL_INTERVAL_SECONDS=300
RUST_LOG=info
# Website requests (Mikan pages/covers, RSS and torrent metadata).
# Unset means direct; HTTP_PROXY/HTTPS_PROXY/ALL_PROXY/NO_PROXY are NOT inherited.
MIKAN_BASE_URL=https://mikanani.me
# WEBSITE_PROXY_URL=http://127.0.0.1:7897
# WEBSITE_PROXY_URL=socks5h://127.0.0.1:7897
# Optional URL-encoded credentials: http://username:password@host:port
# Comma-separated explicit direct exceptions; empty by default.
# WEBSITE_DIRECT_HOSTS=localhost,127.0.0.1,::1
# qBittorrent control connections always remain direct.
# Initial import only, when the database has no downloader instances.
# Existing instances and the global selection are managed through /api/downloaders.
# When upgrading, these MUST identify the original downloader of existing jobs.
# Unset on a fresh installation leaves jobs unassigned until first selection.
# QBIT_URL=http://127.0.0.1:8081
# QBIT_USERNAME=admin
# QBIT_PASSWORD=replace-me
# Path as seen by qBittorrent, not by this Rust process.
# QBIT_SAVE_PATH=/downloads
.gitignore
+8 -1
View File
@@ -51,4 +51,11 @@ Icon?
desktop.ini desktop.ini
# For Cargo documentation # For Cargo documentation
/doc/ /doc/
# Local state and credentials
/data/
.env
*.db
*.db-shm
*.db-wal
Cargo.lock
Generated
+3841
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+21 -1
View File
@@ -1,6 +1,26 @@
[package] [package]
name = "rust_project" name = "rust_project"
version = "0.1.0" version = "0.1.0"
edition = "2024" edition = "2021"
license = "GPL-2.0-only"
description = "A Rust RSS subscription and download automation service"
[dependencies] [dependencies]
actix-web = "4.9"
actix-files = "0.6"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
tokio = { version = "1", features = ["sync", "time", "signal", "rt-multi-thread", "macros"] }
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json", "cookies", "multipart", "socks"] }
scraper = "0.24"
cap-std = "4.0"
sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "sqlite", "derive", "migrate", "macros"] }
feed-rs = "2.3"
regex = "1"
uuid = { version = "1", features = ["v4"] }
thiserror = "2"
sha1 = "0.10"
bendy = "0.3"
data-encoding = "2"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
LICENSE
+339
View File
@@ -0,0 +1,339 @@
GNU GENERAL PUBLIC LICENSE
Version 2, June 1991
Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The licenses for most software are designed to take away your
freedom to share and change it. By contrast, the GNU General Public
License is intended to guarantee your freedom to share and change free
software--to make sure the software is free for all its users. This
General Public License applies to most of the Free Software
Foundation's software and to any other program whose authors commit to
using it. (Some other Free Software Foundation software is covered by
the GNU Lesser General Public License instead.) You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
this service if you wish), that you receive source code or can get it
if you want it, that you can change the software or use pieces of it
in new free programs; and that you know you can do these things.
To protect your rights, we need to make restrictions that forbid
anyone to deny you these rights or to ask you to surrender the rights.
These restrictions translate to certain responsibilities for you if you
distribute copies of the software, or if you modify it.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must give the recipients all the rights that
you have. You must make sure that they, too, receive or can get the
source code. And you must show them these terms so they know their
rights.
We protect your rights with two steps: (1) copyright the software, and
(2) offer you this license which gives you legal permission to copy,
distribute and/or modify the software.
Also, for each author's protection and ours, we want to make certain
that everyone understands that there is no warranty for this free
software. If the software is modified by someone else and passed on, we
want its recipients to know that what they have is not the original, so
that any problems introduced by others will not reflect on the original
authors' reputations.
Finally, any free program is threatened constantly by software
patents. We wish to avoid the danger that redistributors of a free
program will individually obtain patent licenses, in effect making the
program proprietary. To prevent this, we have made it clear that any
patent must be licensed for everyone's free use or not licensed at all.
The precise terms and conditions for copying, distribution and
modification follow.
GNU GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License applies to any program or other work which contains
a notice placed by the copyright holder saying it may be distributed
under the terms of this General Public License. The "Program", below,
refers to any such program or work, and a "work based on the Program"
means either the Program or any derivative work under copyright law:
that is to say, a work containing the Program or a portion of it,
either verbatim or with modifications and/or translated into another
language. (Hereinafter, translation is included without limitation in
the term "modification".) Each licensee is addressed as "you".
Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope. The act of
running the Program is not restricted, and the output from the Program
is covered only if its contents constitute a work based on the
Program (independent of having been made by running the Program).
Whether that is true depends on what the Program does.
1. You may copy and distribute verbatim copies of the Program's
source code as you receive it, in any medium, provided that you
conspicuously and appropriately publish on each copy an appropriate
copyright notice and disclaimer of warranty; keep intact all the
notices that refer to this License and to the absence of any warranty;
and give any other recipients of the Program a copy of this License
along with the Program.
You may charge a fee for the physical act of transferring a copy, and
you may at your option offer warranty protection in exchange for a fee.
2. You may modify your copy or copies of the Program or any portion
of it, thus forming a work based on the Program, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:
a) You must cause the modified files to carry prominent notices
stating that you changed the files and the date of any change.
b) You must cause any work that you distribute or publish, that in
whole or in part contains or is derived from the Program or any
part thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.
c) If the modified program normally reads commands interactively
when run, you must cause it, when started running for such
interactive use in the most ordinary way, to print or display an
announcement including an appropriate copyright notice and a
notice that there is no warranty (or else, saying that you provide
a warranty) and that users may redistribute the program under
these conditions, and telling the user how to view a copy of this
License. (Exception: if the Program itself is interactive but
does not normally print such an announcement, your work based on
the Program is not required to print an announcement.)
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Program,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works. But when you
distribute the same sections as part of a whole which is a work based
on the Program, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote it.
Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Program.
In addition, mere aggregation of another work not based on the Program
with the Program (or with a work based on the Program) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.
3. You may copy and distribute the Program (or a work based on it,
under Section 2) in object code or executable form under the terms of
Sections 1 and 2 above provided that you also do one of the following:
a) Accompany it with the complete corresponding machine-readable
source code, which must be distributed under the terms of Sections
1 and 2 above on a medium customarily used for software interchange; or,
b) Accompany it with a written offer, valid for at least three
years, to give any third party, for a charge no more than your
cost of physically performing source distribution, a complete
machine-readable copy of the corresponding source code, to be
distributed under the terms of Sections 1 and 2 above on a medium
customarily used for software interchange; or,
c) Accompany it with the information you received as to the offer
to distribute corresponding source code. (This alternative is
allowed only for noncommercial distribution and only if you
received the program in object code or executable form with such
an offer, in accord with Subsection b above.)
The source code for a work means the preferred form of the work for
making modifications to it. For an executable work, complete source
code means all the source code for all modules it contains, plus any
associated interface definition files, plus the scripts used to
control compilation and installation of the executable. However, as a
special exception, the source code distributed need not include
anything that is normally distributed (in either source or binary
form) with the major components (compiler, kernel, and so on) of the
operating system on which the executable runs, unless that component
itself accompanies the executable.
If distribution of executable or object code is made by offering
access to copy from a designated place, then offering equivalent
access to copy the source code from the same place counts as
distribution of the source code, even though third parties are not
compelled to copy the source along with the object code.
4. You may not copy, modify, sublicense, or distribute the Program
except as expressly provided under this License. Any attempt
otherwise to copy, modify, sublicense or distribute the Program is
void, and will automatically terminate your rights under this License.
However, parties who have received copies, or rights, from you under
this License will not have their licenses terminated so long as such
parties remain in full compliance.
5. You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Program or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
modifying or distributing the Program (or any work based on the
Program), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Program or works based on it.
6. Each time you redistribute the Program (or any work based on the
Program), the recipient automatically receives a license from the
original licensor to copy, distribute or modify the Program subject to
these terms and conditions. You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties to
this License.
7. If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Program at all. For example, if a patent
license would not permit royalty-free redistribution of the Program by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Program.
If any portion of this section is held invalid or unenforceable under
any particular circumstance, the balance of the section is intended to
apply and the section as a whole is intended to apply in other
circumstances.
It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system, which is
implemented by public license practices. Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
8. If the distribution and/or use of the Program is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Program under this License
may add an explicit geographical distribution limitation excluding
those countries, so that distribution is permitted only in or among
countries not thus excluded. In such case, this License incorporates
the limitation as if written in the body of this License.
9. The Free Software Foundation may publish revised and/or new versions
of the General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the Program
specifies a version number of this License which applies to it and "any
later version", you have the option of following the terms and conditions
either of that version or of any later version published by the Free
Software Foundation. If the Program does not specify a version number of
this License, you may choose any version ever published by the Free Software
Foundation.
10. If you wish to incorporate parts of the Program into other free
programs whose distribution conditions are different, write to the author
to ask for permission. For software which is copyrighted by the Free
Software Foundation, write to the Free Software Foundation; we sometimes
make exceptions for this. Our decision will be guided by the two goals
of preserving the free status of all derivatives of our free software and
of promoting the sharing and reuse of software generally.
NO WARRANTY
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
REPAIR OR CORRECTION.
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
convey the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
Also add information on how to contact you by electronic and paper mail.
If the program is interactive, make it output a short notice like this
when it starts in an interactive mode:
Gnomovision version 69, Copyright (C) year name of author
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, the commands you use may
be called something other than `show w' and `show c'; they could even be
mouse-clicks or menu items--whatever suits your program.
You should also get your employer (if you work as a programmer) or your
school, if any, to sign a "copyright disclaimer" for the program, if
necessary. Here is a sample; alter the names:
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
`Gnomovision' (which makes passes at compilers) written by James Hacker.
<signature of Ty Coon>, 1 April 1989
Ty Coon, President of Vice
This General Public License does not permit incorporating your program into
proprietary programs. If your program is a subroutine library, you may
consider it more useful to permit linking proprietary applications with the
library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License.
README.md
+259
View File
@@ -0,0 +1,259 @@
# Rust RSS 追番服务
使用 Rust 独立设计实现的 RSS 订阅与下载自动化项目，采用 **GPL-2.0-only**。
功能方向参考 [ani-rss](https://github.com/wushuo894/ani-rss)，没有移植其 Java 业务代码或前端资源；本项目不是上游的官方 Rust 版本。
当前为后端 MVP：蜜柑搜索 / 手动 RSS → 订阅管理 → RSS/Atom 预览 → 持久化任务 → qBittorrent 提交与状态同步。
现有 Actix 示例接口继续保留。
## 运行
```sh
cargo run --locked
```
默认监听 `127.0.0.1:8080`，自动创建 `data/ani.db` 并运行数据库迁移。
当前 API 供本机使用，没有用户认证；监听地址固定为回环地址。
首次运行需要下载 Cargo 依赖。已在 Rust 1.93.1 上进行编译验证。
按需配置环境变量，参见 [.env.example](.env.example)。程序不自动读取 `.env`：
```sh
export QBIT_URL='http://127.0.0.1:8081'
export QBIT_USERNAME='admin'
export QBIT_PASSWORD='你的密码'
export QBIT_SAVE_PATH='/downloads'
cargo run --locked
```
qBittorrent 需要启用 Web UI。`QBIT_SAVE_PATH` 是下载器看到的路径。
`QBIT_*` 只在数据库尚无下载器实例时导入一次，并选为全局下载器。已有实例时，重启或修改这些环境变量不会覆盖保存的实例或选择。
全新安装不设置 `QBIT_URL` 时，仍可管理订阅、预览和生成未分配任务；首次通过接口选中下载器后绑定这些任务。清空环境变量不会停用数据库中已选中的下载器。
程序启动后立即执行一轮检查，之后默认每 300 秒检查一次；`POLL_INTERVAL_SECONDS` 最小为 10。
## 使用流程
### 从蜜柑发现订阅
```sh
curl -G 'http://127.0.0.1:8080/api/sources/mikan/search' --data-urlencode 'q=葬送'
curl 'http://127.0.0.1:8080/api/sources/mikan/series/3141'
curl 'http://127.0.0.1:8080/api/sources/mikan/resolve' \
-H 'Content-Type: application/json' \
-d '{"url":"https://mikanani.me/Home/Bangumi/3141"}'
```
搜索返回各季度的独立候选及关键词 RSS。详情返回整部番剧 RSS、字幕组 RSS、站点信息和可用封面地址。
将详情中的字幕组 `rss_url` 传给 `resolve`，可以获得对应字幕组候选。
`resolve` 返回的 `subscription` 对象可直接作为下面预览、保存接口的请求体；默认禁用，`season=1` 只是初始值，需核对季度、偏移和筛选规则。
候选中的来源 ID、字幕组 ID 和范围用于发现流程；当前订阅保存标题、RSS 与规则，尚不单独持久化这些来源元数据。
关键词 RSS 可能跨多部作品，`resolve` 返回 `scope=keyword`、提示和 RSS 地址，`subscription=null`，不会生成单番剧草稿。
建议选择具体番剧；若手工使用关键词 RSS，需先用规则将内容限定为一部作品并预览。
搜索、查看详情、解析链接和预览均不创建下载任务。发布条目通过 RSS 预览获取，不从详情 HTML 重复解析。
这阶段提供后端 API，尚无搜索管理页面或蜜柑账号同步。
### 网站代理
```sh
export WEBSITE_PROXY_URL='http://127.0.0.1:7897'
# 或 socks5h://127.0.0.1:7897，让代理解析目标域名
export MIKAN_BASE_URL='https://mikanani.me'
cargo run --locked
```
网站客户端支持 HTTP、HTTPS、SOCKS5 和 SOCKS5H 代理 URL，可在 URL 内携带编码后的用户名和密码。
`socks5` 本地解析目标域名，`socks5h` 由代理解析。配置在启动时读取，修改后重启。
**不继承** `HTTP_PROXY`、`HTTPS_PROXY`、`ALL_PROXY` 或 `NO_PROXY`；未设置 `WEBSITE_PROXY_URL` 时直连。
可用 `WEBSITE_DIRECT_HOSTS` 设置逗号分隔的显式直连域名/IP/CIDR，例如 `localhost,127.0.0.1,::1`，默认没有例外。
每次重定向重新应用代理及例外；代理失败返回错误，不自动直连。
蜜柑搜索、详情、封面、RSS 和种子元数据请求共用网站策略。封面返回本应用地址，由后端获取，浏览器无需直连蜜柑。
当前仅识别蜜柑同源 `/images/` 下的封面，支持 PNG/JPEG/WebP/GIF；其他图片布局会显示无封面。
qBittorrent 控制连接独立且始终直连；BT 传输由下载器控制。容器中的 `127.0.0.1` 指容器自身。
网站基址必须是无路径、查询参数或凭据的 HTTP(S) origin；粘贴解析仅接受该站点的公开搜索/番剧页面或 RSS。
### 预览、保存和启用
1. 预览真实的 RSS 地址，查看匹配结果。预览不会保存订阅或提交下载。
```sh
curl -s http://127.0.0.1:8080/api/preview \
-H 'Content-Type: application/json' \
-d '{
"title": "示例番剧",
"rss_url": "https://example.com/anime.xml",
"season": 1,
"episode_offset": 0,
"include": ["1080p"],
"exclude": ["合集"],
"enabled": false
}'
```
2. 将相同 JSON 提交到 `POST /api/subscriptions` 保存订阅。订阅默认禁用。
3. 确认预览结果后，向 `PUT /api/subscriptions/{id}` 提交完整 JSON，将 `enabled` 改为 `true`。
4. 等待自动检查，或使用 `POST /api/run` 立即执行一轮 RSS 检查和下载同步。
5. 使用 `GET /api/jobs` 查看状态、失败原因和重试时间。
首次启用会考虑 RSS 当前返回的全部匹配条目，尚未实现“仅从当前最新集开始”。
`include` 为不区分大小写的关键词“全部满足”，`exclude` 为“任意命中则排除”，不是正则表达式。
修改订阅规则只影响后续扫描；已入队任务保留创建时的季度、集数和资源地址。禁用订阅会暂停未提交任务。
## API
| 方法 | 路径 | 用途 |
| --- | --- | --- |
| GET | `/api/status` | 版本及下载器是否配置，不返回凭据 |
| GET / POST | `/api/downloaders` | 列出 / 创建下载器实例，响应不返回密码 |
| PUT | `/api/downloaders/active` | `{"downloader_id":"实例 ID"}` 选择接收新任务的全局下载器 |
| POST | `/api/downloaders/{id}/claim-legacy-jobs` | 将升级后尚无归属的旧任务明确关联到恢复的原实例 |
| GET / POST | `/api/destinations` | 列出 / 创建本地媒体目的地 |
| PUT | `/api/destinations/active` | `{"destination_id":"目的地 ID"}` 设置全局目的地 |
| GET / POST | `/api/destinations/{id}/mappings` | 查看历史映射 / 新增映射版本 |
| GET | `/api/destinations/{id}/entries?path=相对目录` | 只读浏览目录，省略 path 为根目录 |
| GET | `/api/sources/mikan/search?q=关键词` | 番剧搜索和关键词 RSS |
| GET | `/api/sources/mikan/series/{id}` | 番剧、字幕组及对应 RSS |
| GET | `/api/sources/mikan/series/{id}/cover` | 经网站网络策略获取封面 |
| POST | `/api/sources/mikan/resolve` | `{"url":"蜜柑地址"}` → 订阅候选 |
| POST | `/api/preview` | 临时预览，不写库、不下载 |
| GET / POST | `/api/subscriptions` | 列出 / 创建订阅 |
| GET / PUT / DELETE | `/api/subscriptions/{id}` | 查询 / 完整替换 / 删除订阅 |
| GET | `/api/subscriptions/{id}/preview` | 预览已保存订阅，包含去重判断 |
| POST | `/api/subscriptions/{id}/scan` | 扫描并入队，要求订阅已启用 |
| GET | `/api/subscriptions/{id}/releases` | 最近最多 500 条扫描记录 |
| GET | `/api/jobs` | 最近最多 500 个任务 |
| GET | `/api/jobs/{id}/files` | 读取已保存的文件清单与核对状态 |
| POST | `/api/jobs/{id}/files/check` | 发现文件清单（首次）并核对本地文件是否存在 |
| POST | `/api/jobs/{id}/retry` | 手动重试 failed / missing 任务 |
| POST | `/api/run` | 执行完整扫描及下载同步 |
后台处理与手动执行互斥；处理期间修改订阅或再次执行会返回 `409`，稍后重试即可。
删除订阅为软删除，保留去重与任务记录，不删除下载器任务或媒体文件。已提交任务仍会同步状态。
## 下载器实例与切换
```sh
curl 'http://127.0.0.1:8080/api/downloaders' \
-H 'Content-Type: application/json' \
-d '{"name":"NAS qBittorrent","kind":"qbittorrent","connection":{"url":"http://nas:8081","username":"admin","password":"你的密码","save_path":"/downloads"}}'
# 将返回的 id 填入下面的 downloader_id。
curl -X PUT 'http://127.0.0.1:8080/api/downloaders/active' \
-H 'Content-Type: application/json' -d '{"downloader_id":"返回的实例 ID"}'
```
创建实例不会自动切换全局选择。任务在扫描入队的同一个事务中保存 `downloader_id`；已有归属的排队任务、重试和重启恢复都使用原实例。
切换后继续同步原实例；一个实例连接失败会报告该实例 ID，并继续处理其他实例，不把连接失败标成任务丢失。
原实例的地址与写入路径不随全局选择改变。本阶段只实现多个 qBittorrent 实例，不提供实例编辑/删除；第二种工具与凭据维护接口另行扩展。
实例凭据保存在 SQLite 数据库中，尚未加密；数据库及其备份应按含凭据的配置文件管理。列表和创建响应只返回 `password_configured`。
### 从旧版本升级
升级迁移为旧任务增加实例归属字段。首次导入时，应提供**旧任务实际使用的原 `QBIT_*` 配置**，应用会将它与旧任务关联；不能在升级时把这些值替换成另一台下载器。
若升级时没有提供原配置，历史任务保留 `legacy_unassigned=true`，不会自动绑定以后新选中的实例；`/api/status` 的 `legacy_unassigned_jobs` 及运行报告会提示数量。
恢复原实例后，显式调用 `POST /api/downloaders/{原实例ID}/claim-legacy-jobs` 才建立关联。该操作仅认领没有归属的旧任务，不迁移已有归属，也不立即提交下载；后续同步先核对原实例的任务。
## 全局媒体目的地与本地文件核对
先配置并选中下载器，再创建一个本地目的地，配置该下载器的写入路径映射，最后激活目的地：
```sh
curl 'http://127.0.0.1:8080/api/destinations' \
-H 'Content-Type: application/json' \
-d '{"name":"番剧目录","kind":"local","root_path":"/mnt/nas/anime"}'
curl 'http://127.0.0.1:8080/api/destinations/目的地ID/mappings' \
-H 'Content-Type: application/json' \
-d '{"downloader_id":"下载器ID","download_root":"/downloads/anime"}'
curl -X PUT 'http://127.0.0.1:8080/api/destinations/active' \
-H 'Content-Type: application/json' -d '{"destination_id":"目的地ID"}'
```
`root_path` 是 Rust 应用所在环境可访问的绝对目录；`download_root` 是同一个位置在下载器环境中的绝对路径。
配置映射不会挂载、复制或上传文件，也不会创建这些目录。两个路径必须实际指向同一批文件。
同一时刻所有新任务统一使用全局目的地，订阅不能覆盖。切换目的地或下载器前，必须存在所需映射，否则返回 `400`，不会静默使用默认目录。
每次 `POST mappings` 都追加一个版本；新任务使用最新版本，旧任务保留 `destination_id`、`mapping_id` 和 `download_save_path`。
目的地读取根目录和已有映射版本保持不变；更改读取根需创建新目的地。即使任务尚未提交、后来重试或应用重启，也不重新解释其存储位置。
同一下载器里已存在相同 hash，但保存路径与任务快照不一致时，任务报告冲突，不复用错误位置的下载、不自动移动种子。
未启用目的地时继续使用原下载器 `save_path`，保持旧配置可用。旧任务不会自动绑定后来创建的目的地；没有可靠位置的任务核对为 `unchecked`。本阶段没有旧任务目的地补录或文件迁移接口。
```sh
curl -G 'http://127.0.0.1:8080/api/destinations/目的地ID/entries' --data-urlencode 'path=Series'
curl -X POST 'http://127.0.0.1:8080/api/jobs/任务ID/files/check'
curl 'http://127.0.0.1:8080/api/jobs/任务ID/files'
```
首次核对向原下载器读取文件清单，并核实其保存路径与快照一致。清单使用 [qBittorrent 文件接口](https://github.com/qbittorrent/qBittorrent/wiki/WebUI-API-%28qBittorrent-5.0%29#get-torrent-contents) 提供的相对文件名和大小，排除优先级为 0 的文件；不从发布标题猜文件名。
清单成功保存后，后续核对直接访问记录的本地路径，下载器离线也能执行。清单不会随下载器外部改名或移动而自动改写；本阶段不提供清单刷新或路径迁移。
核对由上述 POST 接口显式触发，后台下载同步暂不自动扫描文件系统。
| 状态 | 含义 |
| --- | --- |
| `unchecked` | 尚未核对，或未记录目的地、种子元数据/已选文件尚不可用 |
| `present` | 成功访问并找到所有记录的普通文件 |
| `absent` | 成功访问根目录，至少一个记录的文件不存在 |
| `unavailable` | 根目录不可访问、权限不足、路径越界、映射不匹配或首次读取下载器清单失败 |
汇总时 `unavailable` 优先于 `absent`；每个文件分别记录结果。响应含本次尝试时间 `checked_at` 和最后成功核对时间 `last_success_at`，失败不会抹去后者。
存在状态独立于下载状态；文件缺失不会触发重下载或删除。返回预期/观察大小供查看，`present` 不保证大小一致或内容完整，也不将下载标记为完成。
目录浏览最多返回 1000 项，达到上限时 `truncated=true`；单个任务清单上限 5000 项。不读取文件内容、不提供播放。
本地访问通过 [cap-std 的目录能力接口](https://docs.rs/cap-std/4.0.3/cap_std/fs/struct.Dir.html) 限定在配置根目录内。相对路径拒绝绝对地址、`..`、盘符和反斜杠；符号链接不能越出根目录。根目录若离线而仍保留可访问的空挂载点，本阶段不具备挂载状态识别能力。
P2 仅支持进程可访问的本地目录。SMB 应用直连属于 P3，尚未实现。
## 当前规则与恢复行为
- 支持常见 `[03]`、`- 03`、`EP03`、`第03话`、`S01E03` 以及 `03v2` 标题。
- 合集、分数集数、模糊或冲突的集数标记为 `needs_review`；目前可通过调整源或规则解决，尚无人工集数映射接口。
- RSS enclosure、Atom enclosure、磁力链接及 `.torrent` 链接可作为资源入口；尚未覆盖所有站点私有 XML 扩展。
- 支持 BitTorrent v1 磁力链接（十六进制或 Base32）及 v1/hybrid 种子；纯 v2 尚不支持。
- 同一订阅内按资源地址及“季度 + 集数”去重。相同集数有多个版本时采用 RSS 中先出现的条目，尚不自动洗版。
- 保存原始资源标题、地址、识别结果、过滤原因；种子文件按原始 info 字节计算 hash，并保存实际提交的种子数据。
- 下载前先持久化 hash 和提交意图，再调用下载器。超时或重启后先查询 qBittorrent，按 hash 核对，不盲目重复提交。
- 网络失败按 30 秒起、最多一小时的退避间隔重试；实际执行时间还受扫描周期影响。无效种子进入 `failed`，需手动重试。
- 已提交任务若从下载器消失，标记为 `missing`，不会自动重新下载。完成任务记录保留，不因再次扫描而重建。
- RSS/种子响应限制为 4 MiB，请求超时 30 秒。下载器 Cookie 与 RSS 请求使用不同客户端。
- 蜜柑 HTML 上限 8 MiB，封面上限 4 MiB；成功搜索/详情缓存 60 秒、最多 64 项，最多同时执行 4 个蜜柑请求，超限返回 `429`。HTTP 重定向最多 5 次。
- 无搜索结果返回空列表；页面无法识别、访问受限及上游失败返回 `502`，不会伪装成空列表。公开页面结构变化可能需要更新蜜柑适配器。
本地事务不能与下载器 API 组成原子事务。恢复依赖原下载器实例中的稳定 info hash 和下载器去重；同一数据库仅运行一个服务实例。使用全局选择接口切换不会改变恢复目标。
## 结构与验证
```text
src/api.rs HTTP 接口
src/domain.rs 订阅、资源、任务模型与输入校验
src/parser.rs 标题与集数识别
src/feeds.rs RSS/Atom 获取、解析、过滤
src/mikan.rs 蜜柑搜索、详情、订阅候选与封面
src/media.rs 本地文件访问、路径校验与媒体数据结构
src/media_store.rs 目的地、版本化映射、文件清单与核对记录持久化
src/storage.rs SQLite 持久化与去重
src/downloader.rs 种子标识与 qBittorrent 适配
src/service.rs 扫描、任务恢复与后台调度
migrations/ 数据库迁移
tests/fixtures/ 人工构造的 RSS 测试样本
```
```sh
cargo fmt --check
cargo test --locked
cargo clippy --locked --all-targets -- -D warnings
```
集成测试使用回环地址上的模拟蜜柑 / RSS / 代理 / qBittorrent 服务和临时 SQLite 数据库，不连接真实下载器、不下载媒体。
覆盖过滤、保守集数解析、重复扫描、数据库重开、提交成功但响应失败后的恢复、下载器不可用、禁用订阅、任务移除与 API CRUD。
新增覆盖中文搜索编码、不同季度、字幕组 ID、RSS 候选到保存、访问限制页面、封面与缓存、代理认证及跨域重定向、直连例外和代理失败处理。
下载器归属测试覆盖切换时已排队任务、同 hash 不同实例、单实例离线、重试、重启，以及真实旧版 SQLite 数据库升级后的显式归属恢复。
P2 使用临时本地文件与模拟下载器，覆盖目的地/映射版本快照、实际提交路径、同 hash 不同路径冲突、文件四态、成功核对时间保留、下载器离线核对、目录浏览与路径/符号链接限制。没有连接真实 NAS 或下载器。
已实现全局媒体目的地与本地文件存在核对。后续将支持 SMB 应用直连、第二种下载工具、MySQL、管理页面和部署。
## 许可证
本项目按 GNU General Public License version 2 only 发布，完整条款见 [LICENSE](LICENSE)。
新增 Rust 源文件使用 `SPDX-License-Identifier: GPL-2.0-only` 标识。第三方依赖遵循各自许可证。
migrations/0001_initial.sql
+34
View File
@@ -0,0 +1,34 @@
CREATE TABLE subscriptions (
id TEXT PRIMARY KEY,
spec TEXT NOT NULL,
deleted INTEGER NOT NULL DEFAULT 0,
last_checked_at INTEGER,
last_error TEXT
);
CREATE TABLE releases (
subscription_id TEXT NOT NULL REFERENCES subscriptions(id),
resource_key TEXT NOT NULL,
payload TEXT NOT NULL,
PRIMARY KEY (subscription_id, resource_key)
);
CREATE TABLE download_jobs (
id TEXT PRIMARY KEY,
subscription_id TEXT NOT NULL REFERENCES subscriptions(id),
resource_key TEXT NOT NULL,
title TEXT NOT NULL,
season INTEGER NOT NULL,
episode INTEGER NOT NULL,
download_url TEXT NOT NULL,
torrent_bytes BLOB,
info_hash TEXT,
state TEXT NOT NULL DEFAULT 'queued'
CHECK (state IN ('queued', 'submitting', 'submitted', 'downloading', 'completed', 'missing', 'failed')),
attempts INTEGER NOT NULL DEFAULT 0,
last_error TEXT,
next_attempt_at INTEGER NOT NULL DEFAULT 0,
UNIQUE(subscription_id, resource_key),
UNIQUE(subscription_id, season, episode)
);
CREATE INDEX jobs_state ON download_jobs(state, next_attempt_at);
migrations/0002_downloader_instances.sql
+18
View File
@@ -0,0 +1,18 @@
CREATE TABLE downloader_instances (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
kind TEXT NOT NULL CHECK (kind = 'qbittorrent'),
endpoint TEXT NOT NULL UNIQUE,
config TEXT NOT NULL
);
CREATE TABLE downloader_selection (
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
downloader_id TEXT REFERENCES downloader_instances(id)
);
INSERT INTO downloader_selection(singleton) VALUES (1);
ALTER TABLE download_jobs ADD COLUMN downloader_id TEXT REFERENCES downloader_instances(id);
ALTER TABLE download_jobs ADD COLUMN legacy_unassigned INTEGER NOT NULL DEFAULT 0;
-- Existing jobs have no recorded owner. Do not assign them to an arbitrary new instance.
UPDATE download_jobs SET legacy_unassigned = 1;
CREATE INDEX jobs_downloader ON download_jobs(downloader_id, state);
migrations/0003_media_destinations.sql
+37
View File
@@ -0,0 +1,37 @@
CREATE TABLE media_destinations (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
kind TEXT NOT NULL CHECK (kind = 'local'),
root_path TEXT NOT NULL
);
CREATE TABLE destination_selection (
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
destination_id TEXT REFERENCES media_destinations(id)
);
INSERT INTO destination_selection(singleton) VALUES (1);
CREATE TABLE destination_mappings (
id TEXT PRIMARY KEY,
destination_id TEXT NOT NULL REFERENCES media_destinations(id),
downloader_id TEXT NOT NULL REFERENCES downloader_instances(id),
version INTEGER NOT NULL,
download_root TEXT NOT NULL,
UNIQUE(destination_id, downloader_id, version)
);
ALTER TABLE download_jobs ADD COLUMN destination_id TEXT REFERENCES media_destinations(id);
ALTER TABLE download_jobs ADD COLUMN mapping_id TEXT REFERENCES destination_mappings(id);
ALTER TABLE download_jobs ADD COLUMN download_save_path TEXT;
ALTER TABLE download_jobs ADD COLUMN file_status TEXT NOT NULL DEFAULT 'unchecked'
CHECK (file_status IN ('unchecked', 'present', 'absent', 'unavailable'));
ALTER TABLE download_jobs ADD COLUMN file_checked_at INTEGER;
ALTER TABLE download_jobs ADD COLUMN file_last_success_at INTEGER;
ALTER TABLE download_jobs ADD COLUMN file_error TEXT;
CREATE TABLE media_files (
job_id TEXT NOT NULL REFERENCES download_jobs(id),
relative_path TEXT NOT NULL,
expected_size INTEGER NOT NULL,
observed_size INTEGER,
status TEXT NOT NULL DEFAULT 'unchecked'
CHECK (status IN ('unchecked', 'present', 'absent', 'unavailable')),
error TEXT,
PRIMARY KEY(job_id, relative_path)
);
src/api.rs
+248
View File
@@ -0,0 +1,248 @@
// SPDX-License-Identifier: GPL-2.0-only
use crate::{
domain::{DownloaderInput, SubscriptionInput},
error::{Error, Result},
media::{self, DestinationInput, MappingInput},
service::Service,
};
use actix_web::{web, HttpResponse};
use std::sync::Arc;
type State = web::Data<Arc<Service>>;
pub fn configure(cfg: &mut web::ServiceConfig) {
cfg.service(
web::scope("/api")
.route("/sources/mikan/search", web::get().to(mikan_search))
.route("/sources/mikan/series/{id}", web::get().to(mikan_series))
.route(
"/sources/mikan/series/{id}/cover",
web::get().to(mikan_cover),
)
.route("/sources/mikan/resolve", web::post().to(mikan_resolve))
.route("/status", web::get().to(status))
.route("/destinations", web::get().to(destinations))
.route("/destinations", web::post().to(create_destination))
.route("/destinations/active", web::put().to(activate_destination))
.route("/destinations/{id}/mappings", web::get().to(mappings))
.route(
"/destinations/{id}/mappings",
web::post().to(create_mapping),
)
.route(
"/destinations/{id}/entries",
web::get().to(directory_entries),
)
.route("/downloaders", web::get().to(downloaders))
.route("/downloaders", web::post().to(create_downloader))
.route("/downloaders/active", web::put().to(activate_downloader))
.route(
"/downloaders/{id}/claim-legacy-jobs",
web::post().to(claim_legacy_jobs),
)
.route("/preview", web::post().to(preview))
.route("/subscriptions", web::get().to(list))
.route("/subscriptions", web::post().to(create))
.route("/subscriptions/{id}", web::get().to(get))
.route("/subscriptions/{id}", web::put().to(update))
.route("/subscriptions/{id}", web::delete().to(delete))
.route("/subscriptions/{id}/preview", web::get().to(preview_saved))
.route("/subscriptions/{id}/scan", web::post().to(scan))
.route("/subscriptions/{id}/releases", web::get().to(releases))
.route("/jobs", web::get().to(jobs))
.route("/jobs/{id}/files", web::get().to(job_files))
.route("/jobs/{id}/files/check", web::post().to(check_job_files))
.route("/jobs/{id}/retry", web::post().to(retry))
.route("/run", web::post().to(run)),
);
}
#[derive(serde::Deserialize)]
struct SearchQuery {
q: String,
}
async fn mikan_search(state: State, query: web::Query<SearchQuery>) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(state.mikan.search(&query.q).await?))
}
async fn mikan_series(state: State, id: web::Path<String>) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(state.mikan.series(&id).await?))
}
async fn mikan_cover(state: State, id: web::Path<String>) -> Result<HttpResponse> {
let (mime, bytes) = state.mikan.cover(&id).await?;
Ok(HttpResponse::Ok()
.content_type(mime)
.insert_header(("X-Content-Type-Options", "nosniff"))
.insert_header(("Cache-Control", "private, max-age=60"))
.body(bytes))
}
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct ResolveInput {
url: String,
}
async fn mikan_resolve(state: State, body: web::Json<ResolveInput>) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(state.mikan.resolve(&body.url).await?))
}
async fn status(state: State) -> Result<HttpResponse> {
let active = state.store.active_downloader_id().await?;
Ok(HttpResponse::Ok().json(serde_json::json!({
"downloader_configured": active.is_some(),
"active_downloader_id": active,
"active_destination_id": state.store.active_destination_id().await?,
"legacy_unassigned_jobs": state.store.legacy_unassigned_count().await?,
"version": env!("CARGO_PKG_VERSION")
})))
}
async fn destinations(state: State) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(state.store.destinations().await?))
}
async fn create_destination(
state: State,
body: web::Json<DestinationInput>,
) -> Result<HttpResponse> {
let _guard = state.gate.try_lock().map_err(|_| Error::Busy)?;
Ok(HttpResponse::Created().json(state.store.create_destination(body.into_inner()).await?))
}
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct DestinationSelection {
destination_id: String,
}
async fn activate_destination(
state: State,
body: web::Json<DestinationSelection>,
) -> Result<HttpResponse> {
let _guard = state.gate.try_lock().map_err(|_| Error::Busy)?;
state
.store
.activate_destination(&body.destination_id)
.await?;
Ok(HttpResponse::NoContent().finish())
}
async fn mappings(state: State, id: web::Path<String>) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(state.store.mappings(&id).await?))
}
async fn create_mapping(
state: State,
id: web::Path<String>,
body: web::Json<MappingInput>,
) -> Result<HttpResponse> {
let _guard = state.gate.try_lock().map_err(|_| Error::Busy)?;
Ok(HttpResponse::Created().json(state.store.create_mapping(&id, body.into_inner()).await?))
}
#[derive(serde::Deserialize)]
struct DirectoryQuery {
#[serde(default)]
path: String,
}
async fn directory_entries(
state: State,
id: web::Path<String>,
query: web::Query<DirectoryQuery>,
) -> Result<HttpResponse> {
media::validate_relative(&query.path, true)?;
let root = state.store.destination(&id).await?.root_path;
let path = query.into_inner().path;
let entries = tokio::task::spawn_blocking(move || media::list_local(&root, &path))
.await
.map_err(|_| Error::Upstream("local directory query failed".into()))?;
Ok(HttpResponse::Ok().json(entries))
}
async fn job_files(state: State, id: web::Path<String>) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(state.store.file_report(&id).await?))
}
async fn check_job_files(state: State, id: web::Path<String>) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(state.check_job_files(&id).await?))
}
async fn downloaders(state: State) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(state.store.downloaders().await?))
}
async fn create_downloader(state: State, body: web::Json<DownloaderInput>) -> Result<HttpResponse> {
let _guard = state.gate.try_lock().map_err(|_| Error::Busy)?;
Ok(HttpResponse::Created().json(state.store.create_downloader(body.into_inner()).await?))
}
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct DownloaderSelection {
downloader_id: String,
}
async fn activate_downloader(
state: State,
body: web::Json<DownloaderSelection>,
) -> Result<HttpResponse> {
let _guard = state.gate.try_lock().map_err(|_| Error::Busy)?;
state.store.activate_downloader(&body.downloader_id).await?;
Ok(HttpResponse::NoContent().finish())
}
async fn claim_legacy_jobs(state: State, id: web::Path<String>) -> Result<HttpResponse> {
let _guard = state.gate.try_lock().map_err(|_| Error::Busy)?;
Ok(HttpResponse::Ok()
.json(serde_json::json!({"claimed": state.store.claim_legacy_jobs(&id).await?})))
}
async fn list(state: State) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(state.store.subscriptions().await?))
}
async fn get(state: State, id: web::Path<String>) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(state.store.subscription(&id).await?))
}
async fn create(state: State, body: web::Json<SubscriptionInput>) -> Result<HttpResponse> {
let _guard = state.gate.try_lock().map_err(|_| Error::Busy)?;
Ok(HttpResponse::Created().json(state.store.create(body.into_inner()).await?))
}
async fn update(
state: State,
id: web::Path<String>,
body: web::Json<SubscriptionInput>,
) -> Result<HttpResponse> {
let _guard = state.gate.try_lock().map_err(|_| Error::Busy)?;
Ok(HttpResponse::Ok().json(state.store.update(&id, body.into_inner()).await?))
}
async fn delete(state: State, id: web::Path<String>) -> Result<HttpResponse> {
let _guard = state.gate.try_lock().map_err(|_| Error::Busy)?;
state.store.delete(&id).await?;
Ok(HttpResponse::NoContent().finish())
}
async fn preview(state: State, body: web::Json<SubscriptionInput>) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(state.preview(body.into_inner()).await?))
}
async fn preview_saved(state: State, id: web::Path<String>) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(state.preview_subscription(&id).await?))
}
async fn scan(state: State, id: web::Path<String>) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(serde_json::json!({"queued": state.scan_subscription(&id).await?})))
}
async fn releases(state: State, id: web::Path<String>) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(state.store.releases(&id).await?))
}
async fn jobs(state: State) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(state.store.jobs().await?))
}
async fn retry(state: State, id: web::Path<String>) -> Result<HttpResponse> {
let _guard = state.gate.try_lock().map_err(|_| Error::Busy)?;
state.store.retry(&id).await?;
Ok(HttpResponse::NoContent().finish())
}
async fn run(state: State) -> Result<HttpResponse> {
Ok(HttpResponse::Ok().json(state.cycle().await?))
}
src/config.rs
+106
View File
@@ -0,0 +1,106 @@
// SPDX-License-Identifier: GPL-2.0-only
use crate::error::{Error, Result};
use std::{env, time::Duration};
#[derive(Clone)]
pub struct Config {
pub database_url: String,
pub port: u16,
pub poll_interval: Duration,
pub qbit: Option<QbitConfig>,
pub mikan_base_url: String,
pub website: WebsiteConfig,
}
#[derive(Clone, Default)]
pub struct WebsiteConfig {
pub proxy_url: Option<String>,
pub direct_hosts: String,
}
// Deliberately omit Debug: these fields can contain credentials.
#[derive(Clone, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct QbitConfig {
pub url: String,
pub username: String,
pub password: String,
pub save_path: String,
}
impl QbitConfig {
pub fn validate(&mut self) -> Result<()> {
self.url = self.url.trim().trim_end_matches('/').to_owned();
crate::domain::validate_http_url(&self.url)?;
let url = reqwest::Url::parse(&self.url).expect("validated URL");
if url.query().is_some() || url.fragment().is_some() {
return Err(Error::Invalid(
"downloader URL cannot contain query parameters or a fragment".into(),
));
}
self.url = url.to_string().trim_end_matches('/').to_owned();
if self.username.len() > 1024
|| self.password.len() > 4096
|| self.save_path.trim().is_empty()
|| self.save_path.len() > 4096
|| self.save_path.chars().any(char::is_control)
{
return Err(Error::Invalid(
"invalid downloader credentials or save path".into(),
));
}
Ok(())
}
}
impl Config {
pub fn from_env() -> Result<Self> {
let port = env::var("APP_PORT")
.unwrap_or_else(|_| "8080".into())
.parse()
.map_err(|_| Error::Invalid("APP_PORT must be a valid port".into()))?;
let seconds: u64 = env::var("POLL_INTERVAL_SECONDS")
.unwrap_or_else(|_| "300".into())
.parse()
.map_err(|_| Error::Invalid("POLL_INTERVAL_SECONDS must be an integer".into()))?;
if seconds < 10 {
return Err(Error::Invalid(
"POLL_INTERVAL_SECONDS must be at least 10".into(),
));
}
let qbit = match env::var("QBIT_URL") {
Ok(url) if !url.is_empty() => {
crate::domain::validate_http_url(&url)?;
let base = reqwest::Url::parse(&url).expect("validated URL");
if base.query().is_some() || base.fragment().is_some() {
return Err(Error::Invalid(
"QBIT_URL cannot contain query parameters or a fragment".into(),
));
}
Some(QbitConfig {
url: url.trim_end_matches('/').into(),
username: env::var("QBIT_USERNAME").unwrap_or_default(),
password: env::var("QBIT_PASSWORD").unwrap_or_default(),
save_path: env::var("QBIT_SAVE_PATH").unwrap_or_else(|_| "/downloads".into()),
})
}
_ => None,
};
Ok(Self {
database_url: env::var("DATABASE_URL")
.unwrap_or_else(|_| "sqlite://data/ani.db".into()),
port,
poll_interval: Duration::from_secs(seconds),
qbit,
mikan_base_url: env::var("MIKAN_BASE_URL")
.unwrap_or_else(|_| "https://mikanani.me".into()),
website: WebsiteConfig {
proxy_url: env::var("WEBSITE_PROXY_URL")
.ok()
.map(|s| s.trim().to_owned())
.filter(|s| !s.is_empty()),
direct_hosts: env::var("WEBSITE_DIRECT_HOSTS").unwrap_or_default(),
},
})
}
}
src/domain.rs
+160
View File
@@ -0,0 +1,160 @@
// SPDX-License-Identifier: GPL-2.0-only
use crate::error::{Error, Result};
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct SubscriptionInput {
pub title: String,
pub rss_url: String,
#[serde(default = "one")]
pub season: i32,
#[serde(default)]
pub episode_offset: i32,
#[serde(default)]
pub include: Vec<String>,
#[serde(default)]
pub exclude: Vec<String>,
// Opt in after previewing: the first scan considers all entries in the feed.
#[serde(default)]
pub enabled: bool,
}
fn one() -> i32 {
1
}
impl SubscriptionInput {
pub fn validate(&mut self) -> Result<()> {
self.title = self.title.trim().to_owned();
self.rss_url = self.rss_url.trim().to_owned();
if self.title.is_empty() || self.title.chars().count() > 200 {
return Err(Error::Invalid("title must contain 1–200 characters".into()));
}
validate_http_url(&self.rss_url)?;
if !(1..=999).contains(&self.season) || self.episode_offset.abs_diff(0) > 10000 {
return Err(Error::Invalid(
"season or episode_offset is out of range".into(),
));
}
for rules in [&mut self.include, &mut self.exclude] {
if rules.len() > 32 {
return Err(Error::Invalid("at most 32 rules per list".into()));
}
for rule in rules {
*rule = rule.trim().to_owned();
if rule.is_empty() || rule.len() > 200 {
return Err(Error::Invalid(
"filter keywords must contain 1–200 bytes".into(),
));
}
}
}
Ok(())
}
}
pub fn validate_http_url(value: &str) -> Result<()> {
let url = reqwest::Url::parse(value).map_err(|_| Error::Invalid("invalid URL".into()))?;
if !matches!(url.scheme(), "http" | "https")
|| url.host_str().is_none()
|| !url.username().is_empty()
|| url.password().is_some()
|| value.len() > 8192
{
return Err(Error::Invalid(
"expected an HTTP(S) URL without embedded credentials".into(),
));
}
Ok(())
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Subscription {
pub id: String,
#[serde(flatten)]
pub input: SubscriptionInput,
pub last_checked_at: Option<i64>,
pub last_error: Option<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum Decision {
Ready,
Filtered,
NeedsReview,
AlreadyQueued,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Release {
pub key: String,
pub title: String,
pub download_url: Option<String>,
pub episode: Option<i32>,
pub decision: Decision,
pub reason: String,
}
#[derive(Clone, Debug, Serialize, sqlx::FromRow)]
pub struct DownloadJob {
pub id: String,
pub subscription_id: String,
pub title: String,
pub season: i32,
pub episode: i32,
pub download_url: String,
pub info_hash: Option<String>,
pub state: String,
pub attempts: i32,
pub last_error: Option<String>,
pub next_attempt_at: i64,
pub downloader_id: Option<String>,
pub legacy_unassigned: bool,
pub destination_id: Option<String>,
pub mapping_id: Option<String>,
pub download_save_path: Option<String>,
pub file_status: String,
pub file_checked_at: Option<i64>,
pub file_last_success_at: Option<i64>,
pub file_error: Option<String>,
}
#[derive(Clone, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct DownloaderInput {
pub name: String,
pub kind: DownloaderKind,
pub connection: crate::config::QbitConfig,
}
#[derive(Clone, serde::Deserialize, serde::Serialize)]
#[serde(rename_all = "lowercase")]
pub enum DownloaderKind {
Qbittorrent,
}
impl DownloaderInput {
pub fn validate(&mut self) -> Result<()> {
self.name = self.name.trim().to_owned();
if self.name.is_empty() || self.name.chars().count() > 100 {
return Err(Error::Invalid(
"downloader name must contain 1–100 characters".into(),
));
}
self.connection.validate()
}
}
#[derive(serde::Serialize)]
pub struct DownloaderView {
pub id: String,
pub name: String,
pub kind: DownloaderKind,
pub url: String,
pub username: String,
pub save_path: String,
pub password_configured: bool,
pub is_active: bool,
}
src/downloader.rs
+265
View File
@@ -0,0 +1,265 @@
// SPDX-License-Identifier: GPL-2.0-only
use crate::{
config::QbitConfig,
domain::validate_http_url,
error::{Error, Result},
feeds::bounded_body,
};
use bendy::decoding::{Decoder, Object};
use reqwest::{multipart, Client, Url};
use serde::Deserialize;
use sha1::{Digest, Sha1};
use std::time::Duration;
pub struct PreparedTorrent {
pub hash: String,
pub bytes: Option<Vec<u8>>,
}
pub fn magnet_hash(value: &str) -> Result<String> {
let url = Url::parse(value).map_err(|_| Error::Invalid("invalid magnet URI".into()))?;
if url.scheme() != "magnet" {
return Err(Error::Invalid("expected magnet URI".into()));
}
for (key, value) in url.query_pairs() {
if key != "xt" {
continue;
}
if let Some(hash) = value.strip_prefix("urn:btih:") {
if hash.len() == 40 && hash.bytes().all(|b| b.is_ascii_hexdigit()) {
return Ok(hash.to_ascii_lowercase());
}
if hash.len() == 32 {
if let Ok(bytes) =
data_encoding::BASE32.decode(hash.to_ascii_uppercase().as_bytes())
{
return Ok(data_encoding::HEXLOWER.encode(&bytes));
}
}
}
}
Err(Error::Invalid(
"magnet must contain a valid BitTorrent v1 info hash".into(),
))
}
pub fn torrent_hash(bytes: &[u8]) -> Result<String> {
let invalid = |_| Error::Invalid("invalid or unsupported torrent metadata".into());
let mut decoder = Decoder::new(bytes).with_max_depth(32);
let mut root = decoder
.next_object()
.map_err(invalid)?
.ok_or_else(|| Error::Invalid("empty torrent metadata".into()))?
.try_into_dictionary()
.map_err(invalid)?;
let mut raw_info = None;
while let Some((key, object)) = root.next_pair().map_err(invalid)? {
if key == b"info" {
raw_info = Some(
object
.try_into_dictionary()
.map_err(invalid)?
.into_raw()
.map_err(invalid)?,
);
}
}
drop(root);
if decoder.next_object().map_err(invalid)?.is_some() {
return Err(Error::Invalid("trailing torrent data".into()));
}
let raw =
raw_info.ok_or_else(|| Error::Invalid("torrent is missing info dictionary".into()))?;
let mut info_decoder = Decoder::new(raw).with_max_depth(32);
let mut info = info_decoder
.next_object()
.map_err(invalid)?
.unwrap()
.try_into_dictionary()
.map_err(invalid)?;
let mut v1 = false;
while let Some((key, object)) = info.next_pair().map_err(invalid)? {
if let (b"pieces", Object::Bytes(pieces)) = (key, object) {
v1 = !pieces.is_empty() && pieces.len() % 20 == 0;
}
}
if !v1 {
return Err(Error::Invalid(
"only BitTorrent v1/hybrid torrents are supported".into(),
));
}
// Hash the exact encoded bytes, never a decoded/re-encoded dictionary.
Ok(format!("{:x}", Sha1::digest(raw)))
}
pub async fn prepare(client: &Client, url: &str) -> Result<PreparedTorrent> {
if url.starts_with("magnet:") {
return Ok(PreparedTorrent {
hash: magnet_hash(url)?,
bytes: None,
});
}
validate_http_url(url)?;
let response =
crate::feeds::website_get(client, Url::parse(url).expect("validated URL")).await?;
let bytes = bounded_body(response, 4 * 1024 * 1024).await?;
Ok(PreparedTorrent {
hash: torrent_hash(&bytes)?,
bytes: Some(bytes),
})
}
#[derive(Clone)]
pub struct Qbit {
client: Client,
config: QbitConfig,
}
#[derive(Deserialize)]
pub struct TorrentStatus {
pub hash: String,
pub progress: f64,
pub state: String,
#[serde(default)]
pub save_path: Option<String>,
}
#[derive(Deserialize)]
pub struct TorrentFile {
pub name: String,
pub size: i64,
pub priority: i32,
}
impl Qbit {
pub fn new(config: QbitConfig) -> Result<Self> {
// Keep downloader cookies in a separate client from untrusted RSS hosts.
let client = Client::builder()
.no_proxy()
.cookie_store(true)
.timeout(Duration::from_secs(30))
.redirect(reqwest::redirect::Policy::none())
.build()?;
Ok(Self { client, config })
}
pub async fn login(&self) -> Result<()> {
let response = self
.client
.post(format!("{}/api/v2/auth/login", self.config.url))
.header("Referer", format!("{}/", self.config.url))
.form(&[
("username", &self.config.username),
("password", &self.config.password),
])
.send()
.await?;
let bytes = bounded_body(response, 1024).await?;
if bytes != b"Ok." {
return Err(Error::Upstream("qBittorrent login failed".into()));
}
Ok(())
}
pub async fn torrents(&self) -> Result<Vec<TorrentStatus>> {
let response = self
.client
.get(format!("{}/api/v2/torrents/info", self.config.url))
.header("Referer", format!("{}/", self.config.url))
.send()
.await?;
let bytes = bounded_body(response, 8 * 1024 * 1024).await?;
serde_json::from_slice(&bytes)
.map_err(|_| Error::Upstream("invalid qBittorrent task response".into()))
}
pub async fn files(&self, hash: &str) -> Result<Vec<TorrentFile>> {
let response = self
.client
.get(format!("{}/api/v2/torrents/files", self.config.url))
.header("Referer", format!("{}/", self.config.url))
.query(&[("hash", hash)])
.send()
.await?;
let bytes = bounded_body(response, 8 * 1024 * 1024).await?;
let files: Vec<TorrentFile> = serde_json::from_slice(&bytes)
.map_err(|_| Error::Upstream("invalid qBittorrent file list".into()))?;
if files.len() > 5000 || files.iter().any(|file| file.size < 0) {
return Err(Error::Upstream(
"qBittorrent file list exceeds limits or contains invalid sizes".into(),
));
}
Ok(files)
}
pub async fn add(
&self,
url: &str,
bytes: Option<Vec<u8>>,
save_path: Option<&str>,
) -> Result<()> {
let mut form = multipart::Form::new()
.text(
"savepath",
save_path.unwrap_or(&self.config.save_path).to_owned(),
)
.text("tags", "rust-project")
.text("autoTMM", "false");
form = if let Some(bytes) = bytes {
form.part(
"torrents",
multipart::Part::bytes(bytes)
.file_name("release.torrent")
.mime_str("application/x-bittorrent")?,
)
} else {
form.text("urls", url.to_owned())
};
let response = self
.client
.post(format!("{}/api/v2/torrents/add", self.config.url))
.header("Referer", format!("{}/", self.config.url))
.multipart(form)
.send()
.await?;
let bytes = bounded_body(response, 1024).await?;
if bytes != b"Ok." {
return Err(Error::Upstream("qBittorrent rejected torrent".into()));
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn handles_hex_and_base32_magnets() {
assert_eq!(
magnet_hash("magnet:?xt=urn:btih:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA").unwrap(),
"a".repeat(40)
);
assert_eq!(
magnet_hash("magnet:?xt=urn:btih:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA").unwrap(),
"0".repeat(40)
);
assert!(magnet_hash("magnet:?xt=urn:btih:bad").is_err());
}
#[test]
fn hashes_original_info_bytes_and_rejects_bad_metadata() {
let info = b"d6:lengthi1e4:name1:a12:piece lengthi16384e6:pieces20:aaaaaaaaaaaaaaaaaaaae";
let torrent = [b"d4:info".as_slice(), info, b"e"].concat();
assert_eq!(
torrent_hash(&torrent).unwrap(),
format!("{:x}", Sha1::digest(info))
);
for bad in [
b"<html>error</html>".as_slice(),
b"d4:infodee",
b"d4:info",
&[torrent.as_slice(), b"junk"].concat(),
] {
assert!(torrent_hash(bad).is_err());
}
}
}
src/error.rs
+47
View File
@@ -0,0 +1,47 @@
// SPDX-License-Identifier: GPL-2.0-only
use actix_web::{http::StatusCode, HttpResponse, ResponseError};
pub type Result<T> = std::result::Result<T, Error>;
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("{0}")]
Invalid(String),
#[error("resource not found")]
NotFound,
#[error("a scan or download synchronization is already running")]
Busy,
#[error("too many Mikan requests; retry shortly")]
SourceBusy,
#[error("{0}")]
Upstream(String),
#[error("database operation failed")]
Database(#[from] sqlx::Error),
#[error("stored data could not be decoded")]
Json(#[from] serde_json::Error),
#[error("database migration failed")]
Migration(#[from] sqlx::migrate::MigrateError),
}
impl From<reqwest::Error> for Error {
fn from(value: reqwest::Error) -> Self {
Self::Upstream(value.without_url().to_string())
}
}
impl ResponseError for Error {
fn status_code(&self) -> StatusCode {
match self {
Self::Invalid(_) => StatusCode::BAD_REQUEST,
Self::NotFound => StatusCode::NOT_FOUND,
Self::Busy => StatusCode::CONFLICT,
Self::SourceBusy => StatusCode::TOO_MANY_REQUESTS,
Self::Upstream(_) => StatusCode::BAD_GATEWAY,
_ => StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
HttpResponse::build(self.status_code()).json(serde_json::json!({"error": self.to_string()}))
}
}
src/feeds.rs
+183
View File
@@ -0,0 +1,183 @@
// SPDX-License-Identifier: GPL-2.0-only
use crate::{
domain::*,
error::{Error, Result},
parser,
};
use reqwest::{Client, Url};
use sha1::{Digest, Sha1};
use std::time::Duration;
pub fn http_client() -> Result<Client> {
website_client(&crate::config::WebsiteConfig::default())
}
pub fn website_client(config: &crate::config::WebsiteConfig) -> Result<Client> {
let mut builder = Client::builder()
.no_proxy()
.timeout(Duration::from_secs(30))
.connect_timeout(Duration::from_secs(10))
.user_agent(concat!("rust_project/", env!("CARGO_PKG_VERSION")))
// Rebuild each redirected request so proxy auth and direct-host rules are reapplied.
.redirect(reqwest::redirect::Policy::none());
if let Some(value) = &config.proxy_url {
let invalid = || {
Error::Invalid(
"WEBSITE_PROXY_URL must be an HTTP(S), SOCKS5 or SOCKS5H proxy URL".into(),
)
};
let url = Url::parse(value).map_err(|_| invalid())?;
if !matches!(url.scheme(), "http" | "https" | "socks5" | "socks5h")
|| url.host_str().is_none()
|| url.query().is_some()
|| url.fragment().is_some()
|| !matches!(url.path(), "" | "/")
{
return Err(invalid());
}
let proxy = reqwest::Proxy::all(url)
.map_err(|_| invalid())?
.no_proxy(reqwest::NoProxy::from_string(&config.direct_hosts));
builder = builder.proxy(proxy);
}
Ok(builder.build()?)
}
pub async fn website_get(client: &Client, mut url: Url) -> Result<reqwest::Response> {
tokio::time::timeout(Duration::from_secs(30), async {
for hop in 0..=5 {
validate_http_url(url.as_str())?;
let response = client.get(url.clone()).send().await?;
if !matches!(response.status().as_u16(), 301 | 302 | 303 | 307 | 308) {
return Ok(response);
}
if hop == 5 {
return Err(Error::Upstream("website redirect limit exceeded".into()));
}
let location = response
.headers()
.get(reqwest::header::LOCATION)
.and_then(|v| v.to_str().ok())
.ok_or_else(|| Error::Upstream("invalid website redirect".into()))?;
url = url
.join(location)
.map_err(|_| Error::Upstream("invalid website redirect URL".into()))?;
validate_http_url(url.as_str())
.map_err(|_| Error::Upstream("unsupported website redirect URL".into()))?;
}
unreachable!("redirect count is bounded")
})
.await
.map_err(|_| Error::Upstream("website request timed out".into()))?
}
pub async fn bounded_body(mut response: reqwest::Response, limit: usize) -> Result<Vec<u8>> {
response = response.error_for_status()?;
if response.content_length().is_some_and(|n| n > limit as u64) {
return Err(Error::Upstream("response exceeds size limit".into()));
}
let mut data = Vec::new();
while let Some(chunk) = response.chunk().await? {
if data.len() + chunk.len() > limit {
return Err(Error::Upstream("response exceeds size limit".into()));
}
data.extend_from_slice(&chunk);
}
Ok(data)
}
pub async fn fetch(client: &Client, sub: &SubscriptionInput) -> Result<Vec<Release>> {
let response = website_get(
client,
Url::parse(&sub.rss_url).map_err(|_| Error::Invalid("invalid RSS URL".into()))?,
)
.await?;
let base = response.url().clone();
let bytes = bounded_body(response, 4 * 1024 * 1024).await?;
parse(&bytes, &base, sub)
}
pub fn parse(bytes: &[u8], base: &Url, sub: &SubscriptionInput) -> Result<Vec<Release>> {
let feed = feed_rs::parser::Builder::new()
.base_uri(Some(base.as_str()))
.build()
.parse(bytes)
.map_err(|_| Error::Upstream("invalid RSS/Atom feed".into()))?;
if feed.entries.len() > 2000 {
return Err(Error::Upstream("feed exceeds 2000 entries".into()));
}
Ok(feed
.entries
.into_iter()
.map(|entry| {
let title = entry.title.map(|t| t.content).unwrap_or_default();
let media = entry
.media
.iter()
.flat_map(|m| m.content.iter())
.filter_map(|c| c.url.as_ref().map(|u| u.as_str()));
let links = entry
.links
.iter()
.filter(|l| {
l.rel.as_deref() == Some("enclosure")
|| l.media_type.as_deref() == Some("application/x-bittorrent")
|| l.href.starts_with("magnet:")
|| Url::parse(&l.href)
.ok()
.is_some_and(|u| u.path().ends_with(".torrent"))
})
.map(|l| l.href.as_str());
let download_url = media.chain(links).find_map(|link| {
let url = base.join(link).ok()?;
match url.scheme() {
"magnet" => Some(url.to_string()),
"http" | "https" if validate_http_url(url.as_str()).is_ok() => {
Some(url.to_string())
}
_ => None,
}
});
let key = format!(
"{:x}",
Sha1::digest(download_url.as_deref().unwrap_or(&entry.id).as_bytes())
);
let mut release = Release {
key,
title,
download_url,
episode: None,
decision: Decision::Ready,
reason: "matched subscription rules".into(),
};
let lower = release.title.to_lowercase();
if sub
.exclude
.iter()
.any(|v| lower.contains(&v.to_lowercase()))
{
release.decision = Decision::Filtered;
release.reason = "matched an exclusion keyword".into();
} else if !sub
.include
.iter()
.all(|v| lower.contains(&v.to_lowercase()))
{
release.decision = Decision::Filtered;
release.reason = "missing a required inclusion keyword".into();
} else if release.download_url.is_none() {
release.decision = Decision::NeedsReview;
release.reason = "no supported torrent enclosure or magnet link".into();
} else {
match parser::episode(&release.title, sub.season, sub.episode_offset) {
Ok(ep) => release.episode = Some(ep),
Err(reason) => {
release.decision = Decision::NeedsReview;
release.reason = reason.into();
}
}
}
release
})
.collect())
}
src/lib.rs
+13
View File
@@ -0,0 +1,13 @@
// SPDX-License-Identifier: GPL-2.0-only
pub mod api;
pub mod config;
pub mod domain;
pub mod downloader;
pub mod error;
pub mod feeds;
pub mod media;
mod media_store;
pub mod mikan;
pub mod parser;
pub mod service;
pub mod storage;
src/main.rs
+148 -41
View File
@@ -1,45 +1,152 @@
use std::io; // SPDX-License-Identifier: GPL-2.0-only
use actix_web::{web, App, HttpResponse, HttpServer, Responder};
use rust_project::{
api,
config::Config,
service::{worker, Service},
storage::Store,
};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
fn main() { #[derive(Debug, Serialize, Deserialize)]
println!("欢迎来到增强版 Rust 程序！"); struct Greeting {
message: String,
// 演示变量声明和类型d name: String,
let name = "Rustacean"; // Rustacean 是 Rust 开发者的昵称 }
let age: u32 = 10;
#[derive(Debug, Serialize, Deserialize)]
println!("你好，{}！你使用 Rust 已经 {} 年了。", name, age); struct Health {
status: String,
// 演示数组和循环 version: String,
let numbers = [1, 2, 3, 4, 5]; }
println!("\n数字列表:");
for (index, &value) in numbers.iter().enumerate() { #[derive(Debug, Serialize, Deserialize)]
println!("索引 {}: 值 {}", index, value); struct CalcRequest {
} a: f64,
b: f64,
// 计算数组元素总和 operation: String,
let sum: i32 = numbers.iter().sum(); }
println!("\n数组元素总和: {}", sum);
#[derive(Debug, Serialize, Deserialize)]
// 条件判断 struct CalcResponse {
if sum > 10 { result: f64,
println!("总和大于 10"); operation: String,
} else { }
println!("总和小于或等于 10");
} async fn hello() -> impl Responder {
HttpResponse::Ok().body("Hello World! 欢迎使用 Rust Web 服务器")
// 与用户交互 }
println!("\n请输入您的名字:");
let mut input = String::new(); async fn health_check() -> impl Responder {
let health = Health {
match io::stdin().read_line(&mut input) { status: "ok".to_string(),
Ok(_) => { version: env!("CARGO_PKG_VERSION").to_string(),
let input = input.trim(); // 移除换行符 };
if !input.is_empty() { HttpResponse::Ok().json(health)
println!("很高兴认识您，{}！", input); }
} else {
println!("您没有输入任何内容。"); async fn greet(name: web::Path<String>) -> impl Responder {
let greeting = Greeting {
message: format!("你好，{}！", name),
name: name.into_inner(),
};
HttpResponse::Ok().json(greeting)
}
async fn calculate(web::Json(req): web::Json<CalcRequest>) -> impl Responder {
let result = match req.operation.as_str() {
"add" => req.a + req.b,
"subtract" => req.a - req.b,
"multiply" => req.a * req.b,
"divide" => {
if req.b == 0.0 {
return HttpResponse::BadRequest().json("除数不能为 0");
} }
req.a / req.b
} }
Err(error) => println!("读取输入时发生错误: {}", error), _ => return HttpResponse::BadRequest().json("不支持的操作"),
} };
let response = CalcResponse {
result,
operation: req.operation,
};
HttpResponse::Ok().json(response)
}
async fn echo(data: web::Json<serde_json::Value>) -> impl Responder {
HttpResponse::Ok().json(data)
}
#[actix_web::main]
async fn main() -> std::io::Result<()> {
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()),
)
.init();
let config = Config::from_env().map_err(std::io::Error::other)?;
std::fs::create_dir_all("data")?;
let store = Store::open(&config.database_url)
.await
.map_err(std::io::Error::other)?;
let service = Arc::new(
Service::new(store, &config)
.await
.map_err(std::io::Error::other)?,
);
let app_service = service.clone();
let server = HttpServer::new(move || {
App::new()
.app_data(web::Data::new(app_service.clone()))
.app_data(web::JsonConfig::default().limit(32 * 1024))
.route("/", web::get().to(hello))
.route("/health", web::get().to(health_check))
.route("/greet/{name}", web::get().to(greet))
.route("/api/echo", web::post().to(echo))
.route("/api/calculate", web::post().to(calculate))
.configure(api::configure)
.route(
"/info",
web::get().to(|| async {
HttpResponse::Ok().json(serde_json::json!({
"project": env!("CARGO_PKG_NAME"),
"version": env!("CARGO_PKG_VERSION"),
"endpoints": [
{ "method": "GET", "path": "/", "description": "欢迎页面" },
{ "method": "GET", "path": "/health", "description": "健康检查" },
{ "method": "GET", "path": "/greet/{name}", "description": "问候接口" },
{ "method": "POST", "path": "/api/echo", "description": "回显数据" },
{ "method": "POST", "path": "/api/calculate", "description": "计算接口" }
]
}))
}),
)
})
.bind(("127.0.0.1", config.port))?;
let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false);
let background = tokio::spawn(worker(service, config.poll_interval, shutdown_rx));
println!("\n==================================================");
println!("🚀 Rust Web 服务器已启动");
println!("📡 监听地址: http://127.0.0.1:{}", config.port);
println!(" GET/POST /api/subscriptions - 订阅管理");
println!(" POST /api/preview - RSS 解析预览");
println!(" POST /api/run - 执行订阅与下载同步");
println!(" GET /api/jobs - 下载任务");
println!("📋 可用端点:");
println!(" GET / - 欢迎页面");
println!(" GET /health - 健康检查");
println!(" GET /info - API 信息");
println!(" GET /greet/{{name}} - 问候接口");
println!(" POST /api/echo - 回显数据");
println!(" POST /api/calculate - 计算接口");
println!("==================================================\n");
let result = server.run().await;
let _ = shutdown_tx.send(true);
let _ = background.await;
result
} }
src/media.rs
+269
View File
@@ -0,0 +1,269 @@
// SPDX-License-Identifier: GPL-2.0-only
use crate::error::{Error, Result};
use cap_std::{ambient_authority, fs::Dir};
use serde::{Deserialize, Serialize};
use std::{
io,
path::{Component, Path},
};
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub struct DestinationInput {
pub name: String,
pub kind: LocalKind,
pub root_path: String,
}
#[derive(Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum LocalKind {
Local,
}
impl DestinationInput {
pub fn validate(&mut self) -> Result<()> {
self.name = self.name.trim().to_owned();
if self.name.is_empty()
|| self.name.chars().count() > 100
|| self.root_path.len() > 4096
|| self.root_path.chars().any(char::is_control)
|| !Path::new(&self.root_path).is_absolute()
|| Path::new(&self.root_path)
.components()
.any(|c| matches!(c, Component::ParentDir))
{
return Err(Error::Invalid(
"expected a name and an absolute local root without parent traversal".into(),
));
}
Ok(())
}
}
#[derive(Serialize, sqlx::FromRow)]
pub struct Destination {
pub id: String,
pub name: String,
pub kind: String,
pub root_path: String,
pub is_active: bool,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MappingInput {
pub downloader_id: String,
pub download_root: String,
}
#[derive(Clone, Serialize, sqlx::FromRow)]
pub struct Mapping {
pub id: String,
pub destination_id: String,
pub downloader_id: String,
pub version: i64,
pub download_root: String,
}
#[derive(Clone, Serialize, sqlx::FromRow)]
pub struct MediaFile {
pub relative_path: String,
pub expected_size: i64,
pub observed_size: Option<i64>,
pub status: String,
pub error: Option<String>,
}
#[derive(Serialize)]
pub struct FileReport {
pub job_id: String,
pub destination_id: Option<String>,
pub status: String,
pub checked_at: Option<i64>,
pub last_success_at: Option<i64>,
pub error: Option<String>,
pub files: Vec<MediaFile>,
}
// Wire paths always use '/' and must be relative on both Unix and Windows.
pub fn validate_relative(path: &str, allow_empty: bool) -> Result<()> {
if (path.is_empty() && !allow_empty)
|| path.len() > 4096
|| path.contains(['\\', ':'])
|| path.chars().any(char::is_control)
|| (!path.is_empty()
&& path
.split('/')
.any(|part| part.is_empty() || matches!(part, "." | "..")))
{
return Err(Error::Invalid(
"expected a relative path without empty, dot, parent, drive or backslash components"
.into(),
));
}
Ok(())
}
// qBittorrent may run on a different OS. Do not interpret its paths using the host OS.
pub fn normalize_download_root(value: &str) -> Result<String> {
let path = value.replace('\\', "/");
let drive = path.len() >= 3
&& path.as_bytes()[0].is_ascii_alphabetic()
&& path.as_bytes()[1] == b':'
&& path.as_bytes()[2] == b'/';
if path.len() > 4096 || path.chars().any(char::is_control) || !(path.starts_with('/') || drive)
{
return Err(Error::Invalid(
"downloader root must be an absolute Unix or Windows path".into(),
));
}
let rest = if drive {
&path[3..]
} else {
path.trim_start_matches('/')
};
if rest.contains(':') || rest.split('/').any(|s| matches!(s, "." | "..")) {
return Err(Error::Invalid("invalid downloader root components".into()));
}
let path = path.trim_end_matches('/');
Ok(if path.is_empty() {
"/".into()
} else if drive && path.len() == 2 {
format!("{path}/")
} else {
path.into()
})
}
pub fn same_download_root(expected: &str, actual: &str) -> bool {
match (
normalize_download_root(expected),
normalize_download_root(actual),
) {
(Ok(a), Ok(b)) => a == b,
_ => false,
}
}
fn access_error(error: &io::Error) -> String {
match error.kind() {
io::ErrorKind::PermissionDenied => "access denied or path escapes the configured root",
io::ErrorKind::NotFound => "configured root or directory is unavailable",
_ => "filesystem access failed",
}
.into()
}
pub fn check_local(root: &str, mut files: Vec<MediaFile>) -> Vec<MediaFile> {
let dir = Dir::open_ambient_dir(root, ambient_authority());
for file in &mut files {
file.observed_size = None;
file.error = None;
let result = match &dir {
Ok(dir) => {
if validate_relative(&file.relative_path, false).is_err() {
file.status = "unavailable".into();
file.error = Some("invalid stored relative path".into());
continue;
}
dir.metadata(&file.relative_path)
}
Err(error) => {
file.status = "unavailable".into();
file.error = Some(access_error(error));
continue;
}
};
match result {
Ok(metadata) if metadata.is_file() => {
file.status = "present".into();
file.observed_size = i64::try_from(metadata.len()).ok();
}
Ok(_) => {
file.status = "unavailable".into();
file.error = Some("expected a regular file".into());
}
Err(error) if error.kind() == io::ErrorKind::NotFound => file.status = "absent".into(),
Err(error) => {
file.status = "unavailable".into();
file.error = Some(access_error(&error));
}
}
}
files
}
#[derive(Serialize)]
pub struct DirectoryEntry {
pub name: String,
pub kind: &'static str,
pub size: Option<u64>,
}
#[derive(Serialize)]
pub struct DirectoryListing {
pub status: &'static str,
pub error: Option<String>,
pub entries: Vec<DirectoryEntry>,
pub truncated: bool,
}
pub fn list_local(root: &str, path: &str) -> DirectoryListing {
if let Err(error) = validate_relative(path, true) {
return DirectoryListing {
status: "unavailable",
error: Some(error.to_string()),
entries: vec![],
truncated: false,
};
}
let result = (|| -> io::Result<_> {
let dir = Dir::open_ambient_dir(root, ambient_authority())?;
let mut entries = Vec::new();
let mut truncated = false;
for (index, item) in dir
.read_dir(if path.is_empty() { "." } else { path })?
.enumerate()
{
if index >= 1000 {
truncated = true;
break;
}
let item = item?;
let kind = item.file_type()?;
entries.push(DirectoryEntry {
name: item.file_name().to_string_lossy().into_owned(),
size: if kind.is_file() {
Some(item.metadata()?.len())
} else {
None
},
kind: if kind.is_symlink() {
"symlink"
} else if kind.is_dir() {
"directory"
} else if kind.is_file() {
"file"
} else {
"other"
},
});
}
entries.sort_by(|a, b| a.name.cmp(&b.name));
Ok((entries, truncated))
})();
match result {
Ok((entries, truncated)) => DirectoryListing {
status: "available",
error: None,
entries,
truncated,
},
Err(error) => DirectoryListing {
status: "unavailable",
error: Some(access_error(&error)),
entries: vec![],
truncated: false,
},
}
}
src/media_store.rs
+153
View File
@@ -0,0 +1,153 @@
// SPDX-License-Identifier: GPL-2.0-only
use crate::{
domain::DownloadJob,
error::{Error, Result},
media::*,
storage::{now, Store},
};
impl Store {
pub async fn destinations(&self) -> Result<Vec<Destination>> {
Ok(sqlx::query_as("SELECT d.*, COALESCE(s.destination_id = d.id, 0) AS is_active FROM media_destinations d CROSS JOIN destination_selection s ORDER BY d.rowid")
.fetch_all(&self.pool).await?)
}
pub async fn destination(&self, id: &str) -> Result<Destination> {
self.destinations()
.await?
.into_iter()
.find(|d| d.id == id)
.ok_or(Error::NotFound)
}
pub async fn create_destination(&self, mut input: DestinationInput) -> Result<Destination> {
input.validate()?;
let id = uuid::Uuid::new_v4().to_string();
sqlx::query(
"INSERT INTO media_destinations(id, name, kind, root_path) VALUES (?, ?, 'local', ?)",
)
.bind(&id)
.bind(input.name)
.bind(input.root_path)
.execute(&self.pool)
.await?;
self.destination(&id).await
}
pub async fn active_destination_id(&self) -> Result<Option<String>> {
Ok(sqlx::query_scalar(
"SELECT destination_id FROM destination_selection WHERE singleton = 1",
)
.fetch_one(&self.pool)
.await?)
}
pub async fn activate_destination(&self, id: &str) -> Result<()> {
let mut tx = self.pool.begin().await?;
let exists: bool =
sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM media_destinations WHERE id = ?)")
.bind(id)
.fetch_one(&mut *tx)
.await?;
if !exists {
return Err(Error::NotFound);
}
let mapped: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM destination_mappings m JOIN downloader_selection s ON s.downloader_id = m.downloader_id WHERE m.destination_id = ?)")
.bind(id).fetch_one(&mut *tx).await?;
if !mapped {
return Err(Error::Invalid(
"select a downloader and configure its mapping before activating this destination"
.into(),
));
}
sqlx::query("UPDATE destination_selection SET destination_id = ? WHERE singleton = 1")
.bind(id)
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(())
}
pub async fn mappings(&self, id: &str) -> Result<Vec<Mapping>> {
self.destination(id).await?;
Ok(sqlx::query_as("SELECT * FROM destination_mappings WHERE destination_id = ? ORDER BY version DESC, downloader_id")
.bind(id).fetch_all(&self.pool).await?)
}
pub async fn create_mapping(
&self,
destination_id: &str,
input: MappingInput,
) -> Result<Mapping> {
self.destination(destination_id).await?;
self.downloader_config(&input.downloader_id).await?;
let root = normalize_download_root(&input.download_root)?;
let id = uuid::Uuid::new_v4().to_string();
let mut tx = self.pool.begin().await?;
let version: i64 = sqlx::query_scalar("SELECT COALESCE(MAX(version), 0) + 1 FROM destination_mappings WHERE destination_id = ? AND downloader_id = ?")
.bind(destination_id).bind(&input.downloader_id).fetch_one(&mut *tx).await?;
sqlx::query("INSERT INTO destination_mappings(id, destination_id, downloader_id, version, download_root) VALUES (?, ?, ?, ?, ?)")
.bind(&id).bind(destination_id).bind(&input.downloader_id).bind(version).bind(&root).execute(&mut *tx).await?;
tx.commit().await?;
Ok(Mapping {
id,
destination_id: destination_id.into(),
downloader_id: input.downloader_id,
version,
download_root: root,
})
}
pub async fn job(&self, id: &str) -> Result<DownloadJob> {
sqlx::query_as("SELECT * FROM download_jobs WHERE id = ?")
.bind(id)
.fetch_optional(&self.pool)
.await?
.ok_or(Error::NotFound)
}
pub async fn file_report(&self, id: &str) -> Result<FileReport> {
let job = self.job(id).await?;
let files = sqlx::query_as("SELECT relative_path, expected_size, observed_size, status, error FROM media_files WHERE job_id = ? ORDER BY relative_path")
.bind(id).fetch_all(&self.pool).await?;
Ok(FileReport {
job_id: id.into(),
destination_id: job.destination_id,
status: job.file_status,
checked_at: job.file_checked_at,
last_success_at: job.file_last_success_at,
error: job.file_error,
files,
})
}
pub(crate) async fn save_manifest(&self, id: &str, files: &[MediaFile]) -> Result<()> {
let mut tx = self.pool.begin().await?;
for file in files {
validate_relative(&file.relative_path, false)?;
sqlx::query("INSERT INTO media_files(job_id, relative_path, expected_size) VALUES (?, ?, ?) ON CONFLICT(job_id, relative_path) DO NOTHING")
.bind(id).bind(&file.relative_path).bind(file.expected_size).execute(&mut *tx).await?;
}
tx.commit().await?;
Ok(())
}
pub(crate) async fn save_file_check(
&self,
id: &str,
status: &str,
error: Option<&str>,
files: &[MediaFile],
) -> Result<()> {
let mut tx = self.pool.begin().await?;
let checked_at = now();
for file in files {
sqlx::query("UPDATE media_files SET observed_size = ?, status = ?, error = ? WHERE job_id = ? AND relative_path = ?")
.bind(file.observed_size).bind(&file.status).bind(&file.error).bind(id).bind(&file.relative_path).execute(&mut *tx).await?;
}
sqlx::query("UPDATE download_jobs SET file_status = ?, file_error = ?, file_checked_at = ?, file_last_success_at = CASE WHEN ? IN ('present', 'absent') THEN ? ELSE file_last_success_at END WHERE id = ?")
.bind(status).bind(error).bind(checked_at).bind(status).bind(checked_at).bind(id).execute(&mut *tx).await?;
tx.commit().await?;
Ok(())
}
}
src/mikan.rs
+460
View File
@@ -0,0 +1,460 @@
// SPDX-License-Identifier: GPL-2.0-only
//! Public Mikan discovery; HTML details stay inside this adapter.
use crate::{
domain::{validate_http_url, SubscriptionInput},
error::{Error, Result},
feeds::{bounded_body, website_get},
};
use reqwest::{Client, Url};
use scraper::{Html, Selector};
use serde::Serialize;
use std::{
collections::{HashMap, HashSet},
sync::OnceLock,
time::{Duration, Instant},
};
use tokio::sync::{Mutex, Semaphore};
pub struct Mikan {
client: Client,
base: Url,
cache: Mutex<HashMap<String, (Instant, Cached)>>,
requests: Semaphore,
}
#[derive(Clone)]
enum Cached {
Search(SearchResult),
Series(Box<SeriesDetail>),
}
#[derive(Clone, Serialize)]
pub struct SeriesSummary {
pub series_id: String,
pub title: String,
pub page_url: String,
pub cover_url: Option<String>,
}
#[derive(Clone, Serialize)]
pub struct SearchResult {
pub source_id: &'static str,
pub query: String,
pub series: Vec<SeriesSummary>,
pub keyword_rss_url: String,
}
#[derive(Clone, Serialize)]
pub struct SubtitleGroup {
pub group_id: String,
pub name: String,
pub rss_url: String,
}
#[derive(Clone, Serialize)]
pub struct SeriesDetail {
pub source_id: &'static str,
pub series_id: String,
pub title: String,
pub page_url: String,
pub rss_url: String,
pub groups: Vec<SubtitleGroup>,
pub information: Vec<String>,
pub bangumi_id: Option<String>,
pub cover_url: Option<String>,
#[serde(skip)]
poster: Option<Url>,
}
#[derive(Serialize)]
pub struct Candidate {
pub source_id: &'static str,
pub scope: &'static str,
pub series_id: Option<String>,
pub group_id: Option<String>,
pub page_url: String,
pub rss_url: String,
pub subscription: Option<SubscriptionInput>,
pub warnings: Vec<&'static str>,
}
fn text(element: scraper::ElementRef<'_>) -> String {
element
.text()
.collect::<String>()
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
}
fn parameters(url: &Url) -> Result<HashMap<String, String>> {
let mut params = HashMap::new();
for (key, value) in url.query_pairs() {
if params
.insert(key.to_ascii_lowercase(), value.into_owned())
.is_some()
{
return Err(Error::Invalid("duplicate URL parameter".into()));
}
}
Ok(params)
}
fn selector(value: &str) -> Selector {
Selector::parse(value).expect("static selector")
}
fn valid_id(value: &str) -> bool {
!value.is_empty() && value.len() <= 20 && value.bytes().all(|b| b.is_ascii_digit())
}
impl Mikan {
pub fn new(client: Client, base: &str) -> Result<Self> {
validate_http_url(base)?;
let base = Url::parse(base).expect("validated URL");
if base.path() != "/" || base.query().is_some() || base.fragment().is_some() {
return Err(Error::Invalid(
"MIKAN_BASE_URL must be an HTTP(S) origin without a path or query".into(),
));
}
Ok(Self {
client,
base,
cache: Mutex::new(HashMap::new()),
requests: Semaphore::new(4),
})
}
async fn cached(&self, key: &str) -> Option<Cached> {
self.cache
.lock()
.await
.get(key)
.filter(|(created, _)| created.elapsed() < Duration::from_secs(60))
.map(|(_, value)| value.clone())
}
async fn remember(&self, key: String, value: Cached) {
let mut cache = self.cache.lock().await;
cache.retain(|_, (created, _)| created.elapsed() < Duration::from_secs(60));
if cache.len() >= 64 {
if let Some(oldest) = cache
.iter()
.min_by_key(|(_, (time, _))| *time)
.map(|(key, _)| key.clone())
{
cache.remove(&oldest);
}
}
cache.insert(key, (Instant::now(), value));
}
async fn html(&self, url: Url) -> Result<Html> {
let response = website_get(&self.client, url).await?;
let bytes = bounded_body(response, 8 * 1024 * 1024).await?;
let text = std::str::from_utf8(&bytes)
.map_err(|_| Error::Upstream("Mikan returned invalid UTF-8".into()))?;
Ok(Html::parse_document(text))
}
fn same_origin_url(&self, value: &str) -> Result<Url> {
let url = self
.base
.join(value)
.map_err(|_| Error::Invalid("invalid Mikan URL".into()))?;
validate_http_url(url.as_str())?;
if url.origin() != self.base.origin() {
return Err(Error::Invalid(
"URL must belong to the configured Mikan origin".into(),
));
}
Ok(url)
}
fn rss_link(
&self,
element: scraper::ElementRef<'_>,
series_id: &str,
) -> Option<(Url, Option<String>)> {
let url = self.same_origin_url(element.value().attr("href")?).ok()?;
if url.path() != "/RSS/Bangumi" {
return None;
}
let params = parameters(&url).ok()?;
if params.get("bangumiid").map(String::as_str) != Some(series_id) {
return None;
}
let group = params.get("subgroupid").cloned();
if group.as_ref().is_some_and(|id| !valid_id(id)) {
return None;
}
Some((url, group))
}
pub async fn series(&self, id: &str) -> Result<SeriesDetail> {
if !valid_id(id) {
return Err(Error::Invalid("invalid Mikan series ID".into()));
}
let key = format!("series:{id}");
if let Some(Cached::Series(detail)) = self.cached(&key).await {
return Ok(*detail);
}
let _permit = self.requests.try_acquire().map_err(|_| Error::SourceBusy)?;
let page = self
.base
.join(&format!("/Home/Bangumi/{id}"))
.expect("numeric ID");
let html = self.html(page.clone()).await?;
let title = html
.select(&selector(".bangumi-title"))
.next()
.map(text)
.filter(|name| !name.is_empty())
.ok_or_else(|| {
Error::Upstream("Mikan detail page is unrecognized or access is restricted".into())
})?;
let rss_url = html
.select(&selector(".bangumi-title a.mikan-rss[href]"))
.filter_map(|link| self.rss_link(link, id))
.find(|(_, group)| group.is_none())
.map(|(url, _)| url.to_string())
.ok_or_else(|| Error::Upstream("Mikan series RSS link is missing or invalid".into()))?;
let mut groups = Vec::new();
let mut seen = HashSet::new();
for block in html.select(&selector(".subgroup-text")) {
let Some((url, Some(group_id))) = block
.select(&selector("a.mikan-rss[href]"))
.find_map(|link| self.rss_link(link, id))
else {
continue;
};
let Some(name) = block
.select(&selector("a[href*='/Home/PublishGroup/']"))
.next()
.map(text)
.filter(|name| !name.is_empty())
else {
continue;
};
if seen.insert(group_id.clone()) {
groups.push(SubtitleGroup {
group_id,
name,
rss_url: url.into(),
});
}
}
let information = html.select(&selector(".bangumi-info")).map(text).collect();
let bangumi_id = html
.select(&selector(".bangumi-info a[href]"))
.find_map(|link| {
let url = Url::parse(link.value().attr("href")?).ok()?;
if !matches!(url.host_str()?, "bgm.tv" | "bangumi.tv" | "chii.in") {
return None;
}
url.path()
.strip_prefix("/subject/")
.filter(|id| valid_id(id))
.map(str::to_owned)
});
static CSS_URL: OnceLock<regex::Regex> = OnceLock::new();
let pattern = CSS_URL.get_or_init(|| {
regex::Regex::new(r#"url\(\s*['"]?([^'"\s)]+)['"]?\s*\)"#).expect("static regex")
});
let poster = html
.select(&selector(".bangumi-poster[style]"))
.find_map(|element| {
let captures = pattern.captures(element.value().attr("style")?)?;
let url = self.same_origin_url(captures.get(1)?.as_str()).ok()?;
url.path().starts_with("/images/").then_some(url)
});
let detail = SeriesDetail {
source_id: "mikan",
series_id: id.into(),
title,
page_url: page.into(),
rss_url,
groups,
information,
bangumi_id,
cover_url: poster
.as_ref()
.map(|_| format!("/api/sources/mikan/series/{id}/cover")),
poster,
};
self.remember(key, Cached::Series(Box::new(detail.clone())))
.await;
Ok(detail)
}
pub async fn cover(&self, id: &str) -> Result<(String, Vec<u8>)> {
let url = self.series(id).await?.poster.ok_or(Error::NotFound)?;
let _permit = self.requests.try_acquire().map_err(|_| Error::SourceBusy)?;
let response = website_get(&self.client, url).await?;
let mime = response
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.split(';')
.next()
.unwrap_or("")
.trim()
.to_ascii_lowercase();
if !matches!(
mime.as_str(),
"image/png" | "image/jpeg" | "image/webp" | "image/gif"
) {
return Err(Error::Upstream(
"Mikan cover is not a supported raster image".into(),
));
}
Ok((mime, bounded_body(response, 4 * 1024 * 1024).await?))
}
pub async fn resolve(&self, value: &str) -> Result<Candidate> {
if value.len() > 8192 {
return Err(Error::Invalid("Mikan URL is too long".into()));
}
let url = self.same_origin_url(value.trim())?;
let params = parameters(&url)?;
if matches!(url.path(), "/RSS/Search" | "/Home/Search") {
let query = params
.get("searchstr")
.map(|s| s.trim())
.filter(|s| !s.is_empty() && s.chars().count() <= 100)
.ok_or_else(|| {
Error::Invalid("keyword RSS requires a query of 1–100 characters".into())
})?;
if params.len() != 1 {
return Err(Error::Invalid("unsupported keyword RSS parameters".into()));
}
let mut rss = self.base.join("/RSS/Search").expect("static path");
rss.query_pairs_mut().append_pair("searchstr", query);
let mut page = rss.clone();
page.set_path("/Home/Search");
return Ok(Candidate { source_id: "mikan", scope: "keyword", series_id: None, group_id: None,
page_url: page.into(), rss_url: rss.into(), subscription: None,
warnings: vec!["Keyword RSS can contain multiple series. Select a specific series, or manually preview with filters that restrict the feed to one series before creating a subscription."] });
}
let (id, group) = if let Some(id) = url.path().strip_prefix("/Home/Bangumi/") {
(id.to_owned(), None)
} else if url.path() == "/RSS/Bangumi" {
(
params
.get("bangumiid")
.cloned()
.ok_or_else(|| Error::Invalid("missing bangumiId".into()))?,
params.get("subgroupid").cloned(),
)
} else {
return Err(Error::Invalid(
"expected a Mikan series page or series RSS URL".into(),
));
};
if group.as_ref().is_some_and(|id| !valid_id(id)) {
return Err(Error::Invalid("invalid subtitle group ID".into()));
}
let detail = self.series(&id).await?;
let rss_url = match &group {
Some(group) => detail
.groups
.iter()
.find(|g| &g.group_id == group)
.map(|g| g.rss_url.clone())
.ok_or_else(|| {
Error::Invalid("subtitle group is not available for this series".into())
})?,
None => detail.rss_url,
};
Ok(Candidate {
source_id: "mikan", scope: if group.is_some() { "group" } else { "series" },
series_id: Some(id), group_id: group, page_url: detail.page_url, rss_url: rss_url.clone(),
subscription: Some(SubscriptionInput { title: detail.title, rss_url, season: 1, episode_offset: 0,
include: Vec::new(), exclude: Vec::new(), enabled: false }),
warnings: vec!["Confirm season, episode offset and filters in preview before enabling; season 1 is a default, not inferred metadata."],
})
}
pub async fn search(&self, query: &str) -> Result<SearchResult> {
let query = query.trim();
if query.is_empty() || query.chars().count() > 100 {
return Err(Error::Invalid(
"search query must contain 1–100 characters".into(),
));
}
let key = format!("search:{query}");
if let Some(Cached::Search(result)) = self.cached(&key).await {
return Ok(result);
}
let _permit = self.requests.try_acquire().map_err(|_| Error::SourceBusy)?;
let mut url = self.base.join("/Home/Search").expect("static path");
url.query_pairs_mut().append_pair("searchstr", query);
let html = self.html(url.clone()).await?;
let unrecognized =
|| Error::Upstream("Mikan search page is unrecognized or access is restricted".into());
let search_form = html
.select(&selector(
"form[action='/Home/Search'] input[name='searchstr']",
))
.next()
.is_some();
let results = html.select(&selector(".an-ul")).next().is_some()
|| html
.select(&selector(".header2-text"))
.any(|element| text(element).contains("搜索结果"));
if !search_form || !results {
return Err(unrecognized());
}
let mut seen = HashSet::new();
let mut series = Vec::new();
for link in html.select(&selector(".an-ul a[href]")) {
let Some(url) = link
.value()
.attr("href")
.and_then(|s| self.base.join(s).ok())
else {
continue;
};
if url.origin() != self.base.origin() {
continue;
}
let Some(id) = url
.path()
.strip_prefix("/Home/Bangumi/")
.filter(|id| valid_id(id))
else {
continue;
};
let Some(name) = link.select(&selector(".an-text")).next() else {
return Err(unrecognized());
};
let text = name.text().collect::<String>();
let title = name.value().attr("title").unwrap_or(&text).trim();
if title.is_empty() {
return Err(unrecognized());
}
if !seen.insert(id.to_owned()) {
continue;
}
series.push(SeriesSummary {
series_id: id.into(),
title: title.into(),
page_url: url.to_string(),
cover_url: link
.select(&selector("[data-src], img[src]"))
.next()
.map(|_| format!("/api/sources/mikan/series/{id}/cover")),
});
}
url.set_path("/RSS/Search");
let result = SearchResult {
source_id: "mikan",
query: query.into(),
series,
keyword_rss_url: url.into(),
};
self.remember(key, Cached::Search(result.clone())).await;
Ok(result)
}
}
src/parser.rs
+89
View File
@@ -0,0 +1,89 @@
// SPDX-License-Identifier: GPL-2.0-only
use regex::Regex;
use std::{collections::BTreeSet, sync::LazyLock};
static UNSUPPORTED: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(
r"(?i)(合集|全集|\bbatch\b|\bcomplete\b|\b(?:OVA|SP|NCOP|NCED)(?:[0-9]+\b|\b)|(?:\s-\s*|\[|第|\bEP?\s*)[0-9]+\.[0-9]+|[0-9]+\s*[-~～]\s*[0-9]+)"
).unwrap()
});
static SEASON_EPISODE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(?i)\bS([0-9]{1,3})E([0-9]{1,4})(?:v[0-9]+)?(?:\b|_)").unwrap());
static EPISODE: LazyLock<Vec<Regex>> = LazyLock::new(|| {
[
r"(?i)\[([0-9]{1,3})(?:v[0-9]+)?(?:\s*END)?\]",
r"第\s*([0-9]{1,4})\s*[话話集]",
r"(?i)\bEP?\s*([0-9]{1,4})(?:v[0-9]+)?\b",
r"(?i)\s-\s*([0-9]{1,4})(?:v[0-9]+)?(?:\s|\[|$)",
]
.into_iter()
.map(|r| Regex::new(r).unwrap())
.collect()
});
/// Conservative single-episode parsing. Ambiguity is visible, never guessed.
pub fn episode(title: &str, season: i32, offset: i32) -> std::result::Result<i32, &'static str> {
if UNSUPPORTED.is_match(title) {
return Err("batch, special episode or fractional episode requires manual review");
}
let mut candidates = BTreeSet::new();
for captures in SEASON_EPISODE.captures_iter(title) {
if captures[1].parse::<i32>().ok() != Some(season) {
return Err("title season differs from subscription season");
}
candidates.insert(captures[2].parse::<i32>().unwrap());
}
for pattern in EPISODE.iter() {
for captures in pattern.captures_iter(title) {
candidates.insert(captures[1].parse::<i32>().unwrap());
}
}
if candidates.len() != 1 {
return Err("episode is missing or ambiguous");
}
let value = candidates
.first()
.copied()
.unwrap()
.checked_add(offset)
.ok_or("episode offset overflow")?;
if !(1..=10000).contains(&value) {
return Err("episode is outside supported range after offset");
}
Ok(value)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn single_episode_formats() {
for title in [
"[Group] Example [03][1080p]",
"Example - 03v2 [1080p]",
"Example 第03话",
"Example S01E03",
"Example EP03",
"Example - 03 [1080p HEVC AAC2.0]",
] {
assert_eq!(episode(title, 1, 0), Ok(3), "{title}");
}
assert_eq!(episode("Example - 13 [1080p]", 2, -12), Ok(1));
}
#[test]
fn uncertain_titles_are_not_downloaded() {
for title in [
"Example [1080p]",
"Example [01-12]",
"Example - 12.5",
"Example SP01",
"Example SP01 [03]",
"Example [03] [04]",
"Example S02E03",
"Example S01E03E04",
] {
assert!(episode(title, 1, 0).is_err(), "{title}");
}
assert!(episode("Example [01]", 1, -1).is_err());
}
}
src/service.rs
+380
View File
@@ -0,0 +1,380 @@
// SPDX-License-Identifier: GPL-2.0-only
use crate::{
config::Config,
domain::*,
downloader::{self, Qbit},
error::{Error, Result},
feeds,
media::{self, FileReport, MediaFile},
storage::Store,
};
use serde::Serialize;
use std::{
collections::{HashMap, HashSet},
sync::Arc,
};
use tokio::sync::{watch, Mutex};
pub struct Service {
pub store: Store,
pub gate: Mutex<()>,
pub mikan: crate::mikan::Mikan,
client: reqwest::Client,
}
#[derive(Default, Serialize)]
pub struct CycleReport {
pub scanned: usize,
pub queued: usize,
pub errors: Vec<String>,
pub downloader_configured: bool,
}
impl Service {
pub async fn new(store: Store, config: &Config) -> Result<Self> {
let client = feeds::website_client(&config.website)?;
let mikan = crate::mikan::Mikan::new(client.clone(), &config.mikan_base_url)?;
store.bootstrap_downloader(config.qbit.as_ref()).await?;
Ok(Self {
store,
gate: Mutex::new(()),
mikan,
client,
})
}
pub async fn downloader_configured(&self) -> Result<bool> {
Ok(self.store.active_downloader_id().await?.is_some())
}
pub async fn preview(&self, mut input: SubscriptionInput) -> Result<Vec<Release>> {
input.validate()?;
feeds::fetch(&self.client, &input).await
}
pub async fn preview_subscription(&self, id: &str) -> Result<Vec<Release>> {
let sub = self.store.subscription(id).await?;
let mut releases = self.preview(sub.input.clone()).await?;
let mut episodes = HashSet::new();
// Feed order determines the candidate selected in v0.1; priority rules come later.
for release in &mut releases {
if release.decision == Decision::Ready {
let duplicate = self
.store
.already_queued(id, sub.input.season, release)
.await?;
if duplicate || !episodes.insert(release.episode) {
release.decision = Decision::AlreadyQueued;
release.reason = if duplicate {
"resource or episode already has a task"
} else {
"an earlier feed entry already represents this episode"
}
.into();
}
}
}
Ok(releases)
}
async fn scan(&self, sub: &Subscription) -> Result<usize> {
let result = async {
let releases = self.preview_subscription(&sub.id).await?;
self.store.record_scan(sub, &releases).await
}
.await;
self.store
.checked(
&sub.id,
result.as_ref().err().map(ToString::to_string).as_deref(),
)
.await?;
result
}
pub async fn scan_subscription(&self, id: &str) -> Result<usize> {
let _guard = self.gate.try_lock().map_err(|_| Error::Busy)?;
let sub = self.store.subscription(id).await?;
if !sub.input.enabled {
return Err(Error::Invalid(
"enable the subscription before scanning; preview is available while disabled"
.into(),
));
}
self.scan(&sub).await
}
pub async fn cycle(&self) -> Result<CycleReport> {
let _guard = self.gate.try_lock().map_err(|_| Error::Busy)?;
let mut report = CycleReport {
downloader_configured: self.downloader_configured().await?,
..Default::default()
};
let legacy = self.store.legacy_unassigned_count().await?;
if legacy > 0 {
report.errors.push(format!("{legacy} legacy jobs have no recorded downloader; restore their original instance and explicitly claim them"));
}
for sub in self.store.subscriptions().await? {
if !sub.input.enabled {
continue;
}
match self.scan(&sub).await {
Ok(count) => {
report.scanned += 1;
report.queued += count;
}
Err(error) => report
.errors
.push(format!("subscription {}: {error}", sub.id)),
}
}
match self.sync_downloads().await {
Ok(errors) => report.errors.extend(errors),
Err(error) => report.errors.push(error.to_string()),
}
Ok(report)
}
async fn sync_downloads(&self) -> Result<Vec<String>> {
let jobs = self.store.active_jobs().await?;
let mut groups: std::collections::BTreeMap<String, Vec<DownloadJob>> = Default::default();
for job in jobs {
if let Some(id) = &job.downloader_id {
groups.entry(id.clone()).or_default().push(job);
}
}
let mut errors = Vec::new();
for (id, jobs) in groups {
let result = async {
let qbit = Qbit::new(self.store.downloader_config(&id).await?)?;
self.sync_instance(&qbit, jobs).await
}
.await;
if let Err(error) = result {
if matches!(error, Error::Database(_)) {
return Err(error);
}
errors.push(format!("downloader {id}: {error}"));
}
}
Ok(errors)
}
async fn sync_instance(&self, qbit: &Qbit, jobs: Vec<DownloadJob>) -> Result<()> {
qbit.login().await?;
// Never submit when querying the remote state fails.
let remote: HashMap<_, _> = qbit
.torrents()
.await?
.into_iter()
.map(|torrent| (torrent.hash.to_ascii_lowercase(), torrent))
.collect();
let mut added: HashMap<String, Option<String>> = HashMap::new();
for mut job in jobs {
let sub = self.store.subscription(&job.subscription_id).await;
let enabled = match sub {
Ok(sub) => sub.input.enabled,
Err(Error::NotFound) => false,
Err(error) => return Err(error),
};
let result: Result<()> = async {
if job.info_hash.is_none() {
if !enabled { return Ok(()); }
let prepared = downloader::prepare(&self.client, &job.download_url).await?;
self.store.prepare(&job.id, &prepared.hash, prepared.bytes.as_deref()).await?;
job.info_hash = Some(prepared.hash);
}
let hash = job.info_hash.as_ref().expect("prepared hash");
if let Some(torrent) = remote.get(hash) {
if job.destination_id.is_some() && !job.download_save_path.as_deref().zip(torrent.save_path.as_deref())
.is_some_and(|(expected, actual)| media::same_download_root(expected, actual)) {
return Err(Error::Invalid("remote save path differs from the job snapshot; refusing to move or reuse this torrent".into()));
}
if matches!(torrent.state.as_str(), "error" | "missingFiles") {
return Err(Error::Invalid("qBittorrent reports an error or missing files; inspect the downloader before retrying".into()));
}
let state = if torrent.progress >= 1.0 { "completed" }
else if torrent.progress > 0.0 { "downloading" } else { "submitted" };
self.store.state(&job.id, state).await?;
} else if let Some(path) = added.get(hash) {
if job.destination_id.is_some() && path != &job.download_save_path {
return Err(Error::Invalid("this torrent was submitted to a different path; refusing to reuse it".into()));
}
self.store.state(&job.id, "submitted").await?;
} else if matches!(job.state.as_str(), "submitted" | "downloading") {
// Respect removal in qBittorrent; recreating a task needs an explicit retry.
self.store.state(&job.id, "missing").await?;
} else if enabled {
let bytes = self.store.torrent_bytes(&job.id).await?;
// Commit the intent before the external side effect.
self.store.state(&job.id, "submitting").await?;
qbit.add(&job.download_url, bytes, job.download_save_path.as_deref()).await?;
added.insert(hash.clone(), job.download_save_path.clone());
self.store.state(&job.id, "submitted").await?;
}
Ok(())
}.await;
if let Err(error) = result {
if matches!(error, Error::Database(_)) {
return Err(error);
}
if matches!(error, Error::Invalid(_)) {
self.store.state(&job.id, "failed").await?;
}
self.store.failed(&job, &error.to_string()).await?;
tracing::warn!(job_id = %job.id, error = %error, "download job failed");
}
}
Ok(())
}
async fn discover_files(&self, job: &DownloadJob) -> Result<Vec<MediaFile>> {
let hash = job
.info_hash
.as_ref()
.ok_or_else(|| Error::Invalid("torrent metadata is not available yet".into()))?;
let owner = job
.downloader_id
.as_ref()
.ok_or_else(|| Error::Invalid("downloader ownership is unknown".into()))?;
let qbit = Qbit::new(self.store.downloader_config(owner).await?)?;
qbit.login().await?;
let torrent = qbit
.torrents()
.await?
.into_iter()
.find(|torrent| torrent.hash.eq_ignore_ascii_case(hash))
.ok_or_else(|| {
Error::Upstream("torrent is not available in its original downloader".into())
})?;
if !job
.download_save_path
.as_deref()
.zip(torrent.save_path.as_deref())
.is_some_and(|(expected, actual)| media::same_download_root(expected, actual))
{
return Err(Error::Upstream(
"remote save path does not match the frozen destination mapping".into(),
));
}
let mut files = Vec::new();
let mut paths = HashSet::new();
for file in qbit
.files(hash)
.await?
.into_iter()
.filter(|file| file.priority != 0)
{
media::validate_relative(&file.name, false).map_err(|_| {
Error::Upstream("downloader returned an unsafe relative file path".into())
})?;
if !paths.insert(file.name.clone()) {
return Err(Error::Upstream(
"duplicate path in downloader file list".into(),
));
}
files.push(MediaFile {
relative_path: file.name,
expected_size: file.size,
observed_size: None,
status: "unchecked".into(),
error: None,
});
}
Ok(files)
}
pub async fn check_job_files(&self, id: &str) -> Result<FileReport> {
// Serialize manifest discovery/checks with task synchronization and configuration changes.
let _guard = self.gate.try_lock().map_err(|_| Error::Busy)?;
let job = self.store.job(id).await?;
let Some(destination) = &job.destination_id else {
self.store
.save_file_check(
id,
"unchecked",
Some("job has no recorded media destination"),
&[],
)
.await?;
return self.store.file_report(id).await;
};
let root = self.store.destination(destination).await?.root_path;
let mut files = self.store.file_report(id).await?.files;
if files.is_empty() {
if job.info_hash.is_none() {
self.store
.save_file_check(
id,
"unchecked",
Some("torrent metadata is not available yet"),
&[],
)
.await?;
return self.store.file_report(id).await;
}
match self.discover_files(&job).await {
Ok(manifest) => files = manifest,
Err(error @ Error::Database(_)) => return Err(error),
Err(error) => {
self.store
.save_file_check(id, "unavailable", Some(&error.to_string()), &[])
.await?;
return self.store.file_report(id).await;
}
}
if files.is_empty() {
self.store
.save_file_check(
id,
"unchecked",
Some("no selected files or torrent metadata is still pending"),
&[],
)
.await?;
return self.store.file_report(id).await;
}
self.store.save_manifest(id, &files).await?;
}
let files = tokio::task::spawn_blocking(move || media::check_local(&root, files))
.await
.map_err(|_| Error::Upstream("local file check failed".into()))?;
let status = if files.iter().any(|f| f.status == "unavailable") {
"unavailable"
} else if files.iter().any(|f| f.status == "absent") {
"absent"
} else {
"present"
};
let error = files.iter().find_map(|file| file.error.as_deref());
self.store
.save_file_check(id, status, error, &files)
.await?;
self.store.file_report(id).await
}
}
pub async fn worker(
service: Arc<Service>,
interval: std::time::Duration,
mut shutdown: watch::Receiver<bool>,
) {
let mut timer = tokio::time::interval(interval);
timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
tokio::select! {
_ = shutdown.changed() => break,
_ = timer.tick() => {
// Cancellation can interrupt network I/O; persisted submission intent allows recovery.
tokio::select! {
_ = shutdown.changed() => break,
result = service.cycle() => match result {
Ok(report) => for error in report.errors { tracing::warn!(%error, "background cycle issue"); },
Err(Error::Busy) => {},
Err(error) => tracing::error!(%error, "background cycle failed"),
}
}
}
}
}
}
src/storage.rs
+367
View File
@@ -0,0 +1,367 @@
// SPDX-License-Identifier: GPL-2.0-only
use crate::{
domain::*,
error::{Error, Result},
};
use sqlx::{
sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions},
SqlitePool,
};
use std::{
str::FromStr,
time::{Duration, SystemTime, UNIX_EPOCH},
};
#[derive(Clone)]
pub struct Store {
pub pool: SqlitePool,
}
pub fn now() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs() as i64
}
impl Store {
pub async fn active_downloader_id(&self) -> Result<Option<String>> {
Ok(
sqlx::query_scalar(
"SELECT downloader_id FROM downloader_selection WHERE singleton = 1",
)
.fetch_one(&self.pool)
.await?,
)
}
pub async fn downloaders(&self) -> Result<Vec<DownloaderView>> {
let rows: Vec<(String, String, String, bool)> = sqlx::query_as(
"SELECT d.id, d.name, d.config, COALESCE(s.downloader_id = d.id, 0) FROM downloader_instances d CROSS JOIN downloader_selection s ORDER BY d.rowid")
.fetch_all(&self.pool).await?;
rows.into_iter()
.map(|(id, name, json, is_active)| {
let config: crate::config::QbitConfig = serde_json::from_str(&json)?;
Ok(DownloaderView {
id,
name,
kind: DownloaderKind::Qbittorrent,
url: config.url,
username: config.username,
save_path: config.save_path,
password_configured: !config.password.is_empty(),
is_active,
})
})
.collect()
}
pub async fn create_downloader(&self, mut input: DownloaderInput) -> Result<DownloaderView> {
input.validate()?;
let id = uuid::Uuid::new_v4().to_string();
let result = sqlx::query("INSERT INTO downloader_instances(id, name, kind, endpoint, config) VALUES (?, ?, 'qbittorrent', ?, ?) ON CONFLICT(endpoint) DO NOTHING")
.bind(&id).bind(&input.name).bind(&input.connection.url)
.bind(serde_json::to_string(&input.connection)?).execute(&self.pool).await?;
if result.rows_affected() == 0 {
return Err(Error::Invalid(
"this downloader endpoint is already registered".into(),
));
}
self.downloaders()
.await?
.into_iter()
.find(|d| d.id == id)
.ok_or(Error::NotFound)
}
pub(crate) async fn downloader_config(&self, id: &str) -> Result<crate::config::QbitConfig> {
let json: String =
sqlx::query_scalar("SELECT config FROM downloader_instances WHERE id = ?")
.bind(id)
.fetch_optional(&self.pool)
.await?
.ok_or(Error::NotFound)?;
Ok(serde_json::from_str(&json)?)
}
pub async fn activate_downloader(&self, id: &str) -> Result<()> {
let mut tx = self.pool.begin().await?;
let exists: bool =
sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM downloader_instances WHERE id = ?)")
.bind(id)
.fetch_one(&mut *tx)
.await?;
if !exists {
return Err(Error::NotFound);
}
let destination: Option<String> = sqlx::query_scalar(
"SELECT destination_id FROM destination_selection WHERE singleton = 1",
)
.fetch_one(&mut *tx)
.await?;
if let Some(destination) = destination {
let mapped: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM destination_mappings WHERE destination_id = ? AND downloader_id = ?)")
.bind(destination).bind(id).fetch_one(&mut *tx).await?;
if !mapped {
return Err(Error::Invalid("configure a mapping to the active destination before selecting this downloader".into()));
}
}
sqlx::query("UPDATE downloader_selection SET downloader_id = ? WHERE singleton = 1")
.bind(id)
.execute(&mut *tx)
.await?;
// Bind only new, never-owned work. A switch never rewrites an existing owner.
sqlx::query("UPDATE download_jobs SET downloader_id = ? WHERE downloader_id IS NULL AND legacy_unassigned = 0")
.bind(id).execute(&mut *tx).await?;
tx.commit().await?;
Ok(())
}
pub async fn claim_legacy_jobs(&self, id: &str) -> Result<u64> {
self.downloader_config(id).await?;
Ok(sqlx::query("UPDATE download_jobs SET downloader_id = ?, legacy_unassigned = 0 WHERE downloader_id IS NULL AND legacy_unassigned = 1")
.bind(id).execute(&self.pool).await?.rows_affected())
}
pub async fn legacy_unassigned_count(&self) -> Result<i64> {
Ok(sqlx::query_scalar("SELECT COUNT(*) FROM download_jobs WHERE legacy_unassigned = 1 AND downloader_id IS NULL")
.fetch_one(&self.pool).await?)
}
pub(crate) async fn bootstrap_downloader(
&self,
config: Option<&crate::config::QbitConfig>,
) -> Result<()> {
let Some(config) = config else {
return Ok(());
};
let mut config = config.clone();
config.validate()?;
let mut tx = self.pool.begin().await?;
let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM downloader_instances")
.fetch_one(&mut *tx)
.await?;
if count == 0 {
let id = uuid::Uuid::new_v4().to_string();
sqlx::query("INSERT INTO downloader_instances(id, name, kind, endpoint, config) VALUES (?, 'Initial qBittorrent', 'qbittorrent', ?, ?)")
.bind(&id).bind(&config.url).bind(serde_json::to_string(&config)?).execute(&mut *tx).await?;
sqlx::query("UPDATE downloader_selection SET downloader_id = ? WHERE singleton = 1")
.bind(&id)
.execute(&mut *tx)
.await?;
// Initial import uses the operator-supplied original QBIT_* connection.
sqlx::query("UPDATE download_jobs SET downloader_id = ?, legacy_unassigned = 0 WHERE downloader_id IS NULL")
.bind(&id).execute(&mut *tx).await?;
}
tx.commit().await?;
Ok(())
}
pub async fn open(url: &str) -> Result<Self> {
let options = SqliteConnectOptions::from_str(url)?
.create_if_missing(true)
.foreign_keys(true)
.journal_mode(SqliteJournalMode::Wal)
.busy_timeout(Duration::from_secs(5));
// A single connection also makes sqlite::memory: deterministic in tests.
let pool = SqlitePoolOptions::new()
.max_connections(1)
.connect_with(options)
.await?;
sqlx::migrate!().run(&pool).await?;
Ok(Self { pool })
}
pub async fn subscriptions(&self) -> Result<Vec<Subscription>> {
let rows = sqlx::query_as::<_, (String, String, Option<i64>, Option<String>)>(
"SELECT id, spec, last_checked_at, last_error FROM subscriptions WHERE deleted = 0 ORDER BY rowid"
).fetch_all(&self.pool).await?;
rows.into_iter()
.map(|(id, spec, last_checked_at, last_error)| {
Ok(Subscription {
id,
input: serde_json::from_str(&spec)?,
last_checked_at,
last_error,
})
})
.collect()
}
pub async fn subscription(&self, id: &str) -> Result<Subscription> {
self.subscriptions()
.await?
.into_iter()
.find(|s| s.id == id)
.ok_or(Error::NotFound)
}
pub async fn create(&self, mut input: SubscriptionInput) -> Result<Subscription> {
input.validate()?;
let id = uuid::Uuid::new_v4().to_string();
sqlx::query("INSERT INTO subscriptions(id, spec) VALUES (?, ?)")
.bind(&id)
.bind(serde_json::to_string(&input)?)
.execute(&self.pool)
.await?;
self.subscription(&id).await
}
pub async fn update(&self, id: &str, mut input: SubscriptionInput) -> Result<Subscription> {
input.validate()?;
let result = sqlx::query("UPDATE subscriptions SET spec = ? WHERE id = ? AND deleted = 0")
.bind(serde_json::to_string(&input)?)
.bind(id)
.execute(&self.pool)
.await?;
if result.rows_affected() == 0 {
return Err(Error::NotFound);
}
self.subscription(id).await
}
pub async fn delete(&self, id: &str) -> Result<()> {
let result =
sqlx::query("UPDATE subscriptions SET deleted = 1 WHERE id = ? AND deleted = 0")
.bind(id)
.execute(&self.pool)
.await?;
if result.rows_affected() == 0 {
return Err(Error::NotFound);
}
Ok(())
}
pub async fn checked(&self, id: &str, error: Option<&str>) -> Result<()> {
sqlx::query("UPDATE subscriptions SET last_checked_at = ?, last_error = ? WHERE id = ?")
.bind(now())
.bind(error)
.bind(id)
.execute(&self.pool)
.await?;
Ok(())
}
pub async fn already_queued(&self, id: &str, season: i32, release: &Release) -> Result<bool> {
let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM download_jobs WHERE subscription_id = ? AND (resource_key = ? OR (season = ? AND episode = ?))")
.bind(id).bind(&release.key).bind(season).bind(release.episode).fetch_one(&self.pool).await?;
Ok(count > 0)
}
pub async fn record_scan(&self, sub: &Subscription, releases: &[Release]) -> Result<usize> {
let mut tx = self.pool.begin().await?;
let downloader_id: Option<String> = sqlx::query_scalar(
"SELECT downloader_id FROM downloader_selection WHERE singleton = 1",
)
.fetch_one(&mut *tx)
.await?;
let destination_id: Option<String> = sqlx::query_scalar(
"SELECT destination_id FROM destination_selection WHERE singleton = 1",
)
.fetch_one(&mut *tx)
.await?;
let mut mapping_id = None;
let mut download_save_path = None;
if let Some(destination) = &destination_id {
let mapping: crate::media::Mapping = sqlx::query_as("SELECT * FROM destination_mappings WHERE destination_id = ? AND downloader_id = ? ORDER BY version DESC LIMIT 1")
.bind(destination).bind(&downloader_id).fetch_optional(&mut *tx).await?
.ok_or_else(|| Error::Invalid("active destination has no mapping for the selected downloader".into()))?;
mapping_id = Some(mapping.id);
download_save_path = Some(mapping.download_root);
} else if let Some(downloader) = &downloader_id {
let config: String =
sqlx::query_scalar("SELECT config FROM downloader_instances WHERE id = ?")
.bind(downloader)
.fetch_one(&mut *tx)
.await?;
download_save_path =
Some(serde_json::from_str::<crate::config::QbitConfig>(&config)?.save_path);
}
let mut queued = 0;
for release in releases {
sqlx::query("INSERT INTO releases(subscription_id, resource_key, payload) VALUES (?, ?, ?) ON CONFLICT(subscription_id, resource_key) DO UPDATE SET payload = excluded.payload")
.bind(&sub.id).bind(&release.key).bind(serde_json::to_string(release)?).execute(&mut *tx).await?;
if release.decision == Decision::Ready {
if let (Some(episode), Some(url)) = (release.episode, &release.download_url) {
let result = sqlx::query("INSERT OR IGNORE INTO download_jobs(id, subscription_id, resource_key, title, season, episode, download_url, downloader_id, destination_id, mapping_id, download_save_path) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
.bind(uuid::Uuid::new_v4().to_string()).bind(&sub.id).bind(&release.key)
.bind(&release.title).bind(sub.input.season).bind(episode).bind(url).bind(&downloader_id)
.bind(&destination_id).bind(&mapping_id).bind(&download_save_path).execute(&mut *tx).await?;
queued += result.rows_affected() as usize;
}
}
}
tx.commit().await?;
Ok(queued)
}
pub async fn releases(&self, id: &str) -> Result<Vec<Release>> {
self.subscription(id).await?;
let rows: Vec<String> = sqlx::query_scalar(
"SELECT payload FROM releases WHERE subscription_id = ? ORDER BY rowid DESC LIMIT 500",
)
.bind(id)
.fetch_all(&self.pool)
.await?;
rows.into_iter()
.map(|s| serde_json::from_str(&s).map_err(Error::from))
.collect()
}
pub async fn jobs(&self) -> Result<Vec<DownloadJob>> {
Ok(
sqlx::query_as("SELECT * FROM download_jobs ORDER BY rowid DESC LIMIT 500")
.fetch_all(&self.pool)
.await?,
)
}
pub async fn active_jobs(&self) -> Result<Vec<DownloadJob>> {
Ok(sqlx::query_as("SELECT * FROM download_jobs WHERE state NOT IN ('completed', 'missing', 'failed') AND next_attempt_at <= ? ORDER BY rowid")
.bind(now()).fetch_all(&self.pool).await?)
}
pub async fn prepare(&self, id: &str, hash: &str, bytes: Option<&[u8]>) -> Result<()> {
sqlx::query("UPDATE download_jobs SET info_hash = ?, torrent_bytes = ? WHERE id = ?")
.bind(hash)
.bind(bytes)
.bind(id)
.execute(&self.pool)
.await?;
Ok(())
}
pub async fn torrent_bytes(&self, id: &str) -> Result<Option<Vec<u8>>> {
Ok(
sqlx::query_scalar("SELECT torrent_bytes FROM download_jobs WHERE id = ?")
.bind(id)
.fetch_one(&self.pool)
.await?,
)
}
pub async fn state(&self, id: &str, state: &str) -> Result<()> {
sqlx::query("UPDATE download_jobs SET state = ?, last_error = NULL, next_attempt_at = 0 WHERE id = ?")
.bind(state).bind(id).execute(&self.pool).await?;
Ok(())
}
pub async fn failed(&self, job: &DownloadJob, error: &str) -> Result<()> {
// Keep 'submitting' on an ambiguous network failure, so reconciliation comes first.
let delay = 30 * (1_i64 << job.attempts.clamp(0, 7));
sqlx::query("UPDATE download_jobs SET attempts = attempts + 1, last_error = ?, next_attempt_at = ? WHERE id = ?")
.bind(error).bind(now() + delay.min(3600)).bind(&job.id).execute(&self.pool).await?;
Ok(())
}
pub async fn retry(&self, id: &str) -> Result<()> {
let result = sqlx::query("UPDATE download_jobs SET state = 'queued', attempts = 0, next_attempt_at = 0, last_error = NULL WHERE id = ? AND state IN ('failed', 'missing')")
.bind(id).execute(&self.pool).await?;
if result.rows_affected() == 0 {
return Err(Error::Invalid(
"only failed or missing jobs can be retried".into(),
));
}
Ok(())
}
}
tests/fixtures/releases.xml
Vendored
+28
View File
@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>Local test releases</title>
<link>https://example.invalid/</link>
<description>Synthetic fixtures; no real media</description>
<item>
<guid>release-3</guid>
<title>[Group] Example - 03 [1080p]</title>
<enclosure url="magnet:?xt=urn:btih:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" type="application/x-bittorrent" length="1"/>
</item>
<item>
<guid>release-3-other</guid>
<title>[Group] Example - 03v2 [1080p]</title>
<enclosure url="magnet:?xt=urn:btih:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" type="application/x-bittorrent" length="1"/>
</item>
<item>
<guid>batch</guid>
<title>[Group] Example [01-12] [1080p]</title>
<enclosure url="magnet:?xt=urn:btih:cccccccccccccccccccccccccccccccccccccccc" type="application/x-bittorrent" length="1"/>
</item>
<item>
<guid>low-resolution</guid>
<title>[Group] Example - 04 [720p]</title>
<enclosure url="magnet:?xt=urn:btih:dddddddddddddddddddddddddddddddddddddddd" type="application/x-bittorrent" length="1"/>
</item>
</channel>
</rss>
tests/local_media.rs
+154
View File
@@ -0,0 +1,154 @@
// SPDX-License-Identifier: GPL-2.0-only
use actix_web::{test, web, App};
use rust_project::{api, config::Config, service::Service, storage::Store};
use serde_json::{json, Value};
use std::{sync::Arc, time::Duration};
#[actix_web::test]
async fn directory_api_rejects_traversal_and_does_not_follow_links_outside_root() {
let directory = std::env::temp_dir().join(format!("media-browse-{}", uuid::Uuid::new_v4()));
let root = directory.join("root");
std::fs::create_dir_all(root.join("Series")).unwrap();
std::fs::create_dir(directory.join("outside")).unwrap();
std::fs::write(root.join("Series/ep01.mkv"), b"abc").unwrap();
std::fs::write(directory.join("outside/secret"), b"outside").unwrap();
#[cfg(unix)]
std::os::unix::fs::symlink(directory.join("outside"), root.join("escape")).unwrap();
let config = Config {
database_url: "sqlite::memory:".into(),
port: 8080,
poll_interval: Duration::from_secs(300),
qbit: None,
mikan_base_url: "https://mikanani.me".into(),
website: Default::default(),
};
let service = Arc::new(
Service::new(Store::open("sqlite::memory:").await.unwrap(), &config)
.await
.unwrap(),
);
let app = test::init_service(
App::new()
.app_data(web::Data::new(service))
.configure(api::configure),
)
.await;
for body in [
json!({"name":"bad", "kind":"local", "root_path":"relative"}),
json!({"name":"bad", "kind":"smb", "root_path":"/share"}),
] {
let response = test::call_service(
&app,
test::TestRequest::post()
.uri("/api/destinations")
.set_json(body)
.to_request(),
)
.await;
assert_eq!(response.status(), 400);
}
let response = test::call_service(
&app,
test::TestRequest::post()
.uri("/api/destinations")
.set_json(json!({"name":"Local", "kind":"local", "root_path":root.to_str().unwrap()}))
.to_request(),
)
.await;
assert_eq!(response.status(), 201);
let destination: Value = test::read_body_json(response).await;
let id = destination["id"].as_str().unwrap();
let response = test::call_service(
&app,
test::TestRequest::put()
.uri("/api/destinations/active")
.set_json(json!({"destination_id":id}))
.to_request(),
)
.await;
assert_eq!(response.status(), 400); // No downloader mapping: never silently use a fallback.
let downloader: Value = test::call_and_read_body_json(&app, test::TestRequest::post().uri("/api/downloaders")
.set_json(json!({"name":"Local qBit", "kind":"qbittorrent", "connection":{
"url":"http://127.0.0.1:1", "username":"", "password":"", "save_path":"/downloads"
}})).to_request()).await;
let response = test::call_service(
&app,
test::TestRequest::put()
.uri("/api/downloaders/active")
.set_json(json!({"downloader_id":downloader["id"]}))
.to_request(),
)
.await;
assert_eq!(response.status(), 204);
let response = test::call_service(
&app,
test::TestRequest::post()
.uri(&format!("/api/destinations/{id}/mappings"))
.set_json(json!({"downloader_id":downloader["id"],"download_root":"/downloads/local"}))
.to_request(),
)
.await;
assert_eq!(response.status(), 201);
let mapping: Value = test::read_body_json(response).await;
assert_eq!(mapping["version"], 1);
let response = test::call_service(
&app,
test::TestRequest::put()
.uri("/api/destinations/active")
.set_json(json!({"destination_id":id}))
.to_request(),
)
.await;
assert_eq!(response.status(), 204);
let status: Value = test::call_and_read_body_json(
&app,
test::TestRequest::get().uri("/api/status").to_request(),
)
.await;
assert_eq!(status["active_destination_id"], id);
let base = format!("/api/destinations/{id}/entries");
let listing: Value = test::call_and_read_body_json(
&app,
test::TestRequest::get()
.uri(&format!("{base}?path=Series"))
.to_request(),
)
.await;
assert_eq!(listing["status"], "available");
assert_eq!(listing["entries"][0]["name"], "ep01.mkv");
assert_eq!(listing["entries"][0]["size"], 3);
for path in [
"../outside",
"%2Fetc",
"C%3A%2FWindows",
"Series%5C..%5Coutside",
"Series/../outside",
"Series//bad",
] {
let response = test::call_service(
&app,
test::TestRequest::get()
.uri(&format!("{base}?path={path}"))
.to_request(),
)
.await;
assert_eq!(response.status(), 400, "{path}");
}
#[cfg(unix)]
{
let listing: Value = test::call_and_read_body_json(
&app,
test::TestRequest::get()
.uri(&format!("{base}?path=escape"))
.to_request(),
)
.await;
assert_eq!(listing["status"], "unavailable");
assert_eq!(listing["entries"], json!([]));
}
std::fs::rename(&root, directory.join("offline")).unwrap();
let listing: Value =
test::call_and_read_body_json(&app, test::TestRequest::get().uri(&base).to_request()).await;
assert_eq!(listing["status"], "unavailable");
std::fs::remove_dir_all(directory).unwrap();
}
tests/mikan.rs
+319
View File
@@ -0,0 +1,319 @@
// SPDX-License-Identifier: GPL-2.0-only
use actix_web::{dev::ServerHandle, test, web, App, HttpRequest, HttpResponse, HttpServer};
use rust_project::{api, config::Config, service::Service, storage::Store};
use serde_json::Value;
use std::{net::TcpListener, sync::Arc, time::Duration};
const SEARCH: &str = r#"<form action="/Home/Search"><input name="searchstr"></form>
<div class="central-container"><ul class="list-inline an-ul">
<li><a href="/Home/Bangumi/101"><div class="an-text" title="测试番剧">测试番剧</div></a></li>
<li><a href="/Home/Bangumi/202"><div class="an-text" title="测试番剧 第二季">测试番剧 第二季</div></a></li>
</ul></div>"#;
// Independently authored fixtures; publish-group ID deliberately differs from RSS group ID.
const DETAIL: &str = r#"<p class="bangumi-title">测试番剧
<a class="mikan-rss" href="/RSS/Bangumi?bangumiId=101"></a></p>
<p class="bangumi-info">放送日期：星期五</p>
<p class="bangumi-info">Bangumi番组计划链接：<a href="https://bgm.tv/subject/400602">详情</a></p>
<div class="subgroup-text" id="12"><a href="/Home/PublishGroup/99">字幕组 &amp; A</a>
<a class="mikan-rss" href="/RSS/Bangumi?bangumiId=101&amp;subgroupid=12"></a></div>"#;
async fn mock() -> (String, ServerHandle) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let server = HttpServer::new(|| {
App::new().default_service(web::to(|req: HttpRequest| async move {
if req.path() == "/Home/Bangumi/202" {
return HttpResponse::Ok().content_type("text/html").body(format!(
"{}<div class=\"bangumi-poster\" style=\"background-image: url('/images/Bangumi/cover.png');\"></div>",
DETAIL.replace("101", "202")));
}
if req.path() == "/images/Bangumi/cover.png" {
return HttpResponse::Ok().content_type("image/png").body(b"\x89PNG\r\n\x1a\nfixture".as_slice());
}
if req.path() == "/Home/Bangumi/101" {
return HttpResponse::Ok().content_type("text/html").body(DETAIL);
}
if req.path() == "/RSS/Bangumi" {
assert_eq!(req.query_string(), "bangumiId=101&subgroupid=12");
return HttpResponse::Ok()
.content_type("application/rss+xml")
.body(include_str!("fixtures/releases.xml"));
}
assert_eq!(req.path(), "/Home/Search");
let query = web::Query::<std::collections::HashMap<String, String>>::from_query(
req.query_string(),
)
.unwrap();
match query.get("searchstr").map(String::as_str) {
Some("empty") => return HttpResponse::Ok().content_type("text/html")
.body(r#"<form action="/Home/Search"><input name="searchstr"></form><div class="central-container"><ul class="an-ul"></ul></div>"#),
Some("challenge") => return HttpResponse::Ok().content_type("text/html").body("<h1>Verify your browser</h1>"),
Some("changed") => return HttpResponse::Ok().content_type("text/html")
.body(r#"<form action="/Home/Search"><input name="searchstr"></form><ul class="an-ul"><a href="/Home/Bangumi/101"><span class="new-title">番剧</span></a></ul>"#),
_ => {}
}
assert_eq!(query.get("searchstr").unwrap(), "测试 & S2");
HttpResponse::Ok()
.content_type("text/html; charset=utf-8")
.body(SEARCH)
}))
})
.workers(1)
.listen(listener)
.unwrap()
.run();
let handle = server.handle();
actix_web::rt::spawn(server);
(url, handle)
}
#[actix_web::test]
async fn covers_use_an_application_url_and_details_remain_available_in_bounded_cache() {
let (base, server) = mock().await;
let service = Service::new(
Store::open("sqlite::memory:").await.unwrap(),
&config(&base),
)
.await
.unwrap();
let app = test::init_service(
App::new()
.app_data(web::Data::new(Arc::new(service)))
.configure(api::configure),
)
.await;
let detail: Value = test::call_and_read_body_json(
&app,
test::TestRequest::get()
.uri("/api/sources/mikan/series/202")
.to_request(),
)
.await;
assert_eq!(detail["cover_url"], "/api/sources/mikan/series/202/cover");
let response = test::call_service(
&app,
test::TestRequest::get()
.uri(detail["cover_url"].as_str().unwrap())
.to_request(),
)
.await;
assert_eq!(response.status(), 200);
assert_eq!(response.headers().get("Content-Type").unwrap(), "image/png");
assert_eq!(
test::read_body(response).await.as_ref(),
b"\x89PNG\r\n\x1a\nfixture"
);
server.stop(false).await;
let cached: Value = test::call_and_read_body_json(
&app,
test::TestRequest::get()
.uri("/api/sources/mikan/series/202")
.to_request(),
)
.await;
assert_eq!(cached, detail);
}
#[actix_web::test]
async fn empty_results_are_distinct_from_challenges_or_changed_markup() {
let (base, server) = mock().await;
let service = Service::new(
Store::open("sqlite::memory:").await.unwrap(),
&config(&base),
)
.await
.unwrap();
let app = test::init_service(
App::new()
.app_data(web::Data::new(Arc::new(service)))
.configure(api::configure),
)
.await;
for (query, status) in [
("empty", 200),
("challenge", 502),
("changed", 502),
("%20", 400),
] {
let response = test::call_service(
&app,
test::TestRequest::get()
.uri(&format!("/api/sources/mikan/search?q={query}"))
.to_request(),
)
.await;
assert_eq!(response.status(), status, "query {query}");
if status == 200 {
let value: Value = test::read_body_json(response).await;
assert_eq!(value["series"], serde_json::json!([]));
}
}
server.stop(false).await;
}
#[actix_web::test]
async fn group_candidate_can_be_previewed_and_saved_disabled_without_downloading() {
let (base, server) = mock().await;
let service = Service::new(
Store::open("sqlite::memory:").await.unwrap(),
&config(&base),
)
.await
.unwrap();
let app = test::init_service(
App::new()
.app_data(web::Data::new(Arc::new(service)))
.configure(api::configure),
)
.await;
let response = test::call_service(
&app,
test::TestRequest::get()
.uri("/api/sources/mikan/series/101")
.to_request(),
)
.await;
assert_eq!(response.status(), 200);
let detail: Value = test::read_body_json(response).await;
assert_eq!(detail["title"], "测试番剧");
assert_eq!(detail["bangumi_id"], "400602");
assert_eq!(detail["groups"][0]["group_id"], "12");
assert_eq!(detail["groups"][0]["name"], "字幕组 & A");
assert!(detail["cover_url"].is_null());
let response = test::call_service(
&app,
test::TestRequest::post()
.uri("/api/sources/mikan/resolve")
.set_json(serde_json::json!({"url": detail["groups"][0]["rss_url"]}))
.to_request(),
)
.await;
assert_eq!(response.status(), 200);
let candidate: Value = test::read_body_json(response).await;
assert_eq!(candidate["scope"], "group");
assert_eq!(candidate["series_id"], "101");
assert_eq!(candidate["group_id"], "12");
assert_eq!(candidate["subscription"]["enabled"], false);
let preview: Value = test::call_and_read_body_json(
&app,
test::TestRequest::post()
.uri("/api/preview")
.set_json(&candidate["subscription"])
.to_request(),
)
.await;
assert_eq!(preview[0]["episode"], 3);
let saved = test::call_service(
&app,
test::TestRequest::post()
.uri("/api/subscriptions")
.set_json(&candidate["subscription"])
.to_request(),
)
.await;
assert_eq!(saved.status(), 201);
let jobs: Value =
test::call_and_read_body_json(&app, test::TestRequest::get().uri("/api/jobs").to_request())
.await;
assert_eq!(jobs, serde_json::json!([]));
server.stop(true).await;
}
fn config(base: &str) -> Config {
Config {
database_url: "sqlite::memory:".into(),
port: 8080,
poll_interval: Duration::from_secs(300),
qbit: None,
mikan_base_url: base.into(),
website: Default::default(),
}
}
#[actix_web::test]
async fn resolver_marks_keyword_scope_and_rejects_foreign_or_ambiguous_urls() {
let (base, server) = mock().await;
let service = Service::new(
Store::open("sqlite::memory:").await.unwrap(),
&config(&base),
)
.await
.unwrap();
let app = test::init_service(
App::new()
.app_data(web::Data::new(Arc::new(service)))
.configure(api::configure),
)
.await;
let response = test::call_service(
&app,
test::TestRequest::post()
.uri("/api/sources/mikan/resolve")
.set_json(serde_json::json!({"url": format!("{base}/RSS/Search?searchstr=test")}))
.to_request(),
)
.await;
assert_eq!(response.status(), 200);
let candidate: Value = test::read_body_json(response).await;
assert_eq!(candidate["scope"], "keyword");
assert!(candidate["series_id"].is_null());
assert!(candidate["subscription"].is_null());
assert!(!candidate["warnings"].as_array().unwrap().is_empty());
for url in [
"https://other.invalid/Home/Bangumi/101".into(),
format!("{base}/RSS/Bangumi?bangumiId=101&bangumiId=202"),
format!("{base}/RSS/Bangumi?bangumiId=101&subgroupid=999"),
format!("{base}/Home/Bangumi/not-an-id"),
] {
let response = test::call_service(
&app,
test::TestRequest::post()
.uri("/api/sources/mikan/resolve")
.set_json(serde_json::json!({"url": url}))
.to_request(),
)
.await;
assert_eq!(response.status(), 400);
}
server.stop(false).await;
}
#[actix_web::test]
async fn search_discovers_distinct_series_and_encoded_keyword_rss_without_creating_jobs() {
let (base, server) = mock().await;
let service = Service::new(
Store::open("sqlite::memory:").await.unwrap(),
&config(&base),
)
.await
.unwrap();
let app = test::init_service(
App::new()
.app_data(web::Data::new(Arc::new(service)))
.configure(api::configure),
)
.await;
let response = test::call_service(
&app,
test::TestRequest::get()
.uri("/api/sources/mikan/search?q=%E6%B5%8B%E8%AF%95%20%26%20S2")
.to_request(),
)
.await;
assert_eq!(response.status(), 200);
let body: Value = test::read_body_json(response).await;
assert_eq!(body["series"][0]["series_id"], "101");
assert_eq!(body["series"][1]["title"], "测试番剧 第二季");
let rss = reqwest::Url::parse(body["keyword_rss_url"].as_str().unwrap()).unwrap();
assert_eq!(rss.path(), "/RSS/Search");
assert_eq!(
rss.query_pairs().collect::<Vec<_>>(),
vec![("searchstr".into(), "测试 & S2".into())]
);
let jobs: Value =
test::call_and_read_body_json(&app, test::TestRequest::get().uri("/api/jobs").to_request())
.await;
assert_eq!(jobs, serde_json::json!([]));
server.stop(true).await;
}
tests/network.rs
+154
View File
@@ -0,0 +1,154 @@
// SPDX-License-Identifier: GPL-2.0-only
use actix_web::{dev::ServerHandle, test, web, App, HttpRequest, HttpResponse, HttpServer};
use rust_project::{
api,
config::{Config, QbitConfig},
service::Service,
storage::Store,
};
use serde_json::{json, Value};
use std::{net::TcpListener, sync::Arc, time::Duration};
async fn server(proxy: bool) -> (String, ServerHandle) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let base = format!("http://{}", listener.local_addr().unwrap());
let server = HttpServer::new(move || App::new().default_service(web::to(move |req: HttpRequest| async move {
if proxy {
assert_eq!(req.headers().get("Proxy-Authorization").and_then(|h| h.to_str().ok()), Some("Basic dXNlcjpwYXNz"), "request {}", req.uri());
if req.uri().host() == Some("127.0.0.1") { return HttpResponse::BadGateway().finish(); }
match req.path() {
"/to-direct" => {
let query = web::Query::<std::collections::HashMap<String, String>>::from_query(req.query_string()).unwrap();
HttpResponse::Found().insert_header(("Location", query.get("target").unwrap().as_str())).finish()
},
"/Home/Search" => HttpResponse::Found().insert_header(("Location", "http://assets.invalid/results")).finish(),
"/results" => {
assert_eq!(req.uri().host(), Some("assets.invalid"));
HttpResponse::Ok().content_type("text/html").body(r#"<form action="/Home/Search"><input name="searchstr"></form><ul class="an-ul"><a href="/Home/Bangumi/101"><div class="an-text">经代理搜索</div></a></ul>"#)
},
"/feed" => HttpResponse::Ok().body(include_str!("fixtures/releases.xml")),
_ => HttpResponse::BadGateway().finish(),
}
} else {
assert!(req.headers().get("Proxy-Authorization").is_none());
match req.path() {
"/api/v2/auth/login" => HttpResponse::Ok().insert_header(("Set-Cookie", "SID=test; Path=/")).body("Ok."),
"/api/v2/torrents/info" => HttpResponse::Ok().json(json!([])),
"/feed" => HttpResponse::Ok().body(include_str!("fixtures/releases.xml")),
_ => HttpResponse::NotFound().finish(),
}
}
}))).workers(1).listen(listener).unwrap().run();
let handle = server.handle();
actix_web::rt::spawn(server);
(base, handle)
}
#[actix_web::test]
async fn direct_exceptions_are_explicit_and_proxy_failure_never_falls_back() {
let (proxy, proxy_server) = server(true).await;
let (direct, direct_server) = server(false).await;
let proxy = proxy.replacen("http://", "http://user:pass@", 1);
for (direct_hosts, status) in [("127.0.0.1", 200), ("", 502)] {
let config = Config {
database_url: "sqlite::memory:".into(),
port: 8080,
poll_interval: Duration::from_secs(300),
qbit: None,
mikan_base_url: "http://mikan.invalid".into(),
website: rust_project::config::WebsiteConfig {
proxy_url: Some(proxy.clone()),
direct_hosts: direct_hosts.into(),
},
};
let service = Service::new(Store::open("sqlite::memory:").await.unwrap(), &config)
.await
.unwrap();
let app = test::init_service(
App::new()
.app_data(web::Data::new(Arc::new(service)))
.configure(api::configure),
)
.await;
let mut redirect = reqwest::Url::parse("http://unresolvable.invalid/to-direct").unwrap();
redirect
.query_pairs_mut()
.append_pair("target", &format!("{direct}/feed"));
let response = test::call_service(
&app,
test::TestRequest::post()
.uri("/api/preview")
.set_json(json!({"title":"Test", "rss_url": redirect.as_str()}))
.to_request(),
)
.await;
assert_eq!(response.status(), status);
if status == 200 {
let preview: Value = test::read_body_json(response).await;
assert_eq!(preview[0]["episode"], 3);
} else {
let body = String::from_utf8(test::read_body(response).await.to_vec()).unwrap();
assert!(!body.contains("user:pass"));
}
}
proxy_server.stop(false).await;
direct_server.stop(false).await;
}
#[actix_web::test]
async fn website_proxy_handles_auth_redirects_and_rss_while_downloader_stays_direct() {
let (proxy, proxy_server) = server(true).await;
let (direct, direct_server) = server(false).await;
let mut proxy_url = reqwest::Url::parse(&proxy).unwrap();
proxy_url.set_username("user").unwrap();
proxy_url.set_password(Some("pass")).unwrap();
let config = Config {
database_url: "sqlite::memory:".into(),
port: 8080,
poll_interval: Duration::from_secs(300),
mikan_base_url: "http://mikan.invalid".into(),
website: rust_project::config::WebsiteConfig {
proxy_url: Some(proxy_url.into()),
direct_hosts: String::new(),
},
qbit: Some(QbitConfig {
url: direct.clone(),
username: "test".into(),
password: "test".into(),
save_path: "/downloads".into(),
}),
};
let service = Service::new(Store::open("sqlite::memory:").await.unwrap(), &config)
.await
.unwrap();
let app = test::init_service(
App::new()
.app_data(web::Data::new(Arc::new(service)))
.configure(api::configure),
)
.await;
let search: Value = test::call_and_read_body_json(
&app,
test::TestRequest::get()
.uri("/api/sources/mikan/search?q=test")
.to_request(),
)
.await;
assert_eq!(search["series"][0]["title"], "经代理搜索");
let preview: Value = test::call_and_read_body_json(
&app,
test::TestRequest::post()
.uri("/api/preview")
.set_json(json!({"title":"Test", "rss_url":"http://unresolvable.invalid/feed"}))
.to_request(),
)
.await;
assert_eq!(preview[0]["episode"], 3);
let cycle: Value =
test::call_and_read_body_json(&app, test::TestRequest::post().uri("/api/run").to_request())
.await;
assert_eq!(cycle["errors"], json!([]));
assert_eq!(cycle["downloader_configured"], true);
proxy_server.stop(false).await;
direct_server.stop(false).await;
}
tests/pipeline.rs
+1100
View File
File diff suppressed because it is too large Load Diff