auto backup 2026-07-02 11:00
This commit is contained in:
@@ -0,0 +1,222 @@
|
||||
---
|
||||
name: "calibre-cleaner"
|
||||
description: "清洗 Calibre 书库：修复损坏书名/提取作者/检测编码损坏、小文件的批量删除"
|
||||
status: proposal
|
||||
version: "v1"
|
||||
date: "2026-06-24T02:29:02.278Z"
|
||||
---
|
||||
|
||||
# Calibre 书库清洗 Skill
|
||||
|
||||
## 概述
|
||||
|
||||
Calibre 书库整理三件套：
|
||||
|
||||
1. **书名清洗** —— 修复从 txt 导入产生的 HTML 标签/长文本/乱码书名
|
||||
2. **作者提取** —— 从书名中自动提取作者名（匹配 `作者：xxx`、`-xxx`、`BY xxx` 等格式）
|
||||
3. **脏数据清理** —— 检测并删除编码损坏的 txt、小于 10KB 的垃圾文件
|
||||
|
||||
## 前置条件
|
||||
|
||||
- 书库路径：`/home/yangxuan/calibre/books`
|
||||
- 元数据文件：`/home/yangxuan/calibre/books/metadata.db`
|
||||
- 运行前须停止其他 Calibre 进程（桌面版、Docker Calibre-Web）
|
||||
- 运行前 **必须备份** `metadata.db`
|
||||
|
||||
---
|
||||
|
||||
## 一、书名清洗 & 作者提取
|
||||
|
||||
（详见 skill 原内容，此处保留完整清洗逻辑）
|
||||
|
||||
---
|
||||
|
||||
## 二、脏数据清理
|
||||
|
||||
### 2.1 检测编码损坏的 TXT
|
||||
|
||||
检测条件：中文比例 < 10% 且含有乱码字符（`Ŀ¼ĴƿȿɿʼÈÌÒÙ（` 等 UTF-8 编码错误字）
|
||||
|
||||
```python
|
||||
import sqlite3, os
|
||||
|
||||
bad_char_set = set('Ŀ¼ĴƿȿɿʼÈÌÒÙ（')
|
||||
bad_ids = []
|
||||
|
||||
for bid, title, bpath, fname in rows:
|
||||
full_path = os.path.join('/home/yangxuan/calibre/books', bpath, fname + '.txt')
|
||||
if not os.path.exists(full_path):
|
||||
continue
|
||||
with open(full_path, 'rb') as f:
|
||||
raw = f.read(20480)
|
||||
text = raw.decode('utf-8', errors='replace')
|
||||
chinese = sum(1 for c in text if '\u4e00' <= c <= '\u9fff')
|
||||
total = len(text)
|
||||
ratio = chinese / total if total > 0 else 0
|
||||
has_bad = any(c in bad_char_set for c in text)
|
||||
if ratio < 0.10 and has_bad:
|
||||
bad_ids.append(bid)
|
||||
```
|
||||
|
||||
### 2.2 检测小文件
|
||||
|
||||
```python
|
||||
c.execute('''
|
||||
SELECT b.id, b.path FROM books b
|
||||
JOIN data d ON d.book = b.id
|
||||
WHERE d.format = 'TXT' AND d.uncompressed_size < 10240
|
||||
''')
|
||||
```
|
||||
|
||||
### 2.3 批量删除（通过 SQLite 直写）
|
||||
|
||||
停掉 calibre 进程后，通过 SQLite 删除记录并清理文件系统：
|
||||
|
||||
```sql
|
||||
DELETE FROM data WHERE book = ?
|
||||
DELETE FROM books_authors_link WHERE book = ?
|
||||
DELETE FROM books_languages_link WHERE book = ?
|
||||
DELETE FROM books_publishers_link WHERE book = ?
|
||||
DELETE FROM books_series_link WHERE book = ?
|
||||
DELETE FROM books_tags_link WHERE book = ?
|
||||
DELETE FROM comments WHERE book = ?
|
||||
DELETE FROM identifiers WHERE book = ?
|
||||
DELETE FROM books WHERE id = ?
|
||||
```
|
||||
|
||||
同时删除对应目录：`shutil.rmtree(os.path.join('/home/yangxuan/calibre/books', bpath))`
|
||||
|
||||
### 2.4 完整删除脚本
|
||||
|
||||
```python
|
||||
import sqlite3, os, shutil
|
||||
|
||||
db_path = '/home/yangxuan/calibre/books/metadata.db'
|
||||
conn = sqlite3.connect(db_path)
|
||||
c = conn.cursor()
|
||||
|
||||
c.execute('''
|
||||
SELECT b.id, b.title, b.path, d.name, d.uncompressed_size
|
||||
FROM books b
|
||||
JOIN data d ON d.book = b.id
|
||||
WHERE d.format = 'TXT'
|
||||
''')
|
||||
rows = c.fetchall()
|
||||
|
||||
bad_char_set = set('Ŀ¼ĴƿȿɿʼÈÌÒÙ（')
|
||||
to_delete = []
|
||||
|
||||
for bid, title, bpath, fname, size in rows:
|
||||
full_path = os.path.join('/home/yangxuan/calibre/books', bpath, fname + '.txt')
|
||||
if not os.path.exists(full_path):
|
||||
continue
|
||||
with open(full_path, 'rb') as f:
|
||||
raw = f.read(20480)
|
||||
text = raw.decode('utf-8', errors='replace')
|
||||
chinese = sum(1 for c in text if '\u4e00' <= c <= '\u9fff')
|
||||
ratio = chinese / len(text) if len(text) > 0 else 0
|
||||
has_bad = any(c in bad_char_set for c in text)
|
||||
if (ratio < 0.10 and has_bad) or size < 10240:
|
||||
to_delete.append((bid, bpath))
|
||||
|
||||
print(f"待删除: {len(to_delete)}")
|
||||
for bid, bpath in to_delete:
|
||||
c.execute('DELETE FROM data WHERE book=?', (bid,))
|
||||
c.execute('DELETE FROM books_authors_link WHERE book=?', (bid,))
|
||||
c.execute('DELETE FROM books_languages_link WHERE book=?', (bid,))
|
||||
c.execute('DELETE FROM books_publishers_link WHERE book=?', (bid,))
|
||||
c.execute('DELETE FROM books_series_link WHERE book=?', (bid,))
|
||||
c.execute('DELETE FROM books_tags_link WHERE book=?', (bid,))
|
||||
c.execute('DELETE FROM comments WHERE book=?', (bid,))
|
||||
c.execute('DELETE FROM identifiers WHERE book=?', (bid,))
|
||||
c.execute('DELETE FROM books WHERE id=?', (bid,))
|
||||
full_dir = os.path.join('/home/yangxuan/calibre/books', bpath)
|
||||
if os.path.exists(full_dir):
|
||||
shutil.rmtree(full_dir)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"完成，删除: {len(to_delete)}")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 三、常用操作
|
||||
|
||||
### 3.1 查看 XX 作者的书
|
||||
|
||||
```bash
|
||||
sqlite3 /home/yangxuan/calibre/books/metadata.db "
|
||||
SELECT b.id, b.title
|
||||
FROM books b JOIN books_authors_link l ON b.id = l.book
|
||||
JOIN authors a ON l.author = a.id
|
||||
WHERE a.name = 'Unknown'
|
||||
LIMIT 20;
|
||||
"
|
||||
```
|
||||
|
||||
### 3.2 统计作者分布
|
||||
|
||||
```bash
|
||||
sqlite3 /home/yangxuan/calibre/books/metadata.db "
|
||||
SELECT a.name, COUNT(*) as cnt
|
||||
FROM books b JOIN books_authors_link l ON b.id = l.book
|
||||
JOIN authors a ON l.author = a.id
|
||||
GROUP BY a.name ORDER BY cnt DESC LIMIT 20;
|
||||
"
|
||||
```
|
||||
|
||||
### 3.3 书库统计
|
||||
|
||||
```bash
|
||||
python3 -c "
|
||||
import sqlite3
|
||||
conn = sqlite3.connect('/home/yangxuan/calibre/books/metadata.db')
|
||||
c = conn.cursor()
|
||||
c.execute('SELECT COUNT(*) FROM books')
|
||||
print(f'总书籍: {c.fetchone()[0]}')
|
||||
c.execute('SELECT COUNT(*) FROM data WHERE format=\"TXT\"')
|
||||
print(f'TXT 文件: {c.fetchone()[0]}')
|
||||
conn.close()
|
||||
"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 四、操作流程（推荐顺序）
|
||||
|
||||
```bash
|
||||
cd /home/yangxuan/calibre
|
||||
|
||||
# 1. 停服务 + 备份
|
||||
docker compose down
|
||||
cp books/metadata.db books/metadata.db.backup.$(date +%Y%m%d)
|
||||
killall calibre 2>/dev/null
|
||||
|
||||
# 2. 脏数据清理（编码损坏 + 小文件）
|
||||
python3 /path/to/clean_dirty_data.py
|
||||
|
||||
# 3. 书名清洗 & 作者提取
|
||||
calibre-debug /path/to/fix_titles.py
|
||||
|
||||
# 4. 重启
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 五、已知问题
|
||||
|
||||
### `set_metadata` 报 `Errno 36 文件名过长`
|
||||
|
||||
当 `author_sort` 字段超 80 字节时，Calibre 路径会触及 ext4 255 字节限制。先清理极长作者名：
|
||||
|
||||
```bash
|
||||
sqlite3 /home/yangxuan/calibre/books/metadata.db "
|
||||
UPDATE authors SET name='Unknown' WHERE LENGTH(name) > 30;
|
||||
"
|
||||
```
|
||||
|
||||
### WSL `/mnt/f` 删除 NTFS 文件权限不够
|
||||
|
||||
WSL 的 drvfs 默认不带 metadata 选项，`sudo rm`、`os.remove`、`cmd.exe /c del` 均无法删除 NTFS 文件。不要在 WSL 下尝试跨分区删除操作，直接让 Windows 侧（文件资源管理器/PowerShell）操作。
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":"openclaw.skill-workshop.proposal.v1","id":"calibre-cleaner-20260624-a441798af9","kind":"create","status":"applied","title":"Create calibre-cleaner","description":"清洗 Calibre 书库：修复损坏书名/提取作者/检测编码损坏、小文件的批量删除","createdAt":"2026-06-24T02:29:02.278Z","updatedAt":"2026-06-24T02:29:27.227Z","createdBy":"skill-workshop","origin":{"agentId":"storage","sessionKey":"agent:storage:main","runId":"6e4a8a08-2c78-4b57-866f-c8377d65f96c","messageId":"6e4a8a08-2c78-4b57-866f-c8377d65f96c"},"proposedVersion":"v1","draftFile":"PROPOSAL.md","draftHash":"15703b0a633a8d2adfce44b7b6623d42f0a25778e39c1436b002a42da67c974e","target":{"skillName":"calibre-cleaner","skillKey":"calibre-cleaner","skillDir":"/home/yangxuan/.openclaw/workspace/storage/skills/calibre-cleaner","skillFile":"/home/yangxuan/.openclaw/workspace/storage/skills/calibre-cleaner/SKILL.md","source":"openclaw-workspace"},"scan":{"state":"clean","scannedAt":"2026-06-24T02:29:27.216Z","critical":0,"warn":0,"info":0,"findings":[]},"appliedAt":"2026-06-24T02:29:27.227Z"}
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":"openclaw.skill-workshop.rollback.v1","proposalId":"calibre-cleaner-20260624-a441798af9","writtenAt":"2026-06-24T02:29:27.217Z","targetSkillFile":"/home/yangxuan/.openclaw/workspace/storage/skills/calibre-cleaner/SKILL.md","action":"create"}
|
||||
Reference in New Issue
Block a user