auto backup 2026-07-02 11:00
This commit is contained in:
@@ -0,0 +1,284 @@
|
||||
---
|
||||
name: "calibre-title-cleaner"
|
||||
description: "清洗 Calibre 书库中损坏的 txt 文档书名（HTML标签/长文本/乱码/路径超限修复）"
|
||||
status: proposal
|
||||
version: "v2"
|
||||
date: "2026-06-24T01:12:53.053Z"
|
||||
---
|
||||
|
||||
# Calibre 书库书名清洗 & 作者提取 Skill
|
||||
|
||||
## 概述
|
||||
|
||||
清洗 Calibre 书库中因从 txt 文件自动导入而产生的损坏书名，**同时从书名中提取作者信息**并更新到作者字段。适用于 Calibre 书库路径 `/home/yangxuan/calibre/books`。
|
||||
|
||||
## 前置条件
|
||||
|
||||
- Calibre 数据库路径：`/home/yangxuan/calibre/books`
|
||||
- 元数据文件：`/home/yangxuan/calibre/books/metadata.db`
|
||||
- 需要 `calibre-debug` 或 `calibre` Python 环境（使用 `LibraryDatabase` API）
|
||||
- **运行前须停止其他 Calibre 进程**（桌面版、Calibre-Web Docker）
|
||||
- **运行前必须备份** `metadata.db`
|
||||
|
||||
## 作者提取规则（从书名中提取）
|
||||
|
||||
### 需要匹配的作者格式（基于实际书库数据）
|
||||
|
||||
格式 | 示例 | 正则
|
||||
---|---|---
|
||||
`作者：xxx`（中文冒号） | `...作者：镜欲` | `作者[：:]\s*([^\s,，、\)）]+)`
|
||||
`作者:xxx`（英文冒号） | `...作者:镜欲` | 同上
|
||||
`-xxx`（英文 dash 结尾） | `善良的美艳妈妈-糠麸` | `-([^-()【】\[\]]+)$`
|
||||
`翻译：xxx` | `...翻译：kkmanlg` | `翻译[：:]\s*([^\s,，、\)）]+)`
|
||||
`BY xxx`（英文 BY） | `...BY狱厨` | `\b[Bb][Yy]\s+([^\s,，、\)）]+)`
|
||||
`&xxx`（and 符结尾） | `...&axenic` | `&([^&()【】\[\] ]+)$`
|
||||
|
||||
### 提取策略
|
||||
|
||||
1. **优先匹配精确的"作者："标记**（最可靠）
|
||||
2. 失败后尝试 `-xxx` 结尾（短作者名）
|
||||
3. 再尝试 `翻译：`、`BY`、`&xxx` 等
|
||||
4. 如果提取到的作者名仍是乱填的（如 `正文`、`第01章`、`【内容简介】`），**跳过**，标为 Unknown
|
||||
|
||||
### 排除项（以下不是真作者）
|
||||
|
||||
- `正文`、`内容简介`
|
||||
- `第\d+[章节]`（章节名）
|
||||
- 长度 > 30 字符的内容摘要
|
||||
|
||||
## 书名清洗规则
|
||||
|
||||
### `shorten(s, max_bytes=120)`
|
||||
|
||||
将字符串截断到指定字节数，避免因 UTF-8 中文字符跨字节而损坏。
|
||||
|
||||
```python
|
||||
def shorten(s, max_bytes=120):
|
||||
encoded = s.encode('utf-8')
|
||||
if len(encoded) <= max_bytes:
|
||||
return s
|
||||
try:
|
||||
return encoded[:max_bytes].decode('utf-8')
|
||||
except:
|
||||
return encoded[:max_bytes-3].decode('utf-8', errors='ignore')
|
||||
```
|
||||
|
||||
### `clean_title(title)`
|
||||
|
||||
```python
|
||||
def clean_title(title):
|
||||
t = title
|
||||
# 移除 HTML 标签
|
||||
t = re.sub(r'<[^>]+>', '', t)
|
||||
# 替换 _BR_ / _BR 标记
|
||||
t = re.sub(r'_BR_', ' ', t, flags=re.IGNORECASE)
|
||||
t = re.sub(r'_BR(?![a-zA-Z])', ' ', t)
|
||||
# 移除 _fontxxx_ / _imgxxx_ 标记
|
||||
t = re.sub(r'_font[^_]*_', '', t)
|
||||
t = re.sub(r'_img[^_]*_', '', t)
|
||||
# 合并多余空格
|
||||
t = re.sub(r'\s+', ' ', t).strip()
|
||||
# 超长截短
|
||||
if len(t) > 100:
|
||||
m = re.search(r'[\u4e00-\u9fff]{2,}', t)
|
||||
if m:
|
||||
t = t[m.start():]
|
||||
t = t[:60].rstrip('，。；：、！？…\ufffd\'"').rstrip('.,;:!?\'').strip()
|
||||
# 移除控制字符
|
||||
t = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f]', '', t)
|
||||
return shorten(t, 100).strip() or 'Unknown'
|
||||
```
|
||||
|
||||
### 坏书检测条件
|
||||
|
||||
```python
|
||||
is_bad = (
|
||||
len(title) > 100 or
|
||||
'<img' in title or '<BR' in title or '<font' in title or
|
||||
'\ufffd' in title or
|
||||
len(title.encode('utf-8')) > 150
|
||||
)
|
||||
```
|
||||
|
||||
## 完整脚本示例（含作者提取）
|
||||
|
||||
```python
|
||||
import re
|
||||
from calibre.db.legacy import LibraryDatabase
|
||||
|
||||
def shorten(s, max_bytes=120):
|
||||
encoded = s.encode('utf-8')
|
||||
if len(encoded) <= max_bytes:
|
||||
return s
|
||||
try:
|
||||
return encoded[:max_bytes].decode('utf-8')
|
||||
except:
|
||||
return encoded[:max_bytes-3].decode('utf-8', errors='ignore')
|
||||
|
||||
EXCLUDE_AUTHORS = {'正文', '内容简介', ''}
|
||||
|
||||
def extract_author_from_title(title):
|
||||
"""从书名中提取作者名"""
|
||||
# 1. 作者：xxx
|
||||
m = re.search(r'作者[：:]\s*([^\s,，、\)）《\[]+)', title)
|
||||
if m:
|
||||
name = m.group(1).strip()
|
||||
if name not in EXCLUDE_AUTHORS and not re.match(r'^第\d+[章节]', name):
|
||||
return name
|
||||
# 2. 翻译：xxx
|
||||
m = re.search(r'翻译[：:]\s*([^\s,，、\)）《\[]+)', title)
|
||||
if m:
|
||||
return m.group(1).strip()
|
||||
# 3. BY xxx
|
||||
m = re.search(r'\b[Bb][Yy]\s+([^\s,，、\)）《\[]+)', title)
|
||||
if m:
|
||||
name = m.group(1).strip()
|
||||
if name not in EXCLUDE_AUTHORS:
|
||||
return name
|
||||
# 4. -xxx 结尾（最后一个 dash 后的词）
|
||||
m = re.search(r'-([^-()【】\[\]{}《》\s]{2,30})$', title)
|
||||
if m:
|
||||
name = m.group(1).strip()
|
||||
if name not in EXCLUDE_AUTHORS and not re.match(r'^\d+', name):
|
||||
return name
|
||||
# 5. &xxx 结尾
|
||||
m = re.search(r'&([^&()【】\[\]\s]{2,30})$', title)
|
||||
if m:
|
||||
name = m.group(1).strip()
|
||||
if name not in EXCLUDE_AUTHORS:
|
||||
return name
|
||||
return None
|
||||
|
||||
def clean_title_and_extract_author(n):
|
||||
"""清洗书名并返回 (cleaned_title, extracted_author_or_None)"""
|
||||
t = n
|
||||
t = re.sub(r'<[^>]+>', '', t)
|
||||
t = re.sub(r'_BR_', ' ', t, flags=re.IGNORECASE)
|
||||
t = re.sub(r'_BR(?![a-zA-Z])', ' ', t)
|
||||
t = re.sub(r'_font[^_]*_', '', t)
|
||||
t = re.sub(r'_img[^_]*_', '', t)
|
||||
t = re.sub(r'\s+', ' ', t).strip()
|
||||
|
||||
extracted_author = extract_author_from_title(t)
|
||||
|
||||
# 如果提取到了作者，从书名中移除作者部分
|
||||
if extracted_author:
|
||||
t = re.sub(r'作者[：:]\s*' + re.escape(extracted_author), '', t)
|
||||
t = re.sub(r'翻译[：:]\s*' + re.escape(extracted_author), '', t)
|
||||
t = re.sub(r'\b[Bb][Yy]\s+' + re.escape(extracted_author), '', t)
|
||||
t = re.sub(r'-' + re.escape(extracted_author) + r'$', '', t)
|
||||
t = re.sub(r'&' + re.escape(extracted_author) + r'$', '', t)
|
||||
|
||||
# 超长截短
|
||||
if len(t) > 100:
|
||||
m = re.search(r'[\u4e00-\u9fff]{2,}', t)
|
||||
if m:
|
||||
t = t[m.start():]
|
||||
t = t[:60].rstrip('，。；：、！？…\ufffd\'"').rstrip('.,;:!?\'').strip()
|
||||
|
||||
t = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f]', '', t)
|
||||
t = shorten(t, 100).strip() or 'Unknown'
|
||||
|
||||
return t, extracted_author
|
||||
|
||||
|
||||
db = LibraryDatabase('/home/yangxuan/calibre/books')
|
||||
count_clean = 0
|
||||
count_author = 0
|
||||
errors = []
|
||||
|
||||
for bid in list(db.all_ids()):
|
||||
mi = db.get_metadata(bid)
|
||||
title = mi.title
|
||||
author_name = mi.authors[0] if mi.authors else 'Unknown'
|
||||
|
||||
# 检测是否需要清洗书名
|
||||
need_clean = (
|
||||
len(title) > 100 or
|
||||
'<img' in title or '<BR' in title or
|
||||
'<font' in title or '\ufffd' in title or
|
||||
len(title.encode('utf-8')) > 150
|
||||
)
|
||||
|
||||
# 检测是否需要提取作者（当前作者为 Unknown）
|
||||
need_author = (author_name == 'Unknown')
|
||||
|
||||
if need_clean or need_author:
|
||||
new_title, extracted = clean_title_and_extract_author(title)
|
||||
|
||||
changed = False
|
||||
if new_title and new_title != title:
|
||||
mi.title = new_title
|
||||
changed = True
|
||||
count_clean += 1
|
||||
|
||||
if need_author and extracted:
|
||||
mi.authors = [extracted]
|
||||
changed = True
|
||||
count_author += 1
|
||||
|
||||
if changed:
|
||||
try:
|
||||
db.set_metadata(bid, mi)
|
||||
except Exception as e:
|
||||
errors.append((bid, str(e)[:80]))
|
||||
|
||||
print(f'书名清洗: {count_clean}')
|
||||
print(f'作者提取: {count_author}')
|
||||
print(f'错误: {len(errors)}')
|
||||
```
|
||||
|
||||
## 常用操作
|
||||
|
||||
### 查看 Unknown 作者的书
|
||||
|
||||
```bash
|
||||
sqlite3 /home/yangxuan/calibre/books/metadata.db "
|
||||
SELECT b.id, b.title
|
||||
FROM books b
|
||||
JOIN books_authors_link l ON b.id = l.book
|
||||
JOIN authors a ON l.author = a.id
|
||||
WHERE a.name = 'Unknown'
|
||||
LIMIT 20;
|
||||
"
|
||||
|
||||
# 统计
|
||||
sqlite3 /home/yangxuan/calibre/books/metadata.db "
|
||||
SELECT a.name, COUNT(*) as cnt
|
||||
FROM books b
|
||||
JOIN books_authors_link l ON b.id = l.book
|
||||
JOIN authors a ON l.author = a.id
|
||||
GROUP BY a.name
|
||||
ORDER BY cnt DESC
|
||||
LIMIT 20;
|
||||
"
|
||||
```
|
||||
|
||||
### 运行清洗脚本
|
||||
|
||||
```bash
|
||||
# 停服务 + 备份
|
||||
cd /home/yangxuan/calibre
|
||||
docker compose down
|
||||
cp books/metadata.db books/metadata.db.backup.$(date +%Y%m%d)
|
||||
killall calibre 2>/dev/null
|
||||
|
||||
# 执行
|
||||
calibre-debug /tmp/fix_titles_v2.py
|
||||
|
||||
# 重启
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
## 已知问题
|
||||
|
||||
### `set_metadata` 报 `Errno 36 文件名过长`
|
||||
|
||||
当 `author_sort` 字段（基于作者名生成）超 80 字节时，Calibre 更新路径会触及 ext4 255 字节限制。此时需先清理极长作者名：
|
||||
|
||||
```bash
|
||||
sqlite3 /home/yangxuan/calibre/books/metadata.db "
|
||||
UPDATE authors SET name='Unknown' WHERE LENGTH(name) > 30;
|
||||
"
|
||||
```
|
||||
然后重新执行脚本。
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":"openclaw.skill-workshop.proposal.v1","id":"calibre-title-cleaner-20260624-3760b64e8b","kind":"create","status":"applied","title":"Create calibre-title-cleaner","description":"清洗 Calibre 书库中损坏的 txt 文档书名（HTML标签/长文本/乱码/路径超限修复）","createdAt":"2026-06-24T01:10:56.296Z","updatedAt":"2026-06-24T01:13:39.670Z","createdBy":"skill-workshop","origin":{"agentId":"storage","sessionKey":"agent:storage:main","runId":"af9dc258-69c6-4d62-85d9-80f64c387d85","messageId":"af9dc258-69c6-4d62-85d9-80f64c387d85"},"proposedVersion":"v2","draftFile":"PROPOSAL.md","draftHash":"735afe1e59db22fb6cbffbe074e559ebc91137fa3687e503b12fc91af62f42e1","target":{"skillName":"calibre-title-cleaner","skillKey":"calibre-title-cleaner","skillDir":"/home/yangxuan/.openclaw/workspace/storage/skills/calibre-title-cleaner","skillFile":"/home/yangxuan/.openclaw/workspace/storage/skills/calibre-title-cleaner/SKILL.md","source":"openclaw-workspace"},"scan":{"state":"clean","scannedAt":"2026-06-24T01:13:39.659Z","critical":0,"warn":0,"info":0,"findings":[]},"appliedAt":"2026-06-24T01:13:39.670Z"}
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":"openclaw.skill-workshop.rollback.v1","proposalId":"calibre-title-cleaner-20260624-3760b64e8b","writtenAt":"2026-06-24T01:13:39.660Z","targetSkillFile":"/home/yangxuan/.openclaw/workspace/storage/skills/calibre-title-cleaner/SKILL.md","action":"create"}
|
||||
Reference in New Issue
Block a user