From 0f85da324535e14ecc660a8f5d87d7a2ddbf654d Mon Sep 17 00:00:00 2001 From: openclaw-314 Date: Tue, 28 Jul 2026 14:00:29 +0800 Subject: [PATCH] auto: sync OpenClaw config 2026-07-28 14:00 --- agents/main/agent/models.json | 22 +- openclaw.json | 34 +- ...383758abd4a04a9a827f7b4cfd3e50202.attested | 2 +- workspace-sql/.clawhub/lock.json | 213 ++++++++ workspace-sql/AGENTS.md | 9 + workspace-sql/IDENTITY.md | 6 + workspace-sql/TOOLS.md | 8 +- workspace-sql/create_table.sql | 77 +++ ...dmp_im_stock_in_out_flow_detail_backup.sql | 118 ++++ workspace-sql/insert_data.sql | 1 + workspace-sql/skills/sql/SKILL.md | 224 ++++++++ workspace-sql/skills/sql/_meta.json | 6 + workspace-sql/skills/sql/analytics.md | 205 +++++++ workspace-sql/skills/sql/data-loading.md | 183 +++++++ workspace-sql/skills/sql/datetime.md | 204 +++++++ workspace-sql/skills/sql/debug.md | 148 ++++++ workspace-sql/skills/sql/dialects.md | 163 ++++++ workspace-sql/skills/sql/json.md | 197 +++++++ workspace-sql/skills/sql/memory-template.md | 38 ++ workspace-sql/skills/sql/modeling.md | 135 +++++ workspace-sql/skills/sql/operations.md | 362 +++++++++++++ workspace-sql/skills/sql/orm.md | 107 ++++ workspace-sql/skills/sql/patterns.md | 418 +++++++++++++++ workspace-sql/skills/sql/performance.md | 133 +++++ workspace-sql/skills/sql/scaling.md | 139 +++++ workspace-sql/skills/sql/schemas.md | 503 ++++++++++++++++++ workspace-sql/skills/sql/security.md | 149 ++++++ workspace-sql/skills/sql/setup.md | 33 ++ workspace-sql/skills/sql/skill-card.md | 40 ++ workspace-sql/skills/sql/testing.md | 127 +++++ workspace-sql/skills/sql/transactions.md | 148 ++++++ workspace-sql/temp.sql | 1 + 32 files changed, 4142 insertions(+), 11 deletions(-) create mode 100644 workspace-sql/create_table.sql create mode 100644 workspace-sql/dmp_im_stock_in_out_flow_detail_backup.sql create mode 100644 workspace-sql/insert_data.sql create mode 100644 workspace-sql/skills/sql/SKILL.md create mode 100644 workspace-sql/skills/sql/_meta.json create mode 100644 workspace-sql/skills/sql/analytics.md create mode 100644 workspace-sql/skills/sql/data-loading.md create mode 100644 workspace-sql/skills/sql/datetime.md create mode 100644 workspace-sql/skills/sql/debug.md create mode 100644 workspace-sql/skills/sql/dialects.md create mode 100644 workspace-sql/skills/sql/json.md create mode 100644 workspace-sql/skills/sql/memory-template.md create mode 100644 workspace-sql/skills/sql/modeling.md create mode 100644 workspace-sql/skills/sql/operations.md create mode 100644 workspace-sql/skills/sql/orm.md create mode 100644 workspace-sql/skills/sql/patterns.md create mode 100644 workspace-sql/skills/sql/performance.md create mode 100644 workspace-sql/skills/sql/scaling.md create mode 100644 workspace-sql/skills/sql/schemas.md create mode 100644 workspace-sql/skills/sql/security.md create mode 100644 workspace-sql/skills/sql/setup.md create mode 100644 workspace-sql/skills/sql/skill-card.md create mode 100644 workspace-sql/skills/sql/testing.md create mode 100644 workspace-sql/skills/sql/transactions.md create mode 100644 workspace-sql/temp.sql diff --git a/agents/main/agent/models.json b/agents/main/agent/models.json index 4463fbb..f130151 100644 --- a/agents/main/agent/models.json +++ b/agents/main/agent/models.json @@ -376,7 +376,7 @@ "models": [ { "id": "qwen3.5-plus", - "name": "qwen3.5-plus", + "name": "Qwen 3.5 Plus", "reasoning": false, "input": [ "text" @@ -392,8 +392,24 @@ "api": "openai-completions" }, { - "id": "qwen3.6-flash", - "name": "Qwen 3.6 Flash", + "id": "deepseek-v4-flash", + "name": "DeepSeek V4 Flash", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 1, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 1000000, + "maxTokens": 384000 + }, + { + "id": "qwen3.7-plus", + "name": "Qwen 3.7 Plus", "reasoning": false, "input": [ "text" diff --git a/openclaw.json b/openclaw.json index fd52d83..e503726 100644 --- a/openclaw.json +++ b/openclaw.json @@ -66,7 +66,7 @@ "models": [ { "id": "qwen3.5-plus", - "name": "qwen3.5-plus", + "name": "Qwen 3.5 Plus", "reasoning": false, "input": [ "text" @@ -82,8 +82,24 @@ "api": "openai-completions" }, { - "id": "qwen3.6-flash", - "name": "Qwen 3.6 Flash", + "id": "deepseek-v4-flash", + "name": "DeepSeek V4 Flash", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 1, + "output": 2, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 1000000, + "maxTokens": 384000 + }, + { + "id": "qwen3.7-plus", + "name": "Qwen 3.7 Plus", "reasoning": false, "input": [ "text" @@ -121,6 +137,15 @@ }, "deepseek/deepseek-v4-flash": { "alias": "DeepSeek V4 Flash" + }, + "newapi/deepseek-v4-flash": { + "alias": "DeepSeek V4 Flash" + }, + "newapi/qwen-3.5-plus": { + "alias": "Qwen 3.5 Plus" + }, + "newapi/qwen-3.7-plus": { + "alias": "Qwen 3.7 Plus" } }, "workspace": "/home/yangxuan/.openclaw/workspace", @@ -223,9 +248,6 @@ "name": "SQL", "workspace": "/home/yangxuan/.openclaw/workspace-sql", "agentDir": "/home/yangxuan/.openclaw/agents/sql/agent", - "model": { - "primary": "newapi/qwen3.6-flash" - }, "identity": { "name": "SQL", "avatar": "./avatars/assistant.jpg" diff --git a/workspace-attestations/cd1ae75e24d5dda927dff842caf8683383758abd4a04a9a827f7b4cfd3e50202.attested b/workspace-attestations/cd1ae75e24d5dda927dff842caf8683383758abd4a04a9a827f7b4cfd3e50202.attested index 1203581..a58486e 100644 --- a/workspace-attestations/cd1ae75e24d5dda927dff842caf8683383758abd4a04a9a827f7b4cfd3e50202.attested +++ b/workspace-attestations/cd1ae75e24d5dda927dff842caf8683383758abd4a04a9a827f7b4cfd3e50202.attested @@ -1,2 +1,2 @@ openclaw-workspace-attestation:v1 -2026-07-27T04:00:00.092Z +2026-07-28T05:57:11.420Z diff --git a/workspace-sql/.clawhub/lock.json b/workspace-sql/.clawhub/lock.json index cad9176..7fe60d6 100644 --- a/workspace-sql/.clawhub/lock.json +++ b/workspace-sql/.clawhub/lock.json @@ -97,6 +97,219 @@ "status": "unsigned" } } + }, + "sql": { + "version": "1.0.4", + "installedAt": 1785208687530, + "registry": "https://clawhub.ai", + "ownerHandle": "ivangdavila", + "artifact": { + "kind": "archive", + "sha256": "8dc47f6405be23bd9db66de4e60577ba2c556aec933ceb455b4ae174c00efa17", + "integrity": "sha256-jcR/ZAW+I72dtm3k5gV3uixVauyTPOtFW0rhdMAO+hc=" + }, + "skillFile": { + "path": "SKILL.md", + "sha256": "0e2eacad802a1757d9d4cd949e3191c714015d8c5617655259626b8e3e55eda5" + }, + "verification": { + "schema": "clawhub.skill.verify.v1", + "ok": true, + "decision": "pass", + "reasons": [], + "card": { + "available": true, + "path": "skill-card.md", + "url": "https://wry-manatee-359.convex.site/api/v1/skills/sql/card?ownerHandle=ivangdavila&version=1.0.4", + "sha256": "6c068c3dfd68f81c50a2bd960def08e23f6f939a2208547e489073875a2d0368", + "size": 1890, + "contentType": "text/markdown; charset=utf-8" + }, + "artifact": { + "sourceFingerprint": "60158fc6943698fdbe6402833772a27312ea5c21901f0f279be09cb054de4fc4", + "bundleFingerprints": [ + "767a3960cfec461d9545e368138c82304383ef06a7535a1c812e164982cd7a48", + "d5c91c79977397996b4e51d96191d12a9a2481946209cd23dac10608770b2a10" + ], + "files": [ + { + "path": "SKILL.md", + "size": 20267, + "sha256": "0e2eacad802a1757d9d4cd949e3191c714015d8c5617655259626b8e3e55eda5", + "contentType": "text/markdown" + }, + { + "path": "_meta.json", + "size": 456, + "sha256": "f2853de96d6e5f5802d6bee812895a09627f7bbde6b6c6ef2fe5a57029299919", + "contentType": "application/json" + }, + { + "path": "analytics.md", + "size": 12738, + "sha256": "4edbaf8ab58eb167c7bb4d9e8564b9ebad1f6bae4dcebdf503d25d18e44b1ce6", + "contentType": "text/markdown" + }, + { + "path": "data-loading.md", + "size": 12324, + "sha256": "1b62e0ca28c524b445fb9265bf6b2e99b3860721e24f47cd541ae216fd6e6b0b", + "contentType": "text/markdown" + }, + { + "path": "datetime.md", + "size": 13431, + "sha256": "bd643df4fcfc4940c7be85a6254ef2efc48c0d9c8b72bc31383ff4a323bfa5d9", + "contentType": "text/markdown" + }, + { + "path": "debug.md", + "size": 13646, + "sha256": "7708d77cc0f070c3d23495c29d6a4ee21594ceef54a04c7cac910c1e53e6d812", + "contentType": "text/markdown" + }, + { + "path": "dialects.md", + "size": 12400, + "sha256": "2b6d26477126dfbb99300427fc0523f3bdc2d8aed59f33df0b50e804b29728ea", + "contentType": "text/markdown" + }, + { + "path": "json.md", + "size": 10194, + "sha256": "7cab974e51516987ff77ac3e8c832918c79ff47ce478cbe5a5d50aa500c2eb73", + "contentType": "text/markdown" + }, + { + "path": "memory-template.md", + "size": 976, + "sha256": "3c779610ff0754e1eabd14274fa9a65c35fe90e970f76f09ff0d6138298b6ba6", + "contentType": "text/markdown" + }, + { + "path": "modeling.md", + "size": 11560, + "sha256": "56013e66d57f3306485fa6476a07c9de828f98bdc33901b28255a57f35680a32", + "contentType": "text/markdown" + }, + { + "path": "operations.md", + "size": 21126, + "sha256": "5f0df1c3c60b0a6d53540600941cb8df4ab96aca59da9df0b74eaaf73e3d6400", + "contentType": "text/markdown" + }, + { + "path": "orm.md", + "size": 10254, + "sha256": "538c9105081212a664ea165fbccbd71beeb22a4a0806f91540a9480f2812797d", + "contentType": "text/markdown" + }, + { + "path": "patterns.md", + "size": 18405, + "sha256": "036f508c7e48707d089c0d75eaef709936f4f11dad8b7da8e0fde41fc44be397", + "contentType": "text/markdown" + }, + { + "path": "performance.md", + "size": 11620, + "sha256": "74f48bdee26eef06dbffa244884c9820b86d4726c73b1a6e1c5e072233663d0a", + "contentType": "text/markdown" + }, + { + "path": "scaling.md", + "size": 11083, + "sha256": "9140aa56fc1bc397a7d826bec055e4930b5349ab8817a17a6f27f0c1b2909377", + "contentType": "text/markdown" + }, + { + "path": "schemas.md", + "size": 23974, + "sha256": "a895730ea30482db86543a0ba0a4ca0a150d86d43df7a3433876db6eb98974e0", + "contentType": "text/markdown" + }, + { + "path": "security.md", + "size": 12412, + "sha256": "2e8afd2fc874b35100050c56743afed67300d214f9a11472771506de462a95e8", + "contentType": "text/markdown" + }, + { + "path": "setup.md", + "size": 2692, + "sha256": "277023416cbd67cc8ae5363ed8126f539fab1a89c82f0ce31aae1517d7eeca12", + "contentType": "text/markdown" + }, + { + "path": "testing.md", + "size": 9820, + "sha256": "90e3b8ad6d3a4bfaf6206992ef1f1f6b8884dc72bb42fa42c503af5dafa6a5bf", + "contentType": "text/markdown" + }, + { + "path": "transactions.md", + "size": 10194, + "sha256": "d758e47ffc10aca6ae3c235281cfa9a0ac0fb1403fd310f3bc29853072dcc874", + "contentType": "text/markdown" + } + ] + }, + "provenance": { + "source": "unavailable", + "reason": "No server-resolved GitHub import provenance is stored for this version." + }, + "security": { + "status": "clean", + "passed": true, + "rawStatus": "clean", + "verdict": "benign", + "confidence": "high", + "summary": "This is a coherent SQL guidance skill with disclosed local preference storage and no executable installer or hidden behavior found.", + "model": null, + "checkedAt": 1784978921369, + "signals": { + "staticScan": { + "status": "clean", + "rawStatus": "clean", + "reasonCodes": [], + "summary": "No suspicious patterns detected.", + "engineVersion": "v2.4.26", + "checkedAt": 1784974869470 + }, + "virusTotal": { + "status": "clean", + "rawStatus": "clean", + "verdict": null, + "analysis": null, + "source": "engines", + "scanner": null, + "engineStats": { + "harmless": 0, + "malicious": 0, + "suspicious": 0, + "undetected": 61 + }, + "checkedAt": 1784978227535 + }, + "skillSpector": { + "status": "clean", + "rawStatus": "clean", + "score": 0, + "severity": "LOW", + "recommendation": "SAFE", + "issueCount": 0, + "scannerVersion": "2.3.5", + "summary": null, + "error": null, + "checkedAt": 1784978921369 + }, + "dependencyRegistry": null + } + }, + "signature": { + "status": "unsigned" + } + } } } } diff --git a/workspace-sql/AGENTS.md b/workspace-sql/AGENTS.md index 741c529..f2b4a15 100644 --- a/workspace-sql/AGENTS.md +++ b/workspace-sql/AGENTS.md @@ -28,6 +28,15 @@ - PostgreSQL: `PGPASSWORD` 环境变量 - 优先使用本地 mysql 客户端直接连接，不用 `docker exec` +## 默认数据库 + +- **默认连接**: VPS `101.34.227.188:3306`（主服务器） +- **默认库**: `dmp_serp`（如未指定库，首选此库） +- **连接命令**: `mysql -h 101.34.227.188 -u root -p"${MYSQL_PWD_SQL}"` +- **表名规则**: 始终带库前缀（如 `dmp_serp.table`、`dmp_smdm.table`），因为 VPS 上多库共存 +- **技能 (sql-toolkit) 也遵循此默认**：未指定环境时指向 VPS +- **需要切换环境时明确告知**：本地、备用机、或者特定库 + ## 相关 - [Agent workspace](/concepts/agent-workspace) diff --git a/workspace-sql/IDENTITY.md b/workspace-sql/IDENTITY.md index bfe3a71..7ce1c6e 100644 --- a/workspace-sql/IDENTITY.md +++ b/workspace-sql/IDENTITY.md @@ -25,6 +25,12 @@ 使用前需确保 `${PGPASSWORD}` 和 `${MYSQL_PWD_SQL}` 已在 shell 环境中 export。 +### 默认数据库 +- **默认连接**: VPS `101.34.227.188:3306`（主服务器） +- **默认库**: `dmp_serp`（如未指定库，首选此库） +- **表名规则**: 始终带库前缀（如 `dmp_smdm.dmp_md_item_info`） +- **连接命令**: `mysql -h 101.34.227.188 -u root -p"${MYSQL_PWD_SQL}" -D dmp_serp` + ## Related - [Agent workspace](/concepts/agent-workspace) diff --git a/workspace-sql/TOOLS.md b/workspace-sql/TOOLS.md index b54def5..dadcfa1 100644 --- a/workspace-sql/TOOLS.md +++ b/workspace-sql/TOOLS.md @@ -3,11 +3,17 @@ Skills define _how_ tools work. This file is for _your_ specifics — the stuff that's unique to your setup: camera names and locations, SSH hosts and aliases, preferred TTS voices, speaker/room names, device nicknames, anything environment-specific. ### MySQL 连接 +- **默认 → VPS**: `101.34.227.188:3306` / root / 5gynj20J（主服务器，`dmp_smdm`、`dmp_serp`、`dmp_smes`） - **本地 Docker**: `docker exec mysql mysql -u root -p"${MYSQL_PWD_SQL}" -D dmp_serp` -- **vps**: `101.34.227.188:3306` / root / 5gynj20J（主服务器，库：dmp_smdm、dmp_serp、dmp_smes） - **备用机**: `100.115.195.191:3306` / root / 5gynj20J（库：dmp_smdm、dmp_serp、dmp_smes 等） - **密码环境变量**: `MYSQL_PWD_SQL=5gynj20J`（来自 .env） - ⚠️ **优先使用本地 mysql 客户端直接连接**，不要每次都用 `docker exec`。 +- ⚠️ **默认数据库连接串**: `mysql -h 101.34.227.188 -u root -p"${MYSQL_PWD_SQL}"` + +**默认规则（AGENTS + 技能）：** +- 未指定环境时，默认连接 VPS（101.34.227.188） +- 脚本表名必须带库前缀（`dmp_xxx.table`），因为 VPS 上多库共存 +- 如果操作只涉及本地库（如无 VPS 权限的测试），需明确切换 ## Examples diff --git a/workspace-sql/create_table.sql b/workspace-sql/create_table.sql new file mode 100644 index 0000000..6561ba1 --- /dev/null +++ b/workspace-sql/create_table.sql @@ -0,0 +1,77 @@ +DROP TABLE IF EXISTS `dmp_im_stock_in_out_flow_detail`; +CREATE TABLE `dmp_serp`.`dmp_im_stock_in_out_flow_detail` ( + `id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '主键id', + `ecid` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '企业编码', + `flow_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '流水单id', + `flow_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '流水单号', + `type` varchar(64) COLLATE utf8mb4_general_ci NOT NULL COMMENT '单据类型(stockInOutType)', + `stock_type` varchar(64) COLLATE utf8mb4_general_ci NOT NULL COMMENT '类型(in/out)', + `stock_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '出入库单号', + `stock_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '出入库id', + `stock_detail_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '出入库明细id', + `stock_shelf_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '出入库位关系id', + `shelf_model_id` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '型号库存id', + `material_properties` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '物资属性', + `material_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '物料编码', + `factory_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '厂别code', + `warehouse_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '仓库编码', + `shelf_area_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '库区编码', + `shelf_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '库位编码', + `batch_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '批次号', + `stock_quantity` decimal(16,6) DEFAULT NULL COMMENT '出入库数量', + `stock_amount` decimal(16,6) DEFAULT NULL COMMENT '含税金额', + `sales_amount` decimal(22,6) DEFAULT '0.000000' COMMENT '销售额', + `no_tax_amount` decimal(16,6) DEFAULT NULL COMMENT '未税金额', + `bill_date` date DEFAULT NULL COMMENT '单据日期', + `remark` varchar(2000) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL, + `stock_remark` varchar(2000) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL, + `source_bill_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '来源单号', + `source_row_index` int DEFAULT NULL COMMENT '行号', + `production_task_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '生产任务单号', + `sales_order_no_list` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci COMMENT '销售单号', + `order_detail_row_index` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci COMMENT '销售订单行号', + `purchase_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '采购单号', + `mrp_no` varchar(600) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '计划单号', + `supplier_code` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '供应商code', + `stock_date` date DEFAULT NULL COMMENT '出、入库日期', + `is_deleted` tinyint(1) NOT NULL COMMENT '是否已作废', + `create_time` datetime NOT NULL COMMENT '创建时间', + `created_by` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '创建人', + `update_time` datetime NOT NULL COMMENT '修改时间', + `updated_by` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '修改人', + `version` int NOT NULL COMMENT '版本号', + `pro_batch_no` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '生产批次', + `auxiliary_stock_quantity` decimal(16,6) DEFAULT '0.000000' COMMENT '辅出入库数量', + `inventory_type_code` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '品质类型编码', + `product_request` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '生产要求', + `mat_trans_form_no` varchar(60) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '物料转换单号', + `no_tax_price` decimal(16,6) DEFAULT NULL COMMENT '无税单价', + `ial_stock_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '线边仓退料领料单号', + `dept_code` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '部门编码', + `tax_rate` decimal(12,6) DEFAULT NULL COMMENT '税率', + `invoice_no` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '发票号码', + `om_gift_tag` varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '是否赠品：是|true,否|false', + `factory_code_in_out` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '拨入拨出厂别', + `warehouse_code_in_out` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '拨入拨出仓库', + `shelf_area_code_in_out` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '拨入拨出库区编码', + `shelf_code_in_out` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '拨入拨出库位编码', + `allocation_no` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '调拨单号', + `allocation_row_index` int DEFAULT NULL COMMENT '行号', + `cust_code` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '客户编码', + `aggregate_model` int DEFAULT NULL COMMENT 'doris汇总类型：1仓库+物料 2，物料+仓库+批次 3，物料+仓库+库区/库位 4，库存明细 5，物料+供应商+仓库', + `group_flag` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '分组标记', + `work_shop_code` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '车间编码', + `purchase_type` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '采购类型', + `order_no_str` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci COMMENT '销售订单号(多个，以“，”分隔）', + `order_no_row_str` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci COMMENT '销售订单行号(多个，以“，”分隔）', + `insp_remark` varchar(255) COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '质检备注', + `apply_by` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '申请人 ID', + PRIMARY KEY (`id`), + KEY `idx_im_stock_in_out_flow_detail_batch_no` (`batch_no`), + KEY `idx_dmp_im_stock_in_out_flow_detail_bill_date` (`bill_date`), + KEY `idx_im_stock_in_out_flow_detail_type` (`type`), + KEY `idx_im_stock_in_out_flow_detail_source_bill_no` (`source_bill_no`), + KEY `idx_dmp_im_stock_in_out_flow_detail_stock_no` (`stock_no`), + KEY `idx_im_stock_in_out_flow_detail_production_task_no` (`production_task_no`), + KEY `idx_dmp_im_stock_in_out_flow_detail_ecid_material_no` (`ecid`,`material_no`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci ROW_FORMAT=DYNAMIC COMMENT='出入库记录明细'; \ No newline at end of file diff --git a/workspace-sql/dmp_im_stock_in_out_flow_detail_backup.sql b/workspace-sql/dmp_im_stock_in_out_flow_detail_backup.sql new file mode 100644 index 0000000..2cdf56c --- /dev/null +++ b/workspace-sql/dmp_im_stock_in_out_flow_detail_backup.sql @@ -0,0 +1,118 @@ +/*M!999999\- enable the sandbox mode */ +-- MariaDB dump 10.19 Distrib 10.11.18-MariaDB, for debian-linux-gnu (x86_64) +-- +-- Host: 101.34.227.188 Database: dmp_serp +-- ------------------------------------------------------ +-- Server version 8.4.9 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8mb4 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `dmp_im_stock_in_out_flow_detail` +-- + +DROP TABLE IF EXISTS `dmp_im_stock_in_out_flow_detail`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8mb4 */; +CREATE TABLE `dmp_im_stock_in_out_flow_detail` ( + `id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '主键id', + `ecid` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '企业编码', + `flow_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '流水单id', + `flow_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '流水单号', + `type` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '流水类型', + `stock_type` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '出入库类型', + `stock_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '出入库单号', + `stock_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '出入库id', + `stock_detail_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '出入库明细id', + `stock_shelf_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '出入库位关系id', + `shelf_model_id` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '型号库存id', + `material_properties` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '物资属性', + `material_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '物料编码', + `factory_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '厂别code', + `warehouse_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '仓库编码', + `shelf_area_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '库区编码', + `shelf_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '库位编码', + `batch_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '批次号', + `stock_quantity` decimal(16,6) DEFAULT NULL COMMENT '出入库数量', + `stock_amount` decimal(16,6) DEFAULT NULL COMMENT '含税金额', + `sales_amount` decimal(22,6) DEFAULT '0.000000' COMMENT '销售额', + `no_tax_amount` decimal(16,6) DEFAULT NULL COMMENT '未税金额', + `bill_date` date DEFAULT NULL COMMENT '单据日期', + `remark` varchar(2000) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL, + `stock_remark` varchar(2000) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci DEFAULT NULL COMMENT '出库备注', + `source_bill_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '来源单号', + `source_row_index` int DEFAULT NULL COMMENT '行号', + `production_task_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '生产任务单号', + `sales_order_no_list` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci COMMENT '销售单号', + `purchase_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '采购单号', + `mrp_no` varchar(400) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '计划单号', + `supplier_code` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '供应商code', + `stock_date` date DEFAULT NULL COMMENT '出、入库日期', + `is_deleted` tinyint(1) NOT NULL COMMENT '是否已作废', + `create_time` datetime NOT NULL COMMENT '创建时间', + `created_by` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '创建人', + `update_time` datetime NOT NULL COMMENT '修改时间', + `updated_by` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '修改人', + `version` int NOT NULL COMMENT '版本号', + `pro_batch_no` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '生产批次', + `auxiliary_stock_quantity` decimal(16,6) DEFAULT '0.000000' COMMENT '辅出入库数量', + `inventory_type_code` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '品质类型编码', + `product_request` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '生产要求', + `mat_trans_form_no` varchar(60) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '物料转换单号', + `order_detail_row_index` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '销售订单行号', + `no_tax_price` decimal(16,6) DEFAULT NULL COMMENT '无税单价', + `ial_stock_no` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '线边仓退料领料单号', + `dept_code` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '部门编码', + `tax_rate` decimal(12,6) DEFAULT NULL COMMENT '税率', + `invoice_no` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '发票号码', + `om_gift_tag` varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '是否赠品：是|true,否|false', + `factory_code_in_out` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '拨入拨出厂别', + `warehouse_code_in_out` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '拨入拨出仓库', + `shelf_area_code_in_out` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '拨入拨出库区编码', + `shelf_code_in_out` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '拨入拨出库位编码', + `allocation_no` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '调拨单号', + `allocation_row_index` int DEFAULT NULL COMMENT '行号', + `cust_code` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '客户编码', + `aggregate_model` int DEFAULT NULL COMMENT 'doris汇总类型：1仓库+物料 2，物料+仓库+批次 3，物料+仓库+库区/库位 4，库存明细 5，物料+供应商+仓库', + `group_flag` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '分组标记', + `work_shop_code` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '车间编码', + `purchase_type` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '采购类型', + PRIMARY KEY (`id`) USING BTREE, + KEY `idx_dmp_im_stock_in_out_flow_detail_ecid_material_no` (`ecid`,`material_no`) USING BTREE, + KEY `idx_dmp_im_stock_in_out_flow_detail_bill_date` (`bill_date`) USING BTREE, + KEY `idx_dmp_im_stock_in_out_flow_detail_stock_no` (`stock_no`) USING BTREE, + KEY `idx_im_stock_in_out_flow_detail_type` (`type`) USING BTREE, + KEY `idx_im_stock_in_out_flow_detail_batch_no` (`batch_no`) USING BTREE, + KEY `idx_im_stock_in_out_flow_detail_source_bill_no` (`source_bill_no`) USING BTREE, + KEY `idx_im_stock_in_out_flow_detail_production_task_no` (`production_task_no`) USING BTREE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci ROW_FORMAT=DYNAMIC COMMENT='出入库记录明细'; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `dmp_im_stock_in_out_flow_detail` +-- + +LOCK TABLES `dmp_im_stock_in_out_flow_detail` WRITE; +/*!40000 ALTER TABLE `dmp_im_stock_in_out_flow_detail` DISABLE KEYS */; +/*!40000 ALTER TABLE `dmp_im_stock_in_out_flow_detail` ENABLE KEYS */; +UNLOCK TABLES; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2026-07-28 11:36:26 diff --git a/workspace-sql/insert_data.sql b/workspace-sql/insert_data.sql new file mode 100644 index 0000000..37a8602 --- /dev/null +++ b/workspace-sql/insert_data.sql @@ -0,0 +1 @@ +INSERT INTO `dmp_serp`.`dmp_im_stock_in_out_flow_detail` (`id`, `ecid`, `flow_id`, `flow_no`, `type`, `stock_type`, `stock_no`, `stock_id`, `stock_detail_id`, `stock_shelf_id`, `shelf_model_id`, `material_properties`, `material_no`, `factory_code`, `warehouse_code`, `shelf_area_code`, `shelf_code`, `batch_no`, `stock_quantity`, `stock_amount`, `sales_amount`, `no_tax_amount`, `bill_date`, `remark`, `stock_remark`, `source_bill_no`, `source_row_index`, `production_task_no`, `sales_order_no_list`, `order_detail_row_index`, `purchase_no`, `mrp_no`, `supplier_code`, `stock_date`, `is_deleted`, `create_time`, `created_by`, `update_time`, `updated_by`, `version`, `pro_batch_no`, `auxiliary_stock_quantity`, `inventory_type_code`, `product_request`, `mat_trans_form_no`, `no_tax_price`, `ial_stock_no`, `dept_code`, `tax_rate`, `invoice_no`, `om_gift_tag`, `factory_code_in_out`, `warehouse_code_in_out`, `shelf_area_code_in_out`, `shelf_code_in_out`, `allocation_no`, `allocation_row_index`, `cust_code`, `aggregate_model`, `group_flag`, `work_shop_code`, `purchase_type`, `order_no_str`, `order_no_row_str`, `insp_remark`, `apply_by`) VALUES ('447560004020091209', 'kunzhou', '447560004020091208', 'SIO20260727000073', 'PRODUCTION_IN', 'in', 'PI2607260007', '447166889270589005', '447166889312532191', '447559998936594770', '447559999066618088', 'KUNZHOU-SMDM-1', 'GH032-16-0006', 'KZ', 'KZCK-003', NULL, NULL, 'MSN2607260024', 2.000000, 0.000000, NULL, 0.000000, '2026-07-26', NULL, NULL, 'GD2607220030-GX-JG11-1', NULL, NULL, 'KZGM20260716001', 'KZGM20260716001_005', NULL, 'PLAN202607221255024999150', 'JG', '2026-07-27', 0, '2026-07-27 09:34:47', '颜美莉', '2026-07-27 09:34:47', '颜美莉', 1, NULL, 0.000000, NULL, NULL, NULL, 0.000000, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); \ No newline at end of file diff --git a/workspace-sql/skills/sql/SKILL.md b/workspace-sql/skills/sql/SKILL.md new file mode 100644 index 0000000..5bd6d5c --- /dev/null +++ b/workspace-sql/skills/sql/SKILL.md @@ -0,0 +1,224 @@ +--- +name: sql +slug: sql +version: 1.0.4 +description: >- + Writes, reviews, and optimizes SQL queries; designs schemas, indexes, and constraints; plans migrations for any relational database. + Use when a query is slow, EXPLAIN shows a sequential scan, or an index is ignored; when rows come back duplicated, missing, or with + inflated totals after a JOIN; on deadlocks, lock timeouts, "too many connections", or transactions that never commit; when designing + tables, keys, and column types, normalizing or denormalizing a model, or deciding between a JSON column and real columns; for ALTER + TABLE on a live table, expand-migrate-contract rollouts, backups and restores, replication lag, connection pooling, partitioning, + bulk CSV imports, and moving data between engines; for window functions, CTEs, keyset pagination, upserts, full-text search, + multi-tenancy, row-level security, and timezone handling in MySQL, SQLite, MariaDB, or SQL Server. Not for PostgreSQL server + internals such as vacuum tuning and work_mem sizing, and not for ORM schema modeling inside a framework. +homepage: https://clawic.com/skills/sql +changelog: "Full coverage pass: deeper guides, situation-named files, and per-user configuration" +metadata: + clawdbot: + emoji: 🗄️ + requires: + anyBins: + - sqlite3 + - psql + - mysql + - sqlcmd + os: + - linux + - darwin + - win32 + displayName: SQL + configPaths: + - ~/Clawic/data/sql/ +--- + +User preferences and memory live in `~/Clawic/data/sql/` (see `setup.md` on first use, `memory-template.md` for the file format). If you have data at an old location (`~/sql/` or `~/clawic/sql/`), move it to `~/Clawic/data/sql/`. + +## When To Use + +- Writing, reviewing, or optimizing SQL: queries, joins, CTEs, window functions, upserts +- Designing tables, keys, types, indexes, and constraints, or normalizing an existing model +- Diagnosing a slow query, a deadlock, a lock timeout, wrong totals, or duplicated rows +- Planning migrations and DDL that must not take a live database down +- Operating a database: backups, restores, monitoring, pooling, replication, partitioning +- Moving data in or out: CSV loads, dumps, engine-to-engine migrations +- Not for PostgreSQL server internals (vacuum tuning, `work_mem`, xid wraparound) — that is `pg`; not for ORM-level modeling in a framework — that is `prisma` + +## Quick Reference + +| Situation | Play | +|-----------|------| +| Query slow, cause unknown | `EXPLAIN (ANALYZE, BUFFERS)`, fix the worst node first (→ Reading EXPLAIN, then `performance.md`) | +| Query was fast yesterday | Stats, data growth, or plan flip — the regression chain in `debug.md` | +| Index exists but is not used | Function on the column, type mismatch, wrong column order, or low selectivity (→ Traps, `performance.md`) | +| Totals inflated after adding a JOIN | 1:N fan-out — aggregate before joining (→ Traps) | +| Rows missing after adding a JOIN | `LEFT JOIN` filtered in `WHERE` became an inner join (→ Traps) | +| Paginating past the first few thousand rows | Keyset, never OFFSET (`patterns.md`) | +| Deadlock, lock timeout, or "could not obtain lock" | `transactions.md` — lock ordering and isolation | +| "Too many connections" / app hangs on connect | Pool sizing before `max_connections` (`operations.md`, `orm.md`) | +| Read-modify-write race, job queue | `SELECT ... FOR UPDATE`, add `SKIP LOCKED` for queues (`patterns.md`) | +| Schema change on a live table | Expand → migrate → contract, `lock_timeout` first (`operations.md`) | +| Designing a model from scratch | Keys, cardinality, normal forms, when to denormalize (`modeling.md`) | +| Known shape needed (tenants, tags, audit, state, history) | `schemas.md` | +| Storing or querying JSON / semi-structured data | `json.md` | +| Cohorts, funnels, retention, rollups, materialized views | `analytics.md` | +| Loading a CSV, dump/restore, engine-to-engine move | `data-loading.md` | +| Timestamps off by hours, DST, week/fiscal boundaries | `datetime.md` | +| Statement works on one engine, fails on another | `dialects.md` | +| Grants, least privilege, RLS, PII erasure, encryption | `security.md` | +| Seeding fixtures, isolating tests, testing a migration | `testing.md` | +| ORM emits terrible SQL, N+1, mystery transactions | `orm.md` | +| Single node at its limit: replicas, sharding, caching | `scaling.md` | +| Choosing an engine | SQLite embedded/local · PostgreSQL default for servers · MySQL when the platform dictates it · SQL Server in .NET/Windows shops (`dialects.md`) | +| Anything else | Reproduce on the smallest table that shows it, then: schema-shaped → `modeling.md`/`schemas.md` · query-shaped → `patterns.md` · slow → `performance.md` · ops-shaped → `operations.md` | + +## Core Rules + +1. **Parameterize values; allowlist identifiers.** Placeholders (`?`, `$1`) stop injection for values, but table/column names cannot be bound — when those are dynamic, check them against a hardcoded allowlist, never interpolate user input. Full attack surface, including `LIKE` and `ORDER BY` injection: `security.md`. +2. **BIGINT (or UUIDv7) primary keys by default.** `INT` overflows at 2,147,483,647 — at a sustained 100 inserts/s that is 2.1B ÷ 100/s ≈ 248 days, and the fix is an outage-grade type change. Random UUIDv4 keys fragment the B-tree; UUIDv7/ULID keep insert locality (`modeling.md`). +3. **Index for the query shape: equality columns first, then range/sort.** `(user_id, created_at)` serves `WHERE user_id = ? AND created_at > ?` and `WHERE user_id = ?` alone — never `created_at` alone. A sequential scan on a filter matching more than roughly 5-10% of rows is the planner being right, not broken. +4. **Index every foreign key column yourself.** MySQL/InnoDB creates the index automatically; PostgreSQL, SQLite, and SQL Server do not. Without it, every join on the FK and every parent `DELETE` (worse with `ON DELETE CASCADE`) scans the whole child table — the slowest delete in most schemas is this one missing index. +5. **Transactions stay short and never wait on the outside world.** No HTTP calls, no user input inside `BEGIN...COMMIT`: open transactions hold locks, and in PostgreSQL they also block vacuum, causing table bloat. Anything open past the >1 min monitoring threshold (`operations.md`) gets investigated. +6. **NULL is three-valued.** `NOT IN (subquery)` returns zero rows if the subquery yields a single NULL — use `NOT EXISTS`. `x = NULL` is never true — use `IS NULL`. `COUNT(col)` skips NULLs; `COUNT(*)` counts rows. Aggregates over zero rows return NULL, not 0 — wrap in `COALESCE` when a chart or invariant expects a number. +7. **Types that avoid the next migration.** Money → `NUMERIC`/`DECIMAL` (float money loses cents in aggregation); timestamps → `TIMESTAMPTZ` stored as UTC (`datetime.md`); strings → `TEXT` in PostgreSQL and SQLite (`varchar(255)` is a cargo-cult limit you will later raise); MySQL charset → `utf8mb4` (MySQL's `utf8` is 3-byte and rejects emoji). +8. **Migrations are additive first.** Rename/retype/drop happens over multiple deploys with both versions live in between (expand-migrate-contract, `operations.md`). A single-deploy column rename breaks every instance still running old code. +9. **Rank before you tune.** `pg_stat_statements` ordered by `total_exec_time` (or the MySQL slow query log digested by `pt-query-digest`) tells you which query costs the most overall — usually not the one someone complained about. Total cost = mean latency × call count: a 5 ms query called 10,000×/min (50 s/min) outranks a 2 s report run hourly. Optimizing an unranked query is guessing. + +## Reading EXPLAIN + +```sql +EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM orders WHERE user_id = 5; -- PostgreSQL +EXPLAIN ANALYZE SELECT ...; -- MySQL >=8.0.18 +EXPLAIN QUERY PLAN SELECT * FROM orders WHERE user_id = 5; -- SQLite +SET STATISTICS PROFILE ON; -- SQL Server (or the graphical plan) +``` + +Read actual behavior, not just the plan. Plain `EXPLAIN` shows estimates only, and estimates are the part that lies. + +- `Seq Scan` / `type: ALL` on a large table with a selective filter → missing or unusable index (→ Traps for what disables one) +- `Rows Removed by Filter` high → the index found candidates but the filter did the work; extend the index to cover the filter +- Estimated vs actual rows off by **more than 10×** → stale stats: run `ANALYZE tablename;`. Still off → the planner assumes column independence; declare the correlation (`CREATE STATISTICS` on PostgreSQL >=10, histogram on MySQL 8) +- `Buffers: read` large vs `hit` → data is coming from disk; recheck on a warm cache before concluding +- Nested Loop over thousands of outer rows → usually the >10× misestimate above feeding a bad join choice +- Node-by-node interpretation, join algorithms, and what to change for each: `performance.md` + +## Index Strategy + +```sql +-- Composite: equality columns first, range/sort last (rule 3) +CREATE INDEX idx_orders_user_status ON orders(user_id, status); + +-- Covering: index-only scan, no heap fetch (PostgreSQL >=11, SQL Server INCLUDE) +CREATE INDEX idx_orders_user ON orders(user_id) INCLUDE (total); + +-- Partial/filtered: index only the rows you query (PostgreSQL, SQLite, SQL Server) +CREATE INDEX idx_orders_pending ON orders(user_id) WHERE status = 'pending'; + +-- Expression: make a function sargable (MySQL >=8.0.13 supports functional indexes) +CREATE INDEX idx_users_email_lower ON users(LOWER(email)); +``` + +- A plain B-tree on a low-cardinality column (`status` with 5 values) rarely helps; a partial index on the rare value you actually query does. +- An index on `(a, b)` already serves `WHERE a = ?` — creating a second index on `(a)` alone adds write cost and reclaims nothing. Check for redundant prefixes before adding. +- PostgreSQL with a non-C locale ignores B-tree indexes for `LIKE 'term%'` — add the `text_pattern_ops` opclass for prefix search. +- Index-only scans still hit the heap for pages not marked all-visible; if `EXPLAIN` shows `Heap Fetches` high, the table needs a `VACUUM` (`operations.md`). +- Every index taxes writes and consumes disk: drop unused ones (`pg_stat_user_indexes` where `idx_scan = 0`), but only after uptime covers a full business cycle — month-end reports use indexes nothing else does. + +## Portability + +| Feature | PostgreSQL | MySQL | SQLite | SQL Server | +|---------|------------|-------|--------|------------| +| Limit | LIMIT n | LIMIT n | LIMIT n | TOP n / OFFSET-FETCH | +| Upsert | ON CONFLICT | ON DUPLICATE KEY | ON CONFLICT | MERGE | +| Boolean | true/false | 1/0 (TINYINT) | 1/0 | 1/0 (BIT) | +| Concat | \|\| | CONCAT() | \|\| | + or CONCAT() | +| Auto-id | GENERATED / SERIAL | AUTO_INCREMENT | INTEGER PRIMARY KEY | IDENTITY | +| Returning rows from DML | RETURNING | — (MariaDB has it) | RETURNING (>=3.35) | OUTPUT | +| Aggregate FILTER | Yes | CASE only | Yes (>=3.30) | CASE only | +| Transactional DDL | Yes | No (implicit commit) | Yes | Yes | +| Default string compare | Case-sensitive | Case-insensitive (`_ci` collations) | Case-sensitive | Case-insensitive by default | + +Date functions, quoting, NULL sort order, collation, and the rest of the divergences: `dialects.md`. + +## Output Gates + +Before emitting SQL, verify: + +- Every value is a placeholder, and every dynamic identifier came from an allowlist? +- `UPDATE`/`DELETE` has a `WHERE`, or the full-table effect is explicitly intended? +- Destructive DML was previewed as the equivalent `SELECT` first? +- No 1:N join feeds an aggregate without pre-aggregation, and no `DISTINCT` is papering over one? +- `LIMIT`/`TOP` has a deterministic `ORDER BY` with a unique tiebreaker? +- New table: primary key type per rule 2, timestamps with zone, uniqueness scoped to the right columns, every FK column indexed (rule 4)? +- DDL against a live table: `lock_timeout` set, and the change is expand-only? +- Every construct used exists in the target engine (→ Portability, `dialects.md`)? + +## Configuration + +User-dependent variables. Defaults apply until the user states a preference; store them in `~/Clawic/data/sql/config.yaml`. + +| Variable | Type | Default | Effect | +|---|---|---|---| +| dialect | postgres \| mysql \| mariadb \| sqlite \| sqlserver | postgres | Syntax of every emitted statement and which caveats from Portability and `dialects.md` are surfaced | +| engine_version | text (e.g. `15`, `8.0.35`) | none | Gates features with a version floor (`RETURNING`, functional indexes, `NULLS NOT DISTINCT`); with none set, the conservative form is emitted | +| identifier_style | snake_case \| camelCase \| PascalCase | snake_case | Naming of tables, columns, indexes, and constraints in generated DDL | +| table_naming | plural \| singular | plural | Whether generated tables read `users` or `user` | +| pk_type | bigint-identity \| uuidv7 \| uuidv4 \| natural | bigint-identity | The primary key emitted by every `CREATE TABLE` (rule 2, `modeling.md`) | +| destructive_guard | bool | true | When true, `UPDATE`/`DELETE` without `WHERE`, `DROP`, and `TRUNCATE` are emitted as a transaction-wrapped preview with the matching `SELECT` first | +| migration_tool | text (flyway, alembic, golang-migrate, sqitch, ...) | none | File naming and up/down structure of generated migrations (`operations.md`) | +| timezone_policy | utc \| local | utc | Whether timestamps are stored and compared as UTC and how `datetime.md` examples render | +| lock_timeout | text (duration: `500ms`, `2s`, `10s`) | 2s | The `SET lock_timeout` emitted before every DDL statement against a live table (→ Traps, `operations.md`) | +| batch_size | number (rows, 100-100000) | 5000 | Chunk size for batched `DELETE`/`UPDATE`, backfills, and bulk loads (`patterns.md`, `data-loading.md`, `json.md`) | + +Preference areas — customizable dimensions; a stated preference gets recorded in `config.yaml` and applied: + +- **Tooling** — client (psql, mysql, sqlite3, sqlcmd, a GUI), migration runner, SQL linter/formatter, local database as a container vs installed service — affects `operations.md` and `testing.md` examples +- **Conventions** — index and constraint naming, keyword casing, CTE vs subquery preference, comment style in DDL — affects every emitted statement +- **Platform** — managed service (RDS, Aurora, Cloud SQL, Neon, PlanetScale) vs self-hosted, available extensions, collation and locale — affects which features may be assumed +- **Safety posture** — how much confirmation destructive DDL/DML needs, whether production credentials are ever used directly, read-only-by-default sessions — affects `operations.md` and the `destructive_guard` gate +- **Output format** — full runnable script vs snippet, whether a down/rollback migration accompanies every up, inline comments — affects the shape of every deliverable +- **Work order** — schema-first vs query-first design, review gate before a migration reaches production — affects the sequence in `modeling.md` and `operations.md` +- **Integrations** — ORM in use, warehouse/BI target, monitoring stack — affects `orm.md` and `analytics.md` advice +- **Constraints** — vetoed features (no triggers, no stored procedures, no vendor extensions), compliance regime (retention windows, GDPR erasure) — affects `schemas.md` and `security.md` choices +- **Thresholds** — the operational budgets a shop usually standardizes: `lock_timeout`, `batch_size`, retry cap on deadlock/serialization failures, pool sizing rule, and the alert levels (connection saturation, disk free, transaction age, backup age) — affects `transactions.md` retry loops and the Alert Thresholds table in `operations.md` +- **Cadence** — how often scheduled work runs: restore drills, rollup and materialized-view refresh, the trailing recompute window, data-quality assertion runs, automated partition creation, backup interval — affects `operations.md`, `analytics.md`, `schemas.md`, and `testing.md` schedules + +## Traps + +| Trap | Why it fails | Do instead | +|------|--------------|------------| +| `WHERE YEAR(created_at) = 2024` | Function on the column hides it from the index | Range: `created_at >= '2024-01-01' AND created_at < '2025-01-01'` | +| `BETWEEN '2024-01-01' AND '2024-01-31'` on a timestamp | Upper bound is midnight — the whole last day is excluded with no error | Half-open range: `>= '2024-01-01' AND < '2024-02-01'` | +| `WHERE varchar_col = 123` | Implicit cast applies to the column, disabling its index | Match types: quote the literal or fix the schema | +| `LEFT JOIN t ... WHERE t.col = x` | WHERE runs after the join and filters the NULL rows out → an INNER JOIN, with no error | Move the predicate into `ON`, or test `t.id IS NULL` for an anti-join | +| Join 1:N then `SUM`/`COUNT` | Fan-out duplicates left-side rows before aggregation | Aggregate in a subquery/CTE, then join the result | +| `DISTINCT` added to make duplicates go away | Hides a fan-out bug and forces a sort of the whole result | Find the join that multiplies rows; `DISTINCT` is a diagnosis, not a fix | +| `LIKE '%term'` | Leading wildcard defeats B-tree ordering | Full-text search (`schemas.md`) or a trigram index (`pg_trgm`) | +| Composite `(a, b)` for `WHERE b = ?` | B-tree is sorted by `a` first; `b` alone is unordered | Separate index on `b`, or reorder if `a` is always filtered | +| `WHERE a = ? OR b = ?` | One index cannot serve two independent predicates; usually a full scan | `UNION ALL` of two indexed queries, deduplicated if needed | +| `LIMIT 10` without `ORDER BY` | Row order is undefined — "top 10" changes between runs and after a vacuum | Always `ORDER BY` with a unique tiebreaker column | +| `ORDER BY random() LIMIT n` | Full scan plus sort of the entire table | `TABLESAMPLE` or a random-key probe (`patterns.md`) | +| `SELECT *` in application code | Blocks index-only scans, fetches columns you drop, breaks on schema change | Name the columns | +| DDL without `lock_timeout` | `ALTER TABLE` queues behind one long query and every new query queues behind it — brief lock, full outage | `SET lock_timeout` (default `2s`, → Configuration) then retry (`operations.md`) | +| Wrapping `TRUNCATE`/DDL in a transaction on MySQL | MySQL commits implicitly on DDL: the rollback you planned does not exist | Take a backup or use a copy table; transactional DDL is PostgreSQL/SQLite/SQL Server only | + +## Where Experts Disagree + +- **Surrogate vs natural keys.** Surrogate (`BIGINT`/UUID) is the default: natural keys change, and a changing PK cascades into every child row. Natural keys legitimately win on pure junction tables and on immutable code tables (ISO currency, country) where the extra id buys nothing (`modeling.md`). +- **Foreign keys at scale.** Some high-write shops drop FK constraints because they add per-write lock and index cost and complicate online schema-change tooling. Default: keep them — orphan cleanup costs more than the writes saved. Drop only with a measured write bottleneck and integrity enforced elsewhere. +- **Logic in the database.** Triggers and stored procedures give atomicity nothing in the app can match and centralize rules across many clients; they are also invisible to code review, hard to test, and version-controlled poorly. Default: constraints and simple audit triggers in the database, business workflows in the application (`schemas.md`). +- **Soft delete everywhere.** Undo and audit are real requirements, but a `deleted_at` on every table poisons every query and every unique constraint. Default: hard delete plus an audit log; soft-delete only the tables users actually restore (`schemas.md`). + +## Related Skills +More Clawic skills, get them at https://clawic.com/skills/sql (install if the user confirms): +- `pg` — PostgreSQL server internals: vacuum, work_mem, wraparound +- `mysql` — MySQL and InnoDB specifics +- `sqlite` — SQLite concurrency, pragmas, and type affinity +- `prisma` — Node.js ORM modeling +- `dbt` — warehouse transformations and tests + +## Feedback + +- If useful, star it: https://clawic.com/skills/sql +- Latest version: https://clawic.com/skills/sql + +Part of [Clawic](https://clawic.com), the verified skill library. Get this skill: https://clawic.com/skills/sql. diff --git a/workspace-sql/skills/sql/_meta.json b/workspace-sql/skills/sql/_meta.json new file mode 100644 index 0000000..07f53b6 --- /dev/null +++ b/workspace-sql/skills/sql/_meta.json @@ -0,0 +1,6 @@ +{ + "ownerId": "kn73vp5rarc3b14rc7wjcw8f8580t5d1", + "slug": "sql", + "version": "1.0.4", + "publishedAt": 1784974869476 +} \ No newline at end of file diff --git a/workspace-sql/skills/sql/analytics.md b/workspace-sql/skills/sql/analytics.md new file mode 100644 index 0000000..11d90a2 --- /dev/null +++ b/workspace-sql/skills/sql/analytics.md @@ -0,0 +1,205 @@ +# Analytical Queries — Cohorts, Funnels, and Rollups + +Reporting SQL fails differently from transactional SQL: it is usually correct-looking and wrong. The recurring causes are a missing date spine, a fan-out that double-counts, and a metric whose definition drifted between two queries. + +Contents: Definition First · Date Spine · Cohort Retention · Funnels · Sessionization · Running and Period Comparisons · Distinct Counts · Rollups · Materialized Views · Incremental Refresh · Star Schema · Sampling and Estimates · Traps + +## Define the Metric Before Writing SQL + +Write these down before the first `SELECT`; disagreements between two dashboards are almost always here, not in the code. + +- **Grain**: one row per what? User-day, order, session, user-month. +- **Population**: who is included and excluded (test accounts, internal users, cancelled orders, soft-deleted rows). +- **Timestamp**: which of `created_at`, `paid_at`, `shipped_at`, `updated_at` defines the period — and in which timezone. +- **Deduplication**: is a repeat action within the window one event or many? +- **Late data**: events that arrive after the period closed — does the number get restated or frozen? + +A metric without these five written down cannot be reproduced, and every "the numbers don't match" investigation resolves to one of them. + +## The Date Spine (the bug that never raises an error) + +Grouping by a timestamp column only produces rows for periods that had data. Days with zero activity vanish, so a chart draws a line straight through the outage and an average divides by the wrong denominator. + +```sql +-- PostgreSQL +SELECT d.day, COALESCE(COUNT(o.id), 0) AS orders +FROM generate_series(DATE '2026-01-01', DATE '2026-01-31', INTERVAL '1 day') AS d(day) +LEFT JOIN orders o ON o.created_at >= d.day AND o.created_at < d.day + INTERVAL '1 day' +GROUP BY d.day ORDER BY d.day; + +-- Portable: recursive CTE spine (MySQL >=8.0, SQLite, SQL Server) +WITH RECURSIVE days(day) AS ( + SELECT DATE '2026-01-01' + UNION ALL SELECT day + 1 FROM days WHERE day < DATE '2026-01-31' +) +SELECT days.day, COUNT(o.id) AS orders +FROM days LEFT JOIN orders o ON o.created_at >= days.day AND o.created_at < days.day + 1 +GROUP BY days.day; +``` + +Join the spine on a half-open range, never on a cast (`DATE(o.created_at) = d.day` is not sargable). A permanent `calendar` table with columns for week, month, quarter, fiscal period, and holiday flags removes this boilerplate everywhere and makes fiscal calendars possible at all. + +## Cohort Retention + +```sql +WITH first_seen AS ( + SELECT user_id, DATE_TRUNC('month', MIN(created_at)) AS cohort_month + FROM orders GROUP BY user_id +), +activity AS ( + SELECT DISTINCT o.user_id, DATE_TRUNC('month', o.created_at) AS active_month + FROM orders o +) +SELECT f.cohort_month, + -- whole months between cohort and activity; integer month offset, not day/30 + (DATE_PART('year', a.active_month) - DATE_PART('year', f.cohort_month)) * 12 + + (DATE_PART('month', a.active_month) - DATE_PART('month', f.cohort_month)) AS month_offset, + COUNT(DISTINCT a.user_id) AS active_users, + MAX(COUNT(DISTINCT a.user_id)) OVER (PARTITION BY f.cohort_month) AS cohort_size, + ROUND(100.0 * COUNT(DISTINCT a.user_id) + / MAX(COUNT(DISTINCT a.user_id)) OVER (PARTITION BY f.cohort_month), 1) AS retention_pct +FROM first_seen f +JOIN activity a ON a.user_id = f.user_id +GROUP BY f.cohort_month, month_offset +ORDER BY f.cohort_month, month_offset; +``` + +- The denominator is the cohort's size at offset 0, taken from the same query. Computing it separately guarantees the two drift. +- Truncate the table at the last **complete** period. A cohort whose month is still running always looks like a collapse, and someone will act on it. +- Rolling retention ("active on day 30 or later") and bounded retention ("active on day 30 exactly") give very different curves. Say which one the chart shows. + +## Funnels + +```sql +-- Ordered funnel: each step must occur after the previous one for the same user +WITH steps AS ( + SELECT user_id, + MIN(CASE WHEN event = 'view' THEN occurred_at END) AS t_view, + MIN(CASE WHEN event = 'add_cart' THEN occurred_at END) AS t_cart, + MIN(CASE WHEN event = 'purchase' THEN occurred_at END) AS t_buy + FROM events + WHERE occurred_at >= :from AND occurred_at < :to + GROUP BY user_id +) +SELECT COUNT(*) FILTER (WHERE t_view IS NOT NULL) AS viewed, + COUNT(*) FILTER (WHERE t_cart > t_view) AS carted, + COUNT(*) FILTER (WHERE t_buy > t_cart) AS purchased +FROM steps; +``` + +- Unordered funnels (counting anyone who did each step, in any order) inflate every stage and can report more purchases than views. Enforce the time ordering unless the product genuinely allows entry mid-funnel. +- Decide the attribution window: steps completed a month apart are usually not one funnel. Add `AND t_buy < t_view + INTERVAL '7 days'`. +- Users who entered before the window started are cut off at the left edge; either exclude them or widen the lookback for step 1 only. + +## Sessionization + +```sql +-- 30-minute inactivity gap defines a session boundary +WITH marked AS ( + SELECT user_id, occurred_at, + CASE WHEN occurred_at - LAG(occurred_at) OVER (PARTITION BY user_id ORDER BY occurred_at) + > INTERVAL '30 minutes' + OR LAG(occurred_at) OVER (PARTITION BY user_id ORDER BY occurred_at) IS NULL + THEN 1 ELSE 0 END AS is_new_session + FROM events +) +SELECT user_id, occurred_at, + SUM(is_new_session) OVER (PARTITION BY user_id ORDER BY occurred_at + ROWS UNBOUNDED PRECEDING) AS session_number +FROM marked; +``` + +The 30-minute gap is a convention from web analytics, not a law — it is wrong for products used in long background sessions. State the threshold with the metric, and keep it identical across every query that reports sessions. + +## Running Totals and Period Comparisons + +```sql +SELECT day, revenue, + SUM(revenue) OVER (ORDER BY day ROWS UNBOUNDED PRECEDING) AS running_total, + AVG(revenue) OVER (ORDER BY day ROWS BETWEEN 6 PRECEDING AND CURRENT ROW) AS ma7, + revenue - LAG(revenue, 7) OVER (ORDER BY day) AS wow_change +FROM daily_revenue; +``` + +- The default window frame is `RANGE`, which treats tied ordering values as one peer group — every row of the same day gets the same "running" total. Use `ROWS` for a true row-by-row accumulation. +- `LAG(revenue, 7)` compares to seven **rows** back, which equals seven days only if the spine has no gaps. This is why the date spine comes first. +- A moving average over a partial trailing window (the first 6 days) is computed over fewer rows and is not comparable; null it out or start the chart later. + +## Counting Distinct Things + +- `COUNT(DISTINCT x)` cannot be summed across partitions: daily distinct users do not add up to monthly distinct users. Every period needs its own pass, which is why "monthly active" cannot be derived from a daily rollup table without storing the identities. +- To make it additive, store a sketch instead of a count: PostgreSQL `hll` or `datasketches` extensions, ClickHouse/BigQuery natives. Sketches merge across periods with a stated error bound (HyperLogLog is typically within a couple of percent at default precision). +- Cheaper alternative for modest cardinality: store the distinct id list per period as an array or bitmap, then union at read time. +- `COUNT(DISTINCT)` over a large table is a sort or hash of all values — usually the slowest node in a report. + +## Rollup Tables + +The default answer for a dashboard that reads the same aggregation repeatedly. + +```sql +CREATE TABLE daily_revenue ( + day DATE NOT NULL, + tenant_id BIGINT NOT NULL, + orders INT NOT NULL, + revenue NUMERIC(14,2) NOT NULL, + computed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (day, tenant_id) +); + +-- Idempotent refresh of one day: safe to re-run, safe to backfill +INSERT INTO daily_revenue (day, tenant_id, orders, revenue) +SELECT DATE(created_at), tenant_id, COUNT(*), SUM(total) +FROM orders +WHERE created_at >= :day AND created_at < :day + INTERVAL '1 day' +GROUP BY 1, 2 +ON CONFLICT (day, tenant_id) DO UPDATE +SET orders = EXCLUDED.orders, revenue = EXCLUDED.revenue, computed_at = NOW(); +``` + +- Make every refresh idempotent and re-runnable for an arbitrary date. A rollup you cannot recompute for last Tuesday is unfixable when last Tuesday was wrong. +- Recompute a trailing window (default 3-7 days; a stated refresh Cadence overrides it), not only yesterday: late-arriving events change closed periods. +- Store `computed_at` so a stale dashboard is visibly stale rather than quietly wrong. +- Keep raw data as long as retention allows; a rollup is a cache, and every rollup eventually needs a new dimension. + +## Materialized Views + +```sql +CREATE MATERIALIZED VIEW mv_daily_revenue AS SELECT ...; +CREATE UNIQUE INDEX ON mv_daily_revenue (day, tenant_id); -- required for CONCURRENTLY +REFRESH MATERIALIZED VIEW CONCURRENTLY mv_daily_revenue; +``` + +- Plain `REFRESH` takes an exclusive lock for the full rebuild — the view is unreadable while it runs. `CONCURRENTLY` avoids that but requires a unique index and is slower. +- PostgreSQL materialized views never refresh themselves; something must schedule it, and nothing warns you when that job dies. `computed_at` in a rollup table beats a materialized view precisely because staleness is visible. +- Choose a materialized view when the query is complex and full recomputation is cheap; choose a rollup table when the data is append-heavy and incremental refresh is the point. +- MySQL has no materialized views — use a rollup table. SQL Server indexed views refresh synchronously on write, which shifts the cost onto every insert. + +## Star Schema, When Reporting Grows Up + +- Facts are events with measures and foreign keys (one row per order line); dimensions are the descriptive entities (customer, product, date, store). +- The dimension that repays itself immediately is the date dimension: week/month/quarter, fiscal period, holiday, day-of-week, all pre-computed and joinable. +- Slowly changing dimensions: type 1 overwrites (you lose history); type 2 adds a row per version with validity dates (a fact joins to the version current at the fact's timestamp). Choose per attribute — a customer's name is type 1, their pricing tier is type 2. +- Keep facts at the finest grain you will ever need; aggregation up is trivial, disaggregation is impossible. +- On a transactional database, a star schema is often overkill — rollup tables get you most of the way. Move to a warehouse when reporting load competes with production traffic or when the transformation layer needs testing and lineage (`dbt`). + +## Sampling and Estimates + +- Exact `COUNT(*)` on a huge table scans it; the planner's estimate is instant and accurate to the last `ANALYZE`. +- `TABLESAMPLE SYSTEM (1)` samples pages (fast, clustered bias); `BERNOULLI (1)` samples rows uniformly and scans more. +- A sampled metric needs its error stated. A 1% sample of a 100k-row table gives roughly 1,000 rows; a proportion measured on 1,000 rows has a margin of error of about ±3 percentage points at 95% confidence (≈ 1/√n). Do not report a 0.5% change from that sample. +- Billing, invoices, and compliance figures are never sampled or estimated. + +## Traps + +| Trap | Why it fails | Do instead | +|---|---|---| +| `GROUP BY` a timestamp column with no spine | Empty periods disappear; averages divide by the wrong count | Join a generated date spine or a calendar table | +| Summing daily distinct counts into a monthly figure | Distinct counts are not additive | Recompute per period, or store a sketch | +| Two dashboards, two copies of the metric SQL | Definitions drift; nobody can say which is right | One rollup table or one view as the single definition | +| Joining fact to fact | Two 1:N joins multiply rows before aggregation | Aggregate each fact to a common grain, then join | +| Reporting the current, incomplete period next to complete ones | Always looks like a crash | Exclude it, or label it explicitly as partial | +| Percent change from a tiny base | 1 → 3 is "+200%" and means nothing | Show absolute values alongside, suppress below a stated minimum base | +| Averaging an average | Unweighted means of group means ignore group sizes | Sum the numerators and denominators, then divide | +| Running analytics on the primary at business hours | Long scans evict the OLTP working set from cache | Replica, rollup, or warehouse | + +Every period boundary in this file is half-open for one reason: `BETWEEN` on a timestamp silently drops the last day (→ SKILL.md Traps). diff --git a/workspace-sql/skills/sql/data-loading.md b/workspace-sql/skills/sql/data-loading.md new file mode 100644 index 0000000..c1e6bf6 --- /dev/null +++ b/workspace-sql/skills/sql/data-loading.md @@ -0,0 +1,183 @@ +# Loading and Moving Data + +Imports fail on the data, not the SQL: encodings, quoting, NULL conventions, and types the source never enforced. The workflow that survives is always the same — land raw text, validate, then transform into typed tables. + +Contents: The Staging Pattern · CSV Quirks · Bulk Load by Engine · Load Performance · Idempotent Loads · Validation · Upsert at Scale · Exporting · Dump and Restore · Cross-Engine Migration · Type Mapping · Verification · Traps + +## The Staging Pattern + +1. Create a staging table with **every column as text** and no constraints. A load that fails on row 400,000 because of one bad date has wasted the whole run. +2. Load the file into staging with the fastest bulk mechanism available. +3. Run validation queries against staging: row count, null counts, distinct value sets, parse failures. +4. `INSERT ... SELECT` with explicit casts into the real table, routing rejects to a separate table with the reason. +5. Keep the staging table until the load is confirmed; it is the only copy of what actually arrived. + +This is slower to write and much faster to finish than a load that aborts halfway. + +## CSV Quirks That Cost Hours + +| Symptom | Cause | Handling | +|---|---|---| +| Row count off by one | Header row loaded as data, or missing `HEADER` option | Declare the header explicitly | +| Fields shifted after a specific row | Unescaped delimiter inside a quoted field, or a newline inside a quoted field | Use the engine's real CSV parser (`COPY ... CSV`), never a hand-split | +| Empty strings became NULL (or vice versa) | Engines differ: PostgreSQL `COPY CSV` treats unquoted empty as NULL, quoted empty as `''` | Set `NULL ''` explicitly and decide which one you mean | +| Leading zeros gone | Loaded straight into a numeric column | Load as text into staging; cast only what is truly numeric | +| Dates off by a month | `MM/DD/YYYY` parsed as `DD/MM/YYYY` | Set `DateStyle`/`--date-format` explicitly, or parse in the cast step | +| "invalid byte sequence" | File is Latin-1/Windows-1252, declared UTF-8 | Convert with `iconv` before loading; do not guess row by row | +| First column name has odd prefix characters | UTF-8 BOM | Strip the BOM | +| Line endings appear in the last column | CRLF file loaded as LF | Set the newline convention, or strip `\r` in the cast | +| Numbers with thousands separators or currency symbols | Locale-formatted export | Clean in the cast step: strip, then cast | +| `\N`, `NULL`, `-`, `n/a` all present | Multiple export tools in the source pipeline | Normalize in the cast step, not with a single NULL marker | + +## Bulk Load by Engine + +```bash +# PostgreSQL: server-side COPY (needs server file access); \copy runs client-side +psql -c "\copy staging_users FROM 'users.csv' WITH (FORMAT csv, HEADER, NULL '')" + +# MySQL: LOCAL reads the client's filesystem; the server may require local_infile=1 +mysql -e "LOAD DATA LOCAL INFILE 'users.csv' INTO TABLE staging_users + FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' + LINES TERMINATED BY '\n' IGNORE 1 LINES" + +# SQLite +sqlite3 mydb.sqlite -cmd ".mode csv" -cmd ".import --skip 1 users.csv staging_users" + +# SQL Server +sqlcmd -Q "BULK INSERT staging_users FROM 'users.csv' + WITH (FORMAT='CSV', FIRSTROW=2, FIELDTERMINATOR=',')" +``` + +`COPY`/`LOAD DATA` is typically an order of magnitude faster than the equivalent `INSERT` batches for the same rows; single-row inserts in a loop are two orders slower and dominated by round-trip latency. + +## Load Performance + +- Drop or disable secondary indexes before loading into an **empty** table, rebuild after. On a table that already holds data and stays queryable, keep them — a missing index during the load makes concurrent reads worse than the load itself. +- Load inside one transaction where the engine allows it, so failure leaves nothing behind — but a single enormous transaction bloats WAL/undo. Chunk at `batch_size` rows per commit (default 5,000) for very large loads. +- Defer foreign key checks during the load and validate afterwards: PostgreSQL `SET CONSTRAINTS ALL DEFERRED` (constraints must be declared `DEFERRABLE`), MySQL `SET FOREIGN_KEY_CHECKS = 0` (which does **not** validate retroactively — you must check for orphans yourself). +- `ANALYZE` immediately after the load. Every plan until then is based on the pre-load statistics, and this is the top cause of "the import worked but now everything is slow". +- SQLite specifically: `PRAGMA journal_mode = WAL` and one transaction around the inserts turn a minutes-long load into seconds. + +## Idempotent, Resumable Loads + +- Give every source row a stable natural key or a file-plus-line identifier, and make the target write an upsert. Reruns then converge instead of duplicating. +- Record progress in the same transaction as the work: a `load_batches(file, chunk_no, rows, loaded_at)` row committed with its chunk means a restart resumes exactly where it stopped. +- Never rely on "the file has not changed" — hash the file and store the hash with the batch record. +- Loading the same file twice is normal in production. A load pipeline without a uniqueness constraint on the target will eventually double every number in the business. + +## Validation Before Promotion + +```sql +-- Row count against the source +SELECT COUNT(*) FROM staging_users; + +-- Which columns actually failed to parse +SELECT COUNT(*) FILTER (WHERE amount !~ '^-?\d+(\.\d+)?$') AS bad_amount, + COUNT(*) FILTER (WHERE email NOT LIKE '%@%') AS bad_email, + COUNT(*) FILTER (WHERE created_at IS NULL OR created_at = '') AS missing_date +FROM staging_users; + +-- Duplicates on the intended key, before the unique constraint rejects the load +SELECT email, COUNT(*) FROM staging_users GROUP BY email HAVING COUNT(*) > 1; + +-- Orphans against the parent table +SELECT COUNT(*) FROM staging_orders s +LEFT JOIN users u ON u.id = s.user_id::bigint WHERE u.id IS NULL; +``` + +Compare the loaded row count against the source count every time, and reconcile a control total (a sum of one numeric column) — count alone misses truncated fields. + +## Upsert at Scale + +```sql +-- PostgreSQL / SQLite: one statement, no race +INSERT INTO users (email, name, updated_at) +SELECT email, name, NOW() FROM staging_users +ON CONFLICT (email) DO UPDATE +SET name = EXCLUDED.name, updated_at = EXCLUDED.updated_at +WHERE users.name IS DISTINCT FROM EXCLUDED.name; -- skip no-op writes + +-- MySQL +INSERT INTO users (email, name) SELECT email, name FROM staging_users +ON DUPLICATE KEY UPDATE name = VALUES(name); +``` + +- The `WHERE ... IS DISTINCT FROM` clause matters at scale: without it, an unchanged row is still rewritten, producing dead tuples, WAL, and replication traffic for nothing. +- Deduplicate the source before the upsert. PostgreSQL raises `ON CONFLICT DO UPDATE command cannot affect row a second time` when one statement touches the same key twice. +- `MERGE` (SQL Server, PostgreSQL >=15) reads more naturally for multi-action loads but has more concurrency footguns; prefer `ON CONFLICT` when it expresses the operation. + +## Exporting + +```bash +# PostgreSQL +psql -c "\copy (SELECT * FROM users WHERE created_at >= '2026-01-01') TO 'users.csv' CSV HEADER" + +# MySQL: SELECT ... INTO OUTFILE writes on the SERVER and needs secure_file_priv; +# from the client, pipe a batch-mode query instead +mysql --batch --raw -e "SELECT * FROM users" mydb > users.tsv + +# SQLite +sqlite3 -header -csv mydb.sqlite "SELECT * FROM users" > users.csv + +# SQL Server +sqlcmd -Q "SELECT * FROM users" -s"," -W -o users.csv +``` + +Export the query, not the table, whenever personal data is involved — select the columns you are allowed to share. Quote and escape via the engine's CSV writer; a hand-built `CONCAT` export breaks on the first embedded comma. + +## Dump and Restore + +```bash +pg_dump -Fc mydb > backup.dump # custom format: compressed, selective, parallel restore +pg_restore -d mydb -j 4 backup.dump +mysqldump --single-transaction mydb > backup.sql # consistent snapshot without locking (InnoDB) +sqlite3 mydb.sqlite ".backup backup.sqlite" # safe during writes; plain cp is not +``` + +Full backup strategy, retention, and restore drills route from SKILL.md Quick Reference. For loading purposes the relevant points are that a dump restores a *schema plus data* pair — restoring into a database whose schema has moved on fails on the first mismatch — and that `--schema-only`/`--data-only` splits let you restore data into an already-migrated schema. + +## Cross-Engine Migration + +Order of operations that avoids rework: + +1. **Inventory the incompatibilities first**: auto-increment style, boolean representation, `ENUM`s, unsigned integers, zero dates, character sets, and every stored procedure, trigger, and view. +2. **Translate the schema by hand or with a tool, then review it.** Automated converters map types conservatively — everything becomes `TEXT` and `DOUBLE` unless corrected. +3. **Move data through a neutral format** (CSV per table) or a purpose-built tool (`pgloader` for MySQL→PostgreSQL). Do not use one engine's SQL dump as another's input. +4. **Sequences and identity columns must be reset** after the load, or the first insert collides: `SELECT setval('users_id_seq', (SELECT MAX(id) FROM users))`. +5. **Recreate indexes and constraints after loading**, not before. +6. **Run both systems in parallel** with dual writes and a reconciliation query before cutting over, unless downtime is acceptable. + +## Type Mapping Landmines + +| Source | Target | Problem | +|---|---|---| +| MySQL `TINYINT(1)` | boolean | Holds any value 0-255; audit for values outside 0/1 before casting | +| MySQL `DATETIME '0000-00-00'` | `TIMESTAMPTZ` | Not a valid date anywhere else; decide NULL or a sentinel date before the load | +| MySQL unsigned `BIGINT` | PostgreSQL `BIGINT` | Values above 2^63-1 do not fit; check `MAX()` first | +| MySQL `ENUM` | `TEXT` + CHECK | Ordering was by declaration order, not alphabetical — any `ORDER BY` on it changes meaning | +| PostgreSQL `TEXT[]` | MySQL | No array type; junction table or JSON | +| PostgreSQL `TIMESTAMPTZ` | MySQL `TIMESTAMP` | MySQL's range stops in 2038 and converts using the session timezone; use `DATETIME` in UTC | +| Any `FLOAT` money | `NUMERIC` | Values are already imprecise; the migration cannot restore lost cents | +| SQLite anything | Typed engine | SQLite has type affinity, not enforcement: a `INTEGER` column can hold text. Profile actual values, not the declared type | + +## Verification After Any Move + +- Row counts per table, both sides. +- A control total per table: `SUM` of a numeric column and `COUNT` of NULLs per important column. +- Min/max of every date column — this catches timezone shifts and epoch defaults instantly. +- Checksum a sample: order by primary key, hash the concatenated columns, compare. +- Run the application's ten most important queries against both and diff the results. +- Confirm sequences, defaults, constraints, indexes, and grants exist on the target; data-only migrations routinely lose all five. + +## Traps + +| Trap | Why it fails | Do instead | +|---|---|---| +| Loading straight into the typed target table | One bad row aborts the whole load, and you learn about it at minute 40 | Text staging table first | +| Splitting CSV lines on the delimiter | Quoted fields containing delimiters or newlines shift every subsequent column | The engine's CSV parser | +| `SET FOREIGN_KEY_CHECKS = 0` and forgetting to verify | MySQL does not re-validate when you turn it back on | Explicit orphan query after the load | +| Skipping `ANALYZE` after a large load | Every plan uses the pre-load statistics | `ANALYZE` before declaring the load done | +| One transaction for tens of millions of rows | Bloats WAL/undo, blocks vacuum, and an error loses everything | Chunk and commit; record progress | +| Re-running a load "to be safe" | Duplicates the data unless the target has a unique key and an upsert | Idempotent upsert keyed on a stable source identifier | +| Dumping production to a laptop for a "quick look" | An uncontrolled copy of every personal record you hold | Query the columns you need, on a masked environment | +| Trusting a declared type in a SQLite source | Affinity, not enforcement — the column may hold anything | Profile the real values before mapping types | diff --git a/workspace-sql/skills/sql/datetime.md b/workspace-sql/skills/sql/datetime.md new file mode 100644 index 0000000..33397c7 --- /dev/null +++ b/workspace-sql/skills/sql/datetime.md @@ -0,0 +1,204 @@ +# Dates, Times, and Timezones + +Every timestamp bug traces to one of three questions never answered: what instant is stored, in whose calendar is it interpreted, and where is the boundary of "a day". Answer all three in the schema and most of these disappear. + +Contents: Store UTC · Type Selection · The Wall-Clock Exception · Conversion · Day Boundaries · Truncation and Grouping · DST · Intervals and Age · Ranges · Business Days · Fiscal and ISO Weeks · Comparing and Indexing · Dialect Table · Traps + +## Store the Instant in UTC + +The default with no exceptions worth taking casually: store an absolute instant in UTC, convert at the edges, and never store a local time without also storing which zone it means. + +```sql +CREATE TABLE events ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + occurred_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -- PostgreSQL: UTC instant +); +``` + +An offset is not a timezone. `-05:00` tells you nothing about what that clock reads next November; `America/New_York` does. Store the IANA zone name when the zone matters, never the offset. + +## Type Selection + +| Need | PostgreSQL | MySQL | SQLite | SQL Server | +|---|---|---|---|---| +| Absolute instant | `TIMESTAMPTZ` | `TIMESTAMP` (UTC internally, converted by session zone; range ends 2038) or `DATETIME` holding UTC | `TEXT` ISO-8601 UTC, or INTEGER epoch | `DATETIMEOFFSET` | +| Naive local wall clock | `TIMESTAMP` | `DATETIME` | `TEXT` | `DATETIME2` | +| Calendar date only | `DATE` | `DATE` | `TEXT` `YYYY-MM-DD` | `DATE` | +| Time of day only | `TIME` | `TIME` | `TEXT` | `TIME` | +| Duration | `INTERVAL` | integer seconds | integer seconds | integer seconds | + +- PostgreSQL `TIMESTAMPTZ` does **not** store a zone. It converts the input to UTC on write and renders it in the session's `TimeZone` on read. The name misleads everyone once. +- MySQL `TIMESTAMP` converts on both write and read using the session timezone and tops out in 2038. `DATETIME` stores exactly what you gave it with no conversion — which is why MySQL projects usually standardize on `DATETIME` holding UTC plus an application rule. +- SQLite has no date type; text in strict `YYYY-MM-DD HH:MM:SS` sorts and compares correctly, epoch integers are compact. Pick one per database and never mix. +- Durations as `INTERVAL` are convenient but not portable and awkward to aggregate — integer seconds (or milliseconds, named in the column: `duration_ms`) travel everywhere. + +## The Wall-Clock Exception + +Some events are defined by local time, not by an instant. A 09:00 recurring meeting in Berlin stays at 09:00 through a DST change; the UTC instant moves. + +Store both parts: + +```sql +CREATE TABLE appointments ( + starts_at_local TIMESTAMP NOT NULL, -- naive wall clock + tz TEXT NOT NULL, -- IANA name, e.g. 'Europe/Berlin' + starts_at_utc TIMESTAMPTZ -- resolved instant, recomputed if rules change + GENERATED ALWAYS AS (starts_at_local AT TIME ZONE tz) STORED +); +``` + +The generated column gives you an indexable instant for "what is next" queries while the local pair remains the source of truth. Governments change timezone rules with months of notice — the resolved instants must be recomputable, which is exactly why the local time is stored. + +Birthdays, contract dates, and holidays are `DATE`, not timestamps. Storing a birthday as a timestamp makes it shift a day for some users forever. + +## Conversion + +```sql +-- PostgreSQL: AT TIME ZONE flips meaning based on the input type +SELECT occurred_at AT TIME ZONE 'America/New_York'; -- timestamptz → naive local time +SELECT local_ts AT TIME ZONE 'America/New_York'; -- naive → timestamptz (instant) + +-- Session default for rendering +SET TIME ZONE 'UTC'; + +-- MySQL (requires the timezone tables to be loaded, or only offsets work) +SELECT CONVERT_TZ(created_at, '+00:00', 'America/New_York'); + +-- SQLite +SELECT datetime(occurred_at, 'localtime'); -- server's zone, not the user's + +-- SQL Server +SELECT created_at AT TIME ZONE 'UTC' AT TIME ZONE 'Eastern Standard Time'; +``` + +MySQL's `CONVERT_TZ` returns NULL when the named zone is unknown — a fresh server without `mysql_tzinfo_to_sql` loaded turns every converted timestamp into NULL. Test with a named zone, never with an offset, or the failure hides until production. + +## Day Boundaries + +"Today" is a range in a specific zone, and the range is what the query must express: + +```sql +-- Today in the user's zone, expressed as an instant range (index-usable) +WHERE occurred_at >= (DATE '2026-03-15')::timestamp AT TIME ZONE :user_tz + AND occurred_at < (DATE '2026-03-16')::timestamp AT TIME ZONE :user_tz +``` + +- Never `WHERE DATE(occurred_at) = '2026-03-15'`: it applies a function to the column (no index) and uses the *session* zone, so the same query returns different rows for different connections. +- A daily report and a dashboard that disagree by a few rows are almost always splitting the day at different zones. +- Pick one reporting zone per tenant or per report and record it with the metric. "UTC days" is a legitimate choice; an unstated choice is not. + +## Truncation and Grouping + +```sql +DATE_TRUNC('month', occurred_at) -- PostgreSQL, also 'week','quarter' +DATE_TRUNC('day', occurred_at AT TIME ZONE 'Europe/Madrid') -- truncate in the reporting zone +DATE_FORMAT(created_at, '%Y-%m-01') -- MySQL month bucket +strftime('%Y-%m', occurred_at) -- SQLite +DATETRUNC(month, created_at) -- SQL Server 2022+; earlier: EOMONTH/DATEADD +``` + +`DATE_TRUNC('week', ...)` starts on **Monday** in PostgreSQL (ISO). MySQL's `WEEK()` defaults to Sunday and takes a mode argument controlling both the first day and how the first week of the year is counted. A "weekly" chart that disagrees between two systems is nearly always this. + +Truncate in the reporting zone, then group — grouping UTC-truncated days and relabelling them locally is off by the offset for part of every day. + +## Daylight Saving Time + +- Some local times do not exist (the spring-forward gap) and some occur twice (the autumn repeat). Converting a naive local time to an instant is therefore ambiguous or invalid for two hours a year; engines resolve it by rule, not by asking. Store the intended instant for anything that must not shift. +- A "day" is not always 24 hours: adding `INTERVAL '1 day'` in PostgreSQL adds a calendar day (23 or 25 hours across a transition), while adding `INTERVAL '24 hours'` adds exactly 24. Pick according to whether you mean "same time tomorrow" or "one day later". +- Scheduled jobs at 02:30 local never run on the spring-forward day and run twice on the autumn day. Schedule recurring jobs in UTC. +- Durations computed by subtracting local times cross transitions incorrectly. Subtract instants. +- Zone rules change: countries abolish DST, shift offsets, and split zones. The tzdata package must be updated on database servers too, not only on application hosts. + +## Intervals and Age + +```sql +-- Difference as an interval, and as a number +SELECT ended_at - started_at AS duration; -- PostgreSQL interval +SELECT EXTRACT(EPOCH FROM (ended_at - started_at)) AS seconds; -- PostgreSQL +SELECT TIMESTAMPDIFF(SECOND, started_at, ended_at); -- MySQL +SELECT (julianday(ended_at) - julianday(started_at)) * 86400.0; -- SQLite +SELECT DATEDIFF(second, started_at, ended_at); -- SQL Server +``` + +- SQL Server's `DATEDIFF(year, a, b)` counts **boundary crossings**, not elapsed years: 2025-12-31 to 2026-01-01 is 1 year. MySQL's `TIMESTAMPDIFF` counts complete units. They disagree by design. +- Age in whole years is not `days / 365.25`: leap years make that wrong for some birthdays. Use the engine's age function, or compare the month-day pair explicitly. +- "Months between" is genuinely ambiguous (Jan 31 plus one month). Adding a month clamps to the last valid day in most engines, so `Jan 31 + 1 month - 1 month` does not return Jan 31. Never round-trip through month arithmetic. +- Intervals are not additive with fixed conversions: `INTERVAL '1 month'` has no fixed number of days until anchored to a date. + +## Ranges and Overlap + +```sql +-- Portable overlap test between [a_start, a_end) and [b_start, b_end) +WHERE a_start < b_end AND a_end > b_start + +-- PostgreSQL range types handle inclusivity explicitly +WHERE tstzrange(start_at, end_at, '[)') && tstzrange(:from, :to, '[)') + +-- Prevent double booking in the database rather than in code +ALTER TABLE bookings ADD CONSTRAINT no_overlap + EXCLUDE USING gist (room_id WITH =, tstzrange(start_at, end_at, '[)') WITH &&); +``` + +Use half-open `[)` ranges everywhere: they tile without gaps or overlaps, so back-to-back bookings do not collide and `BETWEEN`'s inclusive upper bound never bites (SKILL.md Traps). Hand-written overlap conditions typically miss the containment case; the two-comparison form above covers all four. + +## Business Days and Holidays + +Business-day arithmetic in pure SQL is either wrong or unreadable. Build a calendar table once: + +```sql +CREATE TABLE calendar ( + day DATE PRIMARY KEY, + is_business_day BOOLEAN NOT NULL, + iso_week INT NOT NULL, + month_start DATE NOT NULL, + fiscal_year INT NOT NULL, + fiscal_quarter INT NOT NULL +); +-- "5 business days after 2026-03-10" +SELECT day FROM calendar WHERE is_business_day AND day > DATE '2026-03-10' +ORDER BY day OFFSET 4 LIMIT 1; +``` + +Holidays are jurisdictional and change yearly; a calendar table is data you maintain, not logic you derive. It also doubles as the date spine for reporting and the date dimension of a star schema. + +## Fiscal Calendars and ISO Weeks + +- ISO 8601: weeks start Monday, and week 1 is the week containing the first Thursday. Consequence: the first days of January can belong to week 52/53 of the previous ISO year. Always pair `EXTRACT(WEEK ...)` with `EXTRACT(ISOYEAR ...)` — using the calendar year with the ISO week misplaces early-January rows by a full year, every year. +- Some years have 53 ISO weeks. Any "week over week, 52 rows" assumption breaks in those years. +- Fiscal years rarely start in January, and retail 4-4-5 calendars are not derivable from dates at all. Put them in the calendar table. +- Quarter boundaries differ between fiscal and calendar quarters; label which one a chart shows. + +## Comparing and Indexing Timestamps + +- Compare instants to instants. A `TIMESTAMPTZ` compared against a naive string literal is interpreted in the session zone — the same query then behaves differently for two connections. +- Half-open ranges keep the index usable and avoid boundary loss; functions on the column disable the index entirely. +- Index the raw column and range-scan it, rather than indexing `DATE(col)` — the raw index serves both day queries and arbitrary ranges. +- For very large append-only tables ordered by time, partition by range on the timestamp and confirm every query filters on the partition key, or BRIN-index it. +- Beware `NOW()` inside a transaction: PostgreSQL's `NOW()`/`CURRENT_TIMESTAMP` is frozen at transaction start, so every row inserted in a long transaction shares one timestamp. `clock_timestamp()` gives real wall time. + +## Dialect Quick Table + +| Operation | PostgreSQL | MySQL | SQLite | SQL Server | +|---|---|---|---|---| +| Now (instant) | `NOW()` / `CURRENT_TIMESTAMP` | `NOW()` / `UTC_TIMESTAMP()` | `datetime('now')` (UTC) | `SYSUTCDATETIME()` | +| Add 1 day | `d + INTERVAL '1 day'` | `DATE_ADD(d, INTERVAL 1 DAY)` | `datetime(d,'+1 day')` | `DATEADD(day,1,d)` | +| Difference | `b - a` (interval) | `TIMESTAMPDIFF(unit,a,b)` | `julianday(b)-julianday(a)` | `DATEDIFF(unit,a,b)` | +| Truncate to month | `DATE_TRUNC('month',d)` | `DATE_FORMAT(d,'%Y-%m-01')` | `date(d,'start of month')` | `DATETRUNC(month,d)` (2022+) | +| Extract part | `EXTRACT(YEAR FROM d)` | `YEAR(d)` | `strftime('%Y',d)` | `DATEPART(year,d)` | +| Format | `TO_CHAR(d,'YYYY-MM-DD')` | `DATE_FORMAT(d,'%Y-%m-%d')` | `strftime('%Y-%m-%d',d)` | `FORMAT(d,'yyyy-MM-dd')` | +| Parse | `TO_TIMESTAMP(s,'...')` | `STR_TO_DATE(s,'...')` | `datetime(s)` | `TRY_CONVERT`/`PARSE` | +| Week start | Monday (ISO) | Mode-dependent, Sunday default | `strftime('%W')`, Monday | `DATEFIRST`-dependent | + +## Traps + +| Trap | Why it fails | Do instead | +|---|---|---| +| `WHERE DATE(ts) = :day` | Not sargable, and uses the session zone | Instant range in the chosen reporting zone | +| Storing a UTC offset instead of a zone name | Offsets do not survive DST | Store the IANA zone name | +| Storing a birthday as a timestamp | Shifts a day for users in other zones | `DATE` | +| `days / 365.25` for age in years | Wrong across leap years for some dates | Engine age function, or month-day comparison | +| `INTERVAL '24 hours'` for "next day" | Off by an hour across DST transitions | `INTERVAL '1 day'` when you mean the calendar day | +| Calendar year with ISO week number | Early-January rows land in the wrong year | Pair `ISOYEAR` with `WEEK` | +| Cron at 02:30 local | Skipped or duplicated on transition days | Schedule in UTC | +| Mixing `TIMESTAMP` and `TIMESTAMPTZ` columns in one schema | Every comparison reinterprets one of them without warning | One convention per database, enforced in review | +| Relying on the server's `localtime` | Server zone is infrastructure, not a user preference | Pass the user's zone explicitly | diff --git a/workspace-sql/skills/sql/debug.md b/workspace-sql/skills/sql/debug.md new file mode 100644 index 0000000..ef1129b --- /dev/null +++ b/workspace-sql/skills/sql/debug.md @@ -0,0 +1,148 @@ +# Debugging — Symptom to Cause + +Work symptom-first. Each chain is ordered by probability, and every step is a check you can run, not a guess. Wrong results outrank slow results: a fast query returning the wrong rows never raises an error. + +Contents: First Three · Wrong Row Count · Wrong Totals · Query Slow · Was Fast Yesterday · Index Ignored · Hangs · Deadlock · Connection Errors · Constraint Violations · Disk Full · Encoding · Works In One Client · Migration Failed · Replica Disagrees + +## The Universal First Three + +1. **Read the actual statement, not the intent.** For ORM-generated SQL, turn on query logging and copy the emitted text; for dynamic SQL, log the final string with parameters bound. +2. **Count before you optimize.** `SELECT COUNT(*)` on each base table with only its own filters. If a base count already surprises you, the bug is upstream of the join. +3. **`EXPLAIN (ANALYZE, BUFFERS)`** the real statement with real parameters. A plan for `user_id = 1` can differ from `user_id = 99` when one value is a most-common-value in the stats. + +## Wrong Row Count (too many, too few, duplicates) + +1. Strip the query to one table plus its `WHERE`. Add joins back one at a time and re-count after each — the join that changes the count names the bug. +2. Count grew → 1:N fan-out. Check the join key's uniqueness: `SELECT key, COUNT(*) FROM t GROUP BY key HAVING COUNT(*) > 1`. Fix by pre-aggregating, not by `DISTINCT` (SKILL.md Traps). +3. Count shrank → a `LEFT JOIN` with a predicate in `WHERE` (now an inner join, with no error), or a `NOT IN` against a subquery containing NULL (SKILL.md rule 6). +4. Count differs between two "equivalent" queries → collation or case sensitivity: MySQL's `_ci` collations match `'A' = 'a'`, PostgreSQL does not. +5. Trailing-space equality: `CHAR(n)` pads, and some engines ignore trailing spaces in comparison. `'a ' = 'a'` is true on MySQL `CHAR`, false on PostgreSQL `TEXT`. +6. Count changes between runs of the same query → concurrent writes, or a non-deterministic `LIMIT` without `ORDER BY`. + +## Wrong Totals (sums, averages, percentages) + +- Sum too high → fan-out (above). Verify by comparing `SUM(x)` against `SUM(x) / COUNT(DISTINCT join_key)`-shaped intuition, then rewrite with a pre-aggregated CTE. +- Average is off → `AVG` skips NULLs, so the denominator is not the row count. `SUM(x) / COUNT(*)` and `AVG(x)` differ whenever `x` is nullable. +- Money off by cents → `FLOAT`/`REAL` storage. `0.1 + 0.2 != 0.3` accumulates over aggregation; the schema needs `NUMERIC`/`DECIMAL` (SKILL.md rule 7). +- Integer division: `SELECT 1/2` is `0` in PostgreSQL, MySQL, and SQL Server for integer operands. Cast one side: `x * 100.0 / y`. +- Percentages don't sum to 100 → rounding each row instead of the total; round once at the end. +- Totals differ from the dashboard → different timezone boundary for "today" or a soft-delete filter one query applies and the other doesn't. + +## Query Slow + +1. Is this query even the problem? Rank by total cost first (SKILL.md rule 9). +2. `EXPLAIN (ANALYZE, BUFFERS)`: find the node with the largest actual time, not the largest estimate. +3. Estimated vs actual rows off by more than 10× → stale statistics; `ANALYZE` the table and re-plan. +4. Sequential scan with a selective filter → the index is missing or disabled (→ Index Ignored). +5. Correct index, still slow → the query returns too many rows to matter. A million rows through the network is slow no matter the plan; paginate or aggregate server-side. +6. Fast standalone, slow in the app → parameter sniffing, a different search_path/session setting, or lock waiting rather than work (→ Hangs). +7. Everything is slow, not one query → server-level: cache hit ratio, connection saturation, or another workload. + +## "It Was Fast Yesterday" + +| Cause | Check | +|---|---| +| Data crossed a size threshold and the plan flipped | Compare current plan against the shape you remember; look for a switch from index scan to seq scan | +| Statistics went stale after a bulk load | `ANALYZE`, re-plan; PostgreSQL: `last_analyze` in `pg_stat_user_tables` | +| An index was dropped or left INVALID by a failed concurrent build | PostgreSQL: `SELECT indexrelid::regclass FROM pg_index WHERE NOT indisvalid` | +| Bloat from a long-running transaction | Oldest `xact_start` in `pg_stat_activity` | +| A new index made writes slow, not reads | Compare index count against write latency; every index is written on every insert | +| Cache no longer holds the working set | Cache hit ratio dropped below the 99% OLTP threshold | +| Someone added a `LEFT JOIN` for one column | Read the diff of the query, not just the plan | +| Version upgrade changed the planner | Compare plans across the versions before blaming data | + +## Index Exists But Is Not Used + +Check in this order; the first four cover most cases. + +1. **Function or cast on the column.** `WHERE lower(email) = ?`, `WHERE created_at::date = ?`, or an implicit cast from a type mismatch all hide the column. Fix the predicate or add an expression index that matches the query text exactly. +2. **Wrong column order.** A composite `(a, b)` cannot serve `WHERE b = ?` (SKILL.md rule 3). +3. **Low selectivity.** Matching more than roughly 5-10% of rows, a scan is genuinely cheaper. Confirm with `SELECT COUNT(*) FILTER (WHERE ) * 100.0 / COUNT(*) FROM t`. +4. **Stale stats** making the planner think the predicate is unselective. `ANALYZE`, retry. +5. **`OR` across columns**, leading wildcard `LIKE '%x'`, or a non-C locale with `LIKE 'x%'` in PostgreSQL (SKILL.md Index Strategy). +6. **Different collation** between the index and the query's comparison (MySQL joins across `utf8mb4_general_ci` and `utf8mb4_0900_ai_ci` cannot use the index). +7. **The index is invalid or being built.** PostgreSQL `indisvalid = false`; MySQL `SHOW INDEX` `Comment` column. +8. Prove it before rewriting: force the choice temporarily (`SET enable_seqscan = off` in PostgreSQL, `FORCE INDEX` in MySQL) and compare actual times. If forcing the index is slower, the planner was right — never ship the hint as the fix. + +## Query Hangs (no result, no error) + +1. Is it running or waiting? PostgreSQL `pg_stat_activity.wait_event_type = 'Lock'` means waiting; MySQL `SHOW PROCESSLIST` state `Waiting for table metadata lock`. +2. Waiting → find the blocker (PostgreSQL `pg_blocking_pids(pid)`, MySQL `sys.innodb_lock_waits`) and decide: cancel it, or wait if it is nearly done. +3. Running with no output → it may be working correctly on too much data; `EXPLAIN` without `ANALYZE` returns instantly and shows the plan it chose. +4. Client shows nothing while the server is idle → the result is buffered by the driver, or the app never fetched; check the driver's cursor/fetch mode. +5. `ALTER TABLE` hangs → the lock queue, not the DDL. Every new query is now queued behind it: cancel, set `lock_timeout`, retry. +6. An open transaction in a REPL or notebook is the most common self-inflicted hang: an uncommitted `BEGIN` in another window holds the lock. + +## Deadlock Detected + +- The engine already picked a victim and rolled it back; the error is the report, not the failure. Read the log: PostgreSQL logs both statements, MySQL exposes the last one in `SHOW ENGINE INNODB STATUS`. +- Root cause is almost always inconsistent lock order across two code paths. Fix by ordering (`ORDER BY id FOR UPDATE`), not by retrying blindly. +- Deadlocks that appear only under load with no obvious two-row pattern → gap/next-key locks in MySQL REPEATABLE READ, or foreign-key locks taken on the parent row. +- Every application that writes concurrently needs a bounded retry on deadlock and serialization errors regardless. + +## Connection Errors + +| Error | Real cause | Move | +|---|---|---| +| "too many connections" / "remaining slots reserved" | App pools × instances exceed the server limit | Size pools from cores, add PgBouncer instead of raising `max_connections` | +| "connection refused" | Server not listening on that address, or wrong port | Check bind address and port before credentials | +| "no pg_hba.conf entry" / "Host is not allowed" | Host-based auth rules, not a bad password | Fix the auth rule for the client's network | +| "password authentication failed" for one app only | Different role than you tested with, or a rotated secret | Compare the role, not the password | +| "SSL required" / cert errors | Managed providers force TLS | Set the driver's SSL mode; do not disable verification to move on | +| Connections work then die after minutes | Idle timeout in a proxy/load balancer below the pool's `max_lifetime` | Set pool `max_lifetime` under the infrastructure timeout | +| Intermittent failures under load only | Pool exhaustion — waiters timing out, not the database refusing | Instrument pool wait time before touching the server | + +## Constraint Violations + +- Unique violation on insert that "should not exist" → a soft-deleted row still occupies the value, or the unique index is on a different column set than assumed. +- Unique violation under concurrency despite a check-then-insert → the check-then-insert race; use `INSERT ... ON CONFLICT`/`ON DUPLICATE KEY` or catch the violation. +- Foreign key violation on delete → children exist; decide `RESTRICT` vs `CASCADE` deliberately, and confirm the child FK column is indexed (SKILL.md rule 4). +- Foreign key violation on insert with a valid-looking id → different tenant, or the parent row was inserted in another uncommitted transaction. +- NOT NULL violation only in production → a default exists in one environment's schema and not the other; diff the schemas, don't reason about them. +- CHECK violation after a data import → the import brought values the constraint never saw; the constraint is right. + +## Disk Full / Database Won't Accept Writes + +1. Locate the consumer before deleting anything: table and index sizes ranked descending. +2. Common consumers in order: an unpartitioned events/audit/log table, bloat from dead tuples pinned by a long transaction, WAL retained by an inactive replication slot, an abandoned temp/sort spill. +3. `DELETE` does not return space to the filesystem on PostgreSQL or MySQL/InnoDB — it creates dead rows the vacuum must clean. Dropping a partition does. +4. Never `DROP` to free space during an incident until the backup is verified restorable. +5. PostgreSQL specifically: an inactive replication slot retains WAL forever and is the classic unannounced disk killer — check `pg_replication_slots` for `active = false`. + +## Encoding and Character Problems + +- `?` or `????` in place of accents/emoji → MySQL `utf8` (3-byte) instead of `utf8mb4`; the column, the table default, AND the client connection charset must all be `utf8mb4`. +- Mojibake (`é` for `é`) → UTF-8 bytes read as Latin-1 somewhere in the chain; find the single wrong layer rather than re-encoding twice. +- Import fails on "invalid byte sequence" → the file is not the encoding you declared; detect and convert before loading. +- Sorting looks wrong → collation, not encoding. `ORDER BY` follows the column's collation; changing it after the fact requires reindexing. +- Invisible characters (zero-width space, non-breaking space) pasted into data make equality fail with identical-looking strings; compare `length()` against the visible character count. + +## Works In One Client, Fails In Another + +| Difference | Check | +|---|---| +| Autocommit on/off | The GUI committed; your script left an open transaction | +| Session settings (`search_path`, `sql_mode`, timezone) | Compare `SHOW ALL` / `SHOW VARIABLES` between sessions | +| Different role or database | Same host, different privileges: diff the `GRANT`s and any RLS policy on the table | +| Statement/lock timeouts set per role | A short `statement_timeout` on the app role only | +| Prepared vs literal statements | The generic plan for a prepared statement can differ from the plan for literals | +| Driver-side type conversion | The driver casts a parameter to a type the index can't use | + +## Migration Failed Halfway + +1. Check the runner's state table first — a "dirty" flag means it knows it stopped mid-way; resolve that before rerunning anything. +2. MySQL has no transactional DDL: half the statements are already committed. Determine what actually applied by inspecting the schema, not the migration file. +3. Re-running a partially applied migration usually fails on "already exists". Write the repair as a NEW migration; never edit an applied one. +4. A migration that timed out may still be running server-side — kill the backend before retrying, or the retry deadlocks against it. +5. Prevention: run every migration against a restored copy of production-shaped data before it reaches production. + +## Replica Returns Different Data + +- Read-after-write on an async replica misses the write that just committed; route read-your-own-writes traffic to the primary. +- A long analytics query on a replica gets canceled by WAL replay conflicts — that is not a query bug. +- Replica lag rises with zero write traffic on the primary: the time-based lag metric grows unbounded when there is nothing to replay; alert only while writes are flowing. +- Counts differ permanently → the replica broke and is no longer replaying, or someone wrote to it. Compare replay position, not row counts. + +## When You Are Truly Stuck + +Reduce to the smallest reproduction: one table, minimal columns, the fewest rows that still show the behavior, on a scratch copy. Most "impossible" SQL bugs resolve at the moment the reproduction stops reproducing — the step you removed is the cause. diff --git a/workspace-sql/skills/sql/dialects.md b/workspace-sql/skills/sql/dialects.md new file mode 100644 index 0000000..9aab6a4 --- /dev/null +++ b/workspace-sql/skills/sql/dialects.md @@ -0,0 +1,163 @@ +# Dialects — Where Engines Actually Differ + +Portable SQL is a smaller language than most people write. This file is the list of places a statement that runs on one engine does something else on another without complaining, plus how to choose an engine in the first place. + +Contents: Choosing an Engine · Identifiers and Quoting · Strings and Collation · NULL Ordering · Types · Auto-Increment · Upsert · Limit and Top · Returning · Grouping Rules · Window and CTE Support · DDL Transactionality · Booleans · Concatenation and Math · Error Behavior · Feature Floors · Portability Strategy + +## Choosing an Engine + +| Engine | Choose when | Real cost | +|---|---|---| +| SQLite | Embedded, single-machine, local-first apps, tests, CLI tools, read-heavy sites on one box | One writer at a time; no network access; type affinity instead of enforcement | +| PostgreSQL | Default for anything server-side: richest types, strictest correctness, extensions | Connection-per-process (needs pooling); more knobs to get wrong | +| MySQL / MariaDB | The platform or host dictates it, or the team's operational muscle is there | No transactional DDL; historically lenient defaults; MariaDB and MySQL have diverged | +| SQL Server | .NET/Windows shops, existing licensing, strong tooling requirements | Licensing; default lock-based isolation until `READ_COMMITTED_SNAPSHOT` is enabled | + +Do not switch engines for a performance problem that is really a missing index or a bad plan. Do switch when the workload shape is wrong: heavy analytical scans belong in a columnar store (`duckdb`, `clickhouse`, a warehouse), not in a tuned OLTP database. + +## Identifiers and Quoting + +| Engine | Quote char | Unquoted case | Consequence | +|---|---|---|---| +| PostgreSQL | `"col"` | Folded to **lower**case | `CREATE TABLE "Users"` must be quoted forever after | +| MySQL | `` `col` `` | Preserved; table-name case sensitivity depends on the filesystem | A schema built on macOS breaks on Linux | +| SQLite | `"col"`, `` `col` ``, `[col]` | Preserved, compared case-insensitively | Very permissive; hides problems until you migrate | +| SQL Server | `[col]` | Preserved, compared by database collation | Usually case-insensitive | + +Use lowercase snake_case unquoted everywhere and the problem never appears. MySQL's `lower_case_table_names` is set at initialization and cannot be safely changed afterwards — decide before the first deploy. + +## Strings and Collation + +- Default comparison: PostgreSQL and SQLite are case-**sensitive**; MySQL (`utf8mb4_0900_ai_ci`) and SQL Server are case-**insensitive** by default. The same `WHERE email = ?` finds different rows across engines. +- PostgreSQL: use `citext` or compare `lower(col)` with a matching expression index. MySQL: choose a `_bin` or `_cs` collation on the column when you need sensitivity. +- MySQL `utf8` is 3-byte and cannot store emoji or some CJK characters. `utf8mb4` is the only correct choice, and it must be set on the column, the table, and the connection (SKILL.md rule 7). +- Mixing collations across a join disables the index and can raise "illegal mix of collations" — align collations when creating tables, not later. +- `CHAR(n)` pads with spaces and some engines ignore trailing spaces in comparison; `VARCHAR`/`TEXT` do not. Avoid `CHAR` except for genuinely fixed-width codes. +- Empty string versus NULL: Oracle treats `''` as NULL; every engine here treats them as different values. Anything importing from Oracle inherits that ambiguity. +- Concatenating anything with NULL yields NULL in PostgreSQL, SQLite, and MySQL; SQL Server's `+` does too unless `CONCAT_NULL_YIELDS_NULL` is off, while `CONCAT()` ignores NULLs everywhere. + +## NULL Ordering + +| Engine | `ORDER BY col ASC` | Override | +|---|---|---| +| PostgreSQL | NULLs **last** | `NULLS FIRST` / `NULLS LAST` | +| SQLite | NULLs first | `NULLS LAST` (>=3.30) | +| MySQL | NULLs first | No clause — sort by `col IS NULL, col` | +| SQL Server | NULLs first | No clause — sort by `CASE WHEN col IS NULL THEN 1 ELSE 0 END, col` | + +Adding an explicit `NULLS LAST` can disable an index that does not carry that ordering, so prefer making the column `NOT NULL` when the ordering matters. + +## Types + +| Concept | PostgreSQL | MySQL | SQLite | SQL Server | +|---|---|---|---|---| +| Unbounded text | `TEXT` | `TEXT`/`LONGTEXT` (cannot be fully indexed; needs a prefix length) | `TEXT` | `NVARCHAR(MAX)` | +| Exact decimal | `NUMERIC(p,s)` | `DECIMAL(p,s)` | `NUMERIC` (affinity only) | `DECIMAL(p,s)` | +| Boolean | `BOOLEAN` | `TINYINT(1)` | `INTEGER` 0/1 | `BIT` | +| UUID | `UUID` | `BINARY(16)` or `CHAR(36)` | `TEXT`/`BLOB` | `UNIQUEIDENTIFIER` | +| Array | `TYPE[]` | none (JSON) | none (JSON) | none (JSON) | +| JSON | `JSONB` | `JSON` | JSON functions over `TEXT` | `NVARCHAR` + JSON functions | +| Enum | native `ENUM` type | inline `ENUM(...)` | none | none | +| Unsigned integers | none | yes | none | none | +| IP / network | `INET`, `CIDR` | none | none | none | + +SQLite's type affinity means declared types are advisory: an `INTEGER` column accepts `'abc'` unless the table is declared `STRICT` (>=3.37). Any migration out of SQLite must profile actual values, not declared types. + +MySQL cannot index a `TEXT` column without a prefix length (`INDEX (col(191))`), and the 191 convention comes from the old 767-byte index limit under `utf8mb4` — modern InnoDB with `DYNAMIC` row format allows 3072 bytes, so the limit is often no longer needed. + +## Auto-Increment Behavior + +| Engine | Syntax | Gaps | Reset after load | +|---|---|---|---| +| PostgreSQL | `GENERATED ALWAYS AS IDENTITY` (prefer over `SERIAL`) | Sequences are non-transactional: rollbacks consume values | `SELECT setval('t_id_seq', (SELECT MAX(id) FROM t))` | +| MySQL | `AUTO_INCREMENT` | Gaps on rollback; InnoDB may reset the counter on restart in older versions | `ALTER TABLE t AUTO_INCREMENT = n` | +| SQLite | `INTEGER PRIMARY KEY` (rowid alias) | Reuses deleted maximum values unless `AUTOINCREMENT` is declared | Managed via `sqlite_sequence` | +| SQL Server | `IDENTITY(1,1)` | Large jumps possible after restart (identity cache) | `DBCC CHECKIDENT` | + +Gaps in generated ids are normal in every engine. Treat them as meaningless: any business logic that counts on contiguous ids (invoice numbering, "records processed") is already broken and needs its own sequence table. + +## Upsert + +```sql +-- PostgreSQL / SQLite +INSERT INTO t (k, v) VALUES (?, ?) +ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v; + +-- MySQL +INSERT INTO t (k, v) VALUES (?, ?) +ON DUPLICATE KEY UPDATE v = VALUES(v); -- MySQL >=8.0.20: v = new.v + +-- SQL Server (and PostgreSQL >=15) +MERGE INTO t AS tgt USING (VALUES (?, ?)) AS src(k, v) ON tgt.k = src.k +WHEN MATCHED THEN UPDATE SET v = src.v +WHEN NOT MATCHED THEN INSERT (k, v) VALUES (src.k, src.v); +``` + +`ON DUPLICATE KEY` fires on **any** unique constraint, not just the one you had in mind — a row can update through a different key than expected. `ON CONFLICT (k)` names the constraint explicitly, which is why it is the safer construct. + +## Limit, Offset, Top + +```sql +SELECT ... ORDER BY id LIMIT 20 OFFSET 40; -- PostgreSQL, MySQL, SQLite +SELECT TOP 20 ... ORDER BY id; -- SQL Server, no offset +SELECT ... ORDER BY id OFFSET 40 ROWS FETCH NEXT 20 ROWS ONLY; -- SQL Server 2012+, standard +``` + +SQL Server requires `ORDER BY` for `OFFSET ... FETCH`. Every engine returns undefined order without `ORDER BY`, and a unique tiebreaker is required for stable pagination. + +## Returning Affected Rows + +- PostgreSQL and SQLite (>=3.35): `INSERT/UPDATE/DELETE ... RETURNING *`. +- SQL Server: `OUTPUT INSERTED.*` / `OUTPUT DELETED.*`, which can also write into a table. +- MySQL: none. Use `LAST_INSERT_ID()` for a single insert, or re-select. MariaDB has `RETURNING`. + +Multi-row inserts on MySQL give you the first generated id from `LAST_INSERT_ID()`; the rest are consecutive only when `innodb_autoinc_lock_mode` guarantees it. Batch inserts needing ids back should insert client-generated keys instead. + +## GROUP BY Strictness + +- Standard, PostgreSQL, and SQL Server: every selected non-aggregated column must appear in `GROUP BY` (PostgreSQL relaxes this when you group by the primary key, since everything else is functionally dependent). +- MySQL historically allowed selecting arbitrary ungrouped columns, returning an unspecified row's value. `ONLY_FULL_GROUP_BY` is in the default `sql_mode` from 5.7 — legacy queries fail on upgrade, and the fix is usually `ANY_VALUE()` or a proper aggregate. +- SQLite permits bare columns and documents that with `MIN`/`MAX` the bare columns come from the matching row — convenient, non-portable. + +## Window Functions and CTEs + +| Feature | Floor | +|---|---| +| Window functions | PostgreSQL 8.4, MySQL 8.0, MariaDB 10.2, SQLite 3.25, SQL Server 2005 | +| Recursive CTE | PostgreSQL 8.4, MySQL 8.0, SQLite 3.8.3, SQL Server 2005 | +| `FILTER (WHERE ...)` on aggregates | PostgreSQL, SQLite 3.30; MySQL and SQL Server need `CASE` | +| `GROUPING SETS`/`CUBE`/`ROLLUP` | PostgreSQL 9.5, SQL Server, MySQL (`WITH ROLLUP` only), SQLite none | +| Generated columns | PostgreSQL 12 (stored only), MySQL 5.7, SQLite 3.31, SQL Server (computed) | +| Partitioned tables | PostgreSQL 10 declarative, MySQL native, SQL Server, SQLite none | +| Full-text | PostgreSQL tsvector, MySQL InnoDB FTS, SQLite FTS5, SQL Server FTS | + +CTE materialization also differs: PostgreSQL inlined CTEs from version 12 (before that every CTE was an optimization fence), MySQL 8 treats them like derived tables, SQL Server always inlines. A query that relied on the fence for performance changes behavior on upgrade. + +## DDL Transactionality + +PostgreSQL, SQLite, and SQL Server run DDL inside transactions — a failed migration rolls back cleanly. MySQL and MariaDB commit implicitly before and after each DDL statement, so a multi-statement migration can end half-applied with no rollback available. + +Consequences for MySQL specifically: one DDL statement per migration file, a verified backup before schema changes, and online schema-change tooling (`gh-ost`, `pt-online-schema-change`) for large tables. + +## Booleans, Concatenation, Math + +- Booleans: PostgreSQL has a real type; MySQL's `TRUE`/`FALSE` are literals for 1/0 in a `TINYINT`; SQL Server has `BIT` and no boolean expression type, so `SELECT (a > b)` is invalid there and needs `CASE`. +- Concatenation: `||` in PostgreSQL and SQLite; `CONCAT()` in MySQL (where `||` means logical OR unless `PIPES_AS_CONCAT` is set); `+` or `CONCAT()` in SQL Server. +- Integer division truncates in PostgreSQL, MySQL, and SQL Server; SQLite also truncates for integer operands. Cast one operand to get a real division. +- Modulo of negatives, rounding mode (half-up vs half-even), and `AVG` of integers all vary. Cast to a decimal type before any arithmetic whose result is reported to a user. + +## Error Behavior on Bad Data + +- Inserting a too-long string: PostgreSQL and SQL Server error; MySQL errors in strict mode (default since 5.7) and truncated without warning before that. +- Invalid dates: MySQL historically accepted `'0000-00-00'`; strict mode rejects it, but old data survives upgrades and breaks every migration out. +- Division by zero: PostgreSQL, SQL Server, and SQLite raise an error; MySQL returns NULL unless `ERROR_FOR_DIVISION_BY_ZERO` is in `sql_mode`. +- Out-of-range numerics: error in PostgreSQL, clamped or rejected in MySQL depending on mode. + +When auditing a MySQL database, read `SELECT @@sql_mode` first — it determines which of these are errors and which are corruption with no error. + +## Portability Strategy + +- **One target engine.** The portable subset is small, and code written for "any database" is usually written well for none. Support a second engine only when a paying requirement says so. +- If you must be portable: no vendor types, no engine-specific functions in shared code, ANSI `CASE` instead of `IF`/`IIF`, explicit `CAST`, no `RETURNING`, no arrays, no upsert syntax — push the differences into a thin per-engine layer. +- Test against the real target engine, not SQLite-in-memory standing in for PostgreSQL: SQLite accepts statements the target rejects, and enforces less. +- Keep a written list of the version floors you rely on (`engine_version` in Configuration) so an upgrade is a checklist, not an archaeology project. diff --git a/workspace-sql/skills/sql/json.md b/workspace-sql/skills/sql/json.md new file mode 100644 index 0000000..2268826 --- /dev/null +++ b/workspace-sql/skills/sql/json.md @@ -0,0 +1,197 @@ +# JSON and Semi-Structured Data + +The decision that matters is not how to query JSON — it is which fields should never have been JSON. A JSON column buys schema flexibility and pays with no per-key statistics, no constraints, and no cheap alteration. + +Contents: Column or JSON · Engine Support · Extraction Syntax · Indexing · Containment vs Path · Updating · Constraints · Arrays and Expansion · Building JSON · Migrating Out · Traps + +## Column Or JSON + +Put it in a real column when any of these is true: + +- You filter, join, sort, or group by it (JSON keys have no statistics, so the planner guesses and picks bad plans) +- It must be `NOT NULL`, unique, checked, or referenced by a foreign key +- It exists on every row +- It has a stable type and unit + +Keep it in JSON when the shape is genuinely per-row and open-ended: third-party webhook payloads kept verbatim, user-defined custom fields, feature-flag blobs, a request snapshot for debugging, sparse attributes across thousands of product categories. + +The productive middle: store the raw document AND promote the three or four fields you query into generated or plain columns. You keep fidelity and get indexes on what matters. + +```sql +-- PostgreSQL >=12: promote a hot key to a real, indexable column +ALTER TABLE events ADD COLUMN user_id BIGINT + GENERATED ALWAYS AS ((payload->>'user_id')::bigint) STORED; +CREATE INDEX idx_events_user ON events(user_id); +``` + +## Engine Support + +| Engine | Type | Notes | +|---|---|---| +| PostgreSQL | `JSONB` (binary, indexed) and `JSON` (text) | Use `JSONB` unless you must preserve key order and whitespace exactly; `JSONB` deduplicates keys and reorders them | +| MySQL >=5.7 | `JSON` (binary internally) | No direct index on the column; index a generated column instead | +| MariaDB | `JSON` is an alias for `LONGTEXT` with a CHECK | Not the same engine-level type as MySQL; expect differences | +| SQLite >=3.38 | JSON functions built in (`json1` earlier as an extension) | Stored as `TEXT`; `JSONB` format added in 3.45 | +| SQL Server | `NVARCHAR` + `JSON_VALUE`/`OPENJSON` | Index a persisted computed column | + +## Extraction Syntax + +```sql +-- PostgreSQL: -> returns JSON, ->> returns text (the distinction that causes most bugs) +SELECT payload->'user'->>'name' AS name, -- text + payload->'items'->0->>'sku' AS first_sku, -- array index is 0-based + payload #>> '{user,address,city}' AS city -- path form +FROM events; + +-- Casting is explicit and required for comparison +WHERE (payload->>'amount')::numeric > 100 + +-- MySQL +SELECT JSON_UNQUOTE(JSON_EXTRACT(payload, '$.user.name')) AS name, + payload->>'$.user.name' AS same_thing -- ->> is the unquoting shorthand +FROM events; + +-- SQLite +SELECT json_extract(payload, '$.user.name') FROM events; + +-- SQL Server +SELECT JSON_VALUE(payload, '$.user.name') FROM events; -- scalars +SELECT JSON_QUERY(payload, '$.items') FROM events; -- objects/arrays +``` + +`->` vs `->>` is the number-one JSON bug: `payload->'amount' = '100'` compares a JSON number against a JSON string and is false. Use `->>` and cast, or compare JSON to JSON (`payload->'amount' = '100'::jsonb`). + +## Indexing + +```sql +-- PostgreSQL: GIN over the whole document, containment queries only +CREATE INDEX idx_events_payload ON events USING GIN (payload jsonb_path_ops); +SELECT * FROM events WHERE payload @> '{"status":"paid"}'; + +-- PostgreSQL: B-tree on one extracted key — supports equality, ranges, and sorting +CREATE INDEX idx_events_status ON events ((payload->>'status')); +SELECT * FROM events WHERE payload->>'status' = 'paid'; + +-- MySQL: index a generated column +ALTER TABLE events ADD COLUMN status VARCHAR(32) + GENERATED ALWAYS AS (payload->>'$.status') STORED; +CREATE INDEX idx_events_status ON events(status); + +-- SQLite: expression index +CREATE INDEX idx_events_status ON events(json_extract(payload, '$.status')); +``` + +- `jsonb_path_ops` is smaller and faster than the default GIN opclass but supports only containment (`@>`) — not key-existence (`?`) or path queries. Choose by the operator you actually use. +- A GIN index does not help `payload->>'k' = 'v'`, and a B-tree expression index does not help `@>`. They are different access paths; adding the wrong one changes nothing. +- The expression in the index must match the query text exactly, including the cast. +- No engine gives per-key statistics inside a JSON document: the planner uses a fixed guess for JSON predicates, so estimates are frequently far off and joins downstream get the wrong algorithm. + +## Containment vs Path Queries + +```sql +-- Containment: "does the document include this subtree" — GIN-indexable, no wildcards +WHERE payload @> '{"user":{"role":"admin"}}' + +-- Key existence +WHERE payload ? 'discount_code' -- top-level key +WHERE payload ?| array['a','b'] -- any of +WHERE payload ?& array['a','b'] -- all of + +-- JSONPath (PostgreSQL >=12): filters and comparisons inside the document +WHERE payload @? '$.items[*] ? (@.price > 100)' +``` + +Containment matches nested structure, so `@> '{"a":1}'` matches `{"a":1,"b":2}` but `@> '{"a":[1]}'` requires the array to contain 1. Array containment ignores order and duplicates — a frequent source of surprise when comparing lists. + +## Updating JSON + +```sql +-- PostgreSQL: merge at the top level, or set a path +UPDATE users SET prefs = prefs || '{"theme":"dark"}'::jsonb WHERE id = 1; +UPDATE users SET prefs = jsonb_set(prefs, '{notify,email}', 'true', true) WHERE id = 1; +UPDATE users SET prefs = prefs #- '{legacy_flag}' WHERE id = 1; -- delete a path + +-- MySQL +UPDATE users SET prefs = JSON_SET(prefs, '$.theme', 'dark') WHERE id = 1; +UPDATE users SET prefs = JSON_REMOVE(prefs, '$.legacy_flag') WHERE id = 1; +``` + +- `||` merges only the top level: `{"a":{"x":1}} || {"a":{"y":2}}` yields `{"a":{"y":2}}`, dropping `x` with no error. Nested merges need `jsonb_set` per path or a recursive helper. +- `jsonb_set` returns NULL if the document is NULL — a single NULL column wipes the update. Guard with `COALESCE(prefs, '{}'::jsonb)`. +- Every JSON update rewrites the entire document: a 200 KB blob updated per request writes 200 KB per request and inflates WAL, replication traffic, and bloat. High-churn keys belong in real columns. +- Concurrent read-modify-write on the same document loses updates exactly like any other read-modify-write. + +## Constraints on JSON + +```sql +-- Require a key and constrain its value +ALTER TABLE events ADD CONSTRAINT events_type_valid + CHECK (payload ? 'type' AND payload->>'type' IN ('click','view','purchase')); + +-- Require it to be an object, not an array or scalar +ALTER TABLE events ADD CONSTRAINT events_payload_object + CHECK (jsonb_typeof(payload) = 'object'); +``` + +CHECK constraints are the only validation a JSON column gets — there is no schema. Validate the handful of fields your code depends on, and treat everything else as untrusted at read time. If you find yourself writing more than three or four such constraints, the fields belong in columns. + +## Arrays and Expansion + +```sql +-- PostgreSQL: one row per array element +SELECT e.id, item->>'sku' AS sku, (item->>'qty')::int AS qty +FROM events e, jsonb_array_elements(e.payload->'items') AS item; + +-- Aggregate back +SELECT id, jsonb_agg(item) FROM ... GROUP BY id; + +-- MySQL >=8.0.4 +SELECT e.id, j.sku FROM events e, +JSON_TABLE(e.payload, '$.items[*]' COLUMNS (sku VARCHAR(64) PATH '$.sku')) j; + +-- SQLite +SELECT e.id, j.value->>'sku' FROM events e, json_each(e.payload, '$.items') j; +``` + +`jsonb_array_elements` on a NULL or non-array value errors or returns nothing depending on the variant — use `jsonb_array_elements(COALESCE(payload->'items','[]'::jsonb))`. In a `LEFT JOIN LATERAL`, an empty array drops the parent row unless the lateral is left-joined explicitly. + +## Building JSON in SQL + +```sql +-- PostgreSQL: assemble a nested result in one round trip +SELECT jsonb_build_object( + 'id', o.id, + 'total', o.total, + 'items', COALESCE(jsonb_agg(jsonb_build_object('sku', i.sku, 'qty', i.qty)) + FILTER (WHERE i.id IS NOT NULL), '[]'::jsonb) + ) +FROM orders o LEFT JOIN order_items i ON i.order_id = o.id +GROUP BY o.id; +``` + +The `FILTER (WHERE i.id IS NOT NULL)` is required: without it, an order with no items produces `[null]` instead of `[]`. This pattern replaces N+1 round trips for nested API responses — but it moves serialization CPU onto the database, so use it for read-heavy endpoints, not for everything. + +## Migrating Out Of JSON + +When a key graduates to a column: + +1. Add the nullable column. +2. Backfill in batches from the JSON (`batch_size` rows per commit). +3. Dual-write both places while old code is live. +4. Switch reads to the column; verify nothing reads the JSON key. +5. Drop the key from the document in batches, or leave it as an inert copy if space is not a concern. + +Steps 3 and 4 are the expand-migrate-contract shape; the only addition is that removing the key from millions of documents is itself a full rewrite of each row, so it is optional and often skipped. + +## Traps + +| Trap | Why it fails | Do instead | +|---|---|---| +| `payload->'x' = 'value'` | Compares JSON to a string literal; never true | `payload->>'x' = 'value'` | +| Ordering by `payload->>'amount'` | Text ordering: `'9' > '100'` | Cast: `ORDER BY (payload->>'amount')::numeric` | +| Filtering hot paths on JSON keys | No statistics, no index unless one was built for that exact expression | Promote to a column | +| `jsonb_set` on a NULL column | Returns NULL, erasing the row's data | `COALESCE(col, '{}'::jsonb)` | +| `||` for a nested merge | Replaces the whole subtree | `jsonb_set` per path | +| Storing numbers as JSON floats | JSON numbers are doubles in many parsers; money loses cents | Store money as a string in JSON, or as a real `NUMERIC` column | +| Using JSON to avoid writing a migration | Every read now branches on missing keys, forever | Write the migration; JSON debt compounds faster than schema debt | +| Huge documents in a hot table | Every update rewrites the whole value; PostgreSQL TOASTs and de-TOASTs it | Split the large blob into its own table, joined only when needed | diff --git a/workspace-sql/skills/sql/memory-template.md b/workspace-sql/skills/sql/memory-template.md new file mode 100644 index 0000000..7fa957c --- /dev/null +++ b/workspace-sql/skills/sql/memory-template.md @@ -0,0 +1,38 @@ +# Memory Template — SQL + +Create `~/Clawic/data/sql/memory.md` with this structure: + +```markdown +# SQL Memory + +## Status +status: ongoing +last: YYYY-MM-DD + +## Context + + + +## Schema Seen + + + +## Pain Points + + +## Preferences + + + +--- +*Updated: YYYY-MM-DD* +``` + +## Status Values + +| Value | Meaning | +|-------|---------| +| `ongoing` | Still learning their schema and environment | +| `complete` | Know the schema and workflow well enough to skip re-asking | + +Never record credentials, connection strings, hostnames, or rows of real data. diff --git a/workspace-sql/skills/sql/modeling.md b/workspace-sql/skills/sql/modeling.md new file mode 100644 index 0000000..035afbe --- /dev/null +++ b/workspace-sql/skills/sql/modeling.md @@ -0,0 +1,135 @@ +# Data Modeling — Keys, Relationships, and Normal Forms + +Design order that survives contact with production: identify entities and their identity → fix cardinality → choose keys → add constraints → then, and only then, denormalize against a measured read. Reversing that order produces schemas whose bugs are unfixable without a migration. + +Contents: Entity Test · Cardinality · Key Choice · Natural Keys · Normal Forms · Denormalization · Nullability · Enumerations · Junctions · Inheritance · Units and Precision · Naming · Review Checklist + +## Is It An Entity Or A Column? + +- It is an entity when it has an independent lifecycle, its own attributes, or something else must reference it. +- It is a column when it only ever describes exactly one parent row and is never referenced. +- The tell for a missed entity: repeated column groups (`address_line1`, `address_line2`, `address_city` appearing on three tables) or numbered columns (`phone1`, `phone2`, `phone3`). Numbered columns are a one-to-many relationship written by hand. +- The tell for an over-modeled entity: a table whose only columns are an id and a name, joined every single time, never referenced from more than one place, never edited. That is an enum (→ Enumerations). + +## Cardinality: Get It Right Before Anything Else + +| Relationship | Implementation | Common mistake | +|---|---|---| +| 1:1 | FK with a UNIQUE constraint, or merge into one table | Splitting for aesthetics; split only for access frequency, permissions, or genuinely optional bulk columns | +| 1:N | FK on the **many** side | Putting an array of child ids on the parent, which no FK can validate | +| M:N | Junction table with a composite PK | A comma-separated string column; every query becomes `LIKE` and nothing is enforceable | +| Optional 1:N | Nullable FK on the many side | Sentinel `0`/`-1` instead of NULL, which breaks the FK | +| Self-referencing | Nullable FK to the same table | No cycle guard; recursive reads then loop forever | + +Ask "can this ever be more than one?" for every 1:1 and "will it always be exactly one?" for every required FK. Both answers change over the product's life, and the M:N junction is cheap to build up front and expensive to retrofit. + +## Primary Key Choice + +| Option | Pick it when | Cost | +|---|---|---| +| `BIGINT GENERATED ALWAYS AS IDENTITY` | Default for almost everything | Values are guessable and leak volume (id 4,120 tells a competitor your order count) | +| UUIDv7 / ULID | Ids generated client-side, offline, or across shards; ids exposed publicly | 16 bytes vs 8 in every index; still time-ordered so inserts stay local | +| UUIDv4 | Only when unpredictability matters more than write locality | Random insert position fragments the B-tree and inflates every index | +| Composite natural key | Pure junction tables | Every child table must carry all the columns | +| `INT` | Never for a growing table (SKILL.md rule 2) | Overflow is an outage-grade migration | + +Public exposure is a separate decision from the primary key: keep `BIGINT` internally and add a unique, indexed external id (UUID or short random slug) for URLs and APIs. That gives compact joins internally and no enumeration externally. + +## Natural Keys: The Honest Frontier + +A natural key is defensible when the value is defined by an outside authority, immutable by that authority, and never re-issued: ISO 3166 country codes, ISO 4217 currency codes. + +Values that look natural and are not: email (users change them, and case/alias normalization is a policy), phone number (reassigned), national identifiers (formats change, and they are regulated personal data), SKU (merchandising renames them), "username" (rename features exist), any code with a check digit whose standard has been revised. + +When in doubt: surrogate PK plus a UNIQUE constraint on the natural key. That gives you the integrity guarantee without cascading a rename through every child row. + +## Normal Forms, In The Order They Bite + +- **1NF** — one value per column, no repeating groups. Violation: a comma-separated `tags` column. Cost: no index, no FK, no counting. +- **2NF** — no non-key column depends on part of a composite key. Violation: `order_items(order_id, product_id, product_name)` — `product_name` depends on `product_id` alone. Cost: renaming a product updates thousands of rows and misses some. +- **3NF** — no non-key column depends on another non-key column. Violation: storing both `zip` and `city` when `zip` determines `city`. Cost: they drift. +- **BCNF** — every determinant is a candidate key. Matters mainly for tables with overlapping candidate keys; reaching 3NF handles the vast majority of real designs. +- **4NF/5NF** — independent multi-valued facts crammed into one table (a person's languages and their skills in one row) produce a cross-product of rows. Rare, but unmistakable once seen: the row count is the product of two unrelated lists. + +Design at 3NF by default. Every deliberate exception gets a comment saying which read it serves and what keeps the copies in sync. + +## Denormalization: The Four Legitimate Kinds + +Denormalize only after the normalized read is measured and too slow, and only with a named synchronization mechanism. + +1. **Cached aggregate** (`posts.comment_count`) — synced by trigger or by the same transaction as the insert. Becomes a lock hotspot on hot rows; shard into N counter rows and SUM if it does. +2. **Copied immutable value** (`order_items.unit_price_at_purchase`) — not denormalization at all: the price at purchase time is a different fact from the current price. Always copy this one. +3. **Materialized rollup** (daily revenue table) — refreshed on a schedule, explicitly stale, queried by dashboards. +4. **Redundant FK for locality** (`comments.post_author_id` to avoid a join in a hot path) — measurable win only on very hot paths, and it must be maintained on parent change. + +Anything outside these four is usually a missing index. + +## Nullability + +- `NOT NULL` is the default posture; nullable is the exception you justify. Every nullable column multiplies the query branches downstream (SKILL.md rule 6). +- NULL means "unknown or not applicable". It does not mean zero, empty string, or false — encoding those as NULL destroys the distinction between "no answer" and "answered zero". +- Never use a sentinel (`-1`, `'N/A'`, `1970-01-01`) to avoid NULL: it defeats aggregates, sorts wrong, and eventually collides with a real value. +- A column nullable "for now, during the migration" needs a scheduled follow-up to set `NOT NULL`, or it is permanent. +- Uniqueness over nullable columns treats NULLs as distinct in most engines — two rows with NULL both pass a UNIQUE constraint. PostgreSQL >=15 offers `UNIQUE NULLS NOT DISTINCT` when you need the opposite. + +## Enumerations: Three Options + +| Option | When | Watch out | +|---|---|---| +| `TEXT` + `CHECK (col IN (...))` | Default — values evolve with normal migrations | Constraint must be dropped and re-added to change the set | +| Native `ENUM` type | Fixed forever, and the storage saving matters | PostgreSQL cannot remove or reorder values; MySQL renumbers on `ALTER` | +| Lookup table + FK | The set has attributes (label, sort order, active flag) or is user-editable | An extra join on every read | + +Store the machine value, not the display label — labels are localized and get edited. Transition rules between states belong in code or a trigger, never in a CHECK, which only sees the new row. + +## Junction Tables + +```sql +CREATE TABLE post_tags ( + post_id BIGINT NOT NULL REFERENCES posts(id) ON DELETE CASCADE, + tag_id BIGINT NOT NULL REFERENCES tags(id) ON DELETE CASCADE, + PRIMARY KEY (post_id, tag_id) +); +CREATE INDEX idx_post_tags_tag ON post_tags(tag_id); -- the reverse lookup (SKILL.md rule 4) +``` + +- The composite PK gives you one direction's index for free; the other direction needs its own index, and its absence is the most common junction-table performance bug. +- Add a surrogate id only when the relationship itself gains attributes (`added_at`, `added_by`, `role`) or something must reference the relationship. +- Column order in the PK is a real decision: put the side you filter on most often first (SKILL.md rule 3). + +## Inheritance / Subtypes + +Three shapes for "a Vehicle is a Car or a Truck": + +- **Single table** — all columns in one table with a `type` discriminator. Simplest queries; subtype-specific columns must be nullable, so the database cannot enforce "trucks have `payload_kg`". Best when subtypes differ by a couple of columns. +- **Table per subtype** with a shared parent table holding the common columns and a FK. Full integrity, one join per read. Best when subtypes have substantial distinct attributes and are queried separately. +- **Table per concrete type**, no shared parent. Fastest per-type reads, but no way to reference "any vehicle" with a FK, and every cross-type query is a `UNION ALL`. + +Default to single table below roughly five subtype-specific columns, table-per-subtype above it. The forcing question: does anything need a foreign key to "any subtype"? If yes, you need a shared parent table. + +## Units, Precision, and Encoding of Values + +- Store money as integer minor units or `NUMERIC(p, s)`, and store the currency next to it. An amount without a currency column is a bug waiting for the second market. +- Put the unit in the column name (`weight_kg`, `duration_ms`, `distance_m`) — a column named `weight` is read as pounds by someone eventually. Convert at the presentation layer only. +- Percentages: pick ratio (0-1) or percent (0-100), state it in the name (`discount_rate`, `discount_pct`), and never mix them in one schema. +- Precision for `NUMERIC(p, s)`: `s` must exceed the smallest unit you will ever aggregate. Interest and tax intermediate values typically need 4-6 decimal places even when the display shows 2. +- Booleans that will grow a third state (`is_approved` → approved/rejected/pending) start as a status enum. The retrofit costs a migration and a search for every truthiness check. + +## Naming That Prevents Bugs + +- One concept, one name, across the whole schema: `user_id` everywhere, never `uid` in one table and `owner` in another. +- FK column = referenced table singular + `_id` (`user_id` → `users.id`). Deviation forces every reader to check. +- Booleans read as assertions (`is_active`, `has_verified_email`); timestamps end in `_at` (`deleted_at`); dates end in `_on`; durations carry the unit. +- Name constraints and indexes explicitly (`uq_users_email_live`, `fk_orders_user`), because the engine's autogenerated name is what appears in production error messages. +- Avoid reserved words (`order`, `user`, `group`, `check`, `end`) — quoting them forever is worse than choosing `orders`, `accounts`, `groups`. Casing and quoting rules differ by engine. + +## Review Checklist + +- Every table has a primary key, and its type follows SKILL.md rule 2. +- Every FK column is indexed (SKILL.md rule 4), and its `ON DELETE` behavior was chosen, not defaulted. +- Every UNIQUE constraint is scoped correctly: per tenant, and excluding soft-deleted rows if they exist. +- Every nullable column has a reason; every NULL means "unknown", not zero. +- No repeating groups, no comma-separated lists, no numbered columns. +- Every timestamp is zone-aware and stored as UTC; every money column has a currency. +- Nothing stores a derived value without a stated sync mechanism. +- The model answers the three or four queries the feature actually needs — write them out and check. diff --git a/workspace-sql/skills/sql/operations.md b/workspace-sql/skills/sql/operations.md new file mode 100644 index 0000000..7047270 --- /dev/null +++ b/workspace-sql/skills/sql/operations.md @@ -0,0 +1,362 @@ +# Operations + +Contents: Migrations · Zero-Downtime Changes · Large-Table DDL · Backup & Restore · RPO/RTO · Restore Drills · Maintenance · Monitoring · Alert Thresholds · Incident Runbook · Connection Pooling · Replication · High Availability · Major Version Upgrades · Capacity Planning + +## Migrations + +``` +migrations/ +├── 001_create_users.sql +├── 002_create_orders.sql +├── 003_add_users_phone.sql +``` + +```sql +CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ DEFAULT NOW() +); +``` + +Use an established runner in production — golang-migrate, Flyway, sqitch, Alembic, Prisma Migrate, Liquibase. They handle what ad-hoc scripts miss: transactional application, concurrent-run locking, dirty-state detection. Rules regardless of tool: + +- Applied migrations are immutable — fix mistakes with a new migration, never by editing an applied file (checksums diverge across environments). +- One logical change per migration: a half-applied 400-line migration is unrecoverable by tooling. On MySQL, one DDL statement per migration, because there is no rollback. +- Migrations run under a lock so two deploying instances do not apply the same file twice. Confirm your runner takes one; some do not. +- Data backfills belong in their own migration (or a job), separated from DDL, and written to be resumable — a backfill that times out mid-run must restart from where it stopped, not from the beginning. +- Test every migration against production-shaped data before it ships. + +## Zero-Downtime Changes + +The failure mode isn't slow DDL — most DDL is instant. It's the lock queue: `ALTER TABLE` needs an exclusive lock, waits behind one long-running query, and every new query then queues behind the ALTER. A 100 ms change becomes a full outage. Cap the wait first: + +```sql +SET lock_timeout = '2s'; -- ALTER fails fast instead of stalling all traffic +ALTER TABLE users ADD COLUMN phone TEXT; +-- on failure: find/kill the blocker (→ Monitoring), retry +``` + +The `2s` is the `lock_timeout` default from SKILL.md Configuration; a house standard replaces it in every DDL block emitted. + +Per-change playbook (PostgreSQL specifics noted): + +```sql +-- Add nullable column: metadata-only, instant +ALTER TABLE users ADD COLUMN phone TEXT; + +-- Add column with a constant default: instant in PostgreSQL >=11 (stored as metadata); +-- a volatile default (now(), gen_random_uuid()) still rewrites the table +ALTER TABLE users ADD COLUMN status TEXT NOT NULL DEFAULT 'active'; + +-- NOT NULL on an existing big column, without a long table lock: +ALTER TABLE users ADD CONSTRAINT users_email_nn + CHECK (email IS NOT NULL) NOT VALID; -- instant, applies to new rows +-- backfill existing NULLs in bounded batches (UPDATE ... WHERE id BETWEEN :lo AND :hi +-- in a loop, keyset-advancing, committing each batch), then: +ALTER TABLE users VALIDATE CONSTRAINT users_email_nn; -- scans without blocking writes + +-- Index on a live table: CONCURRENTLY doesn't block writes. +-- Caveats: cannot run inside a transaction; a failed build leaves an INVALID +-- index that still taxes writes — check and drop: +CREATE INDEX CONCURRENTLY idx_users_email ON users(email); +SELECT indexrelid::regclass FROM pg_index WHERE NOT indisvalid; +``` + +Rename/retype/drop = expand-migrate-contract, because old code runs against the new schema during every deploy. Rename `email` → `email_address` in three deploys: + +1. **Expand**: add `email_address`; dual-write both columns from app code; backfill old rows in batches. +2. **Migrate**: switch all reads to `email_address`; verify no traffic touches `email` (log or `pg_stat` checks). +3. **Contract**: drop the old column in a later deploy. + +Same shape for type changes (new column of new type) and table renames. A one-deploy `ALTER TABLE ... RENAME` is only safe when you control the maintenance window. + +Ordering rule for adding a column the application will write: deploy the schema change first, then the code. For dropping: deploy the code that stops using it first, then the schema change. Getting the order backwards is the most common deploy-time outage. + +## Large-Table DDL + +- PostgreSQL: the online variants above cover most cases; `pg_repack` rebuilds a bloated table without an exclusive lock. +- MySQL: `ALGORITHM=INPLACE, LOCK=NONE` works for many changes but falls back to a copy for others without saying so — state it explicitly so the statement errors instead of locking the table for an hour. For anything it cannot do online, use `gh-ost` or `pt-online-schema-change`, which build a shadow table and swap it. +- SQL Server: `WITH (ONLINE = ON)` for index rebuilds on supported editions. +- Any tool that copies the table needs free disk equal to the table plus its indexes; check before starting, not at 90%. +- Announce and schedule: even online tools double the write load for the duration, and the swap step takes a brief exclusive lock. + +## Backup & Restore + +A backup is untested until you've restored it. Every other property of the backup is secondary. + +### PostgreSQL + +```bash +pg_dump -Fc mydb > backup.dump # custom format: compressed, selective restore +pg_dump -Fc -t users -t orders mydb > partial.dump +pg_dump -Fc --schema-only mydb > schema.dump + +pg_restore -d mydb --clean --if-exists backup.dump +pg_restore -d mydb -j 4 backup.dump # parallel restore (custom/dir formats only) + +pg_dump mydb > backup.sql # plain SQL: portable, no parallel restore +``` + +`pg_dump` is a point-in-time snapshot: everything since the dump is lost on restore, and it does not include roles or other databases (`pg_dumpall --globals-only` does). When the tolerable loss window is minutes, not hours — or dump duration stops fitting the night — move to WAL archiving with pgBackRest or WAL-G (continuous backup, point-in-time recovery). + +### MySQL + +```bash +mysqldump --single-transaction mydb > backup.sql # consistent snapshot without locking (InnoDB) +mysqldump --single-transaction --routines --triggers --events mydb > full.sql +mysql mydb < backup.sql +``` + +`--single-transaction` is consistent for InnoDB only; a MyISAM table in the same database is dumped inconsistently and nothing warns you. Physical backups (Percona XtraBackup) restore far faster than a logical dump at any real size. + +### SQLite + +```bash +sqlite3 mydb.sqlite ".backup backup.sqlite" # safe during writes; plain cp is not +sqlite3 mydb.sqlite .dump > backup.sql +sqlite3 newdb.sqlite < backup.sql +``` + +### SQL Server + +```bash +sqlcmd -S localhost -U sa -Q "BACKUP DATABASE mydb TO DISK='backup.bak'" +sqlcmd -S localhost -U sa -Q "RESTORE DATABASE mydb FROM DISK='backup.bak'" +``` + +## RPO and RTO: Decide the Numbers First + +- **RPO** (recovery point objective) = how much data you can lose. A nightly dump means an RPO of up to 24 hours, whatever anyone assumed. +- **RTO** (recovery time objective) = how long recovery may take. Measure it on a real restore of a real-sized database; logical restores of large databases run for hours, and index rebuilds dominate. +- Write both numbers down and check the backup strategy against them. Nightly dumps satisfy an RPO of 24 hours and nothing tighter; continuous WAL archiving gets to minutes; synchronous replication gets to near-zero at a latency cost. +- Keep multiple generations and at least one copy in a different failure domain. A backup on the same host protects against nothing that actually happens. +- Backups contain everything the database contains: encrypt them and restrict access. + +## Restore Drills + +Schedule a restore into a scratch environment on the user's stated Cadence (default: monthly) and verify, not just that it completed, but that it is right: + +1. Restore to an isolated environment (masked or network-isolated — a drill is a common way production data leaks). +2. Row counts on the largest tables against the source. +3. A control total: `SUM` of a monetary column, `MIN`/`MAX` of the main timestamp column. +4. Run the application's smoke tests against the restored copy. +5. Record how long the whole thing took — that number is your real RTO. + +Automate the drill. A documented but unpracticed restore procedure fails on details (missing roles, extensions, sequences) at the worst moment. + +## Maintenance + +### PostgreSQL + +```sql +ANALYZE users; -- refresh planner stats; first move after bulk load / bad estimates +VACUUM users; -- reclaim dead tuples for reuse (does not shrink the file) +``` + +- `VACUUM FULL` shrinks the file but takes an exclusive lock for the whole rewrite — on a production table use pg_repack (online) instead. +- Autovacuum runs on its own schedule, so a large hot table can carry dead tuples long past the point where scans slow down. The observable symptom is on-disk size growing while the row count does not. +- Long-open transactions (SKILL.md rule 5) pin dead tuples: vacuum can't clean anything newer than the oldest open snapshot. Bloat despite aggressive autovacuum → look for idle-in-transaction sessions and inactive replication slots first. +- Every PostgreSQL knob behind this — autovacuum thresholds, freeze/wraparound, `work_mem`, `shared_buffers` — belongs to `pg`, not here. + +```sql +-- Table and index sizes: what's actually eating disk +SELECT relname, pg_size_pretty(pg_total_relation_size(relid)) +FROM pg_stat_user_tables ORDER BY pg_total_relation_size(relid) DESC; + +-- Unused indexes: pure write tax; verify uptime covers a full workload cycle before dropping +SELECT indexrelname, idx_scan FROM pg_stat_user_indexes WHERE idx_scan = 0; +``` + +### SQLite + +```sql +VACUUM; -- rewrites the file; needs free disk ≈ db size +PRAGMA integrity_check; +PRAGMA optimize; -- run before closing long-lived connections +PRAGMA journal_mode=WAL; -- readers no longer block the writer +``` + +### MySQL + +```sql +ANALYZE TABLE users; -- statistics +OPTIMIZE TABLE users; -- rebuilds table + indexes, reclaims space (locks: online-ish in InnoDB) +SELECT table_name, ROUND(data_length/1024/1024, 2) AS size_mb +FROM information_schema.tables WHERE table_schema = 'mydb' ORDER BY data_length DESC; +``` + +## Monitoring + +Canonical thresholds (referenced from SKILL.md): investigate any transaction or query older than **1 minute**; OLTP cache hit ratio should sit **above 99%** — sustained drops mean the working set outgrew memory. + +### PostgreSQL + +```sql +-- What is running right now +SELECT pid, NOW() - query_start AS duration, state, query +FROM pg_stat_activity +WHERE state != 'idle' AND query_start < NOW() - INTERVAL '1 minute' +ORDER BY duration DESC; + +-- Idle-in-transaction: worse than active — holds locks and blocks vacuum while doing nothing +SELECT pid, NOW() - xact_start AS open_for, query +FROM pg_stat_activity WHERE state = 'idle in transaction'; + +-- Who blocks whom (then decide, then kill) +SELECT blocked.pid AS blocked_pid, blocking.pid AS blocking_pid, + blocked_a.query AS blocked_query, blocking_a.query AS blocking_query +FROM pg_locks blocked +JOIN pg_stat_activity blocked_a ON blocked_a.pid = blocked.pid +JOIN pg_locks blocking ON blocking.locktype = blocked.locktype + AND blocking.database IS NOT DISTINCT FROM blocked.database + AND blocking.relation IS NOT DISTINCT FROM blocked.relation + AND blocking.pid != blocked.pid +JOIN pg_stat_activity blocking_a ON blocking_a.pid = blocking.pid +WHERE NOT blocked.granted; + +SELECT pg_cancel_backend(pid); -- cancel query, keep connection (try first) +SELECT pg_terminate_backend(pid); -- kill connection + +-- Cache hit ratio +SELECT sum(blks_hit)*100/sum(blks_hit+blks_read) AS cache_hit_ratio +FROM pg_stat_database; + +-- Which queries cost the most overall (SKILL.md rule 9); needs +-- shared_preload_libraries = 'pg_stat_statements' +SELECT round(total_exec_time) AS total_ms, calls, round(mean_exec_time, 1) AS mean_ms, query +FROM pg_stat_statements ORDER BY total_exec_time DESC LIMIT 10; +``` + +### MySQL + +```sql +SHOW PROCESSLIST; -- KILL to terminate +SET GLOBAL slow_query_log = 'ON'; +SET GLOBAL long_query_time = 1; -- seconds; then rank by total time with pt-query-digest +SHOW ENGINE INNODB STATUS; -- deadlock section is the useful part +SELECT * FROM sys.schema_unused_indexes; -- performance_schema view +``` + +## Alert Thresholds + +Alert on symptoms a human can act on, not on raw counters. These levels are defaults; a shop with its own alerting standard overrides them under the Thresholds preference area. + +| Signal | Page when | Why this level | +|---|---|---| +| Connections in use / max | > 80% sustained | Above this the next traffic spike fails to connect | +| Longest transaction age | > 1 min investigate, > 15 min page | Blocks vacuum and DDL; the canonical threshold above | +| Cache hit ratio (OLTP) | Sustained < 99% | Working set no longer fits memory | +| Replication lag | > the RPO you committed to | Ties the alert to a promise, not a guess | +| Disk free on the data volume | < 20% warn, < 10% page | A full data volume can wedge the server entirely | +| Deadlocks per minute | Any sustained non-zero rate | Occasional is normal; a rate means a lock-order bug | +| Failed connections / auth errors | Any spike | Credential rotation gone wrong, or an attack | +| Inactive replication slots (PostgreSQL) | Any, immediately | Retains WAL forever and fills the disk with no warning | +| Backup job age | > 1 scheduled interval + margin | A failed backup job is otherwise discovered during the incident | + +Alerting on CPU alone produces noise: databases are supposed to use CPU. Alert on the queue behind the CPU (waiting connections, transaction age) instead. + +## Incident Runbook + +When the database is the suspect, in this order: + +1. **Scope it.** All queries slow, or one? All clients, or one service? Started when, and what deployed then? +2. **Look at activity**, not at averages: the running-query list above, sorted by duration. One blocker at the top explains most incidents. +3. **Check the four resources**: connections in use, disk free, cache hit ratio, replication lag. +4. **Decide before killing.** Cancel first (`pg_cancel_backend`); terminate only if cancel does not work. Killing a long transaction rolls it back, and the rollback can take as long as the work did. +5. **Stop the bleeding before finding the cause**: cancel the runaway report, disable the feature flag, throttle the batch job. Root cause after service is restored. +6. **Do not restart the database** as a first move. It rolls back every open transaction, empties the cache, and produces a slow recovery period that looks like a second incident. +7. **Capture evidence while it is happening** — the activity list, the plan, the lock graph. It is unavailable afterwards. +8. **Write down the trigger and the fix.** Recurring incidents with no recorded cause are the same incident. + +## Connection Pooling + +Size pools from CPU, not from "more is faster": throughput peaks near `cores × 2` active connections (HikariCP guidance) and degrades beyond — hundreds of connections mostly buy context-switching and lock contention. PostgreSQL spawns a process per connection, so many app instances × generous pools exhausts `max_connections` fast; put PgBouncer in front instead of raising it. + +```ini +# pgbouncer.ini +[databases] +mydb = host=localhost dbname=mydb + +[pgbouncer] +listen_addr = 0.0.0.0 +listen_port = 6432 +auth_type = scram-sha-256 +auth_file = /etc/pgbouncer/userlist.txt +pool_mode = transaction # server connection released at COMMIT — highest reuse +max_client_conn = 1000 +default_pool_size = 20 +``` + +Transaction mode breaks anything that assumes a stable session: session-level `SET`, `LISTEN/NOTIFY`, session advisory locks, temp tables — and protocol-level prepared statements before PgBouncer 1.21 (which added support). Audit the app for these before switching from `session` mode; `SET LOCAL` inside a transaction is safe. + +Application pool knobs that matter: pool size (per-instance ceiling — multiply by instance count when sizing the server), `max_lifetime` (recycle before any infra idle-timeout kills connections mid-query), acquisition timeout (fail fast instead of piling up waiters during incidents). + +## Replication + +### PostgreSQL + +```bash +# Primary: postgresql.conf +wal_level = replica +max_wal_senders = 5 +wal_keep_size = 1GB + +# Primary: pg_hba.conf +host replication replicator replica_ip/32 scram-sha-256 + +# Replica (bootstraps a copy and configures streaming) +pg_basebackup -h primary_ip -U replicator -D /var/lib/postgresql/data -P -R +``` + +```sql +-- On primary: per-replica state +SELECT client_addr, state, replay_lsn FROM pg_stat_replication; + +-- On replica: lag as time. Gotcha: with zero write traffic this grows +-- unbounded — alert on lag only while the primary is receiving writes +SELECT NOW() - pg_last_xact_replay_timestamp() AS replication_lag; + +-- Replication slots retain WAL until consumed; an inactive slot fills the disk +SELECT slot_name, active, pg_size_pretty( + pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn)) AS retained +FROM pg_replication_slots; +``` + +### MySQL + +```sql +SHOW REPLICA STATUS\G -- Seconds_Behind_Source, Last_Error, and the two thread states +``` + +MySQL's `Seconds_Behind_Source` measures the applier's position, not real staleness: it reads 0 while the I/O thread is stalled and the replica is minutes behind. Use a heartbeat table written on the primary and compared on the replica for a trustworthy number. Row-based binlog format is the correct default; statement-based replicates non-deterministic functions incorrectly. + +Two facts to design around on any engine: + +- Replication is async by default: an acknowledged commit can be lost on failover, and read-after-write against a replica can miss the write. Route read-your-own-writes traffic to the primary (or use synchronous replication and pay the latency). +- Long replica queries conflict with replay and get canceled. PostgreSQL's `hot_standby_feedback = on` stops the cancellations but lets replica queries hold back vacuum on the primary — pick per workload: analytics replica → on, and accept the primary bloat risk. + +Replication is availability, not backup: a `DROP TABLE` replicates in milliseconds. You still need backups (→ Backup & Restore). Read routing and the point where replicas stop helping are topology questions — route from SKILL.md Quick Reference. + +## High Availability and Failover + +- Automatic failover needs three things: a health check that cannot be fooled by a slow query, a fencing mechanism that stops the old primary from accepting writes, and a way for applications to find the new primary. +- **Split brain** is the failure that loses data: two nodes both believing they are primary, both accepting writes. Any HA setup without fencing (STONITH, a lease, or a quorum) will eventually produce it. +- Application-side discovery: a virtual IP, a DNS record with a short TTL, a proxy (HAProxy, pgpool), or the driver's own multi-host connection string with `target_session_attrs=read-write`. Pick one and test that connections actually move. +- Managed services handle this for you; the useful question to ask them is the measured failover time and whether it is synchronous. +- Practice a failover on purpose, during business hours, before the first unplanned one. An untested failover is a rumor. + +## Major Version Upgrades + +1. Read the release notes for **breaking** changes, not features: removed functions, default changes (`sql_mode`, `ONLY_FULL_GROUP_BY`), collation changes, and planner behavior changes. +2. Restore a copy of production at the current version, upgrade the copy, and run the application's test suite plus the top queries from the ranked list against it. Compare plans, not only results. +3. Collation changes are the one nothing warns about: PostgreSQL text indexes built under a different glibc/ICU collation version can produce wrong results after an OS upgrade. Reindex text indexes when the collation version changes. +4. Plan the rollback before starting. In-place upgrades are usually one-way; a logical-replication upgrade (replicate old → new, then switch) allows cutting back. +5. Upgrade extensions and the client driver too, and re-run `ANALYZE` on the upgraded database before serving traffic — statistics are not always carried over. + +## Capacity Planning + +- Track four series over time: database size, largest table sizes, peak connections, and peak query rate. The trend tells you when, the absolute value tells you what. +- Project disk from growth rate plus retention policy, and add headroom for the maintenance operations that need it: a table rebuild needs free space equal to the table plus its indexes. +- The cliff to watch is the working set exceeding RAM; the cache hit ratio drops before latency does, which makes it a leading indicator. +- Growth is rarely linear: model against the business driver (tenants, orders per day), not against last month's bytes. +- Retention is a capacity decision. Deciding to keep events for 90 days instead of forever is cheaper than every other option on this page. diff --git a/workspace-sql/skills/sql/orm.md b/workspace-sql/skills/sql/orm.md new file mode 100644 index 0000000..aad323e --- /dev/null +++ b/workspace-sql/skills/sql/orm.md @@ -0,0 +1,107 @@ +# ORM-Generated SQL + +The ORM is not the problem; the invisibility is. Every fix below starts with reading the SQL the ORM actually emitted, because the generated statement rarely resembles the method chain that produced it. + +Contents: See the SQL · N+1 · Over-Fetching · Pagination · Bulk Operations · Transaction Boundaries · Lazy Loading Outside the Session · Type and Parameter Mismatches · Pool Configuration · Migration Autogeneration · When to Drop to SQL · Traps + +## First: See the SQL + +Nothing here is diagnosable without the emitted statements and their count per request. + +- Turn on the ORM's query log in development and print the statement plus its duration and the call site. +- Turn on the database's own logging for a bounded window when the ORM's log is not enough: PostgreSQL `log_min_duration_statement = 0` on a session or role, MySQL's general log, SQL Server Extended Events. +- Count queries per request, not just their duration. A page issuing 340 fast queries looks healthy in every per-query metric and is the most common performance failure in ORM applications. +- Add an assertion on the query count for the endpoints that matter — it is the only detector that survives refactoring. + +## N+1 Queries + +The shape: one query for a list, then one query per element for a relation. + +- Fix by eager loading the relation in the same round trip. Two mechanisms exist in most ORMs and they are not interchangeable: + - **Join-based** (a single query with a `JOIN`) — one round trip, but a 1:N join multiplies the parent rows, so hydrating 100 parents with 50 children each transfers 5,000 rows and the framework deduplicates in memory. Fine for 1:1 and small collections. + - **Batched select** (a second query with `WHERE parent_id IN (...)`) — two round trips, no row multiplication. Better for large collections, and the right default for nested relations. +- Nested eager loading of several 1:N relations in one join is a cartesian product: two collections of 50 each produce 2,500 rows per parent. Split into separate batched selects. +- The `IN (...)` list has a limit: PostgreSQL's protocol caps bind parameters at 65,535 per statement, and huge lists plan badly regardless. Most ORMs chunk automatically — verify yours does, and at what size. +- A query count that scales with result size is the signature. If 10 rows issue 11 queries and 100 rows issue 101, no amount of index tuning will help. + +## Over-Fetching + +- `SELECT *` is the ORM default because it must hydrate a full entity. It blocks index-only scans and drags large columns (`TEXT`, JSON blobs) across the wire on every read (SKILL.md Traps). +- Use the ORM's projection facility (select specific columns, or a DTO/tuple query) on hot read paths and on any table with a large column. +- Defer or exclude big columns at the mapping level so they load only when touched. +- Loading an entity to update one field reads the whole row, hydrates an object, and writes every column back. For a targeted change, issue the `UPDATE` directly — it is also atomic, where read-modify-write is not. +- Counting by loading a collection and taking its length transfers every row to count them. Use the ORM's `count` method, which emits `COUNT(*)`. + +## Pagination + +- ORM `.offset(n).limit(m)` maps to SQL `OFFSET`, which reads and discards every skipped row. Deep pages degrade linearly. +- Keyset pagination usually requires dropping to a raw or expression predicate, because most ORMs cannot express a row-value comparison. The trade is worth it for infinite scroll and any API that iterates a full table. +- Ordering must include a unique tiebreaker or rows are skipped and duplicated across pages — a bug that surfaces as "the export is missing records" long after release. +- Counting the total for a page number costs a second full aggregate. Offer "next page" instead of "page 7 of 93" where the product allows it. + +## Bulk Operations + +- Saving N entities in a loop issues N statements and N round trips. Use the ORM's bulk insert, which emits a multi-row `INSERT` — typically an order of magnitude faster for the same rows. +- ORM-level `update_all`/`delete_all`-style methods bypass callbacks, validations, and the identity map. That is the point (they are one statement) and the risk (audit hooks and cache invalidation do not run). Decide explicitly, per call. +- Bulk statements over very large sets need chunking for the same reason hand-written ones do: one transaction holding millions of rows bloats WAL/undo. +- After a large bulk load through the ORM, `ANALYZE` still matters — the planner does not know about the ORM. + +## Transaction Boundaries + +- "Transaction per request" middleware makes template rendering, serialization, and any outbound HTTP call part of the transaction — locks held for the entire request (SKILL.md rule 5). +- Nested `transaction do ... end` blocks usually map to savepoints, not real nested transactions. An inner rollback may leave the outer transaction alive and the object graph inconsistent with the database. +- After-commit hooks are the correct place for side effects. A hook that fires inside the transaction sends the email for a transaction that then rolls back. +- Connection-per-transaction matters under a transaction-mode pooler: session state set through the ORM (`SET`, advisory locks, temp tables) does not survive between statements. +- The ORM's optimistic-locking column is checked by row count; ignoring the "0 rows affected" result discards a user's edit with no error. + +## Lazy Loading Outside the Session + +- Accessing a relation after the session, unit of work, or request has closed raises an error in strict ORMs and issues a surprise query in permissive ones — including inside a template or a serializer, where nobody is looking. +- Fix by loading everything the response needs before the boundary, and by configuring the ORM to raise on lazy loads in test and development so the failure appears at authoring time. +- Serializers are the usual culprit: a serializer that walks associations turns one endpoint into an unbounded query count that varies by payload. + +## Type and Parameter Mismatches + +- A driver that binds a string where the column is an integer forces an implicit cast on the column and disables its index (SKILL.md Traps). It looks like a missing index and no index will fix it. +- Enum mapping: an ORM enum stored as an integer sorts and filters by the declaration order, not by label. Renumbering the enum reinterprets every stored row, with nothing to warn you. +- `NULL` versus an empty string differ in the database but are often conflated by web frameworks binding empty form fields. +- Timestamps: the ORM may send local time while the column expects UTC. Set the connection timezone explicitly and store UTC. +- Decimal columns bound as floats lose precision before the database ever sees the value; map money columns to the language's decimal type. + +## Pool Configuration + +- Pool size is per process. Total connections = instances × workers per instance × pool size — that product is what must fit the server's limit, and it is routinely 10× what anyone intended. +- Sizing follows the same rule as any pool: throughput peaks near `cores × 2` active connections at the database, not at the application. +- Set `max_lifetime` below any infrastructure idle timeout (load balancer, NAT gateway, managed-database idle cutoff), or connections die mid-query. +- Set an acquisition timeout so pool exhaustion fails fast and visibly instead of piling up waiters that look like database slowness. +- Instrument pool wait time. Rising wait time with flat query time means the pool is the bottleneck, and adding database capacity will do nothing. + +## Migration Autogeneration + +- Autogenerated migrations are a draft. Read every one before committing: they miss data backfills, generate destructive operations for renames (drop plus add loses the data), and reorder operations in ways that break dependencies. +- They also generate DDL with no `lock_timeout` and no online variant, which is what turns a routine deploy into an outage on a large table. +- A model rename produces a drop-and-create. The correct sequence is expand-migrate-contract, written by hand (SKILL.md rule 8). +- Index definitions in ORM models frequently drift from the database. Diff the checked-in schema dump against the migrated schema in CI. + +## When to Drop to SQL + +Use a raw or hand-written query when the ORM cannot express it, and keep the boundary explicit: + +- Window functions, recursive CTEs, `LATERAL` joins, `DISTINCT ON`, set operations with ordering — usually inexpressible or unreadable in the query builder. +- Reporting aggregations, upserts with conditional updates, `SKIP LOCKED` queue pulls. +- Anything where the emitted plan matters and you need control over the exact statement. + +Rules for raw SQL through an ORM: always parameterized, never string-interpolated; returned as a plain row/DTO rather than a partially-hydrated entity; and kept in one place (a repository or query object) rather than scattered through controllers. + +## Traps + +| Trap | Why it fails | Do instead | +|---|---|---| +| Judging performance by per-query timing | Every query is fast; the count is the problem | Count queries per request and assert on it | +| Eager loading everything, everywhere | Turns N+1 into one enormous cartesian join | Load the relations the response uses, batched for collections | +| Loading an entity to change one field | Reads the whole row, races other writers | Targeted `UPDATE` statement | +| Trusting an autogenerated migration | Renames become drop-and-create; no lock protection | Read and rewrite before committing | +| Raising pool size to fix timeouts | Multiplies connections against a server limit | Measure pool wait time; size from database cores | +| `update_all` on a table with audit callbacks | Bypasses callbacks and validations, with no error | Decide per call; document which hooks are skipped | +| Serializing associations lazily | Query count varies with payload shape and is invisible in tests | Preload before the serializer; raise on lazy loads in tests | +| `.raw()` with string interpolation | Injection, straight through the ORM's protections | Parameter binding, always | diff --git a/workspace-sql/skills/sql/patterns.md b/workspace-sql/skills/sql/patterns.md new file mode 100644 index 0000000..ff13d4d --- /dev/null +++ b/workspace-sql/skills/sql/patterns.md @@ -0,0 +1,418 @@ +# Query Patterns + +Contents: Pagination · Deduplication · Top-N Per Group · Gaps and Islands · Conditional Aggregation · Missing Values · Window Functions · Pivot/Unpivot · Hierarchies · Graph Traversal · Temporal · Set Operations · Anti-Joins and Semi-Joins · Optional Filters · String Aggregation · Diffing Two Tables · Sampling · Locking and Queues · Bulk Operations + +## Pagination + +OFFSET reads and discards every skipped row: cost grows linearly with page number. Fine for the first few pages, wrong for "jump to page 500" or infinite scroll — switch to keyset. + +```sql +-- Offset (acceptable only for shallow pages) +SELECT * FROM posts ORDER BY created_at DESC LIMIT 20 OFFSET 100; + +-- Keyset: constant cost at any depth. Requires a deterministic order — +-- always tie-break on a unique column (id), or rows with equal +-- created_at get skipped or duplicated across pages. +SELECT * FROM posts ORDER BY created_at DESC, id DESC LIMIT 20; + +SELECT * FROM posts +WHERE (created_at, id) < ('2026-01-15 10:00:00', 12345) -- last row of previous page +ORDER BY created_at DESC, id DESC LIMIT 20; +``` + +Row-value comparison `(a, b) < (x, y)` is index-friendly in PostgreSQL and SQLite. MySQL accepts the syntax but often won't use the index for it — expand to `a < x OR (a = x AND b < y)`. + +Keyset can't show "page 7 of 93". If the UI demands numbered pages, keep OFFSET but cap depth. For a stable cursor across concurrent inserts, encode the sort values (not the page number) in an opaque cursor token so the client cannot craft one. + +## Deduplication + +```sql +-- Keep one row per key (PostgreSQL): DISTINCT ON columns must be the +-- leading ORDER BY columns; the rest of ORDER BY picks WHICH row survives +SELECT DISTINCT ON (user_id) * +FROM orders ORDER BY user_id, created_at DESC; -- latest order per user + +-- Portable equivalent +SELECT * FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY created_at DESC) AS rn + FROM orders +) t WHERE rn = 1; + +-- Find duplicates first — always run this before any DELETE +SELECT email, COUNT(*) FROM users GROUP BY email HAVING COUNT(*) > 1; + +-- Delete duplicates, keep highest id (PostgreSQL USING; adapt elsewhere) +DELETE FROM users a USING users b +WHERE a.id < b.id AND a.email = b.email; + +-- Portable delete via window function +DELETE FROM users WHERE id IN ( + SELECT id FROM ( + SELECT id, ROW_NUMBER() OVER (PARTITION BY email ORDER BY id DESC) AS rn FROM users + ) t WHERE rn > 1 +); +``` + +After deduplicating, add the unique constraint in the same migration — otherwise duplicates return. On a large table, delete in chunks (→ Bulk Operations) rather than in one statement. + +## Top-N Per Group + +```sql +-- N rows per group, portable +SELECT * FROM ( + SELECT o.*, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY total DESC) AS rn + FROM orders o +) t WHERE rn <= 3; + +-- LATERAL: stops after N rows per group, so it beats the window form +-- when there are many groups and an index on (user_id, total DESC) +SELECT u.id, o.* +FROM users u +CROSS JOIN LATERAL ( + SELECT * FROM orders WHERE user_id = u.id ORDER BY total DESC LIMIT 3 +) o; +``` + +`ROW_NUMBER` sorts every row in every partition; `LATERAL` (or `CROSS APPLY` in SQL Server) with a matching index reads only 3 rows per group. Use `LEFT JOIN LATERAL ... ON true` when groups with no rows must still appear. + +`RANK` vs `DENSE_RANK` vs `ROW_NUMBER`: `ROW_NUMBER` always yields exactly N rows and breaks ties arbitrarily; `RANK` returns all tied rows (so "top 3" may return 5) and skips the following numbers; `DENSE_RANK` returns all ties without skipping. Choose by what a tie should mean, and add a tiebreaker column when it should not happen at all. + +## Gaps and Islands + +Consecutive runs — streaks of active days, contiguous id ranges, uninterrupted sessions. The trick is that `value − ROW_NUMBER()` is constant within a run. + +```sql +-- Longest streak of consecutive active days per user +WITH marked AS ( + SELECT user_id, activity_date, + activity_date - (ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY activity_date))::int + AS grp + FROM daily_activity +) +SELECT user_id, MIN(activity_date) AS streak_start, MAX(activity_date) AS streak_end, + COUNT(*) AS streak_days +FROM marked GROUP BY user_id, grp +ORDER BY streak_days DESC; +``` + +For runs defined by a changing status rather than a date sequence, mark the boundary with `LAG` and take a running `SUM` of the boundary flag as the group id (the same shape as sessionization). + +## Conditional Aggregation + +One pass over the table beats N filtered queries. + +```sql +-- FILTER: PostgreSQL, SQLite >=3.30 +SELECT + COUNT(*) AS total, + COUNT(*) FILTER (WHERE status = 'paid') AS paid, + SUM(total) FILTER (WHERE status = 'paid') AS revenue +FROM orders; + +-- CASE: portable everywhere (MySQL, SQL Server) +SELECT + COUNT(*) AS total, + SUM(CASE WHEN status = 'paid' THEN 1 ELSE 0 END) AS paid, + SUM(CASE WHEN status = 'paid' THEN total END) AS revenue +FROM orders; +``` + +`COUNT(CASE WHEN cond THEN 1 END)` counts matches because `COUNT` skips NULLs; `COUNT(CASE WHEN cond THEN 1 ELSE 0 END)` counts every row and is the standard off-by-everything bug (SKILL.md rule 6). + +## Missing Values (Gap Analysis) + +```sql +-- PostgreSQL (generate_series is PostgreSQL-only) +WITH all_ids AS ( + SELECT generate_series(1, (SELECT MAX(id) FROM products)) AS id +) +SELECT a.id FROM all_ids a +LEFT JOIN products p ON a.id = p.id +WHERE p.id IS NULL; + +-- Portable: recursive CTE as series generator (MySQL >=8.0, SQLite, SQL Server) +WITH RECURSIVE seq(n) AS ( + SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < 1000 +) +SELECT n FROM seq LEFT JOIN products p ON p.id = seq.n WHERE p.id IS NULL; +``` + +Gaps in generated id sequences are normal — rollbacks consume values. Investigate only when a gap means lost data, never to "fix" the numbering. For missing dates in a report, the same shape is the date spine. + +## Window Functions + +```sql +-- Running total +SELECT date, amount, + SUM(amount) OVER (ORDER BY date ROWS UNBOUNDED PRECEDING) AS running_total +FROM transactions; + +-- Day-over-day change +SELECT date, amount, + amount - LAG(amount) OVER (ORDER BY date) AS change +FROM daily_metrics; + +-- Percentage of total, and of the partition +SELECT category, amount, + ROUND(amount * 100.0 / SUM(amount) OVER (), 2) AS pct_of_all, + ROUND(amount * 100.0 / SUM(amount) OVER (PARTITION BY region), 2) AS pct_of_region +FROM category_totals; + +-- First and last value in a partition; the frame matters for LAST_VALUE +SELECT user_id, event, + FIRST_VALUE(event) OVER w AS first_event, + LAST_VALUE(event) OVER w AS last_event +FROM events +WINDOW w AS (PARTITION BY user_id ORDER BY occurred_at + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING); +``` + +- Frame trap: the default frame for `SUM() OVER (ORDER BY date)` is `RANGE`, which treats tied dates as peers — all rows with the same date get the same "running" total. Use `ROWS UNBOUNDED PRECEDING` (or a unique ORDER BY) for a true row-by-row total. +- `LAST_VALUE` with the default frame returns the current row, because the frame ends at the current row. It needs the explicit `UNBOUNDED FOLLOWING` above — the most common window-function bug after the RANGE default. +- `WHERE` cannot reference a window function (it runs earlier) — wrap in a subquery/CTE to filter on one. `QUALIFY` exists in some warehouses but not in these four engines. +- Naming the window with a `WINDOW` clause avoids repeating the specification and guarantees the frames match. + +## Pivoting Data + +Default to portable CASE; `crosstab` needs an extension, breaks when a category appears that isn't in the column list, and buys little. + +```sql +SELECT month, + SUM(CASE WHEN category = 'electronics' THEN total END) AS electronics, + SUM(CASE WHEN category = 'clothing' THEN total END) AS clothing, + SUM(CASE WHEN category = 'food' THEN total END) AS food +FROM sales +GROUP BY month; +``` + +Unknown category set → don't pivot in SQL; return `(month, category, total)` rows and pivot in the application. SQL cannot produce a dynamic column list without building the statement as text, which is an injection surface. + +## Unpivoting Data + +```sql +-- PostgreSQL +SELECT id, key, value +FROM metrics, +LATERAL (VALUES + ('metric_a', metric_a), + ('metric_b', metric_b), + ('metric_c', metric_c) +) AS t(key, value); + +-- Portable: UNION ALL, one branch per column +SELECT id, 'metric_a' AS key, metric_a AS value FROM metrics +UNION ALL SELECT id, 'metric_b', metric_b FROM metrics; + +-- SQL Server +SELECT id, metric_name, metric_value +FROM metrics +UNPIVOT (metric_value FOR metric_name IN (metric_a, metric_b, metric_c)) AS u; +``` + +All unpivoted values must share one type; mixing a numeric and a text column forces a cast to text and loses ordering. + +## Hierarchical Data + +Adjacency list is the default: simplest writes, and recursive CTEs make reads workable everywhere (MySQL >=8.0, SQLite >=3.8.3, PostgreSQL, SQL Server). + +```sql +CREATE TABLE categories ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT NOT NULL, + parent_id BIGINT REFERENCES categories(id) +); + +-- Ancestors of node 5 +WITH RECURSIVE ancestors AS ( + SELECT *, 1 AS depth FROM categories WHERE id = 5 + UNION ALL + SELECT c.*, a.depth + 1 FROM categories c + JOIN ancestors a ON c.id = a.parent_id + WHERE a.depth < 50 -- guard: a cycle in the data loops forever without this +) +SELECT * FROM ancestors; +``` + +Cycle protection: the depth guard above works everywhere; PostgreSQL >=14 has a native `CYCLE id SET is_cycle USING path` clause. `UNION` (not `UNION ALL`) also stops exact-duplicate rows but hides the cycle instead of surfacing it — and it forces a deduplication of the whole intermediate result on every iteration. + +```sql +-- Materialized path: fast subtree reads, but every move rewrites descendants +CREATE TABLE categories ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT NOT NULL, + path TEXT NOT NULL -- '1/3/7/15' +); +SELECT * FROM categories WHERE path LIKE '1/3/%'; -- all descendants of 3 +``` + +Choose by read/write ratio: adjacency list when the tree changes often, materialized path when subtree reads dominate and moves are rare, closure table (one row per ancestor-descendant pair) when both are hot and you can afford the write amplification. + +## Graph Traversal + +The same recursive CTE handles arbitrary graphs, but a graph needs visited-set protection rather than a depth guard, because cycles are normal rather than corruption. + +```sql +-- Shortest-hop reachability from node 1, tracking the path to avoid revisits +WITH RECURSIVE reach AS ( + SELECT to_id, 1 AS hops, ARRAY[from_id, to_id] AS path + FROM edges WHERE from_id = 1 + UNION ALL + SELECT e.to_id, r.hops + 1, r.path || e.to_id + FROM edges e JOIN reach r ON e.from_id = r.to_id + WHERE NOT e.to_id = ANY(r.path) AND r.hops < 6 +) +SELECT to_id, MIN(hops) FROM reach GROUP BY to_id; +``` + +Both guards are required: the path check stops cycles, the hop limit stops combinatorial explosion in dense graphs. A relational database handles a few hops well; queries needing many hops or weighted shortest paths belong in a graph database (`neo4j`). + +## Temporal Queries + +```sql +-- Overlapping ranges (PostgreSQL): && handles all four overlap cases; +-- hand-rolled start/end comparisons routinely miss one +SELECT * FROM bookings +WHERE tstzrange(start_at, end_at, '[)') && tstzrange('2026-01-01', '2026-02-01', '[)'); + +-- Portable overlap test between half-open ranges +-- WHERE a_start < b_end AND a_end > b_start +``` + +Exclusion constraint replaces the check-then-insert race for bookings: `EXCLUDE USING gist (room_id WITH =, tstzrange(start_at, end_at, '[)') WITH &&)`. + +## Set Operations + +```sql +SELECT id FROM a UNION SELECT id FROM b; -- distinct: sorts/hashes the whole result +SELECT id FROM a UNION ALL SELECT id FROM b; -- no deduplication, much cheaper +SELECT id FROM a INTERSECT SELECT id FROM b; -- in both +SELECT id FROM a EXCEPT SELECT id FROM b; -- in a, not in b (MINUS in Oracle) +``` + +- Default to `UNION ALL` and add `UNION` only when duplicates are possible and unwanted. +- `ORDER BY` applies to the whole result and must come last; ordering a branch requires wrapping it in a subquery. +- Branches match by position, not by column name — a reordered column list pairs the wrong columns with no error if the types happen to be compatible. +- MySQL gained `INTERSECT`/`EXCEPT` in 8.0.31; SQLite has both; before that, emulate with `EXISTS`/`NOT EXISTS`. + +## Anti-Joins and Semi-Joins + +```sql +-- Semi-join: rows in a that have a match (stops at the first match) +SELECT * FROM users u WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id); + +-- Anti-join: rows in a with no match. NULL-safe, unlike NOT IN +SELECT * FROM users u WHERE NOT EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id); + +-- Equivalent anti-join via LEFT JOIN; same plan in most engines +SELECT u.* FROM users u LEFT JOIN orders o ON o.user_id = u.id WHERE o.id IS NULL; +``` + +Prefer `EXISTS` over `IN (subquery)` for large subqueries and always over `NOT IN` when the subquery column is nullable (SKILL.md rule 6). Do not `SELECT *` inside an `EXISTS` — `SELECT 1` states the intent and avoids expanding the row. + +## Optional Filters (Dynamic Search) + +The convenient form disables indexes: + +```sql +-- Convenient, and slow: the planner cannot know which branch applies +WHERE (:status IS NULL OR status = :status) + AND (:min_total IS NULL OR total >= :min_total); +``` + +With literals unknown at plan time, the engine plans for the general case and typically scans. Three ways out, in order of preference: + +1. **Build the statement from the filters that are actually present**, appending only the supplied predicates and binding their values. This is safe as long as the predicate fragments are hardcoded and only values are bound. +2. Force a fresh plan per execution (`OPTION (RECOMPILE)` in SQL Server, custom plan mode in PostgreSQL). +3. Write one query per common filter combination when there are only two or three. + +## String Aggregation + +```sql +SELECT user_id, STRING_AGG(tag, ',' ORDER BY tag) FROM user_tags GROUP BY user_id; -- PostgreSQL, SQL Server 2017+ +SELECT user_id, GROUP_CONCAT(tag ORDER BY tag) FROM user_tags GROUP BY user_id; -- MySQL, SQLite +SELECT user_id, JSON_AGG(tag ORDER BY tag) FROM user_tags GROUP BY user_id; -- structured, PostgreSQL +``` + +MySQL's `GROUP_CONCAT` truncates without warning at `group_concat_max_len` (1024 bytes by default) — a report that looks fine in testing loses data in production. Prefer returning rows and joining in the application, or aggregate to JSON where the engine supports it. + +## Diffing Two Tables + +```sql +-- Full row-level diff: rows only in one side, tagged +SELECT 'only_in_new' AS side, * FROM (SELECT * FROM new_data EXCEPT SELECT * FROM old_data) a +UNION ALL +SELECT 'only_in_old', * FROM (SELECT * FROM old_data EXCEPT SELECT * FROM new_data) b; + +-- Changed rows by key, with the differing column +SELECT n.id, o.status AS old_status, n.status AS new_status +FROM new_data n JOIN old_data o USING (id) +WHERE n.status IS DISTINCT FROM o.status; +``` + +`IS DISTINCT FROM` is the NULL-safe comparison: `o.status <> n.status` is NULL (not true) when either side is NULL, so rows where a value became NULL are omitted with no error. MySQL spells it `<=>` for the equality form. This is the standard verification step after any data migration. + +## Sampling + +```sql +-- PostgreSQL block sampling: fast; SYSTEM picks whole pages (clustered bias — +-- rows inserted together are sampled together), BERNOULLI picks rows uniformly but scans more +SELECT * FROM large_table TABLESAMPLE SYSTEM (1); -- ~1% of pages +SELECT * FROM large_table TABLESAMPLE BERNOULLI (1); -- ~1% of rows, unbiased + +-- One pseudo-random row from a large table without a full sort: +-- probe by key around a random point, then fall back if the probe misses +SELECT * FROM large_table WHERE id >= (random() * (SELECT MAX(id) FROM large_table))::bigint +ORDER BY id LIMIT 1; + +-- ORDER BY random()/RAND() LIMIT n = full scan + sort; only for small tables +``` + +The key-probe form is biased when ids have gaps (rows after a large gap are picked more often); accept it for "show me a random example" and never for statistics. + +## Locking & Queues + +```sql +-- Read-modify-write without lost updates +SELECT * FROM inventory WHERE product_id = 5 FOR UPDATE; + +-- Job queue: SKIP LOCKED lets N workers pull disjoint jobs with no coordinator +UPDATE jobs SET status = 'running', started_at = NOW() +WHERE id = ( + SELECT id FROM jobs WHERE status = 'pending' + ORDER BY created_at LIMIT 1 + FOR UPDATE SKIP LOCKED +) +RETURNING *; +``` + +Deadlock prevention: when a transaction locks multiple rows, lock them in a consistent order (`ORDER BY id FOR UPDATE`); two transactions locking {1,2} and {2,1} deadlock, {1,2} and {1,2} queue. Isolation levels, retry loops, and the queue table's own schema route from SKILL.md Quick Reference. + +## Bulk Operations + +```sql +-- Multi-row insert. PostgreSQL wire protocol caps bind parameters at 65,535: +-- max rows per statement = 65535 / column_count (4 columns → 16k rows/batch) +INSERT INTO users (email, name) VALUES + ('a@example.com', 'Alice'), + ('b@example.com', 'Bob'); + +-- Bulk upsert (PostgreSQL/SQLite) +INSERT INTO users (email, name) VALUES ('a@example.com', 'Alice') +ON CONFLICT (email) DO UPDATE SET name = EXCLUDED.name; + +-- Update many rows to different values in one statement +UPDATE products p SET price = v.price +FROM (VALUES (1, 9.99), (2, 14.50)) AS v(id, price) +WHERE p.id = v.id; +``` + +Large DELETE/UPDATE: one statement holds locks for the whole run and bloats WAL/undo. Chunk it — delete with `LIMIT` (or a keyed range) in batches of `batch_size` rows (default 5,000; the useful band is 1k-10k), commit between batches, loop until 0 rows affected: + +```sql +DELETE FROM events WHERE id IN ( + SELECT id FROM events WHERE created_at < NOW() - INTERVAL '90 days' LIMIT 5000 +); +-- repeat until DELETE 0 +``` + +Deleting most of a very large table is faster as a rebuild: create a new table with the rows you keep, swap names, drop the old one. If the deletion is periodic and by time, partitioning turns it into an instant `DROP`. diff --git a/workspace-sql/skills/sql/performance.md b/workspace-sql/skills/sql/performance.md new file mode 100644 index 0000000..c13c186 --- /dev/null +++ b/workspace-sql/skills/sql/performance.md @@ -0,0 +1,133 @@ +# Performance — Plans, Indexes, and What To Change + +Tuning order that avoids wasted work: rank queries by total cost (SKILL.md rule 9) → read the plan → fix the schema or the query → re-measure the same plan. Steps skipped in that order produce indexes nobody uses. + +Contents: Plan Nodes · Join Algorithms · Statistics · Index Types · Composite Design · Sargability · Rewrites That Win · Aggregation · Sorting · Memory & Spills · Prepared Statements · Measuring · When Not To Tune + +## Plan Nodes: What Each One Tells You + +| Node | Means | Change it by | +|---|---|---| +| Seq Scan / table scan | Whole table read | Adding a usable index — or accepting it if selectivity is above roughly 5-10% (SKILL.md rule 3) | +| Index Scan | Index walked, heap fetched per row | Making it index-only with `INCLUDE`/covering columns | +| Index Only Scan | No heap access | Watch `Heap Fetches`; high means the visibility map is stale — vacuum | +| Bitmap Heap Scan | Many index matches, heap read in page order | Normal for medium selectivity; if `Rows Removed by Filter` is high, widen the index | +| Nested Loop | Inner side executed per outer row | Cheap when the outer side is tiny AND the inner is indexed; catastrophic when the outer estimate was wrong | +| Hash Join | Build a hash of the smaller side | Fine for large unsorted joins; watch for `Batches > 1` (spilled to disk) | +| Merge Join | Both sides sorted | Good when indexes already provide the order; expensive when it has to sort | +| Sort | Ordering rows | Provide the order via an index, or reduce rows first | +| Materialize / Memoize | Caching an inner result | Usually the planner compensating for repetition; not a bug | +| Aggregate vs HashAggregate | Grouping strategy | HashAggregate spilling means `work_mem`-class pressure (`pg` for tuning it) | + +Read the plan bottom-up and inside-out: the deepest node runs first. In PostgreSQL, `actual time` on a node is **per loop** — multiply by `loops` for total. A node showing `actual time=0.05..0.08 rows=1 loops=200000` costs 16 seconds, not 0.08 ms; missing this is the single most common plan misreading. + +## Join Algorithms: Choosing By Shape + +- Nested loop wins when the outer row count is small (roughly hundreds) and the inner side has an index on the join key. It is also the failure mode of a bad estimate: the planner expected 10 outer rows, got 100,000, and now runs 100,000 index lookups. +- Hash join wins when one side fits in memory and neither is usefully sorted. Build side = the smaller estimated side; a wrong estimate hashes the wrong table. +- Merge join wins when both inputs arrive sorted from indexes; forcing it by adding an explicit sort rarely pays. +- Join order matters more than join algorithm on many-table queries. PostgreSQL exhaustively searches up to `join_collapse_limit` (default 8) tables, then stops optimizing and uses the written order — a 12-table report is partly hand-ordered whether you meant it or not. + +## Statistics: Why The Planner Guessed Wrong + +- Row estimates drive every decision. An estimate off by more than 10× invalidates the plan, not the index. +- After a bulk load, mass delete, or restore, statistics describe the previous shape. `ANALYZE` is the first move, before any index. +- Skewed columns: default histograms track a limited number of most-common values. A `status` column where 99% is `'done'` needs more granularity — PostgreSQL `ALTER TABLE ... ALTER COLUMN ... SET STATISTICS 1000` (default 100), MySQL `ANALYZE TABLE ... UPDATE HISTOGRAM ON col`. +- Correlated columns break the independence assumption: `WHERE city = 'Paris' AND country = 'France'` is estimated as `sel(city) × sel(country)`, thousands of times too low. Declare it: PostgreSQL `CREATE STATISTICS (dependencies) ON city, country FROM addresses` (>=10). +- Expression predicates have no statistics unless a matching expression index exists — creating the index improves the estimate as well as the access path. + +## Index Types Beyond B-tree + +| Type | Use for | Cost | +|---|---|---| +| B-tree | Equality, ranges, sorting, prefix `LIKE 'x%'` | The default; nothing else beats it for ordered data | +| Hash (PostgreSQL) | Equality only | No range, no sort; B-tree is nearly always fine instead | +| GIN | Array containment, JSONB `@>`, full-text | Slow to update; batch writes or accept write amplification | +| GiST | Ranges, geometry, exclusion constraints | Supports overlap operators B-tree cannot | +| BRIN | Huge tables physically ordered by the indexed column (append-only time series) | Tiny index, but useless if physical order does not match the column | +| Full-text / FTS5 | Word and phrase search | Word-based, never substring | +| Trigram (`pg_trgm`) | Substring, fuzzy, `LIKE '%x%'`, typo tolerance | Large index; worth it exactly when B-tree cannot help | + +Filtered/partial and covering variants apply to most of these — see SKILL.md Index Strategy for the syntax. + +## Composite Index Design + +Order columns: **equality first, then the one range/sort column, then included payload columns.** Only the first range column can use the index for ordering; everything after it is filtered, not sought. + +Worked example. Query: `WHERE tenant_id = ? AND status = ? AND created_at > ? ORDER BY created_at DESC LIMIT 20`. + +- `(tenant_id, status, created_at)` — correct. Both equalities seek, `created_at` provides the range and the sort, so the `LIMIT` stops after 20 rows. +- `(created_at, tenant_id, status)` — wrong. The index is ordered by time across all tenants; the engine scans time descending and discards other tenants' rows until it collects 20. +- `(tenant_id, created_at, status)` — half right. Seeks on tenant and time, but `status` is filtered per row: with 5% matching, the engine reads roughly 400 rows to return 20. + +Rule of thumb for how many indexes: a write-heavy OLTP table carrying more than about five indexes is paying more on every insert than most read plans save. Consolidate — one composite often replaces three single-column indexes. + +## Sargability (Can The Predicate Use An Index) + +Sargable: `col = ?`, `col > ?`, `col BETWEEN ? AND ?`, `col IN (...)`, `col LIKE 'prefix%'`, `col IS NULL`. + +Not sargable, with the rewrite: + +| Written | Rewrite | +|---|---| +| `WHERE YEAR(d) = 2024` | `WHERE d >= '2024-01-01' AND d < '2025-01-01'` | +| `WHERE d::date = CURRENT_DATE` | `WHERE d >= CURRENT_DATE AND d < CURRENT_DATE + 1` | +| `WHERE amount * 1.2 > 100` | `WHERE amount > 100 / 1.2` | +| `WHERE lower(email) = ?` | Expression index on `lower(email)`, or store the normalized column | +| `WHERE col LIKE '%x%'` | Trigram index, or full-text if word-level is enough | +| `WHERE a = ? OR b = ?` | `SELECT ... WHERE a = ? UNION ALL SELECT ... WHERE b = ? AND a <> ?` | +| `WHERE col <> 'done'` | Partial index on the values you actually query, or list them with `IN` | +| `WHERE COALESCE(col, 0) > 5` | `WHERE col > 5` plus an explicit `OR col IS NULL` branch if needed | + +## Rewrites That Win + +- `EXISTS` over `IN (subquery)` for large subqueries: `EXISTS` short-circuits at the first match; `IN` may materialize the whole set. `NOT EXISTS` is also NULL-safe (SKILL.md rule 6). +- `LIMIT` with a covering index turns an ordered scan into an early stop. Without the matching order, the engine sorts everything then discards. +- Window function instead of a correlated subquery per row: one pass with `ROW_NUMBER()` replaces N executions. +- `LATERAL` join for top-N-per-group at scale beats `ROW_NUMBER()` filtering when there are many groups and few rows per group, because it can stop per group. +- `UNION ALL` instead of `UNION` when duplicates are impossible: `UNION` sorts the whole result to deduplicate. +- Push filters into subqueries and CTEs rather than filtering the outer result — with an optimization-fence CTE (PostgreSQL <12, or `MATERIALIZED`), the outer predicate cannot reach inside. +- Replace `OFFSET` deep pagination with keyset; this is a plan improvement of a different order than any index. +- Batch N single-row statements into one multi-row statement: round-trip latency usually dominates execution time for small writes. + +## Aggregation Performance + +- Pre-aggregate before joining whenever a 1:N join feeds a `SUM`/`COUNT` — it also fixes the fan-out correctness bug (SKILL.md Traps). +- `COUNT(*)` on a large table is a scan in PostgreSQL (MVCC visibility must be checked per row) but O(1) in MyISAM and cheap in SQL Server. Use the planner estimate for "about how many". +- `GROUP BY` on a low-cardinality column can be served from an index; `GROUP BY` on a high-cardinality expression cannot. +- Materialized views trade freshness for latency; incremental rollup tables trade complexity for both. +- `DISTINCT` and `GROUP BY` on the same columns usually produce the same plan — the readability choice is free. + +## Sorting + +- An index provides order only if the `ORDER BY` matches its column order AND direction pattern. Mixed directions (`a ASC, b DESC`) need an index declared that way, which PostgreSQL and SQL Server support and SQLite supports from 3.3. +- Sorting the full result then applying `LIMIT` is a top-N heap sort in most engines — much cheaper than a full sort, but still touches every row. Only an index removes that. +- NULL sort position differs by engine; adding `NULLS LAST` to a query can disable an index that does not carry that ordering. + +## Memory and Spills + +- A sort or hash that exceeds the per-node work memory spills to disk. In PostgreSQL, `EXPLAIN ANALYZE` reports `Sort Method: external merge Disk: NkB` — that string is the direct signal. +- The budget is per node, not per query: a query with four sorts uses four allocations, and every connection can do this at once. Raising the global value multiplies by connections (`pg` for the sizing formula). +- Cheaper than more memory: fewer rows entering the sort (filter earlier), or an index that supplies the order. + +## Prepared Statements and Plan Caching + +- A prepared statement may switch to a generic plan after several executions. With skewed data, the generic plan is right on average and wrong for the parameter you care about. +- Symptom: identical SQL is fast in a console with literals, slow from the application. +- Fixes in order: keep parameters out of highly skewed predicates, split the query into per-branch statements, or force a custom plan per execution (`plan_cache_mode` in PostgreSQL >=12; `OPTION (RECOMPILE)` in SQL Server). +- Server-side prepared statements also interact with transaction-mode connection poolers. + +## Measuring Honestly + +1. Warm the cache: run the query twice and compare the second run against the second run of the alternative. First runs measure disk, not the change. +2. Compare buffers/reads, not just wall time — a shared machine's wall time is noise. +3. Change one thing at a time, and re-run `EXPLAIN ANALYZE` after each. Two changes at once means you keep an index you did not need. +4. Verify on production-shaped data volumes. A plan chosen over 1,000 rows tells you nothing about 10 million. +5. After deploying an index, confirm it is actually used (`pg_stat_user_indexes.idx_scan` climbing) and that write latency did not regress. + +## When Not To Tune + +- The query runs once a month and takes 40 seconds. Cost of tuning exceeds cost of waiting. +- The table has 5,000 rows. Full scans of small tables are free; adding indexes there adds write cost and clutter. +- The real fix is upstream: pagination the UI does not need, a report that should be a nightly rollup, or an N+1 the ORM emits. +- You are at the limit of one node rather than one query — that is a topology problem. diff --git a/workspace-sql/skills/sql/scaling.md b/workspace-sql/skills/sql/scaling.md new file mode 100644 index 0000000..09f5683 --- /dev/null +++ b/workspace-sql/skills/sql/scaling.md @@ -0,0 +1,139 @@ +# Scaling — When One Node Is Not Enough + +Order the options by cost, and refuse to skip a step. Most "we need to shard" conversations end at step 2 once someone reads the plans. Sharding is the last option because it removes joins, transactions, and unique constraints across the shard key — capabilities you cannot get back. + +Contents: The Ladder · Read The Bottleneck · Vertical First · Read Replicas · Caching · Partitioning · Splitting Workloads · Sharding · Shard Keys · Life Without Cross-Shard Joins · Hot Rows and Contention · Write Amplification · Queues and Backpressure · Migrating to a Warehouse · Signals + +## The Ladder + +1. Fix the queries and indexes — the cheapest capacity ever bought. +2. Fix the access pattern: N+1s, missing pagination, reports running on the primary. +3. Add hardware. Doubling RAM to fit the working set outperforms every architectural change and costs a restart. +4. Pool connections properly. +5. Cache the reads that dominate. +6. Move reads to replicas. +7. Partition large tables. +8. Split workloads onto separate databases by bounded context. +9. Shard. + +## Read The Bottleneck Before Choosing + +| Symptom | Bottleneck | Right rung | +|---|---|---| +| CPU high, few slow queries | Query cost or plan quality | 1 | +| CPU high, thousands of tiny queries | Access pattern, N+1 | 2 | +| Cache hit ratio below the 99% OLTP threshold, heavy disk reads | Working set exceeds RAM | 3 | +| Connections at the ceiling, queries fast | Pool sizing | 4 | +| Reads dominate and are repetitive | Read volume | 5, 6 | +| One table is enormous, queries scan it by time | Table size | 7 | +| One noisy feature starves everything else | Workload mixing | 8 | +| Writes saturate a single node's disk or CPU | Write volume | 9 | + +Measure before choosing. The most expensive scaling projects are the ones that solved a bottleneck the system did not have. + +## Vertical First + +- Fitting the working set in RAM is the single largest performance cliff in databases: below it, reads are memory-speed; above it, every miss is a disk I/O. +- Faster storage (local NVMe over network-attached) changes write latency more than any configuration change. +- More cores help concurrency, not single-query latency, unless the engine parallelizes the query. +- Vertical scaling ends at the largest instance the provider offers, and it does not improve availability. It buys the runway to do the rest properly. + +## Read Replicas + +- Correct fit: read-heavy workloads with tolerance for staleness — dashboards, search pages, exports, analytics. +- Wrong fit: anything that reads its own write. Replication is asynchronous by default, so a redirect after a POST can read the state from before the write. +- Routing patterns, cheapest first: send explicitly-tagged read-only endpoints to replicas; pin a user's session to the primary for a few seconds after they write; or track the commit position and route by it if the driver supports it. +- Replicas do not help write throughput at all, and every replica adds write cost on the primary (WAL shipping) and another thing that can lag or break. +- Long analytics queries on a replica get canceled by replay conflicts unless configured otherwise, which then holds back cleanup on the primary. Dedicate a replica to analytics rather than mixing. + +## Caching + +- Cache the expensive and repeated, not everything. The candidates are visible in the ranked query list (SKILL.md rule 9): high total time driven by high call count. +- Invalidation strategy, in order of reliability: short TTL (simplest, always correct within the window) → write-through on the code path that changes the data → event-driven invalidation. Manual invalidation scattered through the codebase is the one that always misses a path. +- Cache keys must include everything that varies the result: tenant, user permissions, locale, feature flags. A key missing the tenant is a cross-tenant data leak. +- The database's own buffer cache already serves repeated reads well; an application cache pays off when it avoids the query round trip and the serialization, not merely the disk read. +- Thundering herd: when a hot key expires, every request recomputes it at once. Use a lock or stale-while-revalidate. +- Materialized rollups are a cache with a schema and visible staleness — usually better than an opaque cache for aggregate reads. + +## Partitioning + +- One table, many physical pieces, still one database. It solves table size, not node capacity. +- The reason that pays for itself is **retention**: dropping a partition is instant and reclaims disk; `DELETE` on the same rows runs for hours and leaves bloat. +- Every query must filter on the partition key or it scans every partition. Verify pruning in the plan, not in the documentation. +- Partition creation must be automated. The failure mode is inserts failing at midnight because next month's partition does not exist. +- Partition count has a cost: planning time grows with the number of partitions, so hundreds is routine and tens of thousands is not. +- Unique constraints must include the partition key; global uniqueness across partitions is not available in PostgreSQL declarative partitioning. + +## Splitting Workloads + +Before sharding, separate by bounded context: move the analytics tables, the event log, the job queue, or one noisy service's tables to their own database. + +- Each split removes a workload's interference with the rest, and the pieces scale independently. +- The price is losing cross-database joins and transactions between them — the same price sharding charges, but paid once at a natural seam rather than across every table. +- Choose seams where the data genuinely does not need to be transactionally consistent with the rest: an audit log, metrics, sessions, search indexes, the job queue. +- Do this before sharding, always. It is often enough on its own. + +## Sharding + +Horizontal partitioning across independent databases. Everything below is what you give up: + +- **No cross-shard joins.** Queries that span shards are fanned out and merged in the application or a proxy. +- **No cross-shard transactions.** Multi-shard writes need sagas, an outbox, or eventual consistency with reconciliation. +- **No global unique constraints** except on the shard key. Global ids must be generated externally (UUIDv7, snowflake ids). +- **No global `ORDER BY ... LIMIT`** without fetching from every shard and merging. +- **Rebalancing is a project.** Moving data between shards while serving traffic is the hardest part, and it is not optional as shards grow unevenly. +- **Operations multiply.** Every migration, backup, restore drill, and upgrade now runs N times and can partially fail. + +Implementation choices: application-level routing (most control, most code), a proxy layer (Vitess for MySQL, Citus for PostgreSQL), or a distributed SQL engine (CockroachDB, Yugabyte, Spanner-style) that hides sharding at the cost of higher per-transaction latency and its own operational model. + +## Choosing the Shard Key + +The single most consequential and least reversible decision in the system. + +- It must appear in the overwhelming majority of queries, or every read becomes a scatter-gather. +- It must distribute both data volume and traffic. `tenant_id` is natural for B2B and fails when one customer is 40% of the load; hash-based keys distribute evenly but destroy range queries. +- It must be immutable. Changing a row's shard key means moving the row between databases. +- Time is almost always a bad shard key for OLTP: all writes land on the newest shard while the rest idle. +- Write down the queries that will need scatter-gather **before** committing. If the list includes anything on a hot path, the key is wrong. + +## Hot Rows and Contention + +Contention on a few rows caps throughput regardless of how many nodes exist. + +- A single counter row updated by every request serializes all of them. Shard it into N rows keyed by a random bucket and `SUM` at read time; N of 10-100 is the usual range, and the read cost rises with N. +- Sequences and auto-increment on a single table are a contention point at very high insert rates; client-generated UUIDv7 removes it. +- A job queue polled by many workers contends on the same head rows unless `SKIP LOCKED` is used. +- The last page of an index on a monotonically increasing key is a write hotspot — the reason random UUIDv4 is sometimes proposed, and why UUIDv7 (time-ordered but distributed by node) is the better answer. +- Lock waits reported without deadlocks are the signature; look at what every transaction touches in common. + +## Write Amplification + +Before scaling writes, reduce them: + +- Every index is written on every insert, and updating an indexed column writes to every index on the table. Five indexes on a write-heavy table is a real cost. +- Updating unchanged values still writes a new row version. Guard with `WHERE col IS DISTINCT FROM :new`. +- Wide rows and large JSON documents rewrite entirely on any update. +- Audit triggers double the write volume by construction; batch or partition the audit table. +- Synchronous replicas add a network round trip to every commit; asynchronous ones do not. + +## Queues and Backpressure + +- Not every write needs to be synchronous. Moving non-critical writes (analytics events, notifications, denormalized updates) to a queue smooths spikes and takes them off the request path. +- A queue table inside the same database is legitimate and simple up to moderate rates; it stops being right when queue churn dominates the database's own write load. +- Without backpressure a queue just moves the failure: bound the queue, shed load, and make consumers idempotent. +- Batch consumers: 1,000 rows in one statement instead of 1,000 statements is the same order-of-magnitude win as any bulk load. + +## Moving Analytics to a Warehouse + +- The signal is workload shape, not size: full-table scans, wide aggregations, and columnar access patterns competing with OLTP traffic. +- A columnar engine (`duckdb` locally, `clickhouse`, or a cloud warehouse) reads ~25× less data on column pruning alone for a query touching 3 of 80 columns, and more once per-column compression is counted — a storage-format advantage no index gives you on a row store. +- Start with the cheapest form that works: a replica plus rollup tables, then a scheduled extract, then a full pipeline with transformations and tests (`dbt`). +- Keep one definition of each metric across both systems, or the two will disagree and nobody will know which is right. + +## Signals That You Actually Need the Next Rung + +- Rung 1-2 are exhausted only when the ranked query list has no cheap wins left and the plans are good. +- Vertical is exhausted when you are on the largest practical instance and the working set still does not fit. +- Replicas are exhausted when write throughput, not read throughput, is the ceiling. +- Partitioning is exhausted when a single partition is itself too large for one node. +- Only then is sharding the answer — and by then the shard key is obvious from the access patterns you have been measuring all along. diff --git a/workspace-sql/skills/sql/schemas.md b/workspace-sql/skills/sql/schemas.md new file mode 100644 index 0000000..39bc5ed --- /dev/null +++ b/workspace-sql/skills/sql/schemas.md @@ -0,0 +1,503 @@ +# Schema Patterns + +Ready-made shapes for problems that recur in every application, with the trap each one ships with. This file assumes the model is already decided (keys, normal forms, cardinality) and you need the table. + +Contents: Multi-tenancy · Soft Deletes · Audit Logging · Polymorphic Associations · Tags · State Machines · Permissions · Full-Text Search · Versioning · Key-Value Settings · Feature Flags · Job Queue · Idempotency and Outbox · Rate Limits · Ledgers and Balances · Attachments · Recurrence · Time-Series · Counting + +## Multi-tenancy + +Default: shared tables with `tenant_id`. Schema-per-tenant only when tenants need divergent schemas or independent restore, and the tenant count stays in the hundreds — migrations, backups, and catalog overhead all scale per schema. + +```sql +CREATE TABLE users ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + tenant_id BIGINT NOT NULL REFERENCES tenants(id), + email TEXT NOT NULL, + UNIQUE (tenant_id, email) -- uniqueness is per-tenant, never global +); +-- tenant_id leads every composite index (equality-first rule, SKILL.md) +CREATE INDEX idx_users_tenant ON users(tenant_id, email); +``` + +A forgotten `WHERE tenant_id = ?` is a cross-tenant data leak. Enforce in the database with row-level security, not code review: + +```sql +ALTER TABLE users ENABLE ROW LEVEL SECURITY; +ALTER TABLE users FORCE ROW LEVEL SECURITY; -- otherwise the table owner bypasses it +CREATE POLICY tenant_isolation ON users + USING (tenant_id = current_setting('app.tenant_id')::bigint) + WITH CHECK (tenant_id = current_setting('app.tenant_id')::bigint); +-- App sets per-transaction: SET LOCAL app.tenant_id = '42'; +``` + +One noisy tenant starving the rest is a workload problem, not a schema one. + +## Soft Deletes + +```sql +CREATE TABLE posts ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + title TEXT NOT NULL, + deleted_at TIMESTAMPTZ -- NULL = live +); + +CREATE VIEW active_posts AS SELECT * FROM posts WHERE deleted_at IS NULL; +``` + +The trap that ships to prod: a plain `UNIQUE(email)` still counts soft-deleted rows, so a deleted user blocks re-registration forever. Scope uniqueness to live rows: + +```sql +CREATE UNIQUE INDEX idx_users_email_live ON users(email) WHERE deleted_at IS NULL; +``` + +- Soft delete does not cascade: FKs still point at "deleted" parents, and `ON DELETE CASCADE` never fires. Decide per child table whether to soft-delete along or allow orphan-of-deleted. +- Every query must filter, forever. Enforce with a view or a mandatory scope, because the one report that forgets is the one shown to the customer. +- Index `deleted_at` as a partial predicate on the indexes you actually use, not as its own column index — the value is NULL for almost every row. +- If the requirement is retention or audit rather than undo, an audit log plus hard delete is simpler and erasable on request. + +## Audit Logging + +```sql +CREATE TABLE audit_log ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + table_name TEXT NOT NULL, + record_id BIGINT NOT NULL, + action TEXT NOT NULL, -- INSERT, UPDATE, DELETE + old_data JSONB, -- NULL on INSERT + new_data JSONB, -- NULL on DELETE + changed_by BIGINT, -- application user, not the database role + changed_at TIMESTAMPTZ DEFAULT NOW() +); +CREATE INDEX idx_audit_record ON audit_log(table_name, record_id, changed_at); + +CREATE OR REPLACE FUNCTION audit_trigger() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO audit_log (table_name, record_id, action, old_data, new_data, changed_by) + VALUES (TG_TABLE_NAME, COALESCE(NEW.id, OLD.id), TG_OP, + to_jsonb(OLD), to_jsonb(NEW), + nullif(current_setting('app.user_id', true), '')::bigint); + RETURN COALESCE(NEW, OLD); +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER users_audit +AFTER INSERT OR UPDATE OR DELETE ON users +FOR EACH ROW EXECUTE FUNCTION audit_trigger(); +``` + +- With a pooled connection, the database role is the same for every request — the application user must be propagated in a session variable and read by the trigger, or every audit row is useless. +- Audit tables outgrow their source tables (every UPDATE writes a row) — partition by month and drop old partitions (→ Time-Series), and never FK `changed_by` to `users` if users can be hard-deleted. +- Grant the application `INSERT` only on the audit table. An audit log the app can edit proves nothing. +- Storing full row snapshots is simple and large; storing only changed keys is compact and requires a diff at write time. Snapshots win until the table is wide. + +## Polymorphic Associations + +```sql +-- Type + id columns: flexible, but the database cannot enforce that +-- commentable_id points at a real row — orphans accumulate unnoticed +CREATE TABLE comments ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + body TEXT NOT NULL, + commentable_type TEXT NOT NULL, -- 'Post', 'Photo' + commentable_id BIGINT NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW() +); +CREATE INDEX idx_comments_poly ON comments(commentable_type, commentable_id); + +-- One nullable FK per target: real referential integrity; CHECK enforces exactly one +CREATE TABLE comments ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + body TEXT NOT NULL, + post_id BIGINT REFERENCES posts(id), + photo_id BIGINT REFERENCES photos(id), + CHECK ((post_id IS NOT NULL)::int + (photo_id IS NOT NULL)::int = 1) +); +``` + +Pick separate FKs when the target set is small and stable (2-4 types); pick type+id only when types are open-ended — and accept you now own orphan cleanup, which means a scheduled data-quality query. + +## Tags/Labels + +```sql +-- Junction table: portable, FK integrity, tags are first-class rows +CREATE TABLE tags ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT UNIQUE NOT NULL +); +CREATE TABLE post_tags ( + post_id BIGINT REFERENCES posts(id) ON DELETE CASCADE, + tag_id BIGINT REFERENCES tags(id) ON DELETE CASCADE, + PRIMARY KEY (post_id, tag_id) +); +CREATE INDEX idx_post_tags_tag ON post_tags(tag_id); -- reverse lookup (SKILL.md rule 4) + +-- Posts having ALL listed tags: COUNT must equal the list length +SELECT p.* FROM posts p +JOIN post_tags pt ON pt.post_id = p.id +JOIN tags t ON t.id = pt.tag_id +WHERE t.name IN ('sql', 'tutorial') +GROUP BY p.id +HAVING COUNT(DISTINCT t.name) = 2; +``` + +```sql +-- Array column (PostgreSQL): less machinery, GIN-indexed containment — +-- but no FK, so renaming a tag means updating every row that carries it +CREATE TABLE posts ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + title TEXT NOT NULL, + tags TEXT[] DEFAULT '{}' +); +CREATE INDEX idx_posts_tags ON posts USING GIN(tags); +SELECT * FROM posts WHERE tags @> ARRAY['sql', 'tutorial']; -- has all +``` + +Array when tags are free-form labels nobody manages; junction when tags have identity (rename, merge, count, permissions). + +## State Machines + +Native ENUM is append-only in practice: PostgreSQL `ALTER TYPE ... ADD VALUE` works, but values can never be dropped or reordered, and before PostgreSQL 12 it could not run inside a transaction (breaking migration tools). When states will evolve, prefer TEXT + CHECK: + +```sql +CREATE TABLE orders ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + status TEXT NOT NULL DEFAULT 'draft' + CHECK (status IN ('draft', 'pending', 'paid', 'shipped', 'delivered', 'cancelled')) +); +-- Changing states = drop + re-add the CHECK constraint, a normal migration + +-- History table gives auditability and "time in state" queries +CREATE TABLE order_status_history ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + order_id BIGINT REFERENCES orders(id), + from_status TEXT, + to_status TEXT NOT NULL, + changed_at TIMESTAMPTZ DEFAULT NOW() +); +``` + +Valid transitions (draft→pending, not draft→delivered) belong in application code or a trigger — a CHECK constraint sees only the new row, not the transition. Concurrent transitions on the same row need a lock or an optimistic version check, or two workers both move it forward. + +## Permissions (RBAC) + +```sql +CREATE TABLE roles ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT UNIQUE NOT NULL +); +CREATE TABLE permissions ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT UNIQUE NOT NULL -- 'invoice.read', 'invoice.void' +); +CREATE TABLE role_permissions ( + role_id BIGINT REFERENCES roles(id) ON DELETE CASCADE, + permission_id BIGINT REFERENCES permissions(id) ON DELETE CASCADE, + PRIMARY KEY (role_id, permission_id) +); +-- Roles are scoped: a user is an admin of one organization, not of everything +CREATE TABLE user_roles ( + user_id BIGINT REFERENCES users(id) ON DELETE CASCADE, + role_id BIGINT REFERENCES roles(id), + scope_type TEXT NOT NULL, -- 'org', 'project', 'global' + scope_id BIGINT, + PRIMARY KEY (user_id, role_id, scope_type, scope_id) +); +``` + +- Permission names are verbs on resources and are checked by exact string. Never check the role name in application code, or adding a role means editing every call site. +- Scope is the part everyone forgets: an unscoped `user_roles(user_id, role_id)` makes every admin a global admin the first time you add a second organization. +- Resolving effective permissions on every request is a join; cache it per request, and invalidate on role change. +- Row-level filtering (which invoices, not which action) is a different mechanism — RLS or an explicit predicate. +- Deny rules and role inheritance make resolution order-dependent and hard to reason about. Default to grant-only, flat roles; add hierarchy only when the role count makes it unavoidable. + +## Full-Text Search + +```sql +-- PostgreSQL >=12: generated column replaces the old trigger machinery +ALTER TABLE posts ADD COLUMN search_vector tsvector + GENERATED ALWAYS AS ( + setweight(to_tsvector('english', coalesce(title, '')), 'A') || + setweight(to_tsvector('english', coalesce(body, '')), 'B') + ) STORED; +CREATE INDEX idx_posts_search ON posts USING GIN(search_vector); + +-- Query with ranking; websearch_to_tsquery accepts raw user input safely +SELECT *, ts_rank(search_vector, q) AS rank +FROM posts, websearch_to_tsquery('english', 'database performance') q +WHERE search_vector @@ q +ORDER BY rank DESC; +``` + +```sql +-- SQLite FTS5: external-content table needs sync triggers on the base table +CREATE VIRTUAL TABLE posts_fts USING fts5(title, body, content=posts, content_rowid=id); +SELECT * FROM posts_fts WHERE posts_fts MATCH 'database performance'; +``` + +- tsvector search finds words, not substrings — "data" won't match "database". For fuzzy, substring, or typo matching use `pg_trgm`; for relevance beyond `ts_rank`, that is a search engine's job (`elasticsearch`). +- `to_tsquery` throws a syntax error on raw user input (unbalanced quotes, stray operators). `websearch_to_tsquery` (PostgreSQL >=11) parses human search syntax safely; `plainto_tsquery` ANDs all terms. +- The text search configuration (`'english'`) controls stemming and stop words. Indexing with one configuration and querying with another returns nothing at all. +- Multi-language content needs a language column and per-language indexes; one configuration stems the other languages incorrectly. + +## Versioning (Keep History) + +```sql +CREATE TABLE documents ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + title TEXT NOT NULL, + body TEXT NOT NULL, + version INTEGER NOT NULL DEFAULT 1, + created_at TIMESTAMPTZ DEFAULT NOW() +); +CREATE TABLE document_versions ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + document_id BIGINT REFERENCES documents(id), + title TEXT NOT NULL, + body TEXT NOT NULL, + version INTEGER NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE (document_id, version) -- catches double-fire and race bugs +); + +CREATE OR REPLACE FUNCTION save_document_version() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO document_versions (document_id, title, body, version) + VALUES (OLD.id, OLD.title, OLD.body, OLD.version); + NEW.version = OLD.version + 1; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER documents_version +BEFORE UPDATE ON documents +FOR EACH ROW +WHEN (OLD.* IS DISTINCT FROM NEW.*) -- no phantom versions from no-op updates +EXECUTE FUNCTION save_document_version(); +``` + +The `version` column doubles as an optimistic lock: an update guarded by `WHERE version = :expected` fails when someone else saved first. For temporal validity ("what was the price on this date") rather than edit history, use validity ranges instead of a version counter. + +## Settings/Config (Key-Value) + +```sql +CREATE TABLE settings ( + key TEXT PRIMARY KEY, + value JSONB NOT NULL, + updated_at TIMESTAMPTZ DEFAULT NOW() +); +INSERT INTO settings (key, value) VALUES ('user_prefs', '{"theme": "dark"}') +ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = NOW(); + +CREATE TABLE user_settings ( + user_id BIGINT REFERENCES users(id) ON DELETE CASCADE, + key TEXT NOT NULL, + value JSONB NOT NULL, + PRIMARY KEY (user_id, key) +); +``` + +Key-value is for genuinely open-ended settings. The moment a "setting" needs a type, a default, validation, or appears in a WHERE clause across users — promote it to a real column. EAV as the primary data model is a lot of pain for no gain. + +## Feature Flags + +```sql +CREATE TABLE feature_flags ( + key TEXT PRIMARY KEY, + enabled BOOLEAN NOT NULL DEFAULT false, + rollout_percent SMALLINT NOT NULL DEFAULT 0 CHECK (rollout_percent BETWEEN 0 AND 100), + updated_at TIMESTAMPTZ DEFAULT NOW() +); +CREATE TABLE feature_flag_overrides ( + key TEXT REFERENCES feature_flags(key) ON DELETE CASCADE, + scope_type TEXT NOT NULL, -- 'user', 'tenant' + scope_id BIGINT NOT NULL, + enabled BOOLEAN NOT NULL, + PRIMARY KEY (key, scope_type, scope_id) +); +``` + +- Percentage rollout must be deterministic per subject: hash `key || subject_id` and compare against the threshold, so a user does not flip between variants on every request. +- Resolution order is override → percentage → global default, and it must be identical everywhere the flag is read. +- Flags are read constantly and changed rarely: cache them and invalidate on write, rather than querying per request. +- Every flag needs a removal date. A flags table with entries from three years ago is a schema nobody can reason about. + +## Job Queue Table + +```sql +CREATE TABLE jobs ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + queue TEXT NOT NULL DEFAULT 'default', + payload JSONB NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending','running','done','failed')), + attempts SMALLINT NOT NULL DEFAULT 0, + max_attempts SMALLINT NOT NULL DEFAULT 5, + run_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), -- scheduling and backoff + locked_at TIMESTAMPTZ, -- visibility timeout + last_error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); +-- The one index the pull query needs +CREATE INDEX idx_jobs_pull ON jobs(queue, run_at) WHERE status = 'pending'; +``` + +- Pull with `FOR UPDATE SKIP LOCKED` so N workers take disjoint jobs with no coordinator. +- A crashed worker leaves a row `running` forever unless something reclaims it: sweep rows whose `locked_at` is older than the visibility timeout back to `pending`. +- Retries need exponential backoff written into `run_at`, and a terminal state at `max_attempts` — an infinite retry loop on a poison message saturates the queue. +- Jobs must be idempotent: at-least-once delivery is what this design provides. +- Delete or archive completed rows on a schedule. A `jobs` table that keeps every completed row becomes the largest table in the database and slows the pull query. +- This design is correct up to moderate rates; beyond that, queue churn competes with the application's own writes. + +## Idempotency Keys and Outbox + +```sql +-- Deduplicate externally-triggered writes: the unique constraint is the mechanism +CREATE TABLE idempotency_keys ( + key TEXT PRIMARY KEY, + endpoint TEXT NOT NULL, + request_hash TEXT NOT NULL, -- reject a reused key with a different body + response JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Outbox: the external effect is committed atomically with the data change +CREATE TABLE outbox ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + aggregate TEXT NOT NULL, + event_type TEXT NOT NULL, + payload JSONB NOT NULL, + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); +CREATE INDEX idx_outbox_unpublished ON outbox(id) WHERE published_at IS NULL; +``` + +A database transaction cannot span an external system. Writing the event to `outbox` in the same transaction as the business change, and publishing it from a separate poller, converts "the row was saved but the webhook never fired" into "the webhook fires late" — the failure mode you can live with. Consumers must tolerate duplicates. + +Expire idempotency keys on a stated window (24 hours is typical for payment APIs), or the table grows without bound. + +## Rate Limits and Quotas + +```sql +-- Fixed window: one row per subject per window; cheap and slightly unfair at edges +CREATE TABLE rate_limits ( + subject TEXT NOT NULL, -- user id, api key, ip + window_start TIMESTAMPTZ NOT NULL, + count INT NOT NULL DEFAULT 0, + PRIMARY KEY (subject, window_start) +); +INSERT INTO rate_limits (subject, window_start, count) +VALUES (:subject, date_trunc('minute', NOW()), 1) +ON CONFLICT (subject, window_start) DO UPDATE SET count = rate_limits.count + 1 +RETURNING count; +``` + +- Fixed windows allow a burst of up to 2× the limit across a window boundary. A sliding window (weighting the previous window by the elapsed fraction) fixes it at the cost of a second row read. +- The counter row is a contention point per subject under load, which is exactly the intent for one user but a problem for a global limit. +- Rate limiting in the database costs a write per request. Move it to an in-memory store once the request rate matters; keep the database version for quotas measured in days or months, where durability matters more than latency. +- Delete expired windows on a schedule, or partition by window. + +## Ledgers and Balances + +Never store a mutable balance as the source of truth. Store immutable entries and derive: + +```sql +CREATE TABLE ledger_entries ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + account_id BIGINT NOT NULL REFERENCES accounts(id), + transfer_id BIGINT NOT NULL, -- groups the two sides of one movement + amount_cents BIGINT NOT NULL, -- signed: negative = debit + currency CHAR(3) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); +CREATE INDEX idx_ledger_account ON ledger_entries(account_id, created_at); + +-- Double entry: every transfer's entries must sum to zero +-- (enforced by a deferred constraint trigger or by an application invariant check) +``` + +- Entries are append-only: corrections are new compensating entries, never updates or deletes. An editable ledger is not a ledger. +- Balance = `SUM(amount_cents)` for the account. When that scan gets expensive, add periodic snapshot rows (`balance_at(account_id, as_of, balance_cents)`) and sum only entries after the latest snapshot — the derivation stays authoritative. +- Money as integer minor units, currency stored alongside, never `FLOAT`. +- Enforce the non-negative-balance rule inside the transaction that inserts the debit, with the account row locked, or two concurrent withdrawals both pass the check. +- A scheduled reconciliation query asserting that every `transfer_id` group sums to zero catches bugs the constraints cannot. + +## Attachments and Files + +```sql +CREATE TABLE attachments ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + owner_type TEXT NOT NULL, + owner_id BIGINT NOT NULL, + storage_key TEXT NOT NULL UNIQUE, -- path in object storage, not a URL + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + byte_size BIGINT NOT NULL, + checksum TEXT NOT NULL, -- dedupe and integrity + uploaded_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); +``` + +- Store bytes in object storage and metadata in the database. Blobs in table rows inflate every backup, every replica, and every sequential scan. +- Store the storage key, never a full URL: hostnames, CDNs, and signing schemes change; the key does not. +- Deleting the row does not delete the object. Either delete both in a job driven by the outbox pattern, or accept orphans and run a reconciliation sweep. +- Uploads that fail halfway leave rows with no object: mark rows `pending` until the upload is confirmed, and expire stale pending rows. +- The checksum enables deduplication (same bytes, one object, many rows) and detects corruption that raises no error. + +## Recurrence + +```sql +CREATE TABLE recurring_events ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + rrule TEXT NOT NULL, -- RFC 5545 rule, expanded by the application + starts_at_local TIMESTAMP NOT NULL, + tz TEXT NOT NULL, + until_date DATE +); +CREATE TABLE recurring_event_exceptions ( + event_id BIGINT REFERENCES recurring_events(id) ON DELETE CASCADE, + occurrence_date DATE NOT NULL, + cancelled BOOLEAN NOT NULL DEFAULT false, + override_starts_at_local TIMESTAMP, + PRIMARY KEY (event_id, occurrence_date) +); +``` + +Store the rule plus exceptions, and materialize occurrences into a table only for the window you need to query (the next year, say), refreshed on rule change. Expanding every occurrence forever is unbounded; expanding none makes "what is on Tuesday" unanswerable in SQL. The local-time-plus-zone storage is required because recurrence follows the wall clock across DST. + +## Time-Series Data + +```sql +-- Declarative range partitioning (PostgreSQL >=10; indexes propagate to +-- partitions automatically from >=11) +CREATE TABLE metrics ( + recorded_at TIMESTAMPTZ NOT NULL, + metric_name TEXT NOT NULL, + value NUMERIC NOT NULL +) PARTITION BY RANGE (recorded_at); + +CREATE TABLE metrics_2026_01 PARTITION OF metrics +FOR VALUES FROM ('2026-01-01') TO ('2026-02-01'); +``` + +- Retention is the reason to partition: `DROP TABLE metrics_2025_07` is instant and reclaims disk; `DELETE WHERE recorded_at < ...` on the same data runs for hours and leaves bloat. +- Every query should filter on the partition key, or it scans all partitions. +- Automate partition creation (pg_partman or a scheduled migration), running at least one interval ahead of the partition it creates — the outage mode is inserts failing because next month's partition doesn't exist. +- Unique constraints must include the partition key; global uniqueness is unavailable in declarative partitioning. +- Downsample old data rather than keeping raw points forever: per-minute for a week, per-hour for a month, per-day beyond. A purpose-built store (`timescaledb`, `influxdb`, `clickhouse`) does this natively. +- Partitioning pays at scale; a table you could also just index by `recorded_at` doesn't need it yet. + +## Counting (Exact vs Approximate) + +```sql +-- Exact: scans (index or heap) — cost grows with table size +SELECT COUNT(*) FROM large_table; + +-- Approximate (PostgreSQL, instant): planner's row estimate. +-- Accurate to autovacuum's last ANALYZE; wildly off right after a bulk load +SELECT reltuples::bigint AS estimate FROM pg_class WHERE relname = 'large_table'; +``` + +Dashboards and "~1.2M results" UIs take the estimate; billing and invariants take the exact count. If an exact count is hot, maintain a counter row updated in the same transaction as the insert/delete — and expect that counter row to become a lock hotspot under heavy write concurrency (shard it into N rows and SUM if it does). Distinct counts do not aggregate across periods and need their own treatment. diff --git a/workspace-sql/skills/sql/security.md b/workspace-sql/skills/sql/security.md new file mode 100644 index 0000000..8e06260 --- /dev/null +++ b/workspace-sql/skills/sql/security.md @@ -0,0 +1,149 @@ +# Security — Injection, Privileges, and Personal Data + +Three separate jobs that get conflated: keeping untrusted input out of the parser, keeping the application's database role from being able to do damage, and keeping personal data legible only where it must be. + +Contents: Injection Surface · Beyond Placeholders · Dynamic Identifiers · Least Privilege · Role Layout · Row-Level Security · Personal Data · Erasure · Encryption · Auditing · Connection Secrets · Backup Exposure · Review Checklist + +## The Full Injection Surface + +Placeholders bind **values**. Everything else in a statement is code, and every one of these positions has produced real incidents: + +| Position | Bindable? | Safe approach | +|---|---|---| +| `WHERE col = ?` | Yes | Placeholder | +| `IN (?, ?, ?)` | Yes, one placeholder per element | Generate the exact count, or pass an array/`= ANY($1)` where supported | +| Table or column name | No | Allowlist of literal strings in code (SKILL.md rule 1) | +| `ORDER BY ` | No | Map an opaque client token (`"newest"`) to a hardcoded fragment; never pass the column through | +| `LIMIT` / `OFFSET` | Usually yes | Placeholder; otherwise cast to integer and clamp to a maximum | +| `LIKE` pattern | Value is bindable, wildcards are not | Escape `%` and `_` in user input, then add your own wildcards | +| Interval/date arithmetic (`NOW() - INTERVAL '? days'`) | No, the literal is part of the syntax | Bind an integer and multiply an interval, or validate the number | +| Schema/database qualifier | No | Allowlist; never derive from a hostname or subdomain header | +| JSON path expression | Engine-dependent | Treat as an identifier: allowlist | + +## Beyond Placeholders + +- **Second-order injection**: input stored safely, then concatenated into a later query (a report builder reading a saved "filter" column). The stored value is as untrusted as the original request. +- **String building in stored procedures**: `EXECUTE 'SELECT ... ' || col` inside a function is injectable exactly like application code. Use `format('%I', col)` for identifiers and `%L` for literals in PostgreSQL, `QUOTENAME` in SQL Server. +- **ORM escape hatches**: `.raw()`, `.whereRaw()`, `$queryRawUnsafe`, string-built `filter` arguments. The ORM protects the paths you use through it, not around it. +- **Error messages as an oracle**: returning the database error text to the client leaks table and column names and confirms injection attempts. Log the detail, return a generic message. +- **Blind and time-based probes**: an endpoint that returns different response times or row counts is enough. Correct escaping is the defense; hiding errors is not. +- **Batch separators**: drivers that allow multiple statements per call turn a single injection into arbitrary DDL. Disable multi-statement mode unless a migration path needs it. + +## Dynamic Identifiers, Done Correctly + +``` +ALLOWED_SORTS = {"newest": "created_at DESC", "oldest": "created_at ASC", "name": "name ASC"} +order_by = ALLOWED_SORTS.get(request.sort, "created_at DESC") # default, never the raw input +``` + +The allowlist maps an opaque token to a complete, hardcoded fragment. Validating with a regex (`^[a-z_]+$`) is weaker: it still permits any existing column name, so a client can sort by `password_hash` and read it one bit at a time through ordering. + +## Least Privilege for the Application Role + +The default posture on most projects — the app connects as owner or superuser — means one injection is total compromise. Split the roles: + +```sql +-- Owner role: owns objects, runs migrations, used only by the deploy pipeline +-- App role: no DDL, no ownership +CREATE ROLE app_rw LOGIN PASSWORD :'pw'; +GRANT CONNECT ON DATABASE mydb TO app_rw; +GRANT USAGE ON SCHEMA public TO app_rw; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO app_rw; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA public TO app_rw; + +-- Future tables inherit the grant; without this, every new table breaks the app after deploy +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO app_rw; + +-- Read-only role for analytics, dashboards, and humans +CREATE ROLE app_ro LOGIN PASSWORD :'pw2'; +GRANT CONNECT ON DATABASE mydb TO app_ro; +GRANT USAGE ON SCHEMA public TO app_ro; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO app_ro; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO app_ro; +``` + +- On PostgreSQL, `REVOKE CREATE ON SCHEMA public FROM PUBLIC` (already the default from version 15) — otherwise any role can create objects in it. +- MySQL grants are per `user@host`: `GRANT SELECT ON mydb.* TO 'app'@'10.0.%'`. A grant to `'app'@'%'` undoes the network scoping you configured elsewhere. +- Grant `DELETE` only where the app deletes. Many applications only soft-delete, and discover the difference the day an injection tries a hard one. +- Reserve `TRUNCATE`, `DROP`, and `ALTER` for the migration role. That single split turns "attacker drops the table" into "attacker cannot". + +## Row-Level Security + +```sql +ALTER TABLE documents ENABLE ROW LEVEL SECURITY; +ALTER TABLE documents FORCE ROW LEVEL SECURITY; -- also applies to the table owner + +CREATE POLICY tenant_read ON documents FOR SELECT + USING (tenant_id = current_setting('app.tenant_id')::bigint); +CREATE POLICY tenant_write ON documents FOR INSERT + WITH CHECK (tenant_id = current_setting('app.tenant_id')::bigint); +``` + +- `USING` filters rows that are read or targeted; `WITH CHECK` validates rows being written. A policy with only `USING` lets a tenant insert rows attributed to another tenant. +- RLS is bypassed by superusers and, unless `FORCE` is set, by the table owner. An app connecting as the owner gets no protection at all — this is the most common way RLS is deployed, and it then does nothing at all. +- The setting must be applied per connection, inside the transaction, and cleared or overwritten on reuse. With a transaction-mode pooler, use `SET LOCAL` so it cannot leak into the next tenant's transaction. +- Policies are predicates the planner applies: a policy over a non-indexed expression makes every query on the table slow. Index the policy column (it is usually `tenant_id`, which already leads the composite indexes). +- Cross-tenant references still need checking: a FK to another tenant's row passes RLS on insert unless the policy checks the parent too. + +## Personal Data: Classify Before You Design + +- Tag columns holding personal data in the schema itself (a comment or naming convention) so exports, logs, and fixtures can be filtered mechanically rather than by memory. +- Do not log query parameters for statements touching those columns; slow-query logs are the classic accidental PII store. +- Test and staging environments should hold generated or masked data, never a production restore. If a production restore is unavoidable, mask as part of the restore job, not afterwards. +- Store the minimum: a birth year instead of a birth date, a hashed identifier instead of a national id, an age bucket instead of an age, when the product only needs the coarse value. +- Passwords are hashed with a slow, salted, memory-hard function by the application (bcrypt, scrypt, Argon2) — never a database `MD5`/`SHA` call, which is fast by design and often lands in the query log. + +## Deletion and Erasure + +- An erasure request must reach every copy: the row, the audit log, soft-deleted rows, materialized views, rollup tables, backups, replicas, and any exported extract. +- Backups are the hard part. The workable policy is a bounded retention window (state it, e.g. 30 days) plus a documented rule that restores re-apply the erasure list; per-row deletion inside historical backups is not practical. +- Prefer crypto-shredding for data that must be unrecoverable on demand: encrypt each subject's sensitive fields with a per-subject key and delete the key. Every copy becomes unreadable at once, backups included. +- Anonymization must break linkability: replacing a name while keeping a unique id, an exact timestamp, and a postcode re-identifies the person. Generalize or drop the quasi-identifiers too. +- Distinguish erasure from soft delete. `deleted_at IS NOT NULL` is still the data. + +## Encryption + +| Layer | Protects against | Does not protect against | +|---|---|---| +| TLS in transit | Network interception | Anything with valid credentials | +| Disk / tablespace encryption at rest | Stolen disks, discarded hardware | Any authenticated query — the database decrypts transparently | +| Column-level encryption (app-side) | A dump, a read-only leak, an over-privileged analyst | Nothing if the key sits next to the data | +| Deterministic column encryption | Same, while allowing equality lookups | Frequency analysis; equal plaintexts produce equal ciphertexts | + +- Choose per column: randomized encryption for anything you never search; deterministic only for a column you must look up by exact value, accepting the leak. +- Encrypted columns cannot be range-scanned, sorted, or pattern-matched. Design the query set first — retrofitting encryption onto a column with a range filter means changing the query. +- Keys live in a key management service, not in the database, not in the repository, not in an environment variable that is echoed into logs. +- Enforce TLS on the server side (`sslmode=verify-full` on the client, `require_secure_transport` on MySQL). A client that is allowed to fall back to plaintext will, on the day the certificate expires. + +## Auditing Access + +- Two different needs: **data change history** (who changed which row to what — the audit table) and **access logging** (who read what, when, from where — the engine's own audit facility). +- Capture the application user, not just the database role: with a shared pooled role, every row says `app_rw`. Propagate the end-user id in a session variable (`SET LOCAL app.user_id`) and read it in the audit trigger. +- The audit table must be append-only for the app role: `GRANT INSERT` only, no `UPDATE`, no `DELETE`. An audit log the application can edit proves nothing. +- Audit tables outgrow their source tables; partition and expire them on a stated retention window. +- Log failed authentication and privilege-denied events too — successful queries alone hide the reconnaissance. + +## Connection Secrets + +- Credentials belong in a secret manager or the platform's secret store, injected at runtime. Never in the repository, never in a migration file, never in `~/Clawic/data/sql/`. +- Connection strings appear in process listings, crash dumps, ORM debug output, and error pages. Prefer environment-injected components over one URL string, and redact them in every log formatter. +- Rotate by supporting two valid credentials at once (add the new one, deploy, remove the old); rotation with a single credential is an outage. +- Use per-service credentials so one compromised service is one revocation, and so the audit log can attribute activity. +- Bind the database to a private network and require TLS. A managed database with a public endpoint and a strong password is one credential leak from open. + +## Backups Are a Copy of Everything + +- A dump has the same sensitivity as the database, with none of its access control. Encrypt at rest, restrict who can download, and log every access. +- Restore drills into a scratch environment must use masked data or an isolated network — a restore drill is the most common way production data reaches a laptop. +- Verify what a dump contains before sharing it: `pg_dump --schema-only` for schema questions, and a targeted `--table` extract instead of the whole database. + +## Review Checklist + +- Every user value is a placeholder; every dynamic identifier comes from an allowlist mapping to hardcoded fragments. +- The application role cannot run DDL, `TRUNCATE`, or `DROP`; a separate role owns the schema. +- `DELETE` is granted only where the application deletes. +- Multi-tenant tables enforce isolation in the database (RLS with `FORCE` and both `USING` and `WITH CHECK`), not only in code. +- Personal data columns are identified, minimized, and excluded from logs and non-production environments. +- Database errors are logged in full and returned to clients generically. +- Credentials come from a secret store, TLS is required, and the endpoint is not publicly reachable. +- The audit table is append-only for the application role and carries the end-user identity. diff --git a/workspace-sql/skills/sql/setup.md b/workspace-sql/skills/sql/setup.md new file mode 100644 index 0000000..b65dcba --- /dev/null +++ b/workspace-sql/skills/sql/setup.md @@ -0,0 +1,33 @@ +# Setup — SQL + +Read this on first use to load user preferences. Do not interview the user. + +## Your Attitude + +SQL failures are quiet: the query returns rows, they are just the wrong rows. Verify shape before speed, show the read before the write, and never emit a destructive statement the user cannot preview. + +## How To Load Preferences + +1. Read `~/Clawic/data/sql/config.yaml` if it exists. Apply its values. +2. For anything absent, use the defaults in the Configuration table of `SKILL.md` — do not ask. + - `dialect: postgres`, `identifier_style: snake_case`, `table_naming: plural`, `pk_type: bigint-identity`, `destructive_guard: true`, `timezone_policy: utc`, `lock_timeout: 2s`, `batch_size: 5000`; `engine_version` and `migration_tool` unset. +3. Read `~/Clawic/data/sql/memory.md` for prior context (their schema, recurring pain points, engine version). Absence is fine; proceed without comment. +4. Infer, do not interrogate: a pasted `CREATE TABLE`, connection string, error text, or migration file names the dialect and often the version. Inference applies for the session; only a stated preference gets written. + +Work from defaults immediately. Never open with questions about engine, conventions, or how cautious to be. + +## Recording Preferences (only when the user declares one) + +Write to config or memory **only** when the user states a preference in the course of the work — never as a preflight questionnaire. + +- User names an engine, version, migration tool, key type, naming convention, DDL lock budget, or batch size → update the matching key in `~/Clawic/data/sql/config.yaml`. +- User expresses a habit or stance (how much confirmation destructive DDL needs, whether triggers are allowed, whether every migration needs a down script) → record it under the relevant preference area (tooling, conventions, platform, safety posture, output format, work order, integrations, constraints, thresholds, cadence) in `~/Clawic/data/sql/memory.md`. +- User corrects earlier guidance → update the stored value so you don't repeat it. + +If the user has said nothing, store nothing. An observation never overwrites a declared preference without asking. + +## What Memory Holds + +Memory holds five sections: Status, Context, Schema Seen, Pain Points, Preferences. Track the schema you have already seen (table names, key types, tenancy model), their environments (local, staging, production), recurring pain points, and how much explanation they want — but only from what they actually reveal. + +Never store connection strings, passwords, hostnames, or query results containing personal data. Credentials belong in the user's own secret store, never in `~/Clawic/data/sql/`. diff --git a/workspace-sql/skills/sql/skill-card.md b/workspace-sql/skills/sql/skill-card.md new file mode 100644 index 0000000..ce1cc80 --- /dev/null +++ b/workspace-sql/skills/sql/skill-card.md @@ -0,0 +1,40 @@ +## Description:
+Writes, reviews, and optimizes SQL queries; designs schemas, indexes, and constraints; and plans migrations for relational databases.
+ +This skill is ready for commercial/non-commercial use.
+ +## Publisher:
+[ivangdavila](https://clawhub.ai/user/ivangdavila)
+ +### License/Terms of Use:
+MIT-0
+ + +## Use Case:
+Developers and engineers use this skill to generate, review, debug, and optimize SQL; plan schemas, indexes, constraints, migrations, and operational database tasks across common relational engines.
+ +### Deployment Geography for Use:
+Global
+ +## Known Risks and Mitigations:
+Risk: Generated or suggested SQL can affect production data when copied into a privileged database session.
+Mitigation: Review generated SQL before execution, especially backup, restore, migration, DROP, TRUNCATE, UPDATE, and DELETE statements.
+ + +## Reference(s):
+- [ClawHub SQL Skill](https://clawhub.ai/ivangdavila/skills/sql)
+- [Publisher Profile](https://clawhub.ai/user/ivangdavila)
+- [Clawic SQL Skill Homepage](https://clawic.com/skills/sql)
+ + +## Skill Output:
+**Output Type(s):** [text, markdown, code, shell commands, configuration, guidance]
+**Output Format:** [Markdown with SQL, configuration, and shell command snippets]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [May reference local preferences stored under ~/Clawic/data/sql/ and adapts guidance to the selected SQL dialect.]
+ +## Skill Version(s):
+1.0.4 (source: server release metadata and frontmatter)
+ +## Ethical Considerations:
+Users should evaluate whether this skill is appropriate for their environment, review any generated or modified files before relying on them, and apply their organization's safety, security, and compliance requirements before deployment.
diff --git a/workspace-sql/skills/sql/testing.md b/workspace-sql/skills/sql/testing.md new file mode 100644 index 0000000..3705588 --- /dev/null +++ b/workspace-sql/skills/sql/testing.md @@ -0,0 +1,127 @@ +# Testing SQL and Migrations + +Most database test suites verify that the ORM works. The failures that reach production are constraint violations under concurrency, migrations that lock, and queries whose plan collapses at real data volumes — none of which a green suite on 20 seeded rows detects. + +Contents: What To Test · Test Database · Isolation Strategies · Fixtures · Determinism · Constraint Tests · Migration Tests · Performance Tests · Data Quality Checks · CI · Traps + +## What Is Worth Testing + +| Target | Why | Shape of the test | +|---|---|---| +| Constraints and defaults | They are the last line of defense and get lost in migrations unnoticed | Insert violating rows, assert the error | +| Complex queries (aggregates, window functions, recursive CTEs) | Off-by-one and fan-out bugs are invisible in review | Fixed input rows, exact expected output | +| Migrations | The only code that runs once and cannot be retried | Apply, assert schema, apply to a production-shaped copy | +| Uniqueness under concurrency | Check-then-insert races pass every serial test | Two connections, one assertion | +| Query plans on hot paths | Regression from an added join or dropped index | Assert the plan does not contain a sequential scan on a large table | +| The dialect the code actually targets | Test-engine leniency masks target-engine failures | Run against the target engine | + +Not worth testing: that `SELECT * FROM users` returns rows, that the ORM maps a column, or the engine's own behavior. + +## The Test Database + +- Test against the **same engine and major version** as production. SQLite standing in for PostgreSQL accepts statements PostgreSQL rejects, enforces less, and has different NULL ordering and case sensitivity — the substitution hides exactly the bugs a database test exists to catch. +- Create the schema by **running the migrations**, not by loading a snapshot. If the suite loads a dump, nothing ever tests the migration path, and the two drift until a deploy fails. +- Keep one canonical schema dump under version control anyway, regenerated from the migrations, and diff it in CI. A migration that produces a different schema than the checked-in dump is a review finding. +- One database per parallel worker (or one schema per worker) — shared state across parallel tests produces flakes that look like race conditions in the application. +- Never point a test suite at a database that also holds real data. A truncation helper with the wrong connection string is the classic way to lose production. + +## Isolation Between Tests + +| Strategy | Speed | Limitation | +|---|---|---| +| Transaction per test, rolled back at the end | Fastest | Cannot test anything that commits, and nested transaction behavior in the code under test conflicts with the wrapper | +| `TRUNCATE` the touched tables between tests | Fast enough, always correct | Must reset sequences (`TRUNCATE ... RESTART IDENTITY`) or ids leak across tests | +| Recreate the schema per test | Correct, very slow | Only for migration tests | +| Template database / snapshot restore | Fast and complete | Setup complexity; PostgreSQL `CREATE DATABASE ... TEMPLATE` is the common form | + +Default: transaction-rollback for the bulk of the suite, truncation for tests that must commit (concurrency, triggers on commit, anything checking what another connection sees). + +## Fixtures + +- Build entities through factories that fill required fields with valid defaults and let each test override only the field it is about. A test that constructs 15 fields to assert on one is unreadable and breaks on every schema change. +- Insert the minimum: two rows to prove a filter, three to prove ordering. Large fixture files make every test slower and none clearer. +- Never share mutable fixture rows across tests, and never depend on fixture ids being specific numbers. +- Test data must be obviously fake (`user-1@example.test`) so it is recognizable if it ever escapes into a real system. +- Production data as fixtures is a data-protection incident waiting to happen; if realistic distributions are required, generate them or mask a copy inside the restore job. + +## Determinism + +Flaky database tests almost always come from one of these: + +- `SELECT` without `ORDER BY`: row order is undefined and changes after any write or vacuum. Order every assertion's query by a unique column (SKILL.md Traps). +- `NOW()`/`CURRENT_DATE` in fixtures or assertions: freeze time in the application, or assert on ranges, not equality. Tests that only fail near midnight or on the last day of a month are this. +- Depending on generated ids: sequences do not reset with a rollback, so ids differ between runs. +- Locale or timezone of the test machine differing from CI: pin both explicitly. +- Floating-point equality: compare with a tolerance, or use exact decimal types. + +## Testing Constraints and Concurrency + +```sql +-- The constraint exists and bites: expect a unique violation +INSERT INTO users (email) VALUES ('a@example.test'); +INSERT INTO users (email) VALUES ('a@example.test'); -- assert SQLSTATE 23505 + +-- The FK actually blocks the orphan: expect 23503 +INSERT INTO orders (user_id) VALUES (999999); + +-- Soft-delete uniqueness is scoped to live rows only +-- (insert, soft-delete, re-insert the same email → must succeed) +``` + +Concurrency tests need two real connections; a single connection cannot produce a lock conflict with itself. The minimal shape: connection A opens a transaction and takes the lock, connection B attempts the conflicting write with a short `lock_timeout` and asserts either the timeout or the expected serialization failure. Keep these few and targeted — they are slow and easy to make flaky. + +## Testing Migrations + +Every migration gets four checks before it is allowed near production: + +1. **Applies cleanly** to a database at the previous revision. +2. **Reverses cleanly** if the project keeps down migrations — and if it does not, that is a stated decision, not an omission. +3. **Applies to production-shaped data**: a restored copy (masked) or a generated dataset at production row counts. A migration tested on 100 rows tells you nothing about lock duration on 100 million. +4. **Is compatible with the previous application version**, because during a deploy both run at once. This is what expand-migrate-contract exists for (SKILL.md rule 8), and the test is simply running the previous version's test suite against the new schema. + +Additional checks worth automating: the migration does not exceed a stated duration on the production-shaped copy; it sets `lock_timeout`; and for MySQL, it contains a single DDL statement, since a multi-statement migration cannot roll back there. + +## Performance Regression Tests + +- Assert on the **plan**, not on wall time. Wall time on shared CI hardware is noise; "the plan for this query contains no sequential scan on `orders`" is stable. +- Generate a dataset at a realistic order of magnitude for the handful of queries that matter, and `ANALYZE` it so the planner has honest statistics. +- Track index usage over time in production instead of trying to test it: a new index whose `idx_scan` stays at zero after a week is dead weight. +- Catch N+1s structurally: assert the number of queries a request issues, which is the only reliable detector because each individual query is fast. + +## Data Quality Checks in Production + +Tests cover code; assertions cover data. Run these as scheduled queries on the user's stated Cadence (default: hourly for money, daily for the rest) and alert on non-zero results: + +```sql +-- Orphans that a missing or disabled FK allowed in +SELECT COUNT(*) FROM orders o LEFT JOIN users u ON u.id = o.user_id WHERE u.id IS NULL; + +-- Duplicates on a key that should be unique but is not constrained yet +SELECT email, COUNT(*) FROM users GROUP BY email HAVING COUNT(*) > 1; + +-- Invariants the schema cannot express +SELECT COUNT(*) FROM accounts WHERE balance < 0; +SELECT COUNT(*) FROM bookings WHERE end_at <= start_at; +``` + +Each check that fires twice earns a real constraint. The check is the interim measure; the constraint is the fix. + +## In CI + +- Run the database as a service container pinned to the production major version; a floating `latest` tag turns an upstream release into a mystery failure. +- Order: apply migrations → assert the schema matches the checked-in dump → run the suite → run the data-quality queries against the seeded database. +- Fail the build on a migration that is not accompanied by its schema-dump update. +- Keep the whole suite fast enough to run on every commit; the moment it is not, people stop running it and the database tests are the first to be skipped. + +## Traps + +| Trap | Why it fails | Do instead | +|---|---|---| +| SQLite in tests, PostgreSQL in production | Different type enforcement, case sensitivity, NULL ordering, and constraint behavior | Same engine and major version as production | +| Loading a schema dump instead of running migrations | The migration path is never exercised until deploy day | Migrate up from empty; diff against the dump | +| Transaction-rollback isolation for everything | Skips anything that must commit, with no failure, including trigger and concurrency tests | Truncation strategy for those tests | +| Asserting on ids | Sequences do not roll back; ids vary between runs | Assert on business keys | +| `SELECT` in an assertion without `ORDER BY` | Passes until row order changes | Deterministic ordering with a unique tiebreaker | +| Migration tested only on an empty database | Says nothing about lock time or backfill duration | Production-shaped copy | +| Production data copied into test fixtures | Uncontrolled personal data outside production | Generated or masked data | +| Timing assertions on CI hardware | Noise, then a disabled test | Assert on plans and query counts | diff --git a/workspace-sql/skills/sql/transactions.md b/workspace-sql/skills/sql/transactions.md new file mode 100644 index 0000000..b444150 --- /dev/null +++ b/workspace-sql/skills/sql/transactions.md @@ -0,0 +1,148 @@ +# Transactions, Isolation, and Locking + +Concurrency bugs pass every single-user test. The defense is knowing which anomaly your isolation level still allows and closing that one hole explicitly. + +Contents: Isolation Levels · Anomalies · Engine Defaults · Explicit Locks · Lock Ordering · Deadlocks · Retry Loops · Optimistic Locking · Idempotency · Advisory Locks · Long Transactions · Transaction Boundaries + +## Isolation Levels and What They Still Allow + +| Level | Dirty read | Non-repeatable read | Phantom | Write skew | +|---|---|---|---|---| +| READ UNCOMMITTED | Possible (not in PostgreSQL — it behaves as READ COMMITTED) | Possible | Possible | Possible | +| READ COMMITTED | No | Possible | Possible | Possible | +| REPEATABLE READ | No | No | No in PostgreSQL and InnoDB (MVCC snapshot); allowed by the standard | Possible | +| SERIALIZABLE | No | No | No | No | + +Each statement in READ COMMITTED takes a fresh snapshot: a multi-statement report can see two different states of the database and produce totals that never existed. Reports that must be internally consistent run in REPEATABLE READ. + +## The Anomalies, With Their Real Shapes + +- **Lost update**: two sessions read `balance = 100`, both write `balance - 10`, the result is 90 instead of 80. Read-modify-write in application code always has this bug unless locked or made atomic. +- **Non-repeatable read**: the same `SELECT` in one transaction returns different values because another committed in between. Breaks "check then act". +- **Phantom**: a range query returns new rows on re-execution. Breaks "count then enforce a maximum". +- **Write skew**: two transactions each read a set, each verifies an invariant, each writes a *different* row, and together they break the invariant. The classic case: two doctors both cancel their on-call shift because each sees the other still on call. No level below SERIALIZABLE prevents it — and it is the anomaly people assume REPEATABLE READ covers. + +## Engine Defaults (know yours before reasoning) + +| Engine | Default level | Note | +|---|---|---| +| PostgreSQL | READ COMMITTED | SERIALIZABLE uses SSI: no extra locks, but transactions abort with `40001` | +| MySQL / InnoDB | REPEATABLE READ | Uses next-key (gap) locks, so it blocks phantoms by locking ranges — more deadlocks than PostgreSQL at the same level | +| MariaDB | REPEATABLE READ | As InnoDB | +| SQLite | SERIALIZABLE | One writer at a time; `PRAGMA journal_mode=WAL` lets readers proceed during a write | +| SQL Server | READ COMMITTED (lock-based) | Readers block writers unless `READ_COMMITTED_SNAPSHOT ON` — turning it on is usually the single biggest concurrency win on SQL Server | + +## Explicit Row Locks + +```sql +-- Lock the rows you are about to modify; other writers wait +SELECT * FROM inventory WHERE product_id = 5 FOR UPDATE; + +-- Weaker: prevents deletion/key change, allows concurrent non-key updates +SELECT * FROM users WHERE id = 5 FOR SHARE; -- FOR KEY SHARE in PostgreSQL + +-- Fail immediately instead of queueing (interactive paths, health checks) +SELECT * FROM jobs WHERE id = 5 FOR UPDATE NOWAIT; + +-- Skip contended rows (queues; each worker gets a disjoint set) +SELECT id FROM jobs WHERE status = 'pending' +ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED; +``` + +- `FOR UPDATE` locks rows the query *returns*. In READ COMMITTED, a row modified after your snapshot is re-read at the new version — your `WHERE` may no longer be true for it. Re-check the condition after acquiring the lock. +- `FOR UPDATE` combined with `LEFT JOIN` locks rows from the outer table too unless you scope it (`FOR UPDATE OF t`). +- `SKIP LOCKED` changes semantics, not just performance: the query is no longer "the oldest pending job", it is "the oldest one nobody else holds". That is correct for queues and wrong for reports. +- Row locks are released only at COMMIT or ROLLBACK — never at the end of the statement. + +## Lock Ordering (the deadlock prevention that actually works) + +Two transactions locking `{1,2}` and `{2,1}` deadlock; both locking `{1,2}` queue. Impose a total order everywhere rows are locked together: + +```sql +SELECT * FROM accounts WHERE id IN (:a, :b) ORDER BY id FOR UPDATE; +``` + +The order must be consistent across *code paths*, not just within one function — a transfer routine and a batch-settlement job locking the same two accounts in different orders is the standard production deadlock. + +Additional ordering hazards: + +- Parent/child: inserting a child takes a lock on the parent row for FK validation. A job that updates parents and another that inserts children touch the same rows in opposite order. +- Index maintenance: two inserts with no row in common can still contend on the same index page or, in InnoDB, the same gap. +- Upserts: `INSERT ... ON CONFLICT` may take locks in key order determined by the values, so batches should be sorted by key before insert. + +## Reading a Deadlock Report + +- PostgreSQL logs both statements and the lock each was waiting on. The victim is chosen by cost; it is not necessarily the guilty one. +- MySQL: `SHOW ENGINE INNODB STATUS`, section `LATEST DETECTED DEADLOCK` — it holds only the most recent one, so capture it immediately. +- The two statements shown are where the transactions *waited*, not where they took their first lock. Read the whole transaction, not the reported line. +- Deadlock frequency rising with load and no shared rows → gap locks (InnoDB REPEATABLE READ) or missing indexes forcing wide range locks. An index that narrows the scan narrows the locks. + +## The Retry Loop (non-optional at SERIALIZABLE) + +Serialization failures and deadlocks are expected outcomes, not exceptions to log and drop. + +``` +attempt = 0 +while attempt < 3: + begin + try: + do work; commit; break + except sqlstate in ('40001', '40P01'): # serialization failure, deadlock + rollback + attempt += 1 + sleep(random jitter, e.g. 50ms × 2^attempt ± 25%) + except other: + rollback; raise +``` + +- Retry only the whole transaction. Retrying one statement inside a rolled-back transaction executes against nothing. +- The work must be idempotent or wholly inside the transaction — a retry that re-sends an email is worse than the original failure. +- Cap attempts (3 is a reasonable default; a house retry cap is a Thresholds preference) and jitter the backoff; synchronized retries recreate the same collision. +- Without this loop, SERIALIZABLE is strictly worse than REPEATABLE READ: same cost, plus failures you do not handle. + +## Optimistic Locking (no database lock at all) + +```sql +UPDATE documents +SET body = :body, version = version + 1 +WHERE id = :id AND version = :expected_version; +-- 0 rows affected = someone else won; re-read and re-apply or surface a conflict +``` + +- Correct choice when conflicts are rare and the "transaction" spans a user thinking (an edit form open for minutes). Never hold a database lock across user time (SKILL.md rule 5). +- The application must check the affected-row count. Ignoring it discards the user's edit with no error — the most common implementation bug. +- A `updated_at` timestamp works as the version only if its resolution exceeds the update rate; an integer counter has no such failure. + +## Idempotency and Exactly-Once Writes + +- Give externally triggered writes a client-supplied idempotency key with a unique constraint: the second attempt violates the constraint, and the handler returns the first result. +- `INSERT ... ON CONFLICT DO NOTHING RETURNING id` returns no row when the insert was a no-op — fetch the existing row explicitly rather than treating the empty result as failure. +- Database transactions do not extend to external systems. Commit the database first and drive the external effect from a durable outbox row, or accept a window where one succeeded and the other did not. +- Retries of a batch must be safe at any prefix: chunk the work and record progress in the same transaction as the work. + +## Advisory / Application Locks + +```sql +SELECT pg_advisory_xact_lock(hashtext('nightly-rollup')); -- released at commit +SELECT GET_LOCK('nightly-rollup', 10); -- MySQL, with timeout +``` + +- Use for "only one worker runs this job", cron singletons, and serializing a migration across instances — cases where no row represents the resource. +- Prefer the transaction-scoped variant (`pg_advisory_xact_lock`). Session-scoped locks survive commit and leak when a connection is returned to a pool still holding one. +- Advisory locks live in a flat namespace of integers: hash a descriptive string and keep the list of names in one place, or two features end up sharing a lock. +- Under a transaction-mode pooler, session-level advisory locks are unusable. + +## Long Transactions Are a Systemic Cost + +- They hold row locks, block DDL, and in PostgreSQL pin dead tuples so nothing can be vacuumed database-wide — one forgotten `BEGIN` bloats every table. +- `idle in transaction` is worse than a long-running query: it holds everything while doing nothing. Set `idle_in_transaction_session_timeout` on the application role. +- The monitoring threshold is the canonical one: investigate any transaction open past 1 minute. +- A transaction that spans an HTTP call inherits that service's worst-case latency as its lock duration. + +## Where Transaction Boundaries Belong + +- One transaction per unit of business meaning. Framework "transaction per request" middleware makes every slow template render part of the transaction. +- Do not open a transaction to run a single statement — it is already atomic. +- Read-only work does not need a transaction unless it needs a consistent multi-statement snapshot; then declare it (`SET TRANSACTION READ ONLY` also lets the engine skip work). +- Nested transactions do not exist in most engines: an inner `BEGIN` is ignored or errors. Partial rollback needs savepoints, and a savepoint per loop iteration has real cost in PostgreSQL (each consumes a subtransaction slot; past 64 open subtransactions per session, other backends pay a lookup penalty). +- DDL inside a transaction is safe in PostgreSQL, SQLite, and SQL Server; MySQL commits implicitly (SKILL.md Traps). diff --git a/workspace-sql/temp.sql b/workspace-sql/temp.sql new file mode 100644 index 0000000..cec9e77 --- /dev/null +++ b/workspace-sql/temp.sql @@ -0,0 +1 @@ +INSERT INTO `dmp_im_stock_in_out_flow_detail` (`id`, `ecid`, `flow_id`, `flow_no`, `type`, `stock_type`, `stock_no`, `stock_id`, `stock_detail_id`, `stock_shelf_id`, `shelf_model_id`, `material_properties`, `material_no`, `factory_code`, `warehouse_code`, `shelf_area_code`, `shelf_code`, `batch_no`, `stock_quantity`, `stock_amount`, `sales_amount`, `no_tax_amount`, `bill_date`, `remark`, `stock_remark`, `source_bill_no`, `source_row_index`, `production_task_no`, `sales_order_no_list`, `order_detail_row_index`, `purchase_no`, `mrp_no`, `supplier_code`, `stock_date`, `is_deleted`, `create_time`, `created_by`, `update_time`, `updated_by`, `version`, `pro_batch_no`, `auxiliary_stock_quantity`, `inventory_type_code`, `product_request`, `mat_trans_form_no`, `no_tax_price`, `ial_stock_no`, `dept_code`, `tax_rate`, `invoice_no`, `om_gift_tag`, `factory_code_in_out`, `warehouse_code_in_out`, `shelf_area_code_in_out`, `shelf_code_in_out`, `allocation_no`, `allocation_row_index`, `cust_code`, `aggregate_model`, `group_flag`, `work_shop_code`, `purchase_type`, `order_no_str`) VALUES ('447560004020091209', 'kunzhou', '447560004020091208', 'SIO20260727000073', 'PRODUCTION_IN', 'in', 'PI2607260007', '447166889270589005', '447166889312532191', '447559998936594770', 'KUNZHOU-SMDM-1', 'GH032-16-0006', 'KZ', 'KZCK-003', NULL, NULL, 'MSN2607260024', 2.000000, 0.000000, 0.000000, 0.000000, '2026-07-26', '', NULL, 'GD2607220030-GX-JG11-1', NULL, 'KZGM20260716001_005', NULL, NULL, NULL, 'PLAN202607221255024999150', 'JG', '2026-07-27', 0, '2026-07-27 09:34:47', '颜美莉', '2026-07-27 09:34:47', '颜美莉', 1, NULL, 0.000000, NULL, NULL, NULL, NULL, 0.000000, NULL, NULL, 0.000000, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 'ORDER12345'); \ No newline at end of file