From f226f9c9536061764a24b8e824e081a74c59c75c Mon Sep 17 00:00:00 2001 From: chengcheng Date: Fri, 17 Jul 2026 16:59:50 +0800 Subject: [PATCH] ci: enforce production-safe database migrations --- .gitea/workflows/ci.yml | 41 ++- .gitea/workflows/release-ci.yml | 8 +- apps/api/cmd/migrate/main.go | 12 + deploy/ci/production-migration-base | 1 + docs/decisions/001-production-cicd.md | 2 + docs/runbooks/production-ci-cd.md | 11 +- scripts/ci-validate-migrations.mjs | 430 ++++++++++++++++++++++++++ tests/ci/migrations-test.sh | 323 +++++++++++++++++++ tests/ci/pipeline-test.sh | 40 ++- 9 files changed, 863 insertions(+), 5 deletions(-) create mode 100644 deploy/ci/production-migration-base create mode 100755 scripts/ci-validate-migrations.mjs create mode 100755 tests/ci/migrations-test.sh diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 7d863f5..faa929c 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -18,6 +18,8 @@ jobs: CI_SERVER_URL: ${{ github.server_url }} CI_SHA: ${{ github.sha }} CI_JOB_TOKEN: ${{ github.token }} + CI_EVENT_BEFORE: ${{ github.event.before }} + CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} run: | set -eu test -n "$CI_JOB_TOKEN" @@ -25,9 +27,21 @@ jobs: git init . git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \ fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" "$CI_SHA" + for comparison_sha in "$CI_EVENT_BEFORE" "$CI_PR_BASE_SHA"; do + case "$comparison_sha" in + [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*) + if test "${#comparison_sha}" -eq 40 && \ + test "$comparison_sha" != 0000000000000000000000000000000000000000; then + git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \ + fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" "$comparison_sha" + fi + ;; + esac + done unset authorization CI_JOB_TOKEN test ! -f .git/shallow - git checkout --detach FETCH_HEAD + git checkout --detach "$CI_SHA" + test "$(git rev-parse HEAD)" = "$CI_SHA" - name: Verify pinned host toolchains run: | go version @@ -37,6 +51,27 @@ jobs: shellcheck --version trivy --version govulncheck -version + - name: Verify production migration safety + env: + CI_EVENT_NAME: ${{ github.event_name }} + CI_EVENT_BEFORE: ${{ github.event.before }} + CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + production_base=$(cat deploy/ci/production-migration-base) + immutable_base=$CI_EVENT_BEFORE + if test "$CI_EVENT_NAME" = pull_request; then + immutable_base=$CI_PR_BASE_SHA + fi + case "$immutable_base" in + [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*) + test "${#immutable_base}" -eq 40 + test "$immutable_base" != 0000000000000000000000000000000000000000 + ;; + *) exit 1 ;; + esac + git merge-base --is-ancestor "$immutable_base" HEAD + node ./scripts/ci-validate-migrations.mjs \ + "$production_base" "$immutable_base" - name: Verify Go formatting run: | unformatted=$(gofmt -l apps/api) @@ -61,8 +96,10 @@ jobs: docker-compose -f docker-compose.yml config --quiet shellcheck scripts/ci-build-images.sh scripts/ci-validate-semver.sh \ scripts/provision-ci-runner.sh tests/ci/ci-build-images-test.sh \ - tests/ci/pipeline-test.sh tests/ci/semver-test.sh + tests/ci/migrations-test.sh tests/ci/pipeline-test.sh \ + tests/ci/semver-test.sh ./tests/ci/ci-build-images-test.sh + ./tests/ci/migrations-test.sh ./tests/ci/pipeline-test.sh ./tests/ci/semver-test.sh - name: Scan repository diff --git a/.gitea/workflows/release-ci.yml b/.gitea/workflows/release-ci.yml index 807049e..c73b2c9 100644 --- a/.gitea/workflows/release-ci.yml +++ b/.gitea/workflows/release-ci.yml @@ -53,6 +53,10 @@ jobs: test ! -f .git/shallow test "$(git rev-parse HEAD)" = "$CI_SHA" git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main + - name: Verify production migration safety + run: | + production_base=$(cat deploy/ci/production-migration-base) + node ./scripts/ci-validate-migrations.mjs "$production_base" - name: Verify Go formatting run: | unformatted=$(gofmt -l apps/api) @@ -77,8 +81,10 @@ jobs: docker-compose -f docker-compose.yml config --quiet shellcheck scripts/ci-build-images.sh scripts/ci-validate-semver.sh \ scripts/provision-ci-runner.sh tests/ci/ci-build-images-test.sh \ - tests/ci/pipeline-test.sh tests/ci/semver-test.sh + tests/ci/migrations-test.sh tests/ci/pipeline-test.sh \ + tests/ci/semver-test.sh ./tests/ci/ci-build-images-test.sh + ./tests/ci/migrations-test.sh ./tests/ci/pipeline-test.sh ./tests/ci/semver-test.sh - name: Scan repository diff --git a/apps/api/cmd/migrate/main.go b/apps/api/cmd/migrate/main.go index 1a4d4b9..fe5a9f8 100644 --- a/apps/api/cmd/migrate/main.go +++ b/apps/api/cmd/migrate/main.go @@ -24,6 +24,10 @@ func main() { os.Exit(1) } defer conn.Close(ctx) + if _, err := conn.Exec(ctx, "SET standard_conforming_strings = on"); err != nil { + logger.Error("enforce standard SQL string semantics failed", "error", err) + os.Exit(1) + } if _, err := conn.Exec(ctx, ` CREATE TABLE IF NOT EXISTS schema_migrations ( @@ -64,6 +68,14 @@ CREATE TABLE IF NOT EXISTS schema_migrations ( logger.Error("begin migration failed", "version", version, "error", err) os.Exit(1) } + // Pin string parsing semantics inside every migration transaction. A + // previous migration may have changed the session GUC, while each file is + // parsed and executed independently. + if _, err := tx.Exec(ctx, "SET LOCAL standard_conforming_strings = on"); err != nil { + _ = tx.Rollback(ctx) + logger.Error("enforce migration SQL string semantics failed", "version", version, "error", err) + os.Exit(1) + } if _, err := tx.Exec(ctx, string(sqlBytes)); err != nil { _ = tx.Rollback(ctx) logger.Error("execute migration failed", "version", version, "error", err) diff --git a/deploy/ci/production-migration-base b/deploy/ci/production-migration-base new file mode 100644 index 0000000..31e1d52 --- /dev/null +++ b/deploy/ci/production-migration-base @@ -0,0 +1 @@ +2d6c16fec0bec9c0288e5cb142b458af982fff8f diff --git a/docs/decisions/001-production-cicd.md b/docs/decisions/001-production-cicd.md index 9d4d9b6..d41a7d7 100644 --- a/docs/decisions/001-production-cicd.md +++ b/docs/decisions/001-production-cicd.md @@ -22,6 +22,7 @@ AI Gateway 已通过独立部署仓库和 Docker Compose 运行在 `110.42.51.33 - Runner 外层固定为 `gitea/runner:2.0.0-dind-rootless` 的指定 digest；Job 固定为 `node:24.16.0-bookworm` 的指定 digest。外层按官方 rootless DinD 要求使用 `--privileged`，但内层 Docker daemon 和 Runner 以 UID 1000 运行。 - 不把宿主或内层 Docker socket 挂入 Job。Runner 配置使用 `docker_host: "-"`、`privileged: false`，工作流唯一允许的 bind source 是只读 `/opt/easyai-gateway-ci`，其中提供固定 Go、Node、pnpm、ShellCheck、Compose、Trivy 和 govulncheck 工具链。ShellCheck 与 Compose 使用官方静态发行物和固定 SHA-256，不复制宿主的可变插件或在 Job 中临时安装。RootlessKit 强制 `--pidns`，即使 PR 传入 `--pid=host`，Job 也不能进入外层 Runner 的 PID namespace；provision 会实测 namespace inode。 - CI 只执行格式、静态检查、单元测试、依赖审计、Compose 校验、仓库扫描和 CI 自测；不构建发布镜像，不调用 `sudo`，不执行生产发布助手。 +- PR、`main` 与 Tag CI 都把 `deploy/ci/production-migration-base` 中的当前生产提交作为数据库兼容基线：已经存在于生产基线的迁移不得修改或删除，新迁移静态拒绝 DROP、TRUNCATE、DELETE、MERGE、CREATE OR REPLACE、RENAME、列类型/非空收紧等 contract 操作。PR/`main` 还使用事件中不可变的 base/before SHA 比对，确保基线更新稍有滞后时，已经进入 `main` 的新迁移仍不可被改写。 - 版本 Tag 必须是完整 SemVer，且目标提交必须属于 `main` 历史。该检查是发布前置证据，但本身不授予生产权限。 - Runner 注册状态保存在专用 Docker named volume，首次注册 Token 只传给短命注册容器；长期 systemd 服务不保存 Token，也不加入宿主 `docker`/`sudo` 用户组。 @@ -66,3 +67,4 @@ rootless DinD 显著缩小了 Job 到宿主 Docker/root 的直接通路，但不 - 创建受保护版本 Tag 后，部署仓 dispatcher 必须同时取得同 SHA 的 `main` context 和独立 Tag context，不能把旧 `main` 成功状态误当作本次 Tag 验证。 - 首次安装会清理本项目专属 Buildx 缓存并执行 8 GiB 前置、4 GiB 后置磁盘门禁，不会全局 prune 共享 Docker 状态。 - 首次启用 CD 仍需捕获当前运行镜像作为 bootstrap 回滚基线，并保证数据库迁移对上一应用版本向后兼容。 +- 每次生产发布和恢复演练成功后，必须用单独受审 PR 把 `deploy/ci/production-migration-base` 前移到刚刚部署的完整源码 SHA；不得在发布成功前前移。静态门禁不能证明任意数据变换可逆，生产发布仍必须先做可恢复备份并定期执行真实 `pg_restore` 演练。 diff --git a/docs/runbooks/production-ci-cd.md b/docs/runbooks/production-ci-cd.md index c2f3693..cee74b0 100644 --- a/docs/runbooks/production-ci-cd.md +++ b/docs/runbooks/production-ci-cd.md @@ -12,7 +12,7 @@ - 生产入口：`https://ai.51easyai.com` - 部署目录：`/root/easyai-ai-gateway-deploy` -源码仓只执行质量 CI：PR context 是 `ci / verify (pull_request)`，`main` context 是 `ci / verify (push)`，版本 Tag 使用独立 workflow/context `release-ci / verify-tag (push)`。三者都执行 Go 格式/vet/test/govulncheck、前端 lint/test/build、依赖审计、Compose 校验、Trivy 仓库扫描和 CI 脚本自测。Job 没有 Docker socket、`sudo` 或部署目录权限，也不构建发布镜像。 +源码仓只执行质量 CI：PR context 是 `ci / verify (pull_request)`，`main` context 是 `ci / verify (push)`，版本 Tag 使用独立 workflow/context `release-ci / verify-tag (push)`。三者都执行生产迁移兼容门禁、Go 格式/vet/test/govulncheck、前端 lint/test/build、依赖审计、Compose 校验、Trivy 仓库扫描和 CI 脚本自测。Job 没有 Docker socket、`sudo` 或部署目录权限，也不构建发布镜像。 生产 CD 由部署仓的 root-owned dispatcher 执行。它在相同源码 SHA 上同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)` 成功，然后只运行部署仓中固定的 BuildKit 构建、Trivy 镜像扫描、digest 发布、备份、迁移、健康检查和回滚命令；不得执行 Tag checkout 中的 `pnpm`、测试、shell 脚本或应用程序。 @@ -87,16 +87,25 @@ docker exec easyai-gateway-ci-v2-runner docker image inspect \ 3. 合法 SemVer Tag 新产生独立 `release-ci / verify-tag (push)`；非法或不属于 `main` 历史的 Tag 失败，且 Tag context 不能被旧 `main` context 替代。 4. Fork PR 默认不自动调度，维护者先检查 `.gitea/workflows/**` 和容器选项后再批准。 +PR 与 `main` Push workflow 都校验事件携带的 base/before SHA 必须是触发 SHA 的祖先；分支落后或主干历史被改写时 CI 会 fail-closed。PR 先合并最新 `main` 再重跑，Gitea 分支保护同时启用 `block_on_outdated_branch`。 + ## Fork PR 残余风险 rootless DinD 不是 VM 沙箱。外层仍按官方要求使用 `--privileged`，Gitea runner 也会解析 PR 控制的 `jobs..container.options`。配置已禁止 Job privileged、宿主/内层 Docker socket 和任意 bind source，但 Runner、rootless Docker、容器运行时或内核漏洞仍可能突破边界；Job 的出站网络和资源消耗也不是强租户隔离。因此 Fork PR 必须维护者批准，CI 不得注入生产 Secret；未来应迁移到独立 CI 主机或虚拟机。 +## 数据库迁移兼容门禁 + +`deploy/ci/production-migration-base` 必须等于当前生产 API 对应的完整源码 SHA。`scripts/ci-validate-migrations.mjs` 只允许新增、按文件名严格追加且为普通文件的迁移；生产已知迁移以及已经进入 `main` 的迁移不可修改或删除。DROP、TRUNCATE、DELETE、MERGE、CREATE OR REPLACE、动态 EXECUTE、RENAME、列类型变更、增加/设置 `NOT NULL`、删除默认值和 REVOKE 会直接使 CI 失败。 + +该规则允许 INSERT/UPDATE 等数据迁移，因此不能替代备份和恢复验证。每次生产发布成功并完成 `pg_restore` 演练后，创建一个只前移 `deploy/ci/production-migration-base` 的受审 PR；不要把基线指向尚未成功部署的提交。 + ## 发布生产版本 1. 合并到受保护 `main`，确认源码 CI 成功。 2. 在已验证提交上创建完整 SemVer Tag 并推送。 3. 确认同一 SHA 的 `ci / verify (push)` 和本次 `release-ci / verify-tag (push)` 都成功。 4. 由部署仓 root-owned dispatcher 构建、扫描并发布 digest；源码仓工作流不会直接调用生产 helper。 +5. 发布、健康检查和恢复演练完成后，用单独 PR 把 `deploy/ci/production-migration-base` 更新为该生产 Tag 的完整源码 SHA。 ```bash git switch main diff --git a/scripts/ci-validate-migrations.mjs b/scripts/ci-validate-migrations.mjs new file mode 100755 index 0000000..0911cc0 --- /dev/null +++ b/scripts/ci-validate-migrations.mjs @@ -0,0 +1,430 @@ +#!/usr/bin/env node + +import { execFileSync, spawnSync } from 'node:child_process' +import { lstatSync, readFileSync } from 'node:fs' +import { basename } from 'node:path' + +const migrationDirectory = 'apps/api/migrations/' +const baselineFile = 'deploy/ci/production-migration-base' +const base = process.argv[2] || process.env.PRODUCTION_MIGRATION_BASE_SHA || '' +const immutableRef = + process.argv[3] || process.env.MIGRATION_IMMUTABILITY_REF || '' + +function fail(message) { + console.error(`migration_safety=FAIL ${message}`) + process.exit(1) +} + +function git(args, options = {}) { + try { + return execFileSync('git', args, { + encoding: options.encoding ?? 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }) + } catch (error) { + const stderr = error.stderr?.toString().trim() + fail(stderr ? `git_error=${JSON.stringify(stderr)}` : 'git command failed') + } +} + +if (!/^[0-9a-f]{40}$/.test(base)) { + fail('PRODUCTION_MIGRATION_BASE_SHA must be a full lowercase commit SHA') +} + +const root = git(['rev-parse', '--show-toplevel']).trim() +process.chdir(root) + +const resolvedBase = git([ + 'rev-parse', + '--verify', + '--end-of-options', + `${base}^{commit}`, +]).trim() +if (resolvedBase !== base) { + fail('production migration baseline did not resolve exactly') +} + +const ancestry = spawnSync('git', ['merge-base', '--is-ancestor', base, 'HEAD'], { + stdio: 'ignore', +}) +if (ancestry.status !== 0) { + fail('production migration baseline is not an ancestor of HEAD') +} + +const head = git(['rev-parse', '--verify', 'HEAD^{commit}']).trim() + +const baselineTreeEntry = git(['ls-tree', 'HEAD', '--', baselineFile]).trim() +let baselineState +try { + baselineState = lstatSync(baselineFile) +} catch { + fail('production migration baseline file is unavailable in the worktree') +} +if ( + !/^100644 blob [0-9a-f]+\t/.test(baselineTreeEntry) || + !baselineState.isFile() || + baselineState.isSymbolicLink() +) { + fail('production migration baseline must be a mode 100644 regular file') +} +if (readFileSync(baselineFile, 'utf8').trim() !== base) { + fail('production migration baseline argument does not match the reviewed file') +} + +function changedMigrations(from) { + const rawChanges = git( + [ + 'diff', + '--name-status', + '-z', + '--no-renames', + `${from}..HEAD`, + '--', + migrationDirectory, + ], + { encoding: null }, + ) + const fields = rawChanges.toString('utf8').split('\0') + if (fields.at(-1) === '') fields.pop() + if (fields.length % 2 !== 0) { + fail('could not parse changed migration paths') + } + return fields +} + +const fields = changedMigrations(base) + +function executableSql(sql, source = 'SQL input') { + let output = '' + + for (let index = 0; index < sql.length; index += 1) { + const current = sql[index] + const next = sql[index + 1] + + if (current === '-' && next === '-') { + index += 2 + while (index < sql.length && sql[index] !== '\n') index += 1 + output += '\n' + continue + } + + if (current === '/' && next === '*') { + let depth = 1 + index += 2 + while (index < sql.length && depth > 0) { + if (sql[index] === '/' && sql[index + 1] === '*') { + depth += 1 + index += 2 + continue + } + if (sql[index] === '*' && sql[index + 1] === '/') { + depth -= 1 + index += 2 + continue + } + index += 1 + } + if (depth !== 0) fail('unterminated SQL block comment') + index -= 1 + output += ' ' + continue + } + + if (current === "'" || current === '"') { + const prefix = sql.slice(Math.max(0, index - 2), index).toUpperCase() + const previous = sql[index - 1]?.toUpperCase() + const beforePrevious = sql[index - 2] || '' + const beforeUnicodePrefix = sql[index - 3] || '' + const escapeString = + current === "'" && + previous === 'E' && + !/[A-Za-z0-9_$]/.test(beforePrevious) + const unicodeString = + prefix === 'U&' && !/[A-Za-z0-9_$]/.test(beforeUnicodePrefix) + if (escapeString || unicodeString) { + fail(`${source}: PostgreSQL escape/unicode strings are not allowed in migrations`) + } + const quote = current + index += 1 + let closed = false + while (index < sql.length) { + if (sql[index] === quote && sql[index + 1] === quote) { + index += 2 + continue + } + if (sql[index] === quote) { + closed = true + break + } + index += 1 + } + if (!closed) fail('unterminated SQL quoted value') + output += ' ' + continue + } + + if (current === '$') { + const delimiter = sql.slice(index).match(/^\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/)?.[0] + if (delimiter) { + const bodyStart = index + delimiter.length + const bodyEnd = sql.indexOf(delimiter, bodyStart) + if (bodyEnd === -1) fail('unterminated SQL dollar-quoted body') + // PostgreSQL function bodies may contain executable DDL. Scan their + // contents recursively instead of treating them as inert strings. + const bodySql = executableSql(sql.slice(bodyStart, bodyEnd), source) + // A bare END terminates a PL/pgSQL block inside a dollar-quoted body, + // but at the top level PostgreSQL treats END as COMMIT. Mask only the + // block terminator here so the top-level transaction rule stays exact. + const bodyWithoutBlockEnd = bodySql + .split(';') + .map((statement) => (/^\s*END\s*$/i.test(statement) ? ' ' : statement)) + .join(';') + output += ` ${bodyWithoutBlockEnd} ` + index = bodyEnd + delimiter.length - 1 + continue + } + } + + output += current + } + + return output +} + +const destructiveRules = [ + { + name: 'dynamic SQL execution', + pattern: /\bEXECUTE\b/i, + }, + { + name: 'destructive DROP operation', + pattern: /\bDROP\b/i, + }, + { name: 'MERGE operation', pattern: /\bMERGE\b/i }, + { + name: 'CREATE OR REPLACE operation', + pattern: /\bCREATE\s+OR\s+REPLACE\b/i, + }, + { name: 'TRUNCATE operation', pattern: /\bTRUNCATE(?:\s+TABLE)?\b/i }, + { name: 'DELETE FROM operation', pattern: /\bDELETE\s+FROM\b/i }, + { name: 'permission revocation', pattern: /\bREVOKE\b/i }, + { + name: 'transaction control operation', + pattern: + /\b(?:COMMIT|ROLLBACK|ABORT|SAVEPOINT)\b|\bRELEASE\s+SAVEPOINT\b|\bPREPARE\s+TRANSACTION\b|^\s*END(?:\s+(?:WORK|TRANSACTION))?(?:\s+AND\s+(?:NO\s+)?CHAIN)?\s*$/i, + }, + { + name: 'SQL string semantics override', + pattern: + /\bSET\s+(?:LOCAL\s+|SESSION\s+)?STANDARD_CONFORMING_STRINGS\b/i, + }, + { + name: 'object or column rename', + pattern: /\bRENAME\b/i, + }, + { name: 'SET SCHEMA operation', pattern: /\bSET\s+SCHEMA\b/i }, + { + name: 'column type change', + pattern: + /\bALTER\s+TABLE\b[\s\S]*\bALTER\s+(?:COLUMN\s+)?[\s\S]*\b(?:SET\s+DATA\s+)?TYPE\b/i, + }, + { + name: 'SET NOT NULL operation', + pattern: + /\bALTER\s+TABLE\b[\s\S]*\bALTER\s+(?:COLUMN\s+)?[\s\S]*\bSET\s+NOT\s+NULL\b/i, + }, + { + name: 'non-null column addition', + pattern: + /\bALTER\s+TABLE\b[\s\S]*\bADD\s+(?:COLUMN\s+)?[\s\S]*\bNOT\s+NULL\b/i, + }, + { + name: 'DROP DEFAULT operation', + pattern: + /\bALTER\s+TABLE\b[\s\S]*\bALTER\s+(?:COLUMN\s+)?[\s\S]*\bDROP\s+DEFAULT\b/i, + }, +] + +const violations = [] +let addedFiles = 0 +let immutableBase = '' +const orderingCandidates = new Set() + +if (immutableRef) { + if ( + !/^[0-9a-f]{40}$/.test(immutableRef) && + immutableRef !== 'refs/remotes/origin/main' + ) { + fail('MIGRATION_IMMUTABILITY_REF must be a full commit SHA or refs/remotes/origin/main') + } + + const immutableTip = git([ + 'rev-parse', + '--verify', + '--end-of-options', + `${immutableRef}^{commit}`, + ]).trim() + const baseAtEvent = spawnSync( + 'git', + ['show', `${immutableTip}:${baselineFile}`], + { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }, + ) + if (baseAtEvent.status === 0) { + const previousBaselineTreeEntry = git([ + 'ls-tree', + immutableTip, + '--', + baselineFile, + ]).trim() + if (!/^100644 blob [0-9a-f]+\t/.test(previousBaselineTreeEntry)) { + fail('previous production migration baseline must be a mode 100644 regular file') + } + const previousProductionBase = baseAtEvent.stdout.trim() + if (!/^[0-9a-f]{40}$/.test(previousProductionBase)) { + fail('previous production migration baseline is invalid') + } + const baselineAdvance = spawnSync( + 'git', + ['merge-base', '--is-ancestor', previousProductionBase, base], + { stdio: 'ignore' }, + ) + if (baselineAdvance.status !== 0) { + fail('production migration baseline must only move forward') + } + } + const baselineWasAlreadyReviewed = spawnSync( + 'git', + ['merge-base', '--is-ancestor', base, immutableTip], + { stdio: 'ignore' }, + ) + if (baselineWasAlreadyReviewed.status !== 0) { + fail('production migration baseline must belong to the immutable event base history') + } + if (immutableTip === head) { + const parent = spawnSync( + 'git', + ['rev-parse', '--verify', `${immutableTip}^1^{commit}`], + { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }, + ) + immutableBase = parent.status === 0 ? parent.stdout.trim() : '' + } else { + const immutableAncestry = spawnSync( + 'git', + ['merge-base', '--is-ancestor', immutableTip, head], + { stdio: 'ignore' }, + ) + if (immutableAncestry.status !== 0) { + fail('migration immutability base is not an ancestor of HEAD') + } + immutableBase = immutableTip + } + + if (immutableBase) { + const immutableChanges = changedMigrations(immutableBase) + for (let index = 0; index < immutableChanges.length; index += 2) { + const status = immutableChanges[index] + const file = immutableChanges[index + 1] + if (status === 'A') { + orderingCandidates.add(file) + } else { + violations.push( + `${file}: migration present on main is immutable (status ${status})`, + ) + } + } + } +} + +function migrationNamesAt(ref) { + const output = git( + ['ls-tree', '-r', '-z', '--name-only', ref, '--', migrationDirectory], + { encoding: null }, + ) + return output + .toString('utf8') + .split('\0') + .filter(Boolean) + .filter((file) => { + const relative = file.slice(migrationDirectory.length) + return file.startsWith(migrationDirectory) && !relative.includes('/') + }) + .map((file) => basename(file)) + .filter((name) => /^\d{4}_[a-z0-9][a-z0-9_]*\.sql$/.test(name)) + .sort() +} + +const orderingBase = immutableBase || base +const existingNames = migrationNamesAt(orderingBase) +const previousMaximumName = existingNames.at(-1) || '' + +if (!immutableBase) { + for (let index = 0; index < fields.length; index += 2) { + if (fields[index] === 'A') orderingCandidates.add(fields[index + 1]) + } +} + +for (let index = 0; index < fields.length; index += 2) { + const status = fields[index] + const file = fields[index + 1] + + if (status !== 'A') { + violations.push(`${file}: applied migrations are immutable (status ${status})`) + continue + } + + addedFiles += 1 + const filename = basename(file) + if ( + file !== `${migrationDirectory}${filename}` || + !/^\d{4}_[a-z0-9][a-z0-9_]*\.sql$/.test(filename) + ) { + violations.push(`${file}: new migration must use NNNN_lowercase_name.sql`) + continue + } + + if ( + orderingCandidates.has(file) && + previousMaximumName && + filename <= previousMaximumName + ) { + violations.push( + `${file}: new migration name must sort after ${previousMaximumName}`, + ) + continue + } + + const treeEntry = git(['ls-tree', 'HEAD', '--', file]).trim() + let fileState + try { + fileState = lstatSync(file) + } catch { + violations.push(`${file}: new migration is unavailable in the worktree`) + continue + } + if ( + !/^100644 blob [0-9a-f]+\t/.test(treeEntry) || + !fileState.isFile() || + fileState.isSymbolicLink() + ) { + violations.push(`${file}: new migration must be a mode 100644 regular file`) + continue + } + + const statements = executableSql(readFileSync(file, 'utf8'), file).split(';') + for (const statement of statements) { + for (const rule of destructiveRules) { + if (rule.pattern.test(statement)) { + violations.push(`${file}: ${rule.name}`) + } + } + } +} + +if (violations.length > 0) { + for (const violation of [...new Set(violations)]) { + console.error(`migration_safety_violation=${JSON.stringify(violation)}`) + } + fail(`base=${base} violations=${new Set(violations).size}`) +} + +console.log(`migration_safety=PASS base=${base} added_files=${addedFiles}`) diff --git a/tests/ci/migrations-test.sh b/tests/ci/migrations-test.sh new file mode 100755 index 0000000..1988660 --- /dev/null +++ b/tests/ci/migrations-test.sh @@ -0,0 +1,323 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +validator=$root/scripts/ci-validate-migrations.mjs +fixture_root=$(mktemp -d) +trap 'rm -rf "$fixture_root"' EXIT + +new_repo() { + local name=$1 + local repo=$fixture_root/$name + + mkdir -p "$repo/apps/api/migrations" + git -C "$repo" init -q + git -C "$repo" config user.name 'CI Migration Test' + git -C "$repo" config user.email 'ci-migration-test@example.invalid' + printf 'CREATE TABLE accounts (id bigint PRIMARY KEY);\n' > \ + "$repo/apps/api/migrations/0001_init.sql" + git -C "$repo" add apps/api/migrations/0001_init.sql + git -C "$repo" commit -qm baseline + mkdir -p "$repo/deploy/ci" + git -C "$repo" rev-parse HEAD > \ + "$repo/deploy/ci/production-migration-base" + printf '%s\n' "$repo" +} + +commit_all() { + local repo=$1 + local message=$2 + git -C "$repo" add -A + git -C "$repo" commit -qm "$message" +} + +expect_pass() { + local repo=$1 + local base=$2 + local immutable_base=${3:-} + ( + cd "$repo" + if [[ -n $immutable_base ]]; then + node "$validator" "$base" "$immutable_base" + else + node "$validator" "$base" + fi + ) +} + +expect_fail() { + local repo=$1 + local base=$2 + local expected=$3 + local immutable_base=${4:-} + local output status + + if [[ -n $immutable_base ]]; then + set +e + output=$(cd "$repo" && node "$validator" "$base" "$immutable_base" 2>&1) + status=$? + set -e + else + set +e + output=$(cd "$repo" && node "$validator" "$base" 2>&1) + status=$? + set -e + fi + if [[ $status -eq 0 ]]; then + printf 'migration validator unexpectedly passed: %s\n' "$expected" >&2 + exit 1 + fi + grep -Fq "$expected" <<<"$output" || { + printf 'missing expected migration failure %q in:\n%s\n' "$expected" "$output" >&2 + exit 1 + } +} + +repo=$(new_repo safe) +base=$(git -C "$repo" rev-parse HEAD) +cat >"$repo/apps/api/migrations/0002_expand_accounts.sql" <<'SQL' +-- A comment mentioning DROP TABLE must not trigger the gate. +ALTER TABLE accounts ADD COLUMN display_name text; +CREATE TABLE audit_events (id bigint PRIMARY KEY, note text); +INSERT INTO audit_events (id, note) VALUES (1, 'DROP TABLE is documentation'); +UPDATE accounts SET display_name = 'ready' WHERE display_name IS NULL; +SQL +commit_all "$repo" safe +expect_pass "$repo" "$base" + +repo=$(new_repo drop) +base=$(git -C "$repo" rev-parse HEAD) +printf 'DROP\nTABLE accounts;\n' >"$repo/apps/api/migrations/0002_drop.sql" +commit_all "$repo" drop +expect_fail "$repo" "$base" 'destructive DROP operation' + +repo=$(new_repo delete) +base=$(git -C "$repo" rev-parse HEAD) +printf 'DELETE FROM accounts;\n' >"$repo/apps/api/migrations/0002_delete.sql" +commit_all "$repo" delete +expect_fail "$repo" "$base" 'DELETE FROM operation' + +repo=$(new_repo incompatible_alter) +base=$(git -C "$repo" rev-parse HEAD) +cat >"$repo/apps/api/migrations/0002_alter.sql" <<'SQL' +ALTER TABLE accounts ADD COLUMN external_id text NOT NULL; +ALTER TABLE accounts ALTER COLUMN id TYPE numeric; +SQL +commit_all "$repo" alter +expect_fail "$repo" "$base" 'non-null column addition' +expect_fail "$repo" "$base" 'column type change' + +repo=$(new_repo dynamic_sql) +base=$(git -C "$repo" rev-parse HEAD) +cat >"$repo/apps/api/migrations/0002_dynamic.sql" <<'SQL' +DO $$ +BEGIN + EXECUTE 'DROP ' || 'TABLE accounts'; +END +$$; +SQL +commit_all "$repo" dynamic +expect_fail "$repo" "$base" 'dynamic SQL execution' + +repo=$(new_repo escape_string) +base=$(git -C "$repo" rev-parse HEAD) +cat >"$repo/apps/api/migrations/0002_escape.sql" <<'SQL' +SELECT E'foo\'bar'; DROP TABLE accounts; --'; +SQL +commit_all "$repo" escape +expect_fail "$repo" "$base" 'PostgreSQL escape/unicode strings are not allowed' + +repo=$(new_repo string_semantics_override) +base=$(git -C "$repo" rev-parse HEAD) +printf 'SET standard_conforming_strings = off;\n' > \ + "$repo/apps/api/migrations/0002_string_mode.sql" +commit_all "$repo" string_mode +expect_fail "$repo" "$base" 'SQL string semantics override' + +repo=$(new_repo transaction_control) +base=$(git -C "$repo" rev-parse HEAD) +cat >"$repo/apps/api/migrations/0002_commit.sql" <<'SQL' +ALTER TABLE accounts ADD COLUMN display_name text; +COMMIT; +UPDATE accounts SET display_name = 'partially committed'; +SQL +commit_all "$repo" transaction_control +expect_fail "$repo" "$base" 'transaction control operation' + +repo=$(new_repo bare_end_transaction) +base=$(git -C "$repo" rev-parse HEAD) +cat >"$repo/apps/api/migrations/0002_end.sql" <<'SQL' +ALTER TABLE accounts ADD COLUMN display_name text; +END; +UPDATE accounts SET display_name = 'partially committed'; +SQL +commit_all "$repo" bare_end_transaction +expect_fail "$repo" "$base" 'transaction control operation' + +repo=$(new_repo plpgsql_block_end) +base=$(git -C "$repo" rev-parse HEAD) +cat >"$repo/apps/api/migrations/0002_function.sql" <<'SQL' +CREATE FUNCTION account_count() RETURNS bigint +LANGUAGE plpgsql AS $$ +BEGIN + RETURN (SELECT count(*) FROM accounts); +END; +$$; +SQL +commit_all "$repo" plpgsql_block_end +expect_pass "$repo" "$base" + +repo=$(new_repo ordinary_string_boundary) +base=$(git -C "$repo" rev-parse HEAD) +cat >"$repo/apps/api/migrations/0002_string_boundary.sql" <<'SQL' +SELECT '\'; DROP TABLE accounts; --'; +SQL +commit_all "$repo" ordinary_string +expect_fail "$repo" "$base" 'destructive DROP operation' + +repo=$(new_repo unicode_string) +base=$(git -C "$repo" rev-parse HEAD) +printf "SELECT U&'d\\0061ta';\n" >"$repo/apps/api/migrations/0002_unicode.sql" +commit_all "$repo" unicode +expect_fail "$repo" "$base" 'PostgreSQL escape/unicode strings are not allowed' + +repo=$(new_repo merge_delete) +base=$(git -C "$repo" rev-parse HEAD) +cat >"$repo/apps/api/migrations/0002_merge.sql" <<'SQL' +MERGE INTO accounts USING stale_accounts ON accounts.id = stale_accounts.id +WHEN MATCHED THEN DELETE; +SQL +commit_all "$repo" merge +expect_fail "$repo" "$base" 'MERGE operation' + +repo=$(new_repo replace_view) +base=$(git -C "$repo" rev-parse HEAD) +printf 'CREATE OR REPLACE VIEW active_accounts AS SELECT id FROM accounts;\n' > \ + "$repo/apps/api/migrations/0002_replace.sql" +commit_all "$repo" replace +expect_fail "$repo" "$base" 'CREATE OR REPLACE operation' + +repo=$(new_repo rename_function) +base=$(git -C "$repo" rev-parse HEAD) +printf 'ALTER FUNCTION calculate_total() RENAME TO calculate_total_legacy;\n' > \ + "$repo/apps/api/migrations/0002_rename.sql" +commit_all "$repo" rename +expect_fail "$repo" "$base" 'object or column rename' + +repo=$(new_repo set_schema) +base=$(git -C "$repo" rev-parse HEAD) +printf 'ALTER TABLE accounts SET SCHEMA archive;\n' > \ + "$repo/apps/api/migrations/0002_schema.sql" +commit_all "$repo" schema +expect_fail "$repo" "$base" 'SET SCHEMA operation' + +repo=$(new_repo drop_domain) +base=$(git -C "$repo" rev-parse HEAD) +printf 'DROP DOMAIN account_code CASCADE;\n' > \ + "$repo/apps/api/migrations/0002_drop_domain.sql" +commit_all "$repo" drop_domain +expect_fail "$repo" "$base" 'destructive DROP operation' + +repo=$(new_repo immutable) +base=$(git -C "$repo" rev-parse HEAD) +printf '\n-- retroactive edit\n' >>"$repo/apps/api/migrations/0001_init.sql" +commit_all "$repo" edit +expect_fail "$repo" "$base" 'applied migrations are immutable (status M)' + +repo=$(new_repo immutable_after_baseline) +base=$(git -C "$repo" rev-parse HEAD) +printf 'ALTER TABLE accounts ADD COLUMN nickname text;\n' > \ + "$repo/apps/api/migrations/0002_nickname.sql" +commit_all "$repo" add +immutable_base=$(git -C "$repo" rev-parse HEAD) +printf '\nUPDATE accounts SET nickname = NULL;\n' >> \ + "$repo/apps/api/migrations/0002_nickname.sql" +commit_all "$repo" modify +output=$( + cd "$repo" + node "$validator" "$base" "$immutable_base" 2>&1 || true +) +grep -Fq 'migration present on main is immutable (status M)' <<<"$output" || { + printf 'stale production baseline did not preserve main migration immutability:\n%s\n' \ + "$output" >&2 + exit 1 +} + +repo=$(new_repo filename) +base=$(git -C "$repo" rev-parse HEAD) +printf 'SELECT 1;\n' >"$repo/apps/api/migrations/next.SQL" +commit_all "$repo" filename +expect_fail "$repo" "$base" 'new migration must use NNNN_lowercase_name.sql' + +repo=$(new_repo ordering) +base=$(git -C "$repo" rev-parse HEAD) +printf 'SELECT 1;\n' >"$repo/apps/api/migrations/0000_late.sql" +commit_all "$repo" ordering +expect_fail "$repo" "$base" 'new migration name must sort after 0001_init.sql' + +repo=$(new_repo stale_baseline) +base=$(git -C "$repo" rev-parse HEAD) +printf 'ALTER TABLE accounts ADD COLUMN nickname text;\n' > \ + "$repo/apps/api/migrations/0002_nickname.sql" +commit_all "$repo" main_migration +immutable_base=$(git -C "$repo" rev-parse HEAD) +printf 'ordinary code change\n' >"$repo/README.md" +commit_all "$repo" code_only +expect_pass "$repo" "$base" "$immutable_base" + +repo=$(new_repo baseline_advance) +production_base=$(git -C "$repo" rev-parse HEAD) +mkdir -p "$repo/deploy/ci" +printf '%s\n' "$production_base" >"$repo/deploy/ci/production-migration-base" +commit_all "$repo" baseline_file +immutable_base=$(git -C "$repo" rev-parse HEAD) +git -C "$repo" switch -q -c release-baseline-update "$immutable_base" +printf '%s\n' "$immutable_base" >"$repo/deploy/ci/production-migration-base" +commit_all "$repo" valid_baseline_advance +expect_pass "$repo" "$immutable_base" "$immutable_base" + +git -C "$repo" switch -q -c malicious-baseline "$immutable_base" +printf 'DROP TABLE accounts;\n' >"$repo/apps/api/migrations/0002_drop.sql" +commit_all "$repo" hidden_destructive_migration +hidden_sha=$(git -C "$repo" rev-parse HEAD) +printf '%s\n' "$hidden_sha" >"$repo/deploy/ci/production-migration-base" +commit_all "$repo" hide_migration_in_baseline +expect_fail "$repo" "$hidden_sha" \ + 'production migration baseline must belong to the immutable event base history' \ + "$immutable_base" + +repo=$(new_repo nested) +base=$(git -C "$repo" rev-parse HEAD) +mkdir -p "$repo/apps/api/migrations/archive" +printf 'SELECT 1;\n' >"$repo/apps/api/migrations/archive/9999_hidden.sql" +commit_all "$repo" nested +expect_fail "$repo" "$base" 'new migration must use NNNN_lowercase_name.sql' + +repo=$(new_repo symlink) +base=$(git -C "$repo" rev-parse HEAD) +printf 'SELECT 1;\n' >"$repo/outside.sql" +ln -s ../../../outside.sql "$repo/apps/api/migrations/0002_symlink.sql" +commit_all "$repo" symlink +expect_fail "$repo" "$base" 'new migration must be a mode 100644 regular file' + +repo=$(new_repo baseline_symlink) +base=$(git -C "$repo" rev-parse HEAD) +printf '%s\n' "$base" >"$repo/baseline-target" +rm "$repo/deploy/ci/production-migration-base" +ln -s ../../baseline-target "$repo/deploy/ci/production-migration-base" +commit_all "$repo" baseline_symlink +expect_fail "$repo" "$base" \ + 'production migration baseline must be a mode 100644 regular file' + +repo=$(new_repo ancestry) +base=$(git -C "$repo" rev-parse HEAD) +git -C "$repo" switch -q --orphan unrelated +printf 'unrelated\n' >"$repo/unrelated" +git -C "$repo" add unrelated +git -C "$repo" commit -qm unrelated +expect_fail "$repo" "$base" 'production migration baseline is not an ancestor of HEAD' + +repo=$(new_repo invalid_sha) +expect_fail "$repo" deadbeef 'must be a full lowercase commit SHA' + +echo 'migration_tests=PASS' diff --git a/tests/ci/pipeline-test.sh b/tests/ci/pipeline-test.sh index f2738bd..ef4b707 100755 --- a/tests/ci/pipeline-test.sh +++ b/tests/ci/pipeline-test.sh @@ -10,22 +10,45 @@ runner_config=$root/deploy/ci/act-runner-v2-config.yaml provision=$root/scripts/provision-ci-runner.sh build_images=$root/scripts/ci-build-images.sh validate_semver=$root/scripts/ci-validate-semver.sh +validate_migrations=$root/scripts/ci-validate-migrations.mjs build_images_test=$root/tests/ci/ci-build-images-test.sh +migrations_test=$root/tests/ci/migrations-test.sh +migration_base=$root/deploy/ci/production-migration-base adr=$root/docs/decisions/001-production-cicd.md runbook=$root/docs/runbooks/production-ci-cd.md +migrator=$root/apps/api/cmd/migrate/main.go -for file in "$workflow" "$release_workflow" "$runner_service" "$runner_config" "$provision" "$build_images" "$validate_semver" "$build_images_test" "$adr" "$runbook"; do +for file in "$workflow" "$release_workflow" "$runner_service" "$runner_config" "$provision" "$build_images" "$validate_semver" "$validate_migrations" "$build_images_test" "$migrations_test" "$migration_base" "$adr" "$runbook" "$migrator"; do [[ -f $file ]] || { printf 'missing CI/CD file: %s\n' "$file" >&2 exit 1 } done +if [[ $(wc -l <"$migration_base") -ne 1 ]]; then + echo 'production migration baseline must contain exactly one line' >&2 + exit 1 +fi +grep -Fq 'SET standard_conforming_strings = on' "$migrator" || { + echo 'database migrator does not pin standard SQL string semantics' >&2 + exit 1 +} +if ! grep -Eq '^[0-9a-f]{40}$' "$migration_base"; then + echo 'production migration baseline must be a full lowercase SHA' >&2 + exit 1 +fi + grep -q '^name: ci$' "$workflow" grep -q '^ push:$' "$workflow" grep -q '^ branches: \[main\]$' "$workflow" grep -q '^ pull_request:$' "$workflow" grep -q '^ verify:$' "$workflow" +grep -Fq 'git checkout --detach "$CI_SHA"' "$workflow" +grep -Fq 'test "$(git rev-parse HEAD)" = "$CI_SHA"' "$workflow" +if grep -Fq 'git checkout --detach FETCH_HEAD' "$workflow"; then + echo 'main/PR CI must not checkout mutable FETCH_HEAD after comparison fetches' >&2 + exit 1 +fi if grep -Eq "^[[:space:]]+tags:|Verify release tag ancestry" "$workflow" || \ grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$workflow"; then echo 'main/PR CI must not share the release-tag context' >&2 @@ -53,6 +76,7 @@ for quality_workflow in "$workflow" "$release_workflow"; do fi grep -Fq 'test ! -f .git/shallow' "$quality_workflow" grep -Fq './tests/ci/ci-build-images-test.sh' "$quality_workflow" + grep -Fq './tests/ci/migrations-test.sh' "$quality_workflow" grep -Fq './tests/ci/pipeline-test.sh' "$quality_workflow" grep -Fq './tests/ci/semver-test.sh' "$quality_workflow" grep -Fq 'docker-compose version' "$quality_workflow" @@ -81,6 +105,7 @@ for quality_workflow in "$workflow" "$release_workflow"; do fi for gate in \ + 'scripts/ci-validate-migrations.mjs' \ 'gofmt -l' \ 'go vet ./...' \ 'go test ./...' \ @@ -103,6 +128,12 @@ for quality_workflow in "$workflow" "$release_workflow"; do "$quality_workflow" >&2 exit 1 } + grep -Fq 'production_base=$(cat deploy/ci/production-migration-base)' \ + "$quality_workflow" || { + printf '%s does not read the versioned production migration baseline\n' \ + "$quality_workflow" >&2 + exit 1 + } done if [[ $(sed -n '/^ - name: Verify Go formatting$/,$p' "$workflow") != \ @@ -111,6 +142,13 @@ if [[ $(sed -n '/^ - name: Verify Go formatting$/,$p' "$workflow") != \ exit 1 fi +grep -Fq 'CI_EVENT_BEFORE: ${{ github.event.before }}' "$workflow" +grep -Fq 'CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}' "$workflow" +grep -Fq 'git merge-base --is-ancestor "$immutable_base" HEAD' "$workflow" +grep -Fq '"$production_base" "$immutable_base"' "$workflow" +grep -Fq 'node ./scripts/ci-validate-migrations.mjs "$production_base"' \ + "$release_workflow" + runner_image='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a' job_image='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7'