ci: enforce production-safe database migrations
All checks were successful
ci / verify (pull_request) Successful in 8m50s

This commit is contained in:
chengcheng 2026-07-17 16:59:50 +08:00
parent 74c20b1976
commit f226f9c953
9 changed files with 863 additions and 5 deletions

View File

@ -18,6 +18,8 @@ jobs:
CI_SERVER_URL: ${{ github.server_url }}
CI_SHA: ${{ github.sha }}
CI_JOB_TOKEN: ${{ github.token }}
CI_EVENT_BEFORE: ${{ github.event.before }}
CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
set -eu
test -n "$CI_JOB_TOKEN"
@ -25,9 +27,21 @@ jobs:
git init .
git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \
fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" "$CI_SHA"
for comparison_sha in "$CI_EVENT_BEFORE" "$CI_PR_BASE_SHA"; do
case "$comparison_sha" in
[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*)
if test "${#comparison_sha}" -eq 40 && \
test "$comparison_sha" != 0000000000000000000000000000000000000000; then
git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \
fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" "$comparison_sha"
fi
;;
esac
done
unset authorization CI_JOB_TOKEN
test ! -f .git/shallow
git checkout --detach FETCH_HEAD
git checkout --detach "$CI_SHA"
test "$(git rev-parse HEAD)" = "$CI_SHA"
- name: Verify pinned host toolchains
run: |
go version
@ -37,6 +51,27 @@ jobs:
shellcheck --version
trivy --version
govulncheck -version
- name: Verify production migration safety
env:
CI_EVENT_NAME: ${{ github.event_name }}
CI_EVENT_BEFORE: ${{ github.event.before }}
CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
production_base=$(cat deploy/ci/production-migration-base)
immutable_base=$CI_EVENT_BEFORE
if test "$CI_EVENT_NAME" = pull_request; then
immutable_base=$CI_PR_BASE_SHA
fi
case "$immutable_base" in
[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*)
test "${#immutable_base}" -eq 40
test "$immutable_base" != 0000000000000000000000000000000000000000
;;
*) exit 1 ;;
esac
git merge-base --is-ancestor "$immutable_base" HEAD
node ./scripts/ci-validate-migrations.mjs \
"$production_base" "$immutable_base"
- name: Verify Go formatting
run: |
unformatted=$(gofmt -l apps/api)
@ -61,8 +96,10 @@ jobs:
docker-compose -f docker-compose.yml config --quiet
shellcheck scripts/ci-build-images.sh scripts/ci-validate-semver.sh \
scripts/provision-ci-runner.sh tests/ci/ci-build-images-test.sh \
tests/ci/pipeline-test.sh tests/ci/semver-test.sh
tests/ci/migrations-test.sh tests/ci/pipeline-test.sh \
tests/ci/semver-test.sh
./tests/ci/ci-build-images-test.sh
./tests/ci/migrations-test.sh
./tests/ci/pipeline-test.sh
./tests/ci/semver-test.sh
- name: Scan repository

View File

@ -53,6 +53,10 @@ jobs:
test ! -f .git/shallow
test "$(git rev-parse HEAD)" = "$CI_SHA"
git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main
- name: Verify production migration safety
run: |
production_base=$(cat deploy/ci/production-migration-base)
node ./scripts/ci-validate-migrations.mjs "$production_base"
- name: Verify Go formatting
run: |
unformatted=$(gofmt -l apps/api)
@ -77,8 +81,10 @@ jobs:
docker-compose -f docker-compose.yml config --quiet
shellcheck scripts/ci-build-images.sh scripts/ci-validate-semver.sh \
scripts/provision-ci-runner.sh tests/ci/ci-build-images-test.sh \
tests/ci/pipeline-test.sh tests/ci/semver-test.sh
tests/ci/migrations-test.sh tests/ci/pipeline-test.sh \
tests/ci/semver-test.sh
./tests/ci/ci-build-images-test.sh
./tests/ci/migrations-test.sh
./tests/ci/pipeline-test.sh
./tests/ci/semver-test.sh
- name: Scan repository

View File

@ -24,6 +24,10 @@ func main() {
os.Exit(1)
}
defer conn.Close(ctx)
if _, err := conn.Exec(ctx, "SET standard_conforming_strings = on"); err != nil {
logger.Error("enforce standard SQL string semantics failed", "error", err)
os.Exit(1)
}
if _, err := conn.Exec(ctx, `
CREATE TABLE IF NOT EXISTS schema_migrations (
@ -64,6 +68,14 @@ CREATE TABLE IF NOT EXISTS schema_migrations (
logger.Error("begin migration failed", "version", version, "error", err)
os.Exit(1)
}
// Pin string parsing semantics inside every migration transaction. A
// previous migration may have changed the session GUC, while each file is
// parsed and executed independently.
if _, err := tx.Exec(ctx, "SET LOCAL standard_conforming_strings = on"); err != nil {
_ = tx.Rollback(ctx)
logger.Error("enforce migration SQL string semantics failed", "version", version, "error", err)
os.Exit(1)
}
if _, err := tx.Exec(ctx, string(sqlBytes)); err != nil {
_ = tx.Rollback(ctx)
logger.Error("execute migration failed", "version", version, "error", err)

View File

@ -0,0 +1 @@
2d6c16fec0bec9c0288e5cb142b458af982fff8f

View File

@ -22,6 +22,7 @@ AI Gateway 已通过独立部署仓库和 Docker Compose 运行在 `110.42.51.33
- Runner 外层固定为 `gitea/runner:2.0.0-dind-rootless` 的指定 digestJob 固定为 `node:24.16.0-bookworm` 的指定 digest。外层按官方 rootless DinD 要求使用 `--privileged`,但内层 Docker daemon 和 Runner 以 UID 1000 运行。
- 不把宿主或内层 Docker socket 挂入 Job。Runner 配置使用 `docker_host: "-"`、`privileged: false`,工作流唯一允许的 bind source 是只读 `/opt/easyai-gateway-ci`,其中提供固定 Go、Node、pnpm、ShellCheck、Compose、Trivy 和 govulncheck 工具链。ShellCheck 与 Compose 使用官方静态发行物和固定 SHA-256不复制宿主的可变插件或在 Job 中临时安装。RootlessKit 强制 `--pidns`,即使 PR 传入 `--pid=host`Job 也不能进入外层 Runner 的 PID namespaceprovision 会实测 namespace inode。
- CI 只执行格式、静态检查、单元测试、依赖审计、Compose 校验、仓库扫描和 CI 自测;不构建发布镜像,不调用 `sudo`,不执行生产发布助手。
- PR、`main` 与 Tag CI 都把 `deploy/ci/production-migration-base` 中的当前生产提交作为数据库兼容基线:已经存在于生产基线的迁移不得修改或删除,新迁移静态拒绝 DROP、TRUNCATE、DELETE、MERGE、CREATE OR REPLACE、RENAME、列类型/非空收紧等 contract 操作。PR/`main` 还使用事件中不可变的 base/before SHA 比对,确保基线更新稍有滞后时,已经进入 `main` 的新迁移仍不可被改写。
- 版本 Tag 必须是完整 SemVer且目标提交必须属于 `main` 历史。该检查是发布前置证据,但本身不授予生产权限。
- Runner 注册状态保存在专用 Docker named volume首次注册 Token 只传给短命注册容器;长期 systemd 服务不保存 Token也不加入宿主 `docker`/`sudo` 用户组。
@ -66,3 +67,4 @@ rootless DinD 显著缩小了 Job 到宿主 Docker/root 的直接通路,但不
- 创建受保护版本 Tag 后,部署仓 dispatcher 必须同时取得同 SHA 的 `main` context 和独立 Tag context不能把旧 `main` 成功状态误当作本次 Tag 验证。
- 首次安装会清理本项目专属 Buildx 缓存并执行 8 GiB 前置、4 GiB 后置磁盘门禁,不会全局 prune 共享 Docker 状态。
- 首次启用 CD 仍需捕获当前运行镜像作为 bootstrap 回滚基线,并保证数据库迁移对上一应用版本向后兼容。
- 每次生产发布和恢复演练成功后,必须用单独受审 PR 把 `deploy/ci/production-migration-base` 前移到刚刚部署的完整源码 SHA不得在发布成功前前移。静态门禁不能证明任意数据变换可逆生产发布仍必须先做可恢复备份并定期执行真实 `pg_restore` 演练。

View File

@ -12,7 +12,7 @@
- 生产入口:`https://ai.51easyai.com`
- 部署目录:`/root/easyai-ai-gateway-deploy`
源码仓只执行质量 CIPR context 是 `ci / verify (pull_request)``main` context 是 `ci / verify (push)`,版本 Tag 使用独立 workflow/context `release-ci / verify-tag (push)`。三者都执行 Go 格式/vet/test/govulncheck、前端 lint/test/build、依赖审计、Compose 校验、Trivy 仓库扫描和 CI 脚本自测。Job 没有 Docker socket、`sudo` 或部署目录权限,也不构建发布镜像。
源码仓只执行质量 CIPR context 是 `ci / verify (pull_request)``main` context 是 `ci / verify (push)`,版本 Tag 使用独立 workflow/context `release-ci / verify-tag (push)`。三者都执行生产迁移兼容门禁、Go 格式/vet/test/govulncheck、前端 lint/test/build、依赖审计、Compose 校验、Trivy 仓库扫描和 CI 脚本自测。Job 没有 Docker socket、`sudo` 或部署目录权限,也不构建发布镜像。
生产 CD 由部署仓的 root-owned dispatcher 执行。它在相同源码 SHA 上同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)` 成功,然后只运行部署仓中固定的 BuildKit 构建、Trivy 镜像扫描、digest 发布、备份、迁移、健康检查和回滚命令;不得执行 Tag checkout 中的 `pnpm`、测试、shell 脚本或应用程序。
@ -87,16 +87,25 @@ docker exec easyai-gateway-ci-v2-runner docker image inspect \
3. 合法 SemVer Tag 新产生独立 `release-ci / verify-tag (push)`;非法或不属于 `main` 历史的 Tag 失败,且 Tag context 不能被旧 `main` context 替代。
4. Fork PR 默认不自动调度,维护者先检查 `.gitea/workflows/**` 和容器选项后再批准。
PR 与 `main` Push workflow 都校验事件携带的 base/before SHA 必须是触发 SHA 的祖先;分支落后或主干历史被改写时 CI 会 fail-closed。PR 先合并最新 `main` 再重跑Gitea 分支保护同时启用 `block_on_outdated_branch`
## Fork PR 残余风险
rootless DinD 不是 VM 沙箱。外层仍按官方要求使用 `--privileged`Gitea runner 也会解析 PR 控制的 `jobs.<job>.container.options`。配置已禁止 Job privileged、宿主/内层 Docker socket 和任意 bind source但 Runner、rootless Docker、容器运行时或内核漏洞仍可能突破边界Job 的出站网络和资源消耗也不是强租户隔离。因此 Fork PR 必须维护者批准CI 不得注入生产 Secret未来应迁移到独立 CI 主机或虚拟机。
## 数据库迁移兼容门禁
`deploy/ci/production-migration-base` 必须等于当前生产 API 对应的完整源码 SHA。`scripts/ci-validate-migrations.mjs` 只允许新增、按文件名严格追加且为普通文件的迁移;生产已知迁移以及已经进入 `main` 的迁移不可修改或删除。DROP、TRUNCATE、DELETE、MERGE、CREATE OR REPLACE、动态 EXECUTE、RENAME、列类型变更、增加/设置 `NOT NULL`、删除默认值和 REVOKE 会直接使 CI 失败。
该规则允许 INSERT/UPDATE 等数据迁移,因此不能替代备份和恢复验证。每次生产发布成功并完成 `pg_restore` 演练后,创建一个只前移 `deploy/ci/production-migration-base` 的受审 PR不要把基线指向尚未成功部署的提交。
## 发布生产版本
1. 合并到受保护 `main`,确认源码 CI 成功。
2. 在已验证提交上创建完整 SemVer Tag 并推送。
3. 确认同一 SHA 的 `ci / verify (push)` 和本次 `release-ci / verify-tag (push)` 都成功。
4. 由部署仓 root-owned dispatcher 构建、扫描并发布 digest源码仓工作流不会直接调用生产 helper。
5. 发布、健康检查和恢复演练完成后,用单独 PR 把 `deploy/ci/production-migration-base` 更新为该生产 Tag 的完整源码 SHA。
```bash
git switch main

View File

@ -0,0 +1,430 @@
#!/usr/bin/env node
import { execFileSync, spawnSync } from 'node:child_process'
import { lstatSync, readFileSync } from 'node:fs'
import { basename } from 'node:path'
const migrationDirectory = 'apps/api/migrations/'
const baselineFile = 'deploy/ci/production-migration-base'
const base = process.argv[2] || process.env.PRODUCTION_MIGRATION_BASE_SHA || ''
const immutableRef =
process.argv[3] || process.env.MIGRATION_IMMUTABILITY_REF || ''
function fail(message) {
console.error(`migration_safety=FAIL ${message}`)
process.exit(1)
}
function git(args, options = {}) {
try {
return execFileSync('git', args, {
encoding: options.encoding ?? 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
})
} catch (error) {
const stderr = error.stderr?.toString().trim()
fail(stderr ? `git_error=${JSON.stringify(stderr)}` : 'git command failed')
}
}
if (!/^[0-9a-f]{40}$/.test(base)) {
fail('PRODUCTION_MIGRATION_BASE_SHA must be a full lowercase commit SHA')
}
const root = git(['rev-parse', '--show-toplevel']).trim()
process.chdir(root)
const resolvedBase = git([
'rev-parse',
'--verify',
'--end-of-options',
`${base}^{commit}`,
]).trim()
if (resolvedBase !== base) {
fail('production migration baseline did not resolve exactly')
}
const ancestry = spawnSync('git', ['merge-base', '--is-ancestor', base, 'HEAD'], {
stdio: 'ignore',
})
if (ancestry.status !== 0) {
fail('production migration baseline is not an ancestor of HEAD')
}
const head = git(['rev-parse', '--verify', 'HEAD^{commit}']).trim()
const baselineTreeEntry = git(['ls-tree', 'HEAD', '--', baselineFile]).trim()
let baselineState
try {
baselineState = lstatSync(baselineFile)
} catch {
fail('production migration baseline file is unavailable in the worktree')
}
if (
!/^100644 blob [0-9a-f]+\t/.test(baselineTreeEntry) ||
!baselineState.isFile() ||
baselineState.isSymbolicLink()
) {
fail('production migration baseline must be a mode 100644 regular file')
}
if (readFileSync(baselineFile, 'utf8').trim() !== base) {
fail('production migration baseline argument does not match the reviewed file')
}
function changedMigrations(from) {
const rawChanges = git(
[
'diff',
'--name-status',
'-z',
'--no-renames',
`${from}..HEAD`,
'--',
migrationDirectory,
],
{ encoding: null },
)
const fields = rawChanges.toString('utf8').split('\0')
if (fields.at(-1) === '') fields.pop()
if (fields.length % 2 !== 0) {
fail('could not parse changed migration paths')
}
return fields
}
const fields = changedMigrations(base)
function executableSql(sql, source = 'SQL input') {
let output = ''
for (let index = 0; index < sql.length; index += 1) {
const current = sql[index]
const next = sql[index + 1]
if (current === '-' && next === '-') {
index += 2
while (index < sql.length && sql[index] !== '\n') index += 1
output += '\n'
continue
}
if (current === '/' && next === '*') {
let depth = 1
index += 2
while (index < sql.length && depth > 0) {
if (sql[index] === '/' && sql[index + 1] === '*') {
depth += 1
index += 2
continue
}
if (sql[index] === '*' && sql[index + 1] === '/') {
depth -= 1
index += 2
continue
}
index += 1
}
if (depth !== 0) fail('unterminated SQL block comment')
index -= 1
output += ' '
continue
}
if (current === "'" || current === '"') {
const prefix = sql.slice(Math.max(0, index - 2), index).toUpperCase()
const previous = sql[index - 1]?.toUpperCase()
const beforePrevious = sql[index - 2] || ''
const beforeUnicodePrefix = sql[index - 3] || ''
const escapeString =
current === "'" &&
previous === 'E' &&
!/[A-Za-z0-9_$]/.test(beforePrevious)
const unicodeString =
prefix === 'U&' && !/[A-Za-z0-9_$]/.test(beforeUnicodePrefix)
if (escapeString || unicodeString) {
fail(`${source}: PostgreSQL escape/unicode strings are not allowed in migrations`)
}
const quote = current
index += 1
let closed = false
while (index < sql.length) {
if (sql[index] === quote && sql[index + 1] === quote) {
index += 2
continue
}
if (sql[index] === quote) {
closed = true
break
}
index += 1
}
if (!closed) fail('unterminated SQL quoted value')
output += ' '
continue
}
if (current === '$') {
const delimiter = sql.slice(index).match(/^\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/)?.[0]
if (delimiter) {
const bodyStart = index + delimiter.length
const bodyEnd = sql.indexOf(delimiter, bodyStart)
if (bodyEnd === -1) fail('unterminated SQL dollar-quoted body')
// PostgreSQL function bodies may contain executable DDL. Scan their
// contents recursively instead of treating them as inert strings.
const bodySql = executableSql(sql.slice(bodyStart, bodyEnd), source)
// A bare END terminates a PL/pgSQL block inside a dollar-quoted body,
// but at the top level PostgreSQL treats END as COMMIT. Mask only the
// block terminator here so the top-level transaction rule stays exact.
const bodyWithoutBlockEnd = bodySql
.split(';')
.map((statement) => (/^\s*END\s*$/i.test(statement) ? ' ' : statement))
.join(';')
output += ` ${bodyWithoutBlockEnd} `
index = bodyEnd + delimiter.length - 1
continue
}
}
output += current
}
return output
}
const destructiveRules = [
{
name: 'dynamic SQL execution',
pattern: /\bEXECUTE\b/i,
},
{
name: 'destructive DROP operation',
pattern: /\bDROP\b/i,
},
{ name: 'MERGE operation', pattern: /\bMERGE\b/i },
{
name: 'CREATE OR REPLACE operation',
pattern: /\bCREATE\s+OR\s+REPLACE\b/i,
},
{ name: 'TRUNCATE operation', pattern: /\bTRUNCATE(?:\s+TABLE)?\b/i },
{ name: 'DELETE FROM operation', pattern: /\bDELETE\s+FROM\b/i },
{ name: 'permission revocation', pattern: /\bREVOKE\b/i },
{
name: 'transaction control operation',
pattern:
/\b(?:COMMIT|ROLLBACK|ABORT|SAVEPOINT)\b|\bRELEASE\s+SAVEPOINT\b|\bPREPARE\s+TRANSACTION\b|^\s*END(?:\s+(?:WORK|TRANSACTION))?(?:\s+AND\s+(?:NO\s+)?CHAIN)?\s*$/i,
},
{
name: 'SQL string semantics override',
pattern:
/\bSET\s+(?:LOCAL\s+|SESSION\s+)?STANDARD_CONFORMING_STRINGS\b/i,
},
{
name: 'object or column rename',
pattern: /\bRENAME\b/i,
},
{ name: 'SET SCHEMA operation', pattern: /\bSET\s+SCHEMA\b/i },
{
name: 'column type change',
pattern:
/\bALTER\s+TABLE\b[\s\S]*\bALTER\s+(?:COLUMN\s+)?[\s\S]*\b(?:SET\s+DATA\s+)?TYPE\b/i,
},
{
name: 'SET NOT NULL operation',
pattern:
/\bALTER\s+TABLE\b[\s\S]*\bALTER\s+(?:COLUMN\s+)?[\s\S]*\bSET\s+NOT\s+NULL\b/i,
},
{
name: 'non-null column addition',
pattern:
/\bALTER\s+TABLE\b[\s\S]*\bADD\s+(?:COLUMN\s+)?[\s\S]*\bNOT\s+NULL\b/i,
},
{
name: 'DROP DEFAULT operation',
pattern:
/\bALTER\s+TABLE\b[\s\S]*\bALTER\s+(?:COLUMN\s+)?[\s\S]*\bDROP\s+DEFAULT\b/i,
},
]
const violations = []
let addedFiles = 0
let immutableBase = ''
const orderingCandidates = new Set()
if (immutableRef) {
if (
!/^[0-9a-f]{40}$/.test(immutableRef) &&
immutableRef !== 'refs/remotes/origin/main'
) {
fail('MIGRATION_IMMUTABILITY_REF must be a full commit SHA or refs/remotes/origin/main')
}
const immutableTip = git([
'rev-parse',
'--verify',
'--end-of-options',
`${immutableRef}^{commit}`,
]).trim()
const baseAtEvent = spawnSync(
'git',
['show', `${immutableTip}:${baselineFile}`],
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] },
)
if (baseAtEvent.status === 0) {
const previousBaselineTreeEntry = git([
'ls-tree',
immutableTip,
'--',
baselineFile,
]).trim()
if (!/^100644 blob [0-9a-f]+\t/.test(previousBaselineTreeEntry)) {
fail('previous production migration baseline must be a mode 100644 regular file')
}
const previousProductionBase = baseAtEvent.stdout.trim()
if (!/^[0-9a-f]{40}$/.test(previousProductionBase)) {
fail('previous production migration baseline is invalid')
}
const baselineAdvance = spawnSync(
'git',
['merge-base', '--is-ancestor', previousProductionBase, base],
{ stdio: 'ignore' },
)
if (baselineAdvance.status !== 0) {
fail('production migration baseline must only move forward')
}
}
const baselineWasAlreadyReviewed = spawnSync(
'git',
['merge-base', '--is-ancestor', base, immutableTip],
{ stdio: 'ignore' },
)
if (baselineWasAlreadyReviewed.status !== 0) {
fail('production migration baseline must belong to the immutable event base history')
}
if (immutableTip === head) {
const parent = spawnSync(
'git',
['rev-parse', '--verify', `${immutableTip}^1^{commit}`],
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] },
)
immutableBase = parent.status === 0 ? parent.stdout.trim() : ''
} else {
const immutableAncestry = spawnSync(
'git',
['merge-base', '--is-ancestor', immutableTip, head],
{ stdio: 'ignore' },
)
if (immutableAncestry.status !== 0) {
fail('migration immutability base is not an ancestor of HEAD')
}
immutableBase = immutableTip
}
if (immutableBase) {
const immutableChanges = changedMigrations(immutableBase)
for (let index = 0; index < immutableChanges.length; index += 2) {
const status = immutableChanges[index]
const file = immutableChanges[index + 1]
if (status === 'A') {
orderingCandidates.add(file)
} else {
violations.push(
`${file}: migration present on main is immutable (status ${status})`,
)
}
}
}
}
function migrationNamesAt(ref) {
const output = git(
['ls-tree', '-r', '-z', '--name-only', ref, '--', migrationDirectory],
{ encoding: null },
)
return output
.toString('utf8')
.split('\0')
.filter(Boolean)
.filter((file) => {
const relative = file.slice(migrationDirectory.length)
return file.startsWith(migrationDirectory) && !relative.includes('/')
})
.map((file) => basename(file))
.filter((name) => /^\d{4}_[a-z0-9][a-z0-9_]*\.sql$/.test(name))
.sort()
}
const orderingBase = immutableBase || base
const existingNames = migrationNamesAt(orderingBase)
const previousMaximumName = existingNames.at(-1) || ''
if (!immutableBase) {
for (let index = 0; index < fields.length; index += 2) {
if (fields[index] === 'A') orderingCandidates.add(fields[index + 1])
}
}
for (let index = 0; index < fields.length; index += 2) {
const status = fields[index]
const file = fields[index + 1]
if (status !== 'A') {
violations.push(`${file}: applied migrations are immutable (status ${status})`)
continue
}
addedFiles += 1
const filename = basename(file)
if (
file !== `${migrationDirectory}${filename}` ||
!/^\d{4}_[a-z0-9][a-z0-9_]*\.sql$/.test(filename)
) {
violations.push(`${file}: new migration must use NNNN_lowercase_name.sql`)
continue
}
if (
orderingCandidates.has(file) &&
previousMaximumName &&
filename <= previousMaximumName
) {
violations.push(
`${file}: new migration name must sort after ${previousMaximumName}`,
)
continue
}
const treeEntry = git(['ls-tree', 'HEAD', '--', file]).trim()
let fileState
try {
fileState = lstatSync(file)
} catch {
violations.push(`${file}: new migration is unavailable in the worktree`)
continue
}
if (
!/^100644 blob [0-9a-f]+\t/.test(treeEntry) ||
!fileState.isFile() ||
fileState.isSymbolicLink()
) {
violations.push(`${file}: new migration must be a mode 100644 regular file`)
continue
}
const statements = executableSql(readFileSync(file, 'utf8'), file).split(';')
for (const statement of statements) {
for (const rule of destructiveRules) {
if (rule.pattern.test(statement)) {
violations.push(`${file}: ${rule.name}`)
}
}
}
}
if (violations.length > 0) {
for (const violation of [...new Set(violations)]) {
console.error(`migration_safety_violation=${JSON.stringify(violation)}`)
}
fail(`base=${base} violations=${new Set(violations).size}`)
}
console.log(`migration_safety=PASS base=${base} added_files=${addedFiles}`)

323
tests/ci/migrations-test.sh Executable file
View File

@ -0,0 +1,323 @@
#!/usr/bin/env bash
set -euo pipefail
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
validator=$root/scripts/ci-validate-migrations.mjs
fixture_root=$(mktemp -d)
trap 'rm -rf "$fixture_root"' EXIT
new_repo() {
local name=$1
local repo=$fixture_root/$name
mkdir -p "$repo/apps/api/migrations"
git -C "$repo" init -q
git -C "$repo" config user.name 'CI Migration Test'
git -C "$repo" config user.email 'ci-migration-test@example.invalid'
printf 'CREATE TABLE accounts (id bigint PRIMARY KEY);\n' > \
"$repo/apps/api/migrations/0001_init.sql"
git -C "$repo" add apps/api/migrations/0001_init.sql
git -C "$repo" commit -qm baseline
mkdir -p "$repo/deploy/ci"
git -C "$repo" rev-parse HEAD > \
"$repo/deploy/ci/production-migration-base"
printf '%s\n' "$repo"
}
commit_all() {
local repo=$1
local message=$2
git -C "$repo" add -A
git -C "$repo" commit -qm "$message"
}
expect_pass() {
local repo=$1
local base=$2
local immutable_base=${3:-}
(
cd "$repo"
if [[ -n $immutable_base ]]; then
node "$validator" "$base" "$immutable_base"
else
node "$validator" "$base"
fi
)
}
expect_fail() {
local repo=$1
local base=$2
local expected=$3
local immutable_base=${4:-}
local output status
if [[ -n $immutable_base ]]; then
set +e
output=$(cd "$repo" && node "$validator" "$base" "$immutable_base" 2>&1)
status=$?
set -e
else
set +e
output=$(cd "$repo" && node "$validator" "$base" 2>&1)
status=$?
set -e
fi
if [[ $status -eq 0 ]]; then
printf 'migration validator unexpectedly passed: %s\n' "$expected" >&2
exit 1
fi
grep -Fq "$expected" <<<"$output" || {
printf 'missing expected migration failure %q in:\n%s\n' "$expected" "$output" >&2
exit 1
}
}
repo=$(new_repo safe)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_expand_accounts.sql" <<'SQL'
-- A comment mentioning DROP TABLE must not trigger the gate.
ALTER TABLE accounts ADD COLUMN display_name text;
CREATE TABLE audit_events (id bigint PRIMARY KEY, note text);
INSERT INTO audit_events (id, note) VALUES (1, 'DROP TABLE is documentation');
UPDATE accounts SET display_name = 'ready' WHERE display_name IS NULL;
SQL
commit_all "$repo" safe
expect_pass "$repo" "$base"
repo=$(new_repo drop)
base=$(git -C "$repo" rev-parse HEAD)
printf 'DROP\nTABLE accounts;\n' >"$repo/apps/api/migrations/0002_drop.sql"
commit_all "$repo" drop
expect_fail "$repo" "$base" 'destructive DROP operation'
repo=$(new_repo delete)
base=$(git -C "$repo" rev-parse HEAD)
printf 'DELETE FROM accounts;\n' >"$repo/apps/api/migrations/0002_delete.sql"
commit_all "$repo" delete
expect_fail "$repo" "$base" 'DELETE FROM operation'
repo=$(new_repo incompatible_alter)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_alter.sql" <<'SQL'
ALTER TABLE accounts ADD COLUMN external_id text NOT NULL;
ALTER TABLE accounts ALTER COLUMN id TYPE numeric;
SQL
commit_all "$repo" alter
expect_fail "$repo" "$base" 'non-null column addition'
expect_fail "$repo" "$base" 'column type change'
repo=$(new_repo dynamic_sql)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_dynamic.sql" <<'SQL'
DO $$
BEGIN
EXECUTE 'DROP ' || 'TABLE accounts';
END
$$;
SQL
commit_all "$repo" dynamic
expect_fail "$repo" "$base" 'dynamic SQL execution'
repo=$(new_repo escape_string)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_escape.sql" <<'SQL'
SELECT E'foo\'bar'; DROP TABLE accounts; --';
SQL
commit_all "$repo" escape
expect_fail "$repo" "$base" 'PostgreSQL escape/unicode strings are not allowed'
repo=$(new_repo string_semantics_override)
base=$(git -C "$repo" rev-parse HEAD)
printf 'SET standard_conforming_strings = off;\n' > \
"$repo/apps/api/migrations/0002_string_mode.sql"
commit_all "$repo" string_mode
expect_fail "$repo" "$base" 'SQL string semantics override'
repo=$(new_repo transaction_control)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_commit.sql" <<'SQL'
ALTER TABLE accounts ADD COLUMN display_name text;
COMMIT;
UPDATE accounts SET display_name = 'partially committed';
SQL
commit_all "$repo" transaction_control
expect_fail "$repo" "$base" 'transaction control operation'
repo=$(new_repo bare_end_transaction)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_end.sql" <<'SQL'
ALTER TABLE accounts ADD COLUMN display_name text;
END;
UPDATE accounts SET display_name = 'partially committed';
SQL
commit_all "$repo" bare_end_transaction
expect_fail "$repo" "$base" 'transaction control operation'
repo=$(new_repo plpgsql_block_end)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_function.sql" <<'SQL'
CREATE FUNCTION account_count() RETURNS bigint
LANGUAGE plpgsql AS $$
BEGIN
RETURN (SELECT count(*) FROM accounts);
END;
$$;
SQL
commit_all "$repo" plpgsql_block_end
expect_pass "$repo" "$base"
repo=$(new_repo ordinary_string_boundary)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_string_boundary.sql" <<'SQL'
SELECT '\'; DROP TABLE accounts; --';
SQL
commit_all "$repo" ordinary_string
expect_fail "$repo" "$base" 'destructive DROP operation'
repo=$(new_repo unicode_string)
base=$(git -C "$repo" rev-parse HEAD)
printf "SELECT U&'d\\0061ta';\n" >"$repo/apps/api/migrations/0002_unicode.sql"
commit_all "$repo" unicode
expect_fail "$repo" "$base" 'PostgreSQL escape/unicode strings are not allowed'
repo=$(new_repo merge_delete)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_merge.sql" <<'SQL'
MERGE INTO accounts USING stale_accounts ON accounts.id = stale_accounts.id
WHEN MATCHED THEN DELETE;
SQL
commit_all "$repo" merge
expect_fail "$repo" "$base" 'MERGE operation'
repo=$(new_repo replace_view)
base=$(git -C "$repo" rev-parse HEAD)
printf 'CREATE OR REPLACE VIEW active_accounts AS SELECT id FROM accounts;\n' > \
"$repo/apps/api/migrations/0002_replace.sql"
commit_all "$repo" replace
expect_fail "$repo" "$base" 'CREATE OR REPLACE operation'
repo=$(new_repo rename_function)
base=$(git -C "$repo" rev-parse HEAD)
printf 'ALTER FUNCTION calculate_total() RENAME TO calculate_total_legacy;\n' > \
"$repo/apps/api/migrations/0002_rename.sql"
commit_all "$repo" rename
expect_fail "$repo" "$base" 'object or column rename'
repo=$(new_repo set_schema)
base=$(git -C "$repo" rev-parse HEAD)
printf 'ALTER TABLE accounts SET SCHEMA archive;\n' > \
"$repo/apps/api/migrations/0002_schema.sql"
commit_all "$repo" schema
expect_fail "$repo" "$base" 'SET SCHEMA operation'
repo=$(new_repo drop_domain)
base=$(git -C "$repo" rev-parse HEAD)
printf 'DROP DOMAIN account_code CASCADE;\n' > \
"$repo/apps/api/migrations/0002_drop_domain.sql"
commit_all "$repo" drop_domain
expect_fail "$repo" "$base" 'destructive DROP operation'
repo=$(new_repo immutable)
base=$(git -C "$repo" rev-parse HEAD)
printf '\n-- retroactive edit\n' >>"$repo/apps/api/migrations/0001_init.sql"
commit_all "$repo" edit
expect_fail "$repo" "$base" 'applied migrations are immutable (status M)'
repo=$(new_repo immutable_after_baseline)
base=$(git -C "$repo" rev-parse HEAD)
printf 'ALTER TABLE accounts ADD COLUMN nickname text;\n' > \
"$repo/apps/api/migrations/0002_nickname.sql"
commit_all "$repo" add
immutable_base=$(git -C "$repo" rev-parse HEAD)
printf '\nUPDATE accounts SET nickname = NULL;\n' >> \
"$repo/apps/api/migrations/0002_nickname.sql"
commit_all "$repo" modify
output=$(
cd "$repo"
node "$validator" "$base" "$immutable_base" 2>&1 || true
)
grep -Fq 'migration present on main is immutable (status M)' <<<"$output" || {
printf 'stale production baseline did not preserve main migration immutability:\n%s\n' \
"$output" >&2
exit 1
}
repo=$(new_repo filename)
base=$(git -C "$repo" rev-parse HEAD)
printf 'SELECT 1;\n' >"$repo/apps/api/migrations/next.SQL"
commit_all "$repo" filename
expect_fail "$repo" "$base" 'new migration must use NNNN_lowercase_name.sql'
repo=$(new_repo ordering)
base=$(git -C "$repo" rev-parse HEAD)
printf 'SELECT 1;\n' >"$repo/apps/api/migrations/0000_late.sql"
commit_all "$repo" ordering
expect_fail "$repo" "$base" 'new migration name must sort after 0001_init.sql'
repo=$(new_repo stale_baseline)
base=$(git -C "$repo" rev-parse HEAD)
printf 'ALTER TABLE accounts ADD COLUMN nickname text;\n' > \
"$repo/apps/api/migrations/0002_nickname.sql"
commit_all "$repo" main_migration
immutable_base=$(git -C "$repo" rev-parse HEAD)
printf 'ordinary code change\n' >"$repo/README.md"
commit_all "$repo" code_only
expect_pass "$repo" "$base" "$immutable_base"
repo=$(new_repo baseline_advance)
production_base=$(git -C "$repo" rev-parse HEAD)
mkdir -p "$repo/deploy/ci"
printf '%s\n' "$production_base" >"$repo/deploy/ci/production-migration-base"
commit_all "$repo" baseline_file
immutable_base=$(git -C "$repo" rev-parse HEAD)
git -C "$repo" switch -q -c release-baseline-update "$immutable_base"
printf '%s\n' "$immutable_base" >"$repo/deploy/ci/production-migration-base"
commit_all "$repo" valid_baseline_advance
expect_pass "$repo" "$immutable_base" "$immutable_base"
git -C "$repo" switch -q -c malicious-baseline "$immutable_base"
printf 'DROP TABLE accounts;\n' >"$repo/apps/api/migrations/0002_drop.sql"
commit_all "$repo" hidden_destructive_migration
hidden_sha=$(git -C "$repo" rev-parse HEAD)
printf '%s\n' "$hidden_sha" >"$repo/deploy/ci/production-migration-base"
commit_all "$repo" hide_migration_in_baseline
expect_fail "$repo" "$hidden_sha" \
'production migration baseline must belong to the immutable event base history' \
"$immutable_base"
repo=$(new_repo nested)
base=$(git -C "$repo" rev-parse HEAD)
mkdir -p "$repo/apps/api/migrations/archive"
printf 'SELECT 1;\n' >"$repo/apps/api/migrations/archive/9999_hidden.sql"
commit_all "$repo" nested
expect_fail "$repo" "$base" 'new migration must use NNNN_lowercase_name.sql'
repo=$(new_repo symlink)
base=$(git -C "$repo" rev-parse HEAD)
printf 'SELECT 1;\n' >"$repo/outside.sql"
ln -s ../../../outside.sql "$repo/apps/api/migrations/0002_symlink.sql"
commit_all "$repo" symlink
expect_fail "$repo" "$base" 'new migration must be a mode 100644 regular file'
repo=$(new_repo baseline_symlink)
base=$(git -C "$repo" rev-parse HEAD)
printf '%s\n' "$base" >"$repo/baseline-target"
rm "$repo/deploy/ci/production-migration-base"
ln -s ../../baseline-target "$repo/deploy/ci/production-migration-base"
commit_all "$repo" baseline_symlink
expect_fail "$repo" "$base" \
'production migration baseline must be a mode 100644 regular file'
repo=$(new_repo ancestry)
base=$(git -C "$repo" rev-parse HEAD)
git -C "$repo" switch -q --orphan unrelated
printf 'unrelated\n' >"$repo/unrelated"
git -C "$repo" add unrelated
git -C "$repo" commit -qm unrelated
expect_fail "$repo" "$base" 'production migration baseline is not an ancestor of HEAD'
repo=$(new_repo invalid_sha)
expect_fail "$repo" deadbeef 'must be a full lowercase commit SHA'
echo 'migration_tests=PASS'

View File

@ -10,22 +10,45 @@ runner_config=$root/deploy/ci/act-runner-v2-config.yaml
provision=$root/scripts/provision-ci-runner.sh
build_images=$root/scripts/ci-build-images.sh
validate_semver=$root/scripts/ci-validate-semver.sh
validate_migrations=$root/scripts/ci-validate-migrations.mjs
build_images_test=$root/tests/ci/ci-build-images-test.sh
migrations_test=$root/tests/ci/migrations-test.sh
migration_base=$root/deploy/ci/production-migration-base
adr=$root/docs/decisions/001-production-cicd.md
runbook=$root/docs/runbooks/production-ci-cd.md
migrator=$root/apps/api/cmd/migrate/main.go
for file in "$workflow" "$release_workflow" "$runner_service" "$runner_config" "$provision" "$build_images" "$validate_semver" "$build_images_test" "$adr" "$runbook"; do
for file in "$workflow" "$release_workflow" "$runner_service" "$runner_config" "$provision" "$build_images" "$validate_semver" "$validate_migrations" "$build_images_test" "$migrations_test" "$migration_base" "$adr" "$runbook" "$migrator"; do
[[ -f $file ]] || {
printf 'missing CI/CD file: %s\n' "$file" >&2
exit 1
}
done
if [[ $(wc -l <"$migration_base") -ne 1 ]]; then
echo 'production migration baseline must contain exactly one line' >&2
exit 1
fi
grep -Fq 'SET standard_conforming_strings = on' "$migrator" || {
echo 'database migrator does not pin standard SQL string semantics' >&2
exit 1
}
if ! grep -Eq '^[0-9a-f]{40}$' "$migration_base"; then
echo 'production migration baseline must be a full lowercase SHA' >&2
exit 1
fi
grep -q '^name: ci$' "$workflow"
grep -q '^ push:$' "$workflow"
grep -q '^ branches: \[main\]$' "$workflow"
grep -q '^ pull_request:$' "$workflow"
grep -q '^ verify:$' "$workflow"
grep -Fq 'git checkout --detach "$CI_SHA"' "$workflow"
grep -Fq 'test "$(git rev-parse HEAD)" = "$CI_SHA"' "$workflow"
if grep -Fq 'git checkout --detach FETCH_HEAD' "$workflow"; then
echo 'main/PR CI must not checkout mutable FETCH_HEAD after comparison fetches' >&2
exit 1
fi
if grep -Eq "^[[:space:]]+tags:|Verify release tag ancestry" "$workflow" || \
grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$workflow"; then
echo 'main/PR CI must not share the release-tag context' >&2
@ -53,6 +76,7 @@ for quality_workflow in "$workflow" "$release_workflow"; do
fi
grep -Fq 'test ! -f .git/shallow' "$quality_workflow"
grep -Fq './tests/ci/ci-build-images-test.sh' "$quality_workflow"
grep -Fq './tests/ci/migrations-test.sh' "$quality_workflow"
grep -Fq './tests/ci/pipeline-test.sh' "$quality_workflow"
grep -Fq './tests/ci/semver-test.sh' "$quality_workflow"
grep -Fq 'docker-compose version' "$quality_workflow"
@ -81,6 +105,7 @@ for quality_workflow in "$workflow" "$release_workflow"; do
fi
for gate in \
'scripts/ci-validate-migrations.mjs' \
'gofmt -l' \
'go vet ./...' \
'go test ./...' \
@ -103,6 +128,12 @@ for quality_workflow in "$workflow" "$release_workflow"; do
"$quality_workflow" >&2
exit 1
}
grep -Fq 'production_base=$(cat deploy/ci/production-migration-base)' \
"$quality_workflow" || {
printf '%s does not read the versioned production migration baseline\n' \
"$quality_workflow" >&2
exit 1
}
done
if [[ $(sed -n '/^ - name: Verify Go formatting$/,$p' "$workflow") != \
@ -111,6 +142,13 @@ if [[ $(sed -n '/^ - name: Verify Go formatting$/,$p' "$workflow") != \
exit 1
fi
grep -Fq 'CI_EVENT_BEFORE: ${{ github.event.before }}' "$workflow"
grep -Fq 'CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}' "$workflow"
grep -Fq 'git merge-base --is-ancestor "$immutable_base" HEAD' "$workflow"
grep -Fq '"$production_base" "$immutable_base"' "$workflow"
grep -Fq 'node ./scripts/ci-validate-migrations.mjs "$production_base"' \
"$release_workflow"
runner_image='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a'
job_image='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7'