ci: 加固生产质量门禁 #5

Merged
chengcheng merged 14 commits from codex/production-cicd into main 2026-07-17 17:32:09 +08:00
23 changed files with 3161 additions and 842 deletions

109
.gitea/workflows/ci.yml Normal file
View File

@ -0,0 +1,109 @@
name: ci
on:
push:
branches: [main]
pull_request:
branches: [main]
jobs:
verify:
runs-on: easyai-gateway-ci-unprivileged-v2
env:
TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2
steps:
- name: Checkout without external Actions
env:
CI_REPOSITORY: ${{ github.repository }}
CI_SERVER_URL: ${{ github.server_url }}
CI_SHA: ${{ github.sha }}
CI_JOB_TOKEN: ${{ github.token }}
CI_EVENT_BEFORE: ${{ github.event.before }}
CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
set -eu
test -n "$CI_JOB_TOKEN"
authorization=$(printf 'x-access-token:%s' "$CI_JOB_TOKEN" | base64 | tr -d '\n')
git init .
git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \
fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" "$CI_SHA"
for comparison_sha in "$CI_EVENT_BEFORE" "$CI_PR_BASE_SHA"; do
case "$comparison_sha" in
[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*)
if test "${#comparison_sha}" -eq 40 && \
test "$comparison_sha" != 0000000000000000000000000000000000000000; then
git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \
fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" "$comparison_sha"
fi
;;
esac
done
unset authorization CI_JOB_TOKEN
test ! -f .git/shallow
git checkout --detach "$CI_SHA"
test "$(git rev-parse HEAD)" = "$CI_SHA"
- name: Verify pinned host toolchains
run: |
go version
node --version
pnpm --version
docker-compose version
shellcheck --version
trivy --version
govulncheck -version
- name: Verify production migration safety
env:
CI_EVENT_NAME: ${{ github.event_name }}
CI_EVENT_BEFORE: ${{ github.event.before }}
CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
production_base=$(cat deploy/ci/production-migration-base)
immutable_base=$CI_EVENT_BEFORE
if test "$CI_EVENT_NAME" = pull_request; then
immutable_base=$CI_PR_BASE_SHA
fi
case "$immutable_base" in
[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*)
test "${#immutable_base}" -eq 40
test "$immutable_base" != 0000000000000000000000000000000000000000
;;
*) exit 1 ;;
esac
git merge-base --is-ancestor "$immutable_base" HEAD
node ./scripts/ci-validate-migrations.mjs \
"$production_base" "$immutable_base"
- name: Verify Go formatting
run: |
unformatted=$(gofmt -l apps/api)
test -z "$unformatted" || {
printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2
exit 1
}
- name: Verify Go code
working-directory: apps/api
run: |
go vet ./...
go test ./...
govulncheck ./...
- run: pnpm install --frozen-lockfile
- run: pnpm lint
- run: pnpm test
- run: pnpm build
- name: Audit JavaScript dependencies
run: pnpm audit --audit-level high
- name: Validate deployment configuration
run: |
docker-compose -f docker-compose.yml config --quiet
shellcheck scripts/ci-build-images.sh scripts/ci-validate-semver.sh \
scripts/provision-ci-runner.sh tests/ci/ci-build-images-test.sh \
tests/ci/migrations-test.sh tests/ci/pipeline-test.sh \
tests/ci/semver-test.sh
./tests/ci/ci-build-images-test.sh
./tests/ci/migrations-test.sh
./tests/ci/pipeline-test.sh
./tests/ci/semver-test.sh
- name: Scan repository
run: |
trivy fs --scanners vuln,secret,misconfig --severity HIGH,CRITICAL \
--ignore-unfixed --exit-code 1 --timeout 15m --skip-dirs .git \
--skip-dirs node_modules .

View File

@ -0,0 +1,94 @@
name: release-ci
on:
push:
tags: ['v*']
jobs:
verify-tag:
runs-on: easyai-gateway-ci-unprivileged-v2
env:
TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2
steps:
- name: Checkout without external Actions
env:
CI_REPOSITORY: ${{ github.repository }}
CI_SERVER_URL: ${{ github.server_url }}
CI_SHA: ${{ github.sha }}
CI_JOB_TOKEN: ${{ github.token }}
run: |
set -eu
test -n "$CI_JOB_TOKEN"
authorization=$(printf 'x-access-token:%s' "$CI_JOB_TOKEN" | base64 | tr -d '\n')
git init .
git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \
fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" "$CI_SHA"
unset authorization CI_JOB_TOKEN
test ! -f .git/shallow
git checkout --detach FETCH_HEAD
- name: Verify pinned host toolchains
run: |
go version
node --version
pnpm --version
docker-compose version
shellcheck --version
trivy --version
govulncheck -version
- name: Verify release tag ancestry
env:
CI_REPOSITORY: ${{ github.repository }}
CI_SERVER_URL: ${{ github.server_url }}
CI_SHA: ${{ github.sha }}
CI_JOB_TOKEN: ${{ github.token }}
run: |
set -eu
tag_name=${GITHUB_REF#refs/tags/}
./scripts/ci-validate-semver.sh "$tag_name"
authorization=$(printf 'x-access-token:%s' "$CI_JOB_TOKEN" | base64 | tr -d '\n')
git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \
fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" \
+refs/heads/main:refs/remotes/origin/main
unset authorization CI_JOB_TOKEN
test ! -f .git/shallow
test "$(git rev-parse HEAD)" = "$CI_SHA"
git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main
- name: Verify production migration safety
run: |
production_base=$(cat deploy/ci/production-migration-base)
node ./scripts/ci-validate-migrations.mjs "$production_base"
- name: Verify Go formatting
run: |
unformatted=$(gofmt -l apps/api)
test -z "$unformatted" || {
printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2
exit 1
}
- name: Verify Go code
working-directory: apps/api
run: |
go vet ./...
go test ./...
govulncheck ./...
- run: pnpm install --frozen-lockfile
- run: pnpm lint
- run: pnpm test
- run: pnpm build
- name: Audit JavaScript dependencies
run: pnpm audit --audit-level high
- name: Validate deployment configuration
run: |
docker-compose -f docker-compose.yml config --quiet
shellcheck scripts/ci-build-images.sh scripts/ci-validate-semver.sh \
scripts/provision-ci-runner.sh tests/ci/ci-build-images-test.sh \
tests/ci/migrations-test.sh tests/ci/pipeline-test.sh \
tests/ci/semver-test.sh
./tests/ci/ci-build-images-test.sh
./tests/ci/migrations-test.sh
./tests/ci/pipeline-test.sh
./tests/ci/semver-test.sh
- name: Scan repository
run: |
trivy fs --scanners vuln,secret,misconfig --severity HIGH,CRITICAL \
--ignore-unfixed --exit-code 1 --timeout 15m --skip-dirs .git \
--skip-dirs node_modules .

View File

@ -109,6 +109,10 @@ Web 容器的 Nginx 配置通过 bind mount 挂载自仓库文件 [docker/nginx.
docker compose -f docker-compose.yml restart web
```
## 生产 CI/CD
Gitea Actions 会在隔离的 rootless DinD Runner 中对 Pull Request、`main` Push 和版本 Tag 执行完整质量门禁Tag 使用独立的 `release-ci / verify-tag (push)` context不能复用旧的 `main` 成功状态。源码 Job 没有宿主 Docker、`sudo` 或生产部署权限。部署仓的 root-owned dispatcher 只在相同 SHA 的 `main` 与 Tag context 都成功后,用固定命令构建并扫描镜像,再以 Registry digest 发布 `ai.51easyai.com`。安装 Runner、Fork PR 审批、发布验证和回滚步骤见 [生产 CI/CD 运行手册](docs/runbooks/production-ci-cd.md),信任边界见 [ADR-001](docs/decisions/001-production-cicd.md)。
Compose 默认使用独立容器数据库 `postgres:18-alpine`,数据卷会保留在 `postgres_data``api_data`。为避免本地开发 `.env` 中的 `localhost` 数据库地址污染容器部署compose 使用 `AI_GATEWAY_COMPOSE_*` 变量作为容器部署专用覆盖,例如:
```bash

View File

@ -24,6 +24,10 @@ func main() {
os.Exit(1)
}
defer conn.Close(ctx)
if _, err := conn.Exec(ctx, "SET standard_conforming_strings = on"); err != nil {
logger.Error("enforce standard SQL string semantics failed", "error", err)
os.Exit(1)
}
if _, err := conn.Exec(ctx, `
CREATE TABLE IF NOT EXISTS schema_migrations (
@ -64,6 +68,14 @@ CREATE TABLE IF NOT EXISTS schema_migrations (
logger.Error("begin migration failed", "version", version, "error", err)
os.Exit(1)
}
// Pin string parsing semantics inside every migration transaction. A
// previous migration may have changed the session GUC, while each file is
// parsed and executed independently.
if _, err := tx.Exec(ctx, "SET LOCAL standard_conforming_strings = on"); err != nil {
_ = tx.Rollback(ctx)
logger.Error("enforce migration SQL string semantics failed", "version", version, "error", err)
os.Exit(1)
}
if _, err := tx.Exec(ctx, string(sqlBytes)); err != nil {
_ = tx.Rollback(ctx)
logger.Error("execute migration failed", "version", version, "error", err)

View File

@ -13,7 +13,7 @@ require (
github.com/riverqueue/river v0.24.0
github.com/riverqueue/river/riverdriver/riverpgxv5 v0.24.0
github.com/riverqueue/river/rivertype v0.24.0
golang.org/x/crypto v0.37.0
golang.org/x/crypto v0.52.0
golang.org/x/oauth2 v0.36.0
)
@ -36,6 +36,6 @@ require (
github.com/tidwall/sjson v1.2.5 // indirect
go.uber.org/goleak v1.3.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/text v0.36.0 // indirect
golang.org/x/text v0.37.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

View File

@ -67,14 +67,14 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

View File

@ -44,6 +44,6 @@
"@types/react-dom": "^19.0.0",
"tailwindcss": "^4.3.0",
"typescript": "^5.8.0",
"vitest": "3.2.4"
"vitest": "3.2.6"
}
}

View File

@ -0,0 +1,36 @@
log:
level: info
runner:
file: /data/.runner
capacity: 1
envs:
GOPROXY: "https://goproxy.cn,direct"
GOSUMDB: "sum.golang.google.cn"
PATH: "/opt/easyai-gateway-ci/bin:/opt/easyai-gateway-ci/toolchains/go/bin:/opt/easyai-gateway-ci/toolchains/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
timeout: 3h
shutdown_timeout: 5m
insecure: false
fetch_timeout: 5s
fetch_interval: 2s
labels:
- "easyai-gateway-ci-unprivileged-v2:docker://docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7"
cache:
enabled: false
container:
docker_host: "-"
privileged: false
force_pull: false
force_rebuild: false
require_docker: true
docker_timeout: 1m
bind_workdir: false
workdir_parent: workspace
options: "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro"
valid_volumes:
- "/opt/easyai-gateway-ci"
host:
workdir_parent: /data/host-work

View File

@ -0,0 +1,44 @@
[Unit]
Description=EasyAI AI Gateway rootless DinD Gitea Actions runner v2
Requires=docker.service
After=network-online.target docker.service
Wants=network-online.target
[Service]
Type=simple
User=root
Group=root
Environment=RUNNER_SECURITY_OPTIONS=
Environment=RUNNER_IMAGE_REF=
EnvironmentFile=-/etc/easyai-gateway-ci-v2/runner.env
ExecStartPre=-/usr/bin/docker rm --force easyai-gateway-ci-v2-runner
ExecStart=/usr/bin/docker run --rm --pull=never --name easyai-gateway-ci-v2-runner --privileged --cpus 2 --memory 2g --memory-swap 3g --pids-limit 1024 $RUNNER_SECURITY_OPTIONS --volume easyai-gateway-ci-v2-data:/data --volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro --volume /etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro --volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro --env CONFIG_FILE=/config.yaml --env DOCKERD_ROOTLESS_ROOTLESSKIT_NET=slirp4netns --env DOCKERD_ROOTLESS_ROOTLESSKIT_MTU=65520 --env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns $RUNNER_IMAGE_REF
ExecStop=-/usr/bin/docker stop --time 360 easyai-gateway-ci-v2-runner
Restart=always
RestartSec=5s
TimeoutStartSec=60s
TimeoutStopSec=390s
# systemd only launches the host Docker client. Pull-request jobs run inside the
# nested rootless daemon and never receive the host or nested Docker socket.
NoNewPrivileges=true
CapabilityBoundingSet=
AmbientCapabilities=
PrivateTmp=true
PrivateDevices=true
ProtectSystem=strict
ProtectHome=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectKernelLogs=true
ProtectControlGroups=true
ProtectClock=true
ProtectHostname=true
LockPersonality=true
RestrictRealtime=true
RestrictSUIDSGID=true
SystemCallArchitectures=native
UMask=0077
[Install]
WantedBy=multi-user.target

View File

@ -0,0 +1 @@
2d6c16fec0bec9c0288e5cb142b458af982fff8f

View File

@ -0,0 +1,70 @@
# ADR-001: 隔离质量 CI 与 root-owned 生产发布
## 状态
Accepted
## 日期
2026-07-17
## 背景
AI Gateway 已通过独立部署仓库和 Docker Compose 运行在 `110.42.51.33`,公共入口是 `https://ai.51easyai.com`。同一服务器还运行认证中心、K3s 和其他 EasyAI 服务磁盘与信任边界都很紧张。Gitea 1.22 对依赖 Job、`environment`、`permissions` 和 `concurrency` 的支持也不能作为生产发布的安全边界。
源码仓需要为 Pull Request、`main` 和版本 Tag 提供真实质量门禁;生产发布还必须满足源码可追溯、镜像不可变、数据库迁移前有备份、失败可恢复上一应用版本,以及 Tag 控制的脚本不能直接取得宿主 root 权限。
## 决策
### 质量 CI
- `BCAI/easyai-ai-gateway` 使用仓库级 Runner 标签 `easyai-gateway-ci-unprivileged-v2`。PR 与 `main``ci.yml` 执行context 分别为 `ci / verify (pull_request)`、`ci / verify (push)``v*` Tag 必须由独立 `release-ci.yml` 执行context 为 `release-ci / verify-tag (push)`。两份 workflow 执行相同质量门禁,但 Tag 不复用 `main` 的旧 context。
- Runner 外层固定为 `gitea/runner:2.0.0-dind-rootless` 的指定 digestJob 固定为 `node:24.16.0-bookworm` 的指定 digest。外层按官方 rootless DinD 要求使用 `--privileged`,但内层 Docker daemon 和 Runner 以 UID 1000 运行。
- 不把宿主或内层 Docker socket 挂入 Job。Runner 配置使用 `docker_host: "-"`、`privileged: false`,工作流唯一允许的 bind source 是只读 `/opt/easyai-gateway-ci`,其中提供固定 Go、Node、pnpm、ShellCheck、Compose、Trivy 和 govulncheck 工具链。ShellCheck 与 Compose 使用官方静态发行物和固定 SHA-256不复制宿主的可变插件或在 Job 中临时安装。RootlessKit 强制 `--pidns`,即使 PR 传入 `--pid=host`Job 也不能进入外层 Runner 的 PID namespaceprovision 会实测 namespace inode。
- CI 只执行格式、静态检查、单元测试、依赖审计、Compose 校验、仓库扫描和 CI 自测;不构建发布镜像,不调用 `sudo`,不执行生产发布助手。
- PR、`main` 与 Tag CI 都把 `deploy/ci/production-migration-base` 中的当前生产提交作为数据库兼容基线:已经存在于生产基线的迁移不得修改或删除,新迁移静态拒绝 DROP、TRUNCATE、DELETE、MERGE、CREATE OR REPLACE、RENAME、列类型/非空收紧等 contract 操作。PR/`main` 还使用事件中不可变的 base/before SHA 比对,确保基线更新稍有滞后时,已经进入 `main` 的新迁移仍不可被改写。
- 版本 Tag 必须是完整 SemVer且目标提交必须属于 `main` 历史。该检查是发布前置证据,但本身不授予生产权限。
- Runner 注册状态保存在专用 Docker named volume首次注册 Token 只传给短命注册容器;长期 systemd 服务不保存 Token也不加入宿主 `docker`/`sudo` 用户组。
### 生产 CD
- 生产发布由部署仓的 root-owned dispatcher 负责,和源码 CI Runner 分离。dispatcher 只接受受保护的 `main` 历史版本 Tag并在对应源码 SHA 上同时等待 `ci / verify (push)` 与本次 Tag 新产生的 `release-ci / verify-tag (push)` 成功。
- root 服务不执行 Tag 控制的 `pnpm`、测试脚本、源码仓 shell 脚本或应用程序。它只调用部署仓中由 root 固定和审核的命令,通过 BuildKit 构建目标镜像、用 Trivy 扫描镜像、发布 Registry digest再进行备份、迁移、健康检查和回滚。
- API/Web 镜像以完整 Git SHA 作为可追溯 Tag但生产 Compose 使用 Registry digest。服务器为每个 Git SHA 保存 root-only digest 清单,并拒绝把未知的既有 SHA Tag 当作可信发布物。
- 发布助手在迁移前生成并验证 PostgreSQL custom-format 备份,保留最近 5 份。应用健康检查失败时恢复上一组 digest-pinned 镜像;数据库恢复仍需人工确认。
### 残余风险与触发策略
rootless DinD 显著缩小了 Job 到宿主 Docker/root 的直接通路,但不是虚拟机级隔离。外层容器仍需 `--privileged`Runner、rootless Docker、内核或容器运行时漏洞以及 Gitea 对 `jobs.<job>.container.options` 的透传都可能造成容器边界突破。Job 还保留依赖安装所需的出站网络,并可能消耗共享主机的 CPU、内存和磁盘。因此
- 来自同仓受信任分支的 PR 可自动运行;来自 Fork 的 PR 必须先由维护者检查工作流变更并在 Gitea 中批准,不能自动调度。
- CI Job 不接收生产 Secret、宿主 Docker socket、部署目录或 Runner 状态目录。
- Runner 容量固定为 1外层容器设 4096 PID 上限;磁盘在安装前后硬门禁,但共享主机仍不具备完整的 CPU、内存和长期磁盘配额隔离。
- 中长期应把不受信任 PR CI 迁移到独立主机或虚拟机;当前 rootless Runner 不能被描述为强多租户沙箱。
- Ubuntu 22.04 没有受限 user namespace生产主机缺少 `rootlesskit` AppArmor profile 时不硬编码该 profile。provision 会先探测;在启用了 `kernel.apparmor_restrict_unprivileged_userns=1` 的新系统上,如果 profile 缺失则失败,不会降级为 `apparmor=unconfined` 或关闭内核限制。
## 备选方案
### 让源码 Runner 直接挂载宿主 Docker socket
拒绝。Docker socket 等价于宿主 root任何 PR 工作流都可能取得生产服务器控制权Fork 审批也不能把它变成可靠的技术隔离。
### 使用 host executor 但移除 Docker 组
拒绝。host Job 仍可读取宿主可见文件、消耗宿主资源并攻击同一用户进程Gitea 对容器选项的处理也无法为 host executor 提供隔离。
### 让 Tag 工作流通过 `sudo` 调用发布脚本
拒绝。Tag 可以控制 checkout 中的脚本和应用代码root 服务不得执行这些内容。固定部署仓 dispatcher 是独立的受保护信任根。
### `main` 通过后立即发布或只依赖 `latest`
拒绝。语义版本 Tag 是明确的发布授权点digest 才能证明运行中的容器对应不可变产物并可靠选择回滚版本。
## 影响
- PR、`main` 和版本 Tag 都有自动质量证据,但源码仓 CI 成功不会单独获得生产权限。
- 创建受保护版本 Tag 后,部署仓 dispatcher 必须同时取得同 SHA 的 `main` context 和独立 Tag context不能把旧 `main` 成功状态误当作本次 Tag 验证。
- 首次安装会清理本项目专属 Buildx 缓存并执行 8 GiB 前置、4 GiB 后置磁盘门禁,不会全局 prune 共享 Docker 状态。
- 首次启用 CD 仍需捕获当前运行镜像作为 bootstrap 回滚基线,并保证数据库迁移对上一应用版本向后兼容。
- 每次生产发布和恢复演练成功后,必须用单独受审 PR 把 `deploy/ci/production-migration-base` 前移到刚刚部署的完整源码 SHA不得在发布成功前前移。静态门禁不能证明任意数据变换可逆生产发布仍必须先做可恢复备份并定期执行真实 `pg_restore` 演练。

View File

@ -0,0 +1,157 @@
# 生产 CI/CD 运行手册
## 信任边界与固定资源
- 源码仓:`https://git.51easyai.com/BCAI/easyai-ai-gateway`
- CI Runner`easyai-gateway-ci-v2-runner.service`
- CI 标签:`easyai-gateway-ci-unprivileged-v2`
- Runner 状态Docker named volume `easyai-gateway-ci-v2-data` 内的 `/data/.runner`
- 只读工具链:`/opt/easyai-gateway-ci`
- Runner 配置:`/etc/easyai-gateway-ci-v2/config.yaml`
- Runner 运行参数:`/etc/easyai-gateway-ci-v2/runner.env`
- 生产入口:`https://ai.51easyai.com`
- 部署目录:`/root/easyai-ai-gateway-deploy`
源码仓只执行质量 CIPR context 是 `ci / verify (pull_request)``main` context 是 `ci / verify (push)`,版本 Tag 使用独立 workflow/context `release-ci / verify-tag (push)`。三者都执行生产迁移兼容门禁、Go 格式/vet/test/govulncheck、前端 lint/test/build、依赖审计、Compose 校验、Trivy 仓库扫描和 CI 脚本自测。Job 没有 Docker socket、`sudo` 或部署目录权限,也不构建发布镜像。
生产 CD 由部署仓的 root-owned dispatcher 执行。它在相同源码 SHA 上同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)` 成功,然后只运行部署仓中固定的 BuildKit 构建、Trivy 镜像扫描、digest 发布、备份、迁移、健康检查和回滚命令;不得执行 Tag checkout 中的 `pnpm`、测试、shell 脚本或应用程序。
provision 安装 checksum-pinned 的官方静态 ShellCheck `0.11.0` 和 Docker Compose `5.3.1`,不会依赖宿主是否预装这些命令。所有下载都在使用前校验固定 SHA-256并在精确 Node Job 镜像内实际执行版本检查。
## 首次安装或修复 CI Runner
前置条件Linux x86_64、可用 Docker Engine、root 权限,以及 Docker 文件系统至少 8 GiB 可用空间。脚本只压缩本项目专属 `easyai-gateway-ci` Buildx 缓存,不运行全局 `docker system prune`,因为该服务器与其他服务共享 Docker 状态。
从 Gitea 仓库设置的 Actions Runner 页面取得一次性仓库注册 Token在服务器可信的源码 checkout 中执行:
```bash
RUNNER_REGISTRATION_TOKEN='<一次性 Token>' ./scripts/provision-ci-runner.sh
```
已有有效 named volume 时可直接重跑,不再需要 Token
```bash
./scripts/provision-ci-runner.sh
```
注册 Token 通过交互 stdin 送给短命注册容器不出现在宿主或容器的命令行参数中provision 随后立即从环境中清除它,长期 systemd unit 也不包含 Token。不要输出或复制 `/data/.runner` 内容。脚本会停用旧 host Runner并确保旧 Runner 用户和 v2 遗留用户不属于 `docker`/`sudo` 组。
默认磁盘门禁可用环境变量提高,但生产环境不应降低:
```bash
CI_RUNNER_MIN_FREE_GIB=8 \
CI_RUNNER_MIN_POST_INSTALL_FREE_GIB=4 \
RUNNER_REGISTRATION_TOKEN='<一次性 Token>' \
./scripts/provision-ci-runner.sh
```
## AppArmor 分支
Ubuntu 22.04 默认没有 `kernel.apparmor_restrict_unprivileged_userns` 限制,当前服务器没有 `rootlesskit` profile 是受支持路径。provision 会实测 Docker 是否接受该 profile不存在时写入
```text
RUNNER_SECURITY_OPTIONS=""
```
如果新系统的 `/proc/sys/kernel/apparmor_restrict_unprivileged_userns``1`,则必须先由宿主发行版或 Docker 安装包提供并加载 `rootlesskit` profileprovision 只探测和使用,不会自行复制一份 profile。缺失时 provision 会停止;不得手工关闭该 sysctl也不得使用 `apparmor=unconfined` 绕过。
参考:[Docker Rootless Docker-in-Docker](https://docs.docker.com/engine/security/rootless/tips/#rootless-docker-in-docker)、[Ubuntu 安全特性版本矩阵](https://documentation.ubuntu.com/security/security-features/security-features-overview/)。
## Runner 真实验证
安装脚本会先启动一个无 Token、无持久 Runner 状态挂载、无宿主 bind mount 的短命 rootless DinD probe随后启动长期服务确认 Runner 已向 Gitea declare、通过内层 `docker info` 验证 `name=rootless`,再从内层 daemon 拉取固定 digest 的 Job 镜像,并在该镜像中实际运行全部工具。人工复核:
```bash
systemctl is-active easyai-gateway-ci-v2-runner.service
systemctl is-enabled easyai-gateway-ci-v2-runner.service
systemctl cat easyai-gateway-ci-v2-runner.service
journalctl -u easyai-gateway-ci-v2-runner.service --since '15 minutes ago' --no-pager
docker inspect -f 'user={{.Config.User}} privileged={{.HostConfig.Privileged}} apparmor={{.AppArmorProfile}}' \
easyai-gateway-ci-v2-runner
docker inspect -f '{{range .Mounts}}{{println .Source " -> " .Destination .Mode}}{{end}}' \
easyai-gateway-ci-v2-runner
docker exec easyai-gateway-ci-v2-runner docker info \
--format '{{range .SecurityOptions}}{{println .}}{{end}}'
docker exec easyai-gateway-ci-v2-runner docker image inspect \
'docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7' \
--format '{{.Id}}'
```
预期:外层 image user 为 `rootless`、外层因 DinD 为 `privileged=true`、内层安全选项含 `name=rootless`RootlessKit 使用独立 PID namespace内层 `--pid=host` 也不能看到外层 Runner挂载仅包含专用 `/data`、只读配置和只读工具链,不能出现 `/var/run/docker.sock``/run/docker.sock`
在 Gitea 中分别验证:
1. 同仓 PR 的 `ci / verify (pull_request)` 成功。
2. `main` Push 的 `ci / verify (push)` 成功。
3. 合法 SemVer Tag 新产生独立 `release-ci / verify-tag (push)`;非法或不属于 `main` 历史的 Tag 失败,且 Tag context 不能被旧 `main` context 替代。
4. Fork PR 默认不自动调度,维护者先检查 `.gitea/workflows/**` 和容器选项后再批准。
PR 与 `main` Push workflow 都校验事件携带的 base/before SHA 必须是触发 SHA 的祖先;分支落后或主干历史被改写时 CI 会 fail-closed。PR 先合并最新 `main` 再重跑Gitea 分支保护同时启用 `block_on_outdated_branch`
## Fork PR 残余风险
rootless DinD 不是 VM 沙箱。外层仍按官方要求使用 `--privileged`Gitea runner 也会解析 PR 控制的 `jobs.<job>.container.options`。配置已禁止 Job privileged、宿主/内层 Docker socket 和任意 bind source但 Runner、rootless Docker、容器运行时或内核漏洞仍可能突破边界Job 的出站网络和资源消耗也不是强租户隔离。因此 Fork PR 必须维护者批准CI 不得注入生产 Secret未来应迁移到独立 CI 主机或虚拟机。
## 数据库迁移兼容门禁
`deploy/ci/production-migration-base` 必须等于当前生产 API 对应的完整源码 SHA。`scripts/ci-validate-migrations.mjs` 只允许新增、按文件名严格追加且为普通文件的迁移;生产已知迁移以及已经进入 `main` 的迁移不可修改或删除。DROP、TRUNCATE、DELETE、MERGE、CREATE OR REPLACE、动态 EXECUTE、事务控制、DO/函数/过程定义、RENAME、列类型变更、增加/设置 `NOT NULL`、删除默认值和 REVOKE 会直接使 CI 失败。需要过程式迁移时先拆成可审计的声明式 expand/contract SQL不能把过程体藏在普通或 dollar-quoted 字符串中。
该规则允许 INSERT/UPDATE 等数据迁移,因此不能替代备份和恢复验证。每次生产发布成功并完成 `pg_restore` 演练后,创建一个只前移 `deploy/ci/production-migration-base` 的受审 PR不要把基线指向尚未成功部署的提交。
## 发布生产版本
1. 合并到受保护 `main`,确认源码 CI 成功。
2. 在已验证提交上创建完整 SemVer Tag 并推送。
3. 确认同一 SHA 的 `ci / verify (push)` 和本次 `release-ci / verify-tag (push)` 都成功。
4. 由部署仓 root-owned dispatcher 构建、扫描并发布 digest源码仓工作流不会直接调用生产 helper。
5. 发布、健康检查和恢复演练完成后,用单独 PR 把 `deploy/ci/production-migration-base` 更新为该生产 Tag 的完整源码 SHA。
```bash
git switch main
git pull --ff-only
git tag -a v0.1.1 -m 'release: v0.1.1'
git push origin v0.1.1
```
dispatcher 以完整 Git SHA 发布 Registry Tag并把 Registry 返回的 digest 写入服务器 root-only 发布清单。生产 Compose 使用 `repository@sha256:...`,不是 `latest`、版本号或可变 SHA Tag。
## 发布后验证
```bash
curl -fsS https://ai.51easyai.com/gateway-api/healthz
curl -fsS https://ai.51easyai.com/gateway-api/readyz
ssh root@110.42.51.33 'cd /root/easyai-ai-gateway-deploy && ./gateway-ops.sh ps'
```
同时检查服务器磁盘、API 日志和数据库容器健康状态。发布后的首小时如出现错误率翻倍、P95 延迟增加 50%、数据完整性或安全问题,应立即回滚。
## 应用与数据库回滚
发布失败时固定部署脚本会恢复上一组 API/Web digest 并重复健康检查。人工回滚只选择服务器已有 root-only 发布清单的历史 Git SHA
```bash
sudo /usr/local/sbin/easyai-ai-gateway-release <40 位历史 Git SHA>
```
没有 `/root/easyai-ai-gateway-deploy/.releases/<SHA>.env` 的远端 SHA Tag 不应被盲目信任。数据库备份位于 `/var/backups/easyai-ai-gateway`,只保留最近 5 份。应用回滚不会自动恢复数据库;只有停止写入并再次保留当前副本后,操作员才能选择备份执行 `pg_restore`
## Runner 故障恢复
先在 Gitea 确认没有运行中的 Job再检查服务与两个 daemon
```bash
systemctl status easyai-gateway-ci-v2-runner.service --no-pager
docker logs --tail 100 easyai-gateway-ci-v2-runner
docker exec easyai-gateway-ci-v2-runner docker info
df -h /var/lib/docker
```
确认空闲后才执行:
```bash
systemctl restart easyai-gateway-ci-v2-runner.service
journalctl -u easyai-gateway-ci-v2-runner.service -n 100 --no-pager
```
不要删除 `easyai-gateway-ci-v2-data` volume它包含 Runner 注册状态和内层 daemon 缓存。不得用全局 prune 处理磁盘压力。

View File

@ -21,6 +21,7 @@
"dotenv-cli": "^11.0.0",
"nx": "^21.0.0",
"typescript": "^5.8.0",
"vite": "^7.0.0"
"vite": "^7.3.5",
"vitest": "3.2.6"
}
}

File diff suppressed because it is too large Load Diff

View File

@ -1,6 +1,20 @@
packages:
- apps/web
- packages/*
allowBuilds:
esbuild: true
nx: true
overrides:
'@babel/core@<=7.29.0': 7.29.6
'dompurify@<=3.4.10': 3.4.11
'esbuild@>=0.27.3 <0.28.1': 0.28.1
'form-data@>=4.0.0 <4.0.6': 4.0.6
'js-yaml@<3.15.0': 4.1.1
'mermaid@>=11.0.0-alpha.1 <=11.14.0': 11.15.0
'minimatch@>=9.0.0 <9.0.7': 9.0.7
'@nx/js>picomatch': 4.0.5
'@nx/vite>picomatch': 4.0.5
'@nx/workspace>picomatch': 4.0.5
'tmp@<0.2.7': 0.2.7

101
scripts/ci-build-images.sh Executable file
View File

@ -0,0 +1,101 @@
#!/usr/bin/env bash
set -euo pipefail
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
cd "$root"
tag=${IMAGE_TAG:-}
registry=${AI_GATEWAY_IMAGE_REGISTRY:-registry.cn-shanghai.aliyuncs.com/easyaigc}
builder=${AI_GATEWAY_BUILDX_BUILDER:-easyai-gateway-ci}
cache_limit=${AI_GATEWAY_BUILDX_CACHE_LIMIT:-2gb}
platform=${AI_GATEWAY_PLATFORM:-linux/amd64}
keep_images=0
build_complete=0
readonly trivy_db_repository=ghcr.m.daocloud.io/aquasecurity/trivy-db:2
case ${1:-} in
'') ;;
--keep-images) keep_images=1 ;;
*)
echo 'usage: ci-build-images.sh [--keep-images]' >&2
exit 64
;;
esac
if [[ ! $tag =~ ^[0-9a-f]{40}$ ]]; then
echo 'IMAGE_TAG must be a full lowercase Git SHA' >&2
exit 1
fi
if [[ ! $registry =~ ^[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._/-]+)+$ ]]; then
echo 'AI_GATEWAY_IMAGE_REGISTRY has an invalid format' >&2
exit 1
fi
if [[ ! $cache_limit =~ ^[1-9][0-9]*(kb|mb|gb)$ ]]; then
echo 'AI_GATEWAY_BUILDX_CACHE_LIMIT has an invalid format' >&2
exit 1
fi
api_image="$registry/ai-gateway:$tag"
web_image="$registry/ai-gateway-web:$tag"
cleanup() {
original_status=$?
trap - EXIT HUP INT TERM
cleanup_status=0
docker buildx prune --builder "$builder" --force --max-used-space "$cache_limit" >/dev/null || {
cleanup_status=$?
echo 'failed to prune the CI BuildKit cache' >&2
}
if [[ $keep_images -ne 1 || $build_complete -ne 1 ]]; then
docker image rm "$api_image" "$web_image" >/dev/null 2>&1 || {
image_rm_status=$?
[[ $cleanup_status -ne 0 ]] || cleanup_status=$image_rm_status
echo 'failed to remove CI image tags' >&2
}
fi
if [[ $original_status -ne 0 ]]; then
exit "$original_status"
fi
exit "$cleanup_status"
}
trap cleanup EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
if ! docker buildx inspect "$builder" >/dev/null 2>&1; then
docker buildx create --name "$builder" --driver docker-container --use >/dev/null
fi
docker buildx inspect "$builder" --bootstrap >/dev/null
common_args=(
--builder "$builder"
--platform "$platform"
--file Dockerfile
--pull
--load
)
docker buildx build --builder "$builder" \
"${common_args[@]:2}" \
--target api \
--build-arg "GO_BUILD_IMAGE=${AI_GATEWAY_GO_BUILD_IMAGE:-docker.m.daocloud.io/library/golang:1.26.5-alpine}" \
--build-arg "API_RUNTIME_IMAGE=${AI_GATEWAY_API_RUNTIME_IMAGE:-docker.m.daocloud.io/library/alpine:3.22}" \
--tag "$api_image" .
docker buildx build --builder "$builder" \
"${common_args[@]:2}" \
--target web \
--build-arg "NODE_BUILD_IMAGE=${AI_GATEWAY_NODE_BUILD_IMAGE:-docker.m.daocloud.io/library/node:24-alpine}" \
--build-arg "WEB_RUNTIME_IMAGE=${AI_GATEWAY_WEB_RUNTIME_IMAGE:-docker.m.daocloud.io/library/nginx:1.27-alpine}" \
--tag "$web_image" .
for image in "$api_image" "$web_image"; do
trivy image --db-repository "$trivy_db_repository" \
--scanners vuln,secret --severity HIGH,CRITICAL --ignore-unfixed \
--exit-code 1 --timeout 15m "$image"
done
build_complete=1
printf 'gateway_images=PASS git_sha=%s api=%s web=%s\n' "$tag" "$api_image" "$web_image"

View File

@ -0,0 +1,435 @@
#!/usr/bin/env node
import { execFileSync, spawnSync } from 'node:child_process'
import { lstatSync, readFileSync } from 'node:fs'
import { basename } from 'node:path'
const migrationDirectory = 'apps/api/migrations/'
const baselineFile = 'deploy/ci/production-migration-base'
const base = process.argv[2] || process.env.PRODUCTION_MIGRATION_BASE_SHA || ''
const immutableRef =
process.argv[3] || process.env.MIGRATION_IMMUTABILITY_REF || ''
function fail(message) {
console.error(`migration_safety=FAIL ${message}`)
process.exit(1)
}
function git(args, options = {}) {
try {
return execFileSync('git', args, {
encoding: options.encoding ?? 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
})
} catch (error) {
const stderr = error.stderr?.toString().trim()
fail(stderr ? `git_error=${JSON.stringify(stderr)}` : 'git command failed')
}
}
if (!/^[0-9a-f]{40}$/.test(base)) {
fail('PRODUCTION_MIGRATION_BASE_SHA must be a full lowercase commit SHA')
}
const root = git(['rev-parse', '--show-toplevel']).trim()
process.chdir(root)
const resolvedBase = git([
'rev-parse',
'--verify',
'--end-of-options',
`${base}^{commit}`,
]).trim()
if (resolvedBase !== base) {
fail('production migration baseline did not resolve exactly')
}
const ancestry = spawnSync('git', ['merge-base', '--is-ancestor', base, 'HEAD'], {
stdio: 'ignore',
})
if (ancestry.status !== 0) {
fail('production migration baseline is not an ancestor of HEAD')
}
const head = git(['rev-parse', '--verify', 'HEAD^{commit}']).trim()
const baselineTreeEntry = git(['ls-tree', 'HEAD', '--', baselineFile]).trim()
let baselineState
try {
baselineState = lstatSync(baselineFile)
} catch {
fail('production migration baseline file is unavailable in the worktree')
}
if (
!/^100644 blob [0-9a-f]+\t/.test(baselineTreeEntry) ||
!baselineState.isFile() ||
baselineState.isSymbolicLink()
) {
fail('production migration baseline must be a mode 100644 regular file')
}
if (readFileSync(baselineFile, 'utf8').trim() !== base) {
fail('production migration baseline argument does not match the reviewed file')
}
function changedMigrations(from) {
const rawChanges = git(
[
'diff',
'--name-status',
'-z',
'--no-renames',
`${from}..HEAD`,
'--',
migrationDirectory,
],
{ encoding: null },
)
const fields = rawChanges.toString('utf8').split('\0')
if (fields.at(-1) === '') fields.pop()
if (fields.length % 2 !== 0) {
fail('could not parse changed migration paths')
}
return fields
}
const fields = changedMigrations(base)
function executableSql(sql, source = 'SQL input') {
let output = ''
for (let index = 0; index < sql.length; index += 1) {
const current = sql[index]
const next = sql[index + 1]
if (current === '-' && next === '-') {
index += 2
while (index < sql.length && sql[index] !== '\n') index += 1
output += '\n'
continue
}
if (current === '/' && next === '*') {
let depth = 1
index += 2
while (index < sql.length && depth > 0) {
if (sql[index] === '/' && sql[index + 1] === '*') {
depth += 1
index += 2
continue
}
if (sql[index] === '*' && sql[index + 1] === '/') {
depth -= 1
index += 2
continue
}
index += 1
}
if (depth !== 0) fail('unterminated SQL block comment')
index -= 1
output += ' '
continue
}
if (current === "'" || current === '"') {
const prefix = sql.slice(Math.max(0, index - 2), index).toUpperCase()
const previous = sql[index - 1]?.toUpperCase()
const beforePrevious = sql[index - 2] || ''
const beforeUnicodePrefix = sql[index - 3] || ''
const escapeString =
current === "'" &&
previous === 'E' &&
!/[A-Za-z0-9_$]/.test(beforePrevious)
const unicodeString =
prefix === 'U&' && !/[A-Za-z0-9_$]/.test(beforeUnicodePrefix)
if (escapeString || unicodeString) {
fail(`${source}: PostgreSQL escape/unicode strings are not allowed in migrations`)
}
const quote = current
index += 1
let closed = false
while (index < sql.length) {
if (sql[index] === quote && sql[index + 1] === quote) {
index += 2
continue
}
if (sql[index] === quote) {
closed = true
break
}
index += 1
}
if (!closed) fail('unterminated SQL quoted value')
output += ' '
continue
}
if (current === '$') {
const delimiter = sql.slice(index).match(/^\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/)?.[0]
if (delimiter) {
const bodyStart = index + delimiter.length
const bodyEnd = sql.indexOf(delimiter, bodyStart)
if (bodyEnd === -1) fail('unterminated SQL dollar-quoted body')
// PostgreSQL function bodies may contain executable DDL. Scan their
// contents recursively instead of treating them as inert strings.
const bodySql = executableSql(sql.slice(bodyStart, bodyEnd), source)
// A bare END terminates a PL/pgSQL block inside a dollar-quoted body,
// but at the top level PostgreSQL treats END as COMMIT. Mask only the
// block terminator here so the top-level transaction rule stays exact.
const bodyWithoutBlockEnd = bodySql
.split(';')
.map((statement) => (/^\s*END\s*$/i.test(statement) ? ' ' : statement))
.join(';')
output += ` ${bodyWithoutBlockEnd} `
index = bodyEnd + delimiter.length - 1
continue
}
}
output += current
}
return output
}
const destructiveRules = [
{
name: 'procedural SQL body',
pattern:
/^\s*(?:DO\b|CREATE\s+(?:OR\s+REPLACE\s+)?(?:FUNCTION|PROCEDURE)\b)/i,
},
{
name: 'dynamic SQL execution',
pattern: /\bEXECUTE\b/i,
},
{
name: 'destructive DROP operation',
pattern: /\bDROP\b/i,
},
{ name: 'MERGE operation', pattern: /\bMERGE\b/i },
{
name: 'CREATE OR REPLACE operation',
pattern: /\bCREATE\s+OR\s+REPLACE\b/i,
},
{ name: 'TRUNCATE operation', pattern: /\bTRUNCATE(?:\s+TABLE)?\b/i },
{ name: 'DELETE FROM operation', pattern: /\bDELETE\s+FROM\b/i },
{ name: 'permission revocation', pattern: /\bREVOKE\b/i },
{
name: 'transaction control operation',
pattern:
/\b(?:COMMIT|ROLLBACK|ABORT|SAVEPOINT)\b|\bRELEASE\s+SAVEPOINT\b|\bPREPARE\s+TRANSACTION\b|^\s*END(?:\s+(?:WORK|TRANSACTION))?(?:\s+AND\s+(?:NO\s+)?CHAIN)?\s*$/i,
},
{
name: 'SQL string semantics override',
pattern:
/\bSET\s+(?:LOCAL\s+|SESSION\s+)?STANDARD_CONFORMING_STRINGS\b/i,
},
{
name: 'object or column rename',
pattern: /\bRENAME\b/i,
},
{ name: 'SET SCHEMA operation', pattern: /\bSET\s+SCHEMA\b/i },
{
name: 'column type change',
pattern:
/\bALTER\s+TABLE\b[\s\S]*\bALTER\s+(?:COLUMN\s+)?[\s\S]*\b(?:SET\s+DATA\s+)?TYPE\b/i,
},
{
name: 'SET NOT NULL operation',
pattern:
/\bALTER\s+TABLE\b[\s\S]*\bALTER\s+(?:COLUMN\s+)?[\s\S]*\bSET\s+NOT\s+NULL\b/i,
},
{
name: 'non-null column addition',
pattern:
/\bALTER\s+TABLE\b[\s\S]*\bADD\s+(?:COLUMN\s+)?[\s\S]*\bNOT\s+NULL\b/i,
},
{
name: 'DROP DEFAULT operation',
pattern:
/\bALTER\s+TABLE\b[\s\S]*\bALTER\s+(?:COLUMN\s+)?[\s\S]*\bDROP\s+DEFAULT\b/i,
},
]
const violations = []
let addedFiles = 0
let immutableBase = ''
const orderingCandidates = new Set()
if (immutableRef) {
if (
!/^[0-9a-f]{40}$/.test(immutableRef) &&
immutableRef !== 'refs/remotes/origin/main'
) {
fail('MIGRATION_IMMUTABILITY_REF must be a full commit SHA or refs/remotes/origin/main')
}
const immutableTip = git([
'rev-parse',
'--verify',
'--end-of-options',
`${immutableRef}^{commit}`,
]).trim()
const baseAtEvent = spawnSync(
'git',
['show', `${immutableTip}:${baselineFile}`],
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] },
)
if (baseAtEvent.status === 0) {
const previousBaselineTreeEntry = git([
'ls-tree',
immutableTip,
'--',
baselineFile,
]).trim()
if (!/^100644 blob [0-9a-f]+\t/.test(previousBaselineTreeEntry)) {
fail('previous production migration baseline must be a mode 100644 regular file')
}
const previousProductionBase = baseAtEvent.stdout.trim()
if (!/^[0-9a-f]{40}$/.test(previousProductionBase)) {
fail('previous production migration baseline is invalid')
}
const baselineAdvance = spawnSync(
'git',
['merge-base', '--is-ancestor', previousProductionBase, base],
{ stdio: 'ignore' },
)
if (baselineAdvance.status !== 0) {
fail('production migration baseline must only move forward')
}
}
const baselineWasAlreadyReviewed = spawnSync(
'git',
['merge-base', '--is-ancestor', base, immutableTip],
{ stdio: 'ignore' },
)
if (baselineWasAlreadyReviewed.status !== 0) {
fail('production migration baseline must belong to the immutable event base history')
}
if (immutableTip === head) {
const parent = spawnSync(
'git',
['rev-parse', '--verify', `${immutableTip}^1^{commit}`],
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] },
)
immutableBase = parent.status === 0 ? parent.stdout.trim() : ''
} else {
const immutableAncestry = spawnSync(
'git',
['merge-base', '--is-ancestor', immutableTip, head],
{ stdio: 'ignore' },
)
if (immutableAncestry.status !== 0) {
fail('migration immutability base is not an ancestor of HEAD')
}
immutableBase = immutableTip
}
if (immutableBase) {
const immutableChanges = changedMigrations(immutableBase)
for (let index = 0; index < immutableChanges.length; index += 2) {
const status = immutableChanges[index]
const file = immutableChanges[index + 1]
if (status === 'A') {
orderingCandidates.add(file)
} else {
violations.push(
`${file}: migration present on main is immutable (status ${status})`,
)
}
}
}
}
function migrationNamesAt(ref) {
const output = git(
['ls-tree', '-r', '-z', '--name-only', ref, '--', migrationDirectory],
{ encoding: null },
)
return output
.toString('utf8')
.split('\0')
.filter(Boolean)
.filter((file) => {
const relative = file.slice(migrationDirectory.length)
return file.startsWith(migrationDirectory) && !relative.includes('/')
})
.map((file) => basename(file))
.filter((name) => /^\d{4}_[a-z0-9][a-z0-9_]*\.sql$/.test(name))
.sort()
}
const orderingBase = immutableBase || base
const existingNames = migrationNamesAt(orderingBase)
const previousMaximumName = existingNames.at(-1) || ''
if (!immutableBase) {
for (let index = 0; index < fields.length; index += 2) {
if (fields[index] === 'A') orderingCandidates.add(fields[index + 1])
}
}
for (let index = 0; index < fields.length; index += 2) {
const status = fields[index]
const file = fields[index + 1]
if (status !== 'A') {
violations.push(`${file}: applied migrations are immutable (status ${status})`)
continue
}
addedFiles += 1
const filename = basename(file)
if (
file !== `${migrationDirectory}${filename}` ||
!/^\d{4}_[a-z0-9][a-z0-9_]*\.sql$/.test(filename)
) {
violations.push(`${file}: new migration must use NNNN_lowercase_name.sql`)
continue
}
if (
orderingCandidates.has(file) &&
previousMaximumName &&
filename <= previousMaximumName
) {
violations.push(
`${file}: new migration name must sort after ${previousMaximumName}`,
)
continue
}
const treeEntry = git(['ls-tree', 'HEAD', '--', file]).trim()
let fileState
try {
fileState = lstatSync(file)
} catch {
violations.push(`${file}: new migration is unavailable in the worktree`)
continue
}
if (
!/^100644 blob [0-9a-f]+\t/.test(treeEntry) ||
!fileState.isFile() ||
fileState.isSymbolicLink()
) {
violations.push(`${file}: new migration must be a mode 100644 regular file`)
continue
}
const statements = executableSql(readFileSync(file, 'utf8'), file).split(';')
for (const statement of statements) {
for (const rule of destructiveRules) {
if (rule.pattern.test(statement)) {
violations.push(`${file}: ${rule.name}`)
}
}
}
}
if (violations.length > 0) {
for (const violation of [...new Set(violations)]) {
console.error(`migration_safety_violation=${JSON.stringify(violation)}`)
}
fail(`base=${base} violations=${new Set(violations).size}`)
}
console.log(`migration_safety=PASS base=${base} added_files=${addedFiles}`)

22
scripts/ci-validate-semver.sh Executable file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -euo pipefail
tag=${1:-}
die() {
printf 'invalid release tag: %s\n' "$tag" >&2
exit 1
}
[[ $tag == v* ]] || die
version=${tag#v}
# Bash arrays discard trailing empty fields, so parsing first would accept
# values such as v1.2.3. and v1.2.3-alpha. Validate the complete string with
# the SemVer 2.0.0 grammar before doing anything else with the tag.
semver_pattern='^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(\.(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*))?(\+([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
LC_ALL=C
export LC_ALL
[[ $version =~ $semver_pattern ]] || die
printf 'release_tag=PASS tag=%s\n' "$tag"

401
scripts/provision-ci-runner.sh Executable file
View File

@ -0,0 +1,401 @@
#!/bin/sh
set -eu
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
PREFIX=/opt/easyai-gateway-ci
GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL:-https://git.51easyai.com}
RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-ci-unprivileged-v2}
RUNNER_VOLUME=easyai-gateway-ci-v2-data
RUNNER_CONTAINER=easyai-gateway-ci-v2-runner
LEGACY_BUILDER=${AI_GATEWAY_BUILDX_BUILDER:-easyai-gateway-ci}
MIN_FREE_GIB=${CI_RUNNER_MIN_FREE_GIB:-8}
MIN_POST_INSTALL_FREE_GIB=${CI_RUNNER_MIN_POST_INSTALL_FREE_GIB:-4}
RUNNER_CPUS=2
RUNNER_MEMORY=2g
RUNNER_MEMORY_SWAP=3g
RUNNER_PIDS_LIMIT=1024
GITEA_RUNNER_VERSION=2.0.0
GO_VERSION=1.26.5
GO_SHA256=5c2c3b16caefa1d968a94c1daca04a7ca301a496d9b086e17ad77bb81393f053
NODE_VERSION=24.16.0
NODE_SHA256=d804845d34eddc21dc1092b519d643ef40b1f58ec5dec5c22b1f4bd8fabde6c9
TRIVY_VERSION=0.70.0
TRIVY_SHA256=8b4376d5d6befe5c24d503f10ff136d9e0c49f9127a4279fd110b727929a5aa9
SHELLCHECK_VERSION=0.11.0
SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198
COMPOSE_VERSION=5.3.1
COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959
PNPM_VERSION=10.18.1
RUNNER_IMAGE='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a'
JOB_IMAGE='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7'
RUNNER_IMAGE_SOURCE=${CI_RUNNER_IMAGE_SOURCE:-$RUNNER_IMAGE}
NESTED_DOCKER_REGISTRY_MIRROR=${CI_NESTED_DOCKER_REGISTRY_MIRROR:-}
RUNNER_LABEL="easyai-gateway-ci-unprivileged-v2:docker://$JOB_IMAGE"
fail() {
echo "$*" >&2
exit 1
}
case $MIN_FREE_GIB:$MIN_POST_INSTALL_FREE_GIB in
*[!0-9:]* | :* | *:)
fail "CI runner free-space limits must be whole GiB values"
;;
esac
[ "$MIN_FREE_GIB" -ge 8 ] || fail "CI_RUNNER_MIN_FREE_GIB cannot be lower than 8"
[ "$MIN_POST_INSTALL_FREE_GIB" -ge 4 ] || \
fail "CI_RUNNER_MIN_POST_INSTALL_FREE_GIB cannot be lower than 4"
[ "$(id -u)" -eq 0 ] || fail "run as root"
[ "$(uname -s)-$(uname -m)" = "Linux-x86_64" ] || fail "only Linux x86_64 is supported"
case $RUNNER_IMAGE in
*"/runner:${GITEA_RUNNER_VERSION}-dind-rootless@sha256:"*) ;;
*) fail "RUNNER_IMAGE does not match GITEA_RUNNER_VERSION" ;;
esac
runner_manifest_digest=${RUNNER_IMAGE##*@}
case $RUNNER_IMAGE_SOURCE in
*@"$runner_manifest_digest") ;;
*) fail "CI_RUNNER_IMAGE_SOURCE must use the pinned runner manifest digest" ;;
esac
if [ -n "$NESTED_DOCKER_REGISTRY_MIRROR" ] && \
! printf '%s\n' "$NESTED_DOCKER_REGISTRY_MIRROR" | \
grep -Eq '^https://[A-Za-z0-9.-]+(:[0-9]+)?$'; then
fail "CI_NESTED_DOCKER_REGISTRY_MIRROR must be an HTTPS registry origin"
fi
command -v docker >/dev/null 2>&1 || fail "Docker Engine is required"
docker info >/dev/null 2>&1 || fail "Docker Engine is not reachable"
grep -Fq "\"$RUNNER_LABEL\"" "$ROOT/deploy/ci/act-runner-v2-config.yaml" || \
fail "runner label does not match the pinned job image"
# Fail closed before installing anything. Neither the legacy host runner nor an
# earlier v2 unit may poll for work while its replacement is being prepared.
systemctl disable --now easyai-gateway-act-runner.service >/dev/null 2>&1 || true
systemctl disable --now easyai-gateway-ci-v2-runner.service >/dev/null 2>&1 || true
docker rm --force "$RUNNER_CONTAINER" >/dev/null 2>&1 || true
rm -f /etc/systemd/system/easyai-gateway-act-runner.service
if id easyai-gateway-runner >/dev/null 2>&1; then
if getent group docker >/dev/null 2>&1; then
gpasswd -d easyai-gateway-runner docker >/dev/null 2>&1 || true
fi
if getent group sudo >/dev/null 2>&1; then
gpasswd -d easyai-gateway-runner sudo >/dev/null 2>&1 || true
fi
fi
if id easyai-gateway-ci-v2 >/dev/null 2>&1; then
if getent group docker >/dev/null 2>&1; then
gpasswd -d easyai-gateway-ci-v2 docker >/dev/null 2>&1 || true
fi
if getent group sudo >/dev/null 2>&1; then
gpasswd -d easyai-gateway-ci-v2 sudo >/dev/null 2>&1 || true
fi
fi
for account in easyai-gateway-runner easyai-gateway-ci-v2; do
if id "$account" >/dev/null 2>&1 && \
id -nG "$account" | tr ' ' '\n' | grep -Eq '^(docker|sudo)$'; then
fail "$account must not belong to a privileged group"
fi
done
if ! command -v make >/dev/null 2>&1 || ! command -v gcc >/dev/null 2>&1 || \
! command -v jq >/dev/null 2>&1 || ! command -v curl >/dev/null 2>&1 || \
! command -v xz >/dev/null 2>&1; then
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get install -y --no-install-recommends \
build-essential ca-certificates curl git jq perl unzip util-linux xz-utils
fi
docker_root=$(docker info --format '{{.DockerRootDir}}')
[ -n "$docker_root" ] || fail "Docker root directory could not be determined"
require_free_space() {
minimum_gib=$1
phase=$2
available_kib=$(df -Pk "$docker_root" | awk 'NR == 2 { print $4 }')
case $available_kib in
'' | *[!0-9]*) fail "free space could not be determined for $docker_root" ;;
esac
minimum_kib=$((minimum_gib * 1024 * 1024))
if [ "$available_kib" -lt "$minimum_kib" ]; then
echo "$phase requires at least ${minimum_gib} GiB free on the Docker filesystem" >&2
return 1
fi
}
# This builder belonged only to the retired gateway image-building runner. Cap
# its unused cache before adding the nested daemon; never prune global images,
# volumes, or caches owned by another service on this shared production host.
if docker buildx version >/dev/null 2>&1 && \
docker buildx inspect "$LEGACY_BUILDER" >/dev/null 2>&1; then
docker buildx prune --builder "$LEGACY_BUILDER" --force --max-used-space 512mb
fi
require_free_space "$MIN_FREE_GIB" "CI runner installation"
download_verified() {
url=$1
output=$2
checksum=$3
if [ ! -f "$output" ] || ! printf '%s %s\n' "$checksum" "$output" | sha256sum -c - >/dev/null 2>&1; then
rm -f "$output"
curl -fL --retry 5 --retry-delay 2 "$url" -o "$output"
fi
printf '%s %s\n' "$checksum" "$output" | sha256sum -c -
}
install -d -m 0755 "$PREFIX/bin" "$PREFIX/downloads" "$PREFIX/toolchains"
download_verified \
"https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \
"$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \
"$SHELLCHECK_SHA256"
rm -rf "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}"
tar -xJf "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \
-C "$PREFIX/downloads"
install -m 0755 "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "$PREFIX/bin/shellcheck"
download_verified \
"https://github.com/docker/compose/releases/download/v${COMPOSE_VERSION}/docker-compose-linux-x86_64" \
"$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" \
"$COMPOSE_SHA256"
install -m 0755 "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" "$PREFIX/bin/docker-compose"
download_verified \
"https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" \
"$PREFIX/downloads/go${GO_VERSION}.linux-amd64.tar.gz" \
"$GO_SHA256"
rm -rf "$PREFIX/toolchains/go"
tar -xzf "$PREFIX/downloads/go${GO_VERSION}.linux-amd64.tar.gz" -C "$PREFIX/toolchains"
download_verified \
"https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" \
"$PREFIX/downloads/node-v${NODE_VERSION}-linux-x64.tar.xz" \
"$NODE_SHA256"
rm -rf "$PREFIX/toolchains/node"
mkdir -p "$PREFIX/toolchains/node"
tar -xJf "$PREFIX/downloads/node-v${NODE_VERSION}-linux-x64.tar.xz" \
-C "$PREFIX/toolchains/node" --strip-components=1
PATH="$PREFIX/toolchains/node/bin:$PATH" \
"$PREFIX/toolchains/node/bin/npm" install --global --prefix "$PREFIX/toolchains/node" \
--ignore-scripts "pnpm@${PNPM_VERSION}"
download_verified \
"https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" \
"$PREFIX/downloads/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" \
"$TRIVY_SHA256"
tar -xzf "$PREFIX/downloads/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" -C "$PREFIX/bin" trivy
chmod 0755 "$PREFIX/bin/trivy"
GOBIN="$PREFIX/bin" GOPROXY="${GO_MODULE_PROXY:-https://goproxy.cn,direct}" \
"$PREFIX/toolchains/go/bin/go" install golang.org/x/vuln/cmd/govulncheck@v1.6.0
chown -R root:root "$PREFIX"
chmod -R go-w "$PREFIX"
require_free_space "$MIN_FREE_GIB" "runner image pull"
docker pull "$RUNNER_IMAGE_SOURCE"
runner_runtime_image=$(docker image inspect --format '{{.Id}}' "$RUNNER_IMAGE_SOURCE")
printf '%s\n' "$runner_runtime_image" | grep -Eq '^sha256:[0-9a-f]{64}$' || \
fail "pinned runner image did not resolve to an immutable local image ID"
require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed runner image pull"
docker volume create "$RUNNER_VOLUME" >/dev/null
# The registered runner state and the nested rootless daemon cache share one
# dedicated volume. Persisting the pinned job image lets validation finish
# before registration and avoids racing the first queued workflow for layers.
docker run --rm --pull=never --user 0:0 \
--volume "$RUNNER_VOLUME:/data" \
--entrypoint /bin/sh "$runner_runtime_image" \
-ec 'chown 1000:1000 /data && chmod 0700 /data'
install -d -m 0755 /etc/easyai-gateway-ci-v2
if [ -n "$NESTED_DOCKER_REGISTRY_MIRROR" ]; then
printf '{"data-root":"/data/docker","registry-mirrors":["%s"]}\n' "$NESTED_DOCKER_REGISTRY_MIRROR" \
> /etc/easyai-gateway-ci-v2/daemon.json
else
printf '{"data-root":"/data/docker"}\n' > /etc/easyai-gateway-ci-v2/daemon.json
fi
chmod 0644 /etc/easyai-gateway-ci-v2/daemon.json
# Docker's official rootless DinD baseline is --privileged. Gitea's example
# additionally names an AppArmor profile, but Docker does not create that host
# profile. Enable it only when this daemon proves it can start the pinned image
# with the profile; otherwise leave the option empty.
RUNNER_SECURITY_OPTIONS=
if docker run --rm --pull=never --privileged \
--security-opt apparmor=rootlesskit \
--entrypoint /bin/true "$runner_runtime_image" >/dev/null 2>&1; then
RUNNER_SECURITY_OPTIONS="--security-opt=apparmor=rootlesskit"
elif [ -r /proc/sys/kernel/apparmor_restrict_unprivileged_userns ] && \
[ "$(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns)" = "1" ]; then
fail "restricted AppArmor user namespaces require the rootlesskit profile (install it with the host Docker packages)"
fi
# Exercise the actual daemon startup before registration. This probe has no
# token, state volume, host bind mount, or socket mount.
PROBE_CONTAINER="easyai-gateway-ci-v2-probe-$$"
cleanup_probe() {
docker rm --force "$PROBE_CONTAINER" >/dev/null 2>&1 || true
}
trap 'cleanup_probe' EXIT
trap 'cleanup_probe; exit 129' HUP
trap 'cleanup_probe; exit 130' INT
trap 'cleanup_probe; exit 143' TERM
if [ -n "$RUNNER_SECURITY_OPTIONS" ]; then
set -- "$RUNNER_SECURITY_OPTIONS"
else
set --
fi
docker run --detach --rm --pull=never \
--name "$PROBE_CONTAINER" --privileged \
--cpus "$RUNNER_CPUS" --memory "$RUNNER_MEMORY" \
--memory-swap "$RUNNER_MEMORY_SWAP" --pids-limit "$RUNNER_PIDS_LIMIT" "$@" \
--volume "$RUNNER_VOLUME:/data" \
--volume /etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro \
--volume "$PREFIX:$PREFIX:ro" \
--env DOCKERD_ROOTLESS_ROOTLESSKIT_NET=slirp4netns \
--env DOCKERD_ROOTLESS_ROOTLESSKIT_MTU=65520 \
--env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns \
--entrypoint dockerd-entrypoint.sh "$runner_runtime_image" >/dev/null
attempt=0
until docker exec "$PROBE_CONTAINER" docker info >/dev/null 2>&1; do
attempt=$((attempt + 1))
if [ "$attempt" -ge 30 ] || \
[ "$(docker inspect --format '{{.State.Running}}' "$PROBE_CONTAINER" 2>/dev/null || true)" != "true" ]; then
docker logs "$PROBE_CONTAINER" 2>&1 || true
fail "pinned rootless Docker-in-Docker probe did not become ready"
fi
sleep 2
done
[ "$(docker exec "$PROBE_CONTAINER" id -u)" = "1000" ] || \
fail "rootless Docker-in-Docker probe is not uid 1000"
[ "$(docker inspect --format '{{.HostConfig.Memory}}:{{.HostConfig.MemorySwap}}:{{.HostConfig.NanoCpus}}:{{.HostConfig.PidsLimit}}' "$PROBE_CONTAINER")" = \
'2147483648:3221225472:2000000000:1024' ] || \
fail "rootless Docker-in-Docker probe does not have the required aggregate resource limits"
docker exec "$PROBE_CONTAINER" docker info --format '{{range .SecurityOptions}}{{println .}}{{end}}' | \
grep -Fq 'name=rootless' || fail "rootless Docker-in-Docker probe is not rootless"
[ "$(docker exec "$PROBE_CONTAINER" docker info --format '{{.DockerRootDir}}')" = /data/docker ] || \
fail "rootless Docker-in-Docker probe is not using its persistent dedicated data root"
# Complete every slow and capability-sensitive validation before registration;
# otherwise a queued workflow can race the installer for the same image layers.
docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"
outer_sentinel_pid=$(docker exec "$PROBE_CONTAINER" /bin/sh -c \
'sleep 300 >/dev/null 2>&1 & printf "%s\n" "$!"')
case $outer_sentinel_pid in
'' | *[!0-9]*) fail "could not create the outer PID namespace sentinel" ;;
esac
if ! docker exec "$PROBE_CONTAINER" docker run --rm --pull=never \
--pid=host --entrypoint /bin/sh "$JOB_IMAGE" \
-ec 'test ! -e "/proc/$1"' sh "$outer_sentinel_pid"; then
docker exec "$PROBE_CONTAINER" kill "$outer_sentinel_pid" >/dev/null 2>&1 || true
fail "nested --pid=host can see the outer runner PID namespace"
fi
docker exec "$PROBE_CONTAINER" kill "$outer_sentinel_pid" >/dev/null 2>&1 || true
docker exec "$PROBE_CONTAINER" docker run --rm --pull=never \
--volume "$PREFIX:$PREFIX:ro" \
--entrypoint /bin/sh "$JOB_IMAGE" -ec '
export PATH=/opt/easyai-gateway-ci/bin:/opt/easyai-gateway-ci/toolchains/go/bin:/opt/easyai-gateway-ci/toolchains/node/bin:$PATH
go version
node --version
pnpm --version
docker-compose version
shellcheck --version
trivy --version
govulncheck -version
'
cleanup_probe
trap - EXIT HUP INT TERM
install -m 0644 "$ROOT/deploy/ci/act-runner-v2-config.yaml" /etc/easyai-gateway-ci-v2/config.yaml
umask 077
printf 'RUNNER_SECURITY_OPTIONS="%s"\nRUNNER_IMAGE_REF="%s"\n' \
"$RUNNER_SECURITY_OPTIONS" "$runner_runtime_image" \
> /etc/easyai-gateway-ci-v2/runner.env
install -m 0644 "$ROOT/deploy/ci/easyai-gateway-ci-v2-runner.service" \
/etc/systemd/system/easyai-gateway-ci-v2-runner.service
runner_is_registered() {
docker run --rm --pull=never \
--volume "$RUNNER_VOLUME:/data" \
--entrypoint /bin/sh "$runner_runtime_image" \
-ec 'test -s /data/.runner'
}
if ! runner_is_registered; then
[ -n "${RUNNER_REGISTRATION_TOKEN:-}" ] || \
fail "RUNNER_REGISTRATION_TOKEN is required for first registration"
docker rm --force easyai-gateway-ci-v2-register >/dev/null 2>&1 || true
printf '%s\n%s\n%s\n' "$GITEA_INSTANCE_URL" "$RUNNER_REGISTRATION_TOKEN" "$RUNNER_NAME" | \
docker run --rm --interactive --pull=never \
--name easyai-gateway-ci-v2-register \
--volume "$RUNNER_VOLUME:/data" \
--volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro \
--env HOME=/data \
--entrypoint /usr/local/bin/gitea-runner \
"$runner_runtime_image" --config /config.yaml register
fi
unset RUNNER_REGISTRATION_TOKEN
docker run --rm --pull=never \
--volume "$RUNNER_VOLUME:/data" \
--entrypoint /bin/sh "$runner_runtime_image" \
-ec 'test -s /data/.runner && chmod 0600 /data/.runner'
RUNNER_VALIDATED=0
cleanup_unvalidated_runner() {
if [ "$RUNNER_VALIDATED" -ne 1 ]; then
systemctl disable --now easyai-gateway-ci-v2-runner.service >/dev/null 2>&1 || true
docker rm --force "$RUNNER_CONTAINER" >/dev/null 2>&1 || true
fi
}
trap 'cleanup_unvalidated_runner' EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
systemctl daemon-reload
systemctl enable --now easyai-gateway-ci-v2-runner.service
attempt=0
until docker exec "$RUNNER_CONTAINER" docker info >/dev/null 2>&1; do
attempt=$((attempt + 1))
if [ "$attempt" -ge 30 ]; then
systemctl --no-pager --full status easyai-gateway-ci-v2-runner.service || true
docker logs "$RUNNER_CONTAINER" 2>&1 || true
fail "rootless Docker daemon did not become ready"
fi
sleep 2
done
attempt=0
until docker logs "$RUNNER_CONTAINER" 2>&1 | grep -Fq 'declare successfully'; do
attempt=$((attempt + 1))
if [ "$attempt" -ge 30 ] || \
[ "$(docker inspect --format '{{.State.Running}}' "$RUNNER_CONTAINER" 2>/dev/null || true)" != "true" ]; then
docker logs "$RUNNER_CONTAINER" 2>&1 || true
fail "Gitea runner did not declare itself successfully"
fi
sleep 2
done
[ "$(docker inspect --format '{{.Config.User}}' "$RUNNER_CONTAINER")" = "rootless" ] || \
fail "outer runner container is not using the rootless image user"
[ "$(docker inspect --format '{{.HostConfig.Privileged}}' "$RUNNER_CONTAINER")" = "true" ] || \
fail "outer rootless DinD container is not privileged"
[ "$(docker inspect --format '{{.HostConfig.Memory}}:{{.HostConfig.MemorySwap}}:{{.HostConfig.NanoCpus}}:{{.HostConfig.PidsLimit}}' "$RUNNER_CONTAINER")" = \
'2147483648:3221225472:2000000000:1024' ] || \
fail "outer rootless DinD container does not have the required aggregate resource limits"
if docker inspect --format '{{range .Mounts}}{{println .Source}}{{end}}' "$RUNNER_CONTAINER" | \
grep -Eq '/var/run/docker\.sock|/run/docker\.sock'; then
fail "host Docker socket was mounted into the runner"
fi
docker exec "$RUNNER_CONTAINER" docker info --format '{{range .SecurityOptions}}{{println .}}{{end}}' | \
grep -Fq 'name=rootless' || fail "nested Docker daemon is not rootless"
[ "$(docker exec "$RUNNER_CONTAINER" docker info --format '{{.DockerRootDir}}')" = /data/docker ] || \
fail "nested Docker daemon is not using its persistent dedicated data root"
docker exec "$RUNNER_CONTAINER" docker image inspect "$JOB_IMAGE" >/dev/null || \
fail "validated pinned job image is unavailable after runner activation"
if ! require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed CI runner installation"; then
fail "CI runner was stopped because the post-install disk reserve was not met"
fi
systemctl --no-pager --full status easyai-gateway-ci-v2-runner.service
RUNNER_VALIDATED=1
trap - EXIT HUP INT TERM

109
tests/ci/ci-build-images-test.sh Executable file
View File

@ -0,0 +1,109 @@
#!/usr/bin/env bash
set -euo pipefail
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
script=$root/scripts/ci-build-images.sh
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
mkdir -p "$tmp/bin"
cat >"$tmp/bin/docker" <<'MOCK'
#!/usr/bin/env bash
set -euo pipefail
printf 'docker %s\n' "$*" >>"$MOCK_LOG"
case "$*" in
'buildx inspect '*|'buildx create '*) exit 0 ;;
'buildx build '*)
if [[ ${MOCK_BUILD_FAIL:-0} == 1 && $* == *'--target web'* ]]; then
exit 21
fi
;;
'buildx prune '*)
[[ ${MOCK_PRUNE_FAIL:-0} == 0 ]] || exit 22
;;
'image rm '*)
[[ ${MOCK_IMAGE_RM_FAIL:-0} == 0 ]] || exit 23
;;
*)
printf 'unexpected docker invocation: %s\n' "$*" >&2
exit 99
;;
esac
MOCK
cat >"$tmp/bin/trivy" <<'MOCK'
#!/usr/bin/env bash
set -euo pipefail
printf 'trivy %s\n' "$*" >>"$MOCK_LOG"
MOCK
chmod +x "$tmp/bin/docker" "$tmp/bin/trivy"
sha=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
registry=registry.example.com/easyai
api_image=$registry/ai-gateway:$sha
web_image=$registry/ai-gateway-web:$sha
run_build() {
env \
PATH="$tmp/bin:$PATH" \
MOCK_LOG="$tmp/calls.log" \
IMAGE_TAG="$sha" \
AI_GATEWAY_IMAGE_REGISTRY="$registry" \
AI_GATEWAY_BUILDX_BUILDER=gateway-test \
AI_GATEWAY_BUILDX_CACHE_LIMIT=2gb \
"$script" "$@"
}
: >"$tmp/calls.log"
run_build
grep -Fq "docker buildx prune --builder gateway-test --force --max-used-space 2gb" "$tmp/calls.log"
grep -Fq "docker image rm $api_image $web_image" "$tmp/calls.log"
grep -Fq 'trivy image --db-repository ghcr.m.daocloud.io/aquasecurity/trivy-db:2' \
"$tmp/calls.log"
: >"$tmp/calls.log"
run_build --keep-images
grep -Fq "docker buildx prune --builder gateway-test --force --max-used-space 2gb" "$tmp/calls.log"
if grep -Fq 'docker image rm ' "$tmp/calls.log"; then
echo '--keep-images must preserve fully built images' >&2
exit 1
fi
: >"$tmp/calls.log"
if MOCK_PRUNE_FAIL=1 run_build; then
echo 'a prune failure must fail an otherwise successful build' >&2
exit 1
fi
: >"$tmp/calls.log"
if MOCK_IMAGE_RM_FAIL=1 run_build; then
echo 'an image cleanup failure must fail an otherwise successful build' >&2
exit 1
fi
: >"$tmp/calls.log"
if MOCK_BUILD_FAIL=1 run_build; then
echo 'a failed image build must fail the script' >&2
exit 1
fi
grep -Fq "docker image rm $api_image $web_image" "$tmp/calls.log"
for invalid_limit in 0gb 2 2GB -1gb '2gb --all'; do
: >"$tmp/calls.log"
if env \
PATH="$tmp/bin:$PATH" \
MOCK_LOG="$tmp/calls.log" \
IMAGE_TAG="$sha" \
AI_GATEWAY_IMAGE_REGISTRY="$registry" \
AI_GATEWAY_BUILDX_CACHE_LIMIT="$invalid_limit" \
"$script" >/dev/null 2>&1; then
printf 'invalid cache limit was accepted: %s\n' "$invalid_limit" >&2
exit 1
fi
if [[ -s $tmp/calls.log ]]; then
printf 'invalid cache limit reached Docker: %s\n' "$invalid_limit" >&2
exit 1
fi
done
echo 'ci_build_images_tests=PASS'

331
tests/ci/migrations-test.sh Executable file
View File

@ -0,0 +1,331 @@
#!/usr/bin/env bash
set -euo pipefail
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
validator=$root/scripts/ci-validate-migrations.mjs
fixture_root=$(mktemp -d)
trap 'rm -rf "$fixture_root"' EXIT
new_repo() {
local name=$1
local repo=$fixture_root/$name
mkdir -p "$repo/apps/api/migrations"
git -C "$repo" init -q
git -C "$repo" config user.name 'CI Migration Test'
git -C "$repo" config user.email 'ci-migration-test@example.invalid'
printf 'CREATE TABLE accounts (id bigint PRIMARY KEY);\n' > \
"$repo/apps/api/migrations/0001_init.sql"
git -C "$repo" add apps/api/migrations/0001_init.sql
git -C "$repo" commit -qm baseline
mkdir -p "$repo/deploy/ci"
git -C "$repo" rev-parse HEAD > \
"$repo/deploy/ci/production-migration-base"
printf '%s\n' "$repo"
}
commit_all() {
local repo=$1
local message=$2
git -C "$repo" add -A
git -C "$repo" commit -qm "$message"
}
expect_pass() {
local repo=$1
local base=$2
local immutable_base=${3:-}
(
cd "$repo"
if [[ -n $immutable_base ]]; then
node "$validator" "$base" "$immutable_base"
else
node "$validator" "$base"
fi
)
}
expect_fail() {
local repo=$1
local base=$2
local expected=$3
local immutable_base=${4:-}
local output status
if [[ -n $immutable_base ]]; then
set +e
output=$(cd "$repo" && node "$validator" "$base" "$immutable_base" 2>&1)
status=$?
set -e
else
set +e
output=$(cd "$repo" && node "$validator" "$base" 2>&1)
status=$?
set -e
fi
if [[ $status -eq 0 ]]; then
printf 'migration validator unexpectedly passed: %s\n' "$expected" >&2
exit 1
fi
grep -Fq "$expected" <<<"$output" || {
printf 'missing expected migration failure %q in:\n%s\n' "$expected" "$output" >&2
exit 1
}
}
repo=$(new_repo safe)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_expand_accounts.sql" <<'SQL'
-- A comment mentioning DROP TABLE must not trigger the gate.
ALTER TABLE accounts ADD COLUMN display_name text;
CREATE TABLE audit_events (id bigint PRIMARY KEY, note text);
INSERT INTO audit_events (id, note) VALUES (1, 'DROP TABLE is documentation');
UPDATE accounts SET display_name = 'ready' WHERE display_name IS NULL;
SQL
commit_all "$repo" safe
expect_pass "$repo" "$base"
repo=$(new_repo drop)
base=$(git -C "$repo" rev-parse HEAD)
printf 'DROP\nTABLE accounts;\n' >"$repo/apps/api/migrations/0002_drop.sql"
commit_all "$repo" drop
expect_fail "$repo" "$base" 'destructive DROP operation'
repo=$(new_repo delete)
base=$(git -C "$repo" rev-parse HEAD)
printf 'DELETE FROM accounts;\n' >"$repo/apps/api/migrations/0002_delete.sql"
commit_all "$repo" delete
expect_fail "$repo" "$base" 'DELETE FROM operation'
repo=$(new_repo incompatible_alter)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_alter.sql" <<'SQL'
ALTER TABLE accounts ADD COLUMN external_id text NOT NULL;
ALTER TABLE accounts ALTER COLUMN id TYPE numeric;
SQL
commit_all "$repo" alter
expect_fail "$repo" "$base" 'non-null column addition'
expect_fail "$repo" "$base" 'column type change'
repo=$(new_repo dynamic_sql)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_dynamic.sql" <<'SQL'
DO $$
BEGIN
EXECUTE 'DROP ' || 'TABLE accounts';
END
$$;
SQL
commit_all "$repo" dynamic
expect_fail "$repo" "$base" 'dynamic SQL execution'
repo=$(new_repo escape_string)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_escape.sql" <<'SQL'
SELECT E'foo\'bar'; DROP TABLE accounts; --';
SQL
commit_all "$repo" escape
expect_fail "$repo" "$base" 'PostgreSQL escape/unicode strings are not allowed'
repo=$(new_repo string_semantics_override)
base=$(git -C "$repo" rev-parse HEAD)
printf 'SET standard_conforming_strings = off;\n' > \
"$repo/apps/api/migrations/0002_string_mode.sql"
commit_all "$repo" string_mode
expect_fail "$repo" "$base" 'SQL string semantics override'
repo=$(new_repo transaction_control)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_commit.sql" <<'SQL'
ALTER TABLE accounts ADD COLUMN display_name text;
COMMIT;
UPDATE accounts SET display_name = 'partially committed';
SQL
commit_all "$repo" transaction_control
expect_fail "$repo" "$base" 'transaction control operation'
repo=$(new_repo bare_end_transaction)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_end.sql" <<'SQL'
ALTER TABLE accounts ADD COLUMN display_name text;
END;
UPDATE accounts SET display_name = 'partially committed';
SQL
commit_all "$repo" bare_end_transaction
expect_fail "$repo" "$base" 'transaction control operation'
repo=$(new_repo plpgsql_block_end)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_function.sql" <<'SQL'
CREATE FUNCTION account_count() RETURNS bigint
LANGUAGE plpgsql AS $$
BEGIN
RETURN (SELECT count(*) FROM accounts);
END;
$$;
SQL
commit_all "$repo" plpgsql_block_end
expect_fail "$repo" "$base" 'procedural SQL body'
repo=$(new_repo ordinary_string_procedural_body)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_do_string.sql" <<'SQL'
DO LANGUAGE plpgsql 'BEGIN DROP TABLE accounts; END';
SQL
commit_all "$repo" ordinary_string_procedural_body
expect_fail "$repo" "$base" 'procedural SQL body'
repo=$(new_repo ordinary_string_boundary)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_string_boundary.sql" <<'SQL'
SELECT '\'; DROP TABLE accounts; --';
SQL
commit_all "$repo" ordinary_string
expect_fail "$repo" "$base" 'destructive DROP operation'
repo=$(new_repo unicode_string)
base=$(git -C "$repo" rev-parse HEAD)
printf "SELECT U&'d\\0061ta';\n" >"$repo/apps/api/migrations/0002_unicode.sql"
commit_all "$repo" unicode
expect_fail "$repo" "$base" 'PostgreSQL escape/unicode strings are not allowed'
repo=$(new_repo merge_delete)
base=$(git -C "$repo" rev-parse HEAD)
cat >"$repo/apps/api/migrations/0002_merge.sql" <<'SQL'
MERGE INTO accounts USING stale_accounts ON accounts.id = stale_accounts.id
WHEN MATCHED THEN DELETE;
SQL
commit_all "$repo" merge
expect_fail "$repo" "$base" 'MERGE operation'
repo=$(new_repo replace_view)
base=$(git -C "$repo" rev-parse HEAD)
printf 'CREATE OR REPLACE VIEW active_accounts AS SELECT id FROM accounts;\n' > \
"$repo/apps/api/migrations/0002_replace.sql"
commit_all "$repo" replace
expect_fail "$repo" "$base" 'CREATE OR REPLACE operation'
repo=$(new_repo rename_function)
base=$(git -C "$repo" rev-parse HEAD)
printf 'ALTER FUNCTION calculate_total() RENAME TO calculate_total_legacy;\n' > \
"$repo/apps/api/migrations/0002_rename.sql"
commit_all "$repo" rename
expect_fail "$repo" "$base" 'object or column rename'
repo=$(new_repo set_schema)
base=$(git -C "$repo" rev-parse HEAD)
printf 'ALTER TABLE accounts SET SCHEMA archive;\n' > \
"$repo/apps/api/migrations/0002_schema.sql"
commit_all "$repo" schema
expect_fail "$repo" "$base" 'SET SCHEMA operation'
repo=$(new_repo drop_domain)
base=$(git -C "$repo" rev-parse HEAD)
printf 'DROP DOMAIN account_code CASCADE;\n' > \
"$repo/apps/api/migrations/0002_drop_domain.sql"
commit_all "$repo" drop_domain
expect_fail "$repo" "$base" 'destructive DROP operation'
repo=$(new_repo immutable)
base=$(git -C "$repo" rev-parse HEAD)
printf '\n-- retroactive edit\n' >>"$repo/apps/api/migrations/0001_init.sql"
commit_all "$repo" edit
expect_fail "$repo" "$base" 'applied migrations are immutable (status M)'
repo=$(new_repo immutable_after_baseline)
base=$(git -C "$repo" rev-parse HEAD)
printf 'ALTER TABLE accounts ADD COLUMN nickname text;\n' > \
"$repo/apps/api/migrations/0002_nickname.sql"
commit_all "$repo" add
immutable_base=$(git -C "$repo" rev-parse HEAD)
printf '\nUPDATE accounts SET nickname = NULL;\n' >> \
"$repo/apps/api/migrations/0002_nickname.sql"
commit_all "$repo" modify
output=$(
cd "$repo"
node "$validator" "$base" "$immutable_base" 2>&1 || true
)
grep -Fq 'migration present on main is immutable (status M)' <<<"$output" || {
printf 'stale production baseline did not preserve main migration immutability:\n%s\n' \
"$output" >&2
exit 1
}
repo=$(new_repo filename)
base=$(git -C "$repo" rev-parse HEAD)
printf 'SELECT 1;\n' >"$repo/apps/api/migrations/next.SQL"
commit_all "$repo" filename
expect_fail "$repo" "$base" 'new migration must use NNNN_lowercase_name.sql'
repo=$(new_repo ordering)
base=$(git -C "$repo" rev-parse HEAD)
printf 'SELECT 1;\n' >"$repo/apps/api/migrations/0000_late.sql"
commit_all "$repo" ordering
expect_fail "$repo" "$base" 'new migration name must sort after 0001_init.sql'
repo=$(new_repo stale_baseline)
base=$(git -C "$repo" rev-parse HEAD)
printf 'ALTER TABLE accounts ADD COLUMN nickname text;\n' > \
"$repo/apps/api/migrations/0002_nickname.sql"
commit_all "$repo" main_migration
immutable_base=$(git -C "$repo" rev-parse HEAD)
printf 'ordinary code change\n' >"$repo/README.md"
commit_all "$repo" code_only
expect_pass "$repo" "$base" "$immutable_base"
repo=$(new_repo baseline_advance)
production_base=$(git -C "$repo" rev-parse HEAD)
mkdir -p "$repo/deploy/ci"
printf '%s\n' "$production_base" >"$repo/deploy/ci/production-migration-base"
commit_all "$repo" baseline_file
immutable_base=$(git -C "$repo" rev-parse HEAD)
git -C "$repo" switch -q -c release-baseline-update "$immutable_base"
printf '%s\n' "$immutable_base" >"$repo/deploy/ci/production-migration-base"
commit_all "$repo" valid_baseline_advance
expect_pass "$repo" "$immutable_base" "$immutable_base"
git -C "$repo" switch -q -c malicious-baseline "$immutable_base"
printf 'DROP TABLE accounts;\n' >"$repo/apps/api/migrations/0002_drop.sql"
commit_all "$repo" hidden_destructive_migration
hidden_sha=$(git -C "$repo" rev-parse HEAD)
printf '%s\n' "$hidden_sha" >"$repo/deploy/ci/production-migration-base"
commit_all "$repo" hide_migration_in_baseline
expect_fail "$repo" "$hidden_sha" \
'production migration baseline must belong to the immutable event base history' \
"$immutable_base"
repo=$(new_repo nested)
base=$(git -C "$repo" rev-parse HEAD)
mkdir -p "$repo/apps/api/migrations/archive"
printf 'SELECT 1;\n' >"$repo/apps/api/migrations/archive/9999_hidden.sql"
commit_all "$repo" nested
expect_fail "$repo" "$base" 'new migration must use NNNN_lowercase_name.sql'
repo=$(new_repo symlink)
base=$(git -C "$repo" rev-parse HEAD)
printf 'SELECT 1;\n' >"$repo/outside.sql"
ln -s ../../../outside.sql "$repo/apps/api/migrations/0002_symlink.sql"
commit_all "$repo" symlink
expect_fail "$repo" "$base" 'new migration must be a mode 100644 regular file'
repo=$(new_repo baseline_symlink)
base=$(git -C "$repo" rev-parse HEAD)
printf '%s\n' "$base" >"$repo/baseline-target"
rm "$repo/deploy/ci/production-migration-base"
ln -s ../../baseline-target "$repo/deploy/ci/production-migration-base"
commit_all "$repo" baseline_symlink
expect_fail "$repo" "$base" \
'production migration baseline must be a mode 100644 regular file'
repo=$(new_repo ancestry)
base=$(git -C "$repo" rev-parse HEAD)
git -C "$repo" switch -q --orphan unrelated
printf 'unrelated\n' >"$repo/unrelated"
git -C "$repo" add unrelated
git -C "$repo" commit -qm unrelated
expect_fail "$repo" "$base" 'production migration baseline is not an ancestor of HEAD'
repo=$(new_repo invalid_sha)
expect_fail "$repo" deadbeef 'must be a full lowercase commit SHA'
echo 'migration_tests=PASS'

323
tests/ci/pipeline-test.sh Executable file
View File

@ -0,0 +1,323 @@
#!/usr/bin/env bash
# shellcheck disable=SC2016 # Assertions intentionally match literal workflow variables.
set -euo pipefail
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
workflow=$root/.gitea/workflows/ci.yml
release_workflow=$root/.gitea/workflows/release-ci.yml
runner_service=$root/deploy/ci/easyai-gateway-ci-v2-runner.service
runner_config=$root/deploy/ci/act-runner-v2-config.yaml
provision=$root/scripts/provision-ci-runner.sh
build_images=$root/scripts/ci-build-images.sh
validate_semver=$root/scripts/ci-validate-semver.sh
validate_migrations=$root/scripts/ci-validate-migrations.mjs
build_images_test=$root/tests/ci/ci-build-images-test.sh
migrations_test=$root/tests/ci/migrations-test.sh
migration_base=$root/deploy/ci/production-migration-base
adr=$root/docs/decisions/001-production-cicd.md
runbook=$root/docs/runbooks/production-ci-cd.md
migrator=$root/apps/api/cmd/migrate/main.go
for file in "$workflow" "$release_workflow" "$runner_service" "$runner_config" "$provision" "$build_images" "$validate_semver" "$validate_migrations" "$build_images_test" "$migrations_test" "$migration_base" "$adr" "$runbook" "$migrator"; do
[[ -f $file ]] || {
printf 'missing CI/CD file: %s\n' "$file" >&2
exit 1
}
done
if [[ $(wc -l <"$migration_base") -ne 1 ]]; then
echo 'production migration baseline must contain exactly one line' >&2
exit 1
fi
grep -Fq 'SET standard_conforming_strings = on' "$migrator" || {
echo 'database migrator does not pin standard SQL string semantics' >&2
exit 1
}
if ! grep -Eq '^[0-9a-f]{40}$' "$migration_base"; then
echo 'production migration baseline must be a full lowercase SHA' >&2
exit 1
fi
grep -q '^name: ci$' "$workflow"
grep -q '^ push:$' "$workflow"
grep -q '^ branches: \[main\]$' "$workflow"
grep -q '^ pull_request:$' "$workflow"
grep -q '^ verify:$' "$workflow"
grep -Fq 'git checkout --detach "$CI_SHA"' "$workflow"
grep -Fq 'test "$(git rev-parse HEAD)" = "$CI_SHA"' "$workflow"
if grep -Fq 'git checkout --detach FETCH_HEAD' "$workflow"; then
echo 'main/PR CI must not checkout mutable FETCH_HEAD after comparison fetches' >&2
exit 1
fi
if grep -Eq "^[[:space:]]+tags:|Verify release tag ancestry" "$workflow" || \
grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$workflow"; then
echo 'main/PR CI must not share the release-tag context' >&2
exit 1
fi
grep -q '^name: release-ci$' "$release_workflow"
grep -q '^ push:$' "$release_workflow"
grep -q "^ tags: \['v\*'\]$" "$release_workflow"
grep -q '^ verify-tag:$' "$release_workflow"
grep -q '^ - name: Verify release tag ancestry$' "$release_workflow"
grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$release_workflow"
grep -Fq 'git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main' "$release_workflow"
if grep -Eq 'branches:|pull_request:' "$release_workflow"; then
echo 'release CI must be a tag-only workflow/context' >&2
exit 1
fi
for quality_workflow in "$workflow" "$release_workflow"; do
grep -q '^ runs-on: easyai-gateway-ci-unprivileged-v2$' "$quality_workflow"
grep -q '^ - name: Checkout without external Actions$' "$quality_workflow"
if grep -q -- '--depth=' "$quality_workflow"; then
echo 'quality verification must not use a shallow checkout' >&2
exit 1
fi
grep -Fq 'test ! -f .git/shallow' "$quality_workflow"
grep -Fq './tests/ci/ci-build-images-test.sh' "$quality_workflow"
grep -Fq './tests/ci/migrations-test.sh' "$quality_workflow"
grep -Fq './tests/ci/pipeline-test.sh' "$quality_workflow"
grep -Fq './tests/ci/semver-test.sh' "$quality_workflow"
grep -Fq 'docker-compose version' "$quality_workflow"
grep -Fq 'docker-compose -f docker-compose.yml config --quiet' "$quality_workflow"
if grep -Fq 'docker compose' "$quality_workflow"; then
echo 'job container must use the read-only standalone Compose binary' >&2
exit 1
fi
for forbidden in \
'sudo' \
'/usr/local/sbin/easyai-ai-gateway-release' \
'docker version' \
'docker buildx' \
'./scripts/ci-build-images.sh --keep-images' \
'Deploy verified release to Production'; do
if grep -Fq "$forbidden" "$quality_workflow"; then
printf 'unprivileged workflow contains forbidden capability: %s\n' "$forbidden" >&2
exit 1
fi
done
if grep -Eq '^[[:space:]]+\./scripts/ci-build-images\.sh([[:space:]]|$)' "$quality_workflow"; then
echo 'unprivileged workflow must not build container images' >&2
exit 1
fi
for gate in \
'scripts/ci-validate-migrations.mjs' \
'gofmt -l' \
'go vet ./...' \
'go test ./...' \
'pnpm install --frozen-lockfile' \
'pnpm lint' \
'pnpm test' \
'pnpm build' \
'pnpm audit --audit-level high' \
'docker-compose -f docker-compose.yml config --quiet' \
'govulncheck ./...' \
'trivy fs'; do
grep -Fq "$gate" "$quality_workflow" || {
printf '%s is missing quality gate: %s\n' "$quality_workflow" "$gate" >&2
exit 1
}
done
grep -Fq 'TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2' \
"$quality_workflow" || {
printf '%s does not pin the reachable Trivy vulnerability DB mirror\n' \
"$quality_workflow" >&2
exit 1
}
grep -Fq 'production_base=$(cat deploy/ci/production-migration-base)' \
"$quality_workflow" || {
printf '%s does not read the versioned production migration baseline\n' \
"$quality_workflow" >&2
exit 1
}
done
if [[ $(sed -n '/^ - name: Verify Go formatting$/,$p' "$workflow") != \
"$(sed -n '/^ - name: Verify Go formatting$/,$p' "$release_workflow")" ]]; then
echo 'main/PR and release-tag workflows must have identical quality gates' >&2
exit 1
fi
grep -Fq 'CI_EVENT_BEFORE: ${{ github.event.before }}' "$workflow"
grep -Fq 'CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}' "$workflow"
grep -Fq 'git merge-base --is-ancestor "$immutable_base" HEAD' "$workflow"
grep -Fq '"$production_base" "$immutable_base"' "$workflow"
grep -Fq 'node ./scripts/ci-validate-migrations.mjs "$production_base"' \
"$release_workflow"
runner_image='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a'
job_image='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7'
grep -q '^User=root$' "$runner_service"
grep -q '^Group=root$' "$runner_service"
grep -q '^NoNewPrivileges=true$' "$runner_service"
grep -q '^ProtectSystem=strict$' "$runner_service"
grep -q '^ProtectHome=true$' "$runner_service"
grep -q '^CapabilityBoundingSet=$' "$runner_service"
grep -Fq 'Requires=docker.service' "$runner_service"
grep -Fq -- "--pull=never" "$runner_service"
grep -Fq -- "--privileged" "$runner_service"
grep -Fq -- '--pids-limit 1024' "$runner_service"
grep -Fq -- '--cpus 2' "$runner_service"
grep -Fq -- '--memory 2g' "$runner_service"
grep -Fq -- '--memory-swap 3g' "$runner_service"
grep -Fq -- '--env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns' "$runner_service"
grep -Fq 'EnvironmentFile=-/etc/easyai-gateway-ci-v2/runner.env' "$runner_service"
grep -Fq '$RUNNER_SECURITY_OPTIONS' "$runner_service"
grep -Fq -- "--volume easyai-gateway-ci-v2-data:/data" "$runner_service"
grep -Fq -- "--volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro" "$runner_service"
grep -Fq -- "--volume /etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro" "$runner_service"
grep -Fq -- "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro" "$runner_service"
grep -Fq 'Environment=RUNNER_IMAGE_REF=' "$runner_service"
grep -Fq '$RUNNER_IMAGE_REF' "$runner_service"
if grep -Eq 'GITEA_RUNNER_REGISTRATION_TOKEN|/var/run/docker\.sock|/run/docker\.sock' "$runner_service"; then
echo 'persistent runner service exposes a registration token or host Docker socket' >&2
exit 1
fi
if grep -Fq 'apparmor=rootlesskit' "$runner_service"; then
echo 'runner service must not hard-code an AppArmor profile that may be absent' >&2
exit 1
fi
grep -Fq "easyai-gateway-ci-unprivileged-v2:docker://$job_image" "$runner_config"
if grep -Fq ':host' "$runner_config"; then
echo 'CI runner must never use the host executor' >&2
exit 1
fi
grep -q '^ capacity: 1$' "$runner_config"
grep -q '^ file: /data/.runner$' "$runner_config"
grep -q '^ docker_host: "-"$' "$runner_config"
grep -q '^ privileged: false$' "$runner_config"
grep -q '^ require_docker: true$' "$runner_config"
grep -q '^ bind_workdir: false$' "$runner_config"
grep -Fq ' options: "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro"' "$runner_config"
grep -q '^ valid_volumes:$' "$runner_config"
grep -q '^ - "/opt/easyai-gateway-ci"$' "$runner_config"
if awk '
/^ valid_volumes:$/ { in_volumes=1; next }
in_volumes && /^ - / { count++; next }
in_volumes && /^ [A-Za-z_]+:/ { in_volumes=0 }
END { exit count == 1 ? 0 : 1 }
' "$runner_config"; then
:
else
echo 'runner must allow exactly one workflow volume source' >&2
exit 1
fi
if grep -Eq '/var/run/docker\.sock|/run/docker\.sock' "$runner_config"; then
echo 'job configuration must not expose a Docker socket' >&2
exit 1
fi
if grep -Eq 'apparmor=unconfined|sysctl .*(apparmor_restrict_unprivileged_userns)' "$runner_service" "$runner_config" "$provision"; then
echo 'runner must not disable AppArmor isolation or its user-namespace restriction' >&2
exit 1
fi
grep -q '^GITEA_RUNNER_VERSION=2.0.0$' "$provision"
grep -q '^GO_VERSION=1.26.5$' "$provision"
grep -q '^NODE_VERSION=24.16.0$' "$provision"
grep -q '^TRIVY_VERSION=0.70.0$' "$provision"
grep -q '^PREFIX=/opt/easyai-gateway-ci$' "$provision"
grep -q '^RUNNER_VOLUME=easyai-gateway-ci-v2-data$' "$provision"
grep -q '^SHELLCHECK_VERSION=0.11.0$' "$provision"
grep -q '^SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198$' "$provision"
grep -q '^COMPOSE_VERSION=5.3.1$' "$provision"
grep -q '^COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959$' "$provision"
grep -Fq 'RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-ci-unprivileged-v2}' "$provision"
grep -Fq "RUNNER_IMAGE='$runner_image'" "$provision"
grep -Fq 'RUNNER_IMAGE_SOURCE=${CI_RUNNER_IMAGE_SOURCE:-$RUNNER_IMAGE}' "$provision"
grep -Fq 'runner_manifest_digest=${RUNNER_IMAGE##*@}' "$provision"
grep -Fq 'CI_RUNNER_IMAGE_SOURCE must use the pinned runner manifest digest' "$provision"
grep -Fq 'runner_runtime_image=$(docker image inspect' "$provision"
grep -Fq 'RUNNER_IMAGE_REF="%s"' "$provision"
grep -Fq "JOB_IMAGE='$job_image'" "$provision"
grep -Fq 'RUNNER_LABEL="easyai-gateway-ci-unprivileged-v2:docker://$JOB_IMAGE"' "$provision"
grep -Fq 'docker volume create "$RUNNER_VOLUME"' "$provision"
grep -Fq '"data-root":"/data/docker"' "$provision"
grep -Fq -- '--security-opt apparmor=rootlesskit' "$provision"
grep -Fq 'RUNNER_SECURITY_OPTIONS="--security-opt=apparmor=rootlesskit"' "$provision"
grep -Fq 'RUNNER_SECURITY_OPTIONS=' "$provision"
grep -Fq '/proc/sys/kernel/apparmor_restrict_unprivileged_userns' "$provision"
grep -Fq -- '--entrypoint dockerd-entrypoint.sh "$runner_runtime_image"' "$provision"
grep -Fq '/etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro' "$provision"
grep -Fq 'CI_NESTED_DOCKER_REGISTRY_MIRROR must be an HTTPS registry origin' "$provision"
grep -Fq 'PROBE_CONTAINER="easyai-gateway-ci-v2-probe-$$"' "$provision"
grep -Fq -- '--pids-limit "$RUNNER_PIDS_LIMIT"' "$provision"
grep -Fq -- '--cpus "$RUNNER_CPUS"' "$provision"
grep -Fq -- '--memory "$RUNNER_MEMORY"' "$provision"
grep -Fq -- '--memory-swap "$RUNNER_MEMORY_SWAP"' "$provision"
grep -Fq "'2147483648:3221225472:2000000000:1024'" "$provision"
grep -Fq -- '--entrypoint /usr/local/bin/gitea-runner' "$provision"
grep -Fq 'shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz' "$provision"
grep -Fq 'docker-compose-linux-x86_64' "$provision"
grep -Fq 'install -m 0755 "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "$PREFIX/bin/shellcheck"' "$provision"
grep -Fq 'install -m 0755 "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" "$PREFIX/bin/docker-compose"' "$provision"
grep -Fq 'PATH="$PREFIX/toolchains/node/bin:$PATH"' "$provision"
grep -Fq 'gpasswd -d easyai-gateway-ci-v2 docker' "$provision"
grep -Fq 'gpasswd -d easyai-gateway-ci-v2 sudo' "$provision"
if grep -Fq 'usermod -aG docker' "$provision"; then
echo 'unprivileged runner must never join the Docker group' >&2
exit 1
fi
grep -Fq 'systemctl disable --now easyai-gateway-act-runner.service' "$provision"
grep -Fq 'systemctl disable --now easyai-gateway-ci-v2-runner.service' "$provision"
grep -Fq 'RUNNER_REGISTRATION_TOKEN is required for first registration' "$provision"
grep -Fq 'printf '\''%s\n%s\n%s\n'\'' "$GITEA_INSTANCE_URL" "$RUNNER_REGISTRATION_TOKEN" "$RUNNER_NAME"' "$provision"
if grep -Fq -- '--token "$RUNNER_REGISTRATION_TOKEN"' "$provision"; then
echo 'registration token must not be exposed in a host-visible process argv' >&2
exit 1
fi
grep -Fq "grep -Fq 'name=rootless'" "$provision"
grep -Fq "grep -Fq 'declare successfully'" "$provision"
grep -Fq 'docker buildx prune --builder "$LEGACY_BUILDER" --force --max-used-space 512mb' "$provision"
grep -Fq 'require_free_space "$MIN_FREE_GIB" "runner image pull"' "$provision"
grep -Fq 'require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed CI runner installation"' "$provision"
grep -Fq 'CI_RUNNER_MIN_FREE_GIB cannot be lower than 8' "$provision"
grep -Fq 'CI_RUNNER_MIN_POST_INSTALL_FREE_GIB cannot be lower than 4' "$provision"
grep -Fq 'docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"' "$provision"
grep -Fq 'docker exec "$PROBE_CONTAINER" docker run --rm --pull=never' "$provision"
grep -Fq -- '--volume "$RUNNER_VOLUME:/data"' "$provision"
grep -Fq -- '--volume "$PREFIX:$PREFIX:ro"' "$provision"
probe_pull_line=$(grep -nF 'docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"' "$provision" | cut -d: -f1)
registration_line=$(grep -nF 'if ! runner_is_registered; then' "$provision" | cut -d: -f1)
[[ -n $probe_pull_line && -n $registration_line && $probe_pull_line -lt $registration_line ]] || {
echo 'pinned job image and isolation probes must complete before runner registration' >&2
exit 1
}
grep -Fq 'nested --pid=host can see the outer runner PID namespace' "$provision"
grep -Fq 'chown -R root:root "$PREFIX"' "$provision"
grep -Fq 'chmod -R go-w "$PREFIX"' "$provision"
if grep -Eq 'SHELLCHECK_BIN|COMPOSE_PLUGIN|command -v shellcheck|docker compose version' "$provision"; then
echo 'runner provisioning must use checksum-pinned static CI tools, not host copies' >&2
exit 1
fi
if grep -Eq 'docker (system|builder|image|volume) prune' "$provision"; then
echo 'runner provisioning must not globally prune shared Docker state' >&2
exit 1
fi
grep -Fq 'root-owned dispatcher' "$adr"
grep -Fq 'release-ci / verify-tag (push)' "$adr"
grep -Fq '来自 Fork 的 PR 必须先由维护者检查工作流变更并在 Gitea 中批准' "$adr"
grep -Fq 'rootless DinD 不是 VM 沙箱' "$runbook"
grep -Fq '同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)`' "$runbook"
grep -Fq '不得执行 Tag checkout 中的' "$runbook"
grep -Fq 'docker buildx build --builder "$builder"' "$build_images"
grep -q -- '--target api' "$build_images"
grep -q -- '--target web' "$build_images"
grep -Fq 'docker buildx prune --builder "$builder" --force --max-used-space "$cache_limit"' "$build_images"
grep -Fq 'trivy image' "$build_images"
grep -Fq 'docker image rm "$api_image" "$web_image"' "$build_images"
grep -Fq 'AI_GATEWAY_BUILDX_CACHE_LIMIT has an invalid format' "$build_images"
if grep -En '(password|secret|token):[[:space:]]+[^$<{]' "$workflow" "$release_workflow"; then
echo 'workflow contains a literal credential' >&2
exit 1
fi
echo 'gateway_ci_pipeline_tests=PASS'

52
tests/ci/semver-test.sh Executable file
View File

@ -0,0 +1,52 @@
#!/usr/bin/env bash
set -euo pipefail
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
validator=$root/scripts/ci-validate-semver.sh
valid_tags=(
v0.0.0
v1.2.3
v1.2.3-alpha
v1.2.3-alpha.1
v1.2.3-0.3.7
v1.2.3-01a
v1.2.3-alpha-beta
v1.2.3--
v1.2.3-x.7.z.92+build.5
v1.2.3+build.11.e0f985a
v1.2.3+001
)
invalid_tags=(
1.2.3
v01.2.3
v1.02.3
v1.2.03
v1.2.3.foo
v1.2.3.
v1.2.3-.
v1.2.3-alpha.
v1.2.3-alpha..1
v1.2.3-01
v1.2.3-00.1
v1.2.3+
v1.2.3+build.
v1.2.3+build..1
v1.2.3+build+1
'v1.2.3 '
v1.2.3-alpha_1
v1.2
)
for tag in "${valid_tags[@]}"; do
"$validator" "$tag"
done
for tag in "${invalid_tags[@]}"; do
if "$validator" "$tag" >/dev/null 2>&1; then
printf 'invalid SemVer tag was accepted: %s\n' "$tag" >&2
exit 1
fi
done
echo 'semver_tests=PASS'