From 91ea0e6a2d629200df7f5a7801a14804c8c5a887 Mon Sep 17 00:00:00 2001 From: chengcheng Date: Fri, 17 Jul 2026 14:23:49 +0800 Subject: [PATCH] ci: harden production quality gates --- .gitea/workflows/ci.yml | 53 +-- .gitea/workflows/release-ci.yml | 86 +++++ README.md | 2 +- deploy/ci/act-runner-config.yaml | 35 -- deploy/ci/act-runner-v2-config.yaml | 36 ++ deploy/ci/easyai-gateway-act-runner.service | 31 -- deploy/ci/easyai-gateway-ci-v2-runner.service | 43 +++ docs/decisions/001-production-cicd.md | 63 ++-- docs/runbooks/production-ci-cd.md | 124 +++++-- scripts/ci-build-images.sh | 54 ++- scripts/ci-validate-semver.sh | 22 ++ scripts/provision-ci-runner.sh | 324 +++++++++++++++--- tests/ci/ci-build-images-test.sh | 107 ++++++ tests/ci/pipeline-test.sh | 243 +++++++++++-- tests/ci/semver-test.sh | 52 +++ 15 files changed, 1023 insertions(+), 252 deletions(-) create mode 100644 .gitea/workflows/release-ci.yml delete mode 100644 deploy/ci/act-runner-config.yaml create mode 100644 deploy/ci/act-runner-v2-config.yaml delete mode 100644 deploy/ci/easyai-gateway-act-runner.service create mode 100644 deploy/ci/easyai-gateway-ci-v2-runner.service create mode 100755 scripts/ci-validate-semver.sh create mode 100755 tests/ci/ci-build-images-test.sh create mode 100755 tests/ci/semver-test.sh diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 394d0ba..c6db54b 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -3,14 +3,12 @@ name: ci on: push: branches: [main] - tags: ['v*'] pull_request: + branches: [main] jobs: verify: - runs-on: easyai-gateway-linux - env: - IMAGE_TAG: ${{ github.sha }} + runs-on: easyai-gateway-ci-unprivileged-v2 steps: - name: Checkout without external Actions env: @@ -24,44 +22,19 @@ jobs: authorization=$(printf 'x-access-token:%s' "$CI_JOB_TOKEN" | base64 | tr -d '\n') git init . git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \ - fetch --no-tags --depth=1 "$CI_SERVER_URL/$CI_REPOSITORY.git" "$CI_SHA" + fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" "$CI_SHA" unset authorization CI_JOB_TOKEN + test ! -f .git/shallow git checkout --detach FETCH_HEAD - name: Verify pinned host toolchains run: | go version node --version pnpm --version - docker version - docker compose version - docker buildx version + docker-compose version shellcheck --version trivy --version govulncheck -version - - name: Verify Production deployment prerequisites - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - run: | - grep -Eq '^NoNewPrivs:[[:space:]]+0$' /proc/self/status - test -x /usr/local/sbin/easyai-ai-gateway-release - sudo -n -l /usr/local/sbin/easyai-ai-gateway-release "$IMAGE_TAG" >/dev/null - - name: Verify release tag ancestry - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - env: - CI_REPOSITORY: ${{ github.repository }} - CI_SERVER_URL: ${{ github.server_url }} - CI_SHA: ${{ github.sha }} - CI_JOB_TOKEN: ${{ github.token }} - run: | - set -eu - tag_name=${GITHUB_REF#refs/tags/} - printf '%s' "$tag_name" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.-]+)?$' - authorization=$(printf 'x-access-token:%s' "$CI_JOB_TOKEN" | base64 | tr -d '\n') - git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \ - fetch --no-tags --depth=200 "$CI_SERVER_URL/$CI_REPOSITORY.git" \ - main:refs/remotes/origin/main - unset authorization CI_JOB_TOKEN - test "$(git rev-parse HEAD)" = "$CI_SHA" - git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main - name: Verify Go formatting run: | unformatted=$(gofmt -l apps/api) @@ -83,19 +56,15 @@ jobs: run: pnpm audit --audit-level high - name: Validate deployment configuration run: | - docker compose -f docker-compose.yml config --quiet - shellcheck scripts/ci-build-images.sh scripts/provision-ci-runner.sh tests/ci/pipeline-test.sh + docker-compose -f docker-compose.yml config --quiet + shellcheck scripts/ci-build-images.sh scripts/ci-validate-semver.sh \ + scripts/provision-ci-runner.sh tests/ci/ci-build-images-test.sh \ + tests/ci/pipeline-test.sh tests/ci/semver-test.sh + ./tests/ci/ci-build-images-test.sh ./tests/ci/pipeline-test.sh + ./tests/ci/semver-test.sh - name: Scan repository run: | trivy fs --scanners vuln,secret,misconfig --severity HIGH,CRITICAL \ --ignore-unfixed --exit-code 1 --timeout 15m --skip-dirs .git \ --skip-dirs node_modules . - - run: ./scripts/ci-build-images.sh - - name: Deploy verified release to Production - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - env: - CI_SHA: ${{ github.sha }} - run: | - test "$(git rev-parse HEAD)" = "$CI_SHA" - sudo -n /usr/local/sbin/easyai-ai-gateway-release "$CI_SHA" diff --git a/.gitea/workflows/release-ci.yml b/.gitea/workflows/release-ci.yml new file mode 100644 index 0000000..6eb7b5c --- /dev/null +++ b/.gitea/workflows/release-ci.yml @@ -0,0 +1,86 @@ +name: release-ci + +on: + push: + tags: ['v*'] + +jobs: + verify-tag: + runs-on: easyai-gateway-ci-unprivileged-v2 + steps: + - name: Checkout without external Actions + env: + CI_REPOSITORY: ${{ github.repository }} + CI_SERVER_URL: ${{ github.server_url }} + CI_SHA: ${{ github.sha }} + CI_JOB_TOKEN: ${{ github.token }} + run: | + set -eu + test -n "$CI_JOB_TOKEN" + authorization=$(printf 'x-access-token:%s' "$CI_JOB_TOKEN" | base64 | tr -d '\n') + git init . + git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \ + fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" "$CI_SHA" + unset authorization CI_JOB_TOKEN + test ! -f .git/shallow + git checkout --detach FETCH_HEAD + - name: Verify pinned host toolchains + run: | + go version + node --version + pnpm --version + docker-compose version + shellcheck --version + trivy --version + govulncheck -version + - name: Verify release tag ancestry + env: + CI_REPOSITORY: ${{ github.repository }} + CI_SERVER_URL: ${{ github.server_url }} + CI_SHA: ${{ github.sha }} + CI_JOB_TOKEN: ${{ github.token }} + run: | + set -eu + tag_name=${GITHUB_REF#refs/tags/} + ./scripts/ci-validate-semver.sh "$tag_name" + authorization=$(printf 'x-access-token:%s' "$CI_JOB_TOKEN" | base64 | tr -d '\n') + git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \ + fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" \ + +refs/heads/main:refs/remotes/origin/main + unset authorization CI_JOB_TOKEN + test ! -f .git/shallow + test "$(git rev-parse HEAD)" = "$CI_SHA" + git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main + - name: Verify Go formatting + run: | + unformatted=$(gofmt -l apps/api) + test -z "$unformatted" || { + printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2 + exit 1 + } + - name: Verify Go code + working-directory: apps/api + run: | + go vet ./... + go test ./... + govulncheck ./... + - run: pnpm install --frozen-lockfile + - run: pnpm lint + - run: pnpm test + - run: pnpm build + - name: Audit JavaScript dependencies + run: pnpm audit --audit-level high + - name: Validate deployment configuration + run: | + docker-compose -f docker-compose.yml config --quiet + shellcheck scripts/ci-build-images.sh scripts/ci-validate-semver.sh \ + scripts/provision-ci-runner.sh tests/ci/ci-build-images-test.sh \ + tests/ci/pipeline-test.sh tests/ci/semver-test.sh + ./tests/ci/ci-build-images-test.sh + ./tests/ci/pipeline-test.sh + ./tests/ci/semver-test.sh + - name: Scan repository + run: | + trivy fs --scanners vuln,secret,misconfig --severity HIGH,CRITICAL \ + --ignore-unfixed --exit-code 1 --timeout 15m --skip-dirs .git \ + --skip-dirs node_modules . diff --git a/README.md b/README.md index 50b94d6..aa8464d 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ docker compose -f docker-compose.yml restart web ## 生产 CI/CD -Gitea Actions 会对 Pull Request 和 `main` Push 执行完整质量门禁,并以源码完整 Git SHA 构建 API/Web 镜像。`ai.51easyai.com` 只接受来自 `main` 历史的语义版本 Tag 发布,迁移前自动备份数据库,失败时恢复上一应用镜像。安装 Runner、发布、验证和回滚步骤见 [生产 CI/CD 运行手册](docs/runbooks/production-ci-cd.md),设计取舍见 [ADR-001](docs/decisions/001-production-cicd.md)。 +Gitea Actions 会在隔离的 rootless DinD Runner 中对 Pull Request、`main` Push 和版本 Tag 执行完整质量门禁;Tag 使用独立的 `release-ci / verify-tag (push)` context,不能复用旧的 `main` 成功状态。源码 Job 没有宿主 Docker、`sudo` 或生产部署权限。部署仓的 root-owned dispatcher 只在相同 SHA 的 `main` 与 Tag context 都成功后,用固定命令构建并扫描镜像,再以 Registry digest 发布 `ai.51easyai.com`。安装 Runner、Fork PR 审批、发布验证和回滚步骤见 [生产 CI/CD 运行手册](docs/runbooks/production-ci-cd.md),信任边界见 [ADR-001](docs/decisions/001-production-cicd.md)。 Compose 默认使用独立容器数据库 `postgres:18-alpine`,数据卷会保留在 `postgres_data` 和 `api_data`。为避免本地开发 `.env` 中的 `localhost` 数据库地址污染容器部署,compose 使用 `AI_GATEWAY_COMPOSE_*` 变量作为容器部署专用覆盖,例如: diff --git a/deploy/ci/act-runner-config.yaml b/deploy/ci/act-runner-config.yaml deleted file mode 100644 index 028f7cb..0000000 --- a/deploy/ci/act-runner-config.yaml +++ /dev/null @@ -1,35 +0,0 @@ -log: - level: info - -runner: - file: /var/lib/easyai-gateway-runner/.runner - capacity: 1 - envs: - GOPROXY: "https://goproxy.cn,direct" - GOSUMDB: "sum.golang.google.cn" - PATH: "/opt/easyai-gateway-ci/bin:/opt/easyai-gateway-ci/toolchains/go/bin:/opt/easyai-gateway-ci/toolchains/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - AI_GATEWAY_GO_BUILD_IMAGE: "docker.m.daocloud.io/library/golang:1.26.5-alpine" - AI_GATEWAY_API_RUNTIME_IMAGE: "docker.m.daocloud.io/library/alpine:3.22" - AI_GATEWAY_NODE_BUILD_IMAGE: "docker.m.daocloud.io/library/node:24-alpine" - AI_GATEWAY_WEB_RUNTIME_IMAGE: "docker.m.daocloud.io/library/nginx:1.27-alpine" - timeout: 3h - shutdown_timeout: 5m - insecure: false - fetch_timeout: 5s - fetch_interval: 2s - labels: - - "easyai-gateway-linux:host" - -cache: - enabled: true - dir: /var/lib/easyai-gateway-runner/cache - -container: - docker_host: "-" - privileged: false - force_pull: false - force_rebuild: false - valid_volumes: [] - -host: - workdir_parent: /var/lib/easyai-gateway-runner/work diff --git a/deploy/ci/act-runner-v2-config.yaml b/deploy/ci/act-runner-v2-config.yaml new file mode 100644 index 0000000..eb33e05 --- /dev/null +++ b/deploy/ci/act-runner-v2-config.yaml @@ -0,0 +1,36 @@ +log: + level: info + +runner: + file: /data/.runner + capacity: 1 + envs: + GOPROXY: "https://goproxy.cn,direct" + GOSUMDB: "sum.golang.google.cn" + PATH: "/opt/easyai-gateway-ci/bin:/opt/easyai-gateway-ci/toolchains/go/bin:/opt/easyai-gateway-ci/toolchains/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + timeout: 3h + shutdown_timeout: 5m + insecure: false + fetch_timeout: 5s + fetch_interval: 2s + labels: + - "easyai-gateway-ci-unprivileged-v2:docker://docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7" + +cache: + enabled: false + +container: + docker_host: "-" + privileged: false + force_pull: false + force_rebuild: false + require_docker: true + docker_timeout: 1m + bind_workdir: false + workdir_parent: workspace + options: "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro" + valid_volumes: + - "/opt/easyai-gateway-ci" + +host: + workdir_parent: /data/host-work diff --git a/deploy/ci/easyai-gateway-act-runner.service b/deploy/ci/easyai-gateway-act-runner.service deleted file mode 100644 index bd207d6..0000000 --- a/deploy/ci/easyai-gateway-act-runner.service +++ /dev/null @@ -1,31 +0,0 @@ -[Unit] -Description=EasyAI AI Gateway Gitea Actions runner -After=network-online.target docker.service -Wants=network-online.target docker.service - -[Service] -Type=simple -User=easyai-gateway-runner -Group=easyai-gateway-runner -WorkingDirectory=/var/lib/easyai-gateway-runner -Environment=HOME=/var/lib/easyai-gateway-runner -Environment=PATH=/opt/easyai-gateway-ci/bin:/opt/easyai-gateway-ci/toolchains/go/bin:/opt/easyai-gateway-ci/toolchains/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -# Gitea 1.22 can leave a long-running poll stale after a completed host job. -# Process exactly one job, then let Restart=always create a fresh poller. -ExecStart=/opt/easyai-gateway-ci/bin/gitea-runner --config /etc/easyai-gateway-runner/config.yaml daemon --once -Restart=always -RestartSec=5s -TimeoutStopSec=330s -# Tag jobs invoke one SHA-validating root helper. Docker access already gives -# this dedicated repository runner an equivalent host-root trust boundary. -NoNewPrivileges=false -PrivateTmp=true -ProtectSystem=strict -ProtectHome=true -ProtectKernelTunables=true -ProtectKernelModules=true -ProtectControlGroups=true -ReadWritePaths=/var/lib/easyai-gateway-runner - -[Install] -WantedBy=multi-user.target diff --git a/deploy/ci/easyai-gateway-ci-v2-runner.service b/deploy/ci/easyai-gateway-ci-v2-runner.service new file mode 100644 index 0000000..29c8874 --- /dev/null +++ b/deploy/ci/easyai-gateway-ci-v2-runner.service @@ -0,0 +1,43 @@ +[Unit] +Description=EasyAI AI Gateway rootless DinD Gitea Actions runner v2 +Requires=docker.service +After=network-online.target docker.service +Wants=network-online.target + +[Service] +Type=simple +User=root +Group=root +Environment=RUNNER_SECURITY_OPTIONS= +EnvironmentFile=-/etc/easyai-gateway-ci-v2/runner.env +ExecStartPre=-/usr/bin/docker rm --force easyai-gateway-ci-v2-runner +ExecStart=/usr/bin/docker run --rm --pull=never --name easyai-gateway-ci-v2-runner --privileged --pids-limit 4096 $RUNNER_SECURITY_OPTIONS --volume easyai-gateway-ci-v2-data:/data --volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro --volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro --env CONFIG_FILE=/config.yaml --env DOCKERD_ROOTLESS_ROOTLESSKIT_NET=slirp4netns --env DOCKERD_ROOTLESS_ROOTLESSKIT_MTU=65520 --env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a +ExecStop=-/usr/bin/docker stop --time 360 easyai-gateway-ci-v2-runner +Restart=always +RestartSec=5s +TimeoutStartSec=60s +TimeoutStopSec=390s + +# systemd only launches the host Docker client. Pull-request jobs run inside the +# nested rootless daemon and never receive the host or nested Docker socket. +NoNewPrivileges=true +CapabilityBoundingSet= +AmbientCapabilities= +PrivateTmp=true +PrivateDevices=true +ProtectSystem=strict +ProtectHome=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectKernelLogs=true +ProtectControlGroups=true +ProtectClock=true +ProtectHostname=true +LockPersonality=true +RestrictRealtime=true +RestrictSUIDSGID=true +SystemCallArchitectures=native +UMask=0077 + +[Install] +WantedBy=multi-user.target diff --git a/docs/decisions/001-production-cicd.md b/docs/decisions/001-production-cicd.md index d9ae60f..9d4d9b6 100644 --- a/docs/decisions/001-production-cicd.md +++ b/docs/decisions/001-production-cicd.md @@ -1,4 +1,4 @@ -# ADR-001: 使用仓库级 Runner 和不可变镜像发布生产环境 +# ADR-001: 隔离质量 CI 与 root-owned 生产发布 ## 状态 @@ -10,42 +10,59 @@ Accepted ## 背景 -AI Gateway 已通过独立部署仓库和 Docker Compose 手工运行在 `110.42.51.33`,公共入口是 `https://ai.51easyai.com`。同一服务器还运行认证中心、K3s 和其他 EasyAI 服务,磁盘空间有限。Gitea 1.22 的 host Runner 已知不能可靠启动第二个依赖 Job,也不完整支持 GitHub Actions 的 `environment`、`permissions` 和 `concurrency` 语义。 +AI Gateway 已通过独立部署仓库和 Docker Compose 运行在 `110.42.51.33`,公共入口是 `https://ai.51easyai.com`。同一服务器还运行认证中心、K3s 和其他 EasyAI 服务,磁盘与信任边界都很紧张。Gitea 1.22 对依赖 Job、`environment`、`permissions` 和 `concurrency` 的支持也不能作为生产发布的安全边界。 -生产发布必须满足:源码可追溯、数据库迁移前有备份、失败可恢复上一应用版本、Secret 不进入仓库或流水线日志,并且不能让普通分支自动修改生产环境。 +源码仓需要为 Pull Request、`main` 和版本 Tag 提供真实质量门禁;生产发布还必须满足源码可追溯、镜像不可变、数据库迁移前有备份、失败可恢复上一应用版本,以及 Tag 控制的脚本不能直接取得宿主 root 权限。 ## 决策 -- 为 `BCAI/easyai-ai-gateway` 注册独立、仓库级 host Runner。它不与认证中心 Runner 共享身份、状态目录或任务标签。 -- Pull Request 和 `main` Push 执行相同质量门禁与镜像构建;只有符合 `vMAJOR.MINOR.PATCH` 形式、且目标提交属于 `main` 历史的 Tag 才发布生产。 -- API/Web 镜像使用完整 Git SHA 作为运行时 Tag。语义版本 Tag 只是发布授权信号,不作为可变镜像标识。 -- 验证和发布处于同一个 Runner Job。这样部署只使用已通过全部门禁的同一 checkout,并规避当前 Gitea 版本的依赖 Job 领取问题。 -- Runner 使用专属 Buildx builder,缓存上限为 2 GB。服务器 root 现有的阿里云 Registry 凭据只由固定发布助手使用,不复制到 Gitea Secret。 -- 发布助手在迁移前生成 PostgreSQL custom-format 备份并验证目录;保留最近 5 份。应用健康检查失败时恢复上一组 API/Web 镜像。 -- 数据库迁移必须保持向后兼容。自动回滚只覆盖应用镜像;数据库恢复需要人工确认,避免覆盖失败发布后产生的有效写入。 -- Gitea Runner 的 Docker 权限等价于宿主机 root 权限。因此只允许受信任的仓库维护者触发该 Runner,来自 Fork 的 Pull Request 必须先经维护者批准。 +### 质量 CI + +- `BCAI/easyai-ai-gateway` 使用仓库级 Runner 标签 `easyai-gateway-ci-unprivileged-v2`。PR 与 `main` 由 `ci.yml` 执行,context 分别为 `ci / verify (pull_request)`、`ci / verify (push)`;`v*` Tag 必须由独立 `release-ci.yml` 执行,context 为 `release-ci / verify-tag (push)`。两份 workflow 执行相同质量门禁,但 Tag 不复用 `main` 的旧 context。 +- Runner 外层固定为 `gitea/runner:2.0.0-dind-rootless` 的指定 digest;Job 固定为 `node:24.16.0-bookworm` 的指定 digest。外层按官方 rootless DinD 要求使用 `--privileged`,但内层 Docker daemon 和 Runner 以 UID 1000 运行。 +- 不把宿主或内层 Docker socket 挂入 Job。Runner 配置使用 `docker_host: "-"`、`privileged: false`,工作流唯一允许的 bind source 是只读 `/opt/easyai-gateway-ci`,其中提供固定 Go、Node、pnpm、ShellCheck、Compose、Trivy 和 govulncheck 工具链。ShellCheck 与 Compose 使用官方静态发行物和固定 SHA-256,不复制宿主的可变插件或在 Job 中临时安装。RootlessKit 强制 `--pidns`,即使 PR 传入 `--pid=host`,Job 也不能进入外层 Runner 的 PID namespace;provision 会实测 namespace inode。 +- CI 只执行格式、静态检查、单元测试、依赖审计、Compose 校验、仓库扫描和 CI 自测;不构建发布镜像,不调用 `sudo`,不执行生产发布助手。 +- 版本 Tag 必须是完整 SemVer,且目标提交必须属于 `main` 历史。该检查是发布前置证据,但本身不授予生产权限。 +- Runner 注册状态保存在专用 Docker named volume,首次注册 Token 只传给短命注册容器;长期 systemd 服务不保存 Token,也不加入宿主 `docker`/`sudo` 用户组。 + +### 生产 CD + +- 生产发布由部署仓的 root-owned dispatcher 负责,和源码 CI Runner 分离。dispatcher 只接受受保护的 `main` 历史版本 Tag,并在对应源码 SHA 上同时等待 `ci / verify (push)` 与本次 Tag 新产生的 `release-ci / verify-tag (push)` 成功。 +- root 服务不执行 Tag 控制的 `pnpm`、测试脚本、源码仓 shell 脚本或应用程序。它只调用部署仓中由 root 固定和审核的命令,通过 BuildKit 构建目标镜像、用 Trivy 扫描镜像、发布 Registry digest,再进行备份、迁移、健康检查和回滚。 +- API/Web 镜像以完整 Git SHA 作为可追溯 Tag,但生产 Compose 使用 Registry digest。服务器为每个 Git SHA 保存 root-only digest 清单,并拒绝把未知的既有 SHA Tag 当作可信发布物。 +- 发布助手在迁移前生成并验证 PostgreSQL custom-format 备份,保留最近 5 份。应用健康检查失败时恢复上一组 digest-pinned 镜像;数据库恢复仍需人工确认。 + +### 残余风险与触发策略 + +rootless DinD 显著缩小了 Job 到宿主 Docker/root 的直接通路,但不是虚拟机级隔离。外层容器仍需 `--privileged`;Runner、rootless Docker、内核或容器运行时漏洞,以及 Gitea 对 `jobs..container.options` 的透传,都可能造成容器边界突破。Job 还保留依赖安装所需的出站网络,并可能消耗共享主机的 CPU、内存和磁盘。因此: + +- 来自同仓受信任分支的 PR 可自动运行;来自 Fork 的 PR 必须先由维护者检查工作流变更并在 Gitea 中批准,不能自动调度。 +- CI Job 不接收生产 Secret、宿主 Docker socket、部署目录或 Runner 状态目录。 +- Runner 容量固定为 1,外层容器设 4096 PID 上限;磁盘在安装前后硬门禁,但共享主机仍不具备完整的 CPU、内存和长期磁盘配额隔离。 +- 中长期应把不受信任 PR CI 迁移到独立主机或虚拟机;当前 rootless Runner 不能被描述为强多租户沙箱。 +- Ubuntu 22.04 没有受限 user namespace,生产主机缺少 `rootlesskit` AppArmor profile 时不硬编码该 profile。provision 会先探测;在启用了 `kernel.apparmor_restrict_unprivileged_userns=1` 的新系统上,如果 profile 缺失则失败,不会降级为 `apparmor=unconfined` 或关闭内核限制。 ## 备选方案 -### 复用认证中心 Runner +### 让源码 Runner 直接挂载宿主 Docker socket -拒绝。现有 Runner 是认证中心仓库级身份,扩大为组织级 Runner 会让更多仓库获得同一 Docker/root 信任边界。 +拒绝。Docker socket 等价于宿主 root,任何 PR 工作流都可能取得生产服务器控制权,Fork 审批也不能把它变成可靠的技术隔离。 -### 由 Runner 保存 SSH 或 Registry Secret +### 使用 host executor 但移除 Docker 组 -拒绝。Runner 已位于目标服务器,绕回 SSH 会增加长期凭据;复制 Registry Secret 也会扩大暴露面。固定 root helper 可以复用服务器现有登录状态。 +拒绝。host Job 仍可读取宿主可见文件、消耗宿主资源并攻击同一用户进程;Gitea 对容器选项的处理也无法为 host executor 提供隔离。 -### `main` 通过后直接发布生产 +### 让 Tag 工作流通过 `sudo` 调用发布脚本 -拒绝。`ai.51easyai.com` 是生产入口。语义版本 Tag 提供明确的发布授权点,同时仍保留自动化与可审计性。 +拒绝。Tag 可以控制 checkout 中的脚本和应用代码;root 服务不得执行这些内容。固定部署仓 dispatcher 是独立的受保护信任根。 -### 只依赖 `latest` +### `main` 通过后立即发布或只依赖 `latest` -拒绝。`latest` 无法证明运行中的容器对应哪个源码提交,也不能可靠选择回滚版本。 +拒绝。语义版本 Tag 是明确的发布授权点;digest 才能证明运行中的容器对应不可变产物并可靠选择回滚版本。 ## 影响 -- 日常合并不会自动改变生产;创建并推送版本 Tag 才会部署。 -- 首次启用 CD 时需要在服务器捕获当前运行镜像作为 bootstrap 回滚基线。 -- 发布前必须确保数据库迁移对上一应用版本保持兼容。 -- 如果未来增加不受信任的贡献者,应把 CI 构建迁移到隔离的 rootless Runner,并保留独立的受保护部署 Runner。 +- PR、`main` 和版本 Tag 都有自动质量证据,但源码仓 CI 成功不会单独获得生产权限。 +- 创建受保护版本 Tag 后,部署仓 dispatcher 必须同时取得同 SHA 的 `main` context 和独立 Tag context,不能把旧 `main` 成功状态误当作本次 Tag 验证。 +- 首次安装会清理本项目专属 Buildx 缓存并执行 8 GiB 前置、4 GiB 后置磁盘门禁,不会全局 prune 共享 Docker 状态。 +- 首次启用 CD 仍需捕获当前运行镜像作为 bootstrap 回滚基线,并保证数据库迁移对上一应用版本向后兼容。 diff --git a/docs/runbooks/production-ci-cd.md b/docs/runbooks/production-ci-cd.md index e8d3eb5..c2f3693 100644 --- a/docs/runbooks/production-ci-cd.md +++ b/docs/runbooks/production-ci-cd.md @@ -1,42 +1,102 @@ # 生产 CI/CD 运行手册 -## 流水线约定 +## 信任边界与固定资源 -- Runner:`easyai-gateway-act-runner.service` -- Runner 标签:`easyai-gateway-linux:host` -- Runner 状态:`/var/lib/easyai-gateway-runner` -- 工具链:`/opt/easyai-gateway-ci` -- Gitea:`https://git.51easyai.com/BCAI/easyai-ai-gateway` +- 源码仓:`https://git.51easyai.com/BCAI/easyai-ai-gateway` +- CI Runner:`easyai-gateway-ci-v2-runner.service` +- CI 标签:`easyai-gateway-ci-unprivileged-v2` +- Runner 状态:Docker named volume `easyai-gateway-ci-v2-data` 内的 `/data/.runner` +- 只读工具链:`/opt/easyai-gateway-ci` +- Runner 配置:`/etc/easyai-gateway-ci-v2/config.yaml` +- Runner 运行参数:`/etc/easyai-gateway-ci-v2/runner.env` - 生产入口:`https://ai.51easyai.com` - 部署目录:`/root/easyai-ai-gateway-deploy` -PR、`main` Push 和版本 Tag 都会执行格式、Go vet/test、govulncheck、前端 lint/test/build、依赖审计、Compose 校验、Trivy 扫描与 API/Web 镜像构建。只有版本 Tag 会执行最后的生产发布步骤。 +源码仓只执行质量 CI:PR context 是 `ci / verify (pull_request)`,`main` context 是 `ci / verify (push)`,版本 Tag 使用独立 workflow/context `release-ci / verify-tag (push)`。三者都执行 Go 格式/vet/test/govulncheck、前端 lint/test/build、依赖审计、Compose 校验、Trivy 仓库扫描和 CI 脚本自测。Job 没有 Docker socket、`sudo` 或部署目录权限,也不构建发布镜像。 -## 首次安装或修复 Runner +生产 CD 由部署仓的 root-owned dispatcher 执行。它在相同源码 SHA 上同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)` 成功,然后只运行部署仓中固定的 BuildKit 构建、Trivy 镜像扫描、digest 发布、备份、迁移、健康检查和回滚命令;不得执行 Tag checkout 中的 `pnpm`、测试、shell 脚本或应用程序。 -从 Gitea 仓库设置的 Actions Runner 页面取得一次性仓库注册 Token,在服务器可信 checkout 中执行: +provision 安装 checksum-pinned 的官方静态 ShellCheck `0.11.0` 和 Docker Compose `5.3.1`,不会依赖宿主是否预装这些命令。所有下载都在使用前校验固定 SHA-256,并在精确 Node Job 镜像内实际执行版本检查。 + +## 首次安装或修复 CI Runner + +前置条件:Linux x86_64、可用 Docker Engine、root 权限,以及 Docker 文件系统至少 8 GiB 可用空间。脚本只压缩本项目专属 `easyai-gateway-ci` Buildx 缓存,不运行全局 `docker system prune`,因为该服务器与其他服务共享 Docker 状态。 + +从 Gitea 仓库设置的 Actions Runner 页面取得一次性仓库注册 Token,在服务器可信的源码 checkout 中执行: ```bash RUNNER_REGISTRATION_TOKEN='<一次性 Token>' ./scripts/provision-ci-runner.sh ``` -Token 只通过进程环境传入。生成的 `/var/lib/easyai-gateway-runner/.runner` 必须保持 `0600`,不得复制到仓库、报告或聊天记录。 - -验证: +已有有效 named volume 时可直接重跑,不再需要 Token: ```bash -systemctl is-active easyai-gateway-act-runner.service -systemctl is-enabled easyai-gateway-act-runner.service -journalctl -u easyai-gateway-act-runner.service --since '15 minutes ago' +./scripts/provision-ci-runner.sh ``` -Runner 使用 `daemon --once`。每个 Job 结束后进程自然退出,再由 systemd 启动新的长轮询进程。 +注册 Token 通过交互 stdin 送给短命注册容器,不出现在宿主或容器的命令行参数中;provision 随后立即从环境中清除它,长期 systemd unit 也不包含 Token。不要输出或复制 `/data/.runner` 内容。脚本会停用旧 host Runner,并确保旧 Runner 用户和 v2 遗留用户不属于 `docker`/`sudo` 组。 + +默认磁盘门禁可用环境变量提高,但生产环境不应降低: + +```bash +CI_RUNNER_MIN_FREE_GIB=8 \ +CI_RUNNER_MIN_POST_INSTALL_FREE_GIB=4 \ +RUNNER_REGISTRATION_TOKEN='<一次性 Token>' \ +./scripts/provision-ci-runner.sh +``` + +## AppArmor 分支 + +Ubuntu 22.04 默认没有 `kernel.apparmor_restrict_unprivileged_userns` 限制,当前服务器没有 `rootlesskit` profile 是受支持路径。provision 会实测 Docker 是否接受该 profile;不存在时写入: + +```text +RUNNER_SECURITY_OPTIONS="" +``` + +如果新系统的 `/proc/sys/kernel/apparmor_restrict_unprivileged_userns` 为 `1`,则必须先由宿主发行版或 Docker 安装包提供并加载 `rootlesskit` profile;provision 只探测和使用,不会自行复制一份 profile。缺失时 provision 会停止;不得手工关闭该 sysctl,也不得使用 `apparmor=unconfined` 绕过。 + +参考:[Docker Rootless Docker-in-Docker](https://docs.docker.com/engine/security/rootless/tips/#rootless-docker-in-docker)、[Ubuntu 安全特性版本矩阵](https://documentation.ubuntu.com/security/security-features/security-features-overview/)。 + +## Runner 真实验证 + +安装脚本会先启动一个无 Token、无持久 Runner 状态挂载、无宿主 bind mount 的短命 rootless DinD probe;随后启动长期服务,确认 Runner 已向 Gitea declare、通过内层 `docker info` 验证 `name=rootless`,再从内层 daemon 拉取固定 digest 的 Job 镜像,并在该镜像中实际运行全部工具。人工复核: + +```bash +systemctl is-active easyai-gateway-ci-v2-runner.service +systemctl is-enabled easyai-gateway-ci-v2-runner.service +systemctl cat easyai-gateway-ci-v2-runner.service +journalctl -u easyai-gateway-ci-v2-runner.service --since '15 minutes ago' --no-pager + +docker inspect -f 'user={{.Config.User}} privileged={{.HostConfig.Privileged}} apparmor={{.AppArmorProfile}}' \ + easyai-gateway-ci-v2-runner +docker inspect -f '{{range .Mounts}}{{println .Source " -> " .Destination .Mode}}{{end}}' \ + easyai-gateway-ci-v2-runner +docker exec easyai-gateway-ci-v2-runner docker info \ + --format '{{range .SecurityOptions}}{{println .}}{{end}}' +docker exec easyai-gateway-ci-v2-runner docker image inspect \ + 'docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7' \ + --format '{{.Id}}' +``` + +预期:外层 image user 为 `rootless`、外层因 DinD 为 `privileged=true`、内层安全选项含 `name=rootless`;RootlessKit 使用独立 PID namespace,内层 `--pid=host` 也不能看到外层 Runner;挂载仅包含专用 `/data`、只读配置和只读工具链,不能出现 `/var/run/docker.sock` 或 `/run/docker.sock`。 + +在 Gitea 中分别验证: + +1. 同仓 PR 的 `ci / verify (pull_request)` 成功。 +2. `main` Push 的 `ci / verify (push)` 成功。 +3. 合法 SemVer Tag 新产生独立 `release-ci / verify-tag (push)`;非法或不属于 `main` 历史的 Tag 失败,且 Tag context 不能被旧 `main` context 替代。 +4. Fork PR 默认不自动调度,维护者先检查 `.gitea/workflows/**` 和容器选项后再批准。 + +## Fork PR 残余风险 + +rootless DinD 不是 VM 沙箱。外层仍按官方要求使用 `--privileged`,Gitea runner 也会解析 PR 控制的 `jobs..container.options`。配置已禁止 Job privileged、宿主/内层 Docker socket 和任意 bind source,但 Runner、rootless Docker、容器运行时或内核漏洞仍可能突破边界;Job 的出站网络和资源消耗也不是强租户隔离。因此 Fork PR 必须维护者批准,CI 不得注入生产 Secret;未来应迁移到独立 CI 主机或虚拟机。 ## 发布生产版本 -1. 合并到 `main`,等待 `ci / verify (push)` 成功。 -2. 在已验证的 `main` 提交上创建语义版本 Tag。 -3. 推送 Tag,等待同一 CI Job 完成镜像扫描、发布、数据库备份、迁移和公共健康检查。 +1. 合并到受保护 `main`,确认源码 CI 成功。 +2. 在已验证提交上创建完整 SemVer Tag 并推送。 +3. 确认同一 SHA 的 `ci / verify (push)` 和本次 `release-ci / verify-tag (push)` 都成功。 +4. 由部署仓 root-owned dispatcher 构建、扫描并发布 digest;源码仓工作流不会直接调用生产 helper。 ```bash git switch main @@ -45,7 +105,7 @@ git tag -a v0.1.1 -m 'release: v0.1.1' git push origin v0.1.1 ``` -流水线会部署 Tag 指向提交的完整 Git SHA,而不是 `latest` 或版本号镜像。 +dispatcher 以完整 Git SHA 发布 Registry Tag,并把 Registry 返回的 digest 写入服务器 root-only 发布清单。生产 Compose 使用 `repository@sha256:...`,不是 `latest`、版本号或可变 SHA Tag。 ## 发布后验证 @@ -57,34 +117,32 @@ ssh root@110.42.51.33 'cd /root/easyai-ai-gateway-deploy && ./gateway-ops.sh ps' 同时检查服务器磁盘、API 日志和数据库容器健康状态。发布后的首小时如出现错误率翻倍、P95 延迟增加 50%、数据完整性或安全问题,应立即回滚。 -## 应用回滚 +## 应用与数据库回滚 -发布失败时脚本会自动恢复 `.release.env` 中的上一组 API/Web 镜像并重复健康检查。人工回滚到一个仍存在于本机或 Registry 的历史 Git SHA: +发布失败时固定部署脚本会恢复上一组 API/Web digest 并重复健康检查。人工回滚只选择服务器已有 root-only 发布清单的历史 Git SHA: ```bash sudo /usr/local/sbin/easyai-ai-gateway-release <40 位历史 Git SHA> ``` -该操作也会先创建数据库备份。不要把 `latest`、分支名或版本号传给发布助手。 - -## 数据库恢复 - -备份位于 `/var/backups/easyai-ai-gateway`,默认只保留最近 5 份。应用回滚不会自动恢复数据库。只有确认需要回退数据且已经停止写入后,才能由操作员选择备份执行 `pg_restore`;恢复前必须再保留当前数据库副本。 +没有 `/root/easyai-ai-gateway-deploy/.releases/.env` 的远端 SHA Tag 不应被盲目信任。数据库备份位于 `/var/backups/easyai-ai-gateway`,只保留最近 5 份。应用回滚不会自动恢复数据库;只有停止写入并再次保留当前副本后,操作员才能选择备份执行 `pg_restore`。 ## Runner 故障恢复 -重启前先确认没有构建、扫描或部署子进程: +先在 Gitea 确认没有运行中的 Job,再检查服务与两个 daemon: ```bash -ps -u easyai-gateway-runner -o pid,ppid,etime,cmd --forest -systemctl status easyai-gateway-act-runner.service --no-pager +systemctl status easyai-gateway-ci-v2-runner.service --no-pager +docker logs --tail 100 easyai-gateway-ci-v2-runner +docker exec easyai-gateway-ci-v2-runner docker info +df -h /var/lib/docker ``` 确认空闲后才执行: ```bash -systemctl restart easyai-gateway-act-runner.service -journalctl -u easyai-gateway-act-runner.service -n 50 --no-pager +systemctl restart easyai-gateway-ci-v2-runner.service +journalctl -u easyai-gateway-ci-v2-runner.service -n 100 --no-pager ``` -不要因为日志暂时没有新增就用定时器重启 Runner;镜像构建和扫描可长时间无输出。 +不要删除 `easyai-gateway-ci-v2-data` volume;它包含 Runner 注册状态和内层 daemon 缓存。不得用全局 prune 处理磁盘压力。 diff --git a/scripts/ci-build-images.sh b/scripts/ci-build-images.sh index f34ba28..48ce1f0 100755 --- a/scripts/ci-build-images.sh +++ b/scripts/ci-build-images.sh @@ -9,6 +9,17 @@ registry=${AI_GATEWAY_IMAGE_REGISTRY:-registry.cn-shanghai.aliyuncs.com/easyaigc builder=${AI_GATEWAY_BUILDX_BUILDER:-easyai-gateway-ci} cache_limit=${AI_GATEWAY_BUILDX_CACHE_LIMIT:-2gb} platform=${AI_GATEWAY_PLATFORM:-linux/amd64} +keep_images=0 +build_complete=0 + +case ${1:-} in + '') ;; + --keep-images) keep_images=1 ;; + *) + echo 'usage: ci-build-images.sh [--keep-images]' >&2 + exit 64 + ;; +esac if [[ ! $tag =~ ^[0-9a-f]{40}$ ]]; then echo 'IMAGE_TAG must be a full lowercase Git SHA' >&2 @@ -18,19 +29,45 @@ if [[ ! $registry =~ ^[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._/-]+)+$ ]]; then echo 'AI_GATEWAY_IMAGE_REGISTRY has an invalid format' >&2 exit 1 fi +if [[ ! $cache_limit =~ ^[1-9][0-9]*(kb|mb|gb)$ ]]; then + echo 'AI_GATEWAY_BUILDX_CACHE_LIMIT has an invalid format' >&2 + exit 1 +fi + +api_image="$registry/ai-gateway:$tag" +web_image="$registry/ai-gateway-web:$tag" + +cleanup() { + original_status=$? + trap - EXIT HUP INT TERM + cleanup_status=0 + + docker buildx prune --builder "$builder" --force --max-used-space "$cache_limit" >/dev/null || { + cleanup_status=$? + echo 'failed to prune the CI BuildKit cache' >&2 + } + if [[ $keep_images -ne 1 || $build_complete -ne 1 ]]; then + docker image rm "$api_image" "$web_image" >/dev/null 2>&1 || { + image_rm_status=$? + [[ $cleanup_status -ne 0 ]] || cleanup_status=$image_rm_status + echo 'failed to remove CI image tags' >&2 + } + fi + + if [[ $original_status -ne 0 ]]; then + exit "$original_status" + fi + exit "$cleanup_status" +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM if ! docker buildx inspect "$builder" >/dev/null 2>&1; then docker buildx create --name "$builder" --driver docker-container --use >/dev/null fi docker buildx inspect "$builder" --bootstrap >/dev/null - -cleanup_cache() { - docker buildx prune --builder "$builder" --force --max-used-space "$cache_limit" >/dev/null || true -} -trap cleanup_cache EXIT HUP INT TERM - -api_image="$registry/ai-gateway:$tag" -web_image="$registry/ai-gateway-web:$tag" common_args=( --builder "$builder" --platform "$platform" @@ -58,4 +95,5 @@ for image in "$api_image" "$web_image"; do --exit-code 1 --timeout 15m "$image" done +build_complete=1 printf 'gateway_images=PASS git_sha=%s api=%s web=%s\n' "$tag" "$api_image" "$web_image" diff --git a/scripts/ci-validate-semver.sh b/scripts/ci-validate-semver.sh new file mode 100755 index 0000000..167235f --- /dev/null +++ b/scripts/ci-validate-semver.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +tag=${1:-} + +die() { + printf 'invalid release tag: %s\n' "$tag" >&2 + exit 1 +} + +[[ $tag == v* ]] || die +version=${tag#v} + +# Bash arrays discard trailing empty fields, so parsing first would accept +# values such as v1.2.3. and v1.2.3-alpha. Validate the complete string with +# the SemVer 2.0.0 grammar before doing anything else with the tag. +semver_pattern='^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(\.(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*))?(\+([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' +LC_ALL=C +export LC_ALL +[[ $version =~ $semver_pattern ]] || die + +printf 'release_tag=PASS tag=%s\n' "$tag" diff --git a/scripts/provision-ci-runner.sh b/scripts/provision-ci-runner.sh index 23a92a0..4092fdd 100755 --- a/scripts/provision-ci-runner.sh +++ b/scripts/provision-ci-runner.sh @@ -2,36 +2,121 @@ set -eu ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) -PREFIX=${CI_RUNNER_PREFIX:-/opt/easyai-gateway-ci} -RUNNER_HOME=${CI_RUNNER_HOME:-/var/lib/easyai-gateway-runner} -RUNNER_USER=${CI_RUNNER_USER:-easyai-gateway-runner} +PREFIX=/opt/easyai-gateway-ci GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL:-https://git.51easyai.com} -RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-linux} -RUNNER_LABEL=${RUNNER_LABEL:-easyai-gateway-linux:host} +RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-ci-unprivileged-v2} +RUNNER_VOLUME=easyai-gateway-ci-v2-data +RUNNER_CONTAINER=easyai-gateway-ci-v2-runner +LEGACY_BUILDER=${AI_GATEWAY_BUILDX_BUILDER:-easyai-gateway-ci} +MIN_FREE_GIB=${CI_RUNNER_MIN_FREE_GIB:-8} +MIN_POST_INSTALL_FREE_GIB=${CI_RUNNER_MIN_POST_INSTALL_FREE_GIB:-4} GITEA_RUNNER_VERSION=2.0.0 -GITEA_RUNNER_SHA256=447156b33407ee045409f5552bd4a188a315cdd4085b4b498d8d4a9ad26c9f73 GO_VERSION=1.26.5 GO_SHA256=5c2c3b16caefa1d968a94c1daca04a7ca301a496d9b086e17ad77bb81393f053 NODE_VERSION=24.16.0 NODE_SHA256=d804845d34eddc21dc1092b519d643ef40b1f58ec5dec5c22b1f4bd8fabde6c9 TRIVY_VERSION=0.70.0 TRIVY_SHA256=8b4376d5d6befe5c24d503f10ff136d9e0c49f9127a4279fd110b727929a5aa9 +SHELLCHECK_VERSION=0.11.0 +SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198 +COMPOSE_VERSION=5.3.1 +COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959 PNPM_VERSION=10.18.1 +RUNNER_IMAGE='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a' +JOB_IMAGE='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7' +RUNNER_LABEL="easyai-gateway-ci-unprivileged-v2:docker://$JOB_IMAGE" -[ "$(id -u)" -eq 0 ] || { echo "run as root" >&2; exit 1; } -[ "$(uname -s)-$(uname -m)" = "Linux-x86_64" ] || { echo "only Linux x86_64 is supported" >&2; exit 1; } +fail() { + echo "$*" >&2 + exit 1 +} + +case $MIN_FREE_GIB:$MIN_POST_INSTALL_FREE_GIB in + *[!0-9:]* | :* | *:) + fail "CI runner free-space limits must be whole GiB values" + ;; +esac +[ "$MIN_FREE_GIB" -ge 8 ] || fail "CI_RUNNER_MIN_FREE_GIB cannot be lower than 8" +[ "$MIN_POST_INSTALL_FREE_GIB" -ge 4 ] || \ + fail "CI_RUNNER_MIN_POST_INSTALL_FREE_GIB cannot be lower than 4" + +[ "$(id -u)" -eq 0 ] || fail "run as root" +[ "$(uname -s)-$(uname -m)" = "Linux-x86_64" ] || fail "only Linux x86_64 is supported" +case $RUNNER_IMAGE in + *"/runner:${GITEA_RUNNER_VERSION}-dind-rootless@sha256:"*) ;; + *) fail "RUNNER_IMAGE does not match GITEA_RUNNER_VERSION" ;; +esac +command -v docker >/dev/null 2>&1 || fail "Docker Engine is required" +docker info >/dev/null 2>&1 || fail "Docker Engine is not reachable" +grep -Fq "\"$RUNNER_LABEL\"" "$ROOT/deploy/ci/act-runner-v2-config.yaml" || \ + fail "runner label does not match the pinned job image" + +# Fail closed before installing anything. Neither the legacy host runner nor an +# earlier v2 unit may poll for work while its replacement is being prepared. +systemctl disable --now easyai-gateway-act-runner.service >/dev/null 2>&1 || true +systemctl disable --now easyai-gateway-ci-v2-runner.service >/dev/null 2>&1 || true +docker rm --force "$RUNNER_CONTAINER" >/dev/null 2>&1 || true +rm -f /etc/systemd/system/easyai-gateway-act-runner.service + +if id easyai-gateway-runner >/dev/null 2>&1; then + if getent group docker >/dev/null 2>&1; then + gpasswd -d easyai-gateway-runner docker >/dev/null 2>&1 || true + fi + if getent group sudo >/dev/null 2>&1; then + gpasswd -d easyai-gateway-runner sudo >/dev/null 2>&1 || true + fi +fi +if id easyai-gateway-ci-v2 >/dev/null 2>&1; then + if getent group docker >/dev/null 2>&1; then + gpasswd -d easyai-gateway-ci-v2 docker >/dev/null 2>&1 || true + fi + if getent group sudo >/dev/null 2>&1; then + gpasswd -d easyai-gateway-ci-v2 sudo >/dev/null 2>&1 || true + fi +fi +for account in easyai-gateway-runner easyai-gateway-ci-v2; do + if id "$account" >/dev/null 2>&1 && \ + id -nG "$account" | tr ' ' '\n' | grep -Eq '^(docker|sudo)$'; then + fail "$account must not belong to a privileged group" + fi +done if ! command -v make >/dev/null 2>&1 || ! command -v gcc >/dev/null 2>&1 || \ - ! command -v jq >/dev/null 2>&1 || ! command -v shellcheck >/dev/null 2>&1 || \ - ! docker compose version >/dev/null 2>&1 || ! docker buildx version >/dev/null 2>&1; then + ! command -v jq >/dev/null 2>&1 || ! command -v curl >/dev/null 2>&1 || \ + ! command -v xz >/dev/null 2>&1; then export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y --no-install-recommends \ - build-essential ca-certificates curl docker-buildx-plugin docker-compose-plugin \ - git jq perl shellcheck unzip util-linux xz-utils + build-essential ca-certificates curl git jq perl unzip util-linux xz-utils fi +docker_root=$(docker info --format '{{.DockerRootDir}}') +[ -n "$docker_root" ] || fail "Docker root directory could not be determined" + +require_free_space() { + minimum_gib=$1 + phase=$2 + available_kib=$(df -Pk "$docker_root" | awk 'NR == 2 { print $4 }') + case $available_kib in + '' | *[!0-9]*) fail "free space could not be determined for $docker_root" ;; + esac + minimum_kib=$((minimum_gib * 1024 * 1024)) + if [ "$available_kib" -lt "$minimum_kib" ]; then + echo "$phase requires at least ${minimum_gib} GiB free on the Docker filesystem" >&2 + return 1 + fi +} + +# This builder belonged only to the retired gateway image-building runner. Cap +# its unused cache before adding the nested daemon; never prune global images, +# volumes, or caches owned by another service on this shared production host. +if docker buildx version >/dev/null 2>&1 && \ + docker buildx inspect "$LEGACY_BUILDER" >/dev/null 2>&1; then + docker buildx prune --builder "$LEGACY_BUILDER" --force --max-used-space 512mb +fi +require_free_space "$MIN_FREE_GIB" "CI runner installation" + download_verified() { url=$1 output=$2 @@ -43,20 +128,22 @@ download_verified() { printf '%s %s\n' "$checksum" "$output" | sha256sum -c - } -if ! id "$RUNNER_USER" >/dev/null 2>&1; then - useradd --system --home-dir "$RUNNER_HOME" --create-home --shell /usr/sbin/nologin "$RUNNER_USER" -fi -usermod -aG docker "$RUNNER_USER" - install -d -m 0755 "$PREFIX/bin" "$PREFIX/downloads" "$PREFIX/toolchains" -install -d -o "$RUNNER_USER" -g "$RUNNER_USER" -m 0700 \ - "$RUNNER_HOME" "$RUNNER_HOME/cache" "$RUNNER_HOME/work" download_verified \ - "https://gitea.com/gitea/runner/releases/download/v${GITEA_RUNNER_VERSION}/gitea-runner-${GITEA_RUNNER_VERSION}-linux-amd64" \ - "$PREFIX/downloads/gitea-runner-${GITEA_RUNNER_VERSION}-linux-amd64" \ - "$GITEA_RUNNER_SHA256" -install -m 0755 "$PREFIX/downloads/gitea-runner-${GITEA_RUNNER_VERSION}-linux-amd64" "$PREFIX/bin/gitea-runner" + "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \ + "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \ + "$SHELLCHECK_SHA256" +rm -rf "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}" +tar -xJf "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \ + -C "$PREFIX/downloads" +install -m 0755 "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "$PREFIX/bin/shellcheck" + +download_verified \ + "https://github.com/docker/compose/releases/download/v${COMPOSE_VERSION}/docker-compose-linux-x86_64" \ + "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" \ + "$COMPOSE_SHA256" +install -m 0755 "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" "$PREFIX/bin/docker-compose" download_verified \ "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ @@ -84,30 +171,177 @@ tar -xzf "$PREFIX/downloads/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" -C "$PREF chmod 0755 "$PREFIX/bin/trivy" GOBIN="$PREFIX/bin" GOPROXY="${GO_MODULE_PROXY:-https://goproxy.cn,direct}" \ "$PREFIX/toolchains/go/bin/go" install golang.org/x/vuln/cmd/govulncheck@v1.6.0 +chown -R root:root "$PREFIX" +chmod -R go-w "$PREFIX" -install -d -m 0755 /etc/easyai-gateway-runner -install -m 0644 "$ROOT/deploy/ci/act-runner-config.yaml" /etc/easyai-gateway-runner/config.yaml -install -m 0644 "$ROOT/deploy/ci/easyai-gateway-act-runner.service" /etc/systemd/system/easyai-gateway-act-runner.service +require_free_space "$MIN_FREE_GIB" "runner image pull" +docker pull "$RUNNER_IMAGE" +require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed runner image pull" +docker volume create "$RUNNER_VOLUME" >/dev/null -if [ ! -s "$RUNNER_HOME/.runner" ]; then - [ -n "${RUNNER_REGISTRATION_TOKEN:-}" ] || { - echo "RUNNER_REGISTRATION_TOKEN is required for first registration" >&2 - exit 1 - } - runuser -u "$RUNNER_USER" -- env HOME="$RUNNER_HOME" \ - "$PREFIX/bin/gitea-runner" --config /etc/easyai-gateway-runner/config.yaml \ - register --no-interactive \ - --instance "$GITEA_INSTANCE_URL" \ - --token "$RUNNER_REGISTRATION_TOKEN" \ - --name "$RUNNER_NAME" \ - --labels "$RUNNER_LABEL" - chmod 0600 "$RUNNER_HOME/.runner" +# Docker's official rootless DinD baseline is --privileged. Gitea's example +# additionally names an AppArmor profile, but Docker does not create that host +# profile. Enable it only when this daemon proves it can start the pinned image +# with the profile; otherwise leave the option empty. +RUNNER_SECURITY_OPTIONS= +if docker run --rm --pull=never --privileged \ + --security-opt apparmor=rootlesskit \ + --entrypoint /bin/true "$RUNNER_IMAGE" >/dev/null 2>&1; then + RUNNER_SECURITY_OPTIONS="--security-opt=apparmor=rootlesskit" +elif [ -r /proc/sys/kernel/apparmor_restrict_unprivileged_userns ] && \ + [ "$(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns)" = "1" ]; then + fail "restricted AppArmor user namespaces require the rootlesskit profile (install it with the host Docker packages)" fi -chown -R "$RUNNER_USER:$RUNNER_USER" "$RUNNER_HOME" -systemctl daemon-reload -systemctl enable --now easyai-gateway-act-runner.service -systemctl --no-pager --full status easyai-gateway-act-runner.service +# Exercise the actual daemon startup before registration. This probe has no +# token, state volume, host bind mount, or socket mount. +PROBE_CONTAINER="easyai-gateway-ci-v2-probe-$$" +cleanup_probe() { + docker rm --force "$PROBE_CONTAINER" >/dev/null 2>&1 || true +} +trap 'cleanup_probe' EXIT +trap 'cleanup_probe; exit 129' HUP +trap 'cleanup_probe; exit 130' INT +trap 'cleanup_probe; exit 143' TERM +if [ -n "$RUNNER_SECURITY_OPTIONS" ]; then + set -- "$RUNNER_SECURITY_OPTIONS" +else + set -- +fi +docker run --detach --rm --pull=never \ + --name "$PROBE_CONTAINER" --privileged --pids-limit 4096 "$@" \ + --env DOCKERD_ROOTLESS_ROOTLESSKIT_NET=slirp4netns \ + --env DOCKERD_ROOTLESS_ROOTLESSKIT_MTU=65520 \ + --env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns \ + --entrypoint dockerd-entrypoint.sh "$RUNNER_IMAGE" >/dev/null +attempt=0 +until docker exec "$PROBE_CONTAINER" docker info >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge 30 ] || \ + [ "$(docker inspect --format '{{.State.Running}}' "$PROBE_CONTAINER" 2>/dev/null || true)" != "true" ]; then + docker logs "$PROBE_CONTAINER" 2>&1 || true + fail "pinned rootless Docker-in-Docker probe did not become ready" + fi + sleep 2 +done +[ "$(docker exec "$PROBE_CONTAINER" id -u)" = "1000" ] || \ + fail "rootless Docker-in-Docker probe is not uid 1000" +docker exec "$PROBE_CONTAINER" docker info --format '{{range .SecurityOptions}}{{println .}}{{end}}' | \ + grep -Fq 'name=rootless' || fail "rootless Docker-in-Docker probe is not rootless" +cleanup_probe +trap - EXIT HUP INT TERM -env PATH="$PREFIX/bin:$PREFIX/toolchains/go/bin:$PREFIX/toolchains/node/bin:$PATH" \ - sh -c 'gitea-runner --version && go version && node --version && pnpm --version && trivy --version && govulncheck -version' +install -d -m 0755 /etc/easyai-gateway-ci-v2 +install -m 0644 "$ROOT/deploy/ci/act-runner-v2-config.yaml" /etc/easyai-gateway-ci-v2/config.yaml +umask 077 +printf 'RUNNER_SECURITY_OPTIONS="%s"\n' "$RUNNER_SECURITY_OPTIONS" \ + > /etc/easyai-gateway-ci-v2/runner.env +install -m 0644 "$ROOT/deploy/ci/easyai-gateway-ci-v2-runner.service" \ + /etc/systemd/system/easyai-gateway-ci-v2-runner.service + +# The image runs as uid 1000. Initialize only the dedicated named volume as +# root, then register and operate the runner as that image's rootless user. +docker run --rm --pull=never --user 0:0 \ + --volume "$RUNNER_VOLUME:/data" \ + --entrypoint /bin/sh "$RUNNER_IMAGE" \ + -ec 'chown 1000:1000 /data && chmod 0700 /data' + +runner_is_registered() { + docker run --rm --pull=never \ + --volume "$RUNNER_VOLUME:/data" \ + --entrypoint /bin/sh "$RUNNER_IMAGE" \ + -ec 'test -s /data/.runner' +} + +if ! runner_is_registered; then + [ -n "${RUNNER_REGISTRATION_TOKEN:-}" ] || \ + fail "RUNNER_REGISTRATION_TOKEN is required for first registration" + docker rm --force easyai-gateway-ci-v2-register >/dev/null 2>&1 || true + printf '%s\n%s\n%s\n' "$GITEA_INSTANCE_URL" "$RUNNER_REGISTRATION_TOKEN" "$RUNNER_NAME" | \ + docker run --rm --interactive --pull=never \ + --name easyai-gateway-ci-v2-register \ + --volume "$RUNNER_VOLUME:/data" \ + --volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro \ + --env HOME=/data \ + --entrypoint /usr/local/bin/gitea-runner \ + "$RUNNER_IMAGE" --config /config.yaml register +fi +unset RUNNER_REGISTRATION_TOKEN +docker run --rm --pull=never \ + --volume "$RUNNER_VOLUME:/data" \ + --entrypoint /bin/sh "$RUNNER_IMAGE" \ + -ec 'test -s /data/.runner && chmod 0600 /data/.runner' + +RUNNER_VALIDATED=0 +cleanup_unvalidated_runner() { + if [ "$RUNNER_VALIDATED" -ne 1 ]; then + systemctl disable --now easyai-gateway-ci-v2-runner.service >/dev/null 2>&1 || true + docker rm --force "$RUNNER_CONTAINER" >/dev/null 2>&1 || true + fi +} +trap 'cleanup_unvalidated_runner' EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +systemctl daemon-reload +systemctl enable --now easyai-gateway-ci-v2-runner.service + +attempt=0 +until docker exec "$RUNNER_CONTAINER" docker info >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge 30 ]; then + systemctl --no-pager --full status easyai-gateway-ci-v2-runner.service || true + docker logs "$RUNNER_CONTAINER" 2>&1 || true + fail "rootless Docker daemon did not become ready" + fi + sleep 2 +done + +attempt=0 +until docker logs "$RUNNER_CONTAINER" 2>&1 | grep -Fq 'declare successfully'; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge 30 ] || \ + [ "$(docker inspect --format '{{.State.Running}}' "$RUNNER_CONTAINER" 2>/dev/null || true)" != "true" ]; then + docker logs "$RUNNER_CONTAINER" 2>&1 || true + fail "Gitea runner did not declare itself successfully" + fi + sleep 2 +done + +[ "$(docker inspect --format '{{.Config.User}}' "$RUNNER_CONTAINER")" = "rootless" ] || \ + fail "outer runner container is not using the rootless image user" +[ "$(docker inspect --format '{{.HostConfig.Privileged}}' "$RUNNER_CONTAINER")" = "true" ] || \ + fail "outer rootless DinD container is not privileged" +if docker inspect --format '{{range .Mounts}}{{println .Source}}{{end}}' "$RUNNER_CONTAINER" | \ + grep -Eq '/var/run/docker\.sock|/run/docker\.sock'; then + fail "host Docker socket was mounted into the runner" +fi +docker exec "$RUNNER_CONTAINER" docker info --format '{{range .SecurityOptions}}{{println .}}{{end}}' | \ + grep -Fq 'name=rootless' || fail "nested Docker daemon is not rootless" + +# Pre-pull through the nested daemon to prove the digest is reachable and make +# the first real CI job independent of an additional large image pull. +docker exec "$RUNNER_CONTAINER" docker pull "$JOB_IMAGE" +outer_pidns=$(docker exec "$RUNNER_CONTAINER" readlink /proc/1/ns/pid) +inner_host_pidns=$(docker exec "$RUNNER_CONTAINER" docker run --rm --pull=never \ + --pid=host --entrypoint readlink "$JOB_IMAGE" /proc/1/ns/pid) +[ "$outer_pidns" != "$inner_host_pidns" ] || \ + fail "nested --pid=host can see the outer runner PID namespace" +docker exec "$RUNNER_CONTAINER" docker run --rm --pull=never \ + --volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro \ + --entrypoint /bin/sh "$JOB_IMAGE" -ec ' + export PATH=/opt/easyai-gateway-ci/bin:/opt/easyai-gateway-ci/toolchains/go/bin:/opt/easyai-gateway-ci/toolchains/node/bin:$PATH + go version + node --version + pnpm --version + docker-compose version + shellcheck --version + trivy --version + govulncheck -version + ' +if ! require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed CI runner installation"; then + fail "CI runner was stopped because the post-install disk reserve was not met" +fi +systemctl --no-pager --full status easyai-gateway-ci-v2-runner.service +RUNNER_VALIDATED=1 +trap - EXIT HUP INT TERM diff --git a/tests/ci/ci-build-images-test.sh b/tests/ci/ci-build-images-test.sh new file mode 100755 index 0000000..ed1d781 --- /dev/null +++ b/tests/ci/ci-build-images-test.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +script=$root/scripts/ci-build-images.sh +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +mkdir -p "$tmp/bin" +cat >"$tmp/bin/docker" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail +printf 'docker %s\n' "$*" >>"$MOCK_LOG" + +case "$*" in + 'buildx inspect '*|'buildx create '*) exit 0 ;; + 'buildx build '*) + if [[ ${MOCK_BUILD_FAIL:-0} == 1 && $* == *'--target web'* ]]; then + exit 21 + fi + ;; + 'buildx prune '*) + [[ ${MOCK_PRUNE_FAIL:-0} == 0 ]] || exit 22 + ;; + 'image rm '*) + [[ ${MOCK_IMAGE_RM_FAIL:-0} == 0 ]] || exit 23 + ;; + *) + printf 'unexpected docker invocation: %s\n' "$*" >&2 + exit 99 + ;; +esac +MOCK +cat >"$tmp/bin/trivy" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail +printf 'trivy %s\n' "$*" >>"$MOCK_LOG" +MOCK +chmod +x "$tmp/bin/docker" "$tmp/bin/trivy" + +sha=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +registry=registry.example.com/easyai +api_image=$registry/ai-gateway:$sha +web_image=$registry/ai-gateway-web:$sha + +run_build() { + env \ + PATH="$tmp/bin:$PATH" \ + MOCK_LOG="$tmp/calls.log" \ + IMAGE_TAG="$sha" \ + AI_GATEWAY_IMAGE_REGISTRY="$registry" \ + AI_GATEWAY_BUILDX_BUILDER=gateway-test \ + AI_GATEWAY_BUILDX_CACHE_LIMIT=2gb \ + "$script" "$@" +} + +: >"$tmp/calls.log" +run_build +grep -Fq "docker buildx prune --builder gateway-test --force --max-used-space 2gb" "$tmp/calls.log" +grep -Fq "docker image rm $api_image $web_image" "$tmp/calls.log" + +: >"$tmp/calls.log" +run_build --keep-images +grep -Fq "docker buildx prune --builder gateway-test --force --max-used-space 2gb" "$tmp/calls.log" +if grep -Fq 'docker image rm ' "$tmp/calls.log"; then + echo '--keep-images must preserve fully built images' >&2 + exit 1 +fi + +: >"$tmp/calls.log" +if MOCK_PRUNE_FAIL=1 run_build; then + echo 'a prune failure must fail an otherwise successful build' >&2 + exit 1 +fi + +: >"$tmp/calls.log" +if MOCK_IMAGE_RM_FAIL=1 run_build; then + echo 'an image cleanup failure must fail an otherwise successful build' >&2 + exit 1 +fi + +: >"$tmp/calls.log" +if MOCK_BUILD_FAIL=1 run_build; then + echo 'a failed image build must fail the script' >&2 + exit 1 +fi +grep -Fq "docker image rm $api_image $web_image" "$tmp/calls.log" + +for invalid_limit in 0gb 2 2GB -1gb '2gb --all'; do + : >"$tmp/calls.log" + if env \ + PATH="$tmp/bin:$PATH" \ + MOCK_LOG="$tmp/calls.log" \ + IMAGE_TAG="$sha" \ + AI_GATEWAY_IMAGE_REGISTRY="$registry" \ + AI_GATEWAY_BUILDX_CACHE_LIMIT="$invalid_limit" \ + "$script" >/dev/null 2>&1; then + printf 'invalid cache limit was accepted: %s\n' "$invalid_limit" >&2 + exit 1 + fi + if [[ -s $tmp/calls.log ]]; then + printf 'invalid cache limit reached Docker: %s\n' "$invalid_limit" >&2 + exit 1 + fi +done + +echo 'ci_build_images_tests=PASS' diff --git a/tests/ci/pipeline-test.sh b/tests/ci/pipeline-test.sh index 3abc430..e713d45 100755 --- a/tests/ci/pipeline-test.sh +++ b/tests/ci/pipeline-test.sh @@ -4,12 +4,17 @@ set -euo pipefail root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) workflow=$root/.gitea/workflows/ci.yml -runner_service=$root/deploy/ci/easyai-gateway-act-runner.service -runner_config=$root/deploy/ci/act-runner-config.yaml +release_workflow=$root/.gitea/workflows/release-ci.yml +runner_service=$root/deploy/ci/easyai-gateway-ci-v2-runner.service +runner_config=$root/deploy/ci/act-runner-v2-config.yaml provision=$root/scripts/provision-ci-runner.sh build_images=$root/scripts/ci-build-images.sh +validate_semver=$root/scripts/ci-validate-semver.sh +build_images_test=$root/tests/ci/ci-build-images-test.sh +adr=$root/docs/decisions/001-production-cicd.md +runbook=$root/docs/runbooks/production-ci-cd.md -for file in "$workflow" "$runner_service" "$runner_config" "$provision" "$build_images"; do +for file in "$workflow" "$release_workflow" "$runner_service" "$runner_config" "$provision" "$build_images" "$validate_semver" "$build_images_test" "$adr" "$runbook"; do [[ -f $file ]] || { printf 'missing CI/CD file: %s\n' "$file" >&2 exit 1 @@ -17,59 +22,229 @@ for file in "$workflow" "$runner_service" "$runner_config" "$provision" "$build_ done grep -q '^name: ci$' "$workflow" +grep -q '^ push:$' "$workflow" grep -q '^ branches: \[main\]$' "$workflow" -grep -q "^ tags: \['v\*'\]$" "$workflow" grep -q '^ pull_request:$' "$workflow" -grep -q '^ runs-on: easyai-gateway-linux$' "$workflow" -grep -q '^ - name: Checkout without external Actions$' "$workflow" -grep -q '^ - name: Verify release tag ancestry$' "$workflow" -grep -Fq "if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')" "$workflow" -grep -Fq 'git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main' "$workflow" -grep -q '^ - run: ./scripts/ci-build-images.sh$' "$workflow" -grep -q '^ - name: Deploy verified release to Production$' "$workflow" -grep -q '^ sudo -n /usr/local/sbin/easyai-ai-gateway-release "$CI_SHA"$' "$workflow" +grep -q '^ verify:$' "$workflow" +if grep -Eq "^[[:space:]]+tags:|Verify release tag ancestry" "$workflow" || \ + grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$workflow"; then + echo 'main/PR CI must not share the release-tag context' >&2 + exit 1 +fi -for gate in \ - 'gofmt -l' \ - 'go vet ./...' \ - 'go test ./...' \ - 'pnpm install --frozen-lockfile' \ - 'pnpm lint' \ - 'pnpm test' \ - 'pnpm build' \ - 'docker compose -f docker-compose.yml config --quiet' \ - 'govulncheck ./...' \ - 'trivy fs'; do - grep -Fq "$gate" "$workflow" || { - printf 'workflow is missing quality gate: %s\n' "$gate" >&2 +grep -q '^name: release-ci$' "$release_workflow" +grep -q '^ push:$' "$release_workflow" +grep -q "^ tags: \['v\*'\]$" "$release_workflow" +grep -q '^ verify-tag:$' "$release_workflow" +grep -q '^ - name: Verify release tag ancestry$' "$release_workflow" +grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$release_workflow" +grep -Fq 'git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main' "$release_workflow" +if grep -Eq 'branches:|pull_request:' "$release_workflow"; then + echo 'release CI must be a tag-only workflow/context' >&2 + exit 1 +fi + +for quality_workflow in "$workflow" "$release_workflow"; do + grep -q '^ runs-on: easyai-gateway-ci-unprivileged-v2$' "$quality_workflow" + grep -q '^ - name: Checkout without external Actions$' "$quality_workflow" + if grep -q -- '--depth=' "$quality_workflow"; then + echo 'quality verification must not use a shallow checkout' >&2 exit 1 - } + fi + grep -Fq 'test ! -f .git/shallow' "$quality_workflow" + grep -Fq './tests/ci/ci-build-images-test.sh' "$quality_workflow" + grep -Fq './tests/ci/pipeline-test.sh' "$quality_workflow" + grep -Fq './tests/ci/semver-test.sh' "$quality_workflow" + grep -Fq 'docker-compose version' "$quality_workflow" + grep -Fq 'docker-compose -f docker-compose.yml config --quiet' "$quality_workflow" + if grep -Fq 'docker compose' "$quality_workflow"; then + echo 'job container must use the read-only standalone Compose binary' >&2 + exit 1 + fi + + for forbidden in \ + 'sudo' \ + '/usr/local/sbin/easyai-ai-gateway-release' \ + 'docker version' \ + 'docker buildx' \ + './scripts/ci-build-images.sh --keep-images' \ + 'Deploy verified release to Production'; do + if grep -Fq "$forbidden" "$quality_workflow"; then + printf 'unprivileged workflow contains forbidden capability: %s\n' "$forbidden" >&2 + exit 1 + fi + done + + if grep -Eq '^[[:space:]]+\./scripts/ci-build-images\.sh([[:space:]]|$)' "$quality_workflow"; then + echo 'unprivileged workflow must not build container images' >&2 + exit 1 + fi + + for gate in \ + 'gofmt -l' \ + 'go vet ./...' \ + 'go test ./...' \ + 'pnpm install --frozen-lockfile' \ + 'pnpm lint' \ + 'pnpm test' \ + 'pnpm build' \ + 'pnpm audit --audit-level high' \ + 'docker-compose -f docker-compose.yml config --quiet' \ + 'govulncheck ./...' \ + 'trivy fs'; do + grep -Fq "$gate" "$quality_workflow" || { + printf '%s is missing quality gate: %s\n' "$quality_workflow" "$gate" >&2 + exit 1 + } + done done -grep -q '^User=easyai-gateway-runner$' "$runner_service" -grep -q '^Group=easyai-gateway-runner$' "$runner_service" -grep -q '^NoNewPrivileges=false$' "$runner_service" +if [[ $(sed -n '/^ - name: Verify Go formatting$/,$p' "$workflow") != \ + "$(sed -n '/^ - name: Verify Go formatting$/,$p' "$release_workflow")" ]]; then + echo 'main/PR and release-tag workflows must have identical quality gates' >&2 + exit 1 +fi + +runner_image='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a' +job_image='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7' + +grep -q '^User=root$' "$runner_service" +grep -q '^Group=root$' "$runner_service" +grep -q '^NoNewPrivileges=true$' "$runner_service" grep -q '^ProtectSystem=strict$' "$runner_service" grep -q '^ProtectHome=true$' "$runner_service" -grep -q '^ReadWritePaths=/var/lib/easyai-gateway-runner$' "$runner_service" -grep -q '^ExecStart=/opt/easyai-gateway-ci/bin/gitea-runner --config /etc/easyai-gateway-runner/config.yaml daemon --once$' "$runner_service" -grep -q 'easyai-gateway-linux:host' "$runner_config" +grep -q '^CapabilityBoundingSet=$' "$runner_service" +grep -Fq 'Requires=docker.service' "$runner_service" +grep -Fq -- "--pull=never" "$runner_service" +grep -Fq -- "--privileged" "$runner_service" +grep -Fq -- '--pids-limit 4096' "$runner_service" +grep -Fq -- '--env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns' "$runner_service" +grep -Fq 'EnvironmentFile=-/etc/easyai-gateway-ci-v2/runner.env' "$runner_service" +grep -Fq '$RUNNER_SECURITY_OPTIONS' "$runner_service" +grep -Fq -- "--volume easyai-gateway-ci-v2-data:/data" "$runner_service" +grep -Fq -- "--volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro" "$runner_service" +grep -Fq -- "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro" "$runner_service" +grep -Fq "$runner_image" "$runner_service" +if grep -Eq 'GITEA_RUNNER_REGISTRATION_TOKEN|/var/run/docker\.sock|/run/docker\.sock' "$runner_service"; then + echo 'persistent runner service exposes a registration token or host Docker socket' >&2 + exit 1 +fi +if grep -Fq 'apparmor=rootlesskit' "$runner_service"; then + echo 'runner service must not hard-code an AppArmor profile that may be absent' >&2 + exit 1 +fi + +grep -Fq "easyai-gateway-ci-unprivileged-v2:docker://$job_image" "$runner_config" +if grep -Fq ':host' "$runner_config"; then + echo 'CI runner must never use the host executor' >&2 + exit 1 +fi grep -q '^ capacity: 1$' "$runner_config" +grep -q '^ file: /data/.runner$' "$runner_config" +grep -q '^ docker_host: "-"$' "$runner_config" +grep -q '^ privileged: false$' "$runner_config" +grep -q '^ require_docker: true$' "$runner_config" +grep -q '^ bind_workdir: false$' "$runner_config" +grep -Fq ' options: "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro"' "$runner_config" +grep -q '^ valid_volumes:$' "$runner_config" +grep -q '^ - "/opt/easyai-gateway-ci"$' "$runner_config" +if awk ' + /^ valid_volumes:$/ { in_volumes=1; next } + in_volumes && /^ - / { count++; next } + in_volumes && /^ [A-Za-z_]+:/ { in_volumes=0 } + END { exit count == 1 ? 0 : 1 } +' "$runner_config"; then + : +else + echo 'runner must allow exactly one workflow volume source' >&2 + exit 1 +fi +if grep -Eq '/var/run/docker\.sock|/run/docker\.sock' "$runner_config"; then + echo 'job configuration must not expose a Docker socket' >&2 + exit 1 +fi +if grep -Eq 'apparmor=unconfined|sysctl .*(apparmor_restrict_unprivileged_userns)' "$runner_service" "$runner_config" "$provision"; then + echo 'runner must not disable AppArmor isolation or its user-namespace restriction' >&2 + exit 1 +fi grep -q '^GITEA_RUNNER_VERSION=2.0.0$' "$provision" grep -q '^GO_VERSION=1.26.5$' "$provision" grep -q '^NODE_VERSION=24.16.0$' "$provision" grep -q '^TRIVY_VERSION=0.70.0$' "$provision" -grep -Fq 'usermod -aG docker "$RUNNER_USER"' "$provision" +grep -q '^PREFIX=/opt/easyai-gateway-ci$' "$provision" +grep -q '^RUNNER_VOLUME=easyai-gateway-ci-v2-data$' "$provision" +grep -q '^SHELLCHECK_VERSION=0.11.0$' "$provision" +grep -q '^SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198$' "$provision" +grep -q '^COMPOSE_VERSION=5.3.1$' "$provision" +grep -q '^COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959$' "$provision" +grep -Fq 'RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-ci-unprivileged-v2}' "$provision" +grep -Fq "RUNNER_IMAGE='$runner_image'" "$provision" +grep -Fq "JOB_IMAGE='$job_image'" "$provision" +grep -Fq 'RUNNER_LABEL="easyai-gateway-ci-unprivileged-v2:docker://$JOB_IMAGE"' "$provision" +grep -Fq 'docker volume create "$RUNNER_VOLUME"' "$provision" +grep -Fq -- '--security-opt apparmor=rootlesskit' "$provision" +grep -Fq 'RUNNER_SECURITY_OPTIONS="--security-opt=apparmor=rootlesskit"' "$provision" +grep -Fq 'RUNNER_SECURITY_OPTIONS=' "$provision" +grep -Fq '/proc/sys/kernel/apparmor_restrict_unprivileged_userns' "$provision" +grep -Fq -- '--entrypoint dockerd-entrypoint.sh "$RUNNER_IMAGE"' "$provision" +grep -Fq 'PROBE_CONTAINER="easyai-gateway-ci-v2-probe-$$"' "$provision" +grep -Fq -- '--entrypoint /usr/local/bin/gitea-runner' "$provision" +grep -Fq 'shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz' "$provision" +grep -Fq 'docker-compose-linux-x86_64' "$provision" +grep -Fq 'install -m 0755 "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "$PREFIX/bin/shellcheck"' "$provision" +grep -Fq 'install -m 0755 "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" "$PREFIX/bin/docker-compose"' "$provision" +grep -Fq 'gpasswd -d easyai-gateway-ci-v2 docker' "$provision" +grep -Fq 'gpasswd -d easyai-gateway-ci-v2 sudo' "$provision" +if grep -Fq 'usermod -aG docker' "$provision"; then + echo 'unprivileged runner must never join the Docker group' >&2 + exit 1 +fi +grep -Fq 'systemctl disable --now easyai-gateway-act-runner.service' "$provision" +grep -Fq 'systemctl disable --now easyai-gateway-ci-v2-runner.service' "$provision" grep -Fq 'RUNNER_REGISTRATION_TOKEN is required for first registration' "$provision" +grep -Fq 'printf '\''%s\n%s\n%s\n'\'' "$GITEA_INSTANCE_URL" "$RUNNER_REGISTRATION_TOKEN" "$RUNNER_NAME"' "$provision" +if grep -Fq -- '--token "$RUNNER_REGISTRATION_TOKEN"' "$provision"; then + echo 'registration token must not be exposed in a host-visible process argv' >&2 + exit 1 +fi +grep -Fq "grep -Fq 'name=rootless'" "$provision" +grep -Fq "grep -Fq 'declare successfully'" "$provision" +grep -Fq 'docker buildx prune --builder "$LEGACY_BUILDER" --force --max-used-space 512mb' "$provision" +grep -Fq 'require_free_space "$MIN_FREE_GIB" "runner image pull"' "$provision" +grep -Fq 'require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed CI runner installation"' "$provision" +grep -Fq 'CI_RUNNER_MIN_FREE_GIB cannot be lower than 8' "$provision" +grep -Fq 'CI_RUNNER_MIN_POST_INSTALL_FREE_GIB cannot be lower than 4' "$provision" +grep -Fq 'docker exec "$RUNNER_CONTAINER" docker pull "$JOB_IMAGE"' "$provision" +grep -Fq 'docker exec "$RUNNER_CONTAINER" docker run --rm --pull=never' "$provision" +grep -Fq 'nested --pid=host can see the outer runner PID namespace' "$provision" +grep -Fq 'chown -R root:root "$PREFIX"' "$provision" +grep -Fq 'chmod -R go-w "$PREFIX"' "$provision" +if grep -Eq 'SHELLCHECK_BIN|COMPOSE_PLUGIN|command -v shellcheck|docker compose version' "$provision"; then + echo 'runner provisioning must use checksum-pinned static CI tools, not host copies' >&2 + exit 1 +fi +if grep -Eq 'docker (system|builder|image|volume) prune' "$provision"; then + echo 'runner provisioning must not globally prune shared Docker state' >&2 + exit 1 +fi + +grep -Fq 'root-owned dispatcher' "$adr" +grep -Fq 'release-ci / verify-tag (push)' "$adr" +grep -Fq '来自 Fork 的 PR 必须先由维护者检查工作流变更并在 Gitea 中批准' "$adr" +grep -Fq 'rootless DinD 不是 VM 沙箱' "$runbook" +grep -Fq '同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)`' "$runbook" +grep -Fq '不得执行 Tag checkout 中的' "$runbook" grep -Fq 'docker buildx build --builder "$builder"' "$build_images" grep -q -- '--target api' "$build_images" grep -q -- '--target web' "$build_images" grep -Fq 'docker buildx prune --builder "$builder" --force --max-used-space "$cache_limit"' "$build_images" grep -Fq 'trivy image' "$build_images" +grep -Fq 'docker image rm "$api_image" "$web_image"' "$build_images" +grep -Fq 'AI_GATEWAY_BUILDX_CACHE_LIMIT has an invalid format' "$build_images" -if rg -n '(password|secret|token):[[:space:]]+[^$<{]' "$workflow"; then +if grep -En '(password|secret|token):[[:space:]]+[^$<{]' "$workflow" "$release_workflow"; then echo 'workflow contains a literal credential' >&2 exit 1 fi diff --git a/tests/ci/semver-test.sh b/tests/ci/semver-test.sh new file mode 100755 index 0000000..fde3ed3 --- /dev/null +++ b/tests/ci/semver-test.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +validator=$root/scripts/ci-validate-semver.sh + +valid_tags=( + v0.0.0 + v1.2.3 + v1.2.3-alpha + v1.2.3-alpha.1 + v1.2.3-0.3.7 + v1.2.3-01a + v1.2.3-alpha-beta + v1.2.3-- + v1.2.3-x.7.z.92+build.5 + v1.2.3+build.11.e0f985a + v1.2.3+001 +) +invalid_tags=( + 1.2.3 + v01.2.3 + v1.02.3 + v1.2.03 + v1.2.3.foo + v1.2.3. + v1.2.3-. + v1.2.3-alpha. + v1.2.3-alpha..1 + v1.2.3-01 + v1.2.3-00.1 + v1.2.3+ + v1.2.3+build. + v1.2.3+build..1 + v1.2.3+build+1 + 'v1.2.3 ' + v1.2.3-alpha_1 + v1.2 +) + +for tag in "${valid_tags[@]}"; do + "$validator" "$tag" +done + +for tag in "${invalid_tags[@]}"; do + if "$validator" "$tag" >/dev/null 2>&1; then + printf 'invalid SemVer tag was accepted: %s\n' "$tag" >&2 + exit 1 + fi +done + +echo 'semver_tests=PASS'