From dae5d16a5872884d6d1ef187d2a9011576230379 Mon Sep 17 00:00:00 2001 From: easyai Date: Wed, 22 Jul 2026 15:13:40 +0800 Subject: [PATCH] =?UTF-8?q?refactor(release):=20=E6=94=B9=E4=B8=BA=20Agent?= =?UTF-8?q?=20=E5=8F=8C=E9=98=B6=E6=AE=B5=E4=BA=BA=E5=B7=A5=E5=8F=91?= =?UTF-8?q?=E5=B8=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 删除 Gitea Actions、Tag/Main 自动流水线和旧 Runner 配置，取消 Git 操作与发布授权的绑定。\n\n新增本地镜像发布、固定生产部署助手、digest manifest、迁移安全检查、simulation 冒烟及显式回滚流程。\n\n验证：pnpm lint、pnpm test、pnpm build、Go 全量测试、ShellCheck、Compose 配置、人工发布测试和 linux/amd64 完整栈冒烟。 --- .gitea/workflows/ci.yml | 129 ------ .gitea/workflows/release-ci.yml | 114 ----- AGENTS.md | 46 +- Dockerfile | 5 +- README.md | 49 ++- deploy/ci/act-runner-v2-config.yaml | 36 -- deploy/ci/easyai-gateway-ci-v2-runner.service | 44 -- deploy/ci/production-migration-base | 1 - deploy/manual/easyai-ai-gateway-release | 233 ++++++++++ .../easyai-ai-gateway-release.conf.example | 12 + deploy/manual/install-release-helper.sh | 18 + docker-compose.yml | 1 + docs/decisions/001-production-cicd.md | 2 +- docs/decisions/003-manual-agent-release.md | 31 ++ docs/runbooks/production-ci-cd.md | 233 +++++----- package.json | 5 +- scripts/api-release-smoke.mjs | 145 +++++++ scripts/ci-build-images.sh | 101 ----- scripts/ci-validate-migrations.mjs | 126 +----- scripts/ci-validate-semver.sh | 22 - scripts/deploy-compose.sh | 104 +---- scripts/deploy-production-release.sh | 100 +++++ scripts/provision-ci-runner.sh | 402 ------------------ scripts/publish-release-images.sh | 375 ++++++++++++++++ scripts/release-components.mjs | 157 +++++++ scripts/release-manifest.mjs | 227 ++++++++++ scripts/release-preflight.mjs | 42 ++ tests/ci/ci-build-images-test.sh | 109 ----- tests/ci/migrations-test.sh | 263 ++---------- tests/ci/pipeline-test.sh | 365 ---------------- tests/ci/semver-test.sh | 52 --- tests/release/manual-release-test.sh | 261 ++++++++++++ 32 files changed, 1820 insertions(+), 1990 deletions(-) delete mode 100644 .gitea/workflows/ci.yml delete mode 100644 .gitea/workflows/release-ci.yml delete mode 100644 deploy/ci/act-runner-v2-config.yaml delete mode 100644 deploy/ci/easyai-gateway-ci-v2-runner.service delete mode 100644 deploy/ci/production-migration-base create mode 100755 deploy/manual/easyai-ai-gateway-release create mode 100644 deploy/manual/easyai-ai-gateway-release.conf.example create mode 100755 deploy/manual/install-release-helper.sh create mode 100644 docs/decisions/003-manual-agent-release.md create mode 100755 scripts/api-release-smoke.mjs delete mode 100755 scripts/ci-build-images.sh delete mode 100755 scripts/ci-validate-semver.sh create mode 100755 scripts/deploy-production-release.sh delete mode 100755 scripts/provision-ci-runner.sh create mode 100755 scripts/publish-release-images.sh create mode 100755 scripts/release-components.mjs create mode 100755 scripts/release-manifest.mjs create mode 100755 scripts/release-preflight.mjs delete mode 100755 tests/ci/ci-build-images-test.sh delete mode 100755 tests/ci/pipeline-test.sh delete mode 100755 tests/ci/semver-test.sh create mode 100755 tests/release/manual-release-test.sh diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml deleted file mode 100644 index 58f1805..0000000 --- a/.gitea/workflows/ci.yml +++ /dev/null @@ -1,129 +0,0 @@ -name: ci - -on: - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - verify: - runs-on: easyai-gateway-ci-unprivileged-v2 - services: - postgres: - image: docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 - env: - POSTGRES_USER: easyai_test - POSTGRES_HOST_AUTH_METHOD: trust - POSTGRES_DB: easyai_gateway_test - options: >- - --health-cmd "pg_isready -U easyai_test -d easyai_gateway_test" - --health-interval 2s - --health-timeout 5s - --health-retries 30 - env: - TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2 - AI_GATEWAY_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable - AI_GATEWAY_TEST_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable - steps: - - name: Checkout without external Actions - env: - CI_REPOSITORY: ${{ github.repository }} - CI_SERVER_URL: ${{ github.server_url }} - CI_SHA: ${{ github.sha }} - CI_JOB_TOKEN: ${{ github.token }} - CI_EVENT_BEFORE: ${{ github.event.before }} - CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - run: | - set -eu - test -n "$CI_JOB_TOKEN" - authorization=$(printf 'x-access-token:%s' "$CI_JOB_TOKEN" | base64 | tr -d '\n') - git init . - git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \ - fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" "$CI_SHA" - for comparison_sha in "$CI_EVENT_BEFORE" "$CI_PR_BASE_SHA"; do - case "$comparison_sha" in - [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*) - if test "${#comparison_sha}" -eq 40 && \ - test "$comparison_sha" != 0000000000000000000000000000000000000000; then - git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \ - fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" "$comparison_sha" - fi - ;; - esac - done - unset authorization CI_JOB_TOKEN - test ! -f .git/shallow - git checkout --detach "$CI_SHA" - test "$(git rev-parse HEAD)" = "$CI_SHA" - - name: Verify pinned host toolchains - run: | - go version - node --version - pnpm --version - docker-compose version - shellcheck --version - trivy --version - govulncheck -version - - name: Verify production migration safety - env: - CI_EVENT_NAME: ${{ github.event_name }} - CI_EVENT_BEFORE: ${{ github.event.before }} - CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - run: | - production_base=$(cat deploy/ci/production-migration-base) - immutable_base=$CI_EVENT_BEFORE - if test "$CI_EVENT_NAME" = pull_request; then - immutable_base=$CI_PR_BASE_SHA - fi - case "$immutable_base" in - [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*) - test "${#immutable_base}" -eq 40 - test "$immutable_base" != 0000000000000000000000000000000000000000 - ;; - *) exit 1 ;; - esac - git merge-base --is-ancestor "$immutable_base" HEAD - node ./scripts/ci-validate-migrations.mjs \ - "$production_base" "$immutable_base" - - name: Verify Go formatting - run: | - unformatted=$(gofmt -l apps/api) - test -z "$unformatted" || { - printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2 - exit 1 - } - - name: Migrate PostgreSQL 16 integration database - working-directory: apps/api - run: go run ./cmd/migrate - - name: Verify Go code - working-directory: apps/api - env: - GOFLAGS: "-p=1" - GOMAXPROCS: "1" - run: | - go vet ./... - go test ./... - govulncheck ./... - - run: pnpm install --frozen-lockfile - - run: pnpm lint - - run: pnpm test - - run: pnpm build - - name: Audit JavaScript dependencies - run: pnpm audit --audit-level high - - name: Validate deployment configuration - run: | - docker-compose -f docker-compose.yml config --quiet - shellcheck scripts/ci-build-images.sh scripts/ci-validate-semver.sh \ - scripts/provision-ci-runner.sh tests/ci/ci-build-images-test.sh \ - tests/ci/migrations-test.sh tests/ci/pipeline-test.sh \ - tests/ci/semver-test.sh - ./tests/ci/ci-build-images-test.sh - ./tests/ci/migrations-test.sh - ./tests/ci/pipeline-test.sh - ./tests/ci/semver-test.sh - - name: Scan repository - run: | - trivy fs --scanners vuln,secret,misconfig --severity HIGH,CRITICAL \ - --ignore-unfixed --exit-code 1 --timeout 15m --skip-dirs .git \ - --skip-dirs node_modules . diff --git a/.gitea/workflows/release-ci.yml b/.gitea/workflows/release-ci.yml deleted file mode 100644 index 7142922..0000000 --- a/.gitea/workflows/release-ci.yml +++ /dev/null @@ -1,114 +0,0 @@ -name: release-ci - -on: - push: - tags: ['v*'] - -jobs: - verify-tag: - runs-on: easyai-gateway-ci-unprivileged-v2 - services: - postgres: - image: docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 - env: - POSTGRES_USER: easyai_test - POSTGRES_HOST_AUTH_METHOD: trust - POSTGRES_DB: easyai_gateway_test - options: >- - --health-cmd "pg_isready -U easyai_test -d easyai_gateway_test" - --health-interval 2s - --health-timeout 5s - --health-retries 30 - env: - TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2 - AI_GATEWAY_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable - AI_GATEWAY_TEST_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable - steps: - - name: Checkout without external Actions - env: - CI_REPOSITORY: ${{ github.repository }} - CI_SERVER_URL: ${{ github.server_url }} - CI_SHA: ${{ github.sha }} - CI_JOB_TOKEN: ${{ github.token }} - run: | - set -eu - test -n "$CI_JOB_TOKEN" - authorization=$(printf 'x-access-token:%s' "$CI_JOB_TOKEN" | base64 | tr -d '\n') - git init . - git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \ - fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" "$CI_SHA" - unset authorization CI_JOB_TOKEN - test ! -f .git/shallow - git checkout --detach FETCH_HEAD - - name: Verify pinned host toolchains - run: | - go version - node --version - pnpm --version - docker-compose version - shellcheck --version - trivy --version - govulncheck -version - - name: Verify release tag ancestry - env: - CI_REPOSITORY: ${{ github.repository }} - CI_SERVER_URL: ${{ github.server_url }} - CI_SHA: ${{ github.sha }} - CI_JOB_TOKEN: ${{ github.token }} - run: | - set -eu - tag_name=${GITHUB_REF#refs/tags/} - ./scripts/ci-validate-semver.sh "$tag_name" - authorization=$(printf 'x-access-token:%s' "$CI_JOB_TOKEN" | base64 | tr -d '\n') - git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \ - fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" \ - +refs/heads/main:refs/remotes/origin/main - unset authorization CI_JOB_TOKEN - test ! -f .git/shallow - test "$(git rev-parse HEAD)" = "$CI_SHA" - git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main - - name: Verify production migration safety - run: | - production_base=$(cat deploy/ci/production-migration-base) - node ./scripts/ci-validate-migrations.mjs "$production_base" - - name: Verify Go formatting - run: | - unformatted=$(gofmt -l apps/api) - test -z "$unformatted" || { - printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2 - exit 1 - } - - name: Migrate PostgreSQL 16 integration database - working-directory: apps/api - run: go run ./cmd/migrate - - name: Verify Go code - working-directory: apps/api - env: - GOFLAGS: "-p=1" - GOMAXPROCS: "1" - run: | - go vet ./... - go test ./... - govulncheck ./... - - run: pnpm install --frozen-lockfile - - run: pnpm lint - - run: pnpm test - - run: pnpm build - - name: Audit JavaScript dependencies - run: pnpm audit --audit-level high - - name: Validate deployment configuration - run: | - docker-compose -f docker-compose.yml config --quiet - shellcheck scripts/ci-build-images.sh scripts/ci-validate-semver.sh \ - scripts/provision-ci-runner.sh tests/ci/ci-build-images-test.sh \ - tests/ci/migrations-test.sh tests/ci/pipeline-test.sh \ - tests/ci/semver-test.sh - ./tests/ci/ci-build-images-test.sh - ./tests/ci/migrations-test.sh - ./tests/ci/pipeline-test.sh - ./tests/ci/semver-test.sh - - name: Scan repository - run: | - trivy fs --scanners vuln,secret,misconfig --severity HIGH,CRITICAL \ - --ignore-unfixed --exit-code 1 --timeout 15m --skip-dirs .git \ - --skip-dirs node_modules . diff --git a/AGENTS.md b/AGENTS.md index 81dd82b..4263cd8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,19 +9,28 @@ 3. `type` 使用小写英文，可选值为 `feat`、`fix`、`docs`、`test`、`refactor`、`perf`、`build`、`ci`、`chore`、`revert`。 4. 提交摘要和正文必须使用中文；专有名词、协议名称、命令、路径、代码标识符和第三方原始错误可保留原文。 5. 摘要应简洁明确，末尾不加句号，不得使用“更新代码”“修复问题”等无法说明意图的模糊描述。 -6. 非简单变更应在提交正文或 PR 描述中用中文说明原因、影响、风险和验证结果。 +6. 非简单变更应在提交正文中用中文说明原因、影响、风险和验证结果。 ## 仓库边界 1. 后端位于 `apps/api`，前端位于 `apps/web`，共享 TypeScript 契约位于 `packages/contracts`。 2. 修改 HTTP 接口、请求或响应类型后，必须执行 `pnpm openapi` 并提交匹配的 OpenAPI 产物。 -3. 数据库迁移只能新增，禁止修改已经进入生产基线的历史迁移；迁移必须通过生产迁移安全检查。 -4. 不得把 `.env`、密码、Secret、Token、授权码、私钥或生产凭据提交到 Git、日志、测试输出或验收证据中。 +3. 数据库迁移只能新增，禁止修改或删除已经进入 Git 历史的迁移；发布前必须相对当前线上 SHA 执行迁移安全检查。 +4. 不得把 `.env`、密码、Secret、Token、授权码、私钥或生产凭据提交到 Git、日志、测试输出、release manifest 或验收证据中。 5. 当前工作区存在用户改动时必须保留，不得覆盖、清理、重置或混入当前任务提交；需要隔离时使用独立分支、克隆或 worktree。 ## 验证要求 -根据改动范围执行最小充分验证；准备合并或发布时执行完整门禁： +根据改动范围执行最小充分验证。修改 Shell 脚本后必须执行 `bash -n` 和 ShellCheck；修改 Go 文件后必须确认 `gofmt -l` 没有输出。 + +本地发布的快速强制门禁由 `scripts/publish-release-images.sh` 固定执行： + +```bash +cd apps/api && env -u AI_GATEWAY_TEST_DATABASE_URL go test ./... -count=1 +node scripts/ci-validate-migrations.mjs <当前线上完整 Git SHA> +``` + +完整测试、审计和镜像扫描不再由 Gitea 自动运行。需要合并大范围重构、准备安全审计或人工要求完整验证时执行： ```bash cd apps/api && go vet ./... && go test ./... && govulncheck ./... @@ -31,18 +40,27 @@ pnpm test pnpm build pnpm audit --audit-level high docker compose -f docker-compose.yml config --quiet -./tests/ci/ci-build-images-test.sh ./tests/ci/migrations-test.sh -./tests/ci/pipeline-test.sh -./tests/ci/semver-test.sh +./tests/release/manual-release-test.sh ``` -修改 Shell 脚本后还必须执行 `bash -n` 和 ShellCheck。修改 Go 文件后必须确认 `gofmt -l` 没有输出。 +## Git 与人工发布 -## Git 与 CI/CD +1. 仓库不使用 Gitea Actions，不存在 Push、PR、Tag、Webhook、轮询或定时触发的自动构建和自动部署。 +2. `main` 不设置受保护分支或 required status；允许 Agent 在获得用户明确授权后直接提交并推送，但 commit 或 push 永远不等于发布授权。 +3. 发布只能使用工作区干净、已经提交并属于 `origin/main` 历史的完整 SHA。镜像 Tag 使用完整 SHA，线上只能使用 Registry digest，禁止使用 `latest`。 +4. 本地镜像发布必须由用户明确要求后单独执行： -1. 一个提交只包含一个逻辑变更，提交前检查暂存差异并确认不含敏感信息。 -2. `main` 只能通过短生命周期分支和 PR 合并，禁止直接推送或强制推送。 -3. PR 必须通过精确的 `ci / verify (pull_request)` 状态后才能合并；合并后还要确认同一 SHA 的 `ci / verify (push)` 成功。 -4. 生产版本仅使用稳定 SemVer `vMAJOR.MINOR.PATCH` 标签，并同时要求 `release-ci / verify-tag (push)` 成功。 -5. 未验证 protected tag、发布账本、数据库备份、健康检查和回滚路径时，不得声称 CI/CD 或生产发布已经完成。 + ```bash + ./scripts/publish-release-images.sh --components auto + ``` + + 该命令只能构建、冒烟、推送镜像和生成 `dist/releases/.json`，不得修改生产。 +5. publish 成功后，Agent 必须报告 manifest、组件、digest 和验证结果并停止。只有用户再次明确确认上线，才可单独执行： + + ```bash + ./scripts/deploy-production-release.sh dist/releases/.json + ``` + +6. deploy 前后必须验证线上基线、镜像架构/revision、数据库备份条件、内部及公网健康检查和自动应用回滚。未实际完成这些验证时不得声称生产发布成功。 +7. 回滚也必须由用户明确要求，只能选择服务器已有的历史 manifest；应用回滚不自动恢复数据库。 diff --git a/Dockerfile b/Dockerfile index dfe6ed3..96cd46f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,15 +57,16 @@ ENV APP_ENV=production \ CMD ["/app/easyai-ai-gateway"] FROM --platform=$BUILDPLATFORM ${NODE_BUILD_IMAGE} AS web-builder +ARG NPM_CONFIG_REGISTRY=https://registry.npmjs.org WORKDIR /src -RUN npm install -g pnpm@10.18.1 +RUN npm install --registry "$NPM_CONFIG_REGISTRY" -g pnpm@10.18.1 COPY package.json pnpm-lock.yaml pnpm-workspace.yaml nx.json ./ COPY apps/web/package.json apps/web/package.json COPY packages/contracts/package.json packages/contracts/package.json RUN --mount=type=cache,id=pnpm-store,target=/root/.local/share/pnpm/store \ - pnpm install --frozen-lockfile + pnpm install --frozen-lockfile --registry "$NPM_CONFIG_REGISTRY" COPY packages packages COPY apps/web apps/web diff --git a/README.md b/README.md index edf5227..7a6ac21 100644 --- a/README.md +++ b/README.md @@ -59,9 +59,9 @@ OIDC 用户通过签名、Issuer、Audience、`tid`、Scope 和应用角色校 后端热更新可通过 `GO_WATCH_SHUTDOWN_GRACE_MS` 和 `GO_WATCH_RESTART_DELAY_MS` 调整旧进程退出等待时间与重启间隔。 -## Docker Compose 一键部署 +## Docker Compose 本地运行 -仓库内提供了面向 `linux/amd64` 的 Docker Compose 构建和部署脚本，会自动构建 API/Web 镜像、启动 PostgreSQL、执行数据库迁移，并验证 API 与 Web 是否可访问： +仓库内的 Compose 脚本只用于本地构建和运行：启动 PostgreSQL、执行数据库迁移，并验证 API 与 Web 是否可访问。它不再包含 Registry 推送或生产部署能力： ```bash scripts/deploy-compose.sh @@ -78,28 +78,21 @@ scripts/deploy-compose.sh 常用覆盖项： ```bash -AI_GATEWAY_IMAGE_TAG=2026.05.23-1 scripts/deploy-compose.sh -AI_GATEWAY_IMAGE_TAG=2026.05.23-1 AI_GATEWAY_PUSH=1 scripts/deploy-compose.sh -AI_GATEWAY_IMAGE_TAG=2026.05.23-1 scripts/deploy-compose.sh push +AI_GATEWAY_IMAGE_TAG=local-test scripts/deploy-compose.sh AI_GATEWAY_WEB_PORT=8080 AI_GATEWAY_API_PORT=18088 scripts/deploy-compose.sh AI_GATEWAY_GO_PROXY='https://proxy.golang.org,direct' scripts/deploy-compose.sh +AI_GATEWAY_NPM_REGISTRY='https://registry.npmmirror.com' scripts/deploy-compose.sh AI_GATEWAY_SKIP_BUILD=1 scripts/deploy-compose.sh scripts/deploy-compose.sh down scripts/deploy-compose.sh clean ``` -默认镜像地址为： +默认本地镜像地址为： -- API: `registry.cn-shanghai.aliyuncs.com/easyaigc/ai-gateway:latest` -- Web: `registry.cn-shanghai.aliyuncs.com/easyaigc/ai-gateway-web:latest` +- API: `registry.cn-shanghai.aliyuncs.com/easyaigc/ai-gateway:local` +- Web: `registry.cn-shanghai.aliyuncs.com/easyaigc/ai-gateway-web:local` -执行 `scripts/deploy-compose.sh push` 或设置 `AI_GATEWAY_PUSH=1` 时，会同时推送当前版本 tag 和 `latest`。当前版本 tag 优先使用 `AI_GATEWAY_IMAGE_TAG`；如果没有设置，则使用根 `package.json` 里的 `version`。 - -推送前需要先登录阿里云镜像仓库： - -```bash -docker login --username= registry.cn-shanghai.aliyuncs.com -``` +生产镜像禁止使用 `latest`，只能通过后文的人工 publish 命令推送完整 Git SHA Tag。 Web 容器的 Nginx 配置通过 bind mount 挂载自仓库文件 [docker/nginx.conf](docker/nginx.conf)，可直接修改该文件调整静态资源、规范 `/api/v1` 公开入口和旧 `/gateway-api` 兼容反向代理。修改后执行以下命令使配置生效： @@ -107,9 +100,31 @@ Web 容器的 Nginx 配置通过 bind mount 挂载自仓库文件 [docker/nginx. docker compose -f docker-compose.yml restart web ``` -## 生产 CI/CD +## 生产人工发布 -Gitea Actions 会在隔离的 rootless DinD Runner 中对 Pull Request、`main` Push 和版本 Tag 执行完整质量门禁；Tag 使用独立的 `release-ci / verify-tag (push)` context，不能复用旧的 `main` 成功状态。源码 Job 没有宿主 Docker、`sudo` 或生产部署权限。部署仓的 root-owned dispatcher 只在相同 SHA 的 `main` 与 Tag context 都成功后，用固定命令构建并扫描镜像，再以 Registry digest 发布 `ai.51easyai.com`。安装 Runner、Fork PR 审批、发布验证和回滚步骤见 [生产 CI/CD 运行手册](docs/runbooks/production-ci-cd.md)，信任边界见 [ADR-001](docs/decisions/001-production-cicd.md)。 +本仓库没有 Gitea Actions、Webhook、Tag、`main` Push、轮询或定时发布。`main` 不使用受保护分支限制；commit、push 和 Tag 都不会构建镜像或更新生产。 + +生产发布由 Agent 在用户明确指令下分两次执行。第一步在本机构建 `linux/amd64` 镜像、运行临时 PostgreSQL + simulation API 冒烟、推送完整 SHA Tag，并生成带内容完整性校验的 digest-pinned manifest；该命令不会修改生产： + +```bash +docker login --username= registry.cn-shanghai.aliyuncs.com +./scripts/publish-release-images.sh --components auto +``` + +Agent 必须报告 `dist/releases/.json` 后停止。用户再次明确确认上线后，才执行第二步： + +```bash +./scripts/deploy-production-release.sh dist/releases/.json +``` + +查看生产版本和显式回滚： + +```bash +./scripts/deploy-production-release.sh --status +./scripts/deploy-production-release.sh --rollback <历史完整 Git SHA> +``` + +完整安装、验证、失败处理和停用旧自动化步骤见[人工生产发布运行手册](docs/runbooks/production-ci-cd.md)，决策背景见 [ADR-003](docs/decisions/003-manual-agent-release.md)。 Compose 默认使用独立容器数据库 `postgres:18-alpine`，数据卷会保留在 `postgres_data` 和 `api_data`。为避免本地开发 `.env` 中的 `localhost` 数据库地址污染容器部署，compose 使用 `AI_GATEWAY_COMPOSE_*` 变量作为容器部署专用覆盖，例如： diff --git a/deploy/ci/act-runner-v2-config.yaml b/deploy/ci/act-runner-v2-config.yaml deleted file mode 100644 index eb33e05..0000000 --- a/deploy/ci/act-runner-v2-config.yaml +++ /dev/null @@ -1,36 +0,0 @@ -log: - level: info - -runner: - file: /data/.runner - capacity: 1 - envs: - GOPROXY: "https://goproxy.cn,direct" - GOSUMDB: "sum.golang.google.cn" - PATH: "/opt/easyai-gateway-ci/bin:/opt/easyai-gateway-ci/toolchains/go/bin:/opt/easyai-gateway-ci/toolchains/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - timeout: 3h - shutdown_timeout: 5m - insecure: false - fetch_timeout: 5s - fetch_interval: 2s - labels: - - "easyai-gateway-ci-unprivileged-v2:docker://docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7" - -cache: - enabled: false - -container: - docker_host: "-" - privileged: false - force_pull: false - force_rebuild: false - require_docker: true - docker_timeout: 1m - bind_workdir: false - workdir_parent: workspace - options: "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro" - valid_volumes: - - "/opt/easyai-gateway-ci" - -host: - workdir_parent: /data/host-work diff --git a/deploy/ci/easyai-gateway-ci-v2-runner.service b/deploy/ci/easyai-gateway-ci-v2-runner.service deleted file mode 100644 index 3f4c9dc..0000000 --- a/deploy/ci/easyai-gateway-ci-v2-runner.service +++ /dev/null @@ -1,44 +0,0 @@ -[Unit] -Description=EasyAI AI Gateway rootless DinD Gitea Actions runner v2 -Requires=docker.service -After=network-online.target docker.service -Wants=network-online.target - -[Service] -Type=simple -User=root -Group=root -Environment=RUNNER_SECURITY_OPTIONS= -Environment=RUNNER_IMAGE_REF= -EnvironmentFile=-/etc/easyai-gateway-ci-v2/runner.env -ExecStartPre=-/usr/bin/docker rm --force easyai-gateway-ci-v2-runner -ExecStart=/usr/bin/docker run --rm --pull=never --name easyai-gateway-ci-v2-runner --privileged --cpus 2 --memory 2g --memory-swap 3g --pids-limit 1024 $RUNNER_SECURITY_OPTIONS --volume easyai-gateway-ci-v2-data:/data --volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro --volume /etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro --volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro --env CONFIG_FILE=/config.yaml --env DOCKERD_ROOTLESS_ROOTLESSKIT_NET=slirp4netns --env DOCKERD_ROOTLESS_ROOTLESSKIT_MTU=65520 --env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns $RUNNER_IMAGE_REF -ExecStop=-/usr/bin/docker stop --time 360 easyai-gateway-ci-v2-runner -Restart=always -RestartSec=5s -TimeoutStartSec=60s -TimeoutStopSec=390s - -# systemd only launches the host Docker client. Pull-request jobs run inside the -# nested rootless daemon and never receive the host or nested Docker socket. -NoNewPrivileges=true -CapabilityBoundingSet= -AmbientCapabilities= -PrivateTmp=true -PrivateDevices=true -ProtectSystem=strict -ProtectHome=true -ProtectKernelTunables=true -ProtectKernelModules=true -ProtectKernelLogs=true -ProtectControlGroups=true -ProtectClock=true -ProtectHostname=true -LockPersonality=true -RestrictRealtime=true -RestrictSUIDSGID=true -SystemCallArchitectures=native -UMask=0077 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/ci/production-migration-base b/deploy/ci/production-migration-base deleted file mode 100644 index 16720b9..0000000 --- a/deploy/ci/production-migration-base +++ /dev/null @@ -1 +0,0 @@ -fce76a30ba2d7f9aa514418a2e09be7bbe7aaf4e diff --git a/deploy/manual/easyai-ai-gateway-release b/deploy/manual/easyai-ai-gateway-release new file mode 100755 index 0000000..0f39f6e --- /dev/null +++ b/deploy/manual/easyai-ai-gateway-release @@ -0,0 +1,233 @@ +#!/usr/bin/env bash +set -euo pipefail + +[[ $(id -u) -eq 0 ]] || { echo 'run as root' >&2; exit 1; } + +config_file=${AI_GATEWAY_RELEASE_CONFIG:-/etc/easyai-ai-gateway-release.conf} +manifest_tool=${AI_GATEWAY_RELEASE_MANIFEST_TOOL:-/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs} +[[ -f $config_file && ! -L $config_file ]] || { echo 'missing release config' >&2; exit 1; } +[[ -f $manifest_tool && ! -L $manifest_tool ]] || { echo 'missing release manifest validator' >&2; exit 1; } +# shellcheck source=/dev/null +source "$config_file" + +: "${DEPLOY_DIR:?}" +: "${COMPOSE_FILE:?}" +: "${RELEASES_DIR:?}" +: "${BACKUP_DIR:?}" +: "${PUBLIC_BASE_URL:?}" +: "${API_PORT:?}" +: "${WEB_PORT:?}" +: "${POSTGRES_SERVICE:?}" +: "${POSTGRES_USER:?}" +: "${POSTGRES_DATABASE:?}" +: "${BACKUP_RETENTION:?}" + +[[ $DEPLOY_DIR == /* && $COMPOSE_FILE == /* && $RELEASES_DIR == /* && $BACKUP_DIR == /* ]] || { + echo 'release paths must be absolute' >&2 + exit 1 +} +[[ $BACKUP_RETENTION =~ ^[1-9][0-9]*$ ]] || { echo 'invalid BACKUP_RETENTION' >&2; exit 1; } +[[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || { echo 'invalid PUBLIC_BASE_URL' >&2; exit 1; } +[[ -d $DEPLOY_DIR && -f $COMPOSE_FILE ]] || { echo 'deployment directory is unavailable' >&2; exit 1; } + +for command in docker curl node flock install; do + command -v "$command" >/dev/null 2>&1 || { echo "$command is required" >&2; exit 1; } +done +docker info >/dev/null 2>&1 || { echo 'Docker Engine is unavailable' >&2; exit 1; } +docker compose version >/dev/null 2>&1 || { echo 'Docker Compose v2 is required' >&2; exit 1; } + +install -d -m 0700 "$RELEASES_DIR" "$BACKUP_DIR" +exec 9>"$RELEASES_DIR/.release.lock" +flock -n 9 || { echo 'another release operation is running' >&2; exit 1; } + +compose=(docker compose --project-directory "$DEPLOY_DIR" -f "$COMPOSE_FILE") +current_manifest=$RELEASES_DIR/current.json + +manifest_get() { + node "$manifest_tool" get "$1" "$2" +} + +validate_image() { + local image=$1 + local source_sha=$2 + local require_revision=$3 + local revision architecture + + [[ $image =~ @sha256:[0-9a-f]{64}$ ]] || { echo "image is not digest-pinned: $image" >&2; return 1; } + docker pull --platform linux/amd64 "$image" + architecture=$(docker image inspect "$image" --format '{{.Os}}/{{.Architecture}}') + [[ $architecture == linux/amd64 ]] || { echo "unexpected image architecture: $architecture" >&2; return 1; } + if [[ $require_revision == true ]]; then + revision=$(docker image inspect "$image" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}') + [[ $revision == "$source_sha" ]] || { echo "image revision does not match release: $image" >&2; return 1; } + fi +} + +wait_for_url() { + local label=$1 + local url=$2 + local expected=${3:-} + local body= + + for _ in $(seq 1 60); do + if body=$(curl -fsS --max-time 5 "$url" 2>/dev/null); then + if [[ -z $expected || $body == *"$expected"* ]]; then + echo "[release] verified $label" + return 0 + fi + fi + sleep 2 + done + echo "release probe failed: $label ($url)" >&2 + return 1 +} + +verify_runtime() { + wait_for_url 'internal API health' "http://127.0.0.1:$API_PORT/api/v1/healthz" 'easyai-ai-gateway' + wait_for_url 'internal API readiness' "http://127.0.0.1:$API_PORT/api/v1/readyz" '"ok":true' + wait_for_url 'internal OpenAPI' "http://127.0.0.1:$API_PORT/api/v1/openapi.json" '"/api/v1/chat/completions"' + wait_for_url 'Web API reverse proxy' "http://127.0.0.1:$WEB_PORT/api/v1/healthz" 'easyai-ai-gateway' + wait_for_url 'Web application' "http://127.0.0.1:$WEB_PORT/" 'EasyAI AI Gateway' + wait_for_url 'public API health' "$PUBLIC_BASE_URL/api/v1/healthz" 'easyai-ai-gateway' + wait_for_url 'public API readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true' + wait_for_url 'public OpenAPI' "$PUBLIC_BASE_URL/api/v1/openapi.json" '"/api/v1/chat/completions"' +} + +activate_images() { + local api_image=$1 + local web_image=$2 + export AI_GATEWAY_API_IMAGE=$api_image + export AI_GATEWAY_WEB_IMAGE=$web_image + "${compose[@]}" up -d --no-build --pull always api web +} + +restore_previous() { + local previous=$1 + [[ -f $previous ]] || return 1 + local previous_api previous_web + previous_api=$(manifest_get "$previous" images.api) + previous_web=$(manifest_get "$previous" images.web) + echo '[release] restoring previous application images' >&2 + activate_images "$previous_api" "$previous_web" + verify_runtime +} + +backup_database() { + local source_sha=$1 + local temp_backup=$BACKUP_DIR/.${source_sha}.dump.tmp + local final_backup=$BACKUP_DIR/${source_sha}.dump + + umask 077 + "${compose[@]}" exec -T "$POSTGRES_SERVICE" \ + pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DATABASE" -Fc >"$temp_backup" + [[ -s $temp_backup ]] || { echo 'database backup is empty' >&2; rm -f "$temp_backup"; return 1; } + "${compose[@]}" exec -T "$POSTGRES_SERVICE" pg_restore -l <"$temp_backup" >/dev/null + mv "$temp_backup" "$final_backup" + mapfile -t expired < <(find "$BACKUP_DIR" -maxdepth 1 -type f -name '*.dump' -printf '%T@ %p\n' | sort -rn | awk -v keep="$BACKUP_RETENTION" 'NR > keep { sub(/^[^ ]+ /, ""); print }') + if [[ ${#expired[@]} -gt 0 ]]; then + rm -f -- "${expired[@]}" + fi + echo "[release] verified database backup: $final_backup" +} + +record_current() { + local manifest=$1 + local action=$2 + local started_at=$3 + local source_sha recorded temp + source_sha=$(manifest_get "$manifest" sourceSha) + recorded=$RELEASES_DIR/$source_sha.json + temp=$RELEASES_DIR/.${source_sha}.json.tmp + rm -f "$temp" + RELEASE_DEPLOYED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \ + RELEASE_DEPLOY_DURATION_SECONDS=$(( $(date +%s) - started_at )) \ + RELEASE_DEPLOY_ACTION=$action \ + node "$manifest_tool" record-deployment "$manifest" "$temp" >/dev/null + install -m 0600 "$temp" "$recorded" + install -m 0600 "$recorded" "$current_manifest.tmp" + mv "$current_manifest.tmp" "$current_manifest" + rm -f "$temp" +} + +deploy_manifest() { + local manifest=$1 + local action=${2:-deploy} + local source_sha base_sha api_image web_image migrations_changed current_sha + local changed_components api_changed=false web_changed=false + local previous_copy= + local started_at + + [[ -f $manifest && ! -L $manifest ]] || { echo 'manifest must be a regular file' >&2; return 1; } + node "$manifest_tool" validate "$manifest" + source_sha=$(manifest_get "$manifest" sourceSha) + base_sha=$(manifest_get "$manifest" baseReleaseSha) + api_image=$(manifest_get "$manifest" images.api) + web_image=$(manifest_get "$manifest" images.web) + migrations_changed=$(manifest_get "$manifest" migrationsChanged) + changed_components=$(manifest_get "$manifest" components) + [[ $changed_components == *'"api"'* ]] && api_changed=true + [[ $changed_components == *'"web"'* ]] && web_changed=true + + if [[ $action == deploy ]]; then + if [[ -f $current_manifest ]]; then + current_sha=$(manifest_get "$current_manifest" sourceSha) + [[ $current_sha == "$base_sha" ]] || { + echo "stale release manifest: production=$current_sha manifest_base=$base_sha" >&2 + return 1 + } + else + [[ -z $base_sha ]] || { echo 'non-bootstrap manifest has no production base' >&2; return 1; } + fi + fi + + validate_image "$api_image" "$source_sha" "$api_changed" + validate_image "$web_image" "$source_sha" "$web_changed" + + if [[ -f $current_manifest ]]; then + previous_copy=$(mktemp "$RELEASES_DIR/.previous.XXXXXX") + install -m 0600 "$current_manifest" "$previous_copy" + fi + started_at=$(date +%s) + + if [[ $action == deploy && $migrations_changed == true ]]; then + backup_database "$source_sha" + export AI_GATEWAY_API_IMAGE=$api_image + export AI_GATEWAY_WEB_IMAGE=$web_image + "${compose[@]}" run --rm --no-deps migrator + fi + + if ! activate_images "$api_image" "$web_image" || ! verify_runtime; then + if [[ -n $previous_copy ]]; then + restore_previous "$previous_copy" || echo 'automatic application rollback also failed' >&2 + fi + rm -f "$previous_copy" + return 1 + fi + + record_current "$manifest" "$action" "$started_at" + rm -f "$previous_copy" + echo "production_release=PASS action=$action release=$source_sha" +} + +case ${1:-} in + status) + [[ $# -eq 1 ]] || exit 64 + [[ -f $current_manifest ]] || { echo 'no production release manifest' >&2; exit 1; } + node "$manifest_tool" validate "$current_manifest" >/dev/null + cat "$current_manifest" + ;; + deploy) + [[ $# -eq 2 ]] || exit 64 + deploy_manifest "$2" deploy + ;; + rollback) + [[ $# -eq 2 && $2 =~ ^[0-9a-f]{40}$ ]] || exit 64 + target=$RELEASES_DIR/$2.json + [[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; } + deploy_manifest "$target" rollback + ;; + *) + echo 'usage: easyai-ai-gateway-release {status|deploy |rollback }' >&2 + exit 64 + ;; +esac diff --git a/deploy/manual/easyai-ai-gateway-release.conf.example b/deploy/manual/easyai-ai-gateway-release.conf.example new file mode 100644 index 0000000..49be00a --- /dev/null +++ b/deploy/manual/easyai-ai-gateway-release.conf.example @@ -0,0 +1,12 @@ +# Install as /etc/easyai-ai-gateway-release.conf with mode 0600 and root ownership. +DEPLOY_DIR=/root/easyai-ai-gateway-deploy +COMPOSE_FILE=/root/easyai-ai-gateway-deploy/docker-compose.yml +RELEASES_DIR=/root/easyai-ai-gateway-deploy/.releases +BACKUP_DIR=/var/backups/easyai-ai-gateway +PUBLIC_BASE_URL=https://ai.51easyai.com +API_PORT=8088 +WEB_PORT=5178 +POSTGRES_SERVICE=postgres +POSTGRES_USER=easyai +POSTGRES_DATABASE=easyai_ai_gateway +BACKUP_RETENTION=5 diff --git a/deploy/manual/install-release-helper.sh b/deploy/manual/install-release-helper.sh new file mode 100755 index 0000000..2442b47 --- /dev/null +++ b/deploy/manual/install-release-helper.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +[[ $(id -u) -eq 0 ]] || { echo 'run as root on the production server' >&2; exit 1; } + +install -d -m 0755 /usr/local/lib/easyai-ai-gateway-release +install -m 0755 "$root/deploy/manual/easyai-ai-gateway-release" \ + /usr/local/sbin/easyai-ai-gateway-release +install -m 0644 "$root/scripts/release-manifest.mjs" \ + /usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs +if [[ ! -e /etc/easyai-ai-gateway-release.conf ]]; then + install -m 0600 "$root/deploy/manual/easyai-ai-gateway-release.conf.example" \ + /etc/easyai-ai-gateway-release.conf + echo 'created /etc/easyai-ai-gateway-release.conf; review it before the first release' +fi + +echo 'installed manual release helper; no service, timer, webhook, or watcher was enabled' diff --git a/docker-compose.yml b/docker-compose.yml index 5b1f18b..1ddc325 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,6 +105,7 @@ services: VITE_GATEWAY_API_BASE_URL: ${AI_GATEWAY_WEB_API_BASE_URL:-/gateway-api} NODE_BUILD_IMAGE: ${AI_GATEWAY_NODE_BUILD_IMAGE:-node:22-alpine} WEB_RUNTIME_IMAGE: ${AI_GATEWAY_WEB_RUNTIME_IMAGE:-nginx:1.27-alpine} + NPM_CONFIG_REGISTRY: ${AI_GATEWAY_NPM_REGISTRY:-https://registry.npmjs.org} VITE_BASE_PATH: ${AI_GATEWAY_WEB_BASE_PATH:-/} ports: - "${AI_GATEWAY_WEB_PORT:-5178}:80" diff --git a/docs/decisions/001-production-cicd.md b/docs/decisions/001-production-cicd.md index d41a7d7..1fe1497 100644 --- a/docs/decisions/001-production-cicd.md +++ b/docs/decisions/001-production-cicd.md @@ -2,7 +2,7 @@ ## 状态 -Accepted +Superseded by [ADR-003](003-manual-agent-release.md) ## 日期 diff --git a/docs/decisions/003-manual-agent-release.md b/docs/decisions/003-manual-agent-release.md new file mode 100644 index 0000000..e883155 --- /dev/null +++ b/docs/decisions/003-manual-agent-release.md @@ -0,0 +1,31 @@ +# ADR-003: 本地 Agent 双阶段人工发布 + +## 状态 + +Accepted；取代 ADR-001 的自动 CI/CD 决策。 + +## 日期 + +2026-07-22 + +## 背景 + +AI Gateway 的交付重点是 API 制品能够启动、迁移数据库并通过核心兼容接口。原有 PR、`main` 和 Tag 三套重复质量门禁占用单并发 Runner，发布还要等待外部 dispatcher，反馈与上线时间不符合当前小团队由 Agent 主导运维的模式。 + +## 决策 + +- 删除所有 Gitea Actions，不监听 Push、PR、Tag、Webhook、轮询或定时事件。`main` 不设置保护规则或 required status，允许经用户授权的 Agent 直接提交和推送。 +- Git 操作与生产授权彻底分离。commit、push、merge 和 Tag 都不会构建或部署。 +- 发布分成两次独立的人工作业：本地 `publish` 负责验证、构建、simulation 冒烟、推送不可变镜像并生成带 SHA-256 内容完整性校验的 JSON manifest；用户再次确认后，`deploy` 才把该 manifest 应用到生产。 +- 只允许发布工作区干净、已提交且属于 `origin/main` 历史的完整 SHA。Registry Tag 使用完整 SHA，生产 Compose 只接受 `repository@sha256:...`。 +- publish 相对当前线上 manifest 选择 API/Web 变更；未变化的组件沿用现有 digest。两阶段之间如果线上基线变化，客户端和服务器都拒绝过期 manifest。 +- 待发布镜像在本地临时 PostgreSQL 中执行全部迁移，并逐项记录 health、readiness、OpenAPI、鉴权、模型列表、Chat Completions、Responses、Images 和 Web 反代冒烟。生成接口使用 5ms simulation，不调用真实上游。 +- 只有迁移变化时生产才创建并验证 PostgreSQL custom-format 备份。生产探活失败自动恢复上一应用 digest；数据库恢复仍需人工决定。 +- 完整测试、依赖审计和漏洞扫描保留为按需人工命令，不再影响镜像发布关键路径。 + +## 影响 + +- 不再有自动质量证据或自动发布；用户和 Agent 必须严格遵守两次显式授权边界。 +- 允许直接推送 `main` 提高速度，也取消了服务端代码审查强制约束。可追溯性改由干净提交、main 历史、OCI revision、Registry digest 和服务器 release manifest 保证。 +- 本机是 `Darwin arm64`，发布脚本必须使用 Buildx 构建并实际运行 `linux/amd64` 镜像。 +- Registry、SSH 或生产 helper 不可用时发布失败关闭，不允许退回 `latest`、重建制品或直接修改 Compose。 diff --git a/docs/runbooks/production-ci-cd.md b/docs/runbooks/production-ci-cd.md index 9cc9108..4c35da2 100644 --- a/docs/runbooks/production-ci-cd.md +++ b/docs/runbooks/production-ci-cd.md @@ -1,167 +1,134 @@ -# 生产 CI/CD 运行手册 +# 人工生产发布运行手册 -## 信任边界与固定资源 +## 原则 + +生产发布没有自动触发器。允许 Agent 直接提交和推送 `main`，但 Git 操作不会授权 publish 或 deploy： + +1. 用户明确要求发布镜像后，Agent 执行本地 publish。 +2. Agent 报告 manifest、digest、组件和冒烟结果并停止。 +3. 用户再次明确确认上线后，Agent 执行 deploy。 + +不得把两步放进同一个命令、Git hook、Gitea Action、Webhook、服务、Timer 或轮询脚本。 + +## 一次性停用旧自动化 + +源码仓已经删除 `.gitea/workflows` 中的全部工作流，`main` 分支保护和 required status 也必须保持关闭。可用下列只读命令验收： + +```bash +tea branches list --repo BCAI/easyai-ai-gateway --output yaml | sed -n '/^- name: main$/,+3p' +``` + +预期 `protected: "false"` 且 `user-can-push: "true"`。 + +确认没有运行中任务后，在旧 Runner 主机执行： + +```bash +systemctl disable --now easyai-gateway-ci-v2-runner.service +systemctl is-active easyai-gateway-ci-v2-runner.service +``` + +预期第二条命令返回 `inactive`。保留 `easyai-gateway-ci-v2-data` volume 观察一段时间，不立即删除。 + +生产部署仓不得保留 Tag/Main dispatcher、Webhook receiver 或 Timer。先只读列出相关单元，确认精确名称后逐个 `disable --now`，不得用模糊匹配批量停止其他 EasyAI 服务： + +```bash +systemctl list-unit-files --type=service --type=timer | grep -Ei 'easyai.*(release|dispatch|deploy)' +``` + +## 安装固定生产助手 + +生产助手只解析 manifest、拉取 digest、备份/迁移、更新 Compose、探活和回滚；它不是常驻服务，也不读取源码 checkout。 + +把以下三个已审查文件复制到生产服务器： + +- `deploy/manual/easyai-ai-gateway-release` +- `deploy/manual/easyai-ai-gateway-release.conf.example` +- `scripts/release-manifest.mjs` + +在可信 checkout 中可运行一次安装器： + +```bash +sudo ./deploy/manual/install-release-helper.sh +sudoedit /etc/easyai-ai-gateway-release.conf +sudo /usr/local/sbin/easyai-ai-gateway-release status +``` + +配置文件必须为 root 所有、`0600`。助手安装后不会创建或启用 service、timer、webhook 或 watcher。 + +默认生产资源： -- 源码仓：`https://git.51easyai.com/BCAI/easyai-ai-gateway` -- CI Runner：`easyai-gateway-ci-v2-runner.service` -- CI 标签：`easyai-gateway-ci-unprivileged-v2` -- Runner 状态：Docker named volume `easyai-gateway-ci-v2-data` 内的 `/data/.runner` -- 只读工具链：`/opt/easyai-gateway-ci` -- Runner 配置：`/etc/easyai-gateway-ci-v2/config.yaml` -- Runner 运行参数：`/etc/easyai-gateway-ci-v2/runner.env` -- 生产入口：`https://ai.51easyai.com` - 部署目录：`/root/easyai-ai-gateway-deploy` +- 发布清单：`/root/easyai-ai-gateway-deploy/.releases` +- 数据库备份：`/var/backups/easyai-ai-gateway` +- 公网入口：`https://ai.51easyai.com` -源码仓只执行质量 CI：PR context 是 `ci / verify (pull_request)`，`main` context 是 `ci / verify (push)`，版本 Tag 使用独立 workflow/context `release-ci / verify-tag (push)`。三者都执行生产迁移兼容门禁、Go 格式/vet/test/govulncheck、前端 lint/test/build、依赖审计、Compose 校验、Trivy 仓库扫描和 CI 脚本自测。Job 没有 Docker socket、`sudo` 或部署目录权限，也不构建发布镜像。 +## 阶段一：本地 publish -生产 CD 由部署仓的 root-owned dispatcher 执行。它在相同源码 SHA 上同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)` 成功，然后只运行部署仓中固定的 BuildKit 构建、Trivy 镜像扫描、digest 发布、备份、迁移、健康检查和回滚命令；不得执行 Tag checkout 中的 `pnpm`、测试、shell 脚本或应用程序。 - -provision 安装 checksum-pinned 的官方静态 ShellCheck `0.11.0` 和 Docker Compose `5.3.1`，不会依赖宿主是否预装这些命令。所有下载都在使用前校验固定 SHA-256，并在精确 Node Job 镜像内实际执行版本检查。PostgreSQL 16 集成库作为固定 digest 的 Actions service container 运行，PR Job 不接收宿主或内层 Docker API。 - -## 首次安装或修复 CI Runner - -前置条件：Linux x86_64、可用 Docker Engine、root 权限，以及 Docker 文件系统至少 8 GiB 可用空间。脚本只压缩本项目专属 `easyai-gateway-ci` Buildx 缓存，不运行全局 `docker system prune`，因为该服务器与其他服务共享 Docker 状态。 - -从 Gitea 仓库设置的 Actions Runner 页面取得一次性仓库注册 Token，在服务器可信的源码 checkout 中执行： +前置条件：Docker Desktop/Engine、Buildx、Compose v2、Node、Go、可用 Registry 登录和生产只读 SSH。源码必须无 tracked/untracked 改动，HEAD 必须属于最新获取的 `origin/main` 历史。 ```bash -RUNNER_REGISTRATION_TOKEN='<一次性 Token>' ./scripts/provision-ci-runner.sh +docker login --username= registry.cn-shanghai.aliyuncs.com +./scripts/publish-release-images.sh --components auto ``` -已有有效 named volume 时可直接重跑，不再需要 Token： +`auto` 读取线上当前 manifest，与 HEAD 比较后选择： + +- `api`：API、Go workspace 或迁移变化； +- `web`：Web、contracts、pnpm 或 Nginx 容器配置变化； +- `all`：Dockerfile/Compose 等共享运行时变化； +- `none`：无运行时变化，不产生发布物。 + +首次发布或线上状态不可验证时只能显式使用： ```bash -./scripts/provision-ci-runner.sh +./scripts/publish-release-images.sh --components all ``` -宿主无法直连 Docker Hub 时，可使用可信 HTTPS 镜像代理；Runner 来源仍必须携带脚本中完全相同的 manifest digest，内层 daemon 拉取的 Job 与 service 镜像也会继续校验工作流固定的 digest： +此路径保守地把迁移标记为变化：生产助手会先备份数据库，再运行新 API 镜像内的幂等 migrator。 -```bash -CI_RUNNER_IMAGE_SOURCE='docker.m.daocloud.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a' \ -CI_NESTED_DOCKER_REGISTRY_MIRROR='https://docker.m.daocloud.io' \ -./scripts/provision-ci-runner.sh -``` - -注册 Token 通过交互 stdin 送给短命注册容器，不出现在宿主或容器的命令行参数中；provision 随后立即从环境中清除它，长期 systemd unit 也不包含 Token。不要输出或复制 `/data/.runner` 内容。脚本会停用旧 host Runner，并确保旧 Runner 用户和 v2 遗留用户不属于 `docker`/`sudo` 组。 - -默认磁盘门禁可用环境变量提高，但生产环境不应降低： - -```bash -CI_RUNNER_MIN_FREE_GIB=8 \ -CI_RUNNER_MIN_POST_INSTALL_FREE_GIB=4 \ -RUNNER_REGISTRATION_TOKEN='<一次性 Token>' \ -./scripts/provision-ci-runner.sh -``` - -## AppArmor 分支 - -Ubuntu 22.04 默认没有 `kernel.apparmor_restrict_unprivileged_userns` 限制，当前服务器没有 `rootlesskit` profile 是受支持路径。provision 会实测 Docker 是否接受该 profile；不存在时写入： +publish 会执行快速 Go 测试、迁移安全校验、`linux/amd64` 构建、临时完整栈迁移和 API simulation 冒烟。全部通过后才推送完整 SHA Tag，并生成带 SHA-256 内容完整性字段的固定 schema manifest： ```text -RUNNER_SECURITY_OPTIONS="" +dist/releases/<40 位 Git SHA>.json ``` -如果新系统的 `/proc/sys/kernel/apparmor_restrict_unprivileged_userns` 为 `1`，则必须先由宿主发行版或 Docker 安装包提供并加载 `rootlesskit` profile；provision 只探测和使用，不会自行复制一份 profile。缺失时 provision 会停止；不得手工关闭该 sysctl，也不得使用 `apparmor=unconfined` 绕过。 +成功输出必须包含 `production_changed=false`。此时 Agent 应报告结果并停止，不得继续上线。 -参考：[Docker Rootless Docker-in-Docker](https://docs.docker.com/engine/security/rootless/tips/#rootless-docker-in-docker)、[Ubuntu 安全特性版本矩阵](https://documentation.ubuntu.com/security/security-features/security-features-overview/)。 +## 阶段二：人工 deploy -## Runner 真实验证 - -安装脚本会先启动一个无 Token、无持久 Runner 状态挂载、无宿主 bind mount 的短命 rootless DinD probe；随后启动长期服务，确认 Runner 已向 Gitea declare、通过内层 `docker info` 验证 `name=rootless`，再从内层 daemon 拉取固定 digest 的 Job 镜像，并在该镜像中实际运行全部工具。人工复核： +收到用户再次确认后执行： ```bash -systemctl is-active easyai-gateway-ci-v2-runner.service -systemctl is-enabled easyai-gateway-ci-v2-runner.service -systemctl cat easyai-gateway-ci-v2-runner.service -journalctl -u easyai-gateway-ci-v2-runner.service --since '15 minutes ago' --no-pager - -docker inspect -f 'user={{.Config.User}} privileged={{.HostConfig.Privileged}} apparmor={{.AppArmorProfile}}' \ - easyai-gateway-ci-v2-runner -docker inspect -f '{{range .Mounts}}{{println .Source " -> " .Destination .Mode}}{{end}}' \ - easyai-gateway-ci-v2-runner -docker exec easyai-gateway-ci-v2-runner docker info \ - --format '{{range .SecurityOptions}}{{println .}}{{end}}' -docker exec easyai-gateway-ci-v2-runner docker image inspect \ - 'docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7' \ - --format '{{.Id}}' +./scripts/deploy-production-release.sh dist/releases/<40 位 Git SHA>.json ``` -预期：外层 image user 为 `rootless`、外层因 DinD 为 `privileged=true`、内层安全选项含 `name=rootless`；RootlessKit 使用独立 PID namespace，内层 `--pid=host` 也不能看到外层 Runner；挂载仅包含专用 `/data`、只读配置和只读工具链，不能出现 `/var/run/docker.sock` 或 `/run/docker.sock`。 +客户端和生产助手都会确认当前生产 SHA 仍等于 manifest 的 `baseReleaseSha`。不相等说明 publish 之后生产发生变化，必须重新 publish，不能覆盖或使用强制参数。 -在 Gitea 中分别验证： +助手随后验证： -1. 同仓 PR 的 `ci / verify (pull_request)` 成功。 -2. `main` Push 的 `ci / verify (push)` 成功。 -3. 合法 SemVer Tag 新产生独立 `release-ci / verify-tag (push)`；非法或不属于 `main` 历史的 Tag 失败，且 Tag context 不能被旧 `main` context 替代。 -4. Fork PR 默认不自动调度，维护者先检查 `.gitea/workflows/**` 和容器选项后再批准。 +- API/Web 均为完整 Registry digest； +- 镜像为 `linux/amd64`； +- 本次变化组件的 OCI revision 等于 release SHA； +- 迁移变化时备份可被 `pg_restore -l` 读取； +- 内部 API health/readiness/OpenAPI； +- Web 页面和 Web API 反代； +- 公网 health/readiness/OpenAPI。 -PR 与 `main` Push workflow 都校验事件携带的 base/before SHA 必须是触发 SHA 的祖先；分支落后或主干历史被改写时 CI 会 fail-closed。PR 先合并最新 `main` 再重跑，Gitea 分支保护同时启用 `block_on_outdated_branch`。 +失败时自动恢复上一组应用 digest 并重复探活。已经执行的 expand-only 数据库迁移不会自动恢复。 -## Fork PR 残余风险 - -rootless DinD 不是 VM 沙箱。外层仍按官方要求使用 `--privileged`，Gitea runner 也会解析 PR 控制的 `jobs..container.options`。配置已禁止 Job privileged、宿主/内层 Docker socket 和任意 bind source，但 Runner、rootless Docker、容器运行时或内核漏洞仍可能突破边界；Job 的出站网络和资源消耗也不是强租户隔离。因此 Fork PR 必须维护者批准，CI 不得注入生产 Secret；未来应迁移到独立 CI 主机或虚拟机。 - -## 数据库迁移兼容门禁 - -`deploy/ci/production-migration-base` 必须等于当前生产 API 对应的完整源码 SHA。`scripts/ci-validate-migrations.mjs` 只允许新增、按文件名严格追加且为普通文件的迁移；生产已知迁移以及已经进入 `main` 的迁移不可修改或删除。DROP、TRUNCATE、DELETE、MERGE、CREATE OR REPLACE、动态 EXECUTE、事务控制、DO/函数/过程定义、RENAME、列类型变更、增加/设置 `NOT NULL`、删除默认值和 REVOKE 会直接使 CI 失败。需要过程式迁移时先拆成可审计的声明式 expand/contract SQL；不能把过程体藏在普通或 dollar-quoted 字符串中。 - -该规则允许 INSERT/UPDATE 等数据迁移，因此不能替代备份和恢复验证。每次生产发布成功并完成 `pg_restore` 演练后，创建一个只前移 `deploy/ci/production-migration-base` 的受审 PR；不要把基线指向尚未成功部署的提交。 - -## 发布生产版本 - -1. 合并到受保护 `main`，确认源码 CI 成功。 -2. 在已验证提交上创建完整 SemVer Tag 并推送。 -3. 确认同一 SHA 的 `ci / verify (push)` 和本次 `release-ci / verify-tag (push)` 都成功。 -4. 由部署仓 root-owned dispatcher 构建、扫描并发布 digest；源码仓工作流不会直接调用生产 helper。 -5. 发布、健康检查和恢复演练完成后，用单独 PR 把 `deploy/ci/production-migration-base` 更新为该生产 Tag 的完整源码 SHA。 +## 状态与回滚 ```bash -git switch main -git pull --ff-only -git tag -a v0.1.1 -m 'release: v0.1.1' -git push origin v0.1.1 +./scripts/deploy-production-release.sh --status +./scripts/deploy-production-release.sh --rollback <服务器已有的历史 SHA> ``` -dispatcher 以完整 Git SHA 发布 Registry Tag，并把 Registry 返回的 digest 写入服务器 root-only 发布清单。生产 Compose 使用 `repository@sha256:...`，不是 `latest`、版本号或可变 SHA Tag。 +回滚也需要用户明确指令。服务器只接受 `.releases/.json` 中已有版本，拒绝远端任意 Tag、`latest` 或临时镜像。 -## 发布后验证 +## 故障排查 -宿主 Nginx 的 `ai.51easyai.com` TLS server 必须包含仓库中的 `deploy/nginx/ai.51easyai.com-api-v1.inc` 等价规则，保留完整 URI 转发到 `127.0.0.1:8088`。修改前先备份现有配置，执行 `nginx -t` 成功后才能 reload。 - -```bash -curl -fsS https://ai.51easyai.com/api/v1/healthz -curl -fsS https://ai.51easyai.com/api/v1/readyz -ssh root@110.42.51.33 'cd /root/easyai-ai-gateway-deploy && ./gateway-ops.sh ps' -``` - -同时检查服务器磁盘、API 日志和数据库容器健康状态。发布后的首小时如出现错误率翻倍、P95 延迟增加 50%、数据完整性或安全问题，应立即回滚。 - -## 应用与数据库回滚 - -发布失败时固定部署脚本会恢复上一组 API/Web digest 并重复健康检查。人工回滚只选择服务器已有 root-only 发布清单的历史 Git SHA： - -```bash -sudo /usr/local/sbin/easyai-ai-gateway-release <40 位历史 Git SHA> -``` - -没有 `/root/easyai-ai-gateway-deploy/.releases/.env` 的远端 SHA Tag 不应被盲目信任。数据库备份位于 `/var/backups/easyai-ai-gateway`，只保留最近 5 份。应用回滚不会自动恢复数据库；只有停止写入并再次保留当前副本后，操作员才能选择备份执行 `pg_restore`。 - -## Runner 故障恢复 - -先在 Gitea 确认没有运行中的 Job，再检查服务与两个 daemon： - -```bash -systemctl status easyai-gateway-ci-v2-runner.service --no-pager -docker logs --tail 100 easyai-gateway-ci-v2-runner -docker exec easyai-gateway-ci-v2-runner docker info -df -h /var/lib/docker -``` - -确认空闲后才执行： - -```bash -systemctl restart easyai-gateway-ci-v2-runner.service -journalctl -u easyai-gateway-ci-v2-runner.service -n 100 --no-pager -``` - -不要删除 `easyai-gateway-ci-v2-data` volume；它包含 Runner 注册状态和内层 daemon 缓存。不得用全局 prune 处理磁盘压力。 +- publish 前失败：没有推送镜像，也没有可部署 manifest；修复本地环境后重试。 +- 镜像已经部分推送但 manifest 未生成：SHA Tag 被视为占用，先审查 Registry 中的 digest，不得覆盖重推。 +- deploy 报基线过期：重新读取线上状态并重新 publish。 +- 迁移备份失败：生产不切换。 +- 生产探活和自动应用回滚都失败：停止发布，保留日志、当前数据库和 release manifest，人工恢复服务；不得自动执行 `pg_restore`。 diff --git a/package.json b/package.json index e4ac4da..08550d4 100644 --- a/package.json +++ b/package.json @@ -12,8 +12,9 @@ "migrate": "nx run api:migrate", "openapi": "nx run api:openapi", "docker:build": "docker compose -f docker-compose.yml build --pull api web", - "docker:push": "scripts/deploy-compose.sh push-only", - "docker:deploy": "pnpm docker:build && pnpm docker:push" + "release:publish": "scripts/publish-release-images.sh --components auto", + "release:deploy": "scripts/deploy-production-release.sh", + "test:release": "tests/release/manual-release-test.sh && tests/ci/migrations-test.sh" }, "devDependencies": { "@nx/vite": "^21.0.0", diff --git a/scripts/api-release-smoke.mjs b/scripts/api-release-smoke.mjs new file mode 100755 index 0000000..cda490c --- /dev/null +++ b/scripts/api-release-smoke.mjs @@ -0,0 +1,145 @@ +#!/usr/bin/env node + +import { execFileSync } from 'node:child_process' + +function fail(message) { + console.error(`api_release_smoke=FAIL ${message}`) + process.exit(1) +} + +function assert(condition, message) { + if (!condition) fail(message) +} + +const baseURL = (process.env.RELEASE_SMOKE_BASE_URL || '').replace(/\/+$/, '') +const webURL = (process.env.RELEASE_SMOKE_WEB_URL || '').replace(/\/+$/, '') +const composeProject = process.env.RELEASE_SMOKE_COMPOSE_PROJECT || '' +const composeFile = process.env.RELEASE_SMOKE_COMPOSE_FILE || 'docker-compose.yml' +const nonce = process.env.RELEASE_SMOKE_NONCE || Date.now().toString(36) +const databaseUser = process.env.RELEASE_SMOKE_DATABASE_USER || 'easyai' +const databaseName = process.env.RELEASE_SMOKE_DATABASE_NAME || 'easyai_ai_gateway' + +if (!/^http:\/\/127\.0\.0\.1:\d+$/.test(baseURL)) fail('invalid RELEASE_SMOKE_BASE_URL') +if (!/^http:\/\/127\.0\.0\.1:\d+$/.test(webURL)) fail('invalid RELEASE_SMOKE_WEB_URL') +if (!/^[a-z0-9][a-z0-9_-]{2,62}$/.test(composeProject)) fail('invalid RELEASE_SMOKE_COMPOSE_PROJECT') +if (!/^[a-z_][a-z0-9_]{0,62}$/.test(databaseUser)) fail('invalid RELEASE_SMOKE_DATABASE_USER') +if (!/^[a-z_][a-z0-9_]{0,62}$/.test(databaseName)) fail('invalid RELEASE_SMOKE_DATABASE_NAME') + +async function request(path, { method = 'GET', token = '', body } = {}) { + const response = await fetch(`${baseURL}${path}`, { + method, + signal: AbortSignal.timeout(15_000), + headers: { + ...(token ? { Authorization: `Bearer ${token}` } : {}), + ...(body === undefined ? {} : { 'Content-Type': 'application/json' }), + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }) + const text = await response.text() + let payload + try { + payload = text ? JSON.parse(text) : null + } catch { + fail(`${method} ${path} returned non-JSON HTTP ${response.status}`) + } + if (!response.ok) fail(`${method} ${path} returned HTTP ${response.status}`) + return { response, payload } +} + +async function waitForAPI() { + let lastError = 'API did not respond' + for (let attempt = 0; attempt < 60; attempt += 1) { + try { + const response = await fetch(`${baseURL}/api/v1/readyz`, { + signal: AbortSignal.timeout(2_000), + }) + if (response.ok) return + lastError = `readiness returned HTTP ${response.status}` + } catch (error) { + lastError = error.message + } + await new Promise((resolve) => setTimeout(resolve, 1_000)) + } + fail(`API readiness timed out: ${lastError}`) +} + +await waitForAPI() + +const health = await request('/api/v1/healthz') +assert(health.payload?.ok === true && health.payload?.service === 'easyai-ai-gateway', 'health payload mismatch') + +const ready = await request('/api/v1/readyz') +assert(ready.payload?.ok === true, 'readiness payload mismatch') + +const openapi = await request('/api/v1/openapi.json') +for (const path of [ + '/api/v1/healthz', + '/api/v1/readyz', + '/api/v1/models', + '/api/v1/chat/completions', + '/api/v1/responses', + '/api/v1/images/generations', +]) { + assert(openapi.payload?.paths?.[path], `OpenAPI is missing ${path}`) +} + +const username = `release_smoke_${nonce}`.slice(0, 60) +const password = `Release-Smoke-${nonce}-9x!` +const registration = await request('/api/v1/auth/register', { + method: 'POST', + body: { username, email: `${username}@example.invalid`, password }, +}) +assert(typeof registration.payload?.accessToken === 'string', 'registration did not return an access token') + +const login = await request('/api/v1/auth/login', { + method: 'POST', + body: { account: username, password }, +}) +const jwt = login.payload?.accessToken +assert(typeof jwt === 'string' && jwt.length > 20, 'login did not return an access token') + +try { + execFileSync('docker', [ + 'compose', '-p', composeProject, '-f', composeFile, + 'exec', '-T', 'postgres', 'psql', '-v', 'ON_ERROR_STOP=1', + '-U', databaseUser, '-d', databaseName, '-c', + `UPDATE gateway_wallet_accounts SET balance = 100000 WHERE gateway_user_id = (SELECT id FROM gateway_users WHERE username = '${username.replaceAll("'", "''")}')`, + ], { stdio: ['ignore', 'ignore', 'pipe'] }) +} catch { + fail('could not fund the ephemeral smoke wallet') +} + +const apiKeyResult = await request('/api/v1/api-keys', { + method: 'POST', token: jwt, body: { name: 'release smoke key' }, +}) +const apiKey = apiKeyResult.payload?.secret +assert(typeof apiKey === 'string' && apiKey.startsWith('sk-gw-'), 'API key creation failed') + +const models = await request('/api/v1/models', { token: apiKey }) +assert(Array.isArray(models.payload?.items) || Array.isArray(models.payload?.data), 'models response has no items') + +const simulation = { runMode: 'simulation', simulation: true, simulationDurationMs: 5 } +const chat = await request('/api/v1/chat/completions', { + method: 'POST', token: apiKey, + body: { model: 'openai:gpt-4o-mini', messages: [{ role: 'user', content: 'release smoke' }], ...simulation }, +}) +assert(/^chatcmpl-/.test(chat.payload?.id || ''), 'chat simulation response mismatch') + +const responses = await request('/api/v1/responses', { + method: 'POST', token: apiKey, + body: { model: 'openai:gpt-4o-mini', input: 'release smoke', ...simulation }, +}) +assert(/^resp/.test(responses.payload?.id || ''), 'responses simulation response mismatch') + +const images = await request('/api/v1/images/generations', { + method: 'POST', token: apiKey, + body: { model: 'openai:gpt-image-1', prompt: 'release smoke', ...simulation }, +}) +assert(Array.isArray(images.payload?.data) && images.payload.data.length > 0, 'image simulation response mismatch') + +const webHealth = await fetch(`${webURL}/api/v1/healthz`, { signal: AbortSignal.timeout(15_000) }) +assert(webHealth.ok, `web API reverse proxy returned HTTP ${webHealth.status}`) +const webApp = await fetch(`${webURL}/`, { signal: AbortSignal.timeout(15_000) }) +assert(webApp.ok && (await webApp.text()).includes('EasyAI AI Gateway'), 'web application smoke failed') + +console.log('api_release_smoke=PASS health=ok readiness=ok openapi=ok auth=ok chat=ok responses=ok images=ok web=ok') diff --git a/scripts/ci-build-images.sh b/scripts/ci-build-images.sh deleted file mode 100755 index 7061ca9..0000000 --- a/scripts/ci-build-images.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) -cd "$root" - -tag=${IMAGE_TAG:-} -registry=${AI_GATEWAY_IMAGE_REGISTRY:-registry.cn-shanghai.aliyuncs.com/easyaigc} -builder=${AI_GATEWAY_BUILDX_BUILDER:-easyai-gateway-ci} -cache_limit=${AI_GATEWAY_BUILDX_CACHE_LIMIT:-2gb} -platform=${AI_GATEWAY_PLATFORM:-linux/amd64} -keep_images=0 -build_complete=0 -readonly trivy_db_repository=ghcr.m.daocloud.io/aquasecurity/trivy-db:2 - -case ${1:-} in - '') ;; - --keep-images) keep_images=1 ;; - *) - echo 'usage: ci-build-images.sh [--keep-images]' >&2 - exit 64 - ;; -esac - -if [[ ! $tag =~ ^[0-9a-f]{40}$ ]]; then - echo 'IMAGE_TAG must be a full lowercase Git SHA' >&2 - exit 1 -fi -if [[ ! $registry =~ ^[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._/-]+)+$ ]]; then - echo 'AI_GATEWAY_IMAGE_REGISTRY has an invalid format' >&2 - exit 1 -fi -if [[ ! $cache_limit =~ ^[1-9][0-9]*(kb|mb|gb)$ ]]; then - echo 'AI_GATEWAY_BUILDX_CACHE_LIMIT has an invalid format' >&2 - exit 1 -fi - -api_image="$registry/ai-gateway:$tag" -web_image="$registry/ai-gateway-web:$tag" - -cleanup() { - original_status=$? - trap - EXIT HUP INT TERM - cleanup_status=0 - - docker buildx prune --builder "$builder" --force --max-used-space "$cache_limit" >/dev/null || { - cleanup_status=$? - echo 'failed to prune the CI BuildKit cache' >&2 - } - if [[ $keep_images -ne 1 || $build_complete -ne 1 ]]; then - docker image rm "$api_image" "$web_image" >/dev/null 2>&1 || { - image_rm_status=$? - [[ $cleanup_status -ne 0 ]] || cleanup_status=$image_rm_status - echo 'failed to remove CI image tags' >&2 - } - fi - - if [[ $original_status -ne 0 ]]; then - exit "$original_status" - fi - exit "$cleanup_status" -} -trap cleanup EXIT -trap 'exit 129' HUP -trap 'exit 130' INT -trap 'exit 143' TERM - -if ! docker buildx inspect "$builder" >/dev/null 2>&1; then - docker buildx create --name "$builder" --driver docker-container --use >/dev/null -fi -docker buildx inspect "$builder" --bootstrap >/dev/null -common_args=( - --builder "$builder" - --platform "$platform" - --file Dockerfile - --pull - --load -) - -docker buildx build --builder "$builder" \ - "${common_args[@]:2}" \ - --target api \ - --build-arg "GO_BUILD_IMAGE=${AI_GATEWAY_GO_BUILD_IMAGE:-docker.m.daocloud.io/library/golang:1.26.5-alpine}" \ - --build-arg "API_RUNTIME_IMAGE=${AI_GATEWAY_API_RUNTIME_IMAGE:-docker.m.daocloud.io/library/alpine:3.22}" \ - --tag "$api_image" . - -docker buildx build --builder "$builder" \ - "${common_args[@]:2}" \ - --target web \ - --build-arg "NODE_BUILD_IMAGE=${AI_GATEWAY_NODE_BUILD_IMAGE:-docker.m.daocloud.io/library/node:24-alpine}" \ - --build-arg "WEB_RUNTIME_IMAGE=${AI_GATEWAY_WEB_RUNTIME_IMAGE:-docker.m.daocloud.io/library/nginx:1.27-alpine}" \ - --tag "$web_image" . - -for image in "$api_image" "$web_image"; do - trivy image --db-repository "$trivy_db_repository" \ - --scanners vuln,secret --severity HIGH,CRITICAL --ignore-unfixed \ - --exit-code 1 --timeout 15m "$image" -done - -build_complete=1 -printf 'gateway_images=PASS git_sha=%s api=%s web=%s\n' "$tag" "$api_image" "$web_image" diff --git a/scripts/ci-validate-migrations.mjs b/scripts/ci-validate-migrations.mjs index b75b309..25f6492 100755 --- a/scripts/ci-validate-migrations.mjs +++ b/scripts/ci-validate-migrations.mjs @@ -5,10 +5,7 @@ import { lstatSync, readFileSync } from 'node:fs' import { basename } from 'node:path' const migrationDirectory = 'apps/api/migrations/' -const baselineFile = 'deploy/ci/production-migration-base' -const base = process.argv[2] || process.env.PRODUCTION_MIGRATION_BASE_SHA || '' -const immutableRef = - process.argv[3] || process.env.MIGRATION_IMMUTABILITY_REF || '' +const base = process.argv[2] || process.env.RELEASE_BASE_SHA || '' function fail(message) { console.error(`migration_safety=FAIL ${message}`) @@ -28,7 +25,7 @@ function git(args, options = {}) { } if (!/^[0-9a-f]{40}$/.test(base)) { - fail('PRODUCTION_MIGRATION_BASE_SHA must be a full lowercase commit SHA') + fail('RELEASE_BASE_SHA must be a full lowercase commit SHA') } const root = git(['rev-parse', '--show-toplevel']).trim() @@ -41,34 +38,14 @@ const resolvedBase = git([ `${base}^{commit}`, ]).trim() if (resolvedBase !== base) { - fail('production migration baseline did not resolve exactly') + fail('release base did not resolve exactly') } const ancestry = spawnSync('git', ['merge-base', '--is-ancestor', base, 'HEAD'], { stdio: 'ignore', }) if (ancestry.status !== 0) { - fail('production migration baseline is not an ancestor of HEAD') -} - -const head = git(['rev-parse', '--verify', 'HEAD^{commit}']).trim() - -const baselineTreeEntry = git(['ls-tree', 'HEAD', '--', baselineFile]).trim() -let baselineState -try { - baselineState = lstatSync(baselineFile) -} catch { - fail('production migration baseline file is unavailable in the worktree') -} -if ( - !/^100644 blob [0-9a-f]+\t/.test(baselineTreeEntry) || - !baselineState.isFile() || - baselineState.isSymbolicLink() -) { - fail('production migration baseline must be a mode 100644 regular file') -} -if (readFileSync(baselineFile, 'utf8').trim() !== base) { - fail('production migration baseline argument does not match the reviewed file') + fail('release base is not an ancestor of HEAD') } function changedMigrations(from) { @@ -252,94 +229,8 @@ const destructiveRules = [ const violations = [] let addedFiles = 0 -let immutableBase = '' const orderingCandidates = new Set() -if (immutableRef) { - if ( - !/^[0-9a-f]{40}$/.test(immutableRef) && - immutableRef !== 'refs/remotes/origin/main' - ) { - fail('MIGRATION_IMMUTABILITY_REF must be a full commit SHA or refs/remotes/origin/main') - } - - const immutableTip = git([ - 'rev-parse', - '--verify', - '--end-of-options', - `${immutableRef}^{commit}`, - ]).trim() - const baseAtEvent = spawnSync( - 'git', - ['show', `${immutableTip}:${baselineFile}`], - { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }, - ) - if (baseAtEvent.status === 0) { - const previousBaselineTreeEntry = git([ - 'ls-tree', - immutableTip, - '--', - baselineFile, - ]).trim() - if (!/^100644 blob [0-9a-f]+\t/.test(previousBaselineTreeEntry)) { - fail('previous production migration baseline must be a mode 100644 regular file') - } - const previousProductionBase = baseAtEvent.stdout.trim() - if (!/^[0-9a-f]{40}$/.test(previousProductionBase)) { - fail('previous production migration baseline is invalid') - } - const baselineAdvance = spawnSync( - 'git', - ['merge-base', '--is-ancestor', previousProductionBase, base], - { stdio: 'ignore' }, - ) - if (baselineAdvance.status !== 0) { - fail('production migration baseline must only move forward') - } - } - const baselineWasAlreadyReviewed = spawnSync( - 'git', - ['merge-base', '--is-ancestor', base, immutableTip], - { stdio: 'ignore' }, - ) - if (baselineWasAlreadyReviewed.status !== 0) { - fail('production migration baseline must belong to the immutable event base history') - } - if (immutableTip === head) { - const parent = spawnSync( - 'git', - ['rev-parse', '--verify', `${immutableTip}^1^{commit}`], - { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }, - ) - immutableBase = parent.status === 0 ? parent.stdout.trim() : '' - } else { - const immutableAncestry = spawnSync( - 'git', - ['merge-base', '--is-ancestor', immutableTip, head], - { stdio: 'ignore' }, - ) - if (immutableAncestry.status !== 0) { - fail('migration immutability base is not an ancestor of HEAD') - } - immutableBase = immutableTip - } - - if (immutableBase) { - const immutableChanges = changedMigrations(immutableBase) - for (let index = 0; index < immutableChanges.length; index += 2) { - const status = immutableChanges[index] - const file = immutableChanges[index + 1] - if (status === 'A') { - orderingCandidates.add(file) - } else { - violations.push( - `${file}: migration present on main is immutable (status ${status})`, - ) - } - } - } -} - function migrationNamesAt(ref) { const output = git( ['ls-tree', '-r', '-z', '--name-only', ref, '--', migrationDirectory], @@ -358,14 +249,11 @@ function migrationNamesAt(ref) { .sort() } -const orderingBase = immutableBase || base -const existingNames = migrationNamesAt(orderingBase) +const existingNames = migrationNamesAt(base) const previousMaximumName = existingNames.at(-1) || '' -if (!immutableBase) { - for (let index = 0; index < fields.length; index += 2) { - if (fields[index] === 'A') orderingCandidates.add(fields[index + 1]) - } +for (let index = 0; index < fields.length; index += 2) { + if (fields[index] === 'A') orderingCandidates.add(fields[index + 1]) } for (let index = 0; index < fields.length; index += 2) { diff --git a/scripts/ci-validate-semver.sh b/scripts/ci-validate-semver.sh deleted file mode 100755 index 167235f..0000000 --- a/scripts/ci-validate-semver.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -tag=${1:-} - -die() { - printf 'invalid release tag: %s\n' "$tag" >&2 - exit 1 -} - -[[ $tag == v* ]] || die -version=${tag#v} - -# Bash arrays discard trailing empty fields, so parsing first would accept -# values such as v1.2.3. and v1.2.3-alpha. Validate the complete string with -# the SemVer 2.0.0 grammar before doing anything else with the tag. -semver_pattern='^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(\.(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*))?(\+([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' -LC_ALL=C -export LC_ALL -[[ $version =~ $semver_pattern ]] || die - -printf 'release_tag=PASS tag=%s\n' "$tag" diff --git a/scripts/deploy-compose.sh b/scripts/deploy-compose.sh index 618747d..80944e2 100755 --- a/scripts/deploy-compose.sh +++ b/scripts/deploy-compose.sh @@ -6,7 +6,7 @@ COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.yml}" COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-easyai-ai-gateway}" AI_GATEWAY_PLATFORM="${AI_GATEWAY_PLATFORM:-linux/amd64}" AI_GATEWAY_IMAGE_REGISTRY="${AI_GATEWAY_IMAGE_REGISTRY:-registry.cn-shanghai.aliyuncs.com/easyaigc}" -AI_GATEWAY_IMAGE_TAG="${AI_GATEWAY_IMAGE_TAG:-latest}" +AI_GATEWAY_IMAGE_TAG="${AI_GATEWAY_IMAGE_TAG:-local}" AI_GATEWAY_API_IMAGE="${AI_GATEWAY_API_IMAGE:-${AI_GATEWAY_IMAGE_REGISTRY}/ai-gateway:${AI_GATEWAY_IMAGE_TAG}}" AI_GATEWAY_WEB_IMAGE="${AI_GATEWAY_WEB_IMAGE:-${AI_GATEWAY_IMAGE_REGISTRY}/ai-gateway-web:${AI_GATEWAY_IMAGE_TAG}}" ACTION="${1:-deploy}" @@ -22,11 +22,8 @@ compose=(docker compose -f "$COMPOSE_FILE") usage() { cat <<'EOF' Usage: - scripts/deploy-compose.sh Build, migrate, start, and verify + scripts/deploy-compose.sh Build, migrate, start, and verify locally scripts/deploy-compose.sh deploy Same as default - scripts/deploy-compose.sh push Build and push API/Web images - scripts/deploy-compose.sh push-only - Push already-built API/Web images scripts/deploy-compose.sh down Stop containers, keep volumes scripts/deploy-compose.sh clean Stop containers and remove volumes @@ -38,8 +35,10 @@ Useful environment overrides: AI_GATEWAY_WEB_PORT=5178 AI_GATEWAY_API_PORT=8088 AI_GATEWAY_DB_PORT=54329 - AI_GATEWAY_PUSH=1 AI_GATEWAY_SKIP_BUILD=1 + +Production publishing is intentionally unavailable here. Use: + scripts/publish-release-images.sh --components auto EOF } @@ -125,86 +124,6 @@ build_images() { fi } -package_version() { - local version="" - - version="$( - sed -nE 's/^[[:space:]]*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/p' "$PROJECT_ROOT/package.json" \ - | head -n 1 - )" - if [[ -z "$version" ]]; then - echo "[ai-gateway] cannot infer package version from package.json; set AI_GATEWAY_IMAGE_TAG" >&2 - exit 1 - fi - echo "$version" -} - -push_version_tag() { - if [[ "$AI_GATEWAY_IMAGE_TAG" == "latest" ]]; then - package_version - else - echo "$AI_GATEWAY_IMAGE_TAG" - fi -} - -image_repository() { - local image="$1" - echo "${image%:*}" -} - -tag_release_images() { - local version_tag="$1" - local api_repo web_repo - local api_version_image api_latest_image web_version_image web_latest_image - - api_repo="$(image_repository "$AI_GATEWAY_API_IMAGE")" - web_repo="$(image_repository "$AI_GATEWAY_WEB_IMAGE")" - api_version_image="${api_repo}:${version_tag}" - api_latest_image="${api_repo}:latest" - web_version_image="${web_repo}:${version_tag}" - web_latest_image="${web_repo}:latest" - - docker image inspect "$AI_GATEWAY_API_IMAGE" >/dev/null 2>&1 || { - echo "[ai-gateway] missing local API image: ${AI_GATEWAY_API_IMAGE}; build it first" >&2 - exit 1 - } - docker image inspect "$AI_GATEWAY_WEB_IMAGE" >/dev/null 2>&1 || { - echo "[ai-gateway] missing local Web image: ${AI_GATEWAY_WEB_IMAGE}; build it first" >&2 - exit 1 - } - - docker tag "$AI_GATEWAY_API_IMAGE" "$api_version_image" - docker tag "$AI_GATEWAY_API_IMAGE" "$api_latest_image" - docker tag "$AI_GATEWAY_WEB_IMAGE" "$web_version_image" - docker tag "$AI_GATEWAY_WEB_IMAGE" "$web_latest_image" - - RELEASE_IMAGES=( - "$api_version_image" - "$api_latest_image" - "$web_version_image" - "$web_latest_image" - ) -} - -push_images() { - local version_tag - local image - - version_tag="$(push_version_tag)" - echo "[ai-gateway] release image tag: ${version_tag}" - echo "[ai-gateway] latest tag will also be pushed" - tag_release_images "$version_tag" - - for image in "${RELEASE_IMAGES[@]}"; do - echo "[ai-gateway] pushing image: ${image}" - if ! docker push "$image"; then - echo "[ai-gateway] failed to push image; login may be required:" >&2 - echo "[ai-gateway] docker login --username= registry.cn-shanghai.aliyuncs.com" >&2 - exit 1 - fi - done -} - deploy() { cd "$PROJECT_ROOT" @@ -227,10 +146,6 @@ deploy() { build_images - if [[ "${AI_GATEWAY_PUSH:-0}" == "1" ]]; then - push_images - fi - echo "[ai-gateway] starting postgres" "${compose[@]}" up -d postgres wait_for_service_healthy postgres @@ -262,15 +177,6 @@ case "$ACTION" in deploy|up) deploy ;; - push) - cd "$PROJECT_ROOT" - build_images - push_images - ;; - push-only) - cd "$PROJECT_ROOT" - push_images - ;; down) cd "$PROJECT_ROOT" "${compose[@]}" down --remove-orphans diff --git a/scripts/deploy-production-release.sh b/scripts/deploy-production-release.sh new file mode 100755 index 0000000..97e05b1 --- /dev/null +++ b/scripts/deploy-production-release.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +production_host=${AI_GATEWAY_PRODUCTION_HOST:-root@110.42.51.33} +remote_helper=${AI_GATEWAY_REMOTE_RELEASE_HELPER:-/usr/local/sbin/easyai-ai-gateway-release} +action=deploy +value= + +usage() { + cat <<'EOF' +Usage: + scripts/deploy-production-release.sh dist/releases/.json + scripts/deploy-production-release.sh --rollback + scripts/deploy-production-release.sh --status + +This command changes production. Run it only after a separate successful +publish command and explicit user confirmation. +EOF +} + +case ${1:-} in + --status) + action=status + [[ $# -eq 1 ]] || { usage >&2; exit 64; } + ;; + --rollback) + action=rollback + [[ $# -eq 2 ]] || { usage >&2; exit 64; } + value=$2 + ;; + -h|--help|'') + usage + [[ ${1:-} == -h || ${1:-} == --help ]] && exit 0 + exit 64 + ;; + *) + [[ $# -eq 1 ]] || { usage >&2; exit 64; } + value=$1 + ;; +esac + +[[ $production_host =~ ^[A-Za-z0-9._-]+@[A-Za-z0-9.-]+$ ]] || { + echo 'AI_GATEWAY_PRODUCTION_HOST must use user@host syntax' >&2 + exit 1 +} +[[ $remote_helper =~ ^/[A-Za-z0-9._/-]+$ ]] || { + echo 'AI_GATEWAY_REMOTE_RELEASE_HELPER must be an absolute safe path' >&2 + exit 1 +} + +if [[ $action == status ]]; then + exec ssh -o BatchMode=yes "$production_host" "$remote_helper status" +fi + +if [[ $action == rollback ]]; then + [[ $value =~ ^[0-9a-f]{40}$ ]] || { + echo 'rollback release must be a full lowercase Git SHA' >&2 + exit 1 + } + ssh -o BatchMode=yes "$production_host" "$remote_helper rollback $value" + echo "production_rollback=PASS release=$value" + exit 0 +fi + +manifest=$(cd "$(dirname "$value")" 2>/dev/null && pwd)/$(basename "$value") +[[ -f $manifest && ! -L $manifest ]] || { + echo 'release manifest must be a regular file' >&2 + exit 1 +} +node "$root/scripts/release-manifest.mjs" validate "$manifest" +source_sha=$(node "$root/scripts/release-manifest.mjs" get "$manifest" sourceSha) +base_sha=$(node "$root/scripts/release-manifest.mjs" get "$manifest" baseReleaseSha) +remote_manifest=/tmp/easyai-ai-gateway-release-$source_sha.json +status_file=$(mktemp) + +cleanup() { + rm -f "$status_file" + ssh -o BatchMode=yes "$production_host" "rm -f $remote_manifest" >/dev/null 2>&1 || true +} +trap cleanup EXIT HUP INT TERM + +if ssh -o BatchMode=yes "$production_host" "$remote_helper status" >"$status_file" 2>/dev/null; then + node "$root/scripts/release-manifest.mjs" validate "$status_file" >/dev/null + current_sha=$(node "$root/scripts/release-manifest.mjs" get "$status_file" sourceSha) + [[ $current_sha == "$base_sha" ]] || { + printf 'production base changed after publish: current=%s manifest_base=%s\n' \ + "$current_sha" "$base_sha" >&2 + exit 1 + } +else + [[ -z $base_sha ]] || { + echo 'production release status is unavailable for a non-bootstrap manifest' >&2 + exit 1 + } +fi + +scp -q "$manifest" "$production_host:$remote_manifest" +ssh -o BatchMode=yes "$production_host" "$remote_helper deploy $remote_manifest" +echo "production_deploy=PASS release=$source_sha" diff --git a/scripts/provision-ci-runner.sh b/scripts/provision-ci-runner.sh deleted file mode 100755 index 91b8db7..0000000 --- a/scripts/provision-ci-runner.sh +++ /dev/null @@ -1,402 +0,0 @@ -#!/bin/sh -set -eu - -ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) -PREFIX=/opt/easyai-gateway-ci -GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL:-https://git.51easyai.com} -RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-ci-unprivileged-v2} -RUNNER_VOLUME=easyai-gateway-ci-v2-data -RUNNER_CONTAINER=easyai-gateway-ci-v2-runner -LEGACY_BUILDER=${AI_GATEWAY_BUILDX_BUILDER:-easyai-gateway-ci} -MIN_FREE_GIB=${CI_RUNNER_MIN_FREE_GIB:-8} -MIN_POST_INSTALL_FREE_GIB=${CI_RUNNER_MIN_POST_INSTALL_FREE_GIB:-4} -RUNNER_CPUS=2 -RUNNER_MEMORY=2g -RUNNER_MEMORY_SWAP=3g -RUNNER_PIDS_LIMIT=1024 - -GITEA_RUNNER_VERSION=2.0.0 -GO_VERSION=1.26.5 -GO_SHA256=5c2c3b16caefa1d968a94c1daca04a7ca301a496d9b086e17ad77bb81393f053 -NODE_VERSION=24.16.0 -NODE_SHA256=d804845d34eddc21dc1092b519d643ef40b1f58ec5dec5c22b1f4bd8fabde6c9 -TRIVY_VERSION=0.70.0 -TRIVY_SHA256=8b4376d5d6befe5c24d503f10ff136d9e0c49f9127a4279fd110b727929a5aa9 -SHELLCHECK_VERSION=0.11.0 -SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198 -COMPOSE_VERSION=5.3.1 -COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959 -PNPM_VERSION=10.18.1 -RUNNER_IMAGE='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a' -JOB_IMAGE='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7' -RUNNER_IMAGE_SOURCE=${CI_RUNNER_IMAGE_SOURCE:-$RUNNER_IMAGE} -NESTED_DOCKER_REGISTRY_MIRROR=${CI_NESTED_DOCKER_REGISTRY_MIRROR:-} -RUNNER_LABEL="easyai-gateway-ci-unprivileged-v2:docker://$JOB_IMAGE" - -fail() { - echo "$*" >&2 - exit 1 -} - -case $MIN_FREE_GIB:$MIN_POST_INSTALL_FREE_GIB in - *[!0-9:]* | :* | *:) - fail "CI runner free-space limits must be whole GiB values" - ;; -esac -[ "$MIN_FREE_GIB" -ge 8 ] || fail "CI_RUNNER_MIN_FREE_GIB cannot be lower than 8" -[ "$MIN_POST_INSTALL_FREE_GIB" -ge 4 ] || \ - fail "CI_RUNNER_MIN_POST_INSTALL_FREE_GIB cannot be lower than 4" - -[ "$(id -u)" -eq 0 ] || fail "run as root" -[ "$(uname -s)-$(uname -m)" = "Linux-x86_64" ] || fail "only Linux x86_64 is supported" -case $RUNNER_IMAGE in - *"/runner:${GITEA_RUNNER_VERSION}-dind-rootless@sha256:"*) ;; - *) fail "RUNNER_IMAGE does not match GITEA_RUNNER_VERSION" ;; -esac -runner_manifest_digest=${RUNNER_IMAGE##*@} -case $RUNNER_IMAGE_SOURCE in - *@"$runner_manifest_digest") ;; - *) fail "CI_RUNNER_IMAGE_SOURCE must use the pinned runner manifest digest" ;; -esac -if [ -n "$NESTED_DOCKER_REGISTRY_MIRROR" ] && \ - ! printf '%s\n' "$NESTED_DOCKER_REGISTRY_MIRROR" | \ - grep -Eq '^https://[A-Za-z0-9.-]+(:[0-9]+)?$'; then - fail "CI_NESTED_DOCKER_REGISTRY_MIRROR must be an HTTPS registry origin" -fi -command -v docker >/dev/null 2>&1 || fail "Docker Engine is required" -docker info >/dev/null 2>&1 || fail "Docker Engine is not reachable" -grep -Fq "\"$RUNNER_LABEL\"" "$ROOT/deploy/ci/act-runner-v2-config.yaml" || \ - fail "runner label does not match the pinned job image" - -# Fail closed before installing anything. Neither the legacy host runner nor an -# earlier v2 unit may poll for work while its replacement is being prepared. -systemctl disable --now easyai-gateway-act-runner.service >/dev/null 2>&1 || true -systemctl disable --now easyai-gateway-ci-v2-runner.service >/dev/null 2>&1 || true -docker rm --force "$RUNNER_CONTAINER" >/dev/null 2>&1 || true -rm -f /etc/systemd/system/easyai-gateway-act-runner.service - -if id easyai-gateway-runner >/dev/null 2>&1; then - if getent group docker >/dev/null 2>&1; then - gpasswd -d easyai-gateway-runner docker >/dev/null 2>&1 || true - fi - if getent group sudo >/dev/null 2>&1; then - gpasswd -d easyai-gateway-runner sudo >/dev/null 2>&1 || true - fi -fi -if id easyai-gateway-ci-v2 >/dev/null 2>&1; then - if getent group docker >/dev/null 2>&1; then - gpasswd -d easyai-gateway-ci-v2 docker >/dev/null 2>&1 || true - fi - if getent group sudo >/dev/null 2>&1; then - gpasswd -d easyai-gateway-ci-v2 sudo >/dev/null 2>&1 || true - fi -fi -for account in easyai-gateway-runner easyai-gateway-ci-v2; do - if id "$account" >/dev/null 2>&1 && \ - id -nG "$account" | tr ' ' '\n' | grep -Eq '^(docker|sudo)$'; then - fail "$account must not belong to a privileged group" - fi -done - -if ! command -v make >/dev/null 2>&1 || ! command -v gcc >/dev/null 2>&1 || \ - ! command -v jq >/dev/null 2>&1 || ! command -v curl >/dev/null 2>&1 || \ - ! command -v xz >/dev/null 2>&1; then - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y --no-install-recommends \ - build-essential ca-certificates curl git jq perl unzip util-linux xz-utils -fi - -docker_root=$(docker info --format '{{.DockerRootDir}}') -[ -n "$docker_root" ] || fail "Docker root directory could not be determined" - -require_free_space() { - minimum_gib=$1 - phase=$2 - available_kib=$(df -Pk "$docker_root" | awk 'NR == 2 { print $4 }') - case $available_kib in - '' | *[!0-9]*) fail "free space could not be determined for $docker_root" ;; - esac - minimum_kib=$((minimum_gib * 1024 * 1024)) - if [ "$available_kib" -lt "$minimum_kib" ]; then - echo "$phase requires at least ${minimum_gib} GiB free on the Docker filesystem" >&2 - return 1 - fi -} - -# This builder belonged only to the retired gateway image-building runner. Cap -# its unused cache before adding the nested daemon; never prune global images, -# volumes, or caches owned by another service on this shared production host. -if docker buildx version >/dev/null 2>&1 && \ - docker buildx inspect "$LEGACY_BUILDER" >/dev/null 2>&1; then - docker buildx prune --builder "$LEGACY_BUILDER" --force --max-used-space 512mb -fi -require_free_space "$MIN_FREE_GIB" "CI runner installation" - -download_verified() { - url=$1 - output=$2 - checksum=$3 - if [ ! -f "$output" ] || ! printf '%s %s\n' "$checksum" "$output" | sha256sum -c - >/dev/null 2>&1; then - rm -f "$output" - curl -fL --retry 5 --retry-delay 2 "$url" -o "$output" - fi - printf '%s %s\n' "$checksum" "$output" | sha256sum -c - -} - -install -d -m 0755 "$PREFIX/bin" "$PREFIX/downloads" "$PREFIX/toolchains" - -download_verified \ - "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \ - "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \ - "$SHELLCHECK_SHA256" -rm -rf "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}" -tar -xJf "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \ - -C "$PREFIX/downloads" -install -m 0755 "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "$PREFIX/bin/shellcheck" - -download_verified \ - "https://github.com/docker/compose/releases/download/v${COMPOSE_VERSION}/docker-compose-linux-x86_64" \ - "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" \ - "$COMPOSE_SHA256" -install -m 0755 "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" "$PREFIX/bin/docker-compose" - -download_verified \ - "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ - "$PREFIX/downloads/go${GO_VERSION}.linux-amd64.tar.gz" \ - "$GO_SHA256" -rm -rf "$PREFIX/toolchains/go" -tar -xzf "$PREFIX/downloads/go${GO_VERSION}.linux-amd64.tar.gz" -C "$PREFIX/toolchains" - -download_verified \ - "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" \ - "$PREFIX/downloads/node-v${NODE_VERSION}-linux-x64.tar.xz" \ - "$NODE_SHA256" -rm -rf "$PREFIX/toolchains/node" -mkdir -p "$PREFIX/toolchains/node" -tar -xJf "$PREFIX/downloads/node-v${NODE_VERSION}-linux-x64.tar.xz" \ - -C "$PREFIX/toolchains/node" --strip-components=1 -PATH="$PREFIX/toolchains/node/bin:$PATH" \ - "$PREFIX/toolchains/node/bin/npm" install --global --prefix "$PREFIX/toolchains/node" \ - --ignore-scripts "pnpm@${PNPM_VERSION}" - -download_verified \ - "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" \ - "$PREFIX/downloads/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" \ - "$TRIVY_SHA256" -tar -xzf "$PREFIX/downloads/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" -C "$PREFIX/bin" trivy -chmod 0755 "$PREFIX/bin/trivy" -GOBIN="$PREFIX/bin" GOPROXY="${GO_MODULE_PROXY:-https://goproxy.cn,direct}" \ - "$PREFIX/toolchains/go/bin/go" install golang.org/x/vuln/cmd/govulncheck@v1.6.0 - -require_free_space "$MIN_FREE_GIB" "runner image pull" -docker pull "$RUNNER_IMAGE_SOURCE" -runner_runtime_image=$(docker image inspect --format '{{.Id}}' "$RUNNER_IMAGE_SOURCE") -printf '%s\n' "$runner_runtime_image" | grep -Eq '^sha256:[0-9a-f]{64}$' || \ - fail "pinned runner image did not resolve to an immutable local image ID" -chown -R root:root "$PREFIX" -chmod -R go-w "$PREFIX" - -require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed runner image pull" -docker volume create "$RUNNER_VOLUME" >/dev/null - -# The registered runner state and the nested rootless daemon cache share one -# dedicated volume. Persisting the pinned job image lets validation finish -# before registration and avoids racing the first queued workflow for layers. -docker run --rm --pull=never --user 0:0 \ - --volume "$RUNNER_VOLUME:/data" \ - --entrypoint /bin/sh "$runner_runtime_image" \ - -ec 'chown 1000:1000 /data && chmod 0700 /data' - -install -d -m 0755 /etc/easyai-gateway-ci-v2 -if [ -n "$NESTED_DOCKER_REGISTRY_MIRROR" ]; then - printf '{"data-root":"/data/docker","registry-mirrors":["%s"]}\n' "$NESTED_DOCKER_REGISTRY_MIRROR" \ - > /etc/easyai-gateway-ci-v2/daemon.json -else - printf '{"data-root":"/data/docker"}\n' > /etc/easyai-gateway-ci-v2/daemon.json -fi -chmod 0644 /etc/easyai-gateway-ci-v2/daemon.json - -# Docker's official rootless DinD baseline is --privileged. Gitea's example -# additionally names an AppArmor profile, but Docker does not create that host -# profile. Enable it only when this daemon proves it can start the pinned image -# with the profile; otherwise leave the option empty. -RUNNER_SECURITY_OPTIONS= -if docker run --rm --pull=never --privileged \ - --security-opt apparmor=rootlesskit \ - --entrypoint /bin/true "$runner_runtime_image" >/dev/null 2>&1; then - RUNNER_SECURITY_OPTIONS="--security-opt=apparmor=rootlesskit" -elif [ -r /proc/sys/kernel/apparmor_restrict_unprivileged_userns ] && \ - [ "$(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns)" = "1" ]; then - fail "restricted AppArmor user namespaces require the rootlesskit profile (install it with the host Docker packages)" -fi - -# Exercise the actual daemon startup before registration. This probe has no -# token, state volume, host bind mount, or socket mount. -PROBE_CONTAINER="easyai-gateway-ci-v2-probe-$$" -cleanup_probe() { - docker rm --force "$PROBE_CONTAINER" >/dev/null 2>&1 || true -} -trap 'cleanup_probe' EXIT -trap 'cleanup_probe; exit 129' HUP -trap 'cleanup_probe; exit 130' INT -trap 'cleanup_probe; exit 143' TERM -if [ -n "$RUNNER_SECURITY_OPTIONS" ]; then - set -- "$RUNNER_SECURITY_OPTIONS" -else - set -- -fi -docker run --detach --rm --pull=never \ - --name "$PROBE_CONTAINER" --privileged \ - --cpus "$RUNNER_CPUS" --memory "$RUNNER_MEMORY" \ - --memory-swap "$RUNNER_MEMORY_SWAP" --pids-limit "$RUNNER_PIDS_LIMIT" "$@" \ - --volume "$RUNNER_VOLUME:/data" \ - --volume /etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro \ - --volume "$PREFIX:$PREFIX:ro" \ - --env DOCKERD_ROOTLESS_ROOTLESSKIT_NET=slirp4netns \ - --env DOCKERD_ROOTLESS_ROOTLESSKIT_MTU=65520 \ - --env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns \ - --entrypoint dockerd-entrypoint.sh "$runner_runtime_image" >/dev/null -attempt=0 -until docker exec "$PROBE_CONTAINER" docker info >/dev/null 2>&1; do - attempt=$((attempt + 1)) - if [ "$attempt" -ge 30 ] || \ - [ "$(docker inspect --format '{{.State.Running}}' "$PROBE_CONTAINER" 2>/dev/null || true)" != "true" ]; then - docker logs "$PROBE_CONTAINER" 2>&1 || true - fail "pinned rootless Docker-in-Docker probe did not become ready" - fi - sleep 2 -done -[ "$(docker exec "$PROBE_CONTAINER" id -u)" = "1000" ] || \ - fail "rootless Docker-in-Docker probe is not uid 1000" -[ "$(docker inspect --format '{{.HostConfig.Memory}}:{{.HostConfig.MemorySwap}}:{{.HostConfig.NanoCpus}}:{{.HostConfig.PidsLimit}}' "$PROBE_CONTAINER")" = \ - '2147483648:3221225472:2000000000:1024' ] || \ - fail "rootless Docker-in-Docker probe does not have the required aggregate resource limits" -docker exec "$PROBE_CONTAINER" docker info --format '{{range .SecurityOptions}}{{println .}}{{end}}' | \ - grep -Fq 'name=rootless' || fail "rootless Docker-in-Docker probe is not rootless" -[ "$(docker exec "$PROBE_CONTAINER" docker info --format '{{.DockerRootDir}}')" = /data/docker ] || \ - fail "rootless Docker-in-Docker probe is not using its persistent dedicated data root" - -# Complete every slow and capability-sensitive validation before registration; -# otherwise a queued workflow can race the installer for the same image layers. -docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE" -outer_sentinel_pid=$(docker exec "$PROBE_CONTAINER" /bin/sh -c \ - 'sleep 300 >/dev/null 2>&1 & printf "%s\n" "$!"') -case $outer_sentinel_pid in - '' | *[!0-9]*) fail "could not create the outer PID namespace sentinel" ;; -esac -if ! docker exec "$PROBE_CONTAINER" docker run --rm --pull=never \ - --pid=host --entrypoint /bin/sh "$JOB_IMAGE" \ - -ec 'test ! -e "/proc/$1"' sh "$outer_sentinel_pid"; then - docker exec "$PROBE_CONTAINER" kill "$outer_sentinel_pid" >/dev/null 2>&1 || true - fail "nested --pid=host can see the outer runner PID namespace" -fi -docker exec "$PROBE_CONTAINER" kill "$outer_sentinel_pid" >/dev/null 2>&1 || true -docker exec "$PROBE_CONTAINER" docker run --rm --pull=never \ - --volume "$PREFIX:$PREFIX:ro" \ - --entrypoint /bin/sh "$JOB_IMAGE" -ec ' - export PATH=/opt/easyai-gateway-ci/bin:/opt/easyai-gateway-ci/toolchains/go/bin:/opt/easyai-gateway-ci/toolchains/node/bin:$PATH - go version - node --version - pnpm --version - docker-compose version - shellcheck --version - trivy --version - govulncheck -version - ' -cleanup_probe -trap - EXIT HUP INT TERM - -install -m 0644 "$ROOT/deploy/ci/act-runner-v2-config.yaml" /etc/easyai-gateway-ci-v2/config.yaml -umask 077 -printf 'RUNNER_SECURITY_OPTIONS="%s"\nRUNNER_IMAGE_REF="%s"\n' \ - "$RUNNER_SECURITY_OPTIONS" "$runner_runtime_image" \ - > /etc/easyai-gateway-ci-v2/runner.env -install -m 0644 "$ROOT/deploy/ci/easyai-gateway-ci-v2-runner.service" \ - /etc/systemd/system/easyai-gateway-ci-v2-runner.service - -runner_is_registered() { - docker run --rm --pull=never \ - --volume "$RUNNER_VOLUME:/data" \ - --entrypoint /bin/sh "$runner_runtime_image" \ - -ec 'test -s /data/.runner' -} - -if ! runner_is_registered; then - [ -n "${RUNNER_REGISTRATION_TOKEN:-}" ] || \ - fail "RUNNER_REGISTRATION_TOKEN is required for first registration" - docker rm --force easyai-gateway-ci-v2-register >/dev/null 2>&1 || true - printf '%s\n%s\n%s\n' "$GITEA_INSTANCE_URL" "$RUNNER_REGISTRATION_TOKEN" "$RUNNER_NAME" | \ - docker run --rm --interactive --pull=never \ - --name easyai-gateway-ci-v2-register \ - --volume "$RUNNER_VOLUME:/data" \ - --volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro \ - --env HOME=/data \ - --entrypoint /usr/local/bin/gitea-runner \ - "$runner_runtime_image" --config /config.yaml register -fi -unset RUNNER_REGISTRATION_TOKEN -docker run --rm --pull=never \ - --volume "$RUNNER_VOLUME:/data" \ - --entrypoint /bin/sh "$runner_runtime_image" \ - -ec 'test -s /data/.runner && chmod 0600 /data/.runner' - -RUNNER_VALIDATED=0 -cleanup_unvalidated_runner() { - if [ "$RUNNER_VALIDATED" -ne 1 ]; then - systemctl disable --now easyai-gateway-ci-v2-runner.service >/dev/null 2>&1 || true - docker rm --force "$RUNNER_CONTAINER" >/dev/null 2>&1 || true - fi -} -trap 'cleanup_unvalidated_runner' EXIT -trap 'exit 129' HUP -trap 'exit 130' INT -trap 'exit 143' TERM -systemctl daemon-reload -systemctl enable --now easyai-gateway-ci-v2-runner.service - -attempt=0 -until docker exec "$RUNNER_CONTAINER" docker info >/dev/null 2>&1; do - attempt=$((attempt + 1)) - if [ "$attempt" -ge 30 ]; then - systemctl --no-pager --full status easyai-gateway-ci-v2-runner.service || true - docker logs "$RUNNER_CONTAINER" 2>&1 || true - fail "rootless Docker daemon did not become ready" - fi - sleep 2 -done - -attempt=0 -until docker logs "$RUNNER_CONTAINER" 2>&1 | grep -Fq 'declare successfully'; do - attempt=$((attempt + 1)) - if [ "$attempt" -ge 30 ] || \ - [ "$(docker inspect --format '{{.State.Running}}' "$RUNNER_CONTAINER" 2>/dev/null || true)" != "true" ]; then - docker logs "$RUNNER_CONTAINER" 2>&1 || true - fail "Gitea runner did not declare itself successfully" - fi - sleep 2 -done - -[ "$(docker inspect --format '{{.Config.User}}' "$RUNNER_CONTAINER")" = "rootless" ] || \ - fail "outer runner container is not using the rootless image user" -[ "$(docker inspect --format '{{.HostConfig.Privileged}}' "$RUNNER_CONTAINER")" = "true" ] || \ - fail "outer rootless DinD container is not privileged" -[ "$(docker inspect --format '{{.HostConfig.Memory}}:{{.HostConfig.MemorySwap}}:{{.HostConfig.NanoCpus}}:{{.HostConfig.PidsLimit}}' "$RUNNER_CONTAINER")" = \ - '2147483648:3221225472:2000000000:1024' ] || \ - fail "outer rootless DinD container does not have the required aggregate resource limits" -if docker inspect --format '{{range .Mounts}}{{println .Source}}{{end}}' "$RUNNER_CONTAINER" | \ - grep -Eq '/var/run/docker\.sock|/run/docker\.sock'; then - fail "host Docker socket was mounted into the runner" -fi -docker exec "$RUNNER_CONTAINER" docker info --format '{{range .SecurityOptions}}{{println .}}{{end}}' | \ - grep -Fq 'name=rootless' || fail "nested Docker daemon is not rootless" -[ "$(docker exec "$RUNNER_CONTAINER" docker info --format '{{.DockerRootDir}}')" = /data/docker ] || \ - fail "nested Docker daemon is not using its persistent dedicated data root" -docker exec "$RUNNER_CONTAINER" docker image inspect "$JOB_IMAGE" >/dev/null || \ - fail "validated pinned job image is unavailable after runner activation" -if ! require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed CI runner installation"; then - fail "CI runner was stopped because the post-install disk reserve was not met" -fi -systemctl --no-pager --full status easyai-gateway-ci-v2-runner.service -RUNNER_VALIDATED=1 -trap - EXIT HUP INT TERM diff --git a/scripts/publish-release-images.sh b/scripts/publish-release-images.sh new file mode 100755 index 0000000..0d1bbf7 --- /dev/null +++ b/scripts/publish-release-images.sh @@ -0,0 +1,375 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "$root" + +components=auto +production_host=${AI_GATEWAY_PRODUCTION_HOST:-root@110.42.51.33} +remote_helper=${AI_GATEWAY_REMOTE_RELEASE_HELPER:-/usr/local/sbin/easyai-ai-gateway-release} +registry=${AI_GATEWAY_IMAGE_REGISTRY:-registry.cn-shanghai.aliyuncs.com/easyaigc} +platform=linux/amd64 +platform_probe_image=${AI_GATEWAY_PLATFORM_PROBE_IMAGE:-docker.m.daocloud.io/library/alpine:3.22} +status_file_override=${AI_GATEWAY_RELEASE_STATUS_FILE:-} + +usage() { + cat <<'EOF' +Usage: + scripts/publish-release-images.sh [--components auto|api|web|all] + +This command only builds, smoke-tests, and pushes immutable images. It never +changes production. The generated manifest must be deployed separately with +scripts/deploy-production-release.sh after explicit confirmation. + +Environment: + AI_GATEWAY_PRODUCTION_HOST SSH target used for read-only release status + AI_GATEWAY_REMOTE_RELEASE_HELPER Fixed status/deploy helper on the server + AI_GATEWAY_IMAGE_REGISTRY Registry namespace + AI_GATEWAY_RELEASE_STATUS_FILE Local status manifest for offline/testing use +EOF +} + +while [[ $# -gt 0 ]]; do + case $1 in + --components) + [[ $# -ge 2 ]] || { echo 'missing value for --components' >&2; exit 64; } + components=$2 + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + printf 'unknown argument: %s\n' "$1" >&2 + usage >&2 + exit 64 + ;; + esac +done + +case $components in + auto|api|web|all) ;; + *) echo '--components must be auto, api, web, or all' >&2; exit 64 ;; +esac +[[ $production_host =~ ^[A-Za-z0-9._-]+@[A-Za-z0-9.-]+$ ]] || { + echo 'AI_GATEWAY_PRODUCTION_HOST must use user@host syntax' >&2 + exit 1 +} +[[ $remote_helper =~ ^/[A-Za-z0-9._/-]+$ ]] || { + echo 'AI_GATEWAY_REMOTE_RELEASE_HELPER must be an absolute safe path' >&2 + exit 1 +} +[[ $registry =~ ^[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._-]+)+$ ]] || { + echo 'AI_GATEWAY_IMAGE_REGISTRY has an invalid format' >&2 + exit 1 +} +[[ $platform_probe_image =~ ^[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._-]+)*(:[A-Za-z0-9._-]+|@sha256:[0-9a-f]{64})$ ]] || { + echo 'AI_GATEWAY_PLATFORM_PROBE_IMAGE has an invalid format' >&2 + exit 1 +} + +fail() { + echo "release_publish=FAIL $*" >&2 + exit 1 +} + +for command in git node docker curl; do + command -v "$command" >/dev/null 2>&1 || fail "$command is required" +done +docker info >/dev/null 2>&1 || fail 'Docker Engine is not reachable' +docker buildx version >/dev/null 2>&1 || fail 'Docker Buildx is required' +docker compose version >/dev/null 2>&1 || fail 'Docker Compose v2 is required' + +git fetch --quiet origin main +preflight_file=$(mktemp) +node scripts/release-preflight.mjs >"$preflight_file" +source_sha=$(node -e \ + 'const value=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8")); process.stdout.write(value.sourceSha)' \ + "$preflight_file") + +status_file=$(mktemp) +selection_file=$(mktemp) +smoke_project=easyai-release-smoke-${source_sha:0:12} +smoke_started=0 + +cleanup() { + local status=$? + trap - EXIT HUP INT TERM + if [[ $smoke_started -eq 1 ]]; then + COMPOSE_PROJECT_NAME=$smoke_project \ + docker compose -p "$smoke_project" -f docker-compose.yml down -v --remove-orphans \ + >/dev/null 2>&1 || true + fi + rm -f "$status_file" "$selection_file" "$preflight_file" + exit "$status" +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +has_status=0 +if [[ -n $status_file_override ]]; then + [[ -f $status_file_override ]] || fail 'AI_GATEWAY_RELEASE_STATUS_FILE does not exist' + cp "$status_file_override" "$status_file" + has_status=1 +elif ssh -o BatchMode=yes -o ConnectTimeout=8 "$production_host" \ + "$remote_helper status" >"$status_file" 2>/dev/null; then + has_status=1 +fi + +base_sha= +current_api_image= +current_web_image= +if [[ $has_status -eq 1 ]]; then + node scripts/release-manifest.mjs validate "$status_file" >/dev/null + base_sha=$(node scripts/release-manifest.mjs get "$status_file" sourceSha) + current_api_image=$(node scripts/release-manifest.mjs get "$status_file" images.api) + current_web_image=$(node scripts/release-manifest.mjs get "$status_file" images.web) +fi + +if [[ $components != all && $has_status -ne 1 ]]; then + fail 'auto/api/web publishing requires a readable production release; use --components all for the first release' +fi + +required_components=all +migrations_changed=true +base_verifiable=0 +if [[ $has_status -eq 1 ]]; then + if ! git cat-file -e "$base_sha^{commit}" 2>/dev/null; then + git fetch --quiet origin "$base_sha" >/dev/null 2>&1 || true + fi + if git cat-file -e "$base_sha^{commit}" 2>/dev/null; then + git merge-base --is-ancestor "$base_sha" "$source_sha" || \ + fail 'the production release is not an ancestor of the source commit' + base_verifiable=1 + node scripts/release-components.mjs "$base_sha" "$source_sha" >"$selection_file" + required_components=$(node -e \ + 'const value=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8")); process.stdout.write(value.components)' \ + "$selection_file") + migrations_changed=$(node -e \ + 'const value=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8")); process.stdout.write(String(value.migrationsChanged))' \ + "$selection_file") + elif [[ $components != all ]]; then + fail 'the production source SHA cannot be verified; rerun with --components all' + else + echo '[release] production source SHA is unavailable; conservatively building all components and running migrations' >&2 + fi +fi + +if [[ $components == auto ]]; then + components=$required_components +fi +if [[ $components == none ]]; then + echo 'release_publish=SKIP reason=no_runtime_changes' + exit 0 +fi +if [[ $required_components == all && $components != all ]]; then + fail "the source change requires both images; requested $components" +fi +if [[ $required_components == api && $components == web ]]; then + fail 'the source change requires the API image' +fi +if [[ $required_components == web && $components == api ]]; then + fail 'the source change requires the Web image' +fi +if [[ $migrations_changed == true && $components != api && $components != all ]]; then + fail 'migration changes require the API image' +fi + +api_repository=$registry/ai-gateway +web_repository=$registry/ai-gateway-web +api_tag=$api_repository:$source_sha +web_tag=$web_repository:$source_sha + +remote_tag_must_not_exist() { + local image=$1 + local output status + set +e + output=$(docker buildx imagetools inspect "$image" 2>&1) + status=$? + set -e + if [[ $status -eq 0 ]]; then + fail "immutable image tag already exists: $image" + fi + if ! grep -Eqi 'not found|manifest unknown|does not exist' <<<"$output"; then + fail "cannot verify registry login and repository access for $image" + fi +} + +case $components in + api) remote_tag_must_not_exist "$api_tag" ;; + web) remote_tag_must_not_exist "$web_tag" ;; + all) + remote_tag_must_not_exist "$api_tag" + remote_tag_must_not_exist "$web_tag" + ;; +esac + +docker run --rm --platform "$platform" "$platform_probe_image" true >/dev/null 2>&1 || \ + fail "Docker cannot execute $platform containers" + +if [[ $base_verifiable -eq 1 ]]; then + node scripts/ci-validate-migrations.mjs "$base_sha" +fi + +echo '[release] running fast Go tests without database integration variables' +( + cd apps/api + env -u AI_GATEWAY_TEST_DATABASE_URL go test ./... -count=1 +) + +api_image=$current_api_image +web_image=$current_web_image + +build_image() { + local target=$1 + local image=$2 + local -a build_args=() + case $target in + api) + build_args=( + --build-arg "GOPROXY=${AI_GATEWAY_GO_PROXY:-https://goproxy.cn,direct}" + --build-arg "GO_BUILD_IMAGE=${AI_GATEWAY_GO_BUILD_IMAGE:-golang:1.26.3-alpine}" + --build-arg "API_RUNTIME_IMAGE=${AI_GATEWAY_API_RUNTIME_IMAGE:-alpine:3.22}" + ) + ;; + web) + build_args=( + --build-arg "VITE_GATEWAY_API_BASE_URL=${AI_GATEWAY_WEB_API_BASE_URL:-/gateway-api}" + --build-arg "NODE_BUILD_IMAGE=${AI_GATEWAY_NODE_BUILD_IMAGE:-node:22-alpine}" + --build-arg "WEB_RUNTIME_IMAGE=${AI_GATEWAY_WEB_RUNTIME_IMAGE:-nginx:1.27-alpine}" + --build-arg "NPM_CONFIG_REGISTRY=${AI_GATEWAY_NPM_REGISTRY:-https://registry.npmjs.org}" + --build-arg "VITE_BASE_PATH=${AI_GATEWAY_WEB_BASE_PATH:-/}" + ) + ;; + esac + echo "[release] building $target for $platform: $image" + docker buildx build \ + --platform "$platform" \ + --file Dockerfile \ + --target "$target" \ + --label "org.opencontainers.image.revision=$source_sha" \ + --label 'org.opencontainers.image.source=https://git.51easyai.com/BCAI/easyai-ai-gateway' \ + --tag "$image" \ + --load \ + "${build_args[@]}" \ + . + [[ $(docker image inspect "$image" --format '{{.Os}}/{{.Architecture}}') == "$platform" ]] || \ + fail "$image is not $platform" + [[ $(docker image inspect "$image" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}') == "$source_sha" ]] || \ + fail "$image is missing the source revision label" +} + +case $components in + api) + build_image api "$api_tag" + api_image=$api_tag + ;; + web) + build_image web "$web_tag" + web_image=$web_tag + ;; + all) + build_image api "$api_tag" + build_image web "$web_tag" + api_image=$api_tag + web_image=$web_tag + ;; +esac + +for image in "$api_image" "$web_image"; do + docker image inspect "$image" >/dev/null 2>&1 || docker pull --platform "$platform" "$image" +done +build_completed_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ') + +export COMPOSE_PROJECT_NAME=$smoke_project +export AI_GATEWAY_PLATFORM=$platform +export AI_GATEWAY_API_IMAGE=$api_image +export AI_GATEWAY_WEB_IMAGE=$web_image +export AI_GATEWAY_API_PORT=0 +export AI_GATEWAY_WEB_PORT=0 +export AI_GATEWAY_DB_PORT=0 +export AI_GATEWAY_COMPOSE_APP_ENV=test +export AI_GATEWAY_COMPOSE_IDENTITY_MODE=hybrid +export AI_GATEWAY_COMPOSE_DATABASE_NAME=easyai_ai_gateway +export AI_GATEWAY_COMPOSE_PG_USER=easyai +export AI_GATEWAY_COMPOSE_PG_PASSWORD=release-smoke-postgres +export AI_GATEWAY_COMPOSE_DATABASE_URL='postgresql://easyai:release-smoke-postgres@postgres:5432/easyai_ai_gateway?sslmode=disable' +export CONFIG_JWT_SECRET=release-smoke-$source_sha + +smoke_started=1 +docker compose -p "$smoke_project" -f docker-compose.yml up -d --pull never postgres +for _ in $(seq 1 60); do + postgres_id=$(docker compose -p "$smoke_project" -f docker-compose.yml ps -q postgres) + postgres_health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$postgres_id" 2>/dev/null || true) + [[ $postgres_health == healthy ]] && break + sleep 1 +done +[[ ${postgres_health:-} == healthy ]] || fail 'ephemeral PostgreSQL did not become healthy' + +docker compose -p "$smoke_project" -f docker-compose.yml run --rm --no-deps migrator +docker compose -p "$smoke_project" -f docker-compose.yml up -d --pull never --no-build api web +api_port=$(docker compose -p "$smoke_project" -f docker-compose.yml port api 8088 | tail -n 1) +web_port=$(docker compose -p "$smoke_project" -f docker-compose.yml port web 80 | tail -n 1) +api_port=${api_port##*:} +web_port=${web_port##*:} +[[ $api_port =~ ^[0-9]+$ && $web_port =~ ^[0-9]+$ ]] || fail 'could not resolve smoke ports' + +RELEASE_SMOKE_BASE_URL=http://127.0.0.1:$api_port \ +RELEASE_SMOKE_WEB_URL=http://127.0.0.1:$web_port \ +RELEASE_SMOKE_COMPOSE_PROJECT=$smoke_project \ +RELEASE_SMOKE_COMPOSE_FILE=$root/docker-compose.yml \ +RELEASE_SMOKE_NONCE=${source_sha:0:12} \ +RELEASE_SMOKE_DATABASE_USER=$AI_GATEWAY_COMPOSE_PG_USER \ +RELEASE_SMOKE_DATABASE_NAME=$AI_GATEWAY_COMPOSE_DATABASE_NAME \ + node scripts/api-release-smoke.mjs +smoke_completed_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ') + +push_and_resolve() { + local image=$1 + local repository=$2 + local resolved + docker push "$image" >&2 + resolved=$(docker image inspect "$image" --format '{{range .RepoDigests}}{{println .}}{{end}}' | \ + awk -v prefix="$repository@sha256:" 'index($0, prefix) == 1 { print; exit }') + [[ $resolved =~ ^[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._-]+)+@sha256:[0-9a-f]{64}$ ]] || \ + fail "could not resolve registry digest for $image" + printf '%s\n' "$resolved" +} + +case $components in + api) + api_image=$(push_and_resolve "$api_tag" "$api_repository") + component_list=api + ;; + web) + web_image=$(push_and_resolve "$web_tag" "$web_repository") + component_list=web + ;; + all) + api_image=$(push_and_resolve "$api_tag" "$api_repository") + web_image=$(push_and_resolve "$web_tag" "$web_repository") + component_list=api,web + ;; +esac + +mkdir -p dist/releases +manifest=$root/dist/releases/$source_sha.json +[[ ! -e $manifest ]] || fail "release manifest already exists: $manifest" +RELEASE_SOURCE_SHA=$source_sha \ +RELEASE_BASE_SHA=$base_sha \ +RELEASE_COMPONENTS=$component_list \ +RELEASE_MIGRATIONS_CHANGED=$migrations_changed \ +RELEASE_API_IMAGE=$api_image \ +RELEASE_WEB_IMAGE=$web_image \ +RELEASE_BUILD_COMPLETED_AT=$build_completed_at \ +RELEASE_SMOKE_COMPLETED_AT=$smoke_completed_at \ + node scripts/release-manifest.mjs create "$manifest" + +echo "release_publish=PASS release=$source_sha components=$component_list" +echo "release_manifest=$manifest" +echo "api_image=$api_image" +echo "web_image=$web_image" +echo 'production_changed=false' diff --git a/scripts/release-components.mjs b/scripts/release-components.mjs new file mode 100755 index 0000000..6c80cf5 --- /dev/null +++ b/scripts/release-components.mjs @@ -0,0 +1,157 @@ +#!/usr/bin/env node + +import { execFileSync } from 'node:child_process' + +function fail(message) { + console.error(`release_components=FAIL ${message}`) + process.exit(1) +} + +const base = process.argv[2] || '' +const head = process.argv[3] || 'HEAD' + +if (!/^[0-9a-f]{40}$/.test(base)) { + fail('base must be a full lowercase Git SHA') +} + +function git(args) { + try { + return execFileSync('git', args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trim() + } catch (error) { + const stderr = error.stderr?.toString().trim() + fail(stderr ? `git_error=${JSON.stringify(stderr)}` : 'git command failed') + } +} + +const resolvedBase = git(['rev-parse', '--verify', `${base}^{commit}`]) +const resolvedHead = git(['rev-parse', '--verify', `${head}^{commit}`]) +if (resolvedBase !== base) fail('base did not resolve exactly') +if (!/^[0-9a-f]{40}$/.test(resolvedHead)) fail('head did not resolve to a commit') + +try { + execFileSync('git', ['merge-base', '--is-ancestor', base, resolvedHead], { + stdio: 'ignore', + }) +} catch { + fail('base is not an ancestor of head') +} + +const changed = git([ + 'diff', + '--name-only', + '--no-renames', + `${base}..${resolvedHead}`, +]).split('\n').filter(Boolean) + +const apiPatterns = [ + /^apps\/api\//, + /^go\.work(?:\.sum)?$/, +] +const webPatterns = [ + /^apps\/web\//, + /^packages\/contracts\//, + /^pnpm-lock\.yaml$/, + /^pnpm-workspace\.yaml$/, + /^nx\.json$/, + /^package\.json$/, + /^docker\/nginx\.conf$/, +] +const sharedRuntimePatterns = [ + /^\.dockerignore$/, + /^docker-compose(?:\.[^.]+)?\.ya?ml$/, +] + +function dockerfileSections(ref) { + let contents + try { + contents = execFileSync('git', ['show', `${ref}:Dockerfile`], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }) + } catch { + return null + } + const sections = new Map([['global', []]]) + let section = 'global' + for (const line of contents.split('\n')) { + const match = line.match(/^FROM\s+.+?\s+AS\s+([A-Za-z0-9._-]+)(?:\s|$)/i) + if (match) section = match[1].toLowerCase() + if (!sections.has(section)) sections.set(section, []) + sections.get(section).push(line) + } + return sections +} + +function sectionChanged(before, after, name) { + return (before.get(name) || []).join('\n') !== (after.get(name) || []).join('\n') +} + +function classifyDockerfile() { + const before = dockerfileSections(base) + const after = dockerfileSections(resolvedHead) + if (!before || !after) return { apiChanged: true, webChanged: true } + const knownApi = new Set(['api-builder', 'api']) + const knownWeb = new Set(['web-builder', 'web']) + let apiChanged = false + let webChanged = false + + const beforeGlobal = before.get('global') || [] + const afterGlobal = after.get('global') || [] + const globalLines = (lines, pattern) => lines.filter((line) => pattern.test(line)).join('\n') + const apiGlobal = /\b(?:GO_VERSION|GO_BUILD_IMAGE|API_RUNTIME_IMAGE)\b/ + const webGlobal = /\b(?:NODE_VERSION|NODE_BUILD_IMAGE|WEB_RUNTIME_IMAGE)\b/ + if (globalLines(beforeGlobal, apiGlobal) !== globalLines(afterGlobal, apiGlobal)) apiChanged = true + if (globalLines(beforeGlobal, webGlobal) !== globalLines(afterGlobal, webGlobal)) webChanged = true + const sharedGlobal = (lines) => lines.filter((line) => !apiGlobal.test(line) && !webGlobal.test(line)).join('\n') + if (sharedGlobal(beforeGlobal) !== sharedGlobal(afterGlobal)) { + apiChanged = true + webChanged = true + } + + const sectionNames = new Set([...before.keys(), ...after.keys()]) + sectionNames.delete('global') + for (const name of sectionNames) { + if (!sectionChanged(before, after, name)) continue + if (knownApi.has(name)) apiChanged = true + else if (knownWeb.has(name)) webChanged = true + else { + apiChanged = true + webChanged = true + } + } + return { apiChanged, webChanged } +} + +let api = false +let web = false +let migrationsChanged = false + +if (changed.includes('Dockerfile')) { + const dockerfile = classifyDockerfile() + api = dockerfile.apiChanged + web = dockerfile.webChanged +} + +for (const file of changed) { + if (file === 'Dockerfile') continue + if (/^apps\/api\/migrations\//.test(file)) migrationsChanged = true + if (sharedRuntimePatterns.some((pattern) => pattern.test(file))) { + api = true + web = true + continue + } + if (apiPatterns.some((pattern) => pattern.test(file))) api = true + if (webPatterns.some((pattern) => pattern.test(file))) web = true +} + +const components = api && web ? 'all' : api ? 'api' : web ? 'web' : 'none' +console.log(JSON.stringify({ + base, + head: resolvedHead, + components, + migrationsChanged, + changedFiles: changed, +})) diff --git a/scripts/release-manifest.mjs b/scripts/release-manifest.mjs new file mode 100755 index 0000000..6dc8286 --- /dev/null +++ b/scripts/release-manifest.mjs @@ -0,0 +1,227 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto' +import { readFileSync, writeFileSync } from 'node:fs' + +const shaPattern = /^[0-9a-f]{40}$/ +const digestImagePattern = /^[A-Za-z0-9.-]+(?::[0-9]+)?(?:\/[A-Za-z0-9._-]+)+@sha256:[0-9a-f]{64}$/ +const allowedComponents = new Set(['api', 'web']) +const smokeChecks = [ + 'health', + 'readiness', + 'openapi', + 'register', + 'login', + 'apiKey', + 'models', + 'chatCompletions', + 'responses', + 'images', + 'web', +] + +function requireExactKeys(value, keys, path) { + const actual = Object.keys(value).sort() + const expected = [...keys].sort() + if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { + fail(`${path} has unexpected or missing fields`) + } +} + +function releaseIntegrity(manifest) { + const payload = structuredClone(manifest) + delete payload.integrity + delete payload.deployment + return createHash('sha256').update(JSON.stringify(payload)).digest('hex') +} + +function fail(message) { + console.error(`release_manifest=FAIL ${message}`) + process.exit(1) +} + +function load(path) { + try { + return JSON.parse(readFileSync(path, 'utf8')) + } catch (error) { + fail(`cannot read JSON manifest: ${error.message}`) + } +} + +function validate(manifest) { + if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) { + fail('manifest root must be an object') + } + const rootKeys = [ + 'schemaVersion', 'releaseId', 'sourceSha', 'baseReleaseSha', 'sourceRef', + 'createdAt', 'buildCompletedAt', 'targetPlatform', 'components', + 'migrationsChanged', 'images', 'smoke', 'integrity', + ] + if (manifest.deployment !== undefined) rootKeys.push('deployment') + requireExactKeys(manifest, rootKeys, 'manifest') + if (manifest.schemaVersion !== 1) fail('schemaVersion must be 1') + if (!shaPattern.test(manifest.sourceSha || '')) fail('sourceSha must be a full lowercase Git SHA') + if (manifest.releaseId !== manifest.sourceSha) fail('releaseId must equal sourceSha') + if (manifest.sourceRef !== 'refs/heads/main') fail('sourceRef must be refs/heads/main') + if (manifest.baseReleaseSha !== null && !shaPattern.test(manifest.baseReleaseSha || '')) { + fail('baseReleaseSha must be null or a full lowercase Git SHA') + } + if (manifest.baseReleaseSha === manifest.sourceSha) fail('baseReleaseSha must differ from sourceSha') + if (manifest.targetPlatform !== 'linux/amd64') fail('targetPlatform must be linux/amd64') + if (!Number.isFinite(Date.parse(manifest.createdAt))) fail('createdAt must be an ISO timestamp') + if (!Number.isFinite(Date.parse(manifest.buildCompletedAt))) { + fail('buildCompletedAt must be an ISO timestamp') + } + if (!Array.isArray(manifest.components) || manifest.components.length < 1 || manifest.components.length > 2) { + fail('components must contain api, web, or both') + } + const uniqueComponents = [...new Set(manifest.components)] + if (uniqueComponents.length !== manifest.components.length || + uniqueComponents.some((component) => !allowedComponents.has(component))) { + fail('components contains an invalid or duplicate value') + } + if (typeof manifest.migrationsChanged !== 'boolean') fail('migrationsChanged must be boolean') + if (manifest.migrationsChanged && !manifest.components.includes('api')) { + fail('migration changes require the api component') + } + if (!manifest.images || typeof manifest.images !== 'object') fail('images must be an object') + requireExactKeys(manifest.images, ['api', 'web'], 'images') + for (const component of allowedComponents) { + if (!digestImagePattern.test(manifest.images[component] || '')) { + fail(`images.${component} must be a registry digest reference`) + } + } + if (!manifest.smoke || manifest.smoke.status !== 'passed' || manifest.smoke.mode !== 'simulation') { + fail('smoke must record a passed simulation check') + } + requireExactKeys(manifest.smoke, ['status', 'mode', 'completedAt', 'checks'], 'smoke') + if (!Number.isFinite(Date.parse(manifest.smoke.completedAt))) { + fail('smoke.completedAt must be an ISO timestamp') + } + if (!Array.isArray(manifest.smoke.checks) || + manifest.smoke.checks.length !== smokeChecks.length || + manifest.smoke.checks.some((check, index) => check !== smokeChecks[index])) { + fail('smoke.checks is incomplete or out of order') + } + if (!manifest.integrity || typeof manifest.integrity !== 'object') { + fail('integrity must be an object') + } + requireExactKeys(manifest.integrity, ['algorithm', 'value'], 'integrity') + if (manifest.integrity.algorithm !== 'sha256' || !/^[0-9a-f]{64}$/.test(manifest.integrity.value || '')) { + fail('integrity must contain a SHA-256 value') + } + if (releaseIntegrity(manifest) !== manifest.integrity.value) { + fail('manifest content integrity check failed') + } + if (manifest.deployment !== undefined) { + if (!manifest.deployment || typeof manifest.deployment !== 'object') { + fail('deployment must be an object') + } + requireExactKeys(manifest.deployment, ['status', 'action', 'deployedAt', 'durationSeconds'], 'deployment') + if (manifest.deployment.status !== 'passed' || + !['deploy', 'rollback'].includes(manifest.deployment.action) || + !Number.isFinite(Date.parse(manifest.deployment.deployedAt)) || + !Number.isInteger(manifest.deployment.durationSeconds) || + manifest.deployment.durationSeconds < 0) { + fail('deployment record is invalid') + } + } + + const forbiddenKey = /(password|secret|token|credential|private.?key)/i + const visit = (value, path = '') => { + if (!value || typeof value !== 'object') return + for (const [key, child] of Object.entries(value)) { + const childPath = path ? `${path}.${key}` : key + if (forbiddenKey.test(key)) fail(`forbidden sensitive field: ${childPath}`) + visit(child, childPath) + } + } + visit(manifest) + return manifest +} + +function requireEnv(name, pattern) { + const value = process.env[name] || '' + if (!value || (pattern && !pattern.test(value))) fail(`invalid or missing ${name}`) + return value +} + +const [command = 'validate', path, field] = process.argv.slice(2) + +if (command === 'create') { + if (!path) fail('create requires an output path') + const sourceSha = requireEnv('RELEASE_SOURCE_SHA', shaPattern) + const baseReleaseSha = process.env.RELEASE_BASE_SHA || null + if (baseReleaseSha !== null && !shaPattern.test(baseReleaseSha)) fail('invalid RELEASE_BASE_SHA') + const components = requireEnv('RELEASE_COMPONENTS').split(',').filter(Boolean) + const migrationsChanged = process.env.RELEASE_MIGRATIONS_CHANGED === 'true' + const manifest = { + schemaVersion: 1, + releaseId: sourceSha, + sourceSha, + baseReleaseSha, + sourceRef: 'refs/heads/main', + createdAt: new Date().toISOString(), + buildCompletedAt: requireEnv('RELEASE_BUILD_COMPLETED_AT'), + targetPlatform: 'linux/amd64', + components, + migrationsChanged, + images: { + api: requireEnv('RELEASE_API_IMAGE', digestImagePattern), + web: requireEnv('RELEASE_WEB_IMAGE', digestImagePattern), + }, + smoke: { + status: 'passed', + mode: 'simulation', + completedAt: requireEnv('RELEASE_SMOKE_COMPLETED_AT'), + checks: smokeChecks, + }, + integrity: { algorithm: 'sha256', value: '' }, + } + manifest.integrity.value = releaseIntegrity(manifest) + validate(manifest) + writeFileSync(path, `${JSON.stringify(manifest, null, 2)}\n`, { flag: 'wx', mode: 0o600 }) + const digest = createHash('sha256').update(readFileSync(path)).digest('hex') + console.log(`release_manifest=CREATED path=${path} sha256=${digest}`) + process.exit(0) +} + +if (!path) fail(`${command} requires a manifest path`) +const manifest = validate(load(path)) + +if (command === 'record-deployment') { + if (!field) fail('record-deployment requires an output path') + const deployedAt = requireEnv('RELEASE_DEPLOYED_AT') + const durationSeconds = Number(process.env.RELEASE_DEPLOY_DURATION_SECONDS || '') + const action = process.env.RELEASE_DEPLOY_ACTION || 'deploy' + if (!Number.isFinite(Date.parse(deployedAt))) fail('invalid RELEASE_DEPLOYED_AT') + if (!Number.isInteger(durationSeconds) || durationSeconds < 0) { + fail('invalid RELEASE_DEPLOY_DURATION_SECONDS') + } + if (!['deploy', 'rollback'].includes(action)) fail('invalid RELEASE_DEPLOY_ACTION') + manifest.deployment = { status: 'passed', action, deployedAt, durationSeconds } + validate(manifest) + writeFileSync(field, `${JSON.stringify(manifest, null, 2)}\n`, { flag: 'wx', mode: 0o600 }) + console.log(`release_manifest=RECORDED release=${manifest.releaseId} action=${action}`) + process.exit(0) +} + +if (command === 'validate') { + const digest = createHash('sha256').update(readFileSync(path)).digest('hex') + console.log(`release_manifest=PASS release=${manifest.releaseId} sha256=${digest}`) + process.exit(0) +} + +if (command === 'get') { + if (!field || !/^[A-Za-z][A-Za-z0-9]*(?:\.[A-Za-z][A-Za-z0-9]*)*$/.test(field)) { + fail('get requires a safe dotted field path') + } + let value = manifest + for (const segment of field.split('.')) value = value?.[segment] + if (typeof value === 'object') console.log(JSON.stringify(value)) + else if (value === null || value === undefined) console.log('') + else console.log(String(value)) + process.exit(0) +} + +fail(`unknown command: ${command}`) diff --git a/scripts/release-preflight.mjs b/scripts/release-preflight.mjs new file mode 100755 index 0000000..5ba17f3 --- /dev/null +++ b/scripts/release-preflight.mjs @@ -0,0 +1,42 @@ +#!/usr/bin/env node + +import { execFileSync, spawnSync } from 'node:child_process' + +function fail(message) { + console.error(`release_preflight=FAIL ${message}`) + process.exit(1) +} + +function git(args) { + try { + return execFileSync('git', args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trim() + } catch (error) { + const stderr = error.stderr?.toString().trim() + fail(stderr ? `git_error=${JSON.stringify(stderr)}` : 'git command failed') + } +} + +const root = git(['rev-parse', '--show-toplevel']) +process.chdir(root) + +for (const args of [['diff', '--quiet'], ['diff', '--cached', '--quiet']]) { + if (spawnSync('git', args, { stdio: 'ignore' }).status !== 0) { + fail('the worktree must be clean') + } +} +if (git(['ls-files', '--others', '--exclude-standard'])) { + fail('the worktree must not contain untracked files') +} + +const sourceSha = git(['rev-parse', '--verify', 'HEAD^{commit}']) +if (!/^[0-9a-f]{40}$/.test(sourceSha)) fail('HEAD is not a full Git commit') +const mainSha = git(['rev-parse', '--verify', 'refs/remotes/origin/main^{commit}']) +if (!/^[0-9a-f]{40}$/.test(mainSha)) fail('origin/main is unavailable') +if (spawnSync('git', ['merge-base', '--is-ancestor', sourceSha, mainSha], { stdio: 'ignore' }).status !== 0) { + fail('HEAD is not part of origin/main history') +} + +console.log(JSON.stringify({ sourceSha, originMainSha: mainSha })) diff --git a/tests/ci/ci-build-images-test.sh b/tests/ci/ci-build-images-test.sh deleted file mode 100755 index bb12662..0000000 --- a/tests/ci/ci-build-images-test.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) -script=$root/scripts/ci-build-images.sh -tmp=$(mktemp -d) -trap 'rm -rf "$tmp"' EXIT - -mkdir -p "$tmp/bin" -cat >"$tmp/bin/docker" <<'MOCK' -#!/usr/bin/env bash -set -euo pipefail -printf 'docker %s\n' "$*" >>"$MOCK_LOG" - -case "$*" in - 'buildx inspect '*|'buildx create '*) exit 0 ;; - 'buildx build '*) - if [[ ${MOCK_BUILD_FAIL:-0} == 1 && $* == *'--target web'* ]]; then - exit 21 - fi - ;; - 'buildx prune '*) - [[ ${MOCK_PRUNE_FAIL:-0} == 0 ]] || exit 22 - ;; - 'image rm '*) - [[ ${MOCK_IMAGE_RM_FAIL:-0} == 0 ]] || exit 23 - ;; - *) - printf 'unexpected docker invocation: %s\n' "$*" >&2 - exit 99 - ;; -esac -MOCK -cat >"$tmp/bin/trivy" <<'MOCK' -#!/usr/bin/env bash -set -euo pipefail -printf 'trivy %s\n' "$*" >>"$MOCK_LOG" -MOCK -chmod +x "$tmp/bin/docker" "$tmp/bin/trivy" - -sha=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -registry=registry.example.com/easyai -api_image=$registry/ai-gateway:$sha -web_image=$registry/ai-gateway-web:$sha - -run_build() { - env \ - PATH="$tmp/bin:$PATH" \ - MOCK_LOG="$tmp/calls.log" \ - IMAGE_TAG="$sha" \ - AI_GATEWAY_IMAGE_REGISTRY="$registry" \ - AI_GATEWAY_BUILDX_BUILDER=gateway-test \ - AI_GATEWAY_BUILDX_CACHE_LIMIT=2gb \ - "$script" "$@" -} - -: >"$tmp/calls.log" -run_build -grep -Fq "docker buildx prune --builder gateway-test --force --max-used-space 2gb" "$tmp/calls.log" -grep -Fq "docker image rm $api_image $web_image" "$tmp/calls.log" -grep -Fq 'trivy image --db-repository ghcr.m.daocloud.io/aquasecurity/trivy-db:2' \ - "$tmp/calls.log" - -: >"$tmp/calls.log" -run_build --keep-images -grep -Fq "docker buildx prune --builder gateway-test --force --max-used-space 2gb" "$tmp/calls.log" -if grep -Fq 'docker image rm ' "$tmp/calls.log"; then - echo '--keep-images must preserve fully built images' >&2 - exit 1 -fi - -: >"$tmp/calls.log" -if MOCK_PRUNE_FAIL=1 run_build; then - echo 'a prune failure must fail an otherwise successful build' >&2 - exit 1 -fi - -: >"$tmp/calls.log" -if MOCK_IMAGE_RM_FAIL=1 run_build; then - echo 'an image cleanup failure must fail an otherwise successful build' >&2 - exit 1 -fi - -: >"$tmp/calls.log" -if MOCK_BUILD_FAIL=1 run_build; then - echo 'a failed image build must fail the script' >&2 - exit 1 -fi -grep -Fq "docker image rm $api_image $web_image" "$tmp/calls.log" - -for invalid_limit in 0gb 2 2GB -1gb '2gb --all'; do - : >"$tmp/calls.log" - if env \ - PATH="$tmp/bin:$PATH" \ - MOCK_LOG="$tmp/calls.log" \ - IMAGE_TAG="$sha" \ - AI_GATEWAY_IMAGE_REGISTRY="$registry" \ - AI_GATEWAY_BUILDX_CACHE_LIMIT="$invalid_limit" \ - "$script" >/dev/null 2>&1; then - printf 'invalid cache limit was accepted: %s\n' "$invalid_limit" >&2 - exit 1 - fi - if [[ -s $tmp/calls.log ]]; then - printf 'invalid cache limit reached Docker: %s\n' "$invalid_limit" >&2 - exit 1 - fi -done - -echo 'ci_build_images_tests=PASS' diff --git a/tests/ci/migrations-test.sh b/tests/ci/migrations-test.sh index 816ce25..3f168ff 100755 --- a/tests/ci/migrations-test.sh +++ b/tests/ci/migrations-test.sh @@ -12,15 +12,12 @@ new_repo() { mkdir -p "$repo/apps/api/migrations" git -C "$repo" init -q - git -C "$repo" config user.name 'CI Migration Test' - git -C "$repo" config user.email 'ci-migration-test@example.invalid' + git -C "$repo" config user.name 'Release Migration Test' + git -C "$repo" config user.email 'release-migration-test@example.invalid' printf 'CREATE TABLE accounts (id bigint PRIMARY KEY);\n' > \ "$repo/apps/api/migrations/0001_init.sql" git -C "$repo" add apps/api/migrations/0001_init.sql git -C "$repo" commit -qm baseline - mkdir -p "$repo/deploy/ci" - git -C "$repo" rev-parse HEAD > \ - "$repo/deploy/ci/production-migration-base" printf '%s\n' "$repo" } @@ -34,35 +31,19 @@ commit_all() { expect_pass() { local repo=$1 local base=$2 - local immutable_base=${3:-} - ( - cd "$repo" - if [[ -n $immutable_base ]]; then - node "$validator" "$base" "$immutable_base" - else - node "$validator" "$base" - fi - ) + (cd "$repo" && node "$validator" "$base") } expect_fail() { local repo=$1 local base=$2 local expected=$3 - local immutable_base=${4:-} local output status - if [[ -n $immutable_base ]]; then - set +e - output=$(cd "$repo" && node "$validator" "$base" "$immutable_base" 2>&1) - status=$? - set -e - else - set +e - output=$(cd "$repo" && node "$validator" "$base" 2>&1) - status=$? - set -e - fi + set +e + output=$(cd "$repo" && node "$validator" "$base" 2>&1) + status=$? + set -e if [[ $status -eq 0 ]]; then printf 'migration validator unexpectedly passed: %s\n' "$expected" >&2 exit 1 @@ -76,7 +57,7 @@ expect_fail() { repo=$(new_repo safe) base=$(git -C "$repo" rev-parse HEAD) cat >"$repo/apps/api/migrations/0002_expand_accounts.sql" <<'SQL' --- A comment mentioning DROP TABLE must not trigger the gate. +-- DROP in a comment and string is not executable. ALTER TABLE accounts ADD COLUMN display_name text; CREATE TABLE audit_events (id bigint PRIMARY KEY, note text); INSERT INTO audit_events (id, note) VALUES (1, 'DROP TABLE is documentation'); @@ -85,171 +66,37 @@ SQL commit_all "$repo" safe expect_pass "$repo" "$base" -repo=$(new_repo drop) -base=$(git -C "$repo" rev-parse HEAD) -printf 'DROP\nTABLE accounts;\n' >"$repo/apps/api/migrations/0002_drop.sql" -commit_all "$repo" drop -expect_fail "$repo" "$base" 'destructive DROP operation' - -repo=$(new_repo delete) -base=$(git -C "$repo" rev-parse HEAD) -printf 'DELETE FROM accounts;\n' >"$repo/apps/api/migrations/0002_delete.sql" -commit_all "$repo" delete -expect_fail "$repo" "$base" 'DELETE FROM operation' - -repo=$(new_repo incompatible_alter) -base=$(git -C "$repo" rev-parse HEAD) -cat >"$repo/apps/api/migrations/0002_alter.sql" <<'SQL' -ALTER TABLE accounts ADD COLUMN external_id text NOT NULL; -ALTER TABLE accounts ALTER COLUMN id TYPE numeric; -SQL -commit_all "$repo" alter -expect_fail "$repo" "$base" 'non-null column addition' -expect_fail "$repo" "$base" 'column type change' - -repo=$(new_repo dynamic_sql) -base=$(git -C "$repo" rev-parse HEAD) -cat >"$repo/apps/api/migrations/0002_dynamic.sql" <<'SQL' -DO $$ -BEGIN - EXECUTE 'DROP ' || 'TABLE accounts'; -END -$$; -SQL -commit_all "$repo" dynamic -expect_fail "$repo" "$base" 'dynamic SQL execution' - -repo=$(new_repo escape_string) -base=$(git -C "$repo" rev-parse HEAD) -cat >"$repo/apps/api/migrations/0002_escape.sql" <<'SQL' -SELECT E'foo\'bar'; DROP TABLE accounts; --'; -SQL -commit_all "$repo" escape -expect_fail "$repo" "$base" 'PostgreSQL escape/unicode strings are not allowed' - -repo=$(new_repo string_semantics_override) -base=$(git -C "$repo" rev-parse HEAD) -printf 'SET standard_conforming_strings = off;\n' > \ - "$repo/apps/api/migrations/0002_string_mode.sql" -commit_all "$repo" string_mode -expect_fail "$repo" "$base" 'SQL string semantics override' - -repo=$(new_repo transaction_control) -base=$(git -C "$repo" rev-parse HEAD) -cat >"$repo/apps/api/migrations/0002_commit.sql" <<'SQL' -ALTER TABLE accounts ADD COLUMN display_name text; -COMMIT; -UPDATE accounts SET display_name = 'partially committed'; -SQL -commit_all "$repo" transaction_control -expect_fail "$repo" "$base" 'transaction control operation' - -repo=$(new_repo bare_end_transaction) -base=$(git -C "$repo" rev-parse HEAD) -cat >"$repo/apps/api/migrations/0002_end.sql" <<'SQL' -ALTER TABLE accounts ADD COLUMN display_name text; -END; -UPDATE accounts SET display_name = 'partially committed'; -SQL -commit_all "$repo" bare_end_transaction -expect_fail "$repo" "$base" 'transaction control operation' - -repo=$(new_repo plpgsql_block_end) -base=$(git -C "$repo" rev-parse HEAD) -cat >"$repo/apps/api/migrations/0002_function.sql" <<'SQL' -CREATE FUNCTION account_count() RETURNS bigint -LANGUAGE plpgsql AS $$ -BEGIN - RETURN (SELECT count(*) FROM accounts); -END; -$$; -SQL -commit_all "$repo" plpgsql_block_end -expect_fail "$repo" "$base" 'procedural SQL body' - -repo=$(new_repo ordinary_string_procedural_body) -base=$(git -C "$repo" rev-parse HEAD) -cat >"$repo/apps/api/migrations/0002_do_string.sql" <<'SQL' -DO LANGUAGE plpgsql 'BEGIN DROP TABLE accounts; END'; -SQL -commit_all "$repo" ordinary_string_procedural_body -expect_fail "$repo" "$base" 'procedural SQL body' - -repo=$(new_repo ordinary_string_boundary) -base=$(git -C "$repo" rev-parse HEAD) -cat >"$repo/apps/api/migrations/0002_string_boundary.sql" <<'SQL' -SELECT '\'; DROP TABLE accounts; --'; -SQL -commit_all "$repo" ordinary_string -expect_fail "$repo" "$base" 'destructive DROP operation' - -repo=$(new_repo unicode_string) -base=$(git -C "$repo" rev-parse HEAD) -printf "SELECT U&'d\\0061ta';\n" >"$repo/apps/api/migrations/0002_unicode.sql" -commit_all "$repo" unicode -expect_fail "$repo" "$base" 'PostgreSQL escape/unicode strings are not allowed' - -repo=$(new_repo merge_delete) -base=$(git -C "$repo" rev-parse HEAD) -cat >"$repo/apps/api/migrations/0002_merge.sql" <<'SQL' -MERGE INTO accounts USING stale_accounts ON accounts.id = stale_accounts.id -WHEN MATCHED THEN DELETE; -SQL -commit_all "$repo" merge -expect_fail "$repo" "$base" 'MERGE operation' - -repo=$(new_repo replace_view) -base=$(git -C "$repo" rev-parse HEAD) -printf 'CREATE OR REPLACE VIEW active_accounts AS SELECT id FROM accounts;\n' > \ - "$repo/apps/api/migrations/0002_replace.sql" -commit_all "$repo" replace -expect_fail "$repo" "$base" 'CREATE OR REPLACE operation' - -repo=$(new_repo rename_function) -base=$(git -C "$repo" rev-parse HEAD) -printf 'ALTER FUNCTION calculate_total() RENAME TO calculate_total_legacy;\n' > \ - "$repo/apps/api/migrations/0002_rename.sql" -commit_all "$repo" rename -expect_fail "$repo" "$base" 'object or column rename' - -repo=$(new_repo set_schema) -base=$(git -C "$repo" rev-parse HEAD) -printf 'ALTER TABLE accounts SET SCHEMA archive;\n' > \ - "$repo/apps/api/migrations/0002_schema.sql" -commit_all "$repo" schema -expect_fail "$repo" "$base" 'SET SCHEMA operation' - -repo=$(new_repo drop_domain) -base=$(git -C "$repo" rev-parse HEAD) -printf 'DROP DOMAIN account_code CASCADE;\n' > \ - "$repo/apps/api/migrations/0002_drop_domain.sql" -commit_all "$repo" drop_domain -expect_fail "$repo" "$base" 'destructive DROP operation' - repo=$(new_repo immutable) base=$(git -C "$repo" rev-parse HEAD) printf '\n-- retroactive edit\n' >>"$repo/apps/api/migrations/0001_init.sql" -commit_all "$repo" edit +commit_all "$repo" immutable expect_fail "$repo" "$base" 'applied migrations are immutable (status M)' -repo=$(new_repo immutable_after_baseline) +repo=$(new_repo removed) base=$(git -C "$repo" rev-parse HEAD) -printf 'ALTER TABLE accounts ADD COLUMN nickname text;\n' > \ - "$repo/apps/api/migrations/0002_nickname.sql" -commit_all "$repo" add -immutable_base=$(git -C "$repo" rev-parse HEAD) -printf '\nUPDATE accounts SET nickname = NULL;\n' >> \ - "$repo/apps/api/migrations/0002_nickname.sql" -commit_all "$repo" modify -output=$( - cd "$repo" - node "$validator" "$base" "$immutable_base" 2>&1 || true -) -grep -Fq 'migration present on main is immutable (status M)' <<<"$output" || { - printf 'stale production baseline did not preserve main migration immutability:\n%s\n' \ - "$output" >&2 - exit 1 -} +git -C "$repo" rm -q apps/api/migrations/0001_init.sql +git -C "$repo" commit -qm removed +expect_fail "$repo" "$base" 'applied migrations are immutable (status D)' + +repo=$(new_repo destructive) +base=$(git -C "$repo" rev-parse HEAD) +printf 'DROP TABLE accounts;\n' >"$repo/apps/api/migrations/0002_drop.sql" +commit_all "$repo" destructive +expect_fail "$repo" "$base" 'destructive DROP operation' + +repo=$(new_repo incompatible) +base=$(git -C "$repo" rev-parse HEAD) +printf 'ALTER TABLE accounts ADD COLUMN external_id text NOT NULL;\n' > \ + "$repo/apps/api/migrations/0002_incompatible.sql" +commit_all "$repo" incompatible +expect_fail "$repo" "$base" 'non-null column addition' + +repo=$(new_repo transaction) +base=$(git -C "$repo" rev-parse HEAD) +printf 'ALTER TABLE accounts ADD COLUMN nickname text; COMMIT;\n' > \ + "$repo/apps/api/migrations/0002_transaction.sql" +commit_all "$repo" transaction +expect_fail "$repo" "$base" 'transaction control operation' repo=$(new_repo filename) base=$(git -C "$repo" rev-parse HEAD) @@ -263,41 +110,10 @@ printf 'SELECT 1;\n' >"$repo/apps/api/migrations/0000_late.sql" commit_all "$repo" ordering expect_fail "$repo" "$base" 'new migration name must sort after 0001_init.sql' -repo=$(new_repo stale_baseline) -base=$(git -C "$repo" rev-parse HEAD) -printf 'ALTER TABLE accounts ADD COLUMN nickname text;\n' > \ - "$repo/apps/api/migrations/0002_nickname.sql" -commit_all "$repo" main_migration -immutable_base=$(git -C "$repo" rev-parse HEAD) -printf 'ordinary code change\n' >"$repo/README.md" -commit_all "$repo" code_only -expect_pass "$repo" "$base" "$immutable_base" - -repo=$(new_repo baseline_advance) -production_base=$(git -C "$repo" rev-parse HEAD) -mkdir -p "$repo/deploy/ci" -printf '%s\n' "$production_base" >"$repo/deploy/ci/production-migration-base" -commit_all "$repo" baseline_file -immutable_base=$(git -C "$repo" rev-parse HEAD) -git -C "$repo" switch -q -c release-baseline-update "$immutable_base" -printf '%s\n' "$immutable_base" >"$repo/deploy/ci/production-migration-base" -commit_all "$repo" valid_baseline_advance -expect_pass "$repo" "$immutable_base" "$immutable_base" - -git -C "$repo" switch -q -c malicious-baseline "$immutable_base" -printf 'DROP TABLE accounts;\n' >"$repo/apps/api/migrations/0002_drop.sql" -commit_all "$repo" hidden_destructive_migration -hidden_sha=$(git -C "$repo" rev-parse HEAD) -printf '%s\n' "$hidden_sha" >"$repo/deploy/ci/production-migration-base" -commit_all "$repo" hide_migration_in_baseline -expect_fail "$repo" "$hidden_sha" \ - 'production migration baseline must belong to the immutable event base history' \ - "$immutable_base" - repo=$(new_repo nested) base=$(git -C "$repo" rev-parse HEAD) mkdir -p "$repo/apps/api/migrations/archive" -printf 'SELECT 1;\n' >"$repo/apps/api/migrations/archive/9999_hidden.sql" +printf 'SELECT 1;\n' >"$repo/apps/api/migrations/archive/0002_hidden.sql" commit_all "$repo" nested expect_fail "$repo" "$base" 'new migration must use NNNN_lowercase_name.sql' @@ -308,22 +124,13 @@ ln -s ../../../outside.sql "$repo/apps/api/migrations/0002_symlink.sql" commit_all "$repo" symlink expect_fail "$repo" "$base" 'new migration must be a mode 100644 regular file' -repo=$(new_repo baseline_symlink) -base=$(git -C "$repo" rev-parse HEAD) -printf '%s\n' "$base" >"$repo/baseline-target" -rm "$repo/deploy/ci/production-migration-base" -ln -s ../../baseline-target "$repo/deploy/ci/production-migration-base" -commit_all "$repo" baseline_symlink -expect_fail "$repo" "$base" \ - 'production migration baseline must be a mode 100644 regular file' - repo=$(new_repo ancestry) base=$(git -C "$repo" rev-parse HEAD) git -C "$repo" switch -q --orphan unrelated printf 'unrelated\n' >"$repo/unrelated" git -C "$repo" add unrelated git -C "$repo" commit -qm unrelated -expect_fail "$repo" "$base" 'production migration baseline is not an ancestor of HEAD' +expect_fail "$repo" "$base" 'release base is not an ancestor of HEAD' repo=$(new_repo invalid_sha) expect_fail "$repo" deadbeef 'must be a full lowercase commit SHA' diff --git a/tests/ci/pipeline-test.sh b/tests/ci/pipeline-test.sh deleted file mode 100755 index 7ff545d..0000000 --- a/tests/ci/pipeline-test.sh +++ /dev/null @@ -1,365 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC2016 # Assertions intentionally match literal workflow variables. -set -euo pipefail - -root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) -workflow=$root/.gitea/workflows/ci.yml -release_workflow=$root/.gitea/workflows/release-ci.yml -runner_service=$root/deploy/ci/easyai-gateway-ci-v2-runner.service -runner_config=$root/deploy/ci/act-runner-v2-config.yaml -provision=$root/scripts/provision-ci-runner.sh -build_images=$root/scripts/ci-build-images.sh -validate_semver=$root/scripts/ci-validate-semver.sh -validate_migrations=$root/scripts/ci-validate-migrations.mjs -build_images_test=$root/tests/ci/ci-build-images-test.sh -migrations_test=$root/tests/ci/migrations-test.sh -migration_base=$root/deploy/ci/production-migration-base -adr=$root/docs/decisions/001-production-cicd.md -runbook=$root/docs/runbooks/production-ci-cd.md -migrator=$root/apps/api/cmd/migrate/main.go - -assert_bounded_go_verification() { - local quality_workflow=$1 - - awk ' - /^[[:space:]]+GOFLAGS:/ { all_goflags++ } - /^[[:space:]]+GOMAXPROCS:/ { all_gomaxprocs++ } - /^ - name: Verify Go code$/ { - target_steps++ - in_target=1 - in_env=0 - next - } - /^ - / { - in_target=0 - in_env=0 - } - in_target && /^ env:$/ { - env_blocks++ - in_env=1 - next - } - in_target && in_env && /^ GOFLAGS: "-p=1"$/ { target_goflags++ } - in_target && in_env && /^ GOMAXPROCS: "1"$/ { target_gomaxprocs++ } - in_target && in_env && /^ [^ ]/ { in_env=0 } - END { - exit !(target_steps == 1 && env_blocks == 1 && - target_goflags == 1 && target_gomaxprocs == 1 && - all_goflags == 1 && all_gomaxprocs == 1) - } - ' "$quality_workflow" || { - printf '%s must limit Go parallelism exactly once in the Verify Go code step\n' \ - "$quality_workflow" >&2 - exit 1 - } -} - -for file in "$workflow" "$release_workflow" "$runner_service" "$runner_config" "$provision" "$build_images" "$validate_semver" "$validate_migrations" "$build_images_test" "$migrations_test" "$migration_base" "$adr" "$runbook" "$migrator"; do - [[ -f $file ]] || { - printf 'missing CI/CD file: %s\n' "$file" >&2 - exit 1 - } -done - -if [[ $(wc -l <"$migration_base") -ne 1 ]]; then - echo 'production migration baseline must contain exactly one line' >&2 - exit 1 -fi -grep -Fq 'SET standard_conforming_strings = on' "$migrator" || { - echo 'database migrator does not pin standard SQL string semantics' >&2 - exit 1 -} -if ! grep -Eq '^[0-9a-f]{40}$' "$migration_base"; then - echo 'production migration baseline must be a full lowercase SHA' >&2 - exit 1 -fi - -grep -q '^name: ci$' "$workflow" -grep -q '^ push:$' "$workflow" -grep -q '^ branches: \[main\]$' "$workflow" -grep -q '^ pull_request:$' "$workflow" -grep -q '^ verify:$' "$workflow" -grep -Fq 'git checkout --detach "$CI_SHA"' "$workflow" -grep -Fq 'test "$(git rev-parse HEAD)" = "$CI_SHA"' "$workflow" -if grep -Fq 'git checkout --detach FETCH_HEAD' "$workflow"; then - echo 'main/PR CI must not checkout mutable FETCH_HEAD after comparison fetches' >&2 - exit 1 -fi -if grep -Eq "^[[:space:]]+tags:|Verify release tag ancestry" "$workflow" || \ - grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$workflow"; then - echo 'main/PR CI must not share the release-tag context' >&2 - exit 1 -fi - -grep -q '^name: release-ci$' "$release_workflow" -grep -q '^ push:$' "$release_workflow" -grep -q "^ tags: \['v\*'\]$" "$release_workflow" -grep -q '^ verify-tag:$' "$release_workflow" -grep -q '^ - name: Verify release tag ancestry$' "$release_workflow" -grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$release_workflow" -grep -Fq 'git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main' "$release_workflow" -if grep -Eq 'branches:|pull_request:' "$release_workflow"; then - echo 'release CI must be a tag-only workflow/context' >&2 - exit 1 -fi - -for quality_workflow in "$workflow" "$release_workflow"; do - grep -q '^ runs-on: easyai-gateway-ci-unprivileged-v2$' "$quality_workflow" - grep -q '^ - name: Checkout without external Actions$' "$quality_workflow" - if grep -q -- '--depth=' "$quality_workflow"; then - echo 'quality verification must not use a shallow checkout' >&2 - exit 1 - fi - grep -Fq 'test ! -f .git/shallow' "$quality_workflow" - grep -Fq './tests/ci/ci-build-images-test.sh' "$quality_workflow" - grep -Fq './tests/ci/migrations-test.sh' "$quality_workflow" - grep -Fq './tests/ci/pipeline-test.sh' "$quality_workflow" - grep -Fq './tests/ci/semver-test.sh' "$quality_workflow" - grep -Fq 'docker-compose version' "$quality_workflow" - grep -Fq 'docker-compose -f docker-compose.yml config --quiet' "$quality_workflow" - grep -Fq 'services:' "$quality_workflow" - grep -Fq 'image: docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777' "$quality_workflow" - grep -Fq 'POSTGRES_HOST_AUTH_METHOD: trust' "$quality_workflow" - grep -Fq 'AI_GATEWAY_TEST_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable' "$quality_workflow" - grep -Fq 'run: go run ./cmd/migrate' "$quality_workflow" - if grep -Fq 'docker compose' "$quality_workflow"; then - echo 'job container must use the read-only standalone Compose binary' >&2 - exit 1 - fi - - for forbidden in \ - 'sudo' \ - '/usr/local/sbin/easyai-ai-gateway-release' \ - 'docker version' \ - 'docker buildx' \ - './scripts/ci-build-images.sh --keep-images' \ - 'Deploy verified release to Production'; do - if grep -Fq "$forbidden" "$quality_workflow"; then - printf 'unprivileged workflow contains forbidden capability: %s\n' "$forbidden" >&2 - exit 1 - fi - done - - if grep -Eq '^[[:space:]]+\./scripts/ci-build-images\.sh([[:space:]]|$)' "$quality_workflow"; then - echo 'unprivileged workflow must not build container images' >&2 - exit 1 - fi - - for gate in \ - 'scripts/ci-validate-migrations.mjs' \ - 'gofmt -l' \ - 'go vet ./...' \ - 'go test ./...' \ - 'pnpm install --frozen-lockfile' \ - 'pnpm lint' \ - 'pnpm test' \ - 'pnpm build' \ - 'pnpm audit --audit-level high' \ - 'docker-compose -f docker-compose.yml config --quiet' \ - 'govulncheck ./...' \ - 'trivy fs'; do - grep -Fq "$gate" "$quality_workflow" || { - printf '%s is missing quality gate: %s\n' "$quality_workflow" "$gate" >&2 - exit 1 - } - done - grep -Fq 'TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2' \ - "$quality_workflow" || { - printf '%s does not pin the reachable Trivy vulnerability DB mirror\n' \ - "$quality_workflow" >&2 - exit 1 - } - assert_bounded_go_verification "$quality_workflow" - grep -Fq 'production_base=$(cat deploy/ci/production-migration-base)' \ - "$quality_workflow" || { - printf '%s does not read the versioned production migration baseline\n' \ - "$quality_workflow" >&2 - exit 1 - } -done - -if [[ $(sed -n '/^ - name: Verify Go formatting$/,$p' "$workflow") != \ - "$(sed -n '/^ - name: Verify Go formatting$/,$p' "$release_workflow")" ]]; then - echo 'main/PR and release-tag workflows must have identical quality gates' >&2 - exit 1 -fi - -grep -Fq 'CI_EVENT_BEFORE: ${{ github.event.before }}' "$workflow" -grep -Fq 'CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}' "$workflow" -grep -Fq 'git merge-base --is-ancestor "$immutable_base" HEAD' "$workflow" -grep -Fq '"$production_base" "$immutable_base"' "$workflow" -grep -Fq 'node ./scripts/ci-validate-migrations.mjs "$production_base"' \ - "$release_workflow" - -runner_image='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a' -job_image='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7' - -grep -q '^User=root$' "$runner_service" -grep -q '^Group=root$' "$runner_service" -grep -q '^NoNewPrivileges=true$' "$runner_service" -grep -q '^ProtectSystem=strict$' "$runner_service" -grep -q '^ProtectHome=true$' "$runner_service" -grep -q '^CapabilityBoundingSet=$' "$runner_service" -grep -Fq 'Requires=docker.service' "$runner_service" -grep -Fq -- "--pull=never" "$runner_service" -grep -Fq -- "--privileged" "$runner_service" -grep -Fq -- '--pids-limit 1024' "$runner_service" -grep -Fq -- '--cpus 2' "$runner_service" -grep -Fq -- '--memory 2g' "$runner_service" -grep -Fq -- '--memory-swap 3g' "$runner_service" -grep -Fq -- '--env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns' "$runner_service" -grep -Fq 'EnvironmentFile=-/etc/easyai-gateway-ci-v2/runner.env' "$runner_service" -grep -Fq '$RUNNER_SECURITY_OPTIONS' "$runner_service" -grep -Fq -- "--volume easyai-gateway-ci-v2-data:/data" "$runner_service" -grep -Fq -- "--volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro" "$runner_service" -grep -Fq -- "--volume /etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro" "$runner_service" -grep -Fq -- "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro" "$runner_service" -grep -Fq 'Environment=RUNNER_IMAGE_REF=' "$runner_service" -grep -Fq '$RUNNER_IMAGE_REF' "$runner_service" -if grep -Eq 'GITEA_RUNNER_REGISTRATION_TOKEN|/var/run/docker\.sock|/run/docker\.sock' "$runner_service"; then - echo 'persistent runner service exposes a registration token or host Docker socket' >&2 - exit 1 -fi -if grep -Fq 'apparmor=rootlesskit' "$runner_service"; then - echo 'runner service must not hard-code an AppArmor profile that may be absent' >&2 - exit 1 -fi - -grep -Fq "easyai-gateway-ci-unprivileged-v2:docker://$job_image" "$runner_config" -if grep -Fq ':host' "$runner_config"; then - echo 'CI runner must never use the host executor' >&2 - exit 1 -fi -grep -q '^ capacity: 1$' "$runner_config" -grep -q '^ file: /data/.runner$' "$runner_config" -grep -q '^ docker_host: "-"$' "$runner_config" -grep -q '^ privileged: false$' "$runner_config" -grep -q '^ require_docker: true$' "$runner_config" -grep -q '^ bind_workdir: false$' "$runner_config" -grep -Fq ' options: "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro"' "$runner_config" -grep -q '^ valid_volumes:$' "$runner_config" -grep -q '^ - "/opt/easyai-gateway-ci"$' "$runner_config" -if awk ' - /^ valid_volumes:$/ { in_volumes=1; next } - in_volumes && /^ - / { count++; next } - in_volumes && /^ [A-Za-z_]+:/ { in_volumes=0 } - END { exit count == 1 ? 0 : 1 } -' "$runner_config"; then - : -else - echo 'runner must allow exactly one workflow volume source' >&2 - exit 1 -fi -if grep -Eq '/var/run/docker\.sock|/run/docker\.sock' "$runner_config"; then - echo 'job configuration must not expose a Docker socket' >&2 - exit 1 -fi -if grep -Eq 'apparmor=unconfined|sysctl .*(apparmor_restrict_unprivileged_userns)' "$runner_service" "$runner_config" "$provision"; then - echo 'runner must not disable AppArmor isolation or its user-namespace restriction' >&2 - exit 1 -fi - -grep -q '^GITEA_RUNNER_VERSION=2.0.0$' "$provision" -grep -q '^GO_VERSION=1.26.5$' "$provision" -grep -q '^NODE_VERSION=24.16.0$' "$provision" -grep -q '^TRIVY_VERSION=0.70.0$' "$provision" -grep -q '^PREFIX=/opt/easyai-gateway-ci$' "$provision" -grep -q '^RUNNER_VOLUME=easyai-gateway-ci-v2-data$' "$provision" -grep -q '^SHELLCHECK_VERSION=0.11.0$' "$provision" -grep -q '^SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198$' "$provision" -grep -q '^COMPOSE_VERSION=5.3.1$' "$provision" -grep -q '^COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959$' "$provision" -grep -Fq 'RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-ci-unprivileged-v2}' "$provision" -grep -Fq "RUNNER_IMAGE='$runner_image'" "$provision" -grep -Fq 'RUNNER_IMAGE_SOURCE=${CI_RUNNER_IMAGE_SOURCE:-$RUNNER_IMAGE}' "$provision" -grep -Fq 'runner_manifest_digest=${RUNNER_IMAGE##*@}' "$provision" -grep -Fq 'CI_RUNNER_IMAGE_SOURCE must use the pinned runner manifest digest' "$provision" -grep -Fq 'runner_runtime_image=$(docker image inspect' "$provision" -grep -Fq 'RUNNER_IMAGE_REF="%s"' "$provision" -grep -Fq "JOB_IMAGE='$job_image'" "$provision" -grep -Fq 'RUNNER_LABEL="easyai-gateway-ci-unprivileged-v2:docker://$JOB_IMAGE"' "$provision" -grep -Fq 'docker volume create "$RUNNER_VOLUME"' "$provision" -grep -Fq '"data-root":"/data/docker"' "$provision" -grep -Fq -- '--security-opt apparmor=rootlesskit' "$provision" -grep -Fq 'RUNNER_SECURITY_OPTIONS="--security-opt=apparmor=rootlesskit"' "$provision" -grep -Fq 'RUNNER_SECURITY_OPTIONS=' "$provision" -grep -Fq '/proc/sys/kernel/apparmor_restrict_unprivileged_userns' "$provision" -grep -Fq -- '--entrypoint dockerd-entrypoint.sh "$runner_runtime_image"' "$provision" -grep -Fq '/etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro' "$provision" -grep -Fq 'CI_NESTED_DOCKER_REGISTRY_MIRROR must be an HTTPS registry origin' "$provision" -grep -Fq 'PROBE_CONTAINER="easyai-gateway-ci-v2-probe-$$"' "$provision" -grep -Fq -- '--pids-limit "$RUNNER_PIDS_LIMIT"' "$provision" -grep -Fq -- '--cpus "$RUNNER_CPUS"' "$provision" -grep -Fq -- '--memory "$RUNNER_MEMORY"' "$provision" -grep -Fq -- '--memory-swap "$RUNNER_MEMORY_SWAP"' "$provision" -grep -Fq "'2147483648:3221225472:2000000000:1024'" "$provision" -grep -Fq -- '--entrypoint /usr/local/bin/gitea-runner' "$provision" -grep -Fq 'shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz' "$provision" -grep -Fq 'docker-compose-linux-x86_64' "$provision" -grep -Fq 'install -m 0755 "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "$PREFIX/bin/shellcheck"' "$provision" -grep -Fq 'install -m 0755 "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" "$PREFIX/bin/docker-compose"' "$provision" -grep -Fq 'PATH="$PREFIX/toolchains/node/bin:$PATH"' "$provision" -grep -Fq 'gpasswd -d easyai-gateway-ci-v2 docker' "$provision" -grep -Fq 'gpasswd -d easyai-gateway-ci-v2 sudo' "$provision" -if grep -Fq 'usermod -aG docker' "$provision"; then - echo 'unprivileged runner must never join the Docker group' >&2 - exit 1 -fi -grep -Fq 'systemctl disable --now easyai-gateway-act-runner.service' "$provision" -grep -Fq 'systemctl disable --now easyai-gateway-ci-v2-runner.service' "$provision" -grep -Fq 'RUNNER_REGISTRATION_TOKEN is required for first registration' "$provision" -grep -Fq 'printf '\''%s\n%s\n%s\n'\'' "$GITEA_INSTANCE_URL" "$RUNNER_REGISTRATION_TOKEN" "$RUNNER_NAME"' "$provision" -if grep -Fq -- '--token "$RUNNER_REGISTRATION_TOKEN"' "$provision"; then - echo 'registration token must not be exposed in a host-visible process argv' >&2 - exit 1 -fi -grep -Fq "grep -Fq 'name=rootless'" "$provision" -grep -Fq "grep -Fq 'declare successfully'" "$provision" -grep -Fq 'docker buildx prune --builder "$LEGACY_BUILDER" --force --max-used-space 512mb' "$provision" -grep -Fq 'require_free_space "$MIN_FREE_GIB" "runner image pull"' "$provision" -grep -Fq 'require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed CI runner installation"' "$provision" -grep -Fq 'CI_RUNNER_MIN_FREE_GIB cannot be lower than 8' "$provision" -grep -Fq 'CI_RUNNER_MIN_POST_INSTALL_FREE_GIB cannot be lower than 4' "$provision" -grep -Fq 'docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"' "$provision" -grep -Fq 'docker exec "$PROBE_CONTAINER" docker run --rm --pull=never' "$provision" -grep -Fq -- '--volume "$RUNNER_VOLUME:/data"' "$provision" -grep -Fq -- '--volume "$PREFIX:$PREFIX:ro"' "$provision" -probe_pull_line=$(grep -nF 'docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"' "$provision" | cut -d: -f1) -registration_line=$(grep -nF 'if ! runner_is_registered; then' "$provision" | cut -d: -f1) -[[ -n $probe_pull_line && -n $registration_line && $probe_pull_line -lt $registration_line ]] || { - echo 'pinned job image and isolation probes must complete before runner registration' >&2 - exit 1 -} -grep -Fq 'nested --pid=host can see the outer runner PID namespace' "$provision" -grep -Fq 'chown -R root:root "$PREFIX"' "$provision" -grep -Fq 'chmod -R go-w "$PREFIX"' "$provision" -if grep -Eq 'SHELLCHECK_BIN|COMPOSE_PLUGIN|command -v shellcheck|docker compose version' "$provision"; then - echo 'runner provisioning must use checksum-pinned static CI tools, not host copies' >&2 - exit 1 -fi -if grep -Eq 'docker (system|builder|image|volume) prune' "$provision"; then - echo 'runner provisioning must not globally prune shared Docker state' >&2 - exit 1 -fi - -grep -Fq 'root-owned dispatcher' "$adr" -grep -Fq 'release-ci / verify-tag (push)' "$adr" -grep -Fq '来自 Fork 的 PR 必须先由维护者检查工作流变更并在 Gitea 中批准' "$adr" -grep -Fq 'rootless DinD 不是 VM 沙箱' "$runbook" -grep -Fq '同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)`' "$runbook" -grep -Fq '不得执行 Tag checkout 中的' "$runbook" - -grep -Fq 'docker buildx build --builder "$builder"' "$build_images" -grep -q -- '--target api' "$build_images" -grep -q -- '--target web' "$build_images" -grep -Fq 'docker buildx prune --builder "$builder" --force --max-used-space "$cache_limit"' "$build_images" -grep -Fq 'trivy image' "$build_images" -grep -Fq 'docker image rm "$api_image" "$web_image"' "$build_images" -grep -Fq 'AI_GATEWAY_BUILDX_CACHE_LIMIT has an invalid format' "$build_images" - -if grep -En '(password|secret|token):[[:space:]]+[^$<{]' "$workflow" "$release_workflow"; then - echo 'workflow contains a literal credential' >&2 - exit 1 -fi - -echo 'gateway_ci_pipeline_tests=PASS' diff --git a/tests/ci/semver-test.sh b/tests/ci/semver-test.sh deleted file mode 100755 index fde3ed3..0000000 --- a/tests/ci/semver-test.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) -validator=$root/scripts/ci-validate-semver.sh - -valid_tags=( - v0.0.0 - v1.2.3 - v1.2.3-alpha - v1.2.3-alpha.1 - v1.2.3-0.3.7 - v1.2.3-01a - v1.2.3-alpha-beta - v1.2.3-- - v1.2.3-x.7.z.92+build.5 - v1.2.3+build.11.e0f985a - v1.2.3+001 -) -invalid_tags=( - 1.2.3 - v01.2.3 - v1.02.3 - v1.2.03 - v1.2.3.foo - v1.2.3. - v1.2.3-. - v1.2.3-alpha. - v1.2.3-alpha..1 - v1.2.3-01 - v1.2.3-00.1 - v1.2.3+ - v1.2.3+build. - v1.2.3+build..1 - v1.2.3+build+1 - 'v1.2.3 ' - v1.2.3-alpha_1 - v1.2 -) - -for tag in "${valid_tags[@]}"; do - "$validator" "$tag" -done - -for tag in "${invalid_tags[@]}"; do - if "$validator" "$tag" >/dev/null 2>&1; then - printf 'invalid SemVer tag was accepted: %s\n' "$tag" >&2 - exit 1 - fi -done - -echo 'semver_tests=PASS' diff --git a/tests/release/manual-release-test.sh b/tests/release/manual-release-test.sh new file mode 100755 index 0000000..57df7b6 --- /dev/null +++ b/tests/release/manual-release-test.sh @@ -0,0 +1,261 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +new_repo() { + local name=$1 + local repo=$tmp/$name + mkdir -p "$repo/apps/api/migrations" "$repo/apps/web/src" "$repo/packages/contracts/src" "$repo/docs" + git -C "$repo" init -q + git -C "$repo" config user.name 'Manual Release Test' + git -C "$repo" config user.email 'manual-release-test@example.invalid' + printf 'package api\n' >"$repo/apps/api/main.go" + printf 'SELECT 1;\n' >"$repo/apps/api/migrations/0001_init.sql" + printf 'web\n' >"$repo/apps/web/src/app.ts" + printf 'contracts\n' >"$repo/packages/contracts/src/index.ts" + # shellcheck disable=SC2016 + printf 'ARG API_RUNTIME_IMAGE=scratch\nARG WEB_RUNTIME_IMAGE=scratch\nFROM ${API_RUNTIME_IMAGE} AS api\nCOPY apps/api /app\nFROM ${WEB_RUNTIME_IMAGE} AS web\nCOPY apps/web /web\n' >"$repo/Dockerfile" + printf 'docs\n' >"$repo/docs/readme.md" + git -C "$repo" add -A + git -C "$repo" commit -qm baseline + printf '%s\n' "$repo" +} + +classify() { + local repo=$1 + local base=$2 + (cd "$repo" && node "$root/scripts/release-components.mjs" "$base") +} + +repo=$(new_repo api) +base=$(git -C "$repo" rev-parse HEAD) +printf 'package api\n// changed\n' >"$repo/apps/api/main.go" +git -C "$repo" add -A && git -C "$repo" commit -qm api +result=$(classify "$repo" "$base") +node -e 'const value=JSON.parse(process.argv[1]); if(value.components!=="api"||value.migrationsChanged)process.exit(1)' "$result" + +repo=$(new_repo web) +base=$(git -C "$repo" rev-parse HEAD) +printf 'web changed\n' >"$repo/apps/web/src/app.ts" +git -C "$repo" add -A && git -C "$repo" commit -qm web +result=$(classify "$repo" "$base") +node -e 'const value=JSON.parse(process.argv[1]); if(value.components!=="web")process.exit(1)' "$result" + +repo=$(new_repo all) +base=$(git -C "$repo" rev-parse HEAD) +printf 'node_modules\n' >"$repo/.dockerignore" +git -C "$repo" add -A && git -C "$repo" commit -qm all +result=$(classify "$repo" "$base") +node -e 'const value=JSON.parse(process.argv[1]); if(value.components!=="all")process.exit(1)' "$result" + +repo=$(new_repo api-docker-stage) +base=$(git -C "$repo" rev-parse HEAD) +# shellcheck disable=SC2016 +printf 'ARG API_RUNTIME_IMAGE=scratch\nARG WEB_RUNTIME_IMAGE=scratch\nFROM ${API_RUNTIME_IMAGE} AS api\nCOPY apps/api /app\nRUN echo api\nFROM ${WEB_RUNTIME_IMAGE} AS web\nCOPY apps/web /web\n' >"$repo/Dockerfile" +git -C "$repo" add -A && git -C "$repo" commit -qm api-docker-stage +result=$(classify "$repo" "$base") +node -e 'const value=JSON.parse(process.argv[1]); if(value.components!=="api")process.exit(1)' "$result" + +repo=$(new_repo web-docker-stage) +base=$(git -C "$repo" rev-parse HEAD) +printf '\nRUN echo web\n' >>"$repo/Dockerfile" +git -C "$repo" add -A && git -C "$repo" commit -qm web-docker-stage +result=$(classify "$repo" "$base") +node -e 'const value=JSON.parse(process.argv[1]); if(value.components!=="web")process.exit(1)' "$result" + +repo=$(new_repo migration) +base=$(git -C "$repo" rev-parse HEAD) +printf 'ALTER TABLE demo ADD COLUMN note text;\n' >"$repo/apps/api/migrations/0002_note.sql" +git -C "$repo" add -A && git -C "$repo" commit -qm migration +result=$(classify "$repo" "$base") +node -e 'const value=JSON.parse(process.argv[1]); if(value.components!=="api"||!value.migrationsChanged)process.exit(1)' "$result" + +repo=$(new_repo docs) +base=$(git -C "$repo" rev-parse HEAD) +printf 'docs changed\n' >"$repo/docs/readme.md" +git -C "$repo" add -A && git -C "$repo" commit -qm docs +result=$(classify "$repo" "$base") +node -e 'const value=JSON.parse(process.argv[1]); if(value.components!=="none")process.exit(1)' "$result" + +bare=$tmp/origin.git +git init -q --bare "$bare" +repo=$(new_repo preflight) +git -C "$repo" branch -M main +git -C "$repo" remote add origin "$bare" +git -C "$repo" push -q -u origin main +(cd "$repo" && node "$root/scripts/release-preflight.mjs" >/dev/null) +printf 'dirty\n' >>"$repo/docs/readme.md" +if (cd "$repo" && node "$root/scripts/release-preflight.mjs" >/dev/null 2>&1); then + echo 'dirty worktree passed release preflight' >&2 + exit 1 +fi +git -C "$repo" restore docs/readme.md +printf 'untracked\n' >"$repo/untracked" +if (cd "$repo" && node "$root/scripts/release-preflight.mjs" >/dev/null 2>&1); then + echo 'untracked file passed release preflight' >&2 + exit 1 +fi +rm "$repo/untracked" +git -C "$repo" switch -q -c feature +printf 'feature\n' >>"$repo/docs/readme.md" +git -C "$repo" add -A && git -C "$repo" commit -qm feature +if (cd "$repo" && node "$root/scripts/release-preflight.mjs" >/dev/null 2>&1); then + echo 'non-main commit passed release preflight' >&2 + exit 1 +fi + +manifest=$tmp/release.json +recorded=$tmp/recorded.json +source_sha=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +base_sha=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +api_image=registry.example.com/easyai/ai-gateway@sha256:$(printf '1%.0s' {1..64}) +web_image=registry.example.com/easyai/ai-gateway-web@sha256:$(printf '2%.0s' {1..64}) +RELEASE_SOURCE_SHA=$source_sha \ +RELEASE_BASE_SHA=$base_sha \ +RELEASE_COMPONENTS=api \ +RELEASE_MIGRATIONS_CHANGED=false \ +RELEASE_API_IMAGE=$api_image \ +RELEASE_WEB_IMAGE=$web_image \ +RELEASE_BUILD_COMPLETED_AT=2026-07-22T00:00:00Z \ +RELEASE_SMOKE_COMPLETED_AT=2026-07-22T00:00:00Z \ + node "$root/scripts/release-manifest.mjs" create "$manifest" >/dev/null +node "$root/scripts/release-manifest.mjs" validate "$manifest" >/dev/null +[[ $(node "$root/scripts/release-manifest.mjs" get "$manifest" images.api) == "$api_image" ]] +RELEASE_DEPLOYED_AT=2026-07-22T00:01:00Z \ +RELEASE_DEPLOY_DURATION_SECONDS=17 \ +RELEASE_DEPLOY_ACTION=deploy \ + node "$root/scripts/release-manifest.mjs" record-deployment "$manifest" "$recorded" >/dev/null +node "$root/scripts/release-manifest.mjs" validate "$recorded" >/dev/null + +sensitive=$tmp/sensitive.json +node -e 'const fs=require("fs"); const value=JSON.parse(fs.readFileSync(process.argv[1])); value.accessToken="forbidden"; fs.writeFileSync(process.argv[2], JSON.stringify(value))' "$manifest" "$sensitive" +if node "$root/scripts/release-manifest.mjs" validate "$sensitive" >/dev/null 2>&1; then + echo 'sensitive manifest field passed validation' >&2 + exit 1 +fi + +tampered=$tmp/tampered.json +node -e 'const fs=require("fs"); const value=JSON.parse(fs.readFileSync(process.argv[1])); value.images.api=value.images.api.replace(/^registry[.]/, "mirror."); fs.writeFileSync(process.argv[2], JSON.stringify(value))' "$manifest" "$tampered" +if node "$root/scripts/release-manifest.mjs" validate "$tampered" >/dev/null 2>&1; then + echo 'tampered manifest content passed validation' >&2 + exit 1 +fi + +incomplete_digest=$tmp/incomplete-digest.json +node -e 'const fs=require("fs"); const value=JSON.parse(fs.readFileSync(process.argv[1])); value.images.web="registry.example.com/easyai/ai-gateway-web:tag"; fs.writeFileSync(process.argv[2], JSON.stringify(value))' "$manifest" "$incomplete_digest" +if node "$root/scripts/release-manifest.mjs" validate "$incomplete_digest" >/dev/null 2>&1; then + echo 'tag-based image passed manifest validation' >&2 + exit 1 +fi + +if grep -Eqi 'password|secret|token|credential|private.?key' "$manifest"; then + echo 'generated manifest contains a sensitive field name' >&2 + exit 1 +fi + +publisher_repo=$tmp/publisher +mkdir -p "$publisher_repo/scripts" "$publisher_repo/apps/api" "$publisher_repo/fake-bin" +cp "$root/scripts/publish-release-images.sh" \ + "$root/scripts/release-preflight.mjs" \ + "$root/scripts/release-manifest.mjs" \ + "$root/scripts/release-components.mjs" \ + "$root/scripts/ci-validate-migrations.mjs" \ + "$publisher_repo/scripts/" +printf 'FROM scratch\n' >"$publisher_repo/Dockerfile" +printf 'module example.invalid/release-test\n\ngo 1.26\n' >"$publisher_repo/apps/api/go.mod" +cat >"$publisher_repo/fake-bin/docker" <<'EOF' +#!/usr/bin/env bash +case "$*" in + info|'buildx version'|'compose version') exit 0 ;; + 'buildx imagetools inspect '*) + if [[ ${FAKE_DOCKER_MODE:-} == registry ]]; then + echo 'unauthorized: authentication required' >&2 + else + echo 'manifest unknown: not found' >&2 + fi + exit 1 + ;; + 'run --rm --platform linux/amd64 '*) exit 1 ;; +esac +echo "unexpected fake docker call: $*" >&2 +exit 1 +EOF +cat >"$publisher_repo/fake-bin/ssh" <<'EOF' +#!/usr/bin/env bash +exit 1 +EOF +chmod +x "$publisher_repo/scripts/"* "$publisher_repo/fake-bin/"* +git -C "$publisher_repo" init -q +git -C "$publisher_repo" config user.name 'Manual Release Test' +git -C "$publisher_repo" config user.email 'manual-release-test@example.invalid' +git -C "$publisher_repo" add -A +git -C "$publisher_repo" commit -qm baseline +git -C "$publisher_repo" branch -M main +publisher_bare=$tmp/publisher-origin.git +git init -q --bare "$publisher_bare" +git -C "$publisher_repo" remote add origin "$publisher_bare" +git -C "$publisher_repo" push -q -u origin main + +if (cd "$publisher_repo" && PATH="$publisher_repo/fake-bin:$PATH" \ + ./scripts/publish-release-images.sh --components auto >/dev/null 2>&1); then + echo 'first release guessed components in auto mode' >&2 + exit 1 +fi +if (cd "$publisher_repo" && PATH="$publisher_repo/fake-bin:$PATH" FAKE_DOCKER_MODE=registry \ + ./scripts/publish-release-images.sh --components all >/dev/null 2>&1); then + echo 'publisher passed without registry access' >&2 + exit 1 +fi +if (cd "$publisher_repo" && PATH="$publisher_repo/fake-bin:$PATH" FAKE_DOCKER_MODE=platform \ + ./scripts/publish-release-images.sh --components all >/dev/null 2>&1); then + echo 'publisher passed without linux/amd64 execution support' >&2 + exit 1 +fi + +current_manifest=$tmp/current.json +RELEASE_SOURCE_SHA=cccccccccccccccccccccccccccccccccccccccc \ +RELEASE_BASE_SHA=dddddddddddddddddddddddddddddddddddddddd \ +RELEASE_COMPONENTS=web \ +RELEASE_MIGRATIONS_CHANGED=false \ +RELEASE_API_IMAGE=$api_image \ +RELEASE_WEB_IMAGE=$web_image \ +RELEASE_BUILD_COMPLETED_AT=2026-07-22T00:02:00Z \ +RELEASE_SMOKE_COMPLETED_AT=2026-07-22T00:02:00Z \ + node "$root/scripts/release-manifest.mjs" create "$current_manifest" >/dev/null +stale_bin=$tmp/stale-bin +mkdir -p "$stale_bin" +cat >"$stale_bin/ssh" <<'EOF' +#!/usr/bin/env bash +if [[ $* == *' status' ]]; then + cat "$STALE_STATUS_FILE" +fi +exit 0 +EOF +cat >"$stale_bin/scp" <<'EOF' +#!/usr/bin/env bash +: >"$STALE_SCP_MARKER" +exit 0 +EOF +chmod +x "$stale_bin/"* +stale_marker=$tmp/stale-scp-called +if PATH="$stale_bin:$PATH" STALE_STATUS_FILE=$current_manifest STALE_SCP_MARKER=$stale_marker \ + "$root/scripts/deploy-production-release.sh" "$manifest" >/dev/null 2>&1; then + echo 'stale production base passed deploy client validation' >&2 + exit 1 +fi +[[ ! -e $stale_marker ]] || { + echo 'stale deploy uploaded a manifest' >&2 + exit 1 +} + +if find "$root/.gitea/workflows" -type f -print -quit 2>/dev/null | grep -q .; then + echo 'automatic Gitea workflow still exists' >&2 + exit 1 +fi +grep -Fq 'stale release manifest' "$root/deploy/manual/easyai-ai-gateway-release" +grep -Fq 'production_changed=false' "$root/scripts/publish-release-images.sh" + +echo 'manual_release_tests=PASS'