refactor(release): 改为 Agent 双阶段人工发布
删除 Gitea Actions、Tag/Main 自动流水线和旧 Runner 配置，取消 Git 操作与发布授权的绑定。\n\n新增本地镜像发布、固定生产部署助手、digest manifest、迁移安全检查、simulation 冒烟及显式回滚流程。\n\n验证：pnpm lint、pnpm test、pnpm build、Go 全量测试、ShellCheck、Compose 配置、人工发布测试和 linux/amd64 完整栈冒烟。
This commit is contained in:
@@ -1,129 +0,0 @@
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
runs-on: easyai-gateway-ci-unprivileged-v2
|
||||
services:
|
||||
postgres:
|
||||
image: docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777
|
||||
env:
|
||||
POSTGRES_USER: easyai_test
|
||||
POSTGRES_HOST_AUTH_METHOD: trust
|
||||
POSTGRES_DB: easyai_gateway_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U easyai_test -d easyai_gateway_test"
|
||||
--health-interval 2s
|
||||
--health-timeout 5s
|
||||
--health-retries 30
|
||||
env:
|
||||
TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2
|
||||
AI_GATEWAY_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable
|
||||
AI_GATEWAY_TEST_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable
|
||||
steps:
|
||||
- name: Checkout without external Actions
|
||||
env:
|
||||
CI_REPOSITORY: ${{ github.repository }}
|
||||
CI_SERVER_URL: ${{ github.server_url }}
|
||||
CI_SHA: ${{ github.sha }}
|
||||
CI_JOB_TOKEN: ${{ github.token }}
|
||||
CI_EVENT_BEFORE: ${{ github.event.before }}
|
||||
CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
run: |
|
||||
set -eu
|
||||
test -n "$CI_JOB_TOKEN"
|
||||
authorization=$(printf 'x-access-token:%s' "$CI_JOB_TOKEN" | base64 | tr -d '\n')
|
||||
git init .
|
||||
git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \
|
||||
fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" "$CI_SHA"
|
||||
for comparison_sha in "$CI_EVENT_BEFORE" "$CI_PR_BASE_SHA"; do
|
||||
case "$comparison_sha" in
|
||||
[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*)
|
||||
if test "${#comparison_sha}" -eq 40 && \
|
||||
test "$comparison_sha" != 0000000000000000000000000000000000000000; then
|
||||
git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \
|
||||
fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" "$comparison_sha"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
unset authorization CI_JOB_TOKEN
|
||||
test ! -f .git/shallow
|
||||
git checkout --detach "$CI_SHA"
|
||||
test "$(git rev-parse HEAD)" = "$CI_SHA"
|
||||
- name: Verify pinned host toolchains
|
||||
run: |
|
||||
go version
|
||||
node --version
|
||||
pnpm --version
|
||||
docker-compose version
|
||||
shellcheck --version
|
||||
trivy --version
|
||||
govulncheck -version
|
||||
- name: Verify production migration safety
|
||||
env:
|
||||
CI_EVENT_NAME: ${{ github.event_name }}
|
||||
CI_EVENT_BEFORE: ${{ github.event.before }}
|
||||
CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
run: |
|
||||
production_base=$(cat deploy/ci/production-migration-base)
|
||||
immutable_base=$CI_EVENT_BEFORE
|
||||
if test "$CI_EVENT_NAME" = pull_request; then
|
||||
immutable_base=$CI_PR_BASE_SHA
|
||||
fi
|
||||
case "$immutable_base" in
|
||||
[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*)
|
||||
test "${#immutable_base}" -eq 40
|
||||
test "$immutable_base" != 0000000000000000000000000000000000000000
|
||||
;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
git merge-base --is-ancestor "$immutable_base" HEAD
|
||||
node ./scripts/ci-validate-migrations.mjs \
|
||||
"$production_base" "$immutable_base"
|
||||
- name: Verify Go formatting
|
||||
run: |
|
||||
unformatted=$(gofmt -l apps/api)
|
||||
test -z "$unformatted" || {
|
||||
printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2
|
||||
exit 1
|
||||
}
|
||||
- name: Migrate PostgreSQL 16 integration database
|
||||
working-directory: apps/api
|
||||
run: go run ./cmd/migrate
|
||||
- name: Verify Go code
|
||||
working-directory: apps/api
|
||||
env:
|
||||
GOFLAGS: "-p=1"
|
||||
GOMAXPROCS: "1"
|
||||
run: |
|
||||
go vet ./...
|
||||
go test ./...
|
||||
govulncheck ./...
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- run: pnpm lint
|
||||
- run: pnpm test
|
||||
- run: pnpm build
|
||||
- name: Audit JavaScript dependencies
|
||||
run: pnpm audit --audit-level high
|
||||
- name: Validate deployment configuration
|
||||
run: |
|
||||
docker-compose -f docker-compose.yml config --quiet
|
||||
shellcheck scripts/ci-build-images.sh scripts/ci-validate-semver.sh \
|
||||
scripts/provision-ci-runner.sh tests/ci/ci-build-images-test.sh \
|
||||
tests/ci/migrations-test.sh tests/ci/pipeline-test.sh \
|
||||
tests/ci/semver-test.sh
|
||||
./tests/ci/ci-build-images-test.sh
|
||||
./tests/ci/migrations-test.sh
|
||||
./tests/ci/pipeline-test.sh
|
||||
./tests/ci/semver-test.sh
|
||||
- name: Scan repository
|
||||
run: |
|
||||
trivy fs --scanners vuln,secret,misconfig --severity HIGH,CRITICAL \
|
||||
--ignore-unfixed --exit-code 1 --timeout 15m --skip-dirs .git \
|
||||
--skip-dirs node_modules .
|
||||
@@ -1,114 +0,0 @@
|
||||
name: release-ci
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ['v*']
|
||||
|
||||
jobs:
|
||||
verify-tag:
|
||||
runs-on: easyai-gateway-ci-unprivileged-v2
|
||||
services:
|
||||
postgres:
|
||||
image: docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777
|
||||
env:
|
||||
POSTGRES_USER: easyai_test
|
||||
POSTGRES_HOST_AUTH_METHOD: trust
|
||||
POSTGRES_DB: easyai_gateway_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U easyai_test -d easyai_gateway_test"
|
||||
--health-interval 2s
|
||||
--health-timeout 5s
|
||||
--health-retries 30
|
||||
env:
|
||||
TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2
|
||||
AI_GATEWAY_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable
|
||||
AI_GATEWAY_TEST_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable
|
||||
steps:
|
||||
- name: Checkout without external Actions
|
||||
env:
|
||||
CI_REPOSITORY: ${{ github.repository }}
|
||||
CI_SERVER_URL: ${{ github.server_url }}
|
||||
CI_SHA: ${{ github.sha }}
|
||||
CI_JOB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -eu
|
||||
test -n "$CI_JOB_TOKEN"
|
||||
authorization=$(printf 'x-access-token:%s' "$CI_JOB_TOKEN" | base64 | tr -d '\n')
|
||||
git init .
|
||||
git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \
|
||||
fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" "$CI_SHA"
|
||||
unset authorization CI_JOB_TOKEN
|
||||
test ! -f .git/shallow
|
||||
git checkout --detach FETCH_HEAD
|
||||
- name: Verify pinned host toolchains
|
||||
run: |
|
||||
go version
|
||||
node --version
|
||||
pnpm --version
|
||||
docker-compose version
|
||||
shellcheck --version
|
||||
trivy --version
|
||||
govulncheck -version
|
||||
- name: Verify release tag ancestry
|
||||
env:
|
||||
CI_REPOSITORY: ${{ github.repository }}
|
||||
CI_SERVER_URL: ${{ github.server_url }}
|
||||
CI_SHA: ${{ github.sha }}
|
||||
CI_JOB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -eu
|
||||
tag_name=${GITHUB_REF#refs/tags/}
|
||||
./scripts/ci-validate-semver.sh "$tag_name"
|
||||
authorization=$(printf 'x-access-token:%s' "$CI_JOB_TOKEN" | base64 | tr -d '\n')
|
||||
git -c "http.extraHeader=AUTHORIZATION: basic $authorization" \
|
||||
fetch --no-tags "$CI_SERVER_URL/$CI_REPOSITORY.git" \
|
||||
+refs/heads/main:refs/remotes/origin/main
|
||||
unset authorization CI_JOB_TOKEN
|
||||
test ! -f .git/shallow
|
||||
test "$(git rev-parse HEAD)" = "$CI_SHA"
|
||||
git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main
|
||||
- name: Verify production migration safety
|
||||
run: |
|
||||
production_base=$(cat deploy/ci/production-migration-base)
|
||||
node ./scripts/ci-validate-migrations.mjs "$production_base"
|
||||
- name: Verify Go formatting
|
||||
run: |
|
||||
unformatted=$(gofmt -l apps/api)
|
||||
test -z "$unformatted" || {
|
||||
printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2
|
||||
exit 1
|
||||
}
|
||||
- name: Migrate PostgreSQL 16 integration database
|
||||
working-directory: apps/api
|
||||
run: go run ./cmd/migrate
|
||||
- name: Verify Go code
|
||||
working-directory: apps/api
|
||||
env:
|
||||
GOFLAGS: "-p=1"
|
||||
GOMAXPROCS: "1"
|
||||
run: |
|
||||
go vet ./...
|
||||
go test ./...
|
||||
govulncheck ./...
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- run: pnpm lint
|
||||
- run: pnpm test
|
||||
- run: pnpm build
|
||||
- name: Audit JavaScript dependencies
|
||||
run: pnpm audit --audit-level high
|
||||
- name: Validate deployment configuration
|
||||
run: |
|
||||
docker-compose -f docker-compose.yml config --quiet
|
||||
shellcheck scripts/ci-build-images.sh scripts/ci-validate-semver.sh \
|
||||
scripts/provision-ci-runner.sh tests/ci/ci-build-images-test.sh \
|
||||
tests/ci/migrations-test.sh tests/ci/pipeline-test.sh \
|
||||
tests/ci/semver-test.sh
|
||||
./tests/ci/ci-build-images-test.sh
|
||||
./tests/ci/migrations-test.sh
|
||||
./tests/ci/pipeline-test.sh
|
||||
./tests/ci/semver-test.sh
|
||||
- name: Scan repository
|
||||
run: |
|
||||
trivy fs --scanners vuln,secret,misconfig --severity HIGH,CRITICAL \
|
||||
--ignore-unfixed --exit-code 1 --timeout 15m --skip-dirs .git \
|
||||
--skip-dirs node_modules .
|
||||
@@ -9,19 +9,28 @@
|
||||
3. `type` 使用小写英文，可选值为 `feat`、`fix`、`docs`、`test`、`refactor`、`perf`、`build`、`ci`、`chore`、`revert`。
|
||||
4. 提交摘要和正文必须使用中文；专有名词、协议名称、命令、路径、代码标识符和第三方原始错误可保留原文。
|
||||
5. 摘要应简洁明确，末尾不加句号，不得使用“更新代码”“修复问题”等无法说明意图的模糊描述。
|
||||
6. 非简单变更应在提交正文或 PR 描述中用中文说明原因、影响、风险和验证结果。
|
||||
6. 非简单变更应在提交正文中用中文说明原因、影响、风险和验证结果。
|
||||
|
||||
## 仓库边界
|
||||
|
||||
1. 后端位于 `apps/api`，前端位于 `apps/web`，共享 TypeScript 契约位于 `packages/contracts`。
|
||||
2. 修改 HTTP 接口、请求或响应类型后，必须执行 `pnpm openapi` 并提交匹配的 OpenAPI 产物。
|
||||
3. 数据库迁移只能新增，禁止修改已经进入生产基线的历史迁移；迁移必须通过生产迁移安全检查。
|
||||
4. 不得把 `.env`、密码、Secret、Token、授权码、私钥或生产凭据提交到 Git、日志、测试输出或验收证据中。
|
||||
3. 数据库迁移只能新增，禁止修改或删除已经进入 Git 历史的迁移；发布前必须相对当前线上 SHA 执行迁移安全检查。
|
||||
4. 不得把 `.env`、密码、Secret、Token、授权码、私钥或生产凭据提交到 Git、日志、测试输出、release manifest 或验收证据中。
|
||||
5. 当前工作区存在用户改动时必须保留，不得覆盖、清理、重置或混入当前任务提交；需要隔离时使用独立分支、克隆或 worktree。
|
||||
|
||||
## 验证要求
|
||||
|
||||
根据改动范围执行最小充分验证；准备合并或发布时执行完整门禁：
|
||||
根据改动范围执行最小充分验证。修改 Shell 脚本后必须执行 `bash -n` 和 ShellCheck；修改 Go 文件后必须确认 `gofmt -l` 没有输出。
|
||||
|
||||
本地发布的快速强制门禁由 `scripts/publish-release-images.sh` 固定执行：
|
||||
|
||||
```bash
|
||||
cd apps/api && env -u AI_GATEWAY_TEST_DATABASE_URL go test ./... -count=1
|
||||
node scripts/ci-validate-migrations.mjs <当前线上完整 Git SHA>
|
||||
```
|
||||
|
||||
完整测试、审计和镜像扫描不再由 Gitea 自动运行。需要合并大范围重构、准备安全审计或人工要求完整验证时执行：
|
||||
|
||||
```bash
|
||||
cd apps/api && go vet ./... && go test ./... && govulncheck ./...
|
||||
@@ -31,18 +40,27 @@ pnpm test
|
||||
pnpm build
|
||||
pnpm audit --audit-level high
|
||||
docker compose -f docker-compose.yml config --quiet
|
||||
./tests/ci/ci-build-images-test.sh
|
||||
./tests/ci/migrations-test.sh
|
||||
./tests/ci/pipeline-test.sh
|
||||
./tests/ci/semver-test.sh
|
||||
./tests/release/manual-release-test.sh
|
||||
```
|
||||
|
||||
修改 Shell 脚本后还必须执行 `bash -n` 和 ShellCheck。修改 Go 文件后必须确认 `gofmt -l` 没有输出。
|
||||
## Git 与人工发布
|
||||
|
||||
## Git 与 CI/CD
|
||||
1. 仓库不使用 Gitea Actions，不存在 Push、PR、Tag、Webhook、轮询或定时触发的自动构建和自动部署。
|
||||
2. `main` 不设置受保护分支或 required status；允许 Agent 在获得用户明确授权后直接提交并推送，但 commit 或 push 永远不等于发布授权。
|
||||
3. 发布只能使用工作区干净、已经提交并属于 `origin/main` 历史的完整 SHA。镜像 Tag 使用完整 SHA，线上只能使用 Registry digest，禁止使用 `latest`。
|
||||
4. 本地镜像发布必须由用户明确要求后单独执行：
|
||||
|
||||
1. 一个提交只包含一个逻辑变更，提交前检查暂存差异并确认不含敏感信息。
|
||||
2. `main` 只能通过短生命周期分支和 PR 合并，禁止直接推送或强制推送。
|
||||
3. PR 必须通过精确的 `ci / verify (pull_request)` 状态后才能合并；合并后还要确认同一 SHA 的 `ci / verify (push)` 成功。
|
||||
4. 生产版本仅使用稳定 SemVer `vMAJOR.MINOR.PATCH` 标签，并同时要求 `release-ci / verify-tag (push)` 成功。
|
||||
5. 未验证 protected tag、发布账本、数据库备份、健康检查和回滚路径时，不得声称 CI/CD 或生产发布已经完成。
|
||||
```bash
|
||||
./scripts/publish-release-images.sh --components auto
|
||||
```
|
||||
|
||||
该命令只能构建、冒烟、推送镜像和生成 `dist/releases/<SHA>.json`，不得修改生产。
|
||||
5. publish 成功后，Agent 必须报告 manifest、组件、digest 和验证结果并停止。只有用户再次明确确认上线，才可单独执行：
|
||||
|
||||
```bash
|
||||
./scripts/deploy-production-release.sh dist/releases/<SHA>.json
|
||||
```
|
||||
|
||||
6. deploy 前后必须验证线上基线、镜像架构/revision、数据库备份条件、内部及公网健康检查和自动应用回滚。未实际完成这些验证时不得声称生产发布成功。
|
||||
7. 回滚也必须由用户明确要求，只能选择服务器已有的历史 manifest；应用回滚不自动恢复数据库。
|
||||
|
||||
+3
-2
@@ -57,15 +57,16 @@ ENV APP_ENV=production \
|
||||
CMD ["/app/easyai-ai-gateway"]
|
||||
|
||||
FROM --platform=$BUILDPLATFORM ${NODE_BUILD_IMAGE} AS web-builder
|
||||
ARG NPM_CONFIG_REGISTRY=https://registry.npmjs.org
|
||||
|
||||
WORKDIR /src
|
||||
RUN npm install -g pnpm@10.18.1
|
||||
RUN npm install --registry "$NPM_CONFIG_REGISTRY" -g pnpm@10.18.1
|
||||
|
||||
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml nx.json ./
|
||||
COPY apps/web/package.json apps/web/package.json
|
||||
COPY packages/contracts/package.json packages/contracts/package.json
|
||||
RUN --mount=type=cache,id=pnpm-store,target=/root/.local/share/pnpm/store \
|
||||
pnpm install --frozen-lockfile
|
||||
pnpm install --frozen-lockfile --registry "$NPM_CONFIG_REGISTRY"
|
||||
|
||||
COPY packages packages
|
||||
COPY apps/web apps/web
|
||||
|
||||
@@ -59,9 +59,9 @@ OIDC 用户通过签名、Issuer、Audience、`tid`、Scope 和应用角色校
|
||||
|
||||
后端热更新可通过 `GO_WATCH_SHUTDOWN_GRACE_MS` 和 `GO_WATCH_RESTART_DELAY_MS` 调整旧进程退出等待时间与重启间隔。
|
||||
|
||||
## Docker Compose 一键部署
|
||||
## Docker Compose 本地运行
|
||||
|
||||
仓库内提供了面向 `linux/amd64` 的 Docker Compose 构建和部署脚本，会自动构建 API/Web 镜像、启动 PostgreSQL、执行数据库迁移，并验证 API 与 Web 是否可访问：
|
||||
仓库内的 Compose 脚本只用于本地构建和运行：启动 PostgreSQL、执行数据库迁移，并验证 API 与 Web 是否可访问。它不再包含 Registry 推送或生产部署能力：
|
||||
|
||||
```bash
|
||||
scripts/deploy-compose.sh
|
||||
@@ -78,28 +78,21 @@ scripts/deploy-compose.sh
|
||||
常用覆盖项：
|
||||
|
||||
```bash
|
||||
AI_GATEWAY_IMAGE_TAG=2026.05.23-1 scripts/deploy-compose.sh
|
||||
AI_GATEWAY_IMAGE_TAG=2026.05.23-1 AI_GATEWAY_PUSH=1 scripts/deploy-compose.sh
|
||||
AI_GATEWAY_IMAGE_TAG=2026.05.23-1 scripts/deploy-compose.sh push
|
||||
AI_GATEWAY_IMAGE_TAG=local-test scripts/deploy-compose.sh
|
||||
AI_GATEWAY_WEB_PORT=8080 AI_GATEWAY_API_PORT=18088 scripts/deploy-compose.sh
|
||||
AI_GATEWAY_GO_PROXY='https://proxy.golang.org,direct' scripts/deploy-compose.sh
|
||||
AI_GATEWAY_NPM_REGISTRY='https://registry.npmmirror.com' scripts/deploy-compose.sh
|
||||
AI_GATEWAY_SKIP_BUILD=1 scripts/deploy-compose.sh
|
||||
scripts/deploy-compose.sh down
|
||||
scripts/deploy-compose.sh clean
|
||||
```
|
||||
|
||||
默认镜像地址为：
|
||||
默认本地镜像地址为：
|
||||
|
||||
- API: `registry.cn-shanghai.aliyuncs.com/easyaigc/ai-gateway:latest`
|
||||
- Web: `registry.cn-shanghai.aliyuncs.com/easyaigc/ai-gateway-web:latest`
|
||||
- API: `registry.cn-shanghai.aliyuncs.com/easyaigc/ai-gateway:local`
|
||||
- Web: `registry.cn-shanghai.aliyuncs.com/easyaigc/ai-gateway-web:local`
|
||||
|
||||
执行 `scripts/deploy-compose.sh push` 或设置 `AI_GATEWAY_PUSH=1` 时，会同时推送当前版本 tag 和 `latest`。当前版本 tag 优先使用 `AI_GATEWAY_IMAGE_TAG`；如果没有设置，则使用根 `package.json` 里的 `version`。
|
||||
|
||||
推送前需要先登录阿里云镜像仓库：
|
||||
|
||||
```bash
|
||||
docker login --username=<your-aliyun-account> registry.cn-shanghai.aliyuncs.com
|
||||
```
|
||||
生产镜像禁止使用 `latest`，只能通过后文的人工 publish 命令推送完整 Git SHA Tag。
|
||||
|
||||
Web 容器的 Nginx 配置通过 bind mount 挂载自仓库文件 [docker/nginx.conf](docker/nginx.conf)，可直接修改该文件调整静态资源、规范 `/api/v1` 公开入口和旧 `/gateway-api` 兼容反向代理。修改后执行以下命令使配置生效：
|
||||
|
||||
@@ -107,9 +100,31 @@ Web 容器的 Nginx 配置通过 bind mount 挂载自仓库文件 [docker/nginx.
|
||||
docker compose -f docker-compose.yml restart web
|
||||
```
|
||||
|
||||
## 生产 CI/CD
|
||||
## 生产人工发布
|
||||
|
||||
Gitea Actions 会在隔离的 rootless DinD Runner 中对 Pull Request、`main` Push 和版本 Tag 执行完整质量门禁；Tag 使用独立的 `release-ci / verify-tag (push)` context，不能复用旧的 `main` 成功状态。源码 Job 没有宿主 Docker、`sudo` 或生产部署权限。部署仓的 root-owned dispatcher 只在相同 SHA 的 `main` 与 Tag context 都成功后，用固定命令构建并扫描镜像，再以 Registry digest 发布 `ai.51easyai.com`。安装 Runner、Fork PR 审批、发布验证和回滚步骤见 [生产 CI/CD 运行手册](docs/runbooks/production-ci-cd.md)，信任边界见 [ADR-001](docs/decisions/001-production-cicd.md)。
|
||||
本仓库没有 Gitea Actions、Webhook、Tag、`main` Push、轮询或定时发布。`main` 不使用受保护分支限制；commit、push 和 Tag 都不会构建镜像或更新生产。
|
||||
|
||||
生产发布由 Agent 在用户明确指令下分两次执行。第一步在本机构建 `linux/amd64` 镜像、运行临时 PostgreSQL + simulation API 冒烟、推送完整 SHA Tag，并生成带内容完整性校验的 digest-pinned manifest；该命令不会修改生产：
|
||||
|
||||
```bash
|
||||
docker login --username=<your-aliyun-account> registry.cn-shanghai.aliyuncs.com
|
||||
./scripts/publish-release-images.sh --components auto
|
||||
```
|
||||
|
||||
Agent 必须报告 `dist/releases/<SHA>.json` 后停止。用户再次明确确认上线后，才执行第二步：
|
||||
|
||||
```bash
|
||||
./scripts/deploy-production-release.sh dist/releases/<SHA>.json
|
||||
```
|
||||
|
||||
查看生产版本和显式回滚：
|
||||
|
||||
```bash
|
||||
./scripts/deploy-production-release.sh --status
|
||||
./scripts/deploy-production-release.sh --rollback <历史完整 Git SHA>
|
||||
```
|
||||
|
||||
完整安装、验证、失败处理和停用旧自动化步骤见[人工生产发布运行手册](docs/runbooks/production-ci-cd.md)，决策背景见 [ADR-003](docs/decisions/003-manual-agent-release.md)。
|
||||
|
||||
Compose 默认使用独立容器数据库 `postgres:18-alpine`，数据卷会保留在 `postgres_data` 和 `api_data`。为避免本地开发 `.env` 中的 `localhost` 数据库地址污染容器部署，compose 使用 `AI_GATEWAY_COMPOSE_*` 变量作为容器部署专用覆盖，例如：
|
||||
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
log:
|
||||
level: info
|
||||
|
||||
runner:
|
||||
file: /data/.runner
|
||||
capacity: 1
|
||||
envs:
|
||||
GOPROXY: "https://goproxy.cn,direct"
|
||||
GOSUMDB: "sum.golang.google.cn"
|
||||
PATH: "/opt/easyai-gateway-ci/bin:/opt/easyai-gateway-ci/toolchains/go/bin:/opt/easyai-gateway-ci/toolchains/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
timeout: 3h
|
||||
shutdown_timeout: 5m
|
||||
insecure: false
|
||||
fetch_timeout: 5s
|
||||
fetch_interval: 2s
|
||||
labels:
|
||||
- "easyai-gateway-ci-unprivileged-v2:docker://docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7"
|
||||
|
||||
cache:
|
||||
enabled: false
|
||||
|
||||
container:
|
||||
docker_host: "-"
|
||||
privileged: false
|
||||
force_pull: false
|
||||
force_rebuild: false
|
||||
require_docker: true
|
||||
docker_timeout: 1m
|
||||
bind_workdir: false
|
||||
workdir_parent: workspace
|
||||
options: "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro"
|
||||
valid_volumes:
|
||||
- "/opt/easyai-gateway-ci"
|
||||
|
||||
host:
|
||||
workdir_parent: /data/host-work
|
||||
@@ -1,44 +0,0 @@
|
||||
[Unit]
|
||||
Description=EasyAI AI Gateway rootless DinD Gitea Actions runner v2
|
||||
Requires=docker.service
|
||||
After=network-online.target docker.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
Group=root
|
||||
Environment=RUNNER_SECURITY_OPTIONS=
|
||||
Environment=RUNNER_IMAGE_REF=
|
||||
EnvironmentFile=-/etc/easyai-gateway-ci-v2/runner.env
|
||||
ExecStartPre=-/usr/bin/docker rm --force easyai-gateway-ci-v2-runner
|
||||
ExecStart=/usr/bin/docker run --rm --pull=never --name easyai-gateway-ci-v2-runner --privileged --cpus 2 --memory 2g --memory-swap 3g --pids-limit 1024 $RUNNER_SECURITY_OPTIONS --volume easyai-gateway-ci-v2-data:/data --volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro --volume /etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro --volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro --env CONFIG_FILE=/config.yaml --env DOCKERD_ROOTLESS_ROOTLESSKIT_NET=slirp4netns --env DOCKERD_ROOTLESS_ROOTLESSKIT_MTU=65520 --env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns $RUNNER_IMAGE_REF
|
||||
ExecStop=-/usr/bin/docker stop --time 360 easyai-gateway-ci-v2-runner
|
||||
Restart=always
|
||||
RestartSec=5s
|
||||
TimeoutStartSec=60s
|
||||
TimeoutStopSec=390s
|
||||
|
||||
# systemd only launches the host Docker client. Pull-request jobs run inside the
|
||||
# nested rootless daemon and never receive the host or nested Docker socket.
|
||||
NoNewPrivileges=true
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
PrivateTmp=true
|
||||
PrivateDevices=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ProtectKernelTunables=true
|
||||
ProtectKernelModules=true
|
||||
ProtectKernelLogs=true
|
||||
ProtectControlGroups=true
|
||||
ProtectClock=true
|
||||
ProtectHostname=true
|
||||
LockPersonality=true
|
||||
RestrictRealtime=true
|
||||
RestrictSUIDSGID=true
|
||||
SystemCallArchitectures=native
|
||||
UMask=0077
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1 +0,0 @@
|
||||
fce76a30ba2d7f9aa514418a2e09be7bbe7aaf4e
|
||||
Executable
+233
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
[[ $(id -u) -eq 0 ]] || { echo 'run as root' >&2; exit 1; }
|
||||
|
||||
config_file=${AI_GATEWAY_RELEASE_CONFIG:-/etc/easyai-ai-gateway-release.conf}
|
||||
manifest_tool=${AI_GATEWAY_RELEASE_MANIFEST_TOOL:-/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs}
|
||||
[[ -f $config_file && ! -L $config_file ]] || { echo 'missing release config' >&2; exit 1; }
|
||||
[[ -f $manifest_tool && ! -L $manifest_tool ]] || { echo 'missing release manifest validator' >&2; exit 1; }
|
||||
# shellcheck source=/dev/null
|
||||
source "$config_file"
|
||||
|
||||
: "${DEPLOY_DIR:?}"
|
||||
: "${COMPOSE_FILE:?}"
|
||||
: "${RELEASES_DIR:?}"
|
||||
: "${BACKUP_DIR:?}"
|
||||
: "${PUBLIC_BASE_URL:?}"
|
||||
: "${API_PORT:?}"
|
||||
: "${WEB_PORT:?}"
|
||||
: "${POSTGRES_SERVICE:?}"
|
||||
: "${POSTGRES_USER:?}"
|
||||
: "${POSTGRES_DATABASE:?}"
|
||||
: "${BACKUP_RETENTION:?}"
|
||||
|
||||
[[ $DEPLOY_DIR == /* && $COMPOSE_FILE == /* && $RELEASES_DIR == /* && $BACKUP_DIR == /* ]] || {
|
||||
echo 'release paths must be absolute' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ $BACKUP_RETENTION =~ ^[1-9][0-9]*$ ]] || { echo 'invalid BACKUP_RETENTION' >&2; exit 1; }
|
||||
[[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || { echo 'invalid PUBLIC_BASE_URL' >&2; exit 1; }
|
||||
[[ -d $DEPLOY_DIR && -f $COMPOSE_FILE ]] || { echo 'deployment directory is unavailable' >&2; exit 1; }
|
||||
|
||||
for command in docker curl node flock install; do
|
||||
command -v "$command" >/dev/null 2>&1 || { echo "$command is required" >&2; exit 1; }
|
||||
done
|
||||
docker info >/dev/null 2>&1 || { echo 'Docker Engine is unavailable' >&2; exit 1; }
|
||||
docker compose version >/dev/null 2>&1 || { echo 'Docker Compose v2 is required' >&2; exit 1; }
|
||||
|
||||
install -d -m 0700 "$RELEASES_DIR" "$BACKUP_DIR"
|
||||
exec 9>"$RELEASES_DIR/.release.lock"
|
||||
flock -n 9 || { echo 'another release operation is running' >&2; exit 1; }
|
||||
|
||||
compose=(docker compose --project-directory "$DEPLOY_DIR" -f "$COMPOSE_FILE")
|
||||
current_manifest=$RELEASES_DIR/current.json
|
||||
|
||||
manifest_get() {
|
||||
node "$manifest_tool" get "$1" "$2"
|
||||
}
|
||||
|
||||
validate_image() {
|
||||
local image=$1
|
||||
local source_sha=$2
|
||||
local require_revision=$3
|
||||
local revision architecture
|
||||
|
||||
[[ $image =~ @sha256:[0-9a-f]{64}$ ]] || { echo "image is not digest-pinned: $image" >&2; return 1; }
|
||||
docker pull --platform linux/amd64 "$image"
|
||||
architecture=$(docker image inspect "$image" --format '{{.Os}}/{{.Architecture}}')
|
||||
[[ $architecture == linux/amd64 ]] || { echo "unexpected image architecture: $architecture" >&2; return 1; }
|
||||
if [[ $require_revision == true ]]; then
|
||||
revision=$(docker image inspect "$image" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')
|
||||
[[ $revision == "$source_sha" ]] || { echo "image revision does not match release: $image" >&2; return 1; }
|
||||
fi
|
||||
}
|
||||
|
||||
wait_for_url() {
|
||||
local label=$1
|
||||
local url=$2
|
||||
local expected=${3:-}
|
||||
local body=
|
||||
|
||||
for _ in $(seq 1 60); do
|
||||
if body=$(curl -fsS --max-time 5 "$url" 2>/dev/null); then
|
||||
if [[ -z $expected || $body == *"$expected"* ]]; then
|
||||
echo "[release] verified $label"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "release probe failed: $label ($url)" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
verify_runtime() {
|
||||
wait_for_url 'internal API health' "http://127.0.0.1:$API_PORT/api/v1/healthz" 'easyai-ai-gateway'
|
||||
wait_for_url 'internal API readiness' "http://127.0.0.1:$API_PORT/api/v1/readyz" '"ok":true'
|
||||
wait_for_url 'internal OpenAPI' "http://127.0.0.1:$API_PORT/api/v1/openapi.json" '"/api/v1/chat/completions"'
|
||||
wait_for_url 'Web API reverse proxy' "http://127.0.0.1:$WEB_PORT/api/v1/healthz" 'easyai-ai-gateway'
|
||||
wait_for_url 'Web application' "http://127.0.0.1:$WEB_PORT/" 'EasyAI AI Gateway'
|
||||
wait_for_url 'public API health' "$PUBLIC_BASE_URL/api/v1/healthz" 'easyai-ai-gateway'
|
||||
wait_for_url 'public API readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'
|
||||
wait_for_url 'public OpenAPI' "$PUBLIC_BASE_URL/api/v1/openapi.json" '"/api/v1/chat/completions"'
|
||||
}
|
||||
|
||||
activate_images() {
|
||||
local api_image=$1
|
||||
local web_image=$2
|
||||
export AI_GATEWAY_API_IMAGE=$api_image
|
||||
export AI_GATEWAY_WEB_IMAGE=$web_image
|
||||
"${compose[@]}" up -d --no-build --pull always api web
|
||||
}
|
||||
|
||||
restore_previous() {
|
||||
local previous=$1
|
||||
[[ -f $previous ]] || return 1
|
||||
local previous_api previous_web
|
||||
previous_api=$(manifest_get "$previous" images.api)
|
||||
previous_web=$(manifest_get "$previous" images.web)
|
||||
echo '[release] restoring previous application images' >&2
|
||||
activate_images "$previous_api" "$previous_web"
|
||||
verify_runtime
|
||||
}
|
||||
|
||||
backup_database() {
|
||||
local source_sha=$1
|
||||
local temp_backup=$BACKUP_DIR/.${source_sha}.dump.tmp
|
||||
local final_backup=$BACKUP_DIR/${source_sha}.dump
|
||||
|
||||
umask 077
|
||||
"${compose[@]}" exec -T "$POSTGRES_SERVICE" \
|
||||
pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DATABASE" -Fc >"$temp_backup"
|
||||
[[ -s $temp_backup ]] || { echo 'database backup is empty' >&2; rm -f "$temp_backup"; return 1; }
|
||||
"${compose[@]}" exec -T "$POSTGRES_SERVICE" pg_restore -l <"$temp_backup" >/dev/null
|
||||
mv "$temp_backup" "$final_backup"
|
||||
mapfile -t expired < <(find "$BACKUP_DIR" -maxdepth 1 -type f -name '*.dump' -printf '%T@ %p\n' | sort -rn | awk -v keep="$BACKUP_RETENTION" 'NR > keep { sub(/^[^ ]+ /, ""); print }')
|
||||
if [[ ${#expired[@]} -gt 0 ]]; then
|
||||
rm -f -- "${expired[@]}"
|
||||
fi
|
||||
echo "[release] verified database backup: $final_backup"
|
||||
}
|
||||
|
||||
record_current() {
|
||||
local manifest=$1
|
||||
local action=$2
|
||||
local started_at=$3
|
||||
local source_sha recorded temp
|
||||
source_sha=$(manifest_get "$manifest" sourceSha)
|
||||
recorded=$RELEASES_DIR/$source_sha.json
|
||||
temp=$RELEASES_DIR/.${source_sha}.json.tmp
|
||||
rm -f "$temp"
|
||||
RELEASE_DEPLOYED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \
|
||||
RELEASE_DEPLOY_DURATION_SECONDS=$(( $(date +%s) - started_at )) \
|
||||
RELEASE_DEPLOY_ACTION=$action \
|
||||
node "$manifest_tool" record-deployment "$manifest" "$temp" >/dev/null
|
||||
install -m 0600 "$temp" "$recorded"
|
||||
install -m 0600 "$recorded" "$current_manifest.tmp"
|
||||
mv "$current_manifest.tmp" "$current_manifest"
|
||||
rm -f "$temp"
|
||||
}
|
||||
|
||||
deploy_manifest() {
|
||||
local manifest=$1
|
||||
local action=${2:-deploy}
|
||||
local source_sha base_sha api_image web_image migrations_changed current_sha
|
||||
local changed_components api_changed=false web_changed=false
|
||||
local previous_copy=
|
||||
local started_at
|
||||
|
||||
[[ -f $manifest && ! -L $manifest ]] || { echo 'manifest must be a regular file' >&2; return 1; }
|
||||
node "$manifest_tool" validate "$manifest"
|
||||
source_sha=$(manifest_get "$manifest" sourceSha)
|
||||
base_sha=$(manifest_get "$manifest" baseReleaseSha)
|
||||
api_image=$(manifest_get "$manifest" images.api)
|
||||
web_image=$(manifest_get "$manifest" images.web)
|
||||
migrations_changed=$(manifest_get "$manifest" migrationsChanged)
|
||||
changed_components=$(manifest_get "$manifest" components)
|
||||
[[ $changed_components == *'"api"'* ]] && api_changed=true
|
||||
[[ $changed_components == *'"web"'* ]] && web_changed=true
|
||||
|
||||
if [[ $action == deploy ]]; then
|
||||
if [[ -f $current_manifest ]]; then
|
||||
current_sha=$(manifest_get "$current_manifest" sourceSha)
|
||||
[[ $current_sha == "$base_sha" ]] || {
|
||||
echo "stale release manifest: production=$current_sha manifest_base=$base_sha" >&2
|
||||
return 1
|
||||
}
|
||||
else
|
||||
[[ -z $base_sha ]] || { echo 'non-bootstrap manifest has no production base' >&2; return 1; }
|
||||
fi
|
||||
fi
|
||||
|
||||
validate_image "$api_image" "$source_sha" "$api_changed"
|
||||
validate_image "$web_image" "$source_sha" "$web_changed"
|
||||
|
||||
if [[ -f $current_manifest ]]; then
|
||||
previous_copy=$(mktemp "$RELEASES_DIR/.previous.XXXXXX")
|
||||
install -m 0600 "$current_manifest" "$previous_copy"
|
||||
fi
|
||||
started_at=$(date +%s)
|
||||
|
||||
if [[ $action == deploy && $migrations_changed == true ]]; then
|
||||
backup_database "$source_sha"
|
||||
export AI_GATEWAY_API_IMAGE=$api_image
|
||||
export AI_GATEWAY_WEB_IMAGE=$web_image
|
||||
"${compose[@]}" run --rm --no-deps migrator
|
||||
fi
|
||||
|
||||
if ! activate_images "$api_image" "$web_image" || ! verify_runtime; then
|
||||
if [[ -n $previous_copy ]]; then
|
||||
restore_previous "$previous_copy" || echo 'automatic application rollback also failed' >&2
|
||||
fi
|
||||
rm -f "$previous_copy"
|
||||
return 1
|
||||
fi
|
||||
|
||||
record_current "$manifest" "$action" "$started_at"
|
||||
rm -f "$previous_copy"
|
||||
echo "production_release=PASS action=$action release=$source_sha"
|
||||
}
|
||||
|
||||
case ${1:-} in
|
||||
status)
|
||||
[[ $# -eq 1 ]] || exit 64
|
||||
[[ -f $current_manifest ]] || { echo 'no production release manifest' >&2; exit 1; }
|
||||
node "$manifest_tool" validate "$current_manifest" >/dev/null
|
||||
cat "$current_manifest"
|
||||
;;
|
||||
deploy)
|
||||
[[ $# -eq 2 ]] || exit 64
|
||||
deploy_manifest "$2" deploy
|
||||
;;
|
||||
rollback)
|
||||
[[ $# -eq 2 && $2 =~ ^[0-9a-f]{40}$ ]] || exit 64
|
||||
target=$RELEASES_DIR/$2.json
|
||||
[[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; }
|
||||
deploy_manifest "$target" rollback
|
||||
;;
|
||||
*)
|
||||
echo 'usage: easyai-ai-gateway-release {status|deploy <manifest>|rollback <sha>}' >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,12 @@
|
||||
# Install as /etc/easyai-ai-gateway-release.conf with mode 0600 and root ownership.
|
||||
DEPLOY_DIR=/root/easyai-ai-gateway-deploy
|
||||
COMPOSE_FILE=/root/easyai-ai-gateway-deploy/docker-compose.yml
|
||||
RELEASES_DIR=/root/easyai-ai-gateway-deploy/.releases
|
||||
BACKUP_DIR=/var/backups/easyai-ai-gateway
|
||||
PUBLIC_BASE_URL=https://ai.51easyai.com
|
||||
API_PORT=8088
|
||||
WEB_PORT=5178
|
||||
POSTGRES_SERVICE=postgres
|
||||
POSTGRES_USER=easyai
|
||||
POSTGRES_DATABASE=easyai_ai_gateway
|
||||
BACKUP_RETENTION=5
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
[[ $(id -u) -eq 0 ]] || { echo 'run as root on the production server' >&2; exit 1; }
|
||||
|
||||
install -d -m 0755 /usr/local/lib/easyai-ai-gateway-release
|
||||
install -m 0755 "$root/deploy/manual/easyai-ai-gateway-release" \
|
||||
/usr/local/sbin/easyai-ai-gateway-release
|
||||
install -m 0644 "$root/scripts/release-manifest.mjs" \
|
||||
/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs
|
||||
if [[ ! -e /etc/easyai-ai-gateway-release.conf ]]; then
|
||||
install -m 0600 "$root/deploy/manual/easyai-ai-gateway-release.conf.example" \
|
||||
/etc/easyai-ai-gateway-release.conf
|
||||
echo 'created /etc/easyai-ai-gateway-release.conf; review it before the first release'
|
||||
fi
|
||||
|
||||
echo 'installed manual release helper; no service, timer, webhook, or watcher was enabled'
|
||||
@@ -105,6 +105,7 @@ services:
|
||||
VITE_GATEWAY_API_BASE_URL: ${AI_GATEWAY_WEB_API_BASE_URL:-/gateway-api}
|
||||
NODE_BUILD_IMAGE: ${AI_GATEWAY_NODE_BUILD_IMAGE:-node:22-alpine}
|
||||
WEB_RUNTIME_IMAGE: ${AI_GATEWAY_WEB_RUNTIME_IMAGE:-nginx:1.27-alpine}
|
||||
NPM_CONFIG_REGISTRY: ${AI_GATEWAY_NPM_REGISTRY:-https://registry.npmjs.org}
|
||||
VITE_BASE_PATH: ${AI_GATEWAY_WEB_BASE_PATH:-/}
|
||||
ports:
|
||||
- "${AI_GATEWAY_WEB_PORT:-5178}:80"
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
## 状态
|
||||
|
||||
Accepted
|
||||
Superseded by [ADR-003](003-manual-agent-release.md)
|
||||
|
||||
## 日期
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
# ADR-003: 本地 Agent 双阶段人工发布
|
||||
|
||||
## 状态
|
||||
|
||||
Accepted；取代 ADR-001 的自动 CI/CD 决策。
|
||||
|
||||
## 日期
|
||||
|
||||
2026-07-22
|
||||
|
||||
## 背景
|
||||
|
||||
AI Gateway 的交付重点是 API 制品能够启动、迁移数据库并通过核心兼容接口。原有 PR、`main` 和 Tag 三套重复质量门禁占用单并发 Runner，发布还要等待外部 dispatcher，反馈与上线时间不符合当前小团队由 Agent 主导运维的模式。
|
||||
|
||||
## 决策
|
||||
|
||||
- 删除所有 Gitea Actions，不监听 Push、PR、Tag、Webhook、轮询或定时事件。`main` 不设置保护规则或 required status，允许经用户授权的 Agent 直接提交和推送。
|
||||
- Git 操作与生产授权彻底分离。commit、push、merge 和 Tag 都不会构建或部署。
|
||||
- 发布分成两次独立的人工作业：本地 `publish` 负责验证、构建、simulation 冒烟、推送不可变镜像并生成带 SHA-256 内容完整性校验的 JSON manifest；用户再次确认后，`deploy` 才把该 manifest 应用到生产。
|
||||
- 只允许发布工作区干净、已提交且属于 `origin/main` 历史的完整 SHA。Registry Tag 使用完整 SHA，生产 Compose 只接受 `repository@sha256:...`。
|
||||
- publish 相对当前线上 manifest 选择 API/Web 变更；未变化的组件沿用现有 digest。两阶段之间如果线上基线变化，客户端和服务器都拒绝过期 manifest。
|
||||
- 待发布镜像在本地临时 PostgreSQL 中执行全部迁移，并逐项记录 health、readiness、OpenAPI、鉴权、模型列表、Chat Completions、Responses、Images 和 Web 反代冒烟。生成接口使用 5ms simulation，不调用真实上游。
|
||||
- 只有迁移变化时生产才创建并验证 PostgreSQL custom-format 备份。生产探活失败自动恢复上一应用 digest；数据库恢复仍需人工决定。
|
||||
- 完整测试、依赖审计和漏洞扫描保留为按需人工命令，不再影响镜像发布关键路径。
|
||||
|
||||
## 影响
|
||||
|
||||
- 不再有自动质量证据或自动发布；用户和 Agent 必须严格遵守两次显式授权边界。
|
||||
- 允许直接推送 `main` 提高速度，也取消了服务端代码审查强制约束。可追溯性改由干净提交、main 历史、OCI revision、Registry digest 和服务器 release manifest 保证。
|
||||
- 本机是 `Darwin arm64`，发布脚本必须使用 Buildx 构建并实际运行 `linux/amd64` 镜像。
|
||||
- Registry、SSH 或生产 helper 不可用时发布失败关闭，不允许退回 `latest`、重建制品或直接修改 Compose。
|
||||
+100
-133
@@ -1,167 +1,134 @@
|
||||
# 生产 CI/CD 运行手册
|
||||
# 人工生产发布运行手册
|
||||
|
||||
## 信任边界与固定资源
|
||||
## 原则
|
||||
|
||||
生产发布没有自动触发器。允许 Agent 直接提交和推送 `main`，但 Git 操作不会授权 publish 或 deploy：
|
||||
|
||||
1. 用户明确要求发布镜像后，Agent 执行本地 publish。
|
||||
2. Agent 报告 manifest、digest、组件和冒烟结果并停止。
|
||||
3. 用户再次明确确认上线后，Agent 执行 deploy。
|
||||
|
||||
不得把两步放进同一个命令、Git hook、Gitea Action、Webhook、服务、Timer 或轮询脚本。
|
||||
|
||||
## 一次性停用旧自动化
|
||||
|
||||
源码仓已经删除 `.gitea/workflows` 中的全部工作流，`main` 分支保护和 required status 也必须保持关闭。可用下列只读命令验收：
|
||||
|
||||
```bash
|
||||
tea branches list --repo BCAI/easyai-ai-gateway --output yaml | sed -n '/^- name: main$/,+3p'
|
||||
```
|
||||
|
||||
预期 `protected: "false"` 且 `user-can-push: "true"`。
|
||||
|
||||
确认没有运行中任务后，在旧 Runner 主机执行：
|
||||
|
||||
```bash
|
||||
systemctl disable --now easyai-gateway-ci-v2-runner.service
|
||||
systemctl is-active easyai-gateway-ci-v2-runner.service
|
||||
```
|
||||
|
||||
预期第二条命令返回 `inactive`。保留 `easyai-gateway-ci-v2-data` volume 观察一段时间，不立即删除。
|
||||
|
||||
生产部署仓不得保留 Tag/Main dispatcher、Webhook receiver 或 Timer。先只读列出相关单元，确认精确名称后逐个 `disable --now`，不得用模糊匹配批量停止其他 EasyAI 服务：
|
||||
|
||||
```bash
|
||||
systemctl list-unit-files --type=service --type=timer | grep -Ei 'easyai.*(release|dispatch|deploy)'
|
||||
```
|
||||
|
||||
## 安装固定生产助手
|
||||
|
||||
生产助手只解析 manifest、拉取 digest、备份/迁移、更新 Compose、探活和回滚；它不是常驻服务，也不读取源码 checkout。
|
||||
|
||||
把以下三个已审查文件复制到生产服务器：
|
||||
|
||||
- `deploy/manual/easyai-ai-gateway-release`
|
||||
- `deploy/manual/easyai-ai-gateway-release.conf.example`
|
||||
- `scripts/release-manifest.mjs`
|
||||
|
||||
在可信 checkout 中可运行一次安装器：
|
||||
|
||||
```bash
|
||||
sudo ./deploy/manual/install-release-helper.sh
|
||||
sudoedit /etc/easyai-ai-gateway-release.conf
|
||||
sudo /usr/local/sbin/easyai-ai-gateway-release status
|
||||
```
|
||||
|
||||
配置文件必须为 root 所有、`0600`。助手安装后不会创建或启用 service、timer、webhook 或 watcher。
|
||||
|
||||
默认生产资源：
|
||||
|
||||
- 源码仓：`https://git.51easyai.com/BCAI/easyai-ai-gateway`
|
||||
- CI Runner：`easyai-gateway-ci-v2-runner.service`
|
||||
- CI 标签：`easyai-gateway-ci-unprivileged-v2`
|
||||
- Runner 状态：Docker named volume `easyai-gateway-ci-v2-data` 内的 `/data/.runner`
|
||||
- 只读工具链：`/opt/easyai-gateway-ci`
|
||||
- Runner 配置：`/etc/easyai-gateway-ci-v2/config.yaml`
|
||||
- Runner 运行参数：`/etc/easyai-gateway-ci-v2/runner.env`
|
||||
- 生产入口：`https://ai.51easyai.com`
|
||||
- 部署目录：`/root/easyai-ai-gateway-deploy`
|
||||
- 发布清单：`/root/easyai-ai-gateway-deploy/.releases`
|
||||
- 数据库备份：`/var/backups/easyai-ai-gateway`
|
||||
- 公网入口：`https://ai.51easyai.com`
|
||||
|
||||
源码仓只执行质量 CI：PR context 是 `ci / verify (pull_request)`，`main` context 是 `ci / verify (push)`，版本 Tag 使用独立 workflow/context `release-ci / verify-tag (push)`。三者都执行生产迁移兼容门禁、Go 格式/vet/test/govulncheck、前端 lint/test/build、依赖审计、Compose 校验、Trivy 仓库扫描和 CI 脚本自测。Job 没有 Docker socket、`sudo` 或部署目录权限，也不构建发布镜像。
|
||||
## 阶段一：本地 publish
|
||||
|
||||
生产 CD 由部署仓的 root-owned dispatcher 执行。它在相同源码 SHA 上同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)` 成功，然后只运行部署仓中固定的 BuildKit 构建、Trivy 镜像扫描、digest 发布、备份、迁移、健康检查和回滚命令；不得执行 Tag checkout 中的 `pnpm`、测试、shell 脚本或应用程序。
|
||||
|
||||
provision 安装 checksum-pinned 的官方静态 ShellCheck `0.11.0` 和 Docker Compose `5.3.1`，不会依赖宿主是否预装这些命令。所有下载都在使用前校验固定 SHA-256，并在精确 Node Job 镜像内实际执行版本检查。PostgreSQL 16 集成库作为固定 digest 的 Actions service container 运行，PR Job 不接收宿主或内层 Docker API。
|
||||
|
||||
## 首次安装或修复 CI Runner
|
||||
|
||||
前置条件：Linux x86_64、可用 Docker Engine、root 权限，以及 Docker 文件系统至少 8 GiB 可用空间。脚本只压缩本项目专属 `easyai-gateway-ci` Buildx 缓存，不运行全局 `docker system prune`，因为该服务器与其他服务共享 Docker 状态。
|
||||
|
||||
从 Gitea 仓库设置的 Actions Runner 页面取得一次性仓库注册 Token，在服务器可信的源码 checkout 中执行：
|
||||
前置条件：Docker Desktop/Engine、Buildx、Compose v2、Node、Go、可用 Registry 登录和生产只读 SSH。源码必须无 tracked/untracked 改动，HEAD 必须属于最新获取的 `origin/main` 历史。
|
||||
|
||||
```bash
|
||||
RUNNER_REGISTRATION_TOKEN='<一次性 Token>' ./scripts/provision-ci-runner.sh
|
||||
docker login --username=<your-aliyun-account> registry.cn-shanghai.aliyuncs.com
|
||||
./scripts/publish-release-images.sh --components auto
|
||||
```
|
||||
|
||||
已有有效 named volume 时可直接重跑，不再需要 Token：
|
||||
`auto` 读取线上当前 manifest，与 HEAD 比较后选择：
|
||||
|
||||
- `api`：API、Go workspace 或迁移变化；
|
||||
- `web`：Web、contracts、pnpm 或 Nginx 容器配置变化；
|
||||
- `all`：Dockerfile/Compose 等共享运行时变化；
|
||||
- `none`：无运行时变化，不产生发布物。
|
||||
|
||||
首次发布或线上状态不可验证时只能显式使用：
|
||||
|
||||
```bash
|
||||
./scripts/provision-ci-runner.sh
|
||||
./scripts/publish-release-images.sh --components all
|
||||
```
|
||||
|
||||
宿主无法直连 Docker Hub 时，可使用可信 HTTPS 镜像代理；Runner 来源仍必须携带脚本中完全相同的 manifest digest，内层 daemon 拉取的 Job 与 service 镜像也会继续校验工作流固定的 digest：
|
||||
此路径保守地把迁移标记为变化：生产助手会先备份数据库，再运行新 API 镜像内的幂等 migrator。
|
||||
|
||||
```bash
|
||||
CI_RUNNER_IMAGE_SOURCE='docker.m.daocloud.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a' \
|
||||
CI_NESTED_DOCKER_REGISTRY_MIRROR='https://docker.m.daocloud.io' \
|
||||
./scripts/provision-ci-runner.sh
|
||||
```
|
||||
|
||||
注册 Token 通过交互 stdin 送给短命注册容器，不出现在宿主或容器的命令行参数中；provision 随后立即从环境中清除它，长期 systemd unit 也不包含 Token。不要输出或复制 `/data/.runner` 内容。脚本会停用旧 host Runner，并确保旧 Runner 用户和 v2 遗留用户不属于 `docker`/`sudo` 组。
|
||||
|
||||
默认磁盘门禁可用环境变量提高，但生产环境不应降低：
|
||||
|
||||
```bash
|
||||
CI_RUNNER_MIN_FREE_GIB=8 \
|
||||
CI_RUNNER_MIN_POST_INSTALL_FREE_GIB=4 \
|
||||
RUNNER_REGISTRATION_TOKEN='<一次性 Token>' \
|
||||
./scripts/provision-ci-runner.sh
|
||||
```
|
||||
|
||||
## AppArmor 分支
|
||||
|
||||
Ubuntu 22.04 默认没有 `kernel.apparmor_restrict_unprivileged_userns` 限制，当前服务器没有 `rootlesskit` profile 是受支持路径。provision 会实测 Docker 是否接受该 profile；不存在时写入：
|
||||
publish 会执行快速 Go 测试、迁移安全校验、`linux/amd64` 构建、临时完整栈迁移和 API simulation 冒烟。全部通过后才推送完整 SHA Tag，并生成带 SHA-256 内容完整性字段的固定 schema manifest：
|
||||
|
||||
```text
|
||||
RUNNER_SECURITY_OPTIONS=""
|
||||
dist/releases/<40 位 Git SHA>.json
|
||||
```
|
||||
|
||||
如果新系统的 `/proc/sys/kernel/apparmor_restrict_unprivileged_userns` 为 `1`，则必须先由宿主发行版或 Docker 安装包提供并加载 `rootlesskit` profile；provision 只探测和使用，不会自行复制一份 profile。缺失时 provision 会停止；不得手工关闭该 sysctl，也不得使用 `apparmor=unconfined` 绕过。
|
||||
成功输出必须包含 `production_changed=false`。此时 Agent 应报告结果并停止，不得继续上线。
|
||||
|
||||
参考：[Docker Rootless Docker-in-Docker](https://docs.docker.com/engine/security/rootless/tips/#rootless-docker-in-docker)、[Ubuntu 安全特性版本矩阵](https://documentation.ubuntu.com/security/security-features/security-features-overview/)。
|
||||
## 阶段二：人工 deploy
|
||||
|
||||
## Runner 真实验证
|
||||
|
||||
安装脚本会先启动一个无 Token、无持久 Runner 状态挂载、无宿主 bind mount 的短命 rootless DinD probe；随后启动长期服务，确认 Runner 已向 Gitea declare、通过内层 `docker info` 验证 `name=rootless`，再从内层 daemon 拉取固定 digest 的 Job 镜像，并在该镜像中实际运行全部工具。人工复核：
|
||||
收到用户再次确认后执行：
|
||||
|
||||
```bash
|
||||
systemctl is-active easyai-gateway-ci-v2-runner.service
|
||||
systemctl is-enabled easyai-gateway-ci-v2-runner.service
|
||||
systemctl cat easyai-gateway-ci-v2-runner.service
|
||||
journalctl -u easyai-gateway-ci-v2-runner.service --since '15 minutes ago' --no-pager
|
||||
|
||||
docker inspect -f 'user={{.Config.User}} privileged={{.HostConfig.Privileged}} apparmor={{.AppArmorProfile}}' \
|
||||
easyai-gateway-ci-v2-runner
|
||||
docker inspect -f '{{range .Mounts}}{{println .Source " -> " .Destination .Mode}}{{end}}' \
|
||||
easyai-gateway-ci-v2-runner
|
||||
docker exec easyai-gateway-ci-v2-runner docker info \
|
||||
--format '{{range .SecurityOptions}}{{println .}}{{end}}'
|
||||
docker exec easyai-gateway-ci-v2-runner docker image inspect \
|
||||
'docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7' \
|
||||
--format '{{.Id}}'
|
||||
./scripts/deploy-production-release.sh dist/releases/<40 位 Git SHA>.json
|
||||
```
|
||||
|
||||
预期：外层 image user 为 `rootless`、外层因 DinD 为 `privileged=true`、内层安全选项含 `name=rootless`；RootlessKit 使用独立 PID namespace，内层 `--pid=host` 也不能看到外层 Runner；挂载仅包含专用 `/data`、只读配置和只读工具链，不能出现 `/var/run/docker.sock` 或 `/run/docker.sock`。
|
||||
客户端和生产助手都会确认当前生产 SHA 仍等于 manifest 的 `baseReleaseSha`。不相等说明 publish 之后生产发生变化，必须重新 publish，不能覆盖或使用强制参数。
|
||||
|
||||
在 Gitea 中分别验证：
|
||||
助手随后验证：
|
||||
|
||||
1. 同仓 PR 的 `ci / verify (pull_request)` 成功。
|
||||
2. `main` Push 的 `ci / verify (push)` 成功。
|
||||
3. 合法 SemVer Tag 新产生独立 `release-ci / verify-tag (push)`；非法或不属于 `main` 历史的 Tag 失败，且 Tag context 不能被旧 `main` context 替代。
|
||||
4. Fork PR 默认不自动调度，维护者先检查 `.gitea/workflows/**` 和容器选项后再批准。
|
||||
- API/Web 均为完整 Registry digest；
|
||||
- 镜像为 `linux/amd64`；
|
||||
- 本次变化组件的 OCI revision 等于 release SHA；
|
||||
- 迁移变化时备份可被 `pg_restore -l` 读取；
|
||||
- 内部 API health/readiness/OpenAPI；
|
||||
- Web 页面和 Web API 反代；
|
||||
- 公网 health/readiness/OpenAPI。
|
||||
|
||||
PR 与 `main` Push workflow 都校验事件携带的 base/before SHA 必须是触发 SHA 的祖先；分支落后或主干历史被改写时 CI 会 fail-closed。PR 先合并最新 `main` 再重跑，Gitea 分支保护同时启用 `block_on_outdated_branch`。
|
||||
失败时自动恢复上一组应用 digest 并重复探活。已经执行的 expand-only 数据库迁移不会自动恢复。
|
||||
|
||||
## Fork PR 残余风险
|
||||
|
||||
rootless DinD 不是 VM 沙箱。外层仍按官方要求使用 `--privileged`，Gitea runner 也会解析 PR 控制的 `jobs.<job>.container.options`。配置已禁止 Job privileged、宿主/内层 Docker socket 和任意 bind source，但 Runner、rootless Docker、容器运行时或内核漏洞仍可能突破边界；Job 的出站网络和资源消耗也不是强租户隔离。因此 Fork PR 必须维护者批准，CI 不得注入生产 Secret；未来应迁移到独立 CI 主机或虚拟机。
|
||||
|
||||
## 数据库迁移兼容门禁
|
||||
|
||||
`deploy/ci/production-migration-base` 必须等于当前生产 API 对应的完整源码 SHA。`scripts/ci-validate-migrations.mjs` 只允许新增、按文件名严格追加且为普通文件的迁移；生产已知迁移以及已经进入 `main` 的迁移不可修改或删除。DROP、TRUNCATE、DELETE、MERGE、CREATE OR REPLACE、动态 EXECUTE、事务控制、DO/函数/过程定义、RENAME、列类型变更、增加/设置 `NOT NULL`、删除默认值和 REVOKE 会直接使 CI 失败。需要过程式迁移时先拆成可审计的声明式 expand/contract SQL；不能把过程体藏在普通或 dollar-quoted 字符串中。
|
||||
|
||||
该规则允许 INSERT/UPDATE 等数据迁移，因此不能替代备份和恢复验证。每次生产发布成功并完成 `pg_restore` 演练后，创建一个只前移 `deploy/ci/production-migration-base` 的受审 PR；不要把基线指向尚未成功部署的提交。
|
||||
|
||||
## 发布生产版本
|
||||
|
||||
1. 合并到受保护 `main`，确认源码 CI 成功。
|
||||
2. 在已验证提交上创建完整 SemVer Tag 并推送。
|
||||
3. 确认同一 SHA 的 `ci / verify (push)` 和本次 `release-ci / verify-tag (push)` 都成功。
|
||||
4. 由部署仓 root-owned dispatcher 构建、扫描并发布 digest；源码仓工作流不会直接调用生产 helper。
|
||||
5. 发布、健康检查和恢复演练完成后，用单独 PR 把 `deploy/ci/production-migration-base` 更新为该生产 Tag 的完整源码 SHA。
|
||||
## 状态与回滚
|
||||
|
||||
```bash
|
||||
git switch main
|
||||
git pull --ff-only
|
||||
git tag -a v0.1.1 -m 'release: v0.1.1'
|
||||
git push origin v0.1.1
|
||||
./scripts/deploy-production-release.sh --status
|
||||
./scripts/deploy-production-release.sh --rollback <服务器已有的历史 SHA>
|
||||
```
|
||||
|
||||
dispatcher 以完整 Git SHA 发布 Registry Tag，并把 Registry 返回的 digest 写入服务器 root-only 发布清单。生产 Compose 使用 `repository@sha256:...`，不是 `latest`、版本号或可变 SHA Tag。
|
||||
回滚也需要用户明确指令。服务器只接受 `.releases/<SHA>.json` 中已有版本，拒绝远端任意 Tag、`latest` 或临时镜像。
|
||||
|
||||
## 发布后验证
|
||||
## 故障排查
|
||||
|
||||
宿主 Nginx 的 `ai.51easyai.com` TLS server 必须包含仓库中的 `deploy/nginx/ai.51easyai.com-api-v1.inc` 等价规则，保留完整 URI 转发到 `127.0.0.1:8088`。修改前先备份现有配置，执行 `nginx -t` 成功后才能 reload。
|
||||
|
||||
```bash
|
||||
curl -fsS https://ai.51easyai.com/api/v1/healthz
|
||||
curl -fsS https://ai.51easyai.com/api/v1/readyz
|
||||
ssh root@110.42.51.33 'cd /root/easyai-ai-gateway-deploy && ./gateway-ops.sh ps'
|
||||
```
|
||||
|
||||
同时检查服务器磁盘、API 日志和数据库容器健康状态。发布后的首小时如出现错误率翻倍、P95 延迟增加 50%、数据完整性或安全问题，应立即回滚。
|
||||
|
||||
## 应用与数据库回滚
|
||||
|
||||
发布失败时固定部署脚本会恢复上一组 API/Web digest 并重复健康检查。人工回滚只选择服务器已有 root-only 发布清单的历史 Git SHA：
|
||||
|
||||
```bash
|
||||
sudo /usr/local/sbin/easyai-ai-gateway-release <40 位历史 Git SHA>
|
||||
```
|
||||
|
||||
没有 `/root/easyai-ai-gateway-deploy/.releases/<SHA>.env` 的远端 SHA Tag 不应被盲目信任。数据库备份位于 `/var/backups/easyai-ai-gateway`，只保留最近 5 份。应用回滚不会自动恢复数据库；只有停止写入并再次保留当前副本后，操作员才能选择备份执行 `pg_restore`。
|
||||
|
||||
## Runner 故障恢复
|
||||
|
||||
先在 Gitea 确认没有运行中的 Job，再检查服务与两个 daemon：
|
||||
|
||||
```bash
|
||||
systemctl status easyai-gateway-ci-v2-runner.service --no-pager
|
||||
docker logs --tail 100 easyai-gateway-ci-v2-runner
|
||||
docker exec easyai-gateway-ci-v2-runner docker info
|
||||
df -h /var/lib/docker
|
||||
```
|
||||
|
||||
确认空闲后才执行：
|
||||
|
||||
```bash
|
||||
systemctl restart easyai-gateway-ci-v2-runner.service
|
||||
journalctl -u easyai-gateway-ci-v2-runner.service -n 100 --no-pager
|
||||
```
|
||||
|
||||
不要删除 `easyai-gateway-ci-v2-data` volume；它包含 Runner 注册状态和内层 daemon 缓存。不得用全局 prune 处理磁盘压力。
|
||||
- publish 前失败：没有推送镜像，也没有可部署 manifest；修复本地环境后重试。
|
||||
- 镜像已经部分推送但 manifest 未生成：SHA Tag 被视为占用，先审查 Registry 中的 digest，不得覆盖重推。
|
||||
- deploy 报基线过期：重新读取线上状态并重新 publish。
|
||||
- 迁移备份失败：生产不切换。
|
||||
- 生产探活和自动应用回滚都失败：停止发布，保留日志、当前数据库和 release manifest，人工恢复服务；不得自动执行 `pg_restore`。
|
||||
|
||||
+3
-2
@@ -12,8 +12,9 @@
|
||||
"migrate": "nx run api:migrate",
|
||||
"openapi": "nx run api:openapi",
|
||||
"docker:build": "docker compose -f docker-compose.yml build --pull api web",
|
||||
"docker:push": "scripts/deploy-compose.sh push-only",
|
||||
"docker:deploy": "pnpm docker:build && pnpm docker:push"
|
||||
"release:publish": "scripts/publish-release-images.sh --components auto",
|
||||
"release:deploy": "scripts/deploy-production-release.sh",
|
||||
"test:release": "tests/release/manual-release-test.sh && tests/ci/migrations-test.sh"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@nx/vite": "^21.0.0",
|
||||
|
||||
Executable
+145
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync } from 'node:child_process'
|
||||
|
||||
function fail(message) {
|
||||
console.error(`api_release_smoke=FAIL ${message}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
function assert(condition, message) {
|
||||
if (!condition) fail(message)
|
||||
}
|
||||
|
||||
const baseURL = (process.env.RELEASE_SMOKE_BASE_URL || '').replace(/\/+$/, '')
|
||||
const webURL = (process.env.RELEASE_SMOKE_WEB_URL || '').replace(/\/+$/, '')
|
||||
const composeProject = process.env.RELEASE_SMOKE_COMPOSE_PROJECT || ''
|
||||
const composeFile = process.env.RELEASE_SMOKE_COMPOSE_FILE || 'docker-compose.yml'
|
||||
const nonce = process.env.RELEASE_SMOKE_NONCE || Date.now().toString(36)
|
||||
const databaseUser = process.env.RELEASE_SMOKE_DATABASE_USER || 'easyai'
|
||||
const databaseName = process.env.RELEASE_SMOKE_DATABASE_NAME || 'easyai_ai_gateway'
|
||||
|
||||
if (!/^http:\/\/127\.0\.0\.1:\d+$/.test(baseURL)) fail('invalid RELEASE_SMOKE_BASE_URL')
|
||||
if (!/^http:\/\/127\.0\.0\.1:\d+$/.test(webURL)) fail('invalid RELEASE_SMOKE_WEB_URL')
|
||||
if (!/^[a-z0-9][a-z0-9_-]{2,62}$/.test(composeProject)) fail('invalid RELEASE_SMOKE_COMPOSE_PROJECT')
|
||||
if (!/^[a-z_][a-z0-9_]{0,62}$/.test(databaseUser)) fail('invalid RELEASE_SMOKE_DATABASE_USER')
|
||||
if (!/^[a-z_][a-z0-9_]{0,62}$/.test(databaseName)) fail('invalid RELEASE_SMOKE_DATABASE_NAME')
|
||||
|
||||
async function request(path, { method = 'GET', token = '', body } = {}) {
|
||||
const response = await fetch(`${baseURL}${path}`, {
|
||||
method,
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
headers: {
|
||||
...(token ? { Authorization: `Bearer ${token}` } : {}),
|
||||
...(body === undefined ? {} : { 'Content-Type': 'application/json' }),
|
||||
},
|
||||
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
||||
})
|
||||
const text = await response.text()
|
||||
let payload
|
||||
try {
|
||||
payload = text ? JSON.parse(text) : null
|
||||
} catch {
|
||||
fail(`${method} ${path} returned non-JSON HTTP ${response.status}`)
|
||||
}
|
||||
if (!response.ok) fail(`${method} ${path} returned HTTP ${response.status}`)
|
||||
return { response, payload }
|
||||
}
|
||||
|
||||
async function waitForAPI() {
|
||||
let lastError = 'API did not respond'
|
||||
for (let attempt = 0; attempt < 60; attempt += 1) {
|
||||
try {
|
||||
const response = await fetch(`${baseURL}/api/v1/readyz`, {
|
||||
signal: AbortSignal.timeout(2_000),
|
||||
})
|
||||
if (response.ok) return
|
||||
lastError = `readiness returned HTTP ${response.status}`
|
||||
} catch (error) {
|
||||
lastError = error.message
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_000))
|
||||
}
|
||||
fail(`API readiness timed out: ${lastError}`)
|
||||
}
|
||||
|
||||
await waitForAPI()
|
||||
|
||||
const health = await request('/api/v1/healthz')
|
||||
assert(health.payload?.ok === true && health.payload?.service === 'easyai-ai-gateway', 'health payload mismatch')
|
||||
|
||||
const ready = await request('/api/v1/readyz')
|
||||
assert(ready.payload?.ok === true, 'readiness payload mismatch')
|
||||
|
||||
const openapi = await request('/api/v1/openapi.json')
|
||||
for (const path of [
|
||||
'/api/v1/healthz',
|
||||
'/api/v1/readyz',
|
||||
'/api/v1/models',
|
||||
'/api/v1/chat/completions',
|
||||
'/api/v1/responses',
|
||||
'/api/v1/images/generations',
|
||||
]) {
|
||||
assert(openapi.payload?.paths?.[path], `OpenAPI is missing ${path}`)
|
||||
}
|
||||
|
||||
const username = `release_smoke_${nonce}`.slice(0, 60)
|
||||
const password = `Release-Smoke-${nonce}-9x!`
|
||||
const registration = await request('/api/v1/auth/register', {
|
||||
method: 'POST',
|
||||
body: { username, email: `${username}@example.invalid`, password },
|
||||
})
|
||||
assert(typeof registration.payload?.accessToken === 'string', 'registration did not return an access token')
|
||||
|
||||
const login = await request('/api/v1/auth/login', {
|
||||
method: 'POST',
|
||||
body: { account: username, password },
|
||||
})
|
||||
const jwt = login.payload?.accessToken
|
||||
assert(typeof jwt === 'string' && jwt.length > 20, 'login did not return an access token')
|
||||
|
||||
try {
|
||||
execFileSync('docker', [
|
||||
'compose', '-p', composeProject, '-f', composeFile,
|
||||
'exec', '-T', 'postgres', 'psql', '-v', 'ON_ERROR_STOP=1',
|
||||
'-U', databaseUser, '-d', databaseName, '-c',
|
||||
`UPDATE gateway_wallet_accounts SET balance = 100000 WHERE gateway_user_id = (SELECT id FROM gateway_users WHERE username = '${username.replaceAll("'", "''")}')`,
|
||||
], { stdio: ['ignore', 'ignore', 'pipe'] })
|
||||
} catch {
|
||||
fail('could not fund the ephemeral smoke wallet')
|
||||
}
|
||||
|
||||
const apiKeyResult = await request('/api/v1/api-keys', {
|
||||
method: 'POST', token: jwt, body: { name: 'release smoke key' },
|
||||
})
|
||||
const apiKey = apiKeyResult.payload?.secret
|
||||
assert(typeof apiKey === 'string' && apiKey.startsWith('sk-gw-'), 'API key creation failed')
|
||||
|
||||
const models = await request('/api/v1/models', { token: apiKey })
|
||||
assert(Array.isArray(models.payload?.items) || Array.isArray(models.payload?.data), 'models response has no items')
|
||||
|
||||
const simulation = { runMode: 'simulation', simulation: true, simulationDurationMs: 5 }
|
||||
const chat = await request('/api/v1/chat/completions', {
|
||||
method: 'POST', token: apiKey,
|
||||
body: { model: 'openai:gpt-4o-mini', messages: [{ role: 'user', content: 'release smoke' }], ...simulation },
|
||||
})
|
||||
assert(/^chatcmpl-/.test(chat.payload?.id || ''), 'chat simulation response mismatch')
|
||||
|
||||
const responses = await request('/api/v1/responses', {
|
||||
method: 'POST', token: apiKey,
|
||||
body: { model: 'openai:gpt-4o-mini', input: 'release smoke', ...simulation },
|
||||
})
|
||||
assert(/^resp/.test(responses.payload?.id || ''), 'responses simulation response mismatch')
|
||||
|
||||
const images = await request('/api/v1/images/generations', {
|
||||
method: 'POST', token: apiKey,
|
||||
body: { model: 'openai:gpt-image-1', prompt: 'release smoke', ...simulation },
|
||||
})
|
||||
assert(Array.isArray(images.payload?.data) && images.payload.data.length > 0, 'image simulation response mismatch')
|
||||
|
||||
const webHealth = await fetch(`${webURL}/api/v1/healthz`, { signal: AbortSignal.timeout(15_000) })
|
||||
assert(webHealth.ok, `web API reverse proxy returned HTTP ${webHealth.status}`)
|
||||
const webApp = await fetch(`${webURL}/`, { signal: AbortSignal.timeout(15_000) })
|
||||
assert(webApp.ok && (await webApp.text()).includes('EasyAI AI Gateway'), 'web application smoke failed')
|
||||
|
||||
console.log('api_release_smoke=PASS health=ok readiness=ok openapi=ok auth=ok chat=ok responses=ok images=ok web=ok')
|
||||
@@ -1,101 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
cd "$root"
|
||||
|
||||
tag=${IMAGE_TAG:-}
|
||||
registry=${AI_GATEWAY_IMAGE_REGISTRY:-registry.cn-shanghai.aliyuncs.com/easyaigc}
|
||||
builder=${AI_GATEWAY_BUILDX_BUILDER:-easyai-gateway-ci}
|
||||
cache_limit=${AI_GATEWAY_BUILDX_CACHE_LIMIT:-2gb}
|
||||
platform=${AI_GATEWAY_PLATFORM:-linux/amd64}
|
||||
keep_images=0
|
||||
build_complete=0
|
||||
readonly trivy_db_repository=ghcr.m.daocloud.io/aquasecurity/trivy-db:2
|
||||
|
||||
case ${1:-} in
|
||||
'') ;;
|
||||
--keep-images) keep_images=1 ;;
|
||||
*)
|
||||
echo 'usage: ci-build-images.sh [--keep-images]' >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ ! $tag =~ ^[0-9a-f]{40}$ ]]; then
|
||||
echo 'IMAGE_TAG must be a full lowercase Git SHA' >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! $registry =~ ^[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._/-]+)+$ ]]; then
|
||||
echo 'AI_GATEWAY_IMAGE_REGISTRY has an invalid format' >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! $cache_limit =~ ^[1-9][0-9]*(kb|mb|gb)$ ]]; then
|
||||
echo 'AI_GATEWAY_BUILDX_CACHE_LIMIT has an invalid format' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
api_image="$registry/ai-gateway:$tag"
|
||||
web_image="$registry/ai-gateway-web:$tag"
|
||||
|
||||
cleanup() {
|
||||
original_status=$?
|
||||
trap - EXIT HUP INT TERM
|
||||
cleanup_status=0
|
||||
|
||||
docker buildx prune --builder "$builder" --force --max-used-space "$cache_limit" >/dev/null || {
|
||||
cleanup_status=$?
|
||||
echo 'failed to prune the CI BuildKit cache' >&2
|
||||
}
|
||||
if [[ $keep_images -ne 1 || $build_complete -ne 1 ]]; then
|
||||
docker image rm "$api_image" "$web_image" >/dev/null 2>&1 || {
|
||||
image_rm_status=$?
|
||||
[[ $cleanup_status -ne 0 ]] || cleanup_status=$image_rm_status
|
||||
echo 'failed to remove CI image tags' >&2
|
||||
}
|
||||
fi
|
||||
|
||||
if [[ $original_status -ne 0 ]]; then
|
||||
exit "$original_status"
|
||||
fi
|
||||
exit "$cleanup_status"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
if ! docker buildx inspect "$builder" >/dev/null 2>&1; then
|
||||
docker buildx create --name "$builder" --driver docker-container --use >/dev/null
|
||||
fi
|
||||
docker buildx inspect "$builder" --bootstrap >/dev/null
|
||||
common_args=(
|
||||
--builder "$builder"
|
||||
--platform "$platform"
|
||||
--file Dockerfile
|
||||
--pull
|
||||
--load
|
||||
)
|
||||
|
||||
docker buildx build --builder "$builder" \
|
||||
"${common_args[@]:2}" \
|
||||
--target api \
|
||||
--build-arg "GO_BUILD_IMAGE=${AI_GATEWAY_GO_BUILD_IMAGE:-docker.m.daocloud.io/library/golang:1.26.5-alpine}" \
|
||||
--build-arg "API_RUNTIME_IMAGE=${AI_GATEWAY_API_RUNTIME_IMAGE:-docker.m.daocloud.io/library/alpine:3.22}" \
|
||||
--tag "$api_image" .
|
||||
|
||||
docker buildx build --builder "$builder" \
|
||||
"${common_args[@]:2}" \
|
||||
--target web \
|
||||
--build-arg "NODE_BUILD_IMAGE=${AI_GATEWAY_NODE_BUILD_IMAGE:-docker.m.daocloud.io/library/node:24-alpine}" \
|
||||
--build-arg "WEB_RUNTIME_IMAGE=${AI_GATEWAY_WEB_RUNTIME_IMAGE:-docker.m.daocloud.io/library/nginx:1.27-alpine}" \
|
||||
--tag "$web_image" .
|
||||
|
||||
for image in "$api_image" "$web_image"; do
|
||||
trivy image --db-repository "$trivy_db_repository" \
|
||||
--scanners vuln,secret --severity HIGH,CRITICAL --ignore-unfixed \
|
||||
--exit-code 1 --timeout 15m "$image"
|
||||
done
|
||||
|
||||
build_complete=1
|
||||
printf 'gateway_images=PASS git_sha=%s api=%s web=%s\n' "$tag" "$api_image" "$web_image"
|
||||
@@ -5,10 +5,7 @@ import { lstatSync, readFileSync } from 'node:fs'
|
||||
import { basename } from 'node:path'
|
||||
|
||||
const migrationDirectory = 'apps/api/migrations/'
|
||||
const baselineFile = 'deploy/ci/production-migration-base'
|
||||
const base = process.argv[2] || process.env.PRODUCTION_MIGRATION_BASE_SHA || ''
|
||||
const immutableRef =
|
||||
process.argv[3] || process.env.MIGRATION_IMMUTABILITY_REF || ''
|
||||
const base = process.argv[2] || process.env.RELEASE_BASE_SHA || ''
|
||||
|
||||
function fail(message) {
|
||||
console.error(`migration_safety=FAIL ${message}`)
|
||||
@@ -28,7 +25,7 @@ function git(args, options = {}) {
|
||||
}
|
||||
|
||||
if (!/^[0-9a-f]{40}$/.test(base)) {
|
||||
fail('PRODUCTION_MIGRATION_BASE_SHA must be a full lowercase commit SHA')
|
||||
fail('RELEASE_BASE_SHA must be a full lowercase commit SHA')
|
||||
}
|
||||
|
||||
const root = git(['rev-parse', '--show-toplevel']).trim()
|
||||
@@ -41,34 +38,14 @@ const resolvedBase = git([
|
||||
`${base}^{commit}`,
|
||||
]).trim()
|
||||
if (resolvedBase !== base) {
|
||||
fail('production migration baseline did not resolve exactly')
|
||||
fail('release base did not resolve exactly')
|
||||
}
|
||||
|
||||
const ancestry = spawnSync('git', ['merge-base', '--is-ancestor', base, 'HEAD'], {
|
||||
stdio: 'ignore',
|
||||
})
|
||||
if (ancestry.status !== 0) {
|
||||
fail('production migration baseline is not an ancestor of HEAD')
|
||||
}
|
||||
|
||||
const head = git(['rev-parse', '--verify', 'HEAD^{commit}']).trim()
|
||||
|
||||
const baselineTreeEntry = git(['ls-tree', 'HEAD', '--', baselineFile]).trim()
|
||||
let baselineState
|
||||
try {
|
||||
baselineState = lstatSync(baselineFile)
|
||||
} catch {
|
||||
fail('production migration baseline file is unavailable in the worktree')
|
||||
}
|
||||
if (
|
||||
!/^100644 blob [0-9a-f]+\t/.test(baselineTreeEntry) ||
|
||||
!baselineState.isFile() ||
|
||||
baselineState.isSymbolicLink()
|
||||
) {
|
||||
fail('production migration baseline must be a mode 100644 regular file')
|
||||
}
|
||||
if (readFileSync(baselineFile, 'utf8').trim() !== base) {
|
||||
fail('production migration baseline argument does not match the reviewed file')
|
||||
fail('release base is not an ancestor of HEAD')
|
||||
}
|
||||
|
||||
function changedMigrations(from) {
|
||||
@@ -252,94 +229,8 @@ const destructiveRules = [
|
||||
|
||||
const violations = []
|
||||
let addedFiles = 0
|
||||
let immutableBase = ''
|
||||
const orderingCandidates = new Set()
|
||||
|
||||
if (immutableRef) {
|
||||
if (
|
||||
!/^[0-9a-f]{40}$/.test(immutableRef) &&
|
||||
immutableRef !== 'refs/remotes/origin/main'
|
||||
) {
|
||||
fail('MIGRATION_IMMUTABILITY_REF must be a full commit SHA or refs/remotes/origin/main')
|
||||
}
|
||||
|
||||
const immutableTip = git([
|
||||
'rev-parse',
|
||||
'--verify',
|
||||
'--end-of-options',
|
||||
`${immutableRef}^{commit}`,
|
||||
]).trim()
|
||||
const baseAtEvent = spawnSync(
|
||||
'git',
|
||||
['show', `${immutableTip}:${baselineFile}`],
|
||||
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] },
|
||||
)
|
||||
if (baseAtEvent.status === 0) {
|
||||
const previousBaselineTreeEntry = git([
|
||||
'ls-tree',
|
||||
immutableTip,
|
||||
'--',
|
||||
baselineFile,
|
||||
]).trim()
|
||||
if (!/^100644 blob [0-9a-f]+\t/.test(previousBaselineTreeEntry)) {
|
||||
fail('previous production migration baseline must be a mode 100644 regular file')
|
||||
}
|
||||
const previousProductionBase = baseAtEvent.stdout.trim()
|
||||
if (!/^[0-9a-f]{40}$/.test(previousProductionBase)) {
|
||||
fail('previous production migration baseline is invalid')
|
||||
}
|
||||
const baselineAdvance = spawnSync(
|
||||
'git',
|
||||
['merge-base', '--is-ancestor', previousProductionBase, base],
|
||||
{ stdio: 'ignore' },
|
||||
)
|
||||
if (baselineAdvance.status !== 0) {
|
||||
fail('production migration baseline must only move forward')
|
||||
}
|
||||
}
|
||||
const baselineWasAlreadyReviewed = spawnSync(
|
||||
'git',
|
||||
['merge-base', '--is-ancestor', base, immutableTip],
|
||||
{ stdio: 'ignore' },
|
||||
)
|
||||
if (baselineWasAlreadyReviewed.status !== 0) {
|
||||
fail('production migration baseline must belong to the immutable event base history')
|
||||
}
|
||||
if (immutableTip === head) {
|
||||
const parent = spawnSync(
|
||||
'git',
|
||||
['rev-parse', '--verify', `${immutableTip}^1^{commit}`],
|
||||
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] },
|
||||
)
|
||||
immutableBase = parent.status === 0 ? parent.stdout.trim() : ''
|
||||
} else {
|
||||
const immutableAncestry = spawnSync(
|
||||
'git',
|
||||
['merge-base', '--is-ancestor', immutableTip, head],
|
||||
{ stdio: 'ignore' },
|
||||
)
|
||||
if (immutableAncestry.status !== 0) {
|
||||
fail('migration immutability base is not an ancestor of HEAD')
|
||||
}
|
||||
immutableBase = immutableTip
|
||||
}
|
||||
|
||||
if (immutableBase) {
|
||||
const immutableChanges = changedMigrations(immutableBase)
|
||||
for (let index = 0; index < immutableChanges.length; index += 2) {
|
||||
const status = immutableChanges[index]
|
||||
const file = immutableChanges[index + 1]
|
||||
if (status === 'A') {
|
||||
orderingCandidates.add(file)
|
||||
} else {
|
||||
violations.push(
|
||||
`${file}: migration present on main is immutable (status ${status})`,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function migrationNamesAt(ref) {
|
||||
const output = git(
|
||||
['ls-tree', '-r', '-z', '--name-only', ref, '--', migrationDirectory],
|
||||
@@ -358,14 +249,11 @@ function migrationNamesAt(ref) {
|
||||
.sort()
|
||||
}
|
||||
|
||||
const orderingBase = immutableBase || base
|
||||
const existingNames = migrationNamesAt(orderingBase)
|
||||
const existingNames = migrationNamesAt(base)
|
||||
const previousMaximumName = existingNames.at(-1) || ''
|
||||
|
||||
if (!immutableBase) {
|
||||
for (let index = 0; index < fields.length; index += 2) {
|
||||
if (fields[index] === 'A') orderingCandidates.add(fields[index + 1])
|
||||
}
|
||||
for (let index = 0; index < fields.length; index += 2) {
|
||||
if (fields[index] === 'A') orderingCandidates.add(fields[index + 1])
|
||||
}
|
||||
|
||||
for (let index = 0; index < fields.length; index += 2) {
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
tag=${1:-}
|
||||
|
||||
die() {
|
||||
printf 'invalid release tag: %s\n' "$tag" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[[ $tag == v* ]] || die
|
||||
version=${tag#v}
|
||||
|
||||
# Bash arrays discard trailing empty fields, so parsing first would accept
|
||||
# values such as v1.2.3. and v1.2.3-alpha. Validate the complete string with
|
||||
# the SemVer 2.0.0 grammar before doing anything else with the tag.
|
||||
semver_pattern='^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(\.(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*))?(\+([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
|
||||
LC_ALL=C
|
||||
export LC_ALL
|
||||
[[ $version =~ $semver_pattern ]] || die
|
||||
|
||||
printf 'release_tag=PASS tag=%s\n' "$tag"
|
||||
@@ -6,7 +6,7 @@ COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.yml}"
|
||||
COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-easyai-ai-gateway}"
|
||||
AI_GATEWAY_PLATFORM="${AI_GATEWAY_PLATFORM:-linux/amd64}"
|
||||
AI_GATEWAY_IMAGE_REGISTRY="${AI_GATEWAY_IMAGE_REGISTRY:-registry.cn-shanghai.aliyuncs.com/easyaigc}"
|
||||
AI_GATEWAY_IMAGE_TAG="${AI_GATEWAY_IMAGE_TAG:-latest}"
|
||||
AI_GATEWAY_IMAGE_TAG="${AI_GATEWAY_IMAGE_TAG:-local}"
|
||||
AI_GATEWAY_API_IMAGE="${AI_GATEWAY_API_IMAGE:-${AI_GATEWAY_IMAGE_REGISTRY}/ai-gateway:${AI_GATEWAY_IMAGE_TAG}}"
|
||||
AI_GATEWAY_WEB_IMAGE="${AI_GATEWAY_WEB_IMAGE:-${AI_GATEWAY_IMAGE_REGISTRY}/ai-gateway-web:${AI_GATEWAY_IMAGE_TAG}}"
|
||||
ACTION="${1:-deploy}"
|
||||
@@ -22,11 +22,8 @@ compose=(docker compose -f "$COMPOSE_FILE")
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
scripts/deploy-compose.sh Build, migrate, start, and verify
|
||||
scripts/deploy-compose.sh Build, migrate, start, and verify locally
|
||||
scripts/deploy-compose.sh deploy Same as default
|
||||
scripts/deploy-compose.sh push Build and push API/Web images
|
||||
scripts/deploy-compose.sh push-only
|
||||
Push already-built API/Web images
|
||||
scripts/deploy-compose.sh down Stop containers, keep volumes
|
||||
scripts/deploy-compose.sh clean Stop containers and remove volumes
|
||||
|
||||
@@ -38,8 +35,10 @@ Useful environment overrides:
|
||||
AI_GATEWAY_WEB_PORT=5178
|
||||
AI_GATEWAY_API_PORT=8088
|
||||
AI_GATEWAY_DB_PORT=54329
|
||||
AI_GATEWAY_PUSH=1
|
||||
AI_GATEWAY_SKIP_BUILD=1
|
||||
|
||||
Production publishing is intentionally unavailable here. Use:
|
||||
scripts/publish-release-images.sh --components auto
|
||||
EOF
|
||||
}
|
||||
|
||||
@@ -125,86 +124,6 @@ build_images() {
|
||||
fi
|
||||
}
|
||||
|
||||
package_version() {
|
||||
local version=""
|
||||
|
||||
version="$(
|
||||
sed -nE 's/^[[:space:]]*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/p' "$PROJECT_ROOT/package.json" \
|
||||
| head -n 1
|
||||
)"
|
||||
if [[ -z "$version" ]]; then
|
||||
echo "[ai-gateway] cannot infer package version from package.json; set AI_GATEWAY_IMAGE_TAG" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$version"
|
||||
}
|
||||
|
||||
push_version_tag() {
|
||||
if [[ "$AI_GATEWAY_IMAGE_TAG" == "latest" ]]; then
|
||||
package_version
|
||||
else
|
||||
echo "$AI_GATEWAY_IMAGE_TAG"
|
||||
fi
|
||||
}
|
||||
|
||||
image_repository() {
|
||||
local image="$1"
|
||||
echo "${image%:*}"
|
||||
}
|
||||
|
||||
tag_release_images() {
|
||||
local version_tag="$1"
|
||||
local api_repo web_repo
|
||||
local api_version_image api_latest_image web_version_image web_latest_image
|
||||
|
||||
api_repo="$(image_repository "$AI_GATEWAY_API_IMAGE")"
|
||||
web_repo="$(image_repository "$AI_GATEWAY_WEB_IMAGE")"
|
||||
api_version_image="${api_repo}:${version_tag}"
|
||||
api_latest_image="${api_repo}:latest"
|
||||
web_version_image="${web_repo}:${version_tag}"
|
||||
web_latest_image="${web_repo}:latest"
|
||||
|
||||
docker image inspect "$AI_GATEWAY_API_IMAGE" >/dev/null 2>&1 || {
|
||||
echo "[ai-gateway] missing local API image: ${AI_GATEWAY_API_IMAGE}; build it first" >&2
|
||||
exit 1
|
||||
}
|
||||
docker image inspect "$AI_GATEWAY_WEB_IMAGE" >/dev/null 2>&1 || {
|
||||
echo "[ai-gateway] missing local Web image: ${AI_GATEWAY_WEB_IMAGE}; build it first" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
docker tag "$AI_GATEWAY_API_IMAGE" "$api_version_image"
|
||||
docker tag "$AI_GATEWAY_API_IMAGE" "$api_latest_image"
|
||||
docker tag "$AI_GATEWAY_WEB_IMAGE" "$web_version_image"
|
||||
docker tag "$AI_GATEWAY_WEB_IMAGE" "$web_latest_image"
|
||||
|
||||
RELEASE_IMAGES=(
|
||||
"$api_version_image"
|
||||
"$api_latest_image"
|
||||
"$web_version_image"
|
||||
"$web_latest_image"
|
||||
)
|
||||
}
|
||||
|
||||
push_images() {
|
||||
local version_tag
|
||||
local image
|
||||
|
||||
version_tag="$(push_version_tag)"
|
||||
echo "[ai-gateway] release image tag: ${version_tag}"
|
||||
echo "[ai-gateway] latest tag will also be pushed"
|
||||
tag_release_images "$version_tag"
|
||||
|
||||
for image in "${RELEASE_IMAGES[@]}"; do
|
||||
echo "[ai-gateway] pushing image: ${image}"
|
||||
if ! docker push "$image"; then
|
||||
echo "[ai-gateway] failed to push image; login may be required:" >&2
|
||||
echo "[ai-gateway] docker login --username=<your-aliyun-account> registry.cn-shanghai.aliyuncs.com" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
deploy() {
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
@@ -227,10 +146,6 @@ deploy() {
|
||||
|
||||
build_images
|
||||
|
||||
if [[ "${AI_GATEWAY_PUSH:-0}" == "1" ]]; then
|
||||
push_images
|
||||
fi
|
||||
|
||||
echo "[ai-gateway] starting postgres"
|
||||
"${compose[@]}" up -d postgres
|
||||
wait_for_service_healthy postgres
|
||||
@@ -262,15 +177,6 @@ case "$ACTION" in
|
||||
deploy|up)
|
||||
deploy
|
||||
;;
|
||||
push)
|
||||
cd "$PROJECT_ROOT"
|
||||
build_images
|
||||
push_images
|
||||
;;
|
||||
push-only)
|
||||
cd "$PROJECT_ROOT"
|
||||
push_images
|
||||
;;
|
||||
down)
|
||||
cd "$PROJECT_ROOT"
|
||||
"${compose[@]}" down --remove-orphans
|
||||
|
||||
Executable
+100
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
production_host=${AI_GATEWAY_PRODUCTION_HOST:-root@110.42.51.33}
|
||||
remote_helper=${AI_GATEWAY_REMOTE_RELEASE_HELPER:-/usr/local/sbin/easyai-ai-gateway-release}
|
||||
action=deploy
|
||||
value=
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
scripts/deploy-production-release.sh dist/releases/<git-sha>.json
|
||||
scripts/deploy-production-release.sh --rollback <historical-git-sha>
|
||||
scripts/deploy-production-release.sh --status
|
||||
|
||||
This command changes production. Run it only after a separate successful
|
||||
publish command and explicit user confirmation.
|
||||
EOF
|
||||
}
|
||||
|
||||
case ${1:-} in
|
||||
--status)
|
||||
action=status
|
||||
[[ $# -eq 1 ]] || { usage >&2; exit 64; }
|
||||
;;
|
||||
--rollback)
|
||||
action=rollback
|
||||
[[ $# -eq 2 ]] || { usage >&2; exit 64; }
|
||||
value=$2
|
||||
;;
|
||||
-h|--help|'')
|
||||
usage
|
||||
[[ ${1:-} == -h || ${1:-} == --help ]] && exit 0
|
||||
exit 64
|
||||
;;
|
||||
*)
|
||||
[[ $# -eq 1 ]] || { usage >&2; exit 64; }
|
||||
value=$1
|
||||
;;
|
||||
esac
|
||||
|
||||
[[ $production_host =~ ^[A-Za-z0-9._-]+@[A-Za-z0-9.-]+$ ]] || {
|
||||
echo 'AI_GATEWAY_PRODUCTION_HOST must use user@host syntax' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ $remote_helper =~ ^/[A-Za-z0-9._/-]+$ ]] || {
|
||||
echo 'AI_GATEWAY_REMOTE_RELEASE_HELPER must be an absolute safe path' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [[ $action == status ]]; then
|
||||
exec ssh -o BatchMode=yes "$production_host" "$remote_helper status"
|
||||
fi
|
||||
|
||||
if [[ $action == rollback ]]; then
|
||||
[[ $value =~ ^[0-9a-f]{40}$ ]] || {
|
||||
echo 'rollback release must be a full lowercase Git SHA' >&2
|
||||
exit 1
|
||||
}
|
||||
ssh -o BatchMode=yes "$production_host" "$remote_helper rollback $value"
|
||||
echo "production_rollback=PASS release=$value"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
manifest=$(cd "$(dirname "$value")" 2>/dev/null && pwd)/$(basename "$value")
|
||||
[[ -f $manifest && ! -L $manifest ]] || {
|
||||
echo 'release manifest must be a regular file' >&2
|
||||
exit 1
|
||||
}
|
||||
node "$root/scripts/release-manifest.mjs" validate "$manifest"
|
||||
source_sha=$(node "$root/scripts/release-manifest.mjs" get "$manifest" sourceSha)
|
||||
base_sha=$(node "$root/scripts/release-manifest.mjs" get "$manifest" baseReleaseSha)
|
||||
remote_manifest=/tmp/easyai-ai-gateway-release-$source_sha.json
|
||||
status_file=$(mktemp)
|
||||
|
||||
cleanup() {
|
||||
rm -f "$status_file"
|
||||
ssh -o BatchMode=yes "$production_host" "rm -f $remote_manifest" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
if ssh -o BatchMode=yes "$production_host" "$remote_helper status" >"$status_file" 2>/dev/null; then
|
||||
node "$root/scripts/release-manifest.mjs" validate "$status_file" >/dev/null
|
||||
current_sha=$(node "$root/scripts/release-manifest.mjs" get "$status_file" sourceSha)
|
||||
[[ $current_sha == "$base_sha" ]] || {
|
||||
printf 'production base changed after publish: current=%s manifest_base=%s\n' \
|
||||
"$current_sha" "$base_sha" >&2
|
||||
exit 1
|
||||
}
|
||||
else
|
||||
[[ -z $base_sha ]] || {
|
||||
echo 'production release status is unavailable for a non-bootstrap manifest' >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
|
||||
scp -q "$manifest" "$production_host:$remote_manifest"
|
||||
ssh -o BatchMode=yes "$production_host" "$remote_helper deploy $remote_manifest"
|
||||
echo "production_deploy=PASS release=$source_sha"
|
||||
@@ -1,402 +0,0 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
PREFIX=/opt/easyai-gateway-ci
|
||||
GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL:-https://git.51easyai.com}
|
||||
RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-ci-unprivileged-v2}
|
||||
RUNNER_VOLUME=easyai-gateway-ci-v2-data
|
||||
RUNNER_CONTAINER=easyai-gateway-ci-v2-runner
|
||||
LEGACY_BUILDER=${AI_GATEWAY_BUILDX_BUILDER:-easyai-gateway-ci}
|
||||
MIN_FREE_GIB=${CI_RUNNER_MIN_FREE_GIB:-8}
|
||||
MIN_POST_INSTALL_FREE_GIB=${CI_RUNNER_MIN_POST_INSTALL_FREE_GIB:-4}
|
||||
RUNNER_CPUS=2
|
||||
RUNNER_MEMORY=2g
|
||||
RUNNER_MEMORY_SWAP=3g
|
||||
RUNNER_PIDS_LIMIT=1024
|
||||
|
||||
GITEA_RUNNER_VERSION=2.0.0
|
||||
GO_VERSION=1.26.5
|
||||
GO_SHA256=5c2c3b16caefa1d968a94c1daca04a7ca301a496d9b086e17ad77bb81393f053
|
||||
NODE_VERSION=24.16.0
|
||||
NODE_SHA256=d804845d34eddc21dc1092b519d643ef40b1f58ec5dec5c22b1f4bd8fabde6c9
|
||||
TRIVY_VERSION=0.70.0
|
||||
TRIVY_SHA256=8b4376d5d6befe5c24d503f10ff136d9e0c49f9127a4279fd110b727929a5aa9
|
||||
SHELLCHECK_VERSION=0.11.0
|
||||
SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198
|
||||
COMPOSE_VERSION=5.3.1
|
||||
COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959
|
||||
PNPM_VERSION=10.18.1
|
||||
RUNNER_IMAGE='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a'
|
||||
JOB_IMAGE='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7'
|
||||
RUNNER_IMAGE_SOURCE=${CI_RUNNER_IMAGE_SOURCE:-$RUNNER_IMAGE}
|
||||
NESTED_DOCKER_REGISTRY_MIRROR=${CI_NESTED_DOCKER_REGISTRY_MIRROR:-}
|
||||
RUNNER_LABEL="easyai-gateway-ci-unprivileged-v2:docker://$JOB_IMAGE"
|
||||
|
||||
fail() {
|
||||
echo "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
case $MIN_FREE_GIB:$MIN_POST_INSTALL_FREE_GIB in
|
||||
*[!0-9:]* | :* | *:)
|
||||
fail "CI runner free-space limits must be whole GiB values"
|
||||
;;
|
||||
esac
|
||||
[ "$MIN_FREE_GIB" -ge 8 ] || fail "CI_RUNNER_MIN_FREE_GIB cannot be lower than 8"
|
||||
[ "$MIN_POST_INSTALL_FREE_GIB" -ge 4 ] || \
|
||||
fail "CI_RUNNER_MIN_POST_INSTALL_FREE_GIB cannot be lower than 4"
|
||||
|
||||
[ "$(id -u)" -eq 0 ] || fail "run as root"
|
||||
[ "$(uname -s)-$(uname -m)" = "Linux-x86_64" ] || fail "only Linux x86_64 is supported"
|
||||
case $RUNNER_IMAGE in
|
||||
*"/runner:${GITEA_RUNNER_VERSION}-dind-rootless@sha256:"*) ;;
|
||||
*) fail "RUNNER_IMAGE does not match GITEA_RUNNER_VERSION" ;;
|
||||
esac
|
||||
runner_manifest_digest=${RUNNER_IMAGE##*@}
|
||||
case $RUNNER_IMAGE_SOURCE in
|
||||
*@"$runner_manifest_digest") ;;
|
||||
*) fail "CI_RUNNER_IMAGE_SOURCE must use the pinned runner manifest digest" ;;
|
||||
esac
|
||||
if [ -n "$NESTED_DOCKER_REGISTRY_MIRROR" ] && \
|
||||
! printf '%s\n' "$NESTED_DOCKER_REGISTRY_MIRROR" | \
|
||||
grep -Eq '^https://[A-Za-z0-9.-]+(:[0-9]+)?$'; then
|
||||
fail "CI_NESTED_DOCKER_REGISTRY_MIRROR must be an HTTPS registry origin"
|
||||
fi
|
||||
command -v docker >/dev/null 2>&1 || fail "Docker Engine is required"
|
||||
docker info >/dev/null 2>&1 || fail "Docker Engine is not reachable"
|
||||
grep -Fq "\"$RUNNER_LABEL\"" "$ROOT/deploy/ci/act-runner-v2-config.yaml" || \
|
||||
fail "runner label does not match the pinned job image"
|
||||
|
||||
# Fail closed before installing anything. Neither the legacy host runner nor an
|
||||
# earlier v2 unit may poll for work while its replacement is being prepared.
|
||||
systemctl disable --now easyai-gateway-act-runner.service >/dev/null 2>&1 || true
|
||||
systemctl disable --now easyai-gateway-ci-v2-runner.service >/dev/null 2>&1 || true
|
||||
docker rm --force "$RUNNER_CONTAINER" >/dev/null 2>&1 || true
|
||||
rm -f /etc/systemd/system/easyai-gateway-act-runner.service
|
||||
|
||||
if id easyai-gateway-runner >/dev/null 2>&1; then
|
||||
if getent group docker >/dev/null 2>&1; then
|
||||
gpasswd -d easyai-gateway-runner docker >/dev/null 2>&1 || true
|
||||
fi
|
||||
if getent group sudo >/dev/null 2>&1; then
|
||||
gpasswd -d easyai-gateway-runner sudo >/dev/null 2>&1 || true
|
||||
fi
|
||||
fi
|
||||
if id easyai-gateway-ci-v2 >/dev/null 2>&1; then
|
||||
if getent group docker >/dev/null 2>&1; then
|
||||
gpasswd -d easyai-gateway-ci-v2 docker >/dev/null 2>&1 || true
|
||||
fi
|
||||
if getent group sudo >/dev/null 2>&1; then
|
||||
gpasswd -d easyai-gateway-ci-v2 sudo >/dev/null 2>&1 || true
|
||||
fi
|
||||
fi
|
||||
for account in easyai-gateway-runner easyai-gateway-ci-v2; do
|
||||
if id "$account" >/dev/null 2>&1 && \
|
||||
id -nG "$account" | tr ' ' '\n' | grep -Eq '^(docker|sudo)$'; then
|
||||
fail "$account must not belong to a privileged group"
|
||||
fi
|
||||
done
|
||||
|
||||
if ! command -v make >/dev/null 2>&1 || ! command -v gcc >/dev/null 2>&1 || \
|
||||
! command -v jq >/dev/null 2>&1 || ! command -v curl >/dev/null 2>&1 || \
|
||||
! command -v xz >/dev/null 2>&1; then
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends \
|
||||
build-essential ca-certificates curl git jq perl unzip util-linux xz-utils
|
||||
fi
|
||||
|
||||
docker_root=$(docker info --format '{{.DockerRootDir}}')
|
||||
[ -n "$docker_root" ] || fail "Docker root directory could not be determined"
|
||||
|
||||
require_free_space() {
|
||||
minimum_gib=$1
|
||||
phase=$2
|
||||
available_kib=$(df -Pk "$docker_root" | awk 'NR == 2 { print $4 }')
|
||||
case $available_kib in
|
||||
'' | *[!0-9]*) fail "free space could not be determined for $docker_root" ;;
|
||||
esac
|
||||
minimum_kib=$((minimum_gib * 1024 * 1024))
|
||||
if [ "$available_kib" -lt "$minimum_kib" ]; then
|
||||
echo "$phase requires at least ${minimum_gib} GiB free on the Docker filesystem" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# This builder belonged only to the retired gateway image-building runner. Cap
|
||||
# its unused cache before adding the nested daemon; never prune global images,
|
||||
# volumes, or caches owned by another service on this shared production host.
|
||||
if docker buildx version >/dev/null 2>&1 && \
|
||||
docker buildx inspect "$LEGACY_BUILDER" >/dev/null 2>&1; then
|
||||
docker buildx prune --builder "$LEGACY_BUILDER" --force --max-used-space 512mb
|
||||
fi
|
||||
require_free_space "$MIN_FREE_GIB" "CI runner installation"
|
||||
|
||||
download_verified() {
|
||||
url=$1
|
||||
output=$2
|
||||
checksum=$3
|
||||
if [ ! -f "$output" ] || ! printf '%s %s\n' "$checksum" "$output" | sha256sum -c - >/dev/null 2>&1; then
|
||||
rm -f "$output"
|
||||
curl -fL --retry 5 --retry-delay 2 "$url" -o "$output"
|
||||
fi
|
||||
printf '%s %s\n' "$checksum" "$output" | sha256sum -c -
|
||||
}
|
||||
|
||||
install -d -m 0755 "$PREFIX/bin" "$PREFIX/downloads" "$PREFIX/toolchains"
|
||||
|
||||
download_verified \
|
||||
"https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \
|
||||
"$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \
|
||||
"$SHELLCHECK_SHA256"
|
||||
rm -rf "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}"
|
||||
tar -xJf "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \
|
||||
-C "$PREFIX/downloads"
|
||||
install -m 0755 "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "$PREFIX/bin/shellcheck"
|
||||
|
||||
download_verified \
|
||||
"https://github.com/docker/compose/releases/download/v${COMPOSE_VERSION}/docker-compose-linux-x86_64" \
|
||||
"$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" \
|
||||
"$COMPOSE_SHA256"
|
||||
install -m 0755 "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" "$PREFIX/bin/docker-compose"
|
||||
|
||||
download_verified \
|
||||
"https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" \
|
||||
"$PREFIX/downloads/go${GO_VERSION}.linux-amd64.tar.gz" \
|
||||
"$GO_SHA256"
|
||||
rm -rf "$PREFIX/toolchains/go"
|
||||
tar -xzf "$PREFIX/downloads/go${GO_VERSION}.linux-amd64.tar.gz" -C "$PREFIX/toolchains"
|
||||
|
||||
download_verified \
|
||||
"https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" \
|
||||
"$PREFIX/downloads/node-v${NODE_VERSION}-linux-x64.tar.xz" \
|
||||
"$NODE_SHA256"
|
||||
rm -rf "$PREFIX/toolchains/node"
|
||||
mkdir -p "$PREFIX/toolchains/node"
|
||||
tar -xJf "$PREFIX/downloads/node-v${NODE_VERSION}-linux-x64.tar.xz" \
|
||||
-C "$PREFIX/toolchains/node" --strip-components=1
|
||||
PATH="$PREFIX/toolchains/node/bin:$PATH" \
|
||||
"$PREFIX/toolchains/node/bin/npm" install --global --prefix "$PREFIX/toolchains/node" \
|
||||
--ignore-scripts "pnpm@${PNPM_VERSION}"
|
||||
|
||||
download_verified \
|
||||
"https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" \
|
||||
"$PREFIX/downloads/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" \
|
||||
"$TRIVY_SHA256"
|
||||
tar -xzf "$PREFIX/downloads/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" -C "$PREFIX/bin" trivy
|
||||
chmod 0755 "$PREFIX/bin/trivy"
|
||||
GOBIN="$PREFIX/bin" GOPROXY="${GO_MODULE_PROXY:-https://goproxy.cn,direct}" \
|
||||
"$PREFIX/toolchains/go/bin/go" install golang.org/x/vuln/cmd/govulncheck@v1.6.0
|
||||
|
||||
require_free_space "$MIN_FREE_GIB" "runner image pull"
|
||||
docker pull "$RUNNER_IMAGE_SOURCE"
|
||||
runner_runtime_image=$(docker image inspect --format '{{.Id}}' "$RUNNER_IMAGE_SOURCE")
|
||||
printf '%s\n' "$runner_runtime_image" | grep -Eq '^sha256:[0-9a-f]{64}$' || \
|
||||
fail "pinned runner image did not resolve to an immutable local image ID"
|
||||
chown -R root:root "$PREFIX"
|
||||
chmod -R go-w "$PREFIX"
|
||||
|
||||
require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed runner image pull"
|
||||
docker volume create "$RUNNER_VOLUME" >/dev/null
|
||||
|
||||
# The registered runner state and the nested rootless daemon cache share one
|
||||
# dedicated volume. Persisting the pinned job image lets validation finish
|
||||
# before registration and avoids racing the first queued workflow for layers.
|
||||
docker run --rm --pull=never --user 0:0 \
|
||||
--volume "$RUNNER_VOLUME:/data" \
|
||||
--entrypoint /bin/sh "$runner_runtime_image" \
|
||||
-ec 'chown 1000:1000 /data && chmod 0700 /data'
|
||||
|
||||
install -d -m 0755 /etc/easyai-gateway-ci-v2
|
||||
if [ -n "$NESTED_DOCKER_REGISTRY_MIRROR" ]; then
|
||||
printf '{"data-root":"/data/docker","registry-mirrors":["%s"]}\n' "$NESTED_DOCKER_REGISTRY_MIRROR" \
|
||||
> /etc/easyai-gateway-ci-v2/daemon.json
|
||||
else
|
||||
printf '{"data-root":"/data/docker"}\n' > /etc/easyai-gateway-ci-v2/daemon.json
|
||||
fi
|
||||
chmod 0644 /etc/easyai-gateway-ci-v2/daemon.json
|
||||
|
||||
# Docker's official rootless DinD baseline is --privileged. Gitea's example
|
||||
# additionally names an AppArmor profile, but Docker does not create that host
|
||||
# profile. Enable it only when this daemon proves it can start the pinned image
|
||||
# with the profile; otherwise leave the option empty.
|
||||
RUNNER_SECURITY_OPTIONS=
|
||||
if docker run --rm --pull=never --privileged \
|
||||
--security-opt apparmor=rootlesskit \
|
||||
--entrypoint /bin/true "$runner_runtime_image" >/dev/null 2>&1; then
|
||||
RUNNER_SECURITY_OPTIONS="--security-opt=apparmor=rootlesskit"
|
||||
elif [ -r /proc/sys/kernel/apparmor_restrict_unprivileged_userns ] && \
|
||||
[ "$(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns)" = "1" ]; then
|
||||
fail "restricted AppArmor user namespaces require the rootlesskit profile (install it with the host Docker packages)"
|
||||
fi
|
||||
|
||||
# Exercise the actual daemon startup before registration. This probe has no
|
||||
# token, state volume, host bind mount, or socket mount.
|
||||
PROBE_CONTAINER="easyai-gateway-ci-v2-probe-$$"
|
||||
cleanup_probe() {
|
||||
docker rm --force "$PROBE_CONTAINER" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap 'cleanup_probe' EXIT
|
||||
trap 'cleanup_probe; exit 129' HUP
|
||||
trap 'cleanup_probe; exit 130' INT
|
||||
trap 'cleanup_probe; exit 143' TERM
|
||||
if [ -n "$RUNNER_SECURITY_OPTIONS" ]; then
|
||||
set -- "$RUNNER_SECURITY_OPTIONS"
|
||||
else
|
||||
set --
|
||||
fi
|
||||
docker run --detach --rm --pull=never \
|
||||
--name "$PROBE_CONTAINER" --privileged \
|
||||
--cpus "$RUNNER_CPUS" --memory "$RUNNER_MEMORY" \
|
||||
--memory-swap "$RUNNER_MEMORY_SWAP" --pids-limit "$RUNNER_PIDS_LIMIT" "$@" \
|
||||
--volume "$RUNNER_VOLUME:/data" \
|
||||
--volume /etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro \
|
||||
--volume "$PREFIX:$PREFIX:ro" \
|
||||
--env DOCKERD_ROOTLESS_ROOTLESSKIT_NET=slirp4netns \
|
||||
--env DOCKERD_ROOTLESS_ROOTLESSKIT_MTU=65520 \
|
||||
--env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns \
|
||||
--entrypoint dockerd-entrypoint.sh "$runner_runtime_image" >/dev/null
|
||||
attempt=0
|
||||
until docker exec "$PROBE_CONTAINER" docker info >/dev/null 2>&1; do
|
||||
attempt=$((attempt + 1))
|
||||
if [ "$attempt" -ge 30 ] || \
|
||||
[ "$(docker inspect --format '{{.State.Running}}' "$PROBE_CONTAINER" 2>/dev/null || true)" != "true" ]; then
|
||||
docker logs "$PROBE_CONTAINER" 2>&1 || true
|
||||
fail "pinned rootless Docker-in-Docker probe did not become ready"
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
[ "$(docker exec "$PROBE_CONTAINER" id -u)" = "1000" ] || \
|
||||
fail "rootless Docker-in-Docker probe is not uid 1000"
|
||||
[ "$(docker inspect --format '{{.HostConfig.Memory}}:{{.HostConfig.MemorySwap}}:{{.HostConfig.NanoCpus}}:{{.HostConfig.PidsLimit}}' "$PROBE_CONTAINER")" = \
|
||||
'2147483648:3221225472:2000000000:1024' ] || \
|
||||
fail "rootless Docker-in-Docker probe does not have the required aggregate resource limits"
|
||||
docker exec "$PROBE_CONTAINER" docker info --format '{{range .SecurityOptions}}{{println .}}{{end}}' | \
|
||||
grep -Fq 'name=rootless' || fail "rootless Docker-in-Docker probe is not rootless"
|
||||
[ "$(docker exec "$PROBE_CONTAINER" docker info --format '{{.DockerRootDir}}')" = /data/docker ] || \
|
||||
fail "rootless Docker-in-Docker probe is not using its persistent dedicated data root"
|
||||
|
||||
# Complete every slow and capability-sensitive validation before registration;
|
||||
# otherwise a queued workflow can race the installer for the same image layers.
|
||||
docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"
|
||||
outer_sentinel_pid=$(docker exec "$PROBE_CONTAINER" /bin/sh -c \
|
||||
'sleep 300 >/dev/null 2>&1 & printf "%s\n" "$!"')
|
||||
case $outer_sentinel_pid in
|
||||
'' | *[!0-9]*) fail "could not create the outer PID namespace sentinel" ;;
|
||||
esac
|
||||
if ! docker exec "$PROBE_CONTAINER" docker run --rm --pull=never \
|
||||
--pid=host --entrypoint /bin/sh "$JOB_IMAGE" \
|
||||
-ec 'test ! -e "/proc/$1"' sh "$outer_sentinel_pid"; then
|
||||
docker exec "$PROBE_CONTAINER" kill "$outer_sentinel_pid" >/dev/null 2>&1 || true
|
||||
fail "nested --pid=host can see the outer runner PID namespace"
|
||||
fi
|
||||
docker exec "$PROBE_CONTAINER" kill "$outer_sentinel_pid" >/dev/null 2>&1 || true
|
||||
docker exec "$PROBE_CONTAINER" docker run --rm --pull=never \
|
||||
--volume "$PREFIX:$PREFIX:ro" \
|
||||
--entrypoint /bin/sh "$JOB_IMAGE" -ec '
|
||||
export PATH=/opt/easyai-gateway-ci/bin:/opt/easyai-gateway-ci/toolchains/go/bin:/opt/easyai-gateway-ci/toolchains/node/bin:$PATH
|
||||
go version
|
||||
node --version
|
||||
pnpm --version
|
||||
docker-compose version
|
||||
shellcheck --version
|
||||
trivy --version
|
||||
govulncheck -version
|
||||
'
|
||||
cleanup_probe
|
||||
trap - EXIT HUP INT TERM
|
||||
|
||||
install -m 0644 "$ROOT/deploy/ci/act-runner-v2-config.yaml" /etc/easyai-gateway-ci-v2/config.yaml
|
||||
umask 077
|
||||
printf 'RUNNER_SECURITY_OPTIONS="%s"\nRUNNER_IMAGE_REF="%s"\n' \
|
||||
"$RUNNER_SECURITY_OPTIONS" "$runner_runtime_image" \
|
||||
> /etc/easyai-gateway-ci-v2/runner.env
|
||||
install -m 0644 "$ROOT/deploy/ci/easyai-gateway-ci-v2-runner.service" \
|
||||
/etc/systemd/system/easyai-gateway-ci-v2-runner.service
|
||||
|
||||
runner_is_registered() {
|
||||
docker run --rm --pull=never \
|
||||
--volume "$RUNNER_VOLUME:/data" \
|
||||
--entrypoint /bin/sh "$runner_runtime_image" \
|
||||
-ec 'test -s /data/.runner'
|
||||
}
|
||||
|
||||
if ! runner_is_registered; then
|
||||
[ -n "${RUNNER_REGISTRATION_TOKEN:-}" ] || \
|
||||
fail "RUNNER_REGISTRATION_TOKEN is required for first registration"
|
||||
docker rm --force easyai-gateway-ci-v2-register >/dev/null 2>&1 || true
|
||||
printf '%s\n%s\n%s\n' "$GITEA_INSTANCE_URL" "$RUNNER_REGISTRATION_TOKEN" "$RUNNER_NAME" | \
|
||||
docker run --rm --interactive --pull=never \
|
||||
--name easyai-gateway-ci-v2-register \
|
||||
--volume "$RUNNER_VOLUME:/data" \
|
||||
--volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro \
|
||||
--env HOME=/data \
|
||||
--entrypoint /usr/local/bin/gitea-runner \
|
||||
"$runner_runtime_image" --config /config.yaml register
|
||||
fi
|
||||
unset RUNNER_REGISTRATION_TOKEN
|
||||
docker run --rm --pull=never \
|
||||
--volume "$RUNNER_VOLUME:/data" \
|
||||
--entrypoint /bin/sh "$runner_runtime_image" \
|
||||
-ec 'test -s /data/.runner && chmod 0600 /data/.runner'
|
||||
|
||||
RUNNER_VALIDATED=0
|
||||
cleanup_unvalidated_runner() {
|
||||
if [ "$RUNNER_VALIDATED" -ne 1 ]; then
|
||||
systemctl disable --now easyai-gateway-ci-v2-runner.service >/dev/null 2>&1 || true
|
||||
docker rm --force "$RUNNER_CONTAINER" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
trap 'cleanup_unvalidated_runner' EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now easyai-gateway-ci-v2-runner.service
|
||||
|
||||
attempt=0
|
||||
until docker exec "$RUNNER_CONTAINER" docker info >/dev/null 2>&1; do
|
||||
attempt=$((attempt + 1))
|
||||
if [ "$attempt" -ge 30 ]; then
|
||||
systemctl --no-pager --full status easyai-gateway-ci-v2-runner.service || true
|
||||
docker logs "$RUNNER_CONTAINER" 2>&1 || true
|
||||
fail "rootless Docker daemon did not become ready"
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
attempt=0
|
||||
until docker logs "$RUNNER_CONTAINER" 2>&1 | grep -Fq 'declare successfully'; do
|
||||
attempt=$((attempt + 1))
|
||||
if [ "$attempt" -ge 30 ] || \
|
||||
[ "$(docker inspect --format '{{.State.Running}}' "$RUNNER_CONTAINER" 2>/dev/null || true)" != "true" ]; then
|
||||
docker logs "$RUNNER_CONTAINER" 2>&1 || true
|
||||
fail "Gitea runner did not declare itself successfully"
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
[ "$(docker inspect --format '{{.Config.User}}' "$RUNNER_CONTAINER")" = "rootless" ] || \
|
||||
fail "outer runner container is not using the rootless image user"
|
||||
[ "$(docker inspect --format '{{.HostConfig.Privileged}}' "$RUNNER_CONTAINER")" = "true" ] || \
|
||||
fail "outer rootless DinD container is not privileged"
|
||||
[ "$(docker inspect --format '{{.HostConfig.Memory}}:{{.HostConfig.MemorySwap}}:{{.HostConfig.NanoCpus}}:{{.HostConfig.PidsLimit}}' "$RUNNER_CONTAINER")" = \
|
||||
'2147483648:3221225472:2000000000:1024' ] || \
|
||||
fail "outer rootless DinD container does not have the required aggregate resource limits"
|
||||
if docker inspect --format '{{range .Mounts}}{{println .Source}}{{end}}' "$RUNNER_CONTAINER" | \
|
||||
grep -Eq '/var/run/docker\.sock|/run/docker\.sock'; then
|
||||
fail "host Docker socket was mounted into the runner"
|
||||
fi
|
||||
docker exec "$RUNNER_CONTAINER" docker info --format '{{range .SecurityOptions}}{{println .}}{{end}}' | \
|
||||
grep -Fq 'name=rootless' || fail "nested Docker daemon is not rootless"
|
||||
[ "$(docker exec "$RUNNER_CONTAINER" docker info --format '{{.DockerRootDir}}')" = /data/docker ] || \
|
||||
fail "nested Docker daemon is not using its persistent dedicated data root"
|
||||
docker exec "$RUNNER_CONTAINER" docker image inspect "$JOB_IMAGE" >/dev/null || \
|
||||
fail "validated pinned job image is unavailable after runner activation"
|
||||
if ! require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed CI runner installation"; then
|
||||
fail "CI runner was stopped because the post-install disk reserve was not met"
|
||||
fi
|
||||
systemctl --no-pager --full status easyai-gateway-ci-v2-runner.service
|
||||
RUNNER_VALIDATED=1
|
||||
trap - EXIT HUP INT TERM
|
||||
Executable
+375
@@ -0,0 +1,375 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
cd "$root"
|
||||
|
||||
components=auto
|
||||
production_host=${AI_GATEWAY_PRODUCTION_HOST:-root@110.42.51.33}
|
||||
remote_helper=${AI_GATEWAY_REMOTE_RELEASE_HELPER:-/usr/local/sbin/easyai-ai-gateway-release}
|
||||
registry=${AI_GATEWAY_IMAGE_REGISTRY:-registry.cn-shanghai.aliyuncs.com/easyaigc}
|
||||
platform=linux/amd64
|
||||
platform_probe_image=${AI_GATEWAY_PLATFORM_PROBE_IMAGE:-docker.m.daocloud.io/library/alpine:3.22}
|
||||
status_file_override=${AI_GATEWAY_RELEASE_STATUS_FILE:-}
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
scripts/publish-release-images.sh [--components auto|api|web|all]
|
||||
|
||||
This command only builds, smoke-tests, and pushes immutable images. It never
|
||||
changes production. The generated manifest must be deployed separately with
|
||||
scripts/deploy-production-release.sh after explicit confirmation.
|
||||
|
||||
Environment:
|
||||
AI_GATEWAY_PRODUCTION_HOST SSH target used for read-only release status
|
||||
AI_GATEWAY_REMOTE_RELEASE_HELPER Fixed status/deploy helper on the server
|
||||
AI_GATEWAY_IMAGE_REGISTRY Registry namespace
|
||||
AI_GATEWAY_RELEASE_STATUS_FILE Local status manifest for offline/testing use
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--components)
|
||||
[[ $# -ge 2 ]] || { echo 'missing value for --components' >&2; exit 64; }
|
||||
components=$2
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
printf 'unknown argument: %s\n' "$1" >&2
|
||||
usage >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
case $components in
|
||||
auto|api|web|all) ;;
|
||||
*) echo '--components must be auto, api, web, or all' >&2; exit 64 ;;
|
||||
esac
|
||||
[[ $production_host =~ ^[A-Za-z0-9._-]+@[A-Za-z0-9.-]+$ ]] || {
|
||||
echo 'AI_GATEWAY_PRODUCTION_HOST must use user@host syntax' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ $remote_helper =~ ^/[A-Za-z0-9._/-]+$ ]] || {
|
||||
echo 'AI_GATEWAY_REMOTE_RELEASE_HELPER must be an absolute safe path' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ $registry =~ ^[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._-]+)+$ ]] || {
|
||||
echo 'AI_GATEWAY_IMAGE_REGISTRY has an invalid format' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ $platform_probe_image =~ ^[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._-]+)*(:[A-Za-z0-9._-]+|@sha256:[0-9a-f]{64})$ ]] || {
|
||||
echo 'AI_GATEWAY_PLATFORM_PROBE_IMAGE has an invalid format' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "release_publish=FAIL $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
for command in git node docker curl; do
|
||||
command -v "$command" >/dev/null 2>&1 || fail "$command is required"
|
||||
done
|
||||
docker info >/dev/null 2>&1 || fail 'Docker Engine is not reachable'
|
||||
docker buildx version >/dev/null 2>&1 || fail 'Docker Buildx is required'
|
||||
docker compose version >/dev/null 2>&1 || fail 'Docker Compose v2 is required'
|
||||
|
||||
git fetch --quiet origin main
|
||||
preflight_file=$(mktemp)
|
||||
node scripts/release-preflight.mjs >"$preflight_file"
|
||||
source_sha=$(node -e \
|
||||
'const value=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8")); process.stdout.write(value.sourceSha)' \
|
||||
"$preflight_file")
|
||||
|
||||
status_file=$(mktemp)
|
||||
selection_file=$(mktemp)
|
||||
smoke_project=easyai-release-smoke-${source_sha:0:12}
|
||||
smoke_started=0
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
trap - EXIT HUP INT TERM
|
||||
if [[ $smoke_started -eq 1 ]]; then
|
||||
COMPOSE_PROJECT_NAME=$smoke_project \
|
||||
docker compose -p "$smoke_project" -f docker-compose.yml down -v --remove-orphans \
|
||||
>/dev/null 2>&1 || true
|
||||
fi
|
||||
rm -f "$status_file" "$selection_file" "$preflight_file"
|
||||
exit "$status"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
has_status=0
|
||||
if [[ -n $status_file_override ]]; then
|
||||
[[ -f $status_file_override ]] || fail 'AI_GATEWAY_RELEASE_STATUS_FILE does not exist'
|
||||
cp "$status_file_override" "$status_file"
|
||||
has_status=1
|
||||
elif ssh -o BatchMode=yes -o ConnectTimeout=8 "$production_host" \
|
||||
"$remote_helper status" >"$status_file" 2>/dev/null; then
|
||||
has_status=1
|
||||
fi
|
||||
|
||||
base_sha=
|
||||
current_api_image=
|
||||
current_web_image=
|
||||
if [[ $has_status -eq 1 ]]; then
|
||||
node scripts/release-manifest.mjs validate "$status_file" >/dev/null
|
||||
base_sha=$(node scripts/release-manifest.mjs get "$status_file" sourceSha)
|
||||
current_api_image=$(node scripts/release-manifest.mjs get "$status_file" images.api)
|
||||
current_web_image=$(node scripts/release-manifest.mjs get "$status_file" images.web)
|
||||
fi
|
||||
|
||||
if [[ $components != all && $has_status -ne 1 ]]; then
|
||||
fail 'auto/api/web publishing requires a readable production release; use --components all for the first release'
|
||||
fi
|
||||
|
||||
required_components=all
|
||||
migrations_changed=true
|
||||
base_verifiable=0
|
||||
if [[ $has_status -eq 1 ]]; then
|
||||
if ! git cat-file -e "$base_sha^{commit}" 2>/dev/null; then
|
||||
git fetch --quiet origin "$base_sha" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if git cat-file -e "$base_sha^{commit}" 2>/dev/null; then
|
||||
git merge-base --is-ancestor "$base_sha" "$source_sha" || \
|
||||
fail 'the production release is not an ancestor of the source commit'
|
||||
base_verifiable=1
|
||||
node scripts/release-components.mjs "$base_sha" "$source_sha" >"$selection_file"
|
||||
required_components=$(node -e \
|
||||
'const value=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8")); process.stdout.write(value.components)' \
|
||||
"$selection_file")
|
||||
migrations_changed=$(node -e \
|
||||
'const value=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8")); process.stdout.write(String(value.migrationsChanged))' \
|
||||
"$selection_file")
|
||||
elif [[ $components != all ]]; then
|
||||
fail 'the production source SHA cannot be verified; rerun with --components all'
|
||||
else
|
||||
echo '[release] production source SHA is unavailable; conservatively building all components and running migrations' >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $components == auto ]]; then
|
||||
components=$required_components
|
||||
fi
|
||||
if [[ $components == none ]]; then
|
||||
echo 'release_publish=SKIP reason=no_runtime_changes'
|
||||
exit 0
|
||||
fi
|
||||
if [[ $required_components == all && $components != all ]]; then
|
||||
fail "the source change requires both images; requested $components"
|
||||
fi
|
||||
if [[ $required_components == api && $components == web ]]; then
|
||||
fail 'the source change requires the API image'
|
||||
fi
|
||||
if [[ $required_components == web && $components == api ]]; then
|
||||
fail 'the source change requires the Web image'
|
||||
fi
|
||||
if [[ $migrations_changed == true && $components != api && $components != all ]]; then
|
||||
fail 'migration changes require the API image'
|
||||
fi
|
||||
|
||||
api_repository=$registry/ai-gateway
|
||||
web_repository=$registry/ai-gateway-web
|
||||
api_tag=$api_repository:$source_sha
|
||||
web_tag=$web_repository:$source_sha
|
||||
|
||||
remote_tag_must_not_exist() {
|
||||
local image=$1
|
||||
local output status
|
||||
set +e
|
||||
output=$(docker buildx imagetools inspect "$image" 2>&1)
|
||||
status=$?
|
||||
set -e
|
||||
if [[ $status -eq 0 ]]; then
|
||||
fail "immutable image tag already exists: $image"
|
||||
fi
|
||||
if ! grep -Eqi 'not found|manifest unknown|does not exist' <<<"$output"; then
|
||||
fail "cannot verify registry login and repository access for $image"
|
||||
fi
|
||||
}
|
||||
|
||||
case $components in
|
||||
api) remote_tag_must_not_exist "$api_tag" ;;
|
||||
web) remote_tag_must_not_exist "$web_tag" ;;
|
||||
all)
|
||||
remote_tag_must_not_exist "$api_tag"
|
||||
remote_tag_must_not_exist "$web_tag"
|
||||
;;
|
||||
esac
|
||||
|
||||
docker run --rm --platform "$platform" "$platform_probe_image" true >/dev/null 2>&1 || \
|
||||
fail "Docker cannot execute $platform containers"
|
||||
|
||||
if [[ $base_verifiable -eq 1 ]]; then
|
||||
node scripts/ci-validate-migrations.mjs "$base_sha"
|
||||
fi
|
||||
|
||||
echo '[release] running fast Go tests without database integration variables'
|
||||
(
|
||||
cd apps/api
|
||||
env -u AI_GATEWAY_TEST_DATABASE_URL go test ./... -count=1
|
||||
)
|
||||
|
||||
api_image=$current_api_image
|
||||
web_image=$current_web_image
|
||||
|
||||
build_image() {
|
||||
local target=$1
|
||||
local image=$2
|
||||
local -a build_args=()
|
||||
case $target in
|
||||
api)
|
||||
build_args=(
|
||||
--build-arg "GOPROXY=${AI_GATEWAY_GO_PROXY:-https://goproxy.cn,direct}"
|
||||
--build-arg "GO_BUILD_IMAGE=${AI_GATEWAY_GO_BUILD_IMAGE:-golang:1.26.3-alpine}"
|
||||
--build-arg "API_RUNTIME_IMAGE=${AI_GATEWAY_API_RUNTIME_IMAGE:-alpine:3.22}"
|
||||
)
|
||||
;;
|
||||
web)
|
||||
build_args=(
|
||||
--build-arg "VITE_GATEWAY_API_BASE_URL=${AI_GATEWAY_WEB_API_BASE_URL:-/gateway-api}"
|
||||
--build-arg "NODE_BUILD_IMAGE=${AI_GATEWAY_NODE_BUILD_IMAGE:-node:22-alpine}"
|
||||
--build-arg "WEB_RUNTIME_IMAGE=${AI_GATEWAY_WEB_RUNTIME_IMAGE:-nginx:1.27-alpine}"
|
||||
--build-arg "NPM_CONFIG_REGISTRY=${AI_GATEWAY_NPM_REGISTRY:-https://registry.npmjs.org}"
|
||||
--build-arg "VITE_BASE_PATH=${AI_GATEWAY_WEB_BASE_PATH:-/}"
|
||||
)
|
||||
;;
|
||||
esac
|
||||
echo "[release] building $target for $platform: $image"
|
||||
docker buildx build \
|
||||
--platform "$platform" \
|
||||
--file Dockerfile \
|
||||
--target "$target" \
|
||||
--label "org.opencontainers.image.revision=$source_sha" \
|
||||
--label 'org.opencontainers.image.source=https://git.51easyai.com/BCAI/easyai-ai-gateway' \
|
||||
--tag "$image" \
|
||||
--load \
|
||||
"${build_args[@]}" \
|
||||
.
|
||||
[[ $(docker image inspect "$image" --format '{{.Os}}/{{.Architecture}}') == "$platform" ]] || \
|
||||
fail "$image is not $platform"
|
||||
[[ $(docker image inspect "$image" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}') == "$source_sha" ]] || \
|
||||
fail "$image is missing the source revision label"
|
||||
}
|
||||
|
||||
case $components in
|
||||
api)
|
||||
build_image api "$api_tag"
|
||||
api_image=$api_tag
|
||||
;;
|
||||
web)
|
||||
build_image web "$web_tag"
|
||||
web_image=$web_tag
|
||||
;;
|
||||
all)
|
||||
build_image api "$api_tag"
|
||||
build_image web "$web_tag"
|
||||
api_image=$api_tag
|
||||
web_image=$web_tag
|
||||
;;
|
||||
esac
|
||||
|
||||
for image in "$api_image" "$web_image"; do
|
||||
docker image inspect "$image" >/dev/null 2>&1 || docker pull --platform "$platform" "$image"
|
||||
done
|
||||
build_completed_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
export COMPOSE_PROJECT_NAME=$smoke_project
|
||||
export AI_GATEWAY_PLATFORM=$platform
|
||||
export AI_GATEWAY_API_IMAGE=$api_image
|
||||
export AI_GATEWAY_WEB_IMAGE=$web_image
|
||||
export AI_GATEWAY_API_PORT=0
|
||||
export AI_GATEWAY_WEB_PORT=0
|
||||
export AI_GATEWAY_DB_PORT=0
|
||||
export AI_GATEWAY_COMPOSE_APP_ENV=test
|
||||
export AI_GATEWAY_COMPOSE_IDENTITY_MODE=hybrid
|
||||
export AI_GATEWAY_COMPOSE_DATABASE_NAME=easyai_ai_gateway
|
||||
export AI_GATEWAY_COMPOSE_PG_USER=easyai
|
||||
export AI_GATEWAY_COMPOSE_PG_PASSWORD=release-smoke-postgres
|
||||
export AI_GATEWAY_COMPOSE_DATABASE_URL='postgresql://easyai:release-smoke-postgres@postgres:5432/easyai_ai_gateway?sslmode=disable'
|
||||
export CONFIG_JWT_SECRET=release-smoke-$source_sha
|
||||
|
||||
smoke_started=1
|
||||
docker compose -p "$smoke_project" -f docker-compose.yml up -d --pull never postgres
|
||||
for _ in $(seq 1 60); do
|
||||
postgres_id=$(docker compose -p "$smoke_project" -f docker-compose.yml ps -q postgres)
|
||||
postgres_health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$postgres_id" 2>/dev/null || true)
|
||||
[[ $postgres_health == healthy ]] && break
|
||||
sleep 1
|
||||
done
|
||||
[[ ${postgres_health:-} == healthy ]] || fail 'ephemeral PostgreSQL did not become healthy'
|
||||
|
||||
docker compose -p "$smoke_project" -f docker-compose.yml run --rm --no-deps migrator
|
||||
docker compose -p "$smoke_project" -f docker-compose.yml up -d --pull never --no-build api web
|
||||
api_port=$(docker compose -p "$smoke_project" -f docker-compose.yml port api 8088 | tail -n 1)
|
||||
web_port=$(docker compose -p "$smoke_project" -f docker-compose.yml port web 80 | tail -n 1)
|
||||
api_port=${api_port##*:}
|
||||
web_port=${web_port##*:}
|
||||
[[ $api_port =~ ^[0-9]+$ && $web_port =~ ^[0-9]+$ ]] || fail 'could not resolve smoke ports'
|
||||
|
||||
RELEASE_SMOKE_BASE_URL=http://127.0.0.1:$api_port \
|
||||
RELEASE_SMOKE_WEB_URL=http://127.0.0.1:$web_port \
|
||||
RELEASE_SMOKE_COMPOSE_PROJECT=$smoke_project \
|
||||
RELEASE_SMOKE_COMPOSE_FILE=$root/docker-compose.yml \
|
||||
RELEASE_SMOKE_NONCE=${source_sha:0:12} \
|
||||
RELEASE_SMOKE_DATABASE_USER=$AI_GATEWAY_COMPOSE_PG_USER \
|
||||
RELEASE_SMOKE_DATABASE_NAME=$AI_GATEWAY_COMPOSE_DATABASE_NAME \
|
||||
node scripts/api-release-smoke.mjs
|
||||
smoke_completed_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
push_and_resolve() {
|
||||
local image=$1
|
||||
local repository=$2
|
||||
local resolved
|
||||
docker push "$image" >&2
|
||||
resolved=$(docker image inspect "$image" --format '{{range .RepoDigests}}{{println .}}{{end}}' | \
|
||||
awk -v prefix="$repository@sha256:" 'index($0, prefix) == 1 { print; exit }')
|
||||
[[ $resolved =~ ^[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._-]+)+@sha256:[0-9a-f]{64}$ ]] || \
|
||||
fail "could not resolve registry digest for $image"
|
||||
printf '%s\n' "$resolved"
|
||||
}
|
||||
|
||||
case $components in
|
||||
api)
|
||||
api_image=$(push_and_resolve "$api_tag" "$api_repository")
|
||||
component_list=api
|
||||
;;
|
||||
web)
|
||||
web_image=$(push_and_resolve "$web_tag" "$web_repository")
|
||||
component_list=web
|
||||
;;
|
||||
all)
|
||||
api_image=$(push_and_resolve "$api_tag" "$api_repository")
|
||||
web_image=$(push_and_resolve "$web_tag" "$web_repository")
|
||||
component_list=api,web
|
||||
;;
|
||||
esac
|
||||
|
||||
mkdir -p dist/releases
|
||||
manifest=$root/dist/releases/$source_sha.json
|
||||
[[ ! -e $manifest ]] || fail "release manifest already exists: $manifest"
|
||||
RELEASE_SOURCE_SHA=$source_sha \
|
||||
RELEASE_BASE_SHA=$base_sha \
|
||||
RELEASE_COMPONENTS=$component_list \
|
||||
RELEASE_MIGRATIONS_CHANGED=$migrations_changed \
|
||||
RELEASE_API_IMAGE=$api_image \
|
||||
RELEASE_WEB_IMAGE=$web_image \
|
||||
RELEASE_BUILD_COMPLETED_AT=$build_completed_at \
|
||||
RELEASE_SMOKE_COMPLETED_AT=$smoke_completed_at \
|
||||
node scripts/release-manifest.mjs create "$manifest"
|
||||
|
||||
echo "release_publish=PASS release=$source_sha components=$component_list"
|
||||
echo "release_manifest=$manifest"
|
||||
echo "api_image=$api_image"
|
||||
echo "web_image=$web_image"
|
||||
echo 'production_changed=false'
|
||||
Executable
+157
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync } from 'node:child_process'
|
||||
|
||||
function fail(message) {
|
||||
console.error(`release_components=FAIL ${message}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const base = process.argv[2] || ''
|
||||
const head = process.argv[3] || 'HEAD'
|
||||
|
||||
if (!/^[0-9a-f]{40}$/.test(base)) {
|
||||
fail('base must be a full lowercase Git SHA')
|
||||
}
|
||||
|
||||
function git(args) {
|
||||
try {
|
||||
return execFileSync('git', args, {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
}).trim()
|
||||
} catch (error) {
|
||||
const stderr = error.stderr?.toString().trim()
|
||||
fail(stderr ? `git_error=${JSON.stringify(stderr)}` : 'git command failed')
|
||||
}
|
||||
}
|
||||
|
||||
const resolvedBase = git(['rev-parse', '--verify', `${base}^{commit}`])
|
||||
const resolvedHead = git(['rev-parse', '--verify', `${head}^{commit}`])
|
||||
if (resolvedBase !== base) fail('base did not resolve exactly')
|
||||
if (!/^[0-9a-f]{40}$/.test(resolvedHead)) fail('head did not resolve to a commit')
|
||||
|
||||
try {
|
||||
execFileSync('git', ['merge-base', '--is-ancestor', base, resolvedHead], {
|
||||
stdio: 'ignore',
|
||||
})
|
||||
} catch {
|
||||
fail('base is not an ancestor of head')
|
||||
}
|
||||
|
||||
const changed = git([
|
||||
'diff',
|
||||
'--name-only',
|
||||
'--no-renames',
|
||||
`${base}..${resolvedHead}`,
|
||||
]).split('\n').filter(Boolean)
|
||||
|
||||
const apiPatterns = [
|
||||
/^apps\/api\//,
|
||||
/^go\.work(?:\.sum)?$/,
|
||||
]
|
||||
const webPatterns = [
|
||||
/^apps\/web\//,
|
||||
/^packages\/contracts\//,
|
||||
/^pnpm-lock\.yaml$/,
|
||||
/^pnpm-workspace\.yaml$/,
|
||||
/^nx\.json$/,
|
||||
/^package\.json$/,
|
||||
/^docker\/nginx\.conf$/,
|
||||
]
|
||||
const sharedRuntimePatterns = [
|
||||
/^\.dockerignore$/,
|
||||
/^docker-compose(?:\.[^.]+)?\.ya?ml$/,
|
||||
]
|
||||
|
||||
function dockerfileSections(ref) {
|
||||
let contents
|
||||
try {
|
||||
contents = execFileSync('git', ['show', `${ref}:Dockerfile`], {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
})
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
const sections = new Map([['global', []]])
|
||||
let section = 'global'
|
||||
for (const line of contents.split('\n')) {
|
||||
const match = line.match(/^FROM\s+.+?\s+AS\s+([A-Za-z0-9._-]+)(?:\s|$)/i)
|
||||
if (match) section = match[1].toLowerCase()
|
||||
if (!sections.has(section)) sections.set(section, [])
|
||||
sections.get(section).push(line)
|
||||
}
|
||||
return sections
|
||||
}
|
||||
|
||||
function sectionChanged(before, after, name) {
|
||||
return (before.get(name) || []).join('\n') !== (after.get(name) || []).join('\n')
|
||||
}
|
||||
|
||||
function classifyDockerfile() {
|
||||
const before = dockerfileSections(base)
|
||||
const after = dockerfileSections(resolvedHead)
|
||||
if (!before || !after) return { apiChanged: true, webChanged: true }
|
||||
const knownApi = new Set(['api-builder', 'api'])
|
||||
const knownWeb = new Set(['web-builder', 'web'])
|
||||
let apiChanged = false
|
||||
let webChanged = false
|
||||
|
||||
const beforeGlobal = before.get('global') || []
|
||||
const afterGlobal = after.get('global') || []
|
||||
const globalLines = (lines, pattern) => lines.filter((line) => pattern.test(line)).join('\n')
|
||||
const apiGlobal = /\b(?:GO_VERSION|GO_BUILD_IMAGE|API_RUNTIME_IMAGE)\b/
|
||||
const webGlobal = /\b(?:NODE_VERSION|NODE_BUILD_IMAGE|WEB_RUNTIME_IMAGE)\b/
|
||||
if (globalLines(beforeGlobal, apiGlobal) !== globalLines(afterGlobal, apiGlobal)) apiChanged = true
|
||||
if (globalLines(beforeGlobal, webGlobal) !== globalLines(afterGlobal, webGlobal)) webChanged = true
|
||||
const sharedGlobal = (lines) => lines.filter((line) => !apiGlobal.test(line) && !webGlobal.test(line)).join('\n')
|
||||
if (sharedGlobal(beforeGlobal) !== sharedGlobal(afterGlobal)) {
|
||||
apiChanged = true
|
||||
webChanged = true
|
||||
}
|
||||
|
||||
const sectionNames = new Set([...before.keys(), ...after.keys()])
|
||||
sectionNames.delete('global')
|
||||
for (const name of sectionNames) {
|
||||
if (!sectionChanged(before, after, name)) continue
|
||||
if (knownApi.has(name)) apiChanged = true
|
||||
else if (knownWeb.has(name)) webChanged = true
|
||||
else {
|
||||
apiChanged = true
|
||||
webChanged = true
|
||||
}
|
||||
}
|
||||
return { apiChanged, webChanged }
|
||||
}
|
||||
|
||||
let api = false
|
||||
let web = false
|
||||
let migrationsChanged = false
|
||||
|
||||
if (changed.includes('Dockerfile')) {
|
||||
const dockerfile = classifyDockerfile()
|
||||
api = dockerfile.apiChanged
|
||||
web = dockerfile.webChanged
|
||||
}
|
||||
|
||||
for (const file of changed) {
|
||||
if (file === 'Dockerfile') continue
|
||||
if (/^apps\/api\/migrations\//.test(file)) migrationsChanged = true
|
||||
if (sharedRuntimePatterns.some((pattern) => pattern.test(file))) {
|
||||
api = true
|
||||
web = true
|
||||
continue
|
||||
}
|
||||
if (apiPatterns.some((pattern) => pattern.test(file))) api = true
|
||||
if (webPatterns.some((pattern) => pattern.test(file))) web = true
|
||||
}
|
||||
|
||||
const components = api && web ? 'all' : api ? 'api' : web ? 'web' : 'none'
|
||||
console.log(JSON.stringify({
|
||||
base,
|
||||
head: resolvedHead,
|
||||
components,
|
||||
migrationsChanged,
|
||||
changedFiles: changed,
|
||||
}))
|
||||
Executable
+227
@@ -0,0 +1,227 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { createHash } from 'node:crypto'
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
|
||||
const shaPattern = /^[0-9a-f]{40}$/
|
||||
const digestImagePattern = /^[A-Za-z0-9.-]+(?::[0-9]+)?(?:\/[A-Za-z0-9._-]+)+@sha256:[0-9a-f]{64}$/
|
||||
const allowedComponents = new Set(['api', 'web'])
|
||||
const smokeChecks = [
|
||||
'health',
|
||||
'readiness',
|
||||
'openapi',
|
||||
'register',
|
||||
'login',
|
||||
'apiKey',
|
||||
'models',
|
||||
'chatCompletions',
|
||||
'responses',
|
||||
'images',
|
||||
'web',
|
||||
]
|
||||
|
||||
function requireExactKeys(value, keys, path) {
|
||||
const actual = Object.keys(value).sort()
|
||||
const expected = [...keys].sort()
|
||||
if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) {
|
||||
fail(`${path} has unexpected or missing fields`)
|
||||
}
|
||||
}
|
||||
|
||||
function releaseIntegrity(manifest) {
|
||||
const payload = structuredClone(manifest)
|
||||
delete payload.integrity
|
||||
delete payload.deployment
|
||||
return createHash('sha256').update(JSON.stringify(payload)).digest('hex')
|
||||
}
|
||||
|
||||
function fail(message) {
|
||||
console.error(`release_manifest=FAIL ${message}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
function load(path) {
|
||||
try {
|
||||
return JSON.parse(readFileSync(path, 'utf8'))
|
||||
} catch (error) {
|
||||
fail(`cannot read JSON manifest: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
function validate(manifest) {
|
||||
if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) {
|
||||
fail('manifest root must be an object')
|
||||
}
|
||||
const rootKeys = [
|
||||
'schemaVersion', 'releaseId', 'sourceSha', 'baseReleaseSha', 'sourceRef',
|
||||
'createdAt', 'buildCompletedAt', 'targetPlatform', 'components',
|
||||
'migrationsChanged', 'images', 'smoke', 'integrity',
|
||||
]
|
||||
if (manifest.deployment !== undefined) rootKeys.push('deployment')
|
||||
requireExactKeys(manifest, rootKeys, 'manifest')
|
||||
if (manifest.schemaVersion !== 1) fail('schemaVersion must be 1')
|
||||
if (!shaPattern.test(manifest.sourceSha || '')) fail('sourceSha must be a full lowercase Git SHA')
|
||||
if (manifest.releaseId !== manifest.sourceSha) fail('releaseId must equal sourceSha')
|
||||
if (manifest.sourceRef !== 'refs/heads/main') fail('sourceRef must be refs/heads/main')
|
||||
if (manifest.baseReleaseSha !== null && !shaPattern.test(manifest.baseReleaseSha || '')) {
|
||||
fail('baseReleaseSha must be null or a full lowercase Git SHA')
|
||||
}
|
||||
if (manifest.baseReleaseSha === manifest.sourceSha) fail('baseReleaseSha must differ from sourceSha')
|
||||
if (manifest.targetPlatform !== 'linux/amd64') fail('targetPlatform must be linux/amd64')
|
||||
if (!Number.isFinite(Date.parse(manifest.createdAt))) fail('createdAt must be an ISO timestamp')
|
||||
if (!Number.isFinite(Date.parse(manifest.buildCompletedAt))) {
|
||||
fail('buildCompletedAt must be an ISO timestamp')
|
||||
}
|
||||
if (!Array.isArray(manifest.components) || manifest.components.length < 1 || manifest.components.length > 2) {
|
||||
fail('components must contain api, web, or both')
|
||||
}
|
||||
const uniqueComponents = [...new Set(manifest.components)]
|
||||
if (uniqueComponents.length !== manifest.components.length ||
|
||||
uniqueComponents.some((component) => !allowedComponents.has(component))) {
|
||||
fail('components contains an invalid or duplicate value')
|
||||
}
|
||||
if (typeof manifest.migrationsChanged !== 'boolean') fail('migrationsChanged must be boolean')
|
||||
if (manifest.migrationsChanged && !manifest.components.includes('api')) {
|
||||
fail('migration changes require the api component')
|
||||
}
|
||||
if (!manifest.images || typeof manifest.images !== 'object') fail('images must be an object')
|
||||
requireExactKeys(manifest.images, ['api', 'web'], 'images')
|
||||
for (const component of allowedComponents) {
|
||||
if (!digestImagePattern.test(manifest.images[component] || '')) {
|
||||
fail(`images.${component} must be a registry digest reference`)
|
||||
}
|
||||
}
|
||||
if (!manifest.smoke || manifest.smoke.status !== 'passed' || manifest.smoke.mode !== 'simulation') {
|
||||
fail('smoke must record a passed simulation check')
|
||||
}
|
||||
requireExactKeys(manifest.smoke, ['status', 'mode', 'completedAt', 'checks'], 'smoke')
|
||||
if (!Number.isFinite(Date.parse(manifest.smoke.completedAt))) {
|
||||
fail('smoke.completedAt must be an ISO timestamp')
|
||||
}
|
||||
if (!Array.isArray(manifest.smoke.checks) ||
|
||||
manifest.smoke.checks.length !== smokeChecks.length ||
|
||||
manifest.smoke.checks.some((check, index) => check !== smokeChecks[index])) {
|
||||
fail('smoke.checks is incomplete or out of order')
|
||||
}
|
||||
if (!manifest.integrity || typeof manifest.integrity !== 'object') {
|
||||
fail('integrity must be an object')
|
||||
}
|
||||
requireExactKeys(manifest.integrity, ['algorithm', 'value'], 'integrity')
|
||||
if (manifest.integrity.algorithm !== 'sha256' || !/^[0-9a-f]{64}$/.test(manifest.integrity.value || '')) {
|
||||
fail('integrity must contain a SHA-256 value')
|
||||
}
|
||||
if (releaseIntegrity(manifest) !== manifest.integrity.value) {
|
||||
fail('manifest content integrity check failed')
|
||||
}
|
||||
if (manifest.deployment !== undefined) {
|
||||
if (!manifest.deployment || typeof manifest.deployment !== 'object') {
|
||||
fail('deployment must be an object')
|
||||
}
|
||||
requireExactKeys(manifest.deployment, ['status', 'action', 'deployedAt', 'durationSeconds'], 'deployment')
|
||||
if (manifest.deployment.status !== 'passed' ||
|
||||
!['deploy', 'rollback'].includes(manifest.deployment.action) ||
|
||||
!Number.isFinite(Date.parse(manifest.deployment.deployedAt)) ||
|
||||
!Number.isInteger(manifest.deployment.durationSeconds) ||
|
||||
manifest.deployment.durationSeconds < 0) {
|
||||
fail('deployment record is invalid')
|
||||
}
|
||||
}
|
||||
|
||||
const forbiddenKey = /(password|secret|token|credential|private.?key)/i
|
||||
const visit = (value, path = '') => {
|
||||
if (!value || typeof value !== 'object') return
|
||||
for (const [key, child] of Object.entries(value)) {
|
||||
const childPath = path ? `${path}.${key}` : key
|
||||
if (forbiddenKey.test(key)) fail(`forbidden sensitive field: ${childPath}`)
|
||||
visit(child, childPath)
|
||||
}
|
||||
}
|
||||
visit(manifest)
|
||||
return manifest
|
||||
}
|
||||
|
||||
function requireEnv(name, pattern) {
|
||||
const value = process.env[name] || ''
|
||||
if (!value || (pattern && !pattern.test(value))) fail(`invalid or missing ${name}`)
|
||||
return value
|
||||
}
|
||||
|
||||
const [command = 'validate', path, field] = process.argv.slice(2)
|
||||
|
||||
if (command === 'create') {
|
||||
if (!path) fail('create requires an output path')
|
||||
const sourceSha = requireEnv('RELEASE_SOURCE_SHA', shaPattern)
|
||||
const baseReleaseSha = process.env.RELEASE_BASE_SHA || null
|
||||
if (baseReleaseSha !== null && !shaPattern.test(baseReleaseSha)) fail('invalid RELEASE_BASE_SHA')
|
||||
const components = requireEnv('RELEASE_COMPONENTS').split(',').filter(Boolean)
|
||||
const migrationsChanged = process.env.RELEASE_MIGRATIONS_CHANGED === 'true'
|
||||
const manifest = {
|
||||
schemaVersion: 1,
|
||||
releaseId: sourceSha,
|
||||
sourceSha,
|
||||
baseReleaseSha,
|
||||
sourceRef: 'refs/heads/main',
|
||||
createdAt: new Date().toISOString(),
|
||||
buildCompletedAt: requireEnv('RELEASE_BUILD_COMPLETED_AT'),
|
||||
targetPlatform: 'linux/amd64',
|
||||
components,
|
||||
migrationsChanged,
|
||||
images: {
|
||||
api: requireEnv('RELEASE_API_IMAGE', digestImagePattern),
|
||||
web: requireEnv('RELEASE_WEB_IMAGE', digestImagePattern),
|
||||
},
|
||||
smoke: {
|
||||
status: 'passed',
|
||||
mode: 'simulation',
|
||||
completedAt: requireEnv('RELEASE_SMOKE_COMPLETED_AT'),
|
||||
checks: smokeChecks,
|
||||
},
|
||||
integrity: { algorithm: 'sha256', value: '' },
|
||||
}
|
||||
manifest.integrity.value = releaseIntegrity(manifest)
|
||||
validate(manifest)
|
||||
writeFileSync(path, `${JSON.stringify(manifest, null, 2)}\n`, { flag: 'wx', mode: 0o600 })
|
||||
const digest = createHash('sha256').update(readFileSync(path)).digest('hex')
|
||||
console.log(`release_manifest=CREATED path=${path} sha256=${digest}`)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
if (!path) fail(`${command} requires a manifest path`)
|
||||
const manifest = validate(load(path))
|
||||
|
||||
if (command === 'record-deployment') {
|
||||
if (!field) fail('record-deployment requires an output path')
|
||||
const deployedAt = requireEnv('RELEASE_DEPLOYED_AT')
|
||||
const durationSeconds = Number(process.env.RELEASE_DEPLOY_DURATION_SECONDS || '')
|
||||
const action = process.env.RELEASE_DEPLOY_ACTION || 'deploy'
|
||||
if (!Number.isFinite(Date.parse(deployedAt))) fail('invalid RELEASE_DEPLOYED_AT')
|
||||
if (!Number.isInteger(durationSeconds) || durationSeconds < 0) {
|
||||
fail('invalid RELEASE_DEPLOY_DURATION_SECONDS')
|
||||
}
|
||||
if (!['deploy', 'rollback'].includes(action)) fail('invalid RELEASE_DEPLOY_ACTION')
|
||||
manifest.deployment = { status: 'passed', action, deployedAt, durationSeconds }
|
||||
validate(manifest)
|
||||
writeFileSync(field, `${JSON.stringify(manifest, null, 2)}\n`, { flag: 'wx', mode: 0o600 })
|
||||
console.log(`release_manifest=RECORDED release=${manifest.releaseId} action=${action}`)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
if (command === 'validate') {
|
||||
const digest = createHash('sha256').update(readFileSync(path)).digest('hex')
|
||||
console.log(`release_manifest=PASS release=${manifest.releaseId} sha256=${digest}`)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
if (command === 'get') {
|
||||
if (!field || !/^[A-Za-z][A-Za-z0-9]*(?:\.[A-Za-z][A-Za-z0-9]*)*$/.test(field)) {
|
||||
fail('get requires a safe dotted field path')
|
||||
}
|
||||
let value = manifest
|
||||
for (const segment of field.split('.')) value = value?.[segment]
|
||||
if (typeof value === 'object') console.log(JSON.stringify(value))
|
||||
else if (value === null || value === undefined) console.log('')
|
||||
else console.log(String(value))
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
fail(`unknown command: ${command}`)
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync, spawnSync } from 'node:child_process'
|
||||
|
||||
function fail(message) {
|
||||
console.error(`release_preflight=FAIL ${message}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
function git(args) {
|
||||
try {
|
||||
return execFileSync('git', args, {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
}).trim()
|
||||
} catch (error) {
|
||||
const stderr = error.stderr?.toString().trim()
|
||||
fail(stderr ? `git_error=${JSON.stringify(stderr)}` : 'git command failed')
|
||||
}
|
||||
}
|
||||
|
||||
const root = git(['rev-parse', '--show-toplevel'])
|
||||
process.chdir(root)
|
||||
|
||||
for (const args of [['diff', '--quiet'], ['diff', '--cached', '--quiet']]) {
|
||||
if (spawnSync('git', args, { stdio: 'ignore' }).status !== 0) {
|
||||
fail('the worktree must be clean')
|
||||
}
|
||||
}
|
||||
if (git(['ls-files', '--others', '--exclude-standard'])) {
|
||||
fail('the worktree must not contain untracked files')
|
||||
}
|
||||
|
||||
const sourceSha = git(['rev-parse', '--verify', 'HEAD^{commit}'])
|
||||
if (!/^[0-9a-f]{40}$/.test(sourceSha)) fail('HEAD is not a full Git commit')
|
||||
const mainSha = git(['rev-parse', '--verify', 'refs/remotes/origin/main^{commit}'])
|
||||
if (!/^[0-9a-f]{40}$/.test(mainSha)) fail('origin/main is unavailable')
|
||||
if (spawnSync('git', ['merge-base', '--is-ancestor', sourceSha, mainSha], { stdio: 'ignore' }).status !== 0) {
|
||||
fail('HEAD is not part of origin/main history')
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({ sourceSha, originMainSha: mainSha }))
|
||||
@@ -1,109 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
script=$root/scripts/ci-build-images.sh
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
|
||||
mkdir -p "$tmp/bin"
|
||||
cat >"$tmp/bin/docker" <<'MOCK'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
printf 'docker %s\n' "$*" >>"$MOCK_LOG"
|
||||
|
||||
case "$*" in
|
||||
'buildx inspect '*|'buildx create '*) exit 0 ;;
|
||||
'buildx build '*)
|
||||
if [[ ${MOCK_BUILD_FAIL:-0} == 1 && $* == *'--target web'* ]]; then
|
||||
exit 21
|
||||
fi
|
||||
;;
|
||||
'buildx prune '*)
|
||||
[[ ${MOCK_PRUNE_FAIL:-0} == 0 ]] || exit 22
|
||||
;;
|
||||
'image rm '*)
|
||||
[[ ${MOCK_IMAGE_RM_FAIL:-0} == 0 ]] || exit 23
|
||||
;;
|
||||
*)
|
||||
printf 'unexpected docker invocation: %s\n' "$*" >&2
|
||||
exit 99
|
||||
;;
|
||||
esac
|
||||
MOCK
|
||||
cat >"$tmp/bin/trivy" <<'MOCK'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
printf 'trivy %s\n' "$*" >>"$MOCK_LOG"
|
||||
MOCK
|
||||
chmod +x "$tmp/bin/docker" "$tmp/bin/trivy"
|
||||
|
||||
sha=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
registry=registry.example.com/easyai
|
||||
api_image=$registry/ai-gateway:$sha
|
||||
web_image=$registry/ai-gateway-web:$sha
|
||||
|
||||
run_build() {
|
||||
env \
|
||||
PATH="$tmp/bin:$PATH" \
|
||||
MOCK_LOG="$tmp/calls.log" \
|
||||
IMAGE_TAG="$sha" \
|
||||
AI_GATEWAY_IMAGE_REGISTRY="$registry" \
|
||||
AI_GATEWAY_BUILDX_BUILDER=gateway-test \
|
||||
AI_GATEWAY_BUILDX_CACHE_LIMIT=2gb \
|
||||
"$script" "$@"
|
||||
}
|
||||
|
||||
: >"$tmp/calls.log"
|
||||
run_build
|
||||
grep -Fq "docker buildx prune --builder gateway-test --force --max-used-space 2gb" "$tmp/calls.log"
|
||||
grep -Fq "docker image rm $api_image $web_image" "$tmp/calls.log"
|
||||
grep -Fq 'trivy image --db-repository ghcr.m.daocloud.io/aquasecurity/trivy-db:2' \
|
||||
"$tmp/calls.log"
|
||||
|
||||
: >"$tmp/calls.log"
|
||||
run_build --keep-images
|
||||
grep -Fq "docker buildx prune --builder gateway-test --force --max-used-space 2gb" "$tmp/calls.log"
|
||||
if grep -Fq 'docker image rm ' "$tmp/calls.log"; then
|
||||
echo '--keep-images must preserve fully built images' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
: >"$tmp/calls.log"
|
||||
if MOCK_PRUNE_FAIL=1 run_build; then
|
||||
echo 'a prune failure must fail an otherwise successful build' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
: >"$tmp/calls.log"
|
||||
if MOCK_IMAGE_RM_FAIL=1 run_build; then
|
||||
echo 'an image cleanup failure must fail an otherwise successful build' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
: >"$tmp/calls.log"
|
||||
if MOCK_BUILD_FAIL=1 run_build; then
|
||||
echo 'a failed image build must fail the script' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq "docker image rm $api_image $web_image" "$tmp/calls.log"
|
||||
|
||||
for invalid_limit in 0gb 2 2GB -1gb '2gb --all'; do
|
||||
: >"$tmp/calls.log"
|
||||
if env \
|
||||
PATH="$tmp/bin:$PATH" \
|
||||
MOCK_LOG="$tmp/calls.log" \
|
||||
IMAGE_TAG="$sha" \
|
||||
AI_GATEWAY_IMAGE_REGISTRY="$registry" \
|
||||
AI_GATEWAY_BUILDX_CACHE_LIMIT="$invalid_limit" \
|
||||
"$script" >/dev/null 2>&1; then
|
||||
printf 'invalid cache limit was accepted: %s\n' "$invalid_limit" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -s $tmp/calls.log ]]; then
|
||||
printf 'invalid cache limit reached Docker: %s\n' "$invalid_limit" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo 'ci_build_images_tests=PASS'
|
||||
+35
-228
@@ -12,15 +12,12 @@ new_repo() {
|
||||
|
||||
mkdir -p "$repo/apps/api/migrations"
|
||||
git -C "$repo" init -q
|
||||
git -C "$repo" config user.name 'CI Migration Test'
|
||||
git -C "$repo" config user.email 'ci-migration-test@example.invalid'
|
||||
git -C "$repo" config user.name 'Release Migration Test'
|
||||
git -C "$repo" config user.email 'release-migration-test@example.invalid'
|
||||
printf 'CREATE TABLE accounts (id bigint PRIMARY KEY);\n' > \
|
||||
"$repo/apps/api/migrations/0001_init.sql"
|
||||
git -C "$repo" add apps/api/migrations/0001_init.sql
|
||||
git -C "$repo" commit -qm baseline
|
||||
mkdir -p "$repo/deploy/ci"
|
||||
git -C "$repo" rev-parse HEAD > \
|
||||
"$repo/deploy/ci/production-migration-base"
|
||||
printf '%s\n' "$repo"
|
||||
}
|
||||
|
||||
@@ -34,35 +31,19 @@ commit_all() {
|
||||
expect_pass() {
|
||||
local repo=$1
|
||||
local base=$2
|
||||
local immutable_base=${3:-}
|
||||
(
|
||||
cd "$repo"
|
||||
if [[ -n $immutable_base ]]; then
|
||||
node "$validator" "$base" "$immutable_base"
|
||||
else
|
||||
node "$validator" "$base"
|
||||
fi
|
||||
)
|
||||
(cd "$repo" && node "$validator" "$base")
|
||||
}
|
||||
|
||||
expect_fail() {
|
||||
local repo=$1
|
||||
local base=$2
|
||||
local expected=$3
|
||||
local immutable_base=${4:-}
|
||||
local output status
|
||||
|
||||
if [[ -n $immutable_base ]]; then
|
||||
set +e
|
||||
output=$(cd "$repo" && node "$validator" "$base" "$immutable_base" 2>&1)
|
||||
status=$?
|
||||
set -e
|
||||
else
|
||||
set +e
|
||||
output=$(cd "$repo" && node "$validator" "$base" 2>&1)
|
||||
status=$?
|
||||
set -e
|
||||
fi
|
||||
set +e
|
||||
output=$(cd "$repo" && node "$validator" "$base" 2>&1)
|
||||
status=$?
|
||||
set -e
|
||||
if [[ $status -eq 0 ]]; then
|
||||
printf 'migration validator unexpectedly passed: %s\n' "$expected" >&2
|
||||
exit 1
|
||||
@@ -76,7 +57,7 @@ expect_fail() {
|
||||
repo=$(new_repo safe)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
cat >"$repo/apps/api/migrations/0002_expand_accounts.sql" <<'SQL'
|
||||
-- A comment mentioning DROP TABLE must not trigger the gate.
|
||||
-- DROP in a comment and string is not executable.
|
||||
ALTER TABLE accounts ADD COLUMN display_name text;
|
||||
CREATE TABLE audit_events (id bigint PRIMARY KEY, note text);
|
||||
INSERT INTO audit_events (id, note) VALUES (1, 'DROP TABLE is documentation');
|
||||
@@ -85,171 +66,37 @@ SQL
|
||||
commit_all "$repo" safe
|
||||
expect_pass "$repo" "$base"
|
||||
|
||||
repo=$(new_repo drop)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'DROP\nTABLE accounts;\n' >"$repo/apps/api/migrations/0002_drop.sql"
|
||||
commit_all "$repo" drop
|
||||
expect_fail "$repo" "$base" 'destructive DROP operation'
|
||||
|
||||
repo=$(new_repo delete)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'DELETE FROM accounts;\n' >"$repo/apps/api/migrations/0002_delete.sql"
|
||||
commit_all "$repo" delete
|
||||
expect_fail "$repo" "$base" 'DELETE FROM operation'
|
||||
|
||||
repo=$(new_repo incompatible_alter)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
cat >"$repo/apps/api/migrations/0002_alter.sql" <<'SQL'
|
||||
ALTER TABLE accounts ADD COLUMN external_id text NOT NULL;
|
||||
ALTER TABLE accounts ALTER COLUMN id TYPE numeric;
|
||||
SQL
|
||||
commit_all "$repo" alter
|
||||
expect_fail "$repo" "$base" 'non-null column addition'
|
||||
expect_fail "$repo" "$base" 'column type change'
|
||||
|
||||
repo=$(new_repo dynamic_sql)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
cat >"$repo/apps/api/migrations/0002_dynamic.sql" <<'SQL'
|
||||
DO $$
|
||||
BEGIN
|
||||
EXECUTE 'DROP ' || 'TABLE accounts';
|
||||
END
|
||||
$$;
|
||||
SQL
|
||||
commit_all "$repo" dynamic
|
||||
expect_fail "$repo" "$base" 'dynamic SQL execution'
|
||||
|
||||
repo=$(new_repo escape_string)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
cat >"$repo/apps/api/migrations/0002_escape.sql" <<'SQL'
|
||||
SELECT E'foo\'bar'; DROP TABLE accounts; --';
|
||||
SQL
|
||||
commit_all "$repo" escape
|
||||
expect_fail "$repo" "$base" 'PostgreSQL escape/unicode strings are not allowed'
|
||||
|
||||
repo=$(new_repo string_semantics_override)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'SET standard_conforming_strings = off;\n' > \
|
||||
"$repo/apps/api/migrations/0002_string_mode.sql"
|
||||
commit_all "$repo" string_mode
|
||||
expect_fail "$repo" "$base" 'SQL string semantics override'
|
||||
|
||||
repo=$(new_repo transaction_control)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
cat >"$repo/apps/api/migrations/0002_commit.sql" <<'SQL'
|
||||
ALTER TABLE accounts ADD COLUMN display_name text;
|
||||
COMMIT;
|
||||
UPDATE accounts SET display_name = 'partially committed';
|
||||
SQL
|
||||
commit_all "$repo" transaction_control
|
||||
expect_fail "$repo" "$base" 'transaction control operation'
|
||||
|
||||
repo=$(new_repo bare_end_transaction)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
cat >"$repo/apps/api/migrations/0002_end.sql" <<'SQL'
|
||||
ALTER TABLE accounts ADD COLUMN display_name text;
|
||||
END;
|
||||
UPDATE accounts SET display_name = 'partially committed';
|
||||
SQL
|
||||
commit_all "$repo" bare_end_transaction
|
||||
expect_fail "$repo" "$base" 'transaction control operation'
|
||||
|
||||
repo=$(new_repo plpgsql_block_end)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
cat >"$repo/apps/api/migrations/0002_function.sql" <<'SQL'
|
||||
CREATE FUNCTION account_count() RETURNS bigint
|
||||
LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
RETURN (SELECT count(*) FROM accounts);
|
||||
END;
|
||||
$$;
|
||||
SQL
|
||||
commit_all "$repo" plpgsql_block_end
|
||||
expect_fail "$repo" "$base" 'procedural SQL body'
|
||||
|
||||
repo=$(new_repo ordinary_string_procedural_body)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
cat >"$repo/apps/api/migrations/0002_do_string.sql" <<'SQL'
|
||||
DO LANGUAGE plpgsql 'BEGIN DROP TABLE accounts; END';
|
||||
SQL
|
||||
commit_all "$repo" ordinary_string_procedural_body
|
||||
expect_fail "$repo" "$base" 'procedural SQL body'
|
||||
|
||||
repo=$(new_repo ordinary_string_boundary)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
cat >"$repo/apps/api/migrations/0002_string_boundary.sql" <<'SQL'
|
||||
SELECT '\'; DROP TABLE accounts; --';
|
||||
SQL
|
||||
commit_all "$repo" ordinary_string
|
||||
expect_fail "$repo" "$base" 'destructive DROP operation'
|
||||
|
||||
repo=$(new_repo unicode_string)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf "SELECT U&'d\\0061ta';\n" >"$repo/apps/api/migrations/0002_unicode.sql"
|
||||
commit_all "$repo" unicode
|
||||
expect_fail "$repo" "$base" 'PostgreSQL escape/unicode strings are not allowed'
|
||||
|
||||
repo=$(new_repo merge_delete)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
cat >"$repo/apps/api/migrations/0002_merge.sql" <<'SQL'
|
||||
MERGE INTO accounts USING stale_accounts ON accounts.id = stale_accounts.id
|
||||
WHEN MATCHED THEN DELETE;
|
||||
SQL
|
||||
commit_all "$repo" merge
|
||||
expect_fail "$repo" "$base" 'MERGE operation'
|
||||
|
||||
repo=$(new_repo replace_view)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'CREATE OR REPLACE VIEW active_accounts AS SELECT id FROM accounts;\n' > \
|
||||
"$repo/apps/api/migrations/0002_replace.sql"
|
||||
commit_all "$repo" replace
|
||||
expect_fail "$repo" "$base" 'CREATE OR REPLACE operation'
|
||||
|
||||
repo=$(new_repo rename_function)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'ALTER FUNCTION calculate_total() RENAME TO calculate_total_legacy;\n' > \
|
||||
"$repo/apps/api/migrations/0002_rename.sql"
|
||||
commit_all "$repo" rename
|
||||
expect_fail "$repo" "$base" 'object or column rename'
|
||||
|
||||
repo=$(new_repo set_schema)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'ALTER TABLE accounts SET SCHEMA archive;\n' > \
|
||||
"$repo/apps/api/migrations/0002_schema.sql"
|
||||
commit_all "$repo" schema
|
||||
expect_fail "$repo" "$base" 'SET SCHEMA operation'
|
||||
|
||||
repo=$(new_repo drop_domain)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'DROP DOMAIN account_code CASCADE;\n' > \
|
||||
"$repo/apps/api/migrations/0002_drop_domain.sql"
|
||||
commit_all "$repo" drop_domain
|
||||
expect_fail "$repo" "$base" 'destructive DROP operation'
|
||||
|
||||
repo=$(new_repo immutable)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf '\n-- retroactive edit\n' >>"$repo/apps/api/migrations/0001_init.sql"
|
||||
commit_all "$repo" edit
|
||||
commit_all "$repo" immutable
|
||||
expect_fail "$repo" "$base" 'applied migrations are immutable (status M)'
|
||||
|
||||
repo=$(new_repo immutable_after_baseline)
|
||||
repo=$(new_repo removed)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'ALTER TABLE accounts ADD COLUMN nickname text;\n' > \
|
||||
"$repo/apps/api/migrations/0002_nickname.sql"
|
||||
commit_all "$repo" add
|
||||
immutable_base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf '\nUPDATE accounts SET nickname = NULL;\n' >> \
|
||||
"$repo/apps/api/migrations/0002_nickname.sql"
|
||||
commit_all "$repo" modify
|
||||
output=$(
|
||||
cd "$repo"
|
||||
node "$validator" "$base" "$immutable_base" 2>&1 || true
|
||||
)
|
||||
grep -Fq 'migration present on main is immutable (status M)' <<<"$output" || {
|
||||
printf 'stale production baseline did not preserve main migration immutability:\n%s\n' \
|
||||
"$output" >&2
|
||||
exit 1
|
||||
}
|
||||
git -C "$repo" rm -q apps/api/migrations/0001_init.sql
|
||||
git -C "$repo" commit -qm removed
|
||||
expect_fail "$repo" "$base" 'applied migrations are immutable (status D)'
|
||||
|
||||
repo=$(new_repo destructive)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'DROP TABLE accounts;\n' >"$repo/apps/api/migrations/0002_drop.sql"
|
||||
commit_all "$repo" destructive
|
||||
expect_fail "$repo" "$base" 'destructive DROP operation'
|
||||
|
||||
repo=$(new_repo incompatible)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'ALTER TABLE accounts ADD COLUMN external_id text NOT NULL;\n' > \
|
||||
"$repo/apps/api/migrations/0002_incompatible.sql"
|
||||
commit_all "$repo" incompatible
|
||||
expect_fail "$repo" "$base" 'non-null column addition'
|
||||
|
||||
repo=$(new_repo transaction)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'ALTER TABLE accounts ADD COLUMN nickname text; COMMIT;\n' > \
|
||||
"$repo/apps/api/migrations/0002_transaction.sql"
|
||||
commit_all "$repo" transaction
|
||||
expect_fail "$repo" "$base" 'transaction control operation'
|
||||
|
||||
repo=$(new_repo filename)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
@@ -263,41 +110,10 @@ printf 'SELECT 1;\n' >"$repo/apps/api/migrations/0000_late.sql"
|
||||
commit_all "$repo" ordering
|
||||
expect_fail "$repo" "$base" 'new migration name must sort after 0001_init.sql'
|
||||
|
||||
repo=$(new_repo stale_baseline)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'ALTER TABLE accounts ADD COLUMN nickname text;\n' > \
|
||||
"$repo/apps/api/migrations/0002_nickname.sql"
|
||||
commit_all "$repo" main_migration
|
||||
immutable_base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'ordinary code change\n' >"$repo/README.md"
|
||||
commit_all "$repo" code_only
|
||||
expect_pass "$repo" "$base" "$immutable_base"
|
||||
|
||||
repo=$(new_repo baseline_advance)
|
||||
production_base=$(git -C "$repo" rev-parse HEAD)
|
||||
mkdir -p "$repo/deploy/ci"
|
||||
printf '%s\n' "$production_base" >"$repo/deploy/ci/production-migration-base"
|
||||
commit_all "$repo" baseline_file
|
||||
immutable_base=$(git -C "$repo" rev-parse HEAD)
|
||||
git -C "$repo" switch -q -c release-baseline-update "$immutable_base"
|
||||
printf '%s\n' "$immutable_base" >"$repo/deploy/ci/production-migration-base"
|
||||
commit_all "$repo" valid_baseline_advance
|
||||
expect_pass "$repo" "$immutable_base" "$immutable_base"
|
||||
|
||||
git -C "$repo" switch -q -c malicious-baseline "$immutable_base"
|
||||
printf 'DROP TABLE accounts;\n' >"$repo/apps/api/migrations/0002_drop.sql"
|
||||
commit_all "$repo" hidden_destructive_migration
|
||||
hidden_sha=$(git -C "$repo" rev-parse HEAD)
|
||||
printf '%s\n' "$hidden_sha" >"$repo/deploy/ci/production-migration-base"
|
||||
commit_all "$repo" hide_migration_in_baseline
|
||||
expect_fail "$repo" "$hidden_sha" \
|
||||
'production migration baseline must belong to the immutable event base history' \
|
||||
"$immutable_base"
|
||||
|
||||
repo=$(new_repo nested)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
mkdir -p "$repo/apps/api/migrations/archive"
|
||||
printf 'SELECT 1;\n' >"$repo/apps/api/migrations/archive/9999_hidden.sql"
|
||||
printf 'SELECT 1;\n' >"$repo/apps/api/migrations/archive/0002_hidden.sql"
|
||||
commit_all "$repo" nested
|
||||
expect_fail "$repo" "$base" 'new migration must use NNNN_lowercase_name.sql'
|
||||
|
||||
@@ -308,22 +124,13 @@ ln -s ../../../outside.sql "$repo/apps/api/migrations/0002_symlink.sql"
|
||||
commit_all "$repo" symlink
|
||||
expect_fail "$repo" "$base" 'new migration must be a mode 100644 regular file'
|
||||
|
||||
repo=$(new_repo baseline_symlink)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf '%s\n' "$base" >"$repo/baseline-target"
|
||||
rm "$repo/deploy/ci/production-migration-base"
|
||||
ln -s ../../baseline-target "$repo/deploy/ci/production-migration-base"
|
||||
commit_all "$repo" baseline_symlink
|
||||
expect_fail "$repo" "$base" \
|
||||
'production migration baseline must be a mode 100644 regular file'
|
||||
|
||||
repo=$(new_repo ancestry)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
git -C "$repo" switch -q --orphan unrelated
|
||||
printf 'unrelated\n' >"$repo/unrelated"
|
||||
git -C "$repo" add unrelated
|
||||
git -C "$repo" commit -qm unrelated
|
||||
expect_fail "$repo" "$base" 'production migration baseline is not an ancestor of HEAD'
|
||||
expect_fail "$repo" "$base" 'release base is not an ancestor of HEAD'
|
||||
|
||||
repo=$(new_repo invalid_sha)
|
||||
expect_fail "$repo" deadbeef 'must be a full lowercase commit SHA'
|
||||
|
||||
@@ -1,365 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck disable=SC2016 # Assertions intentionally match literal workflow variables.
|
||||
set -euo pipefail
|
||||
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
workflow=$root/.gitea/workflows/ci.yml
|
||||
release_workflow=$root/.gitea/workflows/release-ci.yml
|
||||
runner_service=$root/deploy/ci/easyai-gateway-ci-v2-runner.service
|
||||
runner_config=$root/deploy/ci/act-runner-v2-config.yaml
|
||||
provision=$root/scripts/provision-ci-runner.sh
|
||||
build_images=$root/scripts/ci-build-images.sh
|
||||
validate_semver=$root/scripts/ci-validate-semver.sh
|
||||
validate_migrations=$root/scripts/ci-validate-migrations.mjs
|
||||
build_images_test=$root/tests/ci/ci-build-images-test.sh
|
||||
migrations_test=$root/tests/ci/migrations-test.sh
|
||||
migration_base=$root/deploy/ci/production-migration-base
|
||||
adr=$root/docs/decisions/001-production-cicd.md
|
||||
runbook=$root/docs/runbooks/production-ci-cd.md
|
||||
migrator=$root/apps/api/cmd/migrate/main.go
|
||||
|
||||
assert_bounded_go_verification() {
|
||||
local quality_workflow=$1
|
||||
|
||||
awk '
|
||||
/^[[:space:]]+GOFLAGS:/ { all_goflags++ }
|
||||
/^[[:space:]]+GOMAXPROCS:/ { all_gomaxprocs++ }
|
||||
/^ - name: Verify Go code$/ {
|
||||
target_steps++
|
||||
in_target=1
|
||||
in_env=0
|
||||
next
|
||||
}
|
||||
/^ - / {
|
||||
in_target=0
|
||||
in_env=0
|
||||
}
|
||||
in_target && /^ env:$/ {
|
||||
env_blocks++
|
||||
in_env=1
|
||||
next
|
||||
}
|
||||
in_target && in_env && /^ GOFLAGS: "-p=1"$/ { target_goflags++ }
|
||||
in_target && in_env && /^ GOMAXPROCS: "1"$/ { target_gomaxprocs++ }
|
||||
in_target && in_env && /^ [^ ]/ { in_env=0 }
|
||||
END {
|
||||
exit !(target_steps == 1 && env_blocks == 1 &&
|
||||
target_goflags == 1 && target_gomaxprocs == 1 &&
|
||||
all_goflags == 1 && all_gomaxprocs == 1)
|
||||
}
|
||||
' "$quality_workflow" || {
|
||||
printf '%s must limit Go parallelism exactly once in the Verify Go code step\n' \
|
||||
"$quality_workflow" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
for file in "$workflow" "$release_workflow" "$runner_service" "$runner_config" "$provision" "$build_images" "$validate_semver" "$validate_migrations" "$build_images_test" "$migrations_test" "$migration_base" "$adr" "$runbook" "$migrator"; do
|
||||
[[ -f $file ]] || {
|
||||
printf 'missing CI/CD file: %s\n' "$file" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
if [[ $(wc -l <"$migration_base") -ne 1 ]]; then
|
||||
echo 'production migration baseline must contain exactly one line' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'SET standard_conforming_strings = on' "$migrator" || {
|
||||
echo 'database migrator does not pin standard SQL string semantics' >&2
|
||||
exit 1
|
||||
}
|
||||
if ! grep -Eq '^[0-9a-f]{40}$' "$migration_base"; then
|
||||
echo 'production migration baseline must be a full lowercase SHA' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
grep -q '^name: ci$' "$workflow"
|
||||
grep -q '^ push:$' "$workflow"
|
||||
grep -q '^ branches: \[main\]$' "$workflow"
|
||||
grep -q '^ pull_request:$' "$workflow"
|
||||
grep -q '^ verify:$' "$workflow"
|
||||
grep -Fq 'git checkout --detach "$CI_SHA"' "$workflow"
|
||||
grep -Fq 'test "$(git rev-parse HEAD)" = "$CI_SHA"' "$workflow"
|
||||
if grep -Fq 'git checkout --detach FETCH_HEAD' "$workflow"; then
|
||||
echo 'main/PR CI must not checkout mutable FETCH_HEAD after comparison fetches' >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -Eq "^[[:space:]]+tags:|Verify release tag ancestry" "$workflow" || \
|
||||
grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$workflow"; then
|
||||
echo 'main/PR CI must not share the release-tag context' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
grep -q '^name: release-ci$' "$release_workflow"
|
||||
grep -q '^ push:$' "$release_workflow"
|
||||
grep -q "^ tags: \['v\*'\]$" "$release_workflow"
|
||||
grep -q '^ verify-tag:$' "$release_workflow"
|
||||
grep -q '^ - name: Verify release tag ancestry$' "$release_workflow"
|
||||
grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$release_workflow"
|
||||
grep -Fq 'git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main' "$release_workflow"
|
||||
if grep -Eq 'branches:|pull_request:' "$release_workflow"; then
|
||||
echo 'release CI must be a tag-only workflow/context' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for quality_workflow in "$workflow" "$release_workflow"; do
|
||||
grep -q '^ runs-on: easyai-gateway-ci-unprivileged-v2$' "$quality_workflow"
|
||||
grep -q '^ - name: Checkout without external Actions$' "$quality_workflow"
|
||||
if grep -q -- '--depth=' "$quality_workflow"; then
|
||||
echo 'quality verification must not use a shallow checkout' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'test ! -f .git/shallow' "$quality_workflow"
|
||||
grep -Fq './tests/ci/ci-build-images-test.sh' "$quality_workflow"
|
||||
grep -Fq './tests/ci/migrations-test.sh' "$quality_workflow"
|
||||
grep -Fq './tests/ci/pipeline-test.sh' "$quality_workflow"
|
||||
grep -Fq './tests/ci/semver-test.sh' "$quality_workflow"
|
||||
grep -Fq 'docker-compose version' "$quality_workflow"
|
||||
grep -Fq 'docker-compose -f docker-compose.yml config --quiet' "$quality_workflow"
|
||||
grep -Fq 'services:' "$quality_workflow"
|
||||
grep -Fq 'image: docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777' "$quality_workflow"
|
||||
grep -Fq 'POSTGRES_HOST_AUTH_METHOD: trust' "$quality_workflow"
|
||||
grep -Fq 'AI_GATEWAY_TEST_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable' "$quality_workflow"
|
||||
grep -Fq 'run: go run ./cmd/migrate' "$quality_workflow"
|
||||
if grep -Fq 'docker compose' "$quality_workflow"; then
|
||||
echo 'job container must use the read-only standalone Compose binary' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for forbidden in \
|
||||
'sudo' \
|
||||
'/usr/local/sbin/easyai-ai-gateway-release' \
|
||||
'docker version' \
|
||||
'docker buildx' \
|
||||
'./scripts/ci-build-images.sh --keep-images' \
|
||||
'Deploy verified release to Production'; do
|
||||
if grep -Fq "$forbidden" "$quality_workflow"; then
|
||||
printf 'unprivileged workflow contains forbidden capability: %s\n' "$forbidden" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if grep -Eq '^[[:space:]]+\./scripts/ci-build-images\.sh([[:space:]]|$)' "$quality_workflow"; then
|
||||
echo 'unprivileged workflow must not build container images' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for gate in \
|
||||
'scripts/ci-validate-migrations.mjs' \
|
||||
'gofmt -l' \
|
||||
'go vet ./...' \
|
||||
'go test ./...' \
|
||||
'pnpm install --frozen-lockfile' \
|
||||
'pnpm lint' \
|
||||
'pnpm test' \
|
||||
'pnpm build' \
|
||||
'pnpm audit --audit-level high' \
|
||||
'docker-compose -f docker-compose.yml config --quiet' \
|
||||
'govulncheck ./...' \
|
||||
'trivy fs'; do
|
||||
grep -Fq "$gate" "$quality_workflow" || {
|
||||
printf '%s is missing quality gate: %s\n' "$quality_workflow" "$gate" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
grep -Fq 'TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2' \
|
||||
"$quality_workflow" || {
|
||||
printf '%s does not pin the reachable Trivy vulnerability DB mirror\n' \
|
||||
"$quality_workflow" >&2
|
||||
exit 1
|
||||
}
|
||||
assert_bounded_go_verification "$quality_workflow"
|
||||
grep -Fq 'production_base=$(cat deploy/ci/production-migration-base)' \
|
||||
"$quality_workflow" || {
|
||||
printf '%s does not read the versioned production migration baseline\n' \
|
||||
"$quality_workflow" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
if [[ $(sed -n '/^ - name: Verify Go formatting$/,$p' "$workflow") != \
|
||||
"$(sed -n '/^ - name: Verify Go formatting$/,$p' "$release_workflow")" ]]; then
|
||||
echo 'main/PR and release-tag workflows must have identical quality gates' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
grep -Fq 'CI_EVENT_BEFORE: ${{ github.event.before }}' "$workflow"
|
||||
grep -Fq 'CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}' "$workflow"
|
||||
grep -Fq 'git merge-base --is-ancestor "$immutable_base" HEAD' "$workflow"
|
||||
grep -Fq '"$production_base" "$immutable_base"' "$workflow"
|
||||
grep -Fq 'node ./scripts/ci-validate-migrations.mjs "$production_base"' \
|
||||
"$release_workflow"
|
||||
|
||||
runner_image='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a'
|
||||
job_image='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7'
|
||||
|
||||
grep -q '^User=root$' "$runner_service"
|
||||
grep -q '^Group=root$' "$runner_service"
|
||||
grep -q '^NoNewPrivileges=true$' "$runner_service"
|
||||
grep -q '^ProtectSystem=strict$' "$runner_service"
|
||||
grep -q '^ProtectHome=true$' "$runner_service"
|
||||
grep -q '^CapabilityBoundingSet=$' "$runner_service"
|
||||
grep -Fq 'Requires=docker.service' "$runner_service"
|
||||
grep -Fq -- "--pull=never" "$runner_service"
|
||||
grep -Fq -- "--privileged" "$runner_service"
|
||||
grep -Fq -- '--pids-limit 1024' "$runner_service"
|
||||
grep -Fq -- '--cpus 2' "$runner_service"
|
||||
grep -Fq -- '--memory 2g' "$runner_service"
|
||||
grep -Fq -- '--memory-swap 3g' "$runner_service"
|
||||
grep -Fq -- '--env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns' "$runner_service"
|
||||
grep -Fq 'EnvironmentFile=-/etc/easyai-gateway-ci-v2/runner.env' "$runner_service"
|
||||
grep -Fq '$RUNNER_SECURITY_OPTIONS' "$runner_service"
|
||||
grep -Fq -- "--volume easyai-gateway-ci-v2-data:/data" "$runner_service"
|
||||
grep -Fq -- "--volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro" "$runner_service"
|
||||
grep -Fq -- "--volume /etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro" "$runner_service"
|
||||
grep -Fq -- "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro" "$runner_service"
|
||||
grep -Fq 'Environment=RUNNER_IMAGE_REF=' "$runner_service"
|
||||
grep -Fq '$RUNNER_IMAGE_REF' "$runner_service"
|
||||
if grep -Eq 'GITEA_RUNNER_REGISTRATION_TOKEN|/var/run/docker\.sock|/run/docker\.sock' "$runner_service"; then
|
||||
echo 'persistent runner service exposes a registration token or host Docker socket' >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -Fq 'apparmor=rootlesskit' "$runner_service"; then
|
||||
echo 'runner service must not hard-code an AppArmor profile that may be absent' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
grep -Fq "easyai-gateway-ci-unprivileged-v2:docker://$job_image" "$runner_config"
|
||||
if grep -Fq ':host' "$runner_config"; then
|
||||
echo 'CI runner must never use the host executor' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q '^ capacity: 1$' "$runner_config"
|
||||
grep -q '^ file: /data/.runner$' "$runner_config"
|
||||
grep -q '^ docker_host: "-"$' "$runner_config"
|
||||
grep -q '^ privileged: false$' "$runner_config"
|
||||
grep -q '^ require_docker: true$' "$runner_config"
|
||||
grep -q '^ bind_workdir: false$' "$runner_config"
|
||||
grep -Fq ' options: "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro"' "$runner_config"
|
||||
grep -q '^ valid_volumes:$' "$runner_config"
|
||||
grep -q '^ - "/opt/easyai-gateway-ci"$' "$runner_config"
|
||||
if awk '
|
||||
/^ valid_volumes:$/ { in_volumes=1; next }
|
||||
in_volumes && /^ - / { count++; next }
|
||||
in_volumes && /^ [A-Za-z_]+:/ { in_volumes=0 }
|
||||
END { exit count == 1 ? 0 : 1 }
|
||||
' "$runner_config"; then
|
||||
:
|
||||
else
|
||||
echo 'runner must allow exactly one workflow volume source' >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -Eq '/var/run/docker\.sock|/run/docker\.sock' "$runner_config"; then
|
||||
echo 'job configuration must not expose a Docker socket' >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -Eq 'apparmor=unconfined|sysctl .*(apparmor_restrict_unprivileged_userns)' "$runner_service" "$runner_config" "$provision"; then
|
||||
echo 'runner must not disable AppArmor isolation or its user-namespace restriction' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
grep -q '^GITEA_RUNNER_VERSION=2.0.0$' "$provision"
|
||||
grep -q '^GO_VERSION=1.26.5$' "$provision"
|
||||
grep -q '^NODE_VERSION=24.16.0$' "$provision"
|
||||
grep -q '^TRIVY_VERSION=0.70.0$' "$provision"
|
||||
grep -q '^PREFIX=/opt/easyai-gateway-ci$' "$provision"
|
||||
grep -q '^RUNNER_VOLUME=easyai-gateway-ci-v2-data$' "$provision"
|
||||
grep -q '^SHELLCHECK_VERSION=0.11.0$' "$provision"
|
||||
grep -q '^SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198$' "$provision"
|
||||
grep -q '^COMPOSE_VERSION=5.3.1$' "$provision"
|
||||
grep -q '^COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959$' "$provision"
|
||||
grep -Fq 'RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-ci-unprivileged-v2}' "$provision"
|
||||
grep -Fq "RUNNER_IMAGE='$runner_image'" "$provision"
|
||||
grep -Fq 'RUNNER_IMAGE_SOURCE=${CI_RUNNER_IMAGE_SOURCE:-$RUNNER_IMAGE}' "$provision"
|
||||
grep -Fq 'runner_manifest_digest=${RUNNER_IMAGE##*@}' "$provision"
|
||||
grep -Fq 'CI_RUNNER_IMAGE_SOURCE must use the pinned runner manifest digest' "$provision"
|
||||
grep -Fq 'runner_runtime_image=$(docker image inspect' "$provision"
|
||||
grep -Fq 'RUNNER_IMAGE_REF="%s"' "$provision"
|
||||
grep -Fq "JOB_IMAGE='$job_image'" "$provision"
|
||||
grep -Fq 'RUNNER_LABEL="easyai-gateway-ci-unprivileged-v2:docker://$JOB_IMAGE"' "$provision"
|
||||
grep -Fq 'docker volume create "$RUNNER_VOLUME"' "$provision"
|
||||
grep -Fq '"data-root":"/data/docker"' "$provision"
|
||||
grep -Fq -- '--security-opt apparmor=rootlesskit' "$provision"
|
||||
grep -Fq 'RUNNER_SECURITY_OPTIONS="--security-opt=apparmor=rootlesskit"' "$provision"
|
||||
grep -Fq 'RUNNER_SECURITY_OPTIONS=' "$provision"
|
||||
grep -Fq '/proc/sys/kernel/apparmor_restrict_unprivileged_userns' "$provision"
|
||||
grep -Fq -- '--entrypoint dockerd-entrypoint.sh "$runner_runtime_image"' "$provision"
|
||||
grep -Fq '/etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro' "$provision"
|
||||
grep -Fq 'CI_NESTED_DOCKER_REGISTRY_MIRROR must be an HTTPS registry origin' "$provision"
|
||||
grep -Fq 'PROBE_CONTAINER="easyai-gateway-ci-v2-probe-$$"' "$provision"
|
||||
grep -Fq -- '--pids-limit "$RUNNER_PIDS_LIMIT"' "$provision"
|
||||
grep -Fq -- '--cpus "$RUNNER_CPUS"' "$provision"
|
||||
grep -Fq -- '--memory "$RUNNER_MEMORY"' "$provision"
|
||||
grep -Fq -- '--memory-swap "$RUNNER_MEMORY_SWAP"' "$provision"
|
||||
grep -Fq "'2147483648:3221225472:2000000000:1024'" "$provision"
|
||||
grep -Fq -- '--entrypoint /usr/local/bin/gitea-runner' "$provision"
|
||||
grep -Fq 'shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz' "$provision"
|
||||
grep -Fq 'docker-compose-linux-x86_64' "$provision"
|
||||
grep -Fq 'install -m 0755 "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "$PREFIX/bin/shellcheck"' "$provision"
|
||||
grep -Fq 'install -m 0755 "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" "$PREFIX/bin/docker-compose"' "$provision"
|
||||
grep -Fq 'PATH="$PREFIX/toolchains/node/bin:$PATH"' "$provision"
|
||||
grep -Fq 'gpasswd -d easyai-gateway-ci-v2 docker' "$provision"
|
||||
grep -Fq 'gpasswd -d easyai-gateway-ci-v2 sudo' "$provision"
|
||||
if grep -Fq 'usermod -aG docker' "$provision"; then
|
||||
echo 'unprivileged runner must never join the Docker group' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'systemctl disable --now easyai-gateway-act-runner.service' "$provision"
|
||||
grep -Fq 'systemctl disable --now easyai-gateway-ci-v2-runner.service' "$provision"
|
||||
grep -Fq 'RUNNER_REGISTRATION_TOKEN is required for first registration' "$provision"
|
||||
grep -Fq 'printf '\''%s\n%s\n%s\n'\'' "$GITEA_INSTANCE_URL" "$RUNNER_REGISTRATION_TOKEN" "$RUNNER_NAME"' "$provision"
|
||||
if grep -Fq -- '--token "$RUNNER_REGISTRATION_TOKEN"' "$provision"; then
|
||||
echo 'registration token must not be exposed in a host-visible process argv' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq "grep -Fq 'name=rootless'" "$provision"
|
||||
grep -Fq "grep -Fq 'declare successfully'" "$provision"
|
||||
grep -Fq 'docker buildx prune --builder "$LEGACY_BUILDER" --force --max-used-space 512mb' "$provision"
|
||||
grep -Fq 'require_free_space "$MIN_FREE_GIB" "runner image pull"' "$provision"
|
||||
grep -Fq 'require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed CI runner installation"' "$provision"
|
||||
grep -Fq 'CI_RUNNER_MIN_FREE_GIB cannot be lower than 8' "$provision"
|
||||
grep -Fq 'CI_RUNNER_MIN_POST_INSTALL_FREE_GIB cannot be lower than 4' "$provision"
|
||||
grep -Fq 'docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"' "$provision"
|
||||
grep -Fq 'docker exec "$PROBE_CONTAINER" docker run --rm --pull=never' "$provision"
|
||||
grep -Fq -- '--volume "$RUNNER_VOLUME:/data"' "$provision"
|
||||
grep -Fq -- '--volume "$PREFIX:$PREFIX:ro"' "$provision"
|
||||
probe_pull_line=$(grep -nF 'docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"' "$provision" | cut -d: -f1)
|
||||
registration_line=$(grep -nF 'if ! runner_is_registered; then' "$provision" | cut -d: -f1)
|
||||
[[ -n $probe_pull_line && -n $registration_line && $probe_pull_line -lt $registration_line ]] || {
|
||||
echo 'pinned job image and isolation probes must complete before runner registration' >&2
|
||||
exit 1
|
||||
}
|
||||
grep -Fq 'nested --pid=host can see the outer runner PID namespace' "$provision"
|
||||
grep -Fq 'chown -R root:root "$PREFIX"' "$provision"
|
||||
grep -Fq 'chmod -R go-w "$PREFIX"' "$provision"
|
||||
if grep -Eq 'SHELLCHECK_BIN|COMPOSE_PLUGIN|command -v shellcheck|docker compose version' "$provision"; then
|
||||
echo 'runner provisioning must use checksum-pinned static CI tools, not host copies' >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -Eq 'docker (system|builder|image|volume) prune' "$provision"; then
|
||||
echo 'runner provisioning must not globally prune shared Docker state' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
grep -Fq 'root-owned dispatcher' "$adr"
|
||||
grep -Fq 'release-ci / verify-tag (push)' "$adr"
|
||||
grep -Fq '来自 Fork 的 PR 必须先由维护者检查工作流变更并在 Gitea 中批准' "$adr"
|
||||
grep -Fq 'rootless DinD 不是 VM 沙箱' "$runbook"
|
||||
grep -Fq '同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)`' "$runbook"
|
||||
grep -Fq '不得执行 Tag checkout 中的' "$runbook"
|
||||
|
||||
grep -Fq 'docker buildx build --builder "$builder"' "$build_images"
|
||||
grep -q -- '--target api' "$build_images"
|
||||
grep -q -- '--target web' "$build_images"
|
||||
grep -Fq 'docker buildx prune --builder "$builder" --force --max-used-space "$cache_limit"' "$build_images"
|
||||
grep -Fq 'trivy image' "$build_images"
|
||||
grep -Fq 'docker image rm "$api_image" "$web_image"' "$build_images"
|
||||
grep -Fq 'AI_GATEWAY_BUILDX_CACHE_LIMIT has an invalid format' "$build_images"
|
||||
|
||||
if grep -En '(password|secret|token):[[:space:]]+[^$<{]' "$workflow" "$release_workflow"; then
|
||||
echo 'workflow contains a literal credential' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo 'gateway_ci_pipeline_tests=PASS'
|
||||
@@ -1,52 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
validator=$root/scripts/ci-validate-semver.sh
|
||||
|
||||
valid_tags=(
|
||||
v0.0.0
|
||||
v1.2.3
|
||||
v1.2.3-alpha
|
||||
v1.2.3-alpha.1
|
||||
v1.2.3-0.3.7
|
||||
v1.2.3-01a
|
||||
v1.2.3-alpha-beta
|
||||
v1.2.3--
|
||||
v1.2.3-x.7.z.92+build.5
|
||||
v1.2.3+build.11.e0f985a
|
||||
v1.2.3+001
|
||||
)
|
||||
invalid_tags=(
|
||||
1.2.3
|
||||
v01.2.3
|
||||
v1.02.3
|
||||
v1.2.03
|
||||
v1.2.3.foo
|
||||
v1.2.3.
|
||||
v1.2.3-.
|
||||
v1.2.3-alpha.
|
||||
v1.2.3-alpha..1
|
||||
v1.2.3-01
|
||||
v1.2.3-00.1
|
||||
v1.2.3+
|
||||
v1.2.3+build.
|
||||
v1.2.3+build..1
|
||||
v1.2.3+build+1
|
||||
'v1.2.3 '
|
||||
v1.2.3-alpha_1
|
||||
v1.2
|
||||
)
|
||||
|
||||
for tag in "${valid_tags[@]}"; do
|
||||
"$validator" "$tag"
|
||||
done
|
||||
|
||||
for tag in "${invalid_tags[@]}"; do
|
||||
if "$validator" "$tag" >/dev/null 2>&1; then
|
||||
printf 'invalid SemVer tag was accepted: %s\n' "$tag" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo 'semver_tests=PASS'
|
||||
Executable
+261
@@ -0,0 +1,261 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
|
||||
new_repo() {
|
||||
local name=$1
|
||||
local repo=$tmp/$name
|
||||
mkdir -p "$repo/apps/api/migrations" "$repo/apps/web/src" "$repo/packages/contracts/src" "$repo/docs"
|
||||
git -C "$repo" init -q
|
||||
git -C "$repo" config user.name 'Manual Release Test'
|
||||
git -C "$repo" config user.email 'manual-release-test@example.invalid'
|
||||
printf 'package api\n' >"$repo/apps/api/main.go"
|
||||
printf 'SELECT 1;\n' >"$repo/apps/api/migrations/0001_init.sql"
|
||||
printf 'web\n' >"$repo/apps/web/src/app.ts"
|
||||
printf 'contracts\n' >"$repo/packages/contracts/src/index.ts"
|
||||
# shellcheck disable=SC2016
|
||||
printf 'ARG API_RUNTIME_IMAGE=scratch\nARG WEB_RUNTIME_IMAGE=scratch\nFROM ${API_RUNTIME_IMAGE} AS api\nCOPY apps/api /app\nFROM ${WEB_RUNTIME_IMAGE} AS web\nCOPY apps/web /web\n' >"$repo/Dockerfile"
|
||||
printf 'docs\n' >"$repo/docs/readme.md"
|
||||
git -C "$repo" add -A
|
||||
git -C "$repo" commit -qm baseline
|
||||
printf '%s\n' "$repo"
|
||||
}
|
||||
|
||||
classify() {
|
||||
local repo=$1
|
||||
local base=$2
|
||||
(cd "$repo" && node "$root/scripts/release-components.mjs" "$base")
|
||||
}
|
||||
|
||||
repo=$(new_repo api)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'package api\n// changed\n' >"$repo/apps/api/main.go"
|
||||
git -C "$repo" add -A && git -C "$repo" commit -qm api
|
||||
result=$(classify "$repo" "$base")
|
||||
node -e 'const value=JSON.parse(process.argv[1]); if(value.components!=="api"||value.migrationsChanged)process.exit(1)' "$result"
|
||||
|
||||
repo=$(new_repo web)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'web changed\n' >"$repo/apps/web/src/app.ts"
|
||||
git -C "$repo" add -A && git -C "$repo" commit -qm web
|
||||
result=$(classify "$repo" "$base")
|
||||
node -e 'const value=JSON.parse(process.argv[1]); if(value.components!=="web")process.exit(1)' "$result"
|
||||
|
||||
repo=$(new_repo all)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'node_modules\n' >"$repo/.dockerignore"
|
||||
git -C "$repo" add -A && git -C "$repo" commit -qm all
|
||||
result=$(classify "$repo" "$base")
|
||||
node -e 'const value=JSON.parse(process.argv[1]); if(value.components!=="all")process.exit(1)' "$result"
|
||||
|
||||
repo=$(new_repo api-docker-stage)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
# shellcheck disable=SC2016
|
||||
printf 'ARG API_RUNTIME_IMAGE=scratch\nARG WEB_RUNTIME_IMAGE=scratch\nFROM ${API_RUNTIME_IMAGE} AS api\nCOPY apps/api /app\nRUN echo api\nFROM ${WEB_RUNTIME_IMAGE} AS web\nCOPY apps/web /web\n' >"$repo/Dockerfile"
|
||||
git -C "$repo" add -A && git -C "$repo" commit -qm api-docker-stage
|
||||
result=$(classify "$repo" "$base")
|
||||
node -e 'const value=JSON.parse(process.argv[1]); if(value.components!=="api")process.exit(1)' "$result"
|
||||
|
||||
repo=$(new_repo web-docker-stage)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf '\nRUN echo web\n' >>"$repo/Dockerfile"
|
||||
git -C "$repo" add -A && git -C "$repo" commit -qm web-docker-stage
|
||||
result=$(classify "$repo" "$base")
|
||||
node -e 'const value=JSON.parse(process.argv[1]); if(value.components!=="web")process.exit(1)' "$result"
|
||||
|
||||
repo=$(new_repo migration)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'ALTER TABLE demo ADD COLUMN note text;\n' >"$repo/apps/api/migrations/0002_note.sql"
|
||||
git -C "$repo" add -A && git -C "$repo" commit -qm migration
|
||||
result=$(classify "$repo" "$base")
|
||||
node -e 'const value=JSON.parse(process.argv[1]); if(value.components!=="api"||!value.migrationsChanged)process.exit(1)' "$result"
|
||||
|
||||
repo=$(new_repo docs)
|
||||
base=$(git -C "$repo" rev-parse HEAD)
|
||||
printf 'docs changed\n' >"$repo/docs/readme.md"
|
||||
git -C "$repo" add -A && git -C "$repo" commit -qm docs
|
||||
result=$(classify "$repo" "$base")
|
||||
node -e 'const value=JSON.parse(process.argv[1]); if(value.components!=="none")process.exit(1)' "$result"
|
||||
|
||||
bare=$tmp/origin.git
|
||||
git init -q --bare "$bare"
|
||||
repo=$(new_repo preflight)
|
||||
git -C "$repo" branch -M main
|
||||
git -C "$repo" remote add origin "$bare"
|
||||
git -C "$repo" push -q -u origin main
|
||||
(cd "$repo" && node "$root/scripts/release-preflight.mjs" >/dev/null)
|
||||
printf 'dirty\n' >>"$repo/docs/readme.md"
|
||||
if (cd "$repo" && node "$root/scripts/release-preflight.mjs" >/dev/null 2>&1); then
|
||||
echo 'dirty worktree passed release preflight' >&2
|
||||
exit 1
|
||||
fi
|
||||
git -C "$repo" restore docs/readme.md
|
||||
printf 'untracked\n' >"$repo/untracked"
|
||||
if (cd "$repo" && node "$root/scripts/release-preflight.mjs" >/dev/null 2>&1); then
|
||||
echo 'untracked file passed release preflight' >&2
|
||||
exit 1
|
||||
fi
|
||||
rm "$repo/untracked"
|
||||
git -C "$repo" switch -q -c feature
|
||||
printf 'feature\n' >>"$repo/docs/readme.md"
|
||||
git -C "$repo" add -A && git -C "$repo" commit -qm feature
|
||||
if (cd "$repo" && node "$root/scripts/release-preflight.mjs" >/dev/null 2>&1); then
|
||||
echo 'non-main commit passed release preflight' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
manifest=$tmp/release.json
|
||||
recorded=$tmp/recorded.json
|
||||
source_sha=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
base_sha=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
|
||||
api_image=registry.example.com/easyai/ai-gateway@sha256:$(printf '1%.0s' {1..64})
|
||||
web_image=registry.example.com/easyai/ai-gateway-web@sha256:$(printf '2%.0s' {1..64})
|
||||
RELEASE_SOURCE_SHA=$source_sha \
|
||||
RELEASE_BASE_SHA=$base_sha \
|
||||
RELEASE_COMPONENTS=api \
|
||||
RELEASE_MIGRATIONS_CHANGED=false \
|
||||
RELEASE_API_IMAGE=$api_image \
|
||||
RELEASE_WEB_IMAGE=$web_image \
|
||||
RELEASE_BUILD_COMPLETED_AT=2026-07-22T00:00:00Z \
|
||||
RELEASE_SMOKE_COMPLETED_AT=2026-07-22T00:00:00Z \
|
||||
node "$root/scripts/release-manifest.mjs" create "$manifest" >/dev/null
|
||||
node "$root/scripts/release-manifest.mjs" validate "$manifest" >/dev/null
|
||||
[[ $(node "$root/scripts/release-manifest.mjs" get "$manifest" images.api) == "$api_image" ]]
|
||||
RELEASE_DEPLOYED_AT=2026-07-22T00:01:00Z \
|
||||
RELEASE_DEPLOY_DURATION_SECONDS=17 \
|
||||
RELEASE_DEPLOY_ACTION=deploy \
|
||||
node "$root/scripts/release-manifest.mjs" record-deployment "$manifest" "$recorded" >/dev/null
|
||||
node "$root/scripts/release-manifest.mjs" validate "$recorded" >/dev/null
|
||||
|
||||
sensitive=$tmp/sensitive.json
|
||||
node -e 'const fs=require("fs"); const value=JSON.parse(fs.readFileSync(process.argv[1])); value.accessToken="forbidden"; fs.writeFileSync(process.argv[2], JSON.stringify(value))' "$manifest" "$sensitive"
|
||||
if node "$root/scripts/release-manifest.mjs" validate "$sensitive" >/dev/null 2>&1; then
|
||||
echo 'sensitive manifest field passed validation' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tampered=$tmp/tampered.json
|
||||
node -e 'const fs=require("fs"); const value=JSON.parse(fs.readFileSync(process.argv[1])); value.images.api=value.images.api.replace(/^registry[.]/, "mirror."); fs.writeFileSync(process.argv[2], JSON.stringify(value))' "$manifest" "$tampered"
|
||||
if node "$root/scripts/release-manifest.mjs" validate "$tampered" >/dev/null 2>&1; then
|
||||
echo 'tampered manifest content passed validation' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
incomplete_digest=$tmp/incomplete-digest.json
|
||||
node -e 'const fs=require("fs"); const value=JSON.parse(fs.readFileSync(process.argv[1])); value.images.web="registry.example.com/easyai/ai-gateway-web:tag"; fs.writeFileSync(process.argv[2], JSON.stringify(value))' "$manifest" "$incomplete_digest"
|
||||
if node "$root/scripts/release-manifest.mjs" validate "$incomplete_digest" >/dev/null 2>&1; then
|
||||
echo 'tag-based image passed manifest validation' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if grep -Eqi 'password|secret|token|credential|private.?key' "$manifest"; then
|
||||
echo 'generated manifest contains a sensitive field name' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
publisher_repo=$tmp/publisher
|
||||
mkdir -p "$publisher_repo/scripts" "$publisher_repo/apps/api" "$publisher_repo/fake-bin"
|
||||
cp "$root/scripts/publish-release-images.sh" \
|
||||
"$root/scripts/release-preflight.mjs" \
|
||||
"$root/scripts/release-manifest.mjs" \
|
||||
"$root/scripts/release-components.mjs" \
|
||||
"$root/scripts/ci-validate-migrations.mjs" \
|
||||
"$publisher_repo/scripts/"
|
||||
printf 'FROM scratch\n' >"$publisher_repo/Dockerfile"
|
||||
printf 'module example.invalid/release-test\n\ngo 1.26\n' >"$publisher_repo/apps/api/go.mod"
|
||||
cat >"$publisher_repo/fake-bin/docker" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
case "$*" in
|
||||
info|'buildx version'|'compose version') exit 0 ;;
|
||||
'buildx imagetools inspect '*)
|
||||
if [[ ${FAKE_DOCKER_MODE:-} == registry ]]; then
|
||||
echo 'unauthorized: authentication required' >&2
|
||||
else
|
||||
echo 'manifest unknown: not found' >&2
|
||||
fi
|
||||
exit 1
|
||||
;;
|
||||
'run --rm --platform linux/amd64 '*) exit 1 ;;
|
||||
esac
|
||||
echo "unexpected fake docker call: $*" >&2
|
||||
exit 1
|
||||
EOF
|
||||
cat >"$publisher_repo/fake-bin/ssh" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
exit 1
|
||||
EOF
|
||||
chmod +x "$publisher_repo/scripts/"* "$publisher_repo/fake-bin/"*
|
||||
git -C "$publisher_repo" init -q
|
||||
git -C "$publisher_repo" config user.name 'Manual Release Test'
|
||||
git -C "$publisher_repo" config user.email 'manual-release-test@example.invalid'
|
||||
git -C "$publisher_repo" add -A
|
||||
git -C "$publisher_repo" commit -qm baseline
|
||||
git -C "$publisher_repo" branch -M main
|
||||
publisher_bare=$tmp/publisher-origin.git
|
||||
git init -q --bare "$publisher_bare"
|
||||
git -C "$publisher_repo" remote add origin "$publisher_bare"
|
||||
git -C "$publisher_repo" push -q -u origin main
|
||||
|
||||
if (cd "$publisher_repo" && PATH="$publisher_repo/fake-bin:$PATH" \
|
||||
./scripts/publish-release-images.sh --components auto >/dev/null 2>&1); then
|
||||
echo 'first release guessed components in auto mode' >&2
|
||||
exit 1
|
||||
fi
|
||||
if (cd "$publisher_repo" && PATH="$publisher_repo/fake-bin:$PATH" FAKE_DOCKER_MODE=registry \
|
||||
./scripts/publish-release-images.sh --components all >/dev/null 2>&1); then
|
||||
echo 'publisher passed without registry access' >&2
|
||||
exit 1
|
||||
fi
|
||||
if (cd "$publisher_repo" && PATH="$publisher_repo/fake-bin:$PATH" FAKE_DOCKER_MODE=platform \
|
||||
./scripts/publish-release-images.sh --components all >/dev/null 2>&1); then
|
||||
echo 'publisher passed without linux/amd64 execution support' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
current_manifest=$tmp/current.json
|
||||
RELEASE_SOURCE_SHA=cccccccccccccccccccccccccccccccccccccccc \
|
||||
RELEASE_BASE_SHA=dddddddddddddddddddddddddddddddddddddddd \
|
||||
RELEASE_COMPONENTS=web \
|
||||
RELEASE_MIGRATIONS_CHANGED=false \
|
||||
RELEASE_API_IMAGE=$api_image \
|
||||
RELEASE_WEB_IMAGE=$web_image \
|
||||
RELEASE_BUILD_COMPLETED_AT=2026-07-22T00:02:00Z \
|
||||
RELEASE_SMOKE_COMPLETED_AT=2026-07-22T00:02:00Z \
|
||||
node "$root/scripts/release-manifest.mjs" create "$current_manifest" >/dev/null
|
||||
stale_bin=$tmp/stale-bin
|
||||
mkdir -p "$stale_bin"
|
||||
cat >"$stale_bin/ssh" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
if [[ $* == *' status' ]]; then
|
||||
cat "$STALE_STATUS_FILE"
|
||||
fi
|
||||
exit 0
|
||||
EOF
|
||||
cat >"$stale_bin/scp" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
: >"$STALE_SCP_MARKER"
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$stale_bin/"*
|
||||
stale_marker=$tmp/stale-scp-called
|
||||
if PATH="$stale_bin:$PATH" STALE_STATUS_FILE=$current_manifest STALE_SCP_MARKER=$stale_marker \
|
||||
"$root/scripts/deploy-production-release.sh" "$manifest" >/dev/null 2>&1; then
|
||||
echo 'stale production base passed deploy client validation' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ ! -e $stale_marker ]] || {
|
||||
echo 'stale deploy uploaded a manifest' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if find "$root/.gitea/workflows" -type f -print -quit 2>/dev/null | grep -q .; then
|
||||
echo 'automatic Gitea workflow still exists' >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'stale release manifest' "$root/deploy/manual/easyai-ai-gateway-release"
|
||||
grep -Fq 'production_changed=false' "$root/scripts/publish-release-images.sh"
|
||||
|
||||
echo 'manual_release_tests=PASS'
|
||||
Reference in New Issue
Block a user