From 0aa9b3e88f1341616751994f3e403f065f58f08d Mon Sep 17 00:00:00 2001 From: chengcheng Date: Tue, 21 Jul 2026 10:45:52 +0800 Subject: [PATCH] =?UTF-8?q?fix(ci):=20=E4=BD=BF=E7=94=A8=E9=9A=94=E7=A6=BB?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E8=BF=90=E8=A1=8C=20PostgreSQL=20=E9=9B=86?= =?UTF-8?q?=E6=88=90=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 PostgreSQL 16 改为固定 digest 的 Actions service container，并通过专用 Job 网络向 Go migrator 和集成测试提供数据库。 移除 PR Job 对 Docker CLI 的依赖，继续禁止宿主及内层 Docker API 暴露；同步发布工作流、契约测试和镜像代理恢复说明。 --- .gitea/workflows/ci.yml | 55 ++++++++++--------------------- .gitea/workflows/release-ci.yml | 55 ++++++++++--------------------- docs/runbooks/production-ci-cd.md | 10 +++++- scripts/provision-ci-runner.sh | 17 +--------- tests/ci/pipeline-test.sh | 13 +++----- 5 files changed, 49 insertions(+), 101 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index ff74a71..58f1805 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -9,8 +9,22 @@ on: jobs: verify: runs-on: easyai-gateway-ci-unprivileged-v2 + services: + postgres: + image: docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 + env: + POSTGRES_USER: easyai_test + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: easyai_gateway_test + options: >- + --health-cmd "pg_isready -U easyai_test -d easyai_gateway_test" + --health-interval 2s + --health-timeout 5s + --health-retries 30 env: TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2 + AI_GATEWAY_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable + AI_GATEWAY_TEST_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable steps: - name: Checkout without external Actions env: @@ -47,7 +61,6 @@ jobs: go version node --version pnpm --version - docker --version docker-compose version shellcheck --version trivy --version @@ -80,37 +93,9 @@ jobs: printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2 exit 1 } - - name: Start PostgreSQL 16 integration database - env: - CI_RUN_ID: ${{ github.run_id }} - CI_RUN_NUMBER: ${{ github.run_number }} - run: | - set -eu - container="easyai-gateway-test-pg-${CI_RUN_ID}-${CI_RUN_NUMBER}" - docker run -d --rm --name "$container" \ - -e POSTGRES_USER=easyai_test \ - -e POSTGRES_PASSWORD=easyai_test_only \ - -e POSTGRES_DB=easyai_gateway_test \ - postgres:16-alpine - for attempt in $(seq 1 60); do - if docker exec "$container" pg_isready -U easyai_test -d easyai_gateway_test >/dev/null 2>&1; then - break - fi - test "$attempt" -lt 60 - sleep 1 - done - database_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container") - test -n "$database_ip" - printf 'AI_GATEWAY_TEST_DATABASE_URL=postgresql://easyai_test:easyai_test_only@%s:5432/easyai_gateway_test?sslmode=disable\n' "$database_ip" >> "$GITHUB_ENV" - printf 'AI_GATEWAY_TEST_POSTGRES_CONTAINER=%s\n' "$container" >> "$GITHUB_ENV" - docker exec "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test \ - -c 'CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())' - for migration in apps/api/migrations/*.sql; do - docker exec -i "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test < "$migration" - version=$(basename "$migration" .sql) - docker exec "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test \ - -c "INSERT INTO schema_migrations(version) VALUES ('$version') ON CONFLICT (version) DO NOTHING" - done + - name: Migrate PostgreSQL 16 integration database + working-directory: apps/api + run: go run ./cmd/migrate - name: Verify Go code working-directory: apps/api env: @@ -142,9 +127,3 @@ jobs: trivy fs --scanners vuln,secret,misconfig --severity HIGH,CRITICAL \ --ignore-unfixed --exit-code 1 --timeout 15m --skip-dirs .git \ --skip-dirs node_modules . - - name: Stop PostgreSQL integration database - if: always() - run: | - if test -n "${AI_GATEWAY_TEST_POSTGRES_CONTAINER:-}"; then - docker rm -f "$AI_GATEWAY_TEST_POSTGRES_CONTAINER" >/dev/null 2>&1 || true - fi diff --git a/.gitea/workflows/release-ci.yml b/.gitea/workflows/release-ci.yml index a5dbdef..7142922 100644 --- a/.gitea/workflows/release-ci.yml +++ b/.gitea/workflows/release-ci.yml @@ -7,8 +7,22 @@ on: jobs: verify-tag: runs-on: easyai-gateway-ci-unprivileged-v2 + services: + postgres: + image: docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 + env: + POSTGRES_USER: easyai_test + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: easyai_gateway_test + options: >- + --health-cmd "pg_isready -U easyai_test -d easyai_gateway_test" + --health-interval 2s + --health-timeout 5s + --health-retries 30 env: TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2 + AI_GATEWAY_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable + AI_GATEWAY_TEST_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable steps: - name: Checkout without external Actions env: @@ -31,7 +45,6 @@ jobs: go version node --version pnpm --version - docker --version docker-compose version shellcheck --version trivy --version @@ -65,37 +78,9 @@ jobs: printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2 exit 1 } - - name: Start PostgreSQL 16 integration database - env: - CI_RUN_ID: ${{ github.run_id }} - CI_RUN_NUMBER: ${{ github.run_number }} - run: | - set -eu - container="easyai-gateway-test-pg-${CI_RUN_ID}-${CI_RUN_NUMBER}" - docker run -d --rm --name "$container" \ - -e POSTGRES_USER=easyai_test \ - -e POSTGRES_PASSWORD=easyai_test_only \ - -e POSTGRES_DB=easyai_gateway_test \ - postgres:16-alpine - for attempt in $(seq 1 60); do - if docker exec "$container" pg_isready -U easyai_test -d easyai_gateway_test >/dev/null 2>&1; then - break - fi - test "$attempt" -lt 60 - sleep 1 - done - database_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container") - test -n "$database_ip" - printf 'AI_GATEWAY_TEST_DATABASE_URL=postgresql://easyai_test:easyai_test_only@%s:5432/easyai_gateway_test?sslmode=disable\n' "$database_ip" >> "$GITHUB_ENV" - printf 'AI_GATEWAY_TEST_POSTGRES_CONTAINER=%s\n' "$container" >> "$GITHUB_ENV" - docker exec "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test \ - -c 'CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())' - for migration in apps/api/migrations/*.sql; do - docker exec -i "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test < "$migration" - version=$(basename "$migration" .sql) - docker exec "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test \ - -c "INSERT INTO schema_migrations(version) VALUES ('$version') ON CONFLICT (version) DO NOTHING" - done + - name: Migrate PostgreSQL 16 integration database + working-directory: apps/api + run: go run ./cmd/migrate - name: Verify Go code working-directory: apps/api env: @@ -127,9 +112,3 @@ jobs: trivy fs --scanners vuln,secret,misconfig --severity HIGH,CRITICAL \ --ignore-unfixed --exit-code 1 --timeout 15m --skip-dirs .git \ --skip-dirs node_modules . - - name: Stop PostgreSQL integration database - if: always() - run: | - if test -n "${AI_GATEWAY_TEST_POSTGRES_CONTAINER:-}"; then - docker rm -f "$AI_GATEWAY_TEST_POSTGRES_CONTAINER" >/dev/null 2>&1 || true - fi diff --git a/docs/runbooks/production-ci-cd.md b/docs/runbooks/production-ci-cd.md index 794170f..1fbd25c 100644 --- a/docs/runbooks/production-ci-cd.md +++ b/docs/runbooks/production-ci-cd.md @@ -16,7 +16,7 @@ 生产 CD 由部署仓的 root-owned dispatcher 执行。它在相同源码 SHA 上同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)` 成功，然后只运行部署仓中固定的 BuildKit 构建、Trivy 镜像扫描、digest 发布、备份、迁移、健康检查和回滚命令；不得执行 Tag checkout 中的 `pnpm`、测试、shell 脚本或应用程序。 -provision 安装 checksum-pinned 的官方静态 ShellCheck `0.11.0`、Docker Compose `5.3.1`，并从固定 digest 的 Runner 镜像提取 Docker CLI `29.6.0` 后校验二进制 SHA-256；不会依赖宿主是否预装这些命令。固定 digest 的 Runner 镜像已存在时可离线复用，缺失时才访问镜像仓库。所有工具都在精确 Node Job 镜像内实际执行版本检查。 +provision 安装 checksum-pinned 的官方静态 ShellCheck `0.11.0` 和 Docker Compose `5.3.1`，不会依赖宿主是否预装这些命令。所有下载都在使用前校验固定 SHA-256，并在精确 Node Job 镜像内实际执行版本检查。PostgreSQL 16 集成库作为固定 digest 的 Actions service container 运行，PR Job 不接收宿主或内层 Docker API。 ## 首次安装或修复 CI Runner @@ -34,6 +34,14 @@ RUNNER_REGISTRATION_TOKEN='<一次性 Token>' ./scripts/provision-ci-runner.sh ./scripts/provision-ci-runner.sh ``` +宿主无法直连 Docker Hub 时，可使用可信 HTTPS 镜像代理；Runner 来源仍必须携带脚本中完全相同的 manifest digest，内层 daemon 拉取的 Job 与 service 镜像也会继续校验工作流固定的 digest： + +```bash +CI_RUNNER_IMAGE_SOURCE='docker.m.daocloud.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a' \ +CI_NESTED_DOCKER_REGISTRY_MIRROR='https://docker.m.daocloud.io' \ +./scripts/provision-ci-runner.sh +``` + 注册 Token 通过交互 stdin 送给短命注册容器，不出现在宿主或容器的命令行参数中；provision 随后立即从环境中清除它，长期 systemd unit 也不包含 Token。不要输出或复制 `/data/.runner` 内容。脚本会停用旧 host Runner，并确保旧 Runner 用户和 v2 遗留用户不属于 `docker`/`sudo` 组。 默认磁盘门禁可用环境变量提高，但生产环境不应降低： diff --git a/scripts/provision-ci-runner.sh b/scripts/provision-ci-runner.sh index eae66c7..91b8db7 100755 --- a/scripts/provision-ci-runner.sh +++ b/scripts/provision-ci-runner.sh @@ -26,8 +26,6 @@ SHELLCHECK_VERSION=0.11.0 SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198 COMPOSE_VERSION=5.3.1 COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959 -DOCKER_CLI_VERSION=29.6.0 -DOCKER_CLI_SHA256=80b40fa11d797a71ea15f9a31e7004790ddac95f58a8222a40c08f08a639faba PNPM_VERSION=10.18.1 RUNNER_IMAGE='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a' JOB_IMAGE='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7' @@ -192,22 +190,10 @@ GOBIN="$PREFIX/bin" GOPROXY="${GO_MODULE_PROXY:-https://goproxy.cn,direct}" \ "$PREFIX/toolchains/go/bin/go" install golang.org/x/vuln/cmd/govulncheck@v1.6.0 require_free_space "$MIN_FREE_GIB" "runner image pull" -if ! docker image inspect "$RUNNER_IMAGE_SOURCE" >/dev/null 2>&1; then - docker pull "$RUNNER_IMAGE_SOURCE" -fi +docker pull "$RUNNER_IMAGE_SOURCE" runner_runtime_image=$(docker image inspect --format '{{.Id}}' "$RUNNER_IMAGE_SOURCE") printf '%s\n' "$runner_runtime_image" | grep -Eq '^sha256:[0-9a-f]{64}$' || \ fail "pinned runner image did not resolve to an immutable local image ID" - -# The job image intentionally contains no Docker client. Extract the static CLI -# from the immutable Runner image, then verify its binary digest before exposing -# it through the read-only CI toolchain mount. -docker_cli_download="$PREFIX/downloads/docker-${DOCKER_CLI_VERSION}-linux-x86_64" -docker run --rm --pull=never --entrypoint /bin/cat "$runner_runtime_image" \ - /usr/local/bin/docker > "$docker_cli_download" -printf '%s %s\n' "$DOCKER_CLI_SHA256" "$docker_cli_download" | sha256sum -c - -install -m 0755 "$docker_cli_download" "$PREFIX/bin/docker" -"$PREFIX/bin/docker" --version | grep -Fq "Docker version ${DOCKER_CLI_VERSION}," chown -R root:root "$PREFIX" chmod -R go-w "$PREFIX" @@ -313,7 +299,6 @@ docker exec "$PROBE_CONTAINER" docker run --rm --pull=never \ go version node --version pnpm --version - docker --version docker-compose version shellcheck --version trivy --version diff --git a/tests/ci/pipeline-test.sh b/tests/ci/pipeline-test.sh index 7a609fb..7ff545d 100755 --- a/tests/ci/pipeline-test.sh +++ b/tests/ci/pipeline-test.sh @@ -116,8 +116,12 @@ for quality_workflow in "$workflow" "$release_workflow"; do grep -Fq './tests/ci/pipeline-test.sh' "$quality_workflow" grep -Fq './tests/ci/semver-test.sh' "$quality_workflow" grep -Fq 'docker-compose version' "$quality_workflow" - grep -Fq 'docker --version' "$quality_workflow" grep -Fq 'docker-compose -f docker-compose.yml config --quiet' "$quality_workflow" + grep -Fq 'services:' "$quality_workflow" + grep -Fq 'image: docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777' "$quality_workflow" + grep -Fq 'POSTGRES_HOST_AUTH_METHOD: trust' "$quality_workflow" + grep -Fq 'AI_GATEWAY_TEST_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable' "$quality_workflow" + grep -Fq 'run: go run ./cmd/migrate' "$quality_workflow" if grep -Fq 'docker compose' "$quality_workflow"; then echo 'job container must use the read-only standalone Compose binary' >&2 exit 1 @@ -265,14 +269,11 @@ grep -q '^SHELLCHECK_VERSION=0.11.0$' "$provision" grep -q '^SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198$' "$provision" grep -q '^COMPOSE_VERSION=5.3.1$' "$provision" grep -q '^COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959$' "$provision" -grep -q '^DOCKER_CLI_VERSION=29.6.0$' "$provision" -grep -q '^DOCKER_CLI_SHA256=80b40fa11d797a71ea15f9a31e7004790ddac95f58a8222a40c08f08a639faba$' "$provision" grep -Fq 'RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-ci-unprivileged-v2}' "$provision" grep -Fq "RUNNER_IMAGE='$runner_image'" "$provision" grep -Fq 'RUNNER_IMAGE_SOURCE=${CI_RUNNER_IMAGE_SOURCE:-$RUNNER_IMAGE}' "$provision" grep -Fq 'runner_manifest_digest=${RUNNER_IMAGE##*@}' "$provision" grep -Fq 'CI_RUNNER_IMAGE_SOURCE must use the pinned runner manifest digest' "$provision" -grep -Fq 'if ! docker image inspect "$RUNNER_IMAGE_SOURCE"' "$provision" grep -Fq 'runner_runtime_image=$(docker image inspect' "$provision" grep -Fq 'RUNNER_IMAGE_REF="%s"' "$provision" grep -Fq "JOB_IMAGE='$job_image'" "$provision" @@ -297,10 +298,6 @@ grep -Fq 'shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz' "$provision" grep -Fq 'docker-compose-linux-x86_64' "$provision" grep -Fq 'install -m 0755 "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "$PREFIX/bin/shellcheck"' "$provision" grep -Fq 'install -m 0755 "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" "$PREFIX/bin/docker-compose"' "$provision" -grep -Fq 'docker run --rm --pull=never --entrypoint /bin/cat "$runner_runtime_image"' "$provision" -grep -Fq 'printf '\''%s %s\n'\'' "$DOCKER_CLI_SHA256" "$docker_cli_download" | sha256sum -c -' "$provision" -grep -Fq 'install -m 0755 "$docker_cli_download" "$PREFIX/bin/docker"' "$provision" -grep -Fq '"$PREFIX/bin/docker" --version' "$provision" grep -Fq 'PATH="$PREFIX/toolchains/node/bin:$PATH"' "$provision" grep -Fq 'gpasswd -d easyai-gateway-ci-v2 docker' "$provision" grep -Fq 'gpasswd -d easyai-gateway-ci-v2 sudo' "$provision"