#!/usr/bin/env bash # shellcheck disable=SC2016 # Assertions intentionally match literal workflow variables. set -euo pipefail root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) workflow=$root/.gitea/workflows/ci.yml release_workflow=$root/.gitea/workflows/release-ci.yml runner_service=$root/deploy/ci/easyai-gateway-ci-v2-runner.service runner_config=$root/deploy/ci/act-runner-v2-config.yaml provision=$root/scripts/provision-ci-runner.sh build_images=$root/scripts/ci-build-images.sh validate_semver=$root/scripts/ci-validate-semver.sh validate_migrations=$root/scripts/ci-validate-migrations.mjs build_images_test=$root/tests/ci/ci-build-images-test.sh migrations_test=$root/tests/ci/migrations-test.sh migration_base=$root/deploy/ci/production-migration-base adr=$root/docs/decisions/001-production-cicd.md runbook=$root/docs/runbooks/production-ci-cd.md migrator=$root/apps/api/cmd/migrate/main.go for file in "$workflow" "$release_workflow" "$runner_service" "$runner_config" "$provision" "$build_images" "$validate_semver" "$validate_migrations" "$build_images_test" "$migrations_test" "$migration_base" "$adr" "$runbook" "$migrator"; do [[ -f $file ]] || { printf 'missing CI/CD file: %s\n' "$file" >&2 exit 1 } done if [[ $(wc -l <"$migration_base") -ne 1 ]]; then echo 'production migration baseline must contain exactly one line' >&2 exit 1 fi grep -Fq 'SET standard_conforming_strings = on' "$migrator" || { echo 'database migrator does not pin standard SQL string semantics' >&2 exit 1 } if ! grep -Eq '^[0-9a-f]{40}$' "$migration_base"; then echo 'production migration baseline must be a full lowercase SHA' >&2 exit 1 fi grep -q '^name: ci$' "$workflow" grep -q '^ push:$' "$workflow" grep -q '^ branches: \[main\]$' "$workflow" grep -q '^ pull_request:$' "$workflow" grep -q '^ verify:$' "$workflow" grep -Fq 'git checkout --detach "$CI_SHA"' "$workflow" grep -Fq 'test "$(git rev-parse HEAD)" = "$CI_SHA"' "$workflow" if grep -Fq 'git checkout --detach FETCH_HEAD' "$workflow"; then echo 'main/PR CI must not checkout mutable FETCH_HEAD after comparison fetches' >&2 exit 1 fi if grep -Eq "^[[:space:]]+tags:|Verify release tag ancestry" "$workflow" || \ grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$workflow"; then echo 'main/PR CI must not share the release-tag context' >&2 exit 1 fi grep -q '^name: release-ci$' "$release_workflow" grep -q '^ push:$' "$release_workflow" grep -q "^ tags: \['v\*'\]$" "$release_workflow" grep -q '^ verify-tag:$' "$release_workflow" grep -q '^ - name: Verify release tag ancestry$' "$release_workflow" grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$release_workflow" grep -Fq 'git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main' "$release_workflow" if grep -Eq 'branches:|pull_request:' "$release_workflow"; then echo 'release CI must be a tag-only workflow/context' >&2 exit 1 fi for quality_workflow in "$workflow" "$release_workflow"; do grep -q '^ runs-on: easyai-gateway-ci-unprivileged-v2$' "$quality_workflow" grep -q '^ - name: Checkout without external Actions$' "$quality_workflow" if grep -q -- '--depth=' "$quality_workflow"; then echo 'quality verification must not use a shallow checkout' >&2 exit 1 fi grep -Fq 'test ! -f .git/shallow' "$quality_workflow" grep -Fq './tests/ci/ci-build-images-test.sh' "$quality_workflow" grep -Fq './tests/ci/migrations-test.sh' "$quality_workflow" grep -Fq './tests/ci/pipeline-test.sh' "$quality_workflow" grep -Fq './tests/ci/semver-test.sh' "$quality_workflow" grep -Fq 'docker-compose version' "$quality_workflow" grep -Fq 'docker-compose -f docker-compose.yml config --quiet' "$quality_workflow" if grep -Fq 'docker compose' "$quality_workflow"; then echo 'job container must use the read-only standalone Compose binary' >&2 exit 1 fi for forbidden in \ 'sudo' \ '/usr/local/sbin/easyai-ai-gateway-release' \ 'docker version' \ 'docker buildx' \ './scripts/ci-build-images.sh --keep-images' \ 'Deploy verified release to Production'; do if grep -Fq "$forbidden" "$quality_workflow"; then printf 'unprivileged workflow contains forbidden capability: %s\n' "$forbidden" >&2 exit 1 fi done if grep -Eq '^[[:space:]]+\./scripts/ci-build-images\.sh([[:space:]]|$)' "$quality_workflow"; then echo 'unprivileged workflow must not build container images' >&2 exit 1 fi for gate in \ 'scripts/ci-validate-migrations.mjs' \ 'gofmt -l' \ 'go vet ./...' \ 'go test ./...' \ 'pnpm install --frozen-lockfile' \ 'pnpm lint' \ 'pnpm test' \ 'pnpm build' \ 'pnpm audit --audit-level high' \ 'docker-compose -f docker-compose.yml config --quiet' \ 'govulncheck ./...' \ 'trivy fs'; do grep -Fq "$gate" "$quality_workflow" || { printf '%s is missing quality gate: %s\n' "$quality_workflow" "$gate" >&2 exit 1 } done grep -Fq 'TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2' \ "$quality_workflow" || { printf '%s does not pin the reachable Trivy vulnerability DB mirror\n' \ "$quality_workflow" >&2 exit 1 } grep -Fq 'production_base=$(cat deploy/ci/production-migration-base)' \ "$quality_workflow" || { printf '%s does not read the versioned production migration baseline\n' \ "$quality_workflow" >&2 exit 1 } done if [[ $(sed -n '/^ - name: Verify Go formatting$/,$p' "$workflow") != \ "$(sed -n '/^ - name: Verify Go formatting$/,$p' "$release_workflow")" ]]; then echo 'main/PR and release-tag workflows must have identical quality gates' >&2 exit 1 fi grep -Fq 'CI_EVENT_BEFORE: ${{ github.event.before }}' "$workflow" grep -Fq 'CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}' "$workflow" grep -Fq 'git merge-base --is-ancestor "$immutable_base" HEAD' "$workflow" grep -Fq '"$production_base" "$immutable_base"' "$workflow" grep -Fq 'node ./scripts/ci-validate-migrations.mjs "$production_base"' \ "$release_workflow" runner_image='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a' job_image='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7' grep -q '^User=root$' "$runner_service" grep -q '^Group=root$' "$runner_service" grep -q '^NoNewPrivileges=true$' "$runner_service" grep -q '^ProtectSystem=strict$' "$runner_service" grep -q '^ProtectHome=true$' "$runner_service" grep -q '^CapabilityBoundingSet=$' "$runner_service" grep -Fq 'Requires=docker.service' "$runner_service" grep -Fq -- "--pull=never" "$runner_service" grep -Fq -- "--privileged" "$runner_service" grep -Fq -- '--pids-limit 1024' "$runner_service" grep -Fq -- '--cpus 2' "$runner_service" grep -Fq -- '--memory 2g' "$runner_service" grep -Fq -- '--memory-swap 3g' "$runner_service" grep -Fq -- '--env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns' "$runner_service" grep -Fq 'EnvironmentFile=-/etc/easyai-gateway-ci-v2/runner.env' "$runner_service" grep -Fq '$RUNNER_SECURITY_OPTIONS' "$runner_service" grep -Fq -- "--volume easyai-gateway-ci-v2-data:/data" "$runner_service" grep -Fq -- "--volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro" "$runner_service" grep -Fq -- "--volume /etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro" "$runner_service" grep -Fq -- "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro" "$runner_service" grep -Fq 'Environment=RUNNER_IMAGE_REF=' "$runner_service" grep -Fq '$RUNNER_IMAGE_REF' "$runner_service" if grep -Eq 'GITEA_RUNNER_REGISTRATION_TOKEN|/var/run/docker\.sock|/run/docker\.sock' "$runner_service"; then echo 'persistent runner service exposes a registration token or host Docker socket' >&2 exit 1 fi if grep -Fq 'apparmor=rootlesskit' "$runner_service"; then echo 'runner service must not hard-code an AppArmor profile that may be absent' >&2 exit 1 fi grep -Fq "easyai-gateway-ci-unprivileged-v2:docker://$job_image" "$runner_config" if grep -Fq ':host' "$runner_config"; then echo 'CI runner must never use the host executor' >&2 exit 1 fi grep -q '^ capacity: 1$' "$runner_config" grep -q '^ file: /data/.runner$' "$runner_config" grep -q '^ docker_host: "-"$' "$runner_config" grep -q '^ privileged: false$' "$runner_config" grep -q '^ require_docker: true$' "$runner_config" grep -q '^ bind_workdir: false$' "$runner_config" grep -Fq ' options: "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro"' "$runner_config" grep -q '^ valid_volumes:$' "$runner_config" grep -q '^ - "/opt/easyai-gateway-ci"$' "$runner_config" if awk ' /^ valid_volumes:$/ { in_volumes=1; next } in_volumes && /^ - / { count++; next } in_volumes && /^ [A-Za-z_]+:/ { in_volumes=0 } END { exit count == 1 ? 0 : 1 } ' "$runner_config"; then : else echo 'runner must allow exactly one workflow volume source' >&2 exit 1 fi if grep -Eq '/var/run/docker\.sock|/run/docker\.sock' "$runner_config"; then echo 'job configuration must not expose a Docker socket' >&2 exit 1 fi if grep -Eq 'apparmor=unconfined|sysctl .*(apparmor_restrict_unprivileged_userns)' "$runner_service" "$runner_config" "$provision"; then echo 'runner must not disable AppArmor isolation or its user-namespace restriction' >&2 exit 1 fi grep -q '^GITEA_RUNNER_VERSION=2.0.0$' "$provision" grep -q '^GO_VERSION=1.26.5$' "$provision" grep -q '^NODE_VERSION=24.16.0$' "$provision" grep -q '^TRIVY_VERSION=0.70.0$' "$provision" grep -q '^PREFIX=/opt/easyai-gateway-ci$' "$provision" grep -q '^RUNNER_VOLUME=easyai-gateway-ci-v2-data$' "$provision" grep -q '^SHELLCHECK_VERSION=0.11.0$' "$provision" grep -q '^SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198$' "$provision" grep -q '^COMPOSE_VERSION=5.3.1$' "$provision" grep -q '^COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959$' "$provision" grep -Fq 'RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-ci-unprivileged-v2}' "$provision" grep -Fq "RUNNER_IMAGE='$runner_image'" "$provision" grep -Fq 'RUNNER_IMAGE_SOURCE=${CI_RUNNER_IMAGE_SOURCE:-$RUNNER_IMAGE}' "$provision" grep -Fq 'runner_manifest_digest=${RUNNER_IMAGE##*@}' "$provision" grep -Fq 'CI_RUNNER_IMAGE_SOURCE must use the pinned runner manifest digest' "$provision" grep -Fq 'runner_runtime_image=$(docker image inspect' "$provision" grep -Fq 'RUNNER_IMAGE_REF="%s"' "$provision" grep -Fq "JOB_IMAGE='$job_image'" "$provision" grep -Fq 'RUNNER_LABEL="easyai-gateway-ci-unprivileged-v2:docker://$JOB_IMAGE"' "$provision" grep -Fq 'docker volume create "$RUNNER_VOLUME"' "$provision" grep -Fq '"data-root":"/data/docker"' "$provision" grep -Fq -- '--security-opt apparmor=rootlesskit' "$provision" grep -Fq 'RUNNER_SECURITY_OPTIONS="--security-opt=apparmor=rootlesskit"' "$provision" grep -Fq 'RUNNER_SECURITY_OPTIONS=' "$provision" grep -Fq '/proc/sys/kernel/apparmor_restrict_unprivileged_userns' "$provision" grep -Fq -- '--entrypoint dockerd-entrypoint.sh "$runner_runtime_image"' "$provision" grep -Fq '/etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro' "$provision" grep -Fq 'CI_NESTED_DOCKER_REGISTRY_MIRROR must be an HTTPS registry origin' "$provision" grep -Fq 'PROBE_CONTAINER="easyai-gateway-ci-v2-probe-$$"' "$provision" grep -Fq -- '--pids-limit "$RUNNER_PIDS_LIMIT"' "$provision" grep -Fq -- '--cpus "$RUNNER_CPUS"' "$provision" grep -Fq -- '--memory "$RUNNER_MEMORY"' "$provision" grep -Fq -- '--memory-swap "$RUNNER_MEMORY_SWAP"' "$provision" grep -Fq "'2147483648:3221225472:2000000000:1024'" "$provision" grep -Fq -- '--entrypoint /usr/local/bin/gitea-runner' "$provision" grep -Fq 'shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz' "$provision" grep -Fq 'docker-compose-linux-x86_64' "$provision" grep -Fq 'install -m 0755 "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "$PREFIX/bin/shellcheck"' "$provision" grep -Fq 'install -m 0755 "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" "$PREFIX/bin/docker-compose"' "$provision" grep -Fq 'PATH="$PREFIX/toolchains/node/bin:$PATH"' "$provision" grep -Fq 'gpasswd -d easyai-gateway-ci-v2 docker' "$provision" grep -Fq 'gpasswd -d easyai-gateway-ci-v2 sudo' "$provision" if grep -Fq 'usermod -aG docker' "$provision"; then echo 'unprivileged runner must never join the Docker group' >&2 exit 1 fi grep -Fq 'systemctl disable --now easyai-gateway-act-runner.service' "$provision" grep -Fq 'systemctl disable --now easyai-gateway-ci-v2-runner.service' "$provision" grep -Fq 'RUNNER_REGISTRATION_TOKEN is required for first registration' "$provision" grep -Fq 'printf '\''%s\n%s\n%s\n'\'' "$GITEA_INSTANCE_URL" "$RUNNER_REGISTRATION_TOKEN" "$RUNNER_NAME"' "$provision" if grep -Fq -- '--token "$RUNNER_REGISTRATION_TOKEN"' "$provision"; then echo 'registration token must not be exposed in a host-visible process argv' >&2 exit 1 fi grep -Fq "grep -Fq 'name=rootless'" "$provision" grep -Fq "grep -Fq 'declare successfully'" "$provision" grep -Fq 'docker buildx prune --builder "$LEGACY_BUILDER" --force --max-used-space 512mb' "$provision" grep -Fq 'require_free_space "$MIN_FREE_GIB" "runner image pull"' "$provision" grep -Fq 'require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed CI runner installation"' "$provision" grep -Fq 'CI_RUNNER_MIN_FREE_GIB cannot be lower than 8' "$provision" grep -Fq 'CI_RUNNER_MIN_POST_INSTALL_FREE_GIB cannot be lower than 4' "$provision" grep -Fq 'docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"' "$provision" grep -Fq 'docker exec "$PROBE_CONTAINER" docker run --rm --pull=never' "$provision" grep -Fq -- '--volume "$RUNNER_VOLUME:/data"' "$provision" grep -Fq -- '--volume "$PREFIX:$PREFIX:ro"' "$provision" probe_pull_line=$(grep -nF 'docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"' "$provision" | cut -d: -f1) registration_line=$(grep -nF 'if ! runner_is_registered; then' "$provision" | cut -d: -f1) [[ -n $probe_pull_line && -n $registration_line && $probe_pull_line -lt $registration_line ]] || { echo 'pinned job image and isolation probes must complete before runner registration' >&2 exit 1 } grep -Fq 'nested --pid=host can see the outer runner PID namespace' "$provision" grep -Fq 'chown -R root:root "$PREFIX"' "$provision" grep -Fq 'chmod -R go-w "$PREFIX"' "$provision" if grep -Eq 'SHELLCHECK_BIN|COMPOSE_PLUGIN|command -v shellcheck|docker compose version' "$provision"; then echo 'runner provisioning must use checksum-pinned static CI tools, not host copies' >&2 exit 1 fi if grep -Eq 'docker (system|builder|image|volume) prune' "$provision"; then echo 'runner provisioning must not globally prune shared Docker state' >&2 exit 1 fi grep -Fq 'root-owned dispatcher' "$adr" grep -Fq 'release-ci / verify-tag (push)' "$adr" grep -Fq '来自 Fork 的 PR 必须先由维护者检查工作流变更并在 Gitea 中批准' "$adr" grep -Fq 'rootless DinD 不是 VM 沙箱' "$runbook" grep -Fq '同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)`' "$runbook" grep -Fq '不得执行 Tag checkout 中的' "$runbook" grep -Fq 'docker buildx build --builder "$builder"' "$build_images" grep -q -- '--target api' "$build_images" grep -q -- '--target web' "$build_images" grep -Fq 'docker buildx prune --builder "$builder" --force --max-used-space "$cache_limit"' "$build_images" grep -Fq 'trivy image' "$build_images" grep -Fq 'docker image rm "$api_image" "$web_image"' "$build_images" grep -Fq 'AI_GATEWAY_BUILDX_CACHE_LIMIT has an invalid format' "$build_images" if grep -En '(password|secret|token):[[:space:]]+[^$<{]' "$workflow" "$release_workflow"; then echo 'workflow contains a literal credential' >&2 exit 1 fi echo 'gateway_ci_pipeline_tests=PASS'