324 lines
16 KiB
Bash
Executable File
324 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# shellcheck disable=SC2016 # Assertions intentionally match literal workflow variables.
|
|
set -euo pipefail
|
|
|
|
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
|
workflow=$root/.gitea/workflows/ci.yml
|
|
release_workflow=$root/.gitea/workflows/release-ci.yml
|
|
runner_service=$root/deploy/ci/easyai-gateway-ci-v2-runner.service
|
|
runner_config=$root/deploy/ci/act-runner-v2-config.yaml
|
|
provision=$root/scripts/provision-ci-runner.sh
|
|
build_images=$root/scripts/ci-build-images.sh
|
|
validate_semver=$root/scripts/ci-validate-semver.sh
|
|
validate_migrations=$root/scripts/ci-validate-migrations.mjs
|
|
build_images_test=$root/tests/ci/ci-build-images-test.sh
|
|
migrations_test=$root/tests/ci/migrations-test.sh
|
|
migration_base=$root/deploy/ci/production-migration-base
|
|
adr=$root/docs/decisions/001-production-cicd.md
|
|
runbook=$root/docs/runbooks/production-ci-cd.md
|
|
migrator=$root/apps/api/cmd/migrate/main.go
|
|
|
|
for file in "$workflow" "$release_workflow" "$runner_service" "$runner_config" "$provision" "$build_images" "$validate_semver" "$validate_migrations" "$build_images_test" "$migrations_test" "$migration_base" "$adr" "$runbook" "$migrator"; do
|
|
[[ -f $file ]] || {
|
|
printf 'missing CI/CD file: %s\n' "$file" >&2
|
|
exit 1
|
|
}
|
|
done
|
|
|
|
if [[ $(wc -l <"$migration_base") -ne 1 ]]; then
|
|
echo 'production migration baseline must contain exactly one line' >&2
|
|
exit 1
|
|
fi
|
|
grep -Fq 'SET standard_conforming_strings = on' "$migrator" || {
|
|
echo 'database migrator does not pin standard SQL string semantics' >&2
|
|
exit 1
|
|
}
|
|
if ! grep -Eq '^[0-9a-f]{40}$' "$migration_base"; then
|
|
echo 'production migration baseline must be a full lowercase SHA' >&2
|
|
exit 1
|
|
fi
|
|
|
|
grep -q '^name: ci$' "$workflow"
|
|
grep -q '^ push:$' "$workflow"
|
|
grep -q '^ branches: \[main\]$' "$workflow"
|
|
grep -q '^ pull_request:$' "$workflow"
|
|
grep -q '^ verify:$' "$workflow"
|
|
grep -Fq 'git checkout --detach "$CI_SHA"' "$workflow"
|
|
grep -Fq 'test "$(git rev-parse HEAD)" = "$CI_SHA"' "$workflow"
|
|
if grep -Fq 'git checkout --detach FETCH_HEAD' "$workflow"; then
|
|
echo 'main/PR CI must not checkout mutable FETCH_HEAD after comparison fetches' >&2
|
|
exit 1
|
|
fi
|
|
if grep -Eq "^[[:space:]]+tags:|Verify release tag ancestry" "$workflow" || \
|
|
grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$workflow"; then
|
|
echo 'main/PR CI must not share the release-tag context' >&2
|
|
exit 1
|
|
fi
|
|
|
|
grep -q '^name: release-ci$' "$release_workflow"
|
|
grep -q '^ push:$' "$release_workflow"
|
|
grep -q "^ tags: \['v\*'\]$" "$release_workflow"
|
|
grep -q '^ verify-tag:$' "$release_workflow"
|
|
grep -q '^ - name: Verify release tag ancestry$' "$release_workflow"
|
|
grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$release_workflow"
|
|
grep -Fq 'git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main' "$release_workflow"
|
|
if grep -Eq 'branches:|pull_request:' "$release_workflow"; then
|
|
echo 'release CI must be a tag-only workflow/context' >&2
|
|
exit 1
|
|
fi
|
|
|
|
for quality_workflow in "$workflow" "$release_workflow"; do
|
|
grep -q '^ runs-on: easyai-gateway-ci-unprivileged-v2$' "$quality_workflow"
|
|
grep -q '^ - name: Checkout without external Actions$' "$quality_workflow"
|
|
if grep -q -- '--depth=' "$quality_workflow"; then
|
|
echo 'quality verification must not use a shallow checkout' >&2
|
|
exit 1
|
|
fi
|
|
grep -Fq 'test ! -f .git/shallow' "$quality_workflow"
|
|
grep -Fq './tests/ci/ci-build-images-test.sh' "$quality_workflow"
|
|
grep -Fq './tests/ci/migrations-test.sh' "$quality_workflow"
|
|
grep -Fq './tests/ci/pipeline-test.sh' "$quality_workflow"
|
|
grep -Fq './tests/ci/semver-test.sh' "$quality_workflow"
|
|
grep -Fq 'docker-compose version' "$quality_workflow"
|
|
grep -Fq 'docker-compose -f docker-compose.yml config --quiet' "$quality_workflow"
|
|
if grep -Fq 'docker compose' "$quality_workflow"; then
|
|
echo 'job container must use the read-only standalone Compose binary' >&2
|
|
exit 1
|
|
fi
|
|
|
|
for forbidden in \
|
|
'sudo' \
|
|
'/usr/local/sbin/easyai-ai-gateway-release' \
|
|
'docker version' \
|
|
'docker buildx' \
|
|
'./scripts/ci-build-images.sh --keep-images' \
|
|
'Deploy verified release to Production'; do
|
|
if grep -Fq "$forbidden" "$quality_workflow"; then
|
|
printf 'unprivileged workflow contains forbidden capability: %s\n' "$forbidden" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
if grep -Eq '^[[:space:]]+\./scripts/ci-build-images\.sh([[:space:]]|$)' "$quality_workflow"; then
|
|
echo 'unprivileged workflow must not build container images' >&2
|
|
exit 1
|
|
fi
|
|
|
|
for gate in \
|
|
'scripts/ci-validate-migrations.mjs' \
|
|
'gofmt -l' \
|
|
'go vet ./...' \
|
|
'go test ./...' \
|
|
'pnpm install --frozen-lockfile' \
|
|
'pnpm lint' \
|
|
'pnpm test' \
|
|
'pnpm build' \
|
|
'pnpm audit --audit-level high' \
|
|
'docker-compose -f docker-compose.yml config --quiet' \
|
|
'govulncheck ./...' \
|
|
'trivy fs'; do
|
|
grep -Fq "$gate" "$quality_workflow" || {
|
|
printf '%s is missing quality gate: %s\n' "$quality_workflow" "$gate" >&2
|
|
exit 1
|
|
}
|
|
done
|
|
grep -Fq 'TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2' \
|
|
"$quality_workflow" || {
|
|
printf '%s does not pin the reachable Trivy vulnerability DB mirror\n' \
|
|
"$quality_workflow" >&2
|
|
exit 1
|
|
}
|
|
grep -Fq 'production_base=$(cat deploy/ci/production-migration-base)' \
|
|
"$quality_workflow" || {
|
|
printf '%s does not read the versioned production migration baseline\n' \
|
|
"$quality_workflow" >&2
|
|
exit 1
|
|
}
|
|
done
|
|
|
|
if [[ $(sed -n '/^ - name: Verify Go formatting$/,$p' "$workflow") != \
|
|
"$(sed -n '/^ - name: Verify Go formatting$/,$p' "$release_workflow")" ]]; then
|
|
echo 'main/PR and release-tag workflows must have identical quality gates' >&2
|
|
exit 1
|
|
fi
|
|
|
|
grep -Fq 'CI_EVENT_BEFORE: ${{ github.event.before }}' "$workflow"
|
|
grep -Fq 'CI_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}' "$workflow"
|
|
grep -Fq 'git merge-base --is-ancestor "$immutable_base" HEAD' "$workflow"
|
|
grep -Fq '"$production_base" "$immutable_base"' "$workflow"
|
|
grep -Fq 'node ./scripts/ci-validate-migrations.mjs "$production_base"' \
|
|
"$release_workflow"
|
|
|
|
runner_image='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a'
|
|
job_image='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7'
|
|
|
|
grep -q '^User=root$' "$runner_service"
|
|
grep -q '^Group=root$' "$runner_service"
|
|
grep -q '^NoNewPrivileges=true$' "$runner_service"
|
|
grep -q '^ProtectSystem=strict$' "$runner_service"
|
|
grep -q '^ProtectHome=true$' "$runner_service"
|
|
grep -q '^CapabilityBoundingSet=$' "$runner_service"
|
|
grep -Fq 'Requires=docker.service' "$runner_service"
|
|
grep -Fq -- "--pull=never" "$runner_service"
|
|
grep -Fq -- "--privileged" "$runner_service"
|
|
grep -Fq -- '--pids-limit 1024' "$runner_service"
|
|
grep -Fq -- '--cpus 2' "$runner_service"
|
|
grep -Fq -- '--memory 2g' "$runner_service"
|
|
grep -Fq -- '--memory-swap 3g' "$runner_service"
|
|
grep -Fq -- '--env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns' "$runner_service"
|
|
grep -Fq 'EnvironmentFile=-/etc/easyai-gateway-ci-v2/runner.env' "$runner_service"
|
|
grep -Fq '$RUNNER_SECURITY_OPTIONS' "$runner_service"
|
|
grep -Fq -- "--volume easyai-gateway-ci-v2-data:/data" "$runner_service"
|
|
grep -Fq -- "--volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro" "$runner_service"
|
|
grep -Fq -- "--volume /etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro" "$runner_service"
|
|
grep -Fq -- "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro" "$runner_service"
|
|
grep -Fq 'Environment=RUNNER_IMAGE_REF=' "$runner_service"
|
|
grep -Fq '$RUNNER_IMAGE_REF' "$runner_service"
|
|
if grep -Eq 'GITEA_RUNNER_REGISTRATION_TOKEN|/var/run/docker\.sock|/run/docker\.sock' "$runner_service"; then
|
|
echo 'persistent runner service exposes a registration token or host Docker socket' >&2
|
|
exit 1
|
|
fi
|
|
if grep -Fq 'apparmor=rootlesskit' "$runner_service"; then
|
|
echo 'runner service must not hard-code an AppArmor profile that may be absent' >&2
|
|
exit 1
|
|
fi
|
|
|
|
grep -Fq "easyai-gateway-ci-unprivileged-v2:docker://$job_image" "$runner_config"
|
|
if grep -Fq ':host' "$runner_config"; then
|
|
echo 'CI runner must never use the host executor' >&2
|
|
exit 1
|
|
fi
|
|
grep -q '^ capacity: 1$' "$runner_config"
|
|
grep -q '^ file: /data/.runner$' "$runner_config"
|
|
grep -q '^ docker_host: "-"$' "$runner_config"
|
|
grep -q '^ privileged: false$' "$runner_config"
|
|
grep -q '^ require_docker: true$' "$runner_config"
|
|
grep -q '^ bind_workdir: false$' "$runner_config"
|
|
grep -Fq ' options: "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro"' "$runner_config"
|
|
grep -q '^ valid_volumes:$' "$runner_config"
|
|
grep -q '^ - "/opt/easyai-gateway-ci"$' "$runner_config"
|
|
if awk '
|
|
/^ valid_volumes:$/ { in_volumes=1; next }
|
|
in_volumes && /^ - / { count++; next }
|
|
in_volumes && /^ [A-Za-z_]+:/ { in_volumes=0 }
|
|
END { exit count == 1 ? 0 : 1 }
|
|
' "$runner_config"; then
|
|
:
|
|
else
|
|
echo 'runner must allow exactly one workflow volume source' >&2
|
|
exit 1
|
|
fi
|
|
if grep -Eq '/var/run/docker\.sock|/run/docker\.sock' "$runner_config"; then
|
|
echo 'job configuration must not expose a Docker socket' >&2
|
|
exit 1
|
|
fi
|
|
if grep -Eq 'apparmor=unconfined|sysctl .*(apparmor_restrict_unprivileged_userns)' "$runner_service" "$runner_config" "$provision"; then
|
|
echo 'runner must not disable AppArmor isolation or its user-namespace restriction' >&2
|
|
exit 1
|
|
fi
|
|
|
|
grep -q '^GITEA_RUNNER_VERSION=2.0.0$' "$provision"
|
|
grep -q '^GO_VERSION=1.26.5$' "$provision"
|
|
grep -q '^NODE_VERSION=24.16.0$' "$provision"
|
|
grep -q '^TRIVY_VERSION=0.70.0$' "$provision"
|
|
grep -q '^PREFIX=/opt/easyai-gateway-ci$' "$provision"
|
|
grep -q '^RUNNER_VOLUME=easyai-gateway-ci-v2-data$' "$provision"
|
|
grep -q '^SHELLCHECK_VERSION=0.11.0$' "$provision"
|
|
grep -q '^SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198$' "$provision"
|
|
grep -q '^COMPOSE_VERSION=5.3.1$' "$provision"
|
|
grep -q '^COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959$' "$provision"
|
|
grep -Fq 'RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-ci-unprivileged-v2}' "$provision"
|
|
grep -Fq "RUNNER_IMAGE='$runner_image'" "$provision"
|
|
grep -Fq 'RUNNER_IMAGE_SOURCE=${CI_RUNNER_IMAGE_SOURCE:-$RUNNER_IMAGE}' "$provision"
|
|
grep -Fq 'runner_manifest_digest=${RUNNER_IMAGE##*@}' "$provision"
|
|
grep -Fq 'CI_RUNNER_IMAGE_SOURCE must use the pinned runner manifest digest' "$provision"
|
|
grep -Fq 'runner_runtime_image=$(docker image inspect' "$provision"
|
|
grep -Fq 'RUNNER_IMAGE_REF="%s"' "$provision"
|
|
grep -Fq "JOB_IMAGE='$job_image'" "$provision"
|
|
grep -Fq 'RUNNER_LABEL="easyai-gateway-ci-unprivileged-v2:docker://$JOB_IMAGE"' "$provision"
|
|
grep -Fq 'docker volume create "$RUNNER_VOLUME"' "$provision"
|
|
grep -Fq '"data-root":"/data/docker"' "$provision"
|
|
grep -Fq -- '--security-opt apparmor=rootlesskit' "$provision"
|
|
grep -Fq 'RUNNER_SECURITY_OPTIONS="--security-opt=apparmor=rootlesskit"' "$provision"
|
|
grep -Fq 'RUNNER_SECURITY_OPTIONS=' "$provision"
|
|
grep -Fq '/proc/sys/kernel/apparmor_restrict_unprivileged_userns' "$provision"
|
|
grep -Fq -- '--entrypoint dockerd-entrypoint.sh "$runner_runtime_image"' "$provision"
|
|
grep -Fq '/etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro' "$provision"
|
|
grep -Fq 'CI_NESTED_DOCKER_REGISTRY_MIRROR must be an HTTPS registry origin' "$provision"
|
|
grep -Fq 'PROBE_CONTAINER="easyai-gateway-ci-v2-probe-$$"' "$provision"
|
|
grep -Fq -- '--pids-limit "$RUNNER_PIDS_LIMIT"' "$provision"
|
|
grep -Fq -- '--cpus "$RUNNER_CPUS"' "$provision"
|
|
grep -Fq -- '--memory "$RUNNER_MEMORY"' "$provision"
|
|
grep -Fq -- '--memory-swap "$RUNNER_MEMORY_SWAP"' "$provision"
|
|
grep -Fq "'2147483648:3221225472:2000000000:1024'" "$provision"
|
|
grep -Fq -- '--entrypoint /usr/local/bin/gitea-runner' "$provision"
|
|
grep -Fq 'shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz' "$provision"
|
|
grep -Fq 'docker-compose-linux-x86_64' "$provision"
|
|
grep -Fq 'install -m 0755 "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "$PREFIX/bin/shellcheck"' "$provision"
|
|
grep -Fq 'install -m 0755 "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" "$PREFIX/bin/docker-compose"' "$provision"
|
|
grep -Fq 'PATH="$PREFIX/toolchains/node/bin:$PATH"' "$provision"
|
|
grep -Fq 'gpasswd -d easyai-gateway-ci-v2 docker' "$provision"
|
|
grep -Fq 'gpasswd -d easyai-gateway-ci-v2 sudo' "$provision"
|
|
if grep -Fq 'usermod -aG docker' "$provision"; then
|
|
echo 'unprivileged runner must never join the Docker group' >&2
|
|
exit 1
|
|
fi
|
|
grep -Fq 'systemctl disable --now easyai-gateway-act-runner.service' "$provision"
|
|
grep -Fq 'systemctl disable --now easyai-gateway-ci-v2-runner.service' "$provision"
|
|
grep -Fq 'RUNNER_REGISTRATION_TOKEN is required for first registration' "$provision"
|
|
grep -Fq 'printf '\''%s\n%s\n%s\n'\'' "$GITEA_INSTANCE_URL" "$RUNNER_REGISTRATION_TOKEN" "$RUNNER_NAME"' "$provision"
|
|
if grep -Fq -- '--token "$RUNNER_REGISTRATION_TOKEN"' "$provision"; then
|
|
echo 'registration token must not be exposed in a host-visible process argv' >&2
|
|
exit 1
|
|
fi
|
|
grep -Fq "grep -Fq 'name=rootless'" "$provision"
|
|
grep -Fq "grep -Fq 'declare successfully'" "$provision"
|
|
grep -Fq 'docker buildx prune --builder "$LEGACY_BUILDER" --force --max-used-space 512mb' "$provision"
|
|
grep -Fq 'require_free_space "$MIN_FREE_GIB" "runner image pull"' "$provision"
|
|
grep -Fq 'require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed CI runner installation"' "$provision"
|
|
grep -Fq 'CI_RUNNER_MIN_FREE_GIB cannot be lower than 8' "$provision"
|
|
grep -Fq 'CI_RUNNER_MIN_POST_INSTALL_FREE_GIB cannot be lower than 4' "$provision"
|
|
grep -Fq 'docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"' "$provision"
|
|
grep -Fq 'docker exec "$PROBE_CONTAINER" docker run --rm --pull=never' "$provision"
|
|
grep -Fq -- '--volume "$RUNNER_VOLUME:/data"' "$provision"
|
|
grep -Fq -- '--volume "$PREFIX:$PREFIX:ro"' "$provision"
|
|
probe_pull_line=$(grep -nF 'docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"' "$provision" | cut -d: -f1)
|
|
registration_line=$(grep -nF 'if ! runner_is_registered; then' "$provision" | cut -d: -f1)
|
|
[[ -n $probe_pull_line && -n $registration_line && $probe_pull_line -lt $registration_line ]] || {
|
|
echo 'pinned job image and isolation probes must complete before runner registration' >&2
|
|
exit 1
|
|
}
|
|
grep -Fq 'nested --pid=host can see the outer runner PID namespace' "$provision"
|
|
grep -Fq 'chown -R root:root "$PREFIX"' "$provision"
|
|
grep -Fq 'chmod -R go-w "$PREFIX"' "$provision"
|
|
if grep -Eq 'SHELLCHECK_BIN|COMPOSE_PLUGIN|command -v shellcheck|docker compose version' "$provision"; then
|
|
echo 'runner provisioning must use checksum-pinned static CI tools, not host copies' >&2
|
|
exit 1
|
|
fi
|
|
if grep -Eq 'docker (system|builder|image|volume) prune' "$provision"; then
|
|
echo 'runner provisioning must not globally prune shared Docker state' >&2
|
|
exit 1
|
|
fi
|
|
|
|
grep -Fq 'root-owned dispatcher' "$adr"
|
|
grep -Fq 'release-ci / verify-tag (push)' "$adr"
|
|
grep -Fq '来自 Fork 的 PR 必须先由维护者检查工作流变更并在 Gitea 中批准' "$adr"
|
|
grep -Fq 'rootless DinD 不是 VM 沙箱' "$runbook"
|
|
grep -Fq '同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)`' "$runbook"
|
|
grep -Fq '不得执行 Tag checkout 中的' "$runbook"
|
|
|
|
grep -Fq 'docker buildx build --builder "$builder"' "$build_images"
|
|
grep -q -- '--target api' "$build_images"
|
|
grep -q -- '--target web' "$build_images"
|
|
grep -Fq 'docker buildx prune --builder "$builder" --force --max-used-space "$cache_limit"' "$build_images"
|
|
grep -Fq 'trivy image' "$build_images"
|
|
grep -Fq 'docker image rm "$api_image" "$web_image"' "$build_images"
|
|
grep -Fq 'AI_GATEWAY_BUILDX_CACHE_LIMIT has an invalid format' "$build_images"
|
|
|
|
if grep -En '(password|secret|token):[[:space:]]+[^$<{]' "$workflow" "$release_workflow"; then
|
|
echo 'workflow contains a literal credential' >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo 'gateway_ci_pipeline_tests=PASS'
|