ci: harden production quality gates
ci / verify (pull_request) Failing after 4m18s

This commit is contained in:
2026-07-17 14:23:49 +08:00
parent db85487b73
commit 91ea0e6a2d
15 changed files with 1023 additions and 252 deletions
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env bash
set -euo pipefail
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
script=$root/scripts/ci-build-images.sh
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
mkdir -p "$tmp/bin"
cat >"$tmp/bin/docker" <<'MOCK'
#!/usr/bin/env bash
set -euo pipefail
printf 'docker %s\n' "$*" >>"$MOCK_LOG"
case "$*" in
'buildx inspect '*|'buildx create '*) exit 0 ;;
'buildx build '*)
if [[ ${MOCK_BUILD_FAIL:-0} == 1 && $* == *'--target web'* ]]; then
exit 21
fi
;;
'buildx prune '*)
[[ ${MOCK_PRUNE_FAIL:-0} == 0 ]] || exit 22
;;
'image rm '*)
[[ ${MOCK_IMAGE_RM_FAIL:-0} == 0 ]] || exit 23
;;
*)
printf 'unexpected docker invocation: %s\n' "$*" >&2
exit 99
;;
esac
MOCK
cat >"$tmp/bin/trivy" <<'MOCK'
#!/usr/bin/env bash
set -euo pipefail
printf 'trivy %s\n' "$*" >>"$MOCK_LOG"
MOCK
chmod +x "$tmp/bin/docker" "$tmp/bin/trivy"
sha=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
registry=registry.example.com/easyai
api_image=$registry/ai-gateway:$sha
web_image=$registry/ai-gateway-web:$sha
run_build() {
env \
PATH="$tmp/bin:$PATH" \
MOCK_LOG="$tmp/calls.log" \
IMAGE_TAG="$sha" \
AI_GATEWAY_IMAGE_REGISTRY="$registry" \
AI_GATEWAY_BUILDX_BUILDER=gateway-test \
AI_GATEWAY_BUILDX_CACHE_LIMIT=2gb \
"$script" "$@"
}
: >"$tmp/calls.log"
run_build
grep -Fq "docker buildx prune --builder gateway-test --force --max-used-space 2gb" "$tmp/calls.log"
grep -Fq "docker image rm $api_image $web_image" "$tmp/calls.log"
: >"$tmp/calls.log"
run_build --keep-images
grep -Fq "docker buildx prune --builder gateway-test --force --max-used-space 2gb" "$tmp/calls.log"
if grep -Fq 'docker image rm ' "$tmp/calls.log"; then
echo '--keep-images must preserve fully built images' >&2
exit 1
fi
: >"$tmp/calls.log"
if MOCK_PRUNE_FAIL=1 run_build; then
echo 'a prune failure must fail an otherwise successful build' >&2
exit 1
fi
: >"$tmp/calls.log"
if MOCK_IMAGE_RM_FAIL=1 run_build; then
echo 'an image cleanup failure must fail an otherwise successful build' >&2
exit 1
fi
: >"$tmp/calls.log"
if MOCK_BUILD_FAIL=1 run_build; then
echo 'a failed image build must fail the script' >&2
exit 1
fi
grep -Fq "docker image rm $api_image $web_image" "$tmp/calls.log"
for invalid_limit in 0gb 2 2GB -1gb '2gb --all'; do
: >"$tmp/calls.log"
if env \
PATH="$tmp/bin:$PATH" \
MOCK_LOG="$tmp/calls.log" \
IMAGE_TAG="$sha" \
AI_GATEWAY_IMAGE_REGISTRY="$registry" \
AI_GATEWAY_BUILDX_CACHE_LIMIT="$invalid_limit" \
"$script" >/dev/null 2>&1; then
printf 'invalid cache limit was accepted: %s\n' "$invalid_limit" >&2
exit 1
fi
if [[ -s $tmp/calls.log ]]; then
printf 'invalid cache limit reached Docker: %s\n' "$invalid_limit" >&2
exit 1
fi
done
echo 'ci_build_images_tests=PASS'
+209 -34
View File
@@ -4,12 +4,17 @@ set -euo pipefail
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
workflow=$root/.gitea/workflows/ci.yml
runner_service=$root/deploy/ci/easyai-gateway-act-runner.service
runner_config=$root/deploy/ci/act-runner-config.yaml
release_workflow=$root/.gitea/workflows/release-ci.yml
runner_service=$root/deploy/ci/easyai-gateway-ci-v2-runner.service
runner_config=$root/deploy/ci/act-runner-v2-config.yaml
provision=$root/scripts/provision-ci-runner.sh
build_images=$root/scripts/ci-build-images.sh
validate_semver=$root/scripts/ci-validate-semver.sh
build_images_test=$root/tests/ci/ci-build-images-test.sh
adr=$root/docs/decisions/001-production-cicd.md
runbook=$root/docs/runbooks/production-ci-cd.md
for file in "$workflow" "$runner_service" "$runner_config" "$provision" "$build_images"; do
for file in "$workflow" "$release_workflow" "$runner_service" "$runner_config" "$provision" "$build_images" "$validate_semver" "$build_images_test" "$adr" "$runbook"; do
[[ -f $file ]] || {
printf 'missing CI/CD file: %s\n' "$file" >&2
exit 1
@@ -17,59 +22,229 @@ for file in "$workflow" "$runner_service" "$runner_config" "$provision" "$build_
done
grep -q '^name: ci$' "$workflow"
grep -q '^ push:$' "$workflow"
grep -q '^ branches: \[main\]$' "$workflow"
grep -q "^ tags: \['v\*'\]$" "$workflow"
grep -q '^ pull_request:$' "$workflow"
grep -q '^ runs-on: easyai-gateway-linux$' "$workflow"
grep -q '^ - name: Checkout without external Actions$' "$workflow"
grep -q '^ - name: Verify release tag ancestry$' "$workflow"
grep -Fq "if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')" "$workflow"
grep -Fq 'git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main' "$workflow"
grep -q '^ - run: ./scripts/ci-build-images.sh$' "$workflow"
grep -q '^ - name: Deploy verified release to Production$' "$workflow"
grep -q '^ sudo -n /usr/local/sbin/easyai-ai-gateway-release "$CI_SHA"$' "$workflow"
grep -q '^ verify:$' "$workflow"
if grep -Eq "^[[:space:]]+tags:|Verify release tag ancestry" "$workflow" || \
grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$workflow"; then
echo 'main/PR CI must not share the release-tag context' >&2
exit 1
fi
for gate in \
'gofmt -l' \
'go vet ./...' \
'go test ./...' \
'pnpm install --frozen-lockfile' \
'pnpm lint' \
'pnpm test' \
'pnpm build' \
'docker compose -f docker-compose.yml config --quiet' \
'govulncheck ./...' \
'trivy fs'; do
grep -Fq "$gate" "$workflow" || {
printf 'workflow is missing quality gate: %s\n' "$gate" >&2
grep -q '^name: release-ci$' "$release_workflow"
grep -q '^ push:$' "$release_workflow"
grep -q "^ tags: \['v\*'\]$" "$release_workflow"
grep -q '^ verify-tag:$' "$release_workflow"
grep -q '^ - name: Verify release tag ancestry$' "$release_workflow"
grep -Fq './scripts/ci-validate-semver.sh "$tag_name"' "$release_workflow"
grep -Fq 'git merge-base --is-ancestor "$CI_SHA" refs/remotes/origin/main' "$release_workflow"
if grep -Eq 'branches:|pull_request:' "$release_workflow"; then
echo 'release CI must be a tag-only workflow/context' >&2
exit 1
fi
for quality_workflow in "$workflow" "$release_workflow"; do
grep -q '^ runs-on: easyai-gateway-ci-unprivileged-v2$' "$quality_workflow"
grep -q '^ - name: Checkout without external Actions$' "$quality_workflow"
if grep -q -- '--depth=' "$quality_workflow"; then
echo 'quality verification must not use a shallow checkout' >&2
exit 1
}
fi
grep -Fq 'test ! -f .git/shallow' "$quality_workflow"
grep -Fq './tests/ci/ci-build-images-test.sh' "$quality_workflow"
grep -Fq './tests/ci/pipeline-test.sh' "$quality_workflow"
grep -Fq './tests/ci/semver-test.sh' "$quality_workflow"
grep -Fq 'docker-compose version' "$quality_workflow"
grep -Fq 'docker-compose -f docker-compose.yml config --quiet' "$quality_workflow"
if grep -Fq 'docker compose' "$quality_workflow"; then
echo 'job container must use the read-only standalone Compose binary' >&2
exit 1
fi
for forbidden in \
'sudo' \
'/usr/local/sbin/easyai-ai-gateway-release' \
'docker version' \
'docker buildx' \
'./scripts/ci-build-images.sh --keep-images' \
'Deploy verified release to Production'; do
if grep -Fq "$forbidden" "$quality_workflow"; then
printf 'unprivileged workflow contains forbidden capability: %s\n' "$forbidden" >&2
exit 1
fi
done
if grep -Eq '^[[:space:]]+\./scripts/ci-build-images\.sh([[:space:]]|$)' "$quality_workflow"; then
echo 'unprivileged workflow must not build container images' >&2
exit 1
fi
for gate in \
'gofmt -l' \
'go vet ./...' \
'go test ./...' \
'pnpm install --frozen-lockfile' \
'pnpm lint' \
'pnpm test' \
'pnpm build' \
'pnpm audit --audit-level high' \
'docker-compose -f docker-compose.yml config --quiet' \
'govulncheck ./...' \
'trivy fs'; do
grep -Fq "$gate" "$quality_workflow" || {
printf '%s is missing quality gate: %s\n' "$quality_workflow" "$gate" >&2
exit 1
}
done
done
grep -q '^User=easyai-gateway-runner$' "$runner_service"
grep -q '^Group=easyai-gateway-runner$' "$runner_service"
grep -q '^NoNewPrivileges=false$' "$runner_service"
if [[ $(sed -n '/^ - name: Verify Go formatting$/,$p' "$workflow") != \
"$(sed -n '/^ - name: Verify Go formatting$/,$p' "$release_workflow")" ]]; then
echo 'main/PR and release-tag workflows must have identical quality gates' >&2
exit 1
fi
runner_image='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a'
job_image='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7'
grep -q '^User=root$' "$runner_service"
grep -q '^Group=root$' "$runner_service"
grep -q '^NoNewPrivileges=true$' "$runner_service"
grep -q '^ProtectSystem=strict$' "$runner_service"
grep -q '^ProtectHome=true$' "$runner_service"
grep -q '^ReadWritePaths=/var/lib/easyai-gateway-runner$' "$runner_service"
grep -q '^ExecStart=/opt/easyai-gateway-ci/bin/gitea-runner --config /etc/easyai-gateway-runner/config.yaml daemon --once$' "$runner_service"
grep -q 'easyai-gateway-linux:host' "$runner_config"
grep -q '^CapabilityBoundingSet=$' "$runner_service"
grep -Fq 'Requires=docker.service' "$runner_service"
grep -Fq -- "--pull=never" "$runner_service"
grep -Fq -- "--privileged" "$runner_service"
grep -Fq -- '--pids-limit 4096' "$runner_service"
grep -Fq -- '--env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns' "$runner_service"
grep -Fq 'EnvironmentFile=-/etc/easyai-gateway-ci-v2/runner.env' "$runner_service"
grep -Fq '$RUNNER_SECURITY_OPTIONS' "$runner_service"
grep -Fq -- "--volume easyai-gateway-ci-v2-data:/data" "$runner_service"
grep -Fq -- "--volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro" "$runner_service"
grep -Fq -- "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro" "$runner_service"
grep -Fq "$runner_image" "$runner_service"
if grep -Eq 'GITEA_RUNNER_REGISTRATION_TOKEN|/var/run/docker\.sock|/run/docker\.sock' "$runner_service"; then
echo 'persistent runner service exposes a registration token or host Docker socket' >&2
exit 1
fi
if grep -Fq 'apparmor=rootlesskit' "$runner_service"; then
echo 'runner service must not hard-code an AppArmor profile that may be absent' >&2
exit 1
fi
grep -Fq "easyai-gateway-ci-unprivileged-v2:docker://$job_image" "$runner_config"
if grep -Fq ':host' "$runner_config"; then
echo 'CI runner must never use the host executor' >&2
exit 1
fi
grep -q '^ capacity: 1$' "$runner_config"
grep -q '^ file: /data/.runner$' "$runner_config"
grep -q '^ docker_host: "-"$' "$runner_config"
grep -q '^ privileged: false$' "$runner_config"
grep -q '^ require_docker: true$' "$runner_config"
grep -q '^ bind_workdir: false$' "$runner_config"
grep -Fq ' options: "--volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro"' "$runner_config"
grep -q '^ valid_volumes:$' "$runner_config"
grep -q '^ - "/opt/easyai-gateway-ci"$' "$runner_config"
if awk '
/^ valid_volumes:$/ { in_volumes=1; next }
in_volumes && /^ - / { count++; next }
in_volumes && /^ [A-Za-z_]+:/ { in_volumes=0 }
END { exit count == 1 ? 0 : 1 }
' "$runner_config"; then
:
else
echo 'runner must allow exactly one workflow volume source' >&2
exit 1
fi
if grep -Eq '/var/run/docker\.sock|/run/docker\.sock' "$runner_config"; then
echo 'job configuration must not expose a Docker socket' >&2
exit 1
fi
if grep -Eq 'apparmor=unconfined|sysctl .*(apparmor_restrict_unprivileged_userns)' "$runner_service" "$runner_config" "$provision"; then
echo 'runner must not disable AppArmor isolation or its user-namespace restriction' >&2
exit 1
fi
grep -q '^GITEA_RUNNER_VERSION=2.0.0$' "$provision"
grep -q '^GO_VERSION=1.26.5$' "$provision"
grep -q '^NODE_VERSION=24.16.0$' "$provision"
grep -q '^TRIVY_VERSION=0.70.0$' "$provision"
grep -Fq 'usermod -aG docker "$RUNNER_USER"' "$provision"
grep -q '^PREFIX=/opt/easyai-gateway-ci$' "$provision"
grep -q '^RUNNER_VOLUME=easyai-gateway-ci-v2-data$' "$provision"
grep -q '^SHELLCHECK_VERSION=0.11.0$' "$provision"
grep -q '^SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198$' "$provision"
grep -q '^COMPOSE_VERSION=5.3.1$' "$provision"
grep -q '^COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959$' "$provision"
grep -Fq 'RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-ci-unprivileged-v2}' "$provision"
grep -Fq "RUNNER_IMAGE='$runner_image'" "$provision"
grep -Fq "JOB_IMAGE='$job_image'" "$provision"
grep -Fq 'RUNNER_LABEL="easyai-gateway-ci-unprivileged-v2:docker://$JOB_IMAGE"' "$provision"
grep -Fq 'docker volume create "$RUNNER_VOLUME"' "$provision"
grep -Fq -- '--security-opt apparmor=rootlesskit' "$provision"
grep -Fq 'RUNNER_SECURITY_OPTIONS="--security-opt=apparmor=rootlesskit"' "$provision"
grep -Fq 'RUNNER_SECURITY_OPTIONS=' "$provision"
grep -Fq '/proc/sys/kernel/apparmor_restrict_unprivileged_userns' "$provision"
grep -Fq -- '--entrypoint dockerd-entrypoint.sh "$RUNNER_IMAGE"' "$provision"
grep -Fq 'PROBE_CONTAINER="easyai-gateway-ci-v2-probe-$$"' "$provision"
grep -Fq -- '--entrypoint /usr/local/bin/gitea-runner' "$provision"
grep -Fq 'shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz' "$provision"
grep -Fq 'docker-compose-linux-x86_64' "$provision"
grep -Fq 'install -m 0755 "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "$PREFIX/bin/shellcheck"' "$provision"
grep -Fq 'install -m 0755 "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" "$PREFIX/bin/docker-compose"' "$provision"
grep -Fq 'gpasswd -d easyai-gateway-ci-v2 docker' "$provision"
grep -Fq 'gpasswd -d easyai-gateway-ci-v2 sudo' "$provision"
if grep -Fq 'usermod -aG docker' "$provision"; then
echo 'unprivileged runner must never join the Docker group' >&2
exit 1
fi
grep -Fq 'systemctl disable --now easyai-gateway-act-runner.service' "$provision"
grep -Fq 'systemctl disable --now easyai-gateway-ci-v2-runner.service' "$provision"
grep -Fq 'RUNNER_REGISTRATION_TOKEN is required for first registration' "$provision"
grep -Fq 'printf '\''%s\n%s\n%s\n'\'' "$GITEA_INSTANCE_URL" "$RUNNER_REGISTRATION_TOKEN" "$RUNNER_NAME"' "$provision"
if grep -Fq -- '--token "$RUNNER_REGISTRATION_TOKEN"' "$provision"; then
echo 'registration token must not be exposed in a host-visible process argv' >&2
exit 1
fi
grep -Fq "grep -Fq 'name=rootless'" "$provision"
grep -Fq "grep -Fq 'declare successfully'" "$provision"
grep -Fq 'docker buildx prune --builder "$LEGACY_BUILDER" --force --max-used-space 512mb' "$provision"
grep -Fq 'require_free_space "$MIN_FREE_GIB" "runner image pull"' "$provision"
grep -Fq 'require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed CI runner installation"' "$provision"
grep -Fq 'CI_RUNNER_MIN_FREE_GIB cannot be lower than 8' "$provision"
grep -Fq 'CI_RUNNER_MIN_POST_INSTALL_FREE_GIB cannot be lower than 4' "$provision"
grep -Fq 'docker exec "$RUNNER_CONTAINER" docker pull "$JOB_IMAGE"' "$provision"
grep -Fq 'docker exec "$RUNNER_CONTAINER" docker run --rm --pull=never' "$provision"
grep -Fq 'nested --pid=host can see the outer runner PID namespace' "$provision"
grep -Fq 'chown -R root:root "$PREFIX"' "$provision"
grep -Fq 'chmod -R go-w "$PREFIX"' "$provision"
if grep -Eq 'SHELLCHECK_BIN|COMPOSE_PLUGIN|command -v shellcheck|docker compose version' "$provision"; then
echo 'runner provisioning must use checksum-pinned static CI tools, not host copies' >&2
exit 1
fi
if grep -Eq 'docker (system|builder|image|volume) prune' "$provision"; then
echo 'runner provisioning must not globally prune shared Docker state' >&2
exit 1
fi
grep -Fq 'root-owned dispatcher' "$adr"
grep -Fq 'release-ci / verify-tag (push)' "$adr"
grep -Fq '来自 Fork 的 PR 必须先由维护者检查工作流变更并在 Gitea 中批准' "$adr"
grep -Fq 'rootless DinD 不是 VM 沙箱' "$runbook"
grep -Fq '同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)`' "$runbook"
grep -Fq '不得执行 Tag checkout 中的' "$runbook"
grep -Fq 'docker buildx build --builder "$builder"' "$build_images"
grep -q -- '--target api' "$build_images"
grep -q -- '--target web' "$build_images"
grep -Fq 'docker buildx prune --builder "$builder" --force --max-used-space "$cache_limit"' "$build_images"
grep -Fq 'trivy image' "$build_images"
grep -Fq 'docker image rm "$api_image" "$web_image"' "$build_images"
grep -Fq 'AI_GATEWAY_BUILDX_CACHE_LIMIT has an invalid format' "$build_images"
if rg -n '(password|secret|token):[[:space:]]+[^$<{]' "$workflow"; then
if grep -En '(password|secret|token):[[:space:]]+[^$<{]' "$workflow" "$release_workflow"; then
echo 'workflow contains a literal credential' >&2
exit 1
fi
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env bash
set -euo pipefail
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
validator=$root/scripts/ci-validate-semver.sh
valid_tags=(
v0.0.0
v1.2.3
v1.2.3-alpha
v1.2.3-alpha.1
v1.2.3-0.3.7
v1.2.3-01a
v1.2.3-alpha-beta
v1.2.3--
v1.2.3-x.7.z.92+build.5
v1.2.3+build.11.e0f985a
v1.2.3+001
)
invalid_tags=(
1.2.3
v01.2.3
v1.02.3
v1.2.03
v1.2.3.foo
v1.2.3.
v1.2.3-.
v1.2.3-alpha.
v1.2.3-alpha..1
v1.2.3-01
v1.2.3-00.1
v1.2.3+
v1.2.3+build.
v1.2.3+build..1
v1.2.3+build+1
'v1.2.3 '
v1.2.3-alpha_1
v1.2
)
for tag in "${valid_tags[@]}"; do
"$validator" "$tag"
done
for tag in "${invalid_tags[@]}"; do
if "$validator" "$tag" >/dev/null 2>&1; then
printf 'invalid SemVer tag was accepted: %s\n' "$tag" >&2
exit 1
fi
done
echo 'semver_tests=PASS'