easyai-ai-gateway/scripts/provision-ci-runner.sh

#!/bin/sh
set -eu

ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
PREFIX=/opt/easyai-gateway-ci
GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL:-https://git.51easyai.com}
RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-ci-unprivileged-v2}
RUNNER_VOLUME=easyai-gateway-ci-v2-data
RUNNER_CONTAINER=easyai-gateway-ci-v2-runner
LEGACY_BUILDER=${AI_GATEWAY_BUILDX_BUILDER:-easyai-gateway-ci}
MIN_FREE_GIB=${CI_RUNNER_MIN_FREE_GIB:-8}
MIN_POST_INSTALL_FREE_GIB=${CI_RUNNER_MIN_POST_INSTALL_FREE_GIB:-4}
RUNNER_CPUS=2
RUNNER_MEMORY=2g
RUNNER_MEMORY_SWAP=3g
RUNNER_PIDS_LIMIT=1024

GITEA_RUNNER_VERSION=2.0.0
GO_VERSION=1.26.5
GO_SHA256=5c2c3b16caefa1d968a94c1daca04a7ca301a496d9b086e17ad77bb81393f053
NODE_VERSION=24.16.0
NODE_SHA256=d804845d34eddc21dc1092b519d643ef40b1f58ec5dec5c22b1f4bd8fabde6c9
TRIVY_VERSION=0.70.0
TRIVY_SHA256=8b4376d5d6befe5c24d503f10ff136d9e0c49f9127a4279fd110b727929a5aa9
SHELLCHECK_VERSION=0.11.0
SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198
COMPOSE_VERSION=5.3.1
COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959
PNPM_VERSION=10.18.1
RUNNER_IMAGE='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a'
JOB_IMAGE='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7'
RUNNER_IMAGE_SOURCE=${CI_RUNNER_IMAGE_SOURCE:-$RUNNER_IMAGE}
NESTED_DOCKER_REGISTRY_MIRROR=${CI_NESTED_DOCKER_REGISTRY_MIRROR:-}
RUNNER_LABEL="easyai-gateway-ci-unprivileged-v2:docker://$JOB_IMAGE"

fail() {
  echo "$*" >&2
  exit 1
}

case $MIN_FREE_GIB:$MIN_POST_INSTALL_FREE_GIB in
  *[!0-9:]* | :* | *:)
    fail "CI runner free-space limits must be whole GiB values"
    ;;
esac
[ "$MIN_FREE_GIB" -ge 8 ] || fail "CI_RUNNER_MIN_FREE_GIB cannot be lower than 8"
[ "$MIN_POST_INSTALL_FREE_GIB" -ge 4 ] || \
  fail "CI_RUNNER_MIN_POST_INSTALL_FREE_GIB cannot be lower than 4"

[ "$(id -u)" -eq 0 ] || fail "run as root"
[ "$(uname -s)-$(uname -m)" = "Linux-x86_64" ] || fail "only Linux x86_64 is supported"
case $RUNNER_IMAGE in
  *"/runner:${GITEA_RUNNER_VERSION}-dind-rootless@sha256:"*) ;;
  *) fail "RUNNER_IMAGE does not match GITEA_RUNNER_VERSION" ;;
esac
runner_manifest_digest=${RUNNER_IMAGE##*@}
case $RUNNER_IMAGE_SOURCE in
  *@"$runner_manifest_digest") ;;
  *) fail "CI_RUNNER_IMAGE_SOURCE must use the pinned runner manifest digest" ;;
esac
if [ -n "$NESTED_DOCKER_REGISTRY_MIRROR" ] && \
  ! printf '%s\n' "$NESTED_DOCKER_REGISTRY_MIRROR" | \
    grep -Eq '^https://[A-Za-z0-9.-]+(:[0-9]+)?$'; then
  fail "CI_NESTED_DOCKER_REGISTRY_MIRROR must be an HTTPS registry origin"
fi
command -v docker >/dev/null 2>&1 || fail "Docker Engine is required"
docker info >/dev/null 2>&1 || fail "Docker Engine is not reachable"
grep -Fq "\"$RUNNER_LABEL\"" "$ROOT/deploy/ci/act-runner-v2-config.yaml" || \
  fail "runner label does not match the pinned job image"

# Fail closed before installing anything. Neither the legacy host runner nor an
# earlier v2 unit may poll for work while its replacement is being prepared.
systemctl disable --now easyai-gateway-act-runner.service >/dev/null 2>&1 || true
systemctl disable --now easyai-gateway-ci-v2-runner.service >/dev/null 2>&1 || true
docker rm --force "$RUNNER_CONTAINER" >/dev/null 2>&1 || true
rm -f /etc/systemd/system/easyai-gateway-act-runner.service

if id easyai-gateway-runner >/dev/null 2>&1; then
  if getent group docker >/dev/null 2>&1; then
    gpasswd -d easyai-gateway-runner docker >/dev/null 2>&1 || true
  fi
  if getent group sudo >/dev/null 2>&1; then
    gpasswd -d easyai-gateway-runner sudo >/dev/null 2>&1 || true
  fi
fi
if id easyai-gateway-ci-v2 >/dev/null 2>&1; then
  if getent group docker >/dev/null 2>&1; then
    gpasswd -d easyai-gateway-ci-v2 docker >/dev/null 2>&1 || true
  fi
  if getent group sudo >/dev/null 2>&1; then
    gpasswd -d easyai-gateway-ci-v2 sudo >/dev/null 2>&1 || true
  fi
fi
for account in easyai-gateway-runner easyai-gateway-ci-v2; do
  if id "$account" >/dev/null 2>&1 && \
    id -nG "$account" | tr ' ' '\n' | grep -Eq '^(docker|sudo)$'; then
    fail "$account must not belong to a privileged group"
  fi
done

if ! command -v make >/dev/null 2>&1 || ! command -v gcc >/dev/null 2>&1 || \
  ! command -v jq >/dev/null 2>&1 || ! command -v curl >/dev/null 2>&1 || \
  ! command -v xz >/dev/null 2>&1; then
  export DEBIAN_FRONTEND=noninteractive
  apt-get update
  apt-get install -y --no-install-recommends \
    build-essential ca-certificates curl git jq perl unzip util-linux xz-utils
fi

docker_root=$(docker info --format '{{.DockerRootDir}}')
[ -n "$docker_root" ] || fail "Docker root directory could not be determined"

require_free_space() {
  minimum_gib=$1
  phase=$2
  available_kib=$(df -Pk "$docker_root" | awk 'NR == 2 { print $4 }')
  case $available_kib in
    '' | *[!0-9]*) fail "free space could not be determined for $docker_root" ;;
  esac
  minimum_kib=$((minimum_gib * 1024 * 1024))
  if [ "$available_kib" -lt "$minimum_kib" ]; then
    echo "$phase requires at least ${minimum_gib} GiB free on the Docker filesystem" >&2
    return 1
  fi
}

# This builder belonged only to the retired gateway image-building runner. Cap
# its unused cache before adding the nested daemon; never prune global images,
# volumes, or caches owned by another service on this shared production host.
if docker buildx version >/dev/null 2>&1 && \
  docker buildx inspect "$LEGACY_BUILDER" >/dev/null 2>&1; then
  docker buildx prune --builder "$LEGACY_BUILDER" --force --max-used-space 512mb
fi
require_free_space "$MIN_FREE_GIB" "CI runner installation"

download_verified() {
  url=$1
  output=$2
  checksum=$3
  if [ ! -f "$output" ] || ! printf '%s  %s\n' "$checksum" "$output" | sha256sum -c - >/dev/null 2>&1; then
    rm -f "$output"
    curl -fL --retry 5 --retry-delay 2 "$url" -o "$output"
  fi
  printf '%s  %s\n' "$checksum" "$output" | sha256sum -c -
}

install -d -m 0755 "$PREFIX/bin" "$PREFIX/downloads" "$PREFIX/toolchains"

download_verified \
  "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \
  "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \
  "$SHELLCHECK_SHA256"
rm -rf "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}"
tar -xJf "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \
  -C "$PREFIX/downloads"
install -m 0755 "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "$PREFIX/bin/shellcheck"

download_verified \
  "https://github.com/docker/compose/releases/download/v${COMPOSE_VERSION}/docker-compose-linux-x86_64" \
  "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" \
  "$COMPOSE_SHA256"
install -m 0755 "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" "$PREFIX/bin/docker-compose"

download_verified \
  "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" \
  "$PREFIX/downloads/go${GO_VERSION}.linux-amd64.tar.gz" \
  "$GO_SHA256"
rm -rf "$PREFIX/toolchains/go"
tar -xzf "$PREFIX/downloads/go${GO_VERSION}.linux-amd64.tar.gz" -C "$PREFIX/toolchains"

download_verified \
  "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" \
  "$PREFIX/downloads/node-v${NODE_VERSION}-linux-x64.tar.xz" \
  "$NODE_SHA256"
rm -rf "$PREFIX/toolchains/node"
mkdir -p "$PREFIX/toolchains/node"
tar -xJf "$PREFIX/downloads/node-v${NODE_VERSION}-linux-x64.tar.xz" \
  -C "$PREFIX/toolchains/node" --strip-components=1
PATH="$PREFIX/toolchains/node/bin:$PATH" \
  "$PREFIX/toolchains/node/bin/npm" install --global --prefix "$PREFIX/toolchains/node" \
  --ignore-scripts "pnpm@${PNPM_VERSION}"

download_verified \
  "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" \
  "$PREFIX/downloads/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" \
  "$TRIVY_SHA256"
tar -xzf "$PREFIX/downloads/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" -C "$PREFIX/bin" trivy
chmod 0755 "$PREFIX/bin/trivy"
GOBIN="$PREFIX/bin" GOPROXY="${GO_MODULE_PROXY:-https://goproxy.cn,direct}" \
  "$PREFIX/toolchains/go/bin/go" install golang.org/x/vuln/cmd/govulncheck@v1.6.0
chown -R root:root "$PREFIX"
chmod -R go-w "$PREFIX"

require_free_space "$MIN_FREE_GIB" "runner image pull"
docker pull "$RUNNER_IMAGE_SOURCE"
runner_runtime_image=$(docker image inspect --format '{{.Id}}' "$RUNNER_IMAGE_SOURCE")
printf '%s\n' "$runner_runtime_image" | grep -Eq '^sha256:[0-9a-f]{64}$' || \
  fail "pinned runner image did not resolve to an immutable local image ID"
require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed runner image pull"
docker volume create "$RUNNER_VOLUME" >/dev/null

# The registered runner state and the nested rootless daemon cache share one
# dedicated volume. Persisting the pinned job image lets validation finish
# before registration and avoids racing the first queued workflow for layers.
docker run --rm --pull=never --user 0:0 \
  --volume "$RUNNER_VOLUME:/data" \
  --entrypoint /bin/sh "$runner_runtime_image" \
  -ec 'chown 1000:1000 /data && chmod 0700 /data'

install -d -m 0755 /etc/easyai-gateway-ci-v2
if [ -n "$NESTED_DOCKER_REGISTRY_MIRROR" ]; then
  printf '{"data-root":"/data/docker","registry-mirrors":["%s"]}\n' "$NESTED_DOCKER_REGISTRY_MIRROR" \
    > /etc/easyai-gateway-ci-v2/daemon.json
else
  printf '{"data-root":"/data/docker"}\n' > /etc/easyai-gateway-ci-v2/daemon.json
fi
chmod 0644 /etc/easyai-gateway-ci-v2/daemon.json

# Docker's official rootless DinD baseline is --privileged. Gitea's example
# additionally names an AppArmor profile, but Docker does not create that host
# profile. Enable it only when this daemon proves it can start the pinned image
# with the profile; otherwise leave the option empty.
RUNNER_SECURITY_OPTIONS=
if docker run --rm --pull=never --privileged \
  --security-opt apparmor=rootlesskit \
  --entrypoint /bin/true "$runner_runtime_image" >/dev/null 2>&1; then
  RUNNER_SECURITY_OPTIONS="--security-opt=apparmor=rootlesskit"
elif [ -r /proc/sys/kernel/apparmor_restrict_unprivileged_userns ] && \
  [ "$(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns)" = "1" ]; then
  fail "restricted AppArmor user namespaces require the rootlesskit profile (install it with the host Docker packages)"
fi

# Exercise the actual daemon startup before registration. This probe has no
# token, state volume, host bind mount, or socket mount.
PROBE_CONTAINER="easyai-gateway-ci-v2-probe-$$"
cleanup_probe() {
  docker rm --force "$PROBE_CONTAINER" >/dev/null 2>&1 || true
}
trap 'cleanup_probe' EXIT
trap 'cleanup_probe; exit 129' HUP
trap 'cleanup_probe; exit 130' INT
trap 'cleanup_probe; exit 143' TERM
if [ -n "$RUNNER_SECURITY_OPTIONS" ]; then
  set -- "$RUNNER_SECURITY_OPTIONS"
else
  set --
fi
docker run --detach --rm --pull=never \
  --name "$PROBE_CONTAINER" --privileged \
  --cpus "$RUNNER_CPUS" --memory "$RUNNER_MEMORY" \
  --memory-swap "$RUNNER_MEMORY_SWAP" --pids-limit "$RUNNER_PIDS_LIMIT" "$@" \
  --volume "$RUNNER_VOLUME:/data" \
  --volume /etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro \
  --volume "$PREFIX:$PREFIX:ro" \
  --env DOCKERD_ROOTLESS_ROOTLESSKIT_NET=slirp4netns \
  --env DOCKERD_ROOTLESS_ROOTLESSKIT_MTU=65520 \
  --env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns \
  --entrypoint dockerd-entrypoint.sh "$runner_runtime_image" >/dev/null
attempt=0
until docker exec "$PROBE_CONTAINER" docker info >/dev/null 2>&1; do
  attempt=$((attempt + 1))
  if [ "$attempt" -ge 30 ] || \
    [ "$(docker inspect --format '{{.State.Running}}' "$PROBE_CONTAINER" 2>/dev/null || true)" != "true" ]; then
    docker logs "$PROBE_CONTAINER" 2>&1 || true
    fail "pinned rootless Docker-in-Docker probe did not become ready"
  fi
  sleep 2
done
[ "$(docker exec "$PROBE_CONTAINER" id -u)" = "1000" ] || \
  fail "rootless Docker-in-Docker probe is not uid 1000"
[ "$(docker inspect --format '{{.HostConfig.Memory}}:{{.HostConfig.MemorySwap}}:{{.HostConfig.NanoCpus}}:{{.HostConfig.PidsLimit}}' "$PROBE_CONTAINER")" = \
  '2147483648:3221225472:2000000000:1024' ] || \
  fail "rootless Docker-in-Docker probe does not have the required aggregate resource limits"
docker exec "$PROBE_CONTAINER" docker info --format '{{range .SecurityOptions}}{{println .}}{{end}}' | \
  grep -Fq 'name=rootless' || fail "rootless Docker-in-Docker probe is not rootless"
[ "$(docker exec "$PROBE_CONTAINER" docker info --format '{{.DockerRootDir}}')" = /data/docker ] || \
  fail "rootless Docker-in-Docker probe is not using its persistent dedicated data root"

# Complete every slow and capability-sensitive validation before registration;
# otherwise a queued workflow can race the installer for the same image layers.
docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"
outer_sentinel_pid=$(docker exec "$PROBE_CONTAINER" /bin/sh -c \
  'sleep 300 >/dev/null 2>&1 & printf "%s\n" "$!"')
case $outer_sentinel_pid in
  '' | *[!0-9]*) fail "could not create the outer PID namespace sentinel" ;;
esac
if ! docker exec "$PROBE_CONTAINER" docker run --rm --pull=never \
  --pid=host --entrypoint /bin/sh "$JOB_IMAGE" \
  -ec 'test ! -e "/proc/$1"' sh "$outer_sentinel_pid"; then
  docker exec "$PROBE_CONTAINER" kill "$outer_sentinel_pid" >/dev/null 2>&1 || true
  fail "nested --pid=host can see the outer runner PID namespace"
fi
docker exec "$PROBE_CONTAINER" kill "$outer_sentinel_pid" >/dev/null 2>&1 || true
docker exec "$PROBE_CONTAINER" docker run --rm --pull=never \
  --volume "$PREFIX:$PREFIX:ro" \
  --entrypoint /bin/sh "$JOB_IMAGE" -ec '
    export PATH=/opt/easyai-gateway-ci/bin:/opt/easyai-gateway-ci/toolchains/go/bin:/opt/easyai-gateway-ci/toolchains/node/bin:$PATH
    go version
    node --version
    pnpm --version
    docker-compose version
    shellcheck --version
    trivy --version
    govulncheck -version
  '
cleanup_probe
trap - EXIT HUP INT TERM

install -m 0644 "$ROOT/deploy/ci/act-runner-v2-config.yaml" /etc/easyai-gateway-ci-v2/config.yaml
umask 077
printf 'RUNNER_SECURITY_OPTIONS="%s"\nRUNNER_IMAGE_REF="%s"\n' \
  "$RUNNER_SECURITY_OPTIONS" "$runner_runtime_image" \
  > /etc/easyai-gateway-ci-v2/runner.env
install -m 0644 "$ROOT/deploy/ci/easyai-gateway-ci-v2-runner.service" \
  /etc/systemd/system/easyai-gateway-ci-v2-runner.service

runner_is_registered() {
  docker run --rm --pull=never \
    --volume "$RUNNER_VOLUME:/data" \
    --entrypoint /bin/sh "$runner_runtime_image" \
    -ec 'test -s /data/.runner'
}

if ! runner_is_registered; then
  [ -n "${RUNNER_REGISTRATION_TOKEN:-}" ] || \
    fail "RUNNER_REGISTRATION_TOKEN is required for first registration"
  docker rm --force easyai-gateway-ci-v2-register >/dev/null 2>&1 || true
  printf '%s\n%s\n%s\n' "$GITEA_INSTANCE_URL" "$RUNNER_REGISTRATION_TOKEN" "$RUNNER_NAME" | \
    docker run --rm --interactive --pull=never \
    --name easyai-gateway-ci-v2-register \
    --volume "$RUNNER_VOLUME:/data" \
    --volume /etc/easyai-gateway-ci-v2/config.yaml:/config.yaml:ro \
    --env HOME=/data \
    --entrypoint /usr/local/bin/gitea-runner \
    "$runner_runtime_image" --config /config.yaml register
fi
unset RUNNER_REGISTRATION_TOKEN
docker run --rm --pull=never \
  --volume "$RUNNER_VOLUME:/data" \
  --entrypoint /bin/sh "$runner_runtime_image" \
  -ec 'test -s /data/.runner && chmod 0600 /data/.runner'

RUNNER_VALIDATED=0
cleanup_unvalidated_runner() {
  if [ "$RUNNER_VALIDATED" -ne 1 ]; then
    systemctl disable --now easyai-gateway-ci-v2-runner.service >/dev/null 2>&1 || true
    docker rm --force "$RUNNER_CONTAINER" >/dev/null 2>&1 || true
  fi
}
trap 'cleanup_unvalidated_runner' EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
systemctl daemon-reload
systemctl enable --now easyai-gateway-ci-v2-runner.service

attempt=0
until docker exec "$RUNNER_CONTAINER" docker info >/dev/null 2>&1; do
  attempt=$((attempt + 1))
  if [ "$attempt" -ge 30 ]; then
    systemctl --no-pager --full status easyai-gateway-ci-v2-runner.service || true
    docker logs "$RUNNER_CONTAINER" 2>&1 || true
    fail "rootless Docker daemon did not become ready"
  fi
  sleep 2
done

attempt=0
until docker logs "$RUNNER_CONTAINER" 2>&1 | grep -Fq 'declare successfully'; do
  attempt=$((attempt + 1))
  if [ "$attempt" -ge 30 ] || \
    [ "$(docker inspect --format '{{.State.Running}}' "$RUNNER_CONTAINER" 2>/dev/null || true)" != "true" ]; then
    docker logs "$RUNNER_CONTAINER" 2>&1 || true
    fail "Gitea runner did not declare itself successfully"
  fi
  sleep 2
done

[ "$(docker inspect --format '{{.Config.User}}' "$RUNNER_CONTAINER")" = "rootless" ] || \
  fail "outer runner container is not using the rootless image user"
[ "$(docker inspect --format '{{.HostConfig.Privileged}}' "$RUNNER_CONTAINER")" = "true" ] || \
  fail "outer rootless DinD container is not privileged"
[ "$(docker inspect --format '{{.HostConfig.Memory}}:{{.HostConfig.MemorySwap}}:{{.HostConfig.NanoCpus}}:{{.HostConfig.PidsLimit}}' "$RUNNER_CONTAINER")" = \
  '2147483648:3221225472:2000000000:1024' ] || \
  fail "outer rootless DinD container does not have the required aggregate resource limits"
if docker inspect --format '{{range .Mounts}}{{println .Source}}{{end}}' "$RUNNER_CONTAINER" | \
  grep -Eq '/var/run/docker\.sock|/run/docker\.sock'; then
  fail "host Docker socket was mounted into the runner"
fi
docker exec "$RUNNER_CONTAINER" docker info --format '{{range .SecurityOptions}}{{println .}}{{end}}' | \
  grep -Fq 'name=rootless' || fail "nested Docker daemon is not rootless"
[ "$(docker exec "$RUNNER_CONTAINER" docker info --format '{{.DockerRootDir}}')" = /data/docker ] || \
  fail "nested Docker daemon is not using its persistent dedicated data root"
docker exec "$RUNNER_CONTAINER" docker image inspect "$JOB_IMAGE" >/dev/null || \
  fail "validated pinned job image is unavailable after runner activation"
if ! require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed CI runner installation"; then
  fail "CI runner was stopped because the post-install disk reserve was not met"
fi
systemctl --no-pager --full status easyai-gateway-ci-v2-runner.service
RUNNER_VALIDATED=1
trap - EXIT HUP INT TERM