ci: validate runner before registration

2026-07-17 15:08:37 +08:00 · 2026-07-17 15:08:37 +08:00 · 06f95c5d86
commit 06f95c5d86
parent 1f8c5a3d03
2 changed files with 50 additions and 32 deletions
--- a/scripts/provision-ci-runner.sh
+++ b/scripts/provision-ci-runner.sh
@ -195,12 +195,20 @@ printf '%s\n' "$runner_runtime_image" | grep -Eq '^sha256:[0-9a-f]{64}$' || \
 require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed runner image pull"
 docker volume create "$RUNNER_VOLUME" >/dev/null

+# The registered runner state and the nested rootless daemon cache share one
+# dedicated volume. Persisting the pinned job image lets validation finish
+# before registration and avoids racing the first queued workflow for layers.
+docker run --rm --pull=never --user 0:0 \
+  --volume "$RUNNER_VOLUME:/data" \
+  --entrypoint /bin/sh "$runner_runtime_image" \
+  -ec 'chown 1000:1000 /data && chmod 0700 /data'
+
 install -d -m 0755 /etc/easyai-gateway-ci-v2
 if [ -n "$NESTED_DOCKER_REGISTRY_MIRROR" ]; then
-  printf '{"registry-mirrors":["%s"]}\n' "$NESTED_DOCKER_REGISTRY_MIRROR" \
+  printf '{"data-root":"/data/docker","registry-mirrors":["%s"]}\n' "$NESTED_DOCKER_REGISTRY_MIRROR" \
    > /etc/easyai-gateway-ci-v2/daemon.json
 else
-  printf '{}\n' > /etc/easyai-gateway-ci-v2/daemon.json
+  printf '{"data-root":"/data/docker"}\n' > /etc/easyai-gateway-ci-v2/daemon.json
 fi
 chmod 0644 /etc/easyai-gateway-ci-v2/daemon.json

@ -235,7 +243,9 @@ else
 fi
 docker run --detach --rm --pull=never \
  --name "$PROBE_CONTAINER" --privileged --pids-limit 4096 "$@" \
+  --volume "$RUNNER_VOLUME:/data" \
  --volume /etc/easyai-gateway-ci-v2/daemon.json:/home/rootless/.config/docker/daemon.json:ro \
+  --volume "$PREFIX:$PREFIX:ro" \
  --env DOCKERD_ROOTLESS_ROOTLESSKIT_NET=slirp4netns \
  --env DOCKERD_ROOTLESS_ROOTLESSKIT_MTU=65520 \
  --env DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS=--pidns \
@ -254,6 +264,29 @@ done
  fail "rootless Docker-in-Docker probe is not uid 1000"
 docker exec "$PROBE_CONTAINER" docker info --format '{{range .SecurityOptions}}{{println .}}{{end}}' | \
  grep -Fq 'name=rootless' || fail "rootless Docker-in-Docker probe is not rootless"
+[ "$(docker exec "$PROBE_CONTAINER" docker info --format '{{.DockerRootDir}}')" = /data/docker ] || \
+  fail "rootless Docker-in-Docker probe is not using its persistent dedicated data root"
+
+# Complete every slow and capability-sensitive validation before registration;
+# otherwise a queued workflow can race the installer for the same image layers.
+docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"
+outer_pidns=$(docker exec "$PROBE_CONTAINER" readlink /proc/1/ns/pid)
+inner_host_pidns=$(docker exec "$PROBE_CONTAINER" docker run --rm --pull=never \
+  --pid=host --entrypoint readlink "$JOB_IMAGE" /proc/1/ns/pid)
+[ "$outer_pidns" != "$inner_host_pidns" ] || \
+  fail "nested --pid=host can see the outer runner PID namespace"
+docker exec "$PROBE_CONTAINER" docker run --rm --pull=never \
+  --volume "$PREFIX:$PREFIX:ro" \
+  --entrypoint /bin/sh "$JOB_IMAGE" -ec '
+    export PATH=/opt/easyai-gateway-ci/bin:/opt/easyai-gateway-ci/toolchains/go/bin:/opt/easyai-gateway-ci/toolchains/node/bin:$PATH
+    go version
+    node --version
+    pnpm --version
+    docker-compose version
+    shellcheck --version
+    trivy --version
+    govulncheck -version
+  '
 cleanup_probe
 trap - EXIT HUP INT TERM

@ -265,13 +298,6 @@ printf 'RUNNER_SECURITY_OPTIONS="%s"\nRUNNER_IMAGE_REF="%s"\n' \
 install -m 0644 "$ROOT/deploy/ci/easyai-gateway-ci-v2-runner.service" \
  /etc/systemd/system/easyai-gateway-ci-v2-runner.service

-# The image runs as uid 1000. Initialize only the dedicated named volume as
-# root, then register and operate the runner as that image's rootless user.
-docker run --rm --pull=never --user 0:0 \
-  --volume "$RUNNER_VOLUME:/data" \
-  --entrypoint /bin/sh "$runner_runtime_image" \
-  -ec 'chown 1000:1000 /data && chmod 0700 /data'
-
 runner_is_registered() {
  docker run --rm --pull=never \
    --volume "$RUNNER_VOLUME:/data" \
@ -344,27 +370,10 @@ if docker inspect --format '{{range .Mounts}}{{println .Source}}{{end}}' "$RUNNE
 fi
 docker exec "$RUNNER_CONTAINER" docker info --format '{{range .SecurityOptions}}{{println .}}{{end}}' | \
  grep -Fq 'name=rootless' || fail "nested Docker daemon is not rootless"
-
-# Pre-pull through the nested daemon to prove the digest is reachable and make
-# the first real CI job independent of an additional large image pull.
-docker exec "$RUNNER_CONTAINER" docker pull "$JOB_IMAGE"
-outer_pidns=$(docker exec "$RUNNER_CONTAINER" readlink /proc/1/ns/pid)
-inner_host_pidns=$(docker exec "$RUNNER_CONTAINER" docker run --rm --pull=never \
-  --pid=host --entrypoint readlink "$JOB_IMAGE" /proc/1/ns/pid)
-[ "$outer_pidns" != "$inner_host_pidns" ] || \
-  fail "nested --pid=host can see the outer runner PID namespace"
-docker exec "$RUNNER_CONTAINER" docker run --rm --pull=never \
-  --volume /opt/easyai-gateway-ci:/opt/easyai-gateway-ci:ro \
-  --entrypoint /bin/sh "$JOB_IMAGE" -ec '
-    export PATH=/opt/easyai-gateway-ci/bin:/opt/easyai-gateway-ci/toolchains/go/bin:/opt/easyai-gateway-ci/toolchains/node/bin:$PATH
-    go version
-    node --version
-    pnpm --version
-    docker-compose version
-    shellcheck --version
-    trivy --version
-    govulncheck -version
-  '
+[ "$(docker exec "$RUNNER_CONTAINER" docker info --format '{{.DockerRootDir}}')" = /data/docker ] || \
+  fail "nested Docker daemon is not using its persistent dedicated data root"
+docker exec "$RUNNER_CONTAINER" docker image inspect "$JOB_IMAGE" >/dev/null || \
+  fail "validated pinned job image is unavailable after runner activation"
 if ! require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed CI runner installation"; then
  fail "CI runner was stopped because the post-install disk reserve was not met"
 fi
--- a/tests/ci/pipeline-test.sh
+++ b/tests/ci/pipeline-test.sh
@ -190,6 +190,7 @@ grep -Fq 'RUNNER_IMAGE_REF="%s"' "$provision"
 grep -Fq "JOB_IMAGE='$job_image'" "$provision"
 grep -Fq 'RUNNER_LABEL="easyai-gateway-ci-unprivileged-v2:docker://$JOB_IMAGE"' "$provision"
 grep -Fq 'docker volume create "$RUNNER_VOLUME"' "$provision"
+grep -Fq '"data-root":"/data/docker"' "$provision"
 grep -Fq -- '--security-opt apparmor=rootlesskit' "$provision"
 grep -Fq 'RUNNER_SECURITY_OPTIONS="--security-opt=apparmor=rootlesskit"' "$provision"
 grep -Fq 'RUNNER_SECURITY_OPTIONS=' "$provision"
@ -225,8 +226,16 @@ grep -Fq 'require_free_space "$MIN_FREE_GIB" "runner image pull"' "$provision"
 grep -Fq 'require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed CI runner installation"' "$provision"
 grep -Fq 'CI_RUNNER_MIN_FREE_GIB cannot be lower than 8' "$provision"
 grep -Fq 'CI_RUNNER_MIN_POST_INSTALL_FREE_GIB cannot be lower than 4' "$provision"
-grep -Fq 'docker exec "$RUNNER_CONTAINER" docker pull "$JOB_IMAGE"' "$provision"
-grep -Fq 'docker exec "$RUNNER_CONTAINER" docker run --rm --pull=never' "$provision"
+grep -Fq 'docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"' "$provision"
+grep -Fq 'docker exec "$PROBE_CONTAINER" docker run --rm --pull=never' "$provision"
+grep -Fq -- '--volume "$RUNNER_VOLUME:/data"' "$provision"
+grep -Fq -- '--volume "$PREFIX:$PREFIX:ro"' "$provision"
+probe_pull_line=$(grep -nF 'docker exec "$PROBE_CONTAINER" docker pull "$JOB_IMAGE"' "$provision" | cut -d: -f1)
+registration_line=$(grep -nF 'if ! runner_is_registered; then' "$provision" | cut -d: -f1)
+[[ -n $probe_pull_line && -n $registration_line && $probe_pull_line -lt $registration_line ]] || {
+  echo 'pinned job image and isolation probes must complete before runner registration' >&2
+  exit 1
+}
 grep -Fq 'nested --pid=host can see the outer runner PID namespace' "$provision"
 grep -Fq 'chown -R root:root "$PREFIX"' "$provision"
 grep -Fq 'chmod -R go-w "$PREFIX"' "$provision"