feat(deploy): 增加三节点 K3s 高可用迁移能力

新增 WireGuard 全互联、三 server embedded-etcd K3s、CloudNativePG 双实例、Barman OSS 备份、双 NGINX、Kubernetes Secret/RBAC 与本地旧文件按严格 24 小时清理。新增维护窗口数据迁移、digest 固定滚动发布、应用回滚、跨节点文件 E2E、节点和数据库故障演练、CNPG 恢复与 etcd 快照验收脚本；洛杉矶仅作为带 NoSchedule 污点的仲裁节点。所有生产 Secret 只在执行时从 0600 本地环境和旧生产容器导入，仓库不保存凭据；公网入口保持人工 DNS 故障切换边界。验证：bash -n、ShellCheck、kubectl kustomize、Node 语法检查、Secret 扫描、OSS put/head/delete 实测。
2026-07-28 04:37:31 +08:00
parent edf7e66941
commit 971540a2a4
38 changed files with 3386 additions and 3 deletions
@@ -104,14 +104,14 @@ docker compose -f docker-compose.yml restart web

 本仓库没有 Gitea Actions、Webhook、Tag、`main` Push、轮询或定时发布。`main` 不使用受保护分支限制；commit、push 和 Tag 都不会构建镜像或更新生产。

-生产发布由 Agent 在用户明确指令下分两次执行。第一步在本机构建 `linux/amd64` 镜像、运行临时 PostgreSQL + simulation API 冒烟、推送完整 SHA Tag，并生成带内容完整性校验的 digest-pinned manifest；该命令不会修改生产：
+生产发布由 Agent 在用户明确指令下按两个阶段连续执行。第一步在本机构建 `linux/amd64` 镜像、运行临时 PostgreSQL + simulation API 冒烟、推送完整 SHA Tag，并生成带内容完整性校验的 digest-pinned manifest；该命令不会修改生产：

 ```bash
 docker login --username=<your-aliyun-account> registry.cn-shanghai.aliyuncs.com
 ./scripts/publish-release-images.sh --components auto
 ```

-Agent 必须报告 `dist/releases/<SHA>.json` 后停止。用户再次明确确认上线后，才执行第二步：
+用户要求“发布”“上线”或“部署到线上”时，Agent 核验并报告 `dist/releases/<SHA>.json`、组件和 digest 后直接执行第二步，无需再次确认；用户仅要求构建或推送镜像时才在第一步后停止：

 ```bash
 ./scripts/deploy-production-release.sh dist/releases/<SHA>.json
@@ -124,7 +124,10 @@ Agent 必须报告 `dist/releases/<SHA>.json` 后停止。用户再次明确确
 ./scripts/deploy-production-release.sh --rollback <历史完整 Git SHA>
 ```

-完整安装、验证、失败处理和停用旧自动化步骤见[人工生产发布运行手册](docs/runbooks/production-ci-cd.md)，决策背景见 [ADR-003](docs/decisions/003-manual-agent-release.md)。
+完整安装、验证、失败处理和停用旧自动化步骤见[人工生产发布运行手册](docs/runbooks/production-ci-cd.md)；
+三节点 K3s、CloudNativePG、双 NGINX、备份恢复和跨节点文件验收见
+[K3s 高可用运行手册](docs/operations/k3s-ha-runbook.md)。决策背景见
+[ADR-003](docs/decisions/003-manual-agent-release.md)。

 Compose 默认使用独立容器数据库 `postgres:18-alpine`，数据卷会保留在 `postgres_data` 和 `api_data`。为避免本地开发 `.env` 中的 `localhost` 数据库地址污染容器部署，compose 使用 `AI_GATEWAY_COMPOSE_*` 变量作为容器部署专用覆盖，例如：

@@ -0,0 +1,279 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+[[ $(id -u) -eq 0 ]] || { echo 'run as root' >&2; exit 1; }
+
+config_file=${AI_GATEWAY_CLUSTER_RELEASE_CONFIG:-/etc/easyai-ai-gateway-cluster-release.conf}
+manifest_tool=${AI_GATEWAY_RELEASE_MANIFEST_TOOL:-/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs}
+[[ -f $config_file && ! -L $config_file ]] || { echo 'missing cluster release config' >&2; exit 1; }
+[[ -f $manifest_tool && ! -L $manifest_tool ]] || { echo 'missing release manifest validator' >&2; exit 1; }
+# shellcheck source=/dev/null
+source "$config_file"
+
+: "${RELEASES_DIR:?}"
+: "${NAMESPACE:=easyai}"
+: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}"
+: "${K3S_BIN:=/usr/local/bin/k3s}"
+[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || {
+  echo 'invalid cluster release configuration' >&2
+  exit 1
+}
+[[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || {
+  echo 'invalid public base URL' >&2
+  exit 1
+}
+for command in "$K3S_BIN" curl flock install jq node; do
+  command -v "$command" >/dev/null 2>&1 || { echo "$command is required" >&2; exit 1; }
+done
+
+kubectl=("$K3S_BIN" kubectl)
+install -d -m 0700 "$RELEASES_DIR"
+exec 9>"$RELEASES_DIR/.release.lock"
+flock -n 9 || { echo 'another cluster release operation is running' >&2; exit 1; }
+
+manifest_get() {
+  node "$manifest_tool" get "$1" "$2"
+}
+
+current_to_file() {
+  local output=$1
+  "${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \
+    -o jsonpath='{.data.manifest\.json}' >"$output"
+  [[ -s $output ]]
+  node "$manifest_tool" validate "$output" >/dev/null
+}
+
+wait_for_url() {
+  local label=$1
+  local url=$2
+  local expected=${3:-}
+  local body
+  for _ in $(seq 1 60); do
+    if body=$(curl -fsS --max-time 5 "$url" 2>/dev/null); then
+      if [[ -z $expected || $body == *"$expected"* ]]; then
+        echo "[cluster-release] verified $label"
+        return 0
+      fi
+    fi
+    sleep 2
+  done
+  echo "release probe failed: $label ($url)" >&2
+  return 1
+}
+
+verify_site() {
+  local site=$1
+  local address
+  case $site in
+    hongkong) address=10.77.0.2 ;;
+    ningbo) address=10.77.0.1 ;;
+    *) return 1 ;;
+  esac
+  wait_for_url "$site API health" "http://$address:30088/api/v1/healthz" easyai-ai-gateway
+  wait_for_url "$site API readiness" "http://$address:30088/api/v1/readyz" '"ok":true'
+  wait_for_url "$site Web" "http://$address:30178/" 'EasyAI AI Gateway'
+}
+
+rollout_site() {
+  local site=$1
+  local api_image=$2
+  local web_image=$3
+  local api_changed=$4
+  local web_changed=$5
+  if [[ $api_changed == true ]]; then
+    "${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \
+      "api=$api_image"
+    "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s
+  fi
+  if [[ $web_changed == true ]]; then
+    "${kubectl[@]}" set image "deployment/easyai-web-$site" -n "$NAMESPACE" \
+      "web=$web_image"
+    "${kubectl[@]}" rollout status "deployment/easyai-web-$site" -n "$NAMESPACE" --timeout=300s
+  fi
+  verify_site "$site"
+}
+
+run_backup() {
+  local source_sha=$1
+  local backup=easyai-pre-release-${source_sha:0:12}
+  cat <<EOF | "${kubectl[@]}" apply -f - >/dev/null
+apiVersion: postgresql.cnpg.io/v1
+kind: Backup
+metadata:
+  name: $backup
+  namespace: $NAMESPACE
+spec:
+  cluster:
+    name: easyai-postgres
+  method: plugin
+  pluginConfiguration:
+    name: barman-cloud.cloudnative-pg.io
+EOF
+  for _ in $(seq 1 120); do
+    phase=$("${kubectl[@]}" get backup "$backup" -n "$NAMESPACE" \
+      -o jsonpath='{.status.phase}' 2>/dev/null || true)
+    [[ ${phase,,} == completed ]] && return 0
+    [[ ${phase,,} == failed ]] && break
+    sleep 5
+  done
+  echo "pre-migration backup failed: $backup phase=${phase:-unknown}" >&2
+  return 1
+}
+
+run_migrator() {
+  local source_sha=$1
+  local api_image=$2
+  local job=easyai-migrator-${source_sha:0:12}
+  "${kubectl[@]}" delete job "$job" -n "$NAMESPACE" \
+    --ignore-not-found=true --wait=true >/dev/null
+  cat <<EOF | "${kubectl[@]}" apply -f - >/dev/null
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: $job
+  namespace: $NAMESPACE
+  labels:
+    easyai.io/release: $source_sha
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 86400
+  template:
+    spec:
+      restartPolicy: Never
+      imagePullSecrets:
+        - name: easyai-registry
+      nodeSelector:
+        easyai.io/site: ningbo
+      containers:
+        - name: migrator
+          image: $api_image
+          command: ["/app/easyai-ai-gateway-migrate"]
+          envFrom:
+            - secretRef:
+                name: easyai-ai-gateway-runtime
+EOF
+  "${kubectl[@]}" wait --for=condition=complete "job/$job" \
+    -n "$NAMESPACE" --timeout=300s
+}
+
+record_current() {
+  local manifest=$1
+  local action=$2
+  local started_at=$3
+  local source_sha recorded temporary
+  source_sha=$(manifest_get "$manifest" sourceSha)
+  recorded=$RELEASES_DIR/$source_sha.json
+  temporary=$(mktemp "$RELEASES_DIR/.recorded.XXXXXX")
+  rm -f -- "$temporary"
+  RELEASE_DEPLOYED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \
+  RELEASE_DEPLOY_DURATION_SECONDS=$(( $(date +%s) - started_at )) \
+  RELEASE_DEPLOY_ACTION=$action \
+    node "$manifest_tool" record-deployment "$manifest" "$temporary" >/dev/null
+  install -m 0600 "$temporary" "$recorded"
+  "${kubectl[@]}" create configmap easyai-gateway-release -n "$NAMESPACE" \
+    --from-file=manifest.json="$recorded" --dry-run=client -o yaml |
+    "${kubectl[@]}" apply -f - >/dev/null
+  rm -f -- "$temporary"
+}
+
+activate_manifest() {
+  local manifest=$1
+  local action=${2:-deploy}
+  local source_sha base_sha api_image web_image migrations_changed components
+  local api_changed=false web_changed=false current_file previous_api previous_web started_at
+  [[ -f $manifest && ! -L $manifest ]] || { echo 'manifest must be a regular file' >&2; return 1; }
+  node "$manifest_tool" validate "$manifest" >/dev/null
+  source_sha=$(manifest_get "$manifest" sourceSha)
+  base_sha=$(manifest_get "$manifest" baseReleaseSha)
+  api_image=$(manifest_get "$manifest" images.api)
+  web_image=$(manifest_get "$manifest" images.web)
+  migrations_changed=$(manifest_get "$manifest" migrationsChanged)
+  components=$(manifest_get "$manifest" components)
+  [[ $api_image =~ @sha256:[0-9a-f]{64}$ && $web_image =~ @sha256:[0-9a-f]{64}$ ]] || {
+    echo 'release images must be digest-pinned' >&2
+    return 1
+  }
+  [[ $components == *'"api"'* ]] && api_changed=true
+  [[ $components == *'"web"'* ]] && web_changed=true
+  if [[ $action == rollback ]]; then
+    api_changed=true
+    web_changed=true
+  fi
+
+  current_file=$(mktemp "$RELEASES_DIR/.current.XXXXXX")
+  if current_to_file "$current_file"; then
+    current_sha=$(manifest_get "$current_file" sourceSha)
+    previous_api=$(manifest_get "$current_file" images.api)
+    previous_web=$(manifest_get "$current_file" images.web)
+    if [[ $action == deploy && $current_sha != "$base_sha" ]]; then
+      echo "stale release manifest: production=$current_sha manifest_base=$base_sha" >&2
+      rm -f -- "$current_file"
+      return 1
+    fi
+  else
+    if [[ $action == deploy && -n $base_sha ]]; then
+      echo 'cluster release status is unavailable for a non-bootstrap manifest' >&2
+      rm -f -- "$current_file"
+      return 1
+    fi
+    previous_api=
+    previous_web=
+  fi
+  started_at=$(date +%s)
+
+  if [[ $action == deploy && $migrations_changed == true ]]; then
+    run_backup "$source_sha"
+    run_migrator "$source_sha" "$api_image"
+  fi
+
+  if ! rollout_site hongkong "$api_image" "$web_image" "$api_changed" "$web_changed" ||
+      ! rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" ||
+      ! wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then
+    if [[ -n $previous_api && -n $previous_web ]]; then
+      echo '[cluster-release] restoring previous application images' >&2
+      rollout_site hongkong "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true
+      rollout_site ningbo "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true
+    fi
+    rm -f -- "$current_file"
+    return 1
+  fi
+
+  record_current "$manifest" "$action" "$started_at"
+  rm -f -- "$current_file"
+  echo "production_release=PASS action=$action release=$source_sha"
+}
+
+case ${1:-} in
+  status)
+    [[ $# -eq 1 ]] || exit 64
+    status_file=$(mktemp "$RELEASES_DIR/.status.XXXXXX")
+    current_to_file "$status_file"
+    cat "$status_file"
+    rm -f -- "$status_file"
+    ;;
+  adopt)
+    [[ $# -eq 2 ]] || exit 64
+    node "$manifest_tool" validate "$2" >/dev/null
+    api_image=$(manifest_get "$2" images.api)
+    web_image=$(manifest_get "$2" images.web)
+    [[ $("${kubectl[@]}" get deployment easyai-api-ningbo -n "$NAMESPACE" \
+      -o jsonpath='{.spec.template.spec.containers[?(@.name=="api")].image}') == "$api_image" ]]
+    [[ $("${kubectl[@]}" get deployment easyai-web-ningbo -n "$NAMESPACE" \
+      -o jsonpath='{.spec.template.spec.containers[?(@.name=="web")].image}') == "$web_image" ]]
+    record_current "$2" deploy "$(date +%s)"
+    ;;
+  deploy)
+    [[ $# -eq 2 ]] || exit 64
+    activate_manifest "$2" deploy
+    ;;
+  rollback)
+    [[ $# -eq 2 && $2 =~ ^[0-9a-f]{40}$ ]] || exit 64
+    target=$RELEASES_DIR/$2.json
+    [[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; }
+    activate_manifest "$target" rollback
+    ;;
+  *)
+    echo 'usage: easyai-ai-gateway-cluster-release {status|adopt <manifest>|deploy <manifest>|rollback <sha>}' >&2
+    exit 64
+    ;;
+esac
@@ -0,0 +1,4 @@
+RELEASES_DIR=/var/lib/easyai-ai-gateway-cluster-release/releases
+NAMESPACE=easyai
+PUBLIC_BASE_URL=https://ai.51easyai.com
+K3S_BIN=/usr/local/bin/k3s
@@ -0,0 +1,39 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: easyai-ai-gateway-config
+  labels:
+    app.kubernetes.io/part-of: easyai-ai-gateway
+data:
+  APP_ENV: production
+  HTTP_ADDR: ":8088"
+  IDENTITY_MODE: hybrid
+  IDENTITY_SECRET_STORE: kubernetes
+  IDENTITY_KUBERNETES_NAMESPACE: easyai
+  IDENTITY_KUBERNETES_SECRET_NAME: easyai-gateway-security-events
+  IDENTITY_SECURITY_EVENTS_HEARTBEAT_INTERVAL_SECONDS: "60"
+  IDENTITY_SECURITY_EVENTS_STALE_AFTER_SECONDS: "180"
+  IDENTITY_SECURITY_EVENTS_CLOCK_SKEW_SECONDS: "60"
+  SERVER_MAIN_BASE_URL: http://10.77.0.1:3000
+  TASK_PROGRESS_CALLBACK_ENABLED: "true"
+  TASK_PROGRESS_CALLBACK_URL: http://10.77.0.1:3000/internal/platform/task-progress-callbacks
+  TASK_PROGRESS_CALLBACK_TIMEOUT_MS: "5000"
+  TASK_PROGRESS_CALLBACK_MAX_ATTEMPTS: "10"
+  AI_GATEWAY_TASK_CLEANUP_ENABLED: "false"
+  AI_GATEWAY_TASK_RETENTION_DAYS: "30"
+  AI_GATEWAY_TASK_ANALYSIS_RETENTION_DAYS: "7"
+  AI_GATEWAY_TASK_CLEANUP_INTERVAL_SECONDS: "300"
+  AI_GATEWAY_TASK_CLEANUP_BATCH_SIZE: "1000"
+  CORS_ALLOWED_ORIGIN: https://ai.51easyai.com
+  AI_GATEWAY_PUBLIC_BASE_URL: https://ai.51easyai.com
+  AI_GATEWAY_WEB_BASE_URL: https://ai.51easyai.com
+  AI_GATEWAY_GENERATED_STORAGE_DIR: /app/data/static/generated
+  AI_GATEWAY_UPLOADED_STORAGE_DIR: /app/data/static/uploaded
+  AI_GATEWAY_LOCAL_TEMP_ASSET_TTL_HOURS: "24"
+  AI_GATEWAY_LOCAL_RESULT_TTL_HOURS: "24"
+  AI_GATEWAY_LOCAL_RESULT_MIN_FREE_BYTES: "10737418240"
+  AI_GATEWAY_LOCAL_RESULT_MAX_BYTES: "268435456"
+  AI_GATEWAY_LOCAL_RESULT_MAX_TASK_BYTES: "536870912"
+  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED: "true"
+  AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT: "2048"
+  AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS: "5"
@@ -0,0 +1,439 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: easyai-api-ningbo
+  labels:
+    app.kubernetes.io/name: easyai-api
+    app.kubernetes.io/component: api
+    app.kubernetes.io/part-of: easyai-ai-gateway
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: easyai-api
+      easyai.io/site: ningbo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: easyai-api
+        app.kubernetes.io/component: api
+        app.kubernetes.io/part-of: easyai-ai-gateway
+        easyai.io/site: ningbo
+    spec:
+      serviceAccountName: easyai-ai-gateway
+      terminationGracePeriodSeconds: 90
+      nodeSelector:
+        easyai.io/site: ningbo
+        easyai.io/workload: "true"
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: api
+          image: easyai-api
+          imagePullPolicy: IfNotPresent
+          envFrom:
+            - configMapRef:
+                name: easyai-ai-gateway-config
+            - secretRef:
+                name: easyai-ai-gateway-runtime
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          ports:
+            - name: http
+              containerPort: 8088
+          readinessProbe:
+            httpGet:
+              path: /api/v1/readyz
+              port: http
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /api/v1/healthz
+              port: http
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 6
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "sleep 10"]
+          resources:
+            requests:
+              cpu: 250m
+              memory: 512Mi
+            limits:
+              cpu: "2"
+              memory: 2Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: application-data
+              mountPath: /app/data
+            - name: temporary-files
+              mountPath: /tmp
+      volumes:
+        - name: application-data
+          emptyDir:
+            sizeLimit: 5Gi
+        - name: temporary-files
+          emptyDir:
+            sizeLimit: 1Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: easyai-api-hongkong
+  labels:
+    app.kubernetes.io/name: easyai-api
+    app.kubernetes.io/component: api
+    app.kubernetes.io/part-of: easyai-ai-gateway
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: easyai-api
+      easyai.io/site: hongkong
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: easyai-api
+        app.kubernetes.io/component: api
+        app.kubernetes.io/part-of: easyai-ai-gateway
+        easyai.io/site: hongkong
+    spec:
+      serviceAccountName: easyai-ai-gateway
+      terminationGracePeriodSeconds: 90
+      nodeSelector:
+        easyai.io/site: hongkong
+        easyai.io/workload: "true"
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: api
+          image: easyai-api
+          imagePullPolicy: IfNotPresent
+          envFrom:
+            - configMapRef:
+                name: easyai-ai-gateway-config
+            - secretRef:
+                name: easyai-ai-gateway-runtime
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          ports:
+            - name: http
+              containerPort: 8088
+          readinessProbe:
+            httpGet:
+              path: /api/v1/readyz
+              port: http
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /api/v1/healthz
+              port: http
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 6
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "sleep 10"]
+          resources:
+            requests:
+              cpu: 250m
+              memory: 512Mi
+            limits:
+              cpu: "2"
+              memory: 2Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: application-data
+              mountPath: /app/data
+            - name: temporary-files
+              mountPath: /tmp
+      volumes:
+        - name: application-data
+          emptyDir:
+            sizeLimit: 5Gi
+        - name: temporary-files
+          emptyDir:
+            sizeLimit: 1Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: easyai-web-ningbo
+  labels:
+    app.kubernetes.io/name: easyai-web
+    app.kubernetes.io/component: web
+    app.kubernetes.io/part-of: easyai-ai-gateway
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: easyai-web
+      easyai.io/site: ningbo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: easyai-web
+        app.kubernetes.io/component: web
+        app.kubernetes.io/part-of: easyai-ai-gateway
+        easyai.io/site: ningbo
+    spec:
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 30
+      nodeSelector:
+        easyai.io/site: ningbo
+        easyai.io/workload: "true"
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 101
+        runAsGroup: 101
+        fsGroup: 101
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: web
+          image: easyai-web
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 80
+          readinessProbe:
+            httpGet:
+              path: /
+              port: http
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /
+              port: http
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 6
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 500m
+              memory: 256Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+              add: ["NET_BIND_SERVICE"]
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: nginx-cache
+              mountPath: /var/cache/nginx
+            - name: nginx-run
+              mountPath: /var/run
+      volumes:
+        - name: nginx-cache
+          emptyDir:
+            sizeLimit: 128Mi
+        - name: nginx-run
+          emptyDir:
+            sizeLimit: 16Mi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: easyai-web-hongkong
+  labels:
+    app.kubernetes.io/name: easyai-web
+    app.kubernetes.io/component: web
+    app.kubernetes.io/part-of: easyai-ai-gateway
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: easyai-web
+      easyai.io/site: hongkong
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: easyai-web
+        app.kubernetes.io/component: web
+        app.kubernetes.io/part-of: easyai-ai-gateway
+        easyai.io/site: hongkong
+    spec:
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 30
+      nodeSelector:
+        easyai.io/site: hongkong
+        easyai.io/workload: "true"
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 101
+        runAsGroup: 101
+        fsGroup: 101
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: web
+          image: easyai-web
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 80
+          readinessProbe:
+            httpGet:
+              path: /
+              port: http
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /
+              port: http
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 6
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 500m
+              memory: 256Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+              add: ["NET_BIND_SERVICE"]
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: nginx-cache
+              mountPath: /var/cache/nginx
+            - name: nginx-run
+              mountPath: /var/run
+      volumes:
+        - name: nginx-cache
+          emptyDir:
+            sizeLimit: 128Mi
+        - name: nginx-run
+          emptyDir:
+            sizeLimit: 16Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: api
+  labels:
+    app.kubernetes.io/name: easyai-api
+    app.kubernetes.io/component: api
+    app.kubernetes.io/part-of: easyai-ai-gateway
+spec:
+  type: NodePort
+  externalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: easyai-api
+  ports:
+    - name: http
+      port: 8088
+      targetPort: http
+      nodePort: 30088
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: web
+  labels:
+    app.kubernetes.io/name: easyai-web
+    app.kubernetes.io/component: web
+    app.kubernetes.io/part-of: easyai-ai-gateway
+spec:
+  type: NodePort
+  externalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: easyai-web
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
+      nodePort: 30178
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: easyai-api
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: easyai-api
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: easyai-web
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: easyai-web
@@ -0,0 +1,113 @@
+apiVersion: barmancloud.cnpg.io/v1
+kind: ObjectStore
+metadata:
+  name: easyai-postgres-oss
+  labels:
+    app.kubernetes.io/part-of: easyai-ai-gateway
+spec:
+  retentionPolicy: 14d
+  configuration:
+    destinationPath: s3://wangbo0808/easyai-ai-gateway/production/postgresql
+    endpointURL: https://oss-cn-shanghai.aliyuncs.com
+    s3Credentials:
+      accessKeyId:
+        name: easyai-oss-backup
+        key: ACCESS_KEY_ID
+      secretAccessKey:
+        name: easyai-oss-backup
+        key: ACCESS_SECRET_KEY
+    wal:
+      compression: gzip
+      maxParallel: 2
+    data:
+      compression: gzip
+      jobs: 2
+  instanceSidecarConfiguration:
+    env:
+      - name: AWS_DEFAULT_REGION
+        value: oss-cn-shanghai
+      - name: AWS_REQUEST_CHECKSUM_CALCULATION
+        value: when_required
+      - name: AWS_RESPONSE_CHECKSUM_VALIDATION
+        value: when_required
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: easyai-postgres
+  labels:
+    app.kubernetes.io/name: easyai-postgres
+    app.kubernetes.io/part-of: easyai-ai-gateway
+spec:
+  instances: 2
+  imageName: ghcr.io/cloudnative-pg/postgresql:18.4-standard-trixie@sha256:4587df73024408f5b2be9b4dd6ba2ccee8c9e5dc0c9a87c274c292291cc8a68c
+  imagePullPolicy: IfNotPresent
+  primaryUpdateStrategy: unsupervised
+  primaryUpdateMethod: switchover
+  failoverDelay: 0
+  switchoverDelay: 60
+  stopDelay: 90
+  smartShutdownTimeout: 30
+  bootstrap:
+    initdb:
+      database: easyai_ai_gateway
+      owner: easyai
+      secret:
+        name: easyai-postgres-app
+      postInitSQL:
+        - CREATE EXTENSION IF NOT EXISTS pgcrypto;
+  storage:
+    storageClass: local-path
+    size: 20Gi
+  walStorage:
+    storageClass: local-path
+    size: 5Gi
+  affinity:
+    enablePodAntiAffinity: true
+    podAntiAffinityType: required
+    topologyKey: easyai.io/site
+    nodeSelector:
+      easyai.io/database: "true"
+  resources:
+    requests:
+      cpu: 500m
+      memory: 1Gi
+    limits:
+      cpu: "2"
+      memory: 3Gi
+  postgresql:
+    synchronous:
+      method: any
+      number: 1
+      dataDurability: preferred
+    parameters:
+      archive_timeout: 60s
+      max_connections: "200"
+      shared_buffers: 512MB
+      wal_compression: "on"
+      wal_keep_size: 1GB
+  monitoring:
+    enablePodMonitor: false
+  plugins:
+    - name: barman-cloud.cloudnative-pg.io
+      isWALArchiver: true
+      parameters:
+        barmanObjectName: easyai-postgres-oss
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: easyai-postgres-daily
+  labels:
+    app.kubernetes.io/part-of: easyai-ai-gateway
+spec:
+  # Six-field cron, UTC: 18:30 UTC is 02:30 Asia/Shanghai.
+  schedule: "0 30 18 * * *"
+  backupOwnerReference: self
+  cluster:
+    name: easyai-postgres
+  method: plugin
+  pluginConfiguration:
+    name: barman-cloud.cloudnative-pg.io
+  immediate: false
+  suspend: false
@@ -0,0 +1,19 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: easyai
+resources:
+  - namespace.yaml
+  - service-account-rbac.yaml
+  - application-config.yaml
+  - application.yaml
+  - database.yaml
+  - network-policy.yaml
+images:
+  # Deployment tooling must replace both sentinel digests with release-manifest
+  # digests before server-side apply.
+  - name: easyai-api
+    newName: registry.cn-shanghai.aliyuncs.com/easyaigc/ai-gateway
+    digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
+  - name: easyai-web
+    newName: registry.cn-shanghai.aliyuncs.com/easyaigc/ai-gateway-web
+    digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: easyai
+  labels:
+    app.kubernetes.io/part-of: easyai-ai-gateway
+    pod-security.kubernetes.io/enforce: baseline
+    pod-security.kubernetes.io/audit: restricted
+    pod-security.kubernetes.io/warn: restricted
@@ -0,0 +1,23 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: easyai-application-ingress
+spec:
+  podSelector:
+    matchExpressions:
+      - key: app.kubernetes.io/name
+        operator: In
+        values: [easyai-api, easyai-web]
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: easyai
+        - ipBlock:
+            cidr: 10.77.0.0/24
+      ports:
+        - protocol: TCP
+          port: 8088
+        - protocol: TCP
+          port: 80
@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: easyai-ai-gateway
+  labels:
+    app.kubernetes.io/part-of: easyai-ai-gateway
+automountServiceAccountToken: true
+imagePullSecrets:
+  - name: easyai-registry
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: easyai-gateway-security-events
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["easyai-gateway-security-events"]
+    verbs: ["get", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: easyai-gateway-security-events
+subjects:
+  - kind: ServiceAccount
+    name: easyai-ai-gateway
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: easyai-gateway-security-events
@@ -0,0 +1,106 @@
+upstream easyai_gateway_api {
+  least_conn;
+  keepalive 64;
+  server 10.77.0.1:30088 max_fails=2 fail_timeout=5s;
+  server 10.77.0.2:30088 max_fails=2 fail_timeout=5s;
+}
+
+upstream easyai_gateway_web {
+  least_conn;
+  keepalive 32;
+  server 10.77.0.1:30178 max_fails=2 fail_timeout=5s;
+  server 10.77.0.2:30178 max_fails=2 fail_timeout=5s;
+}
+
+server {
+  listen 80;
+  listen [::]:80;
+  server_name ai.51easyai.com;
+  return 301 https://$host$request_uri;
+}
+
+server {
+  listen 443 ssl http2;
+  listen [::]:443 ssl http2;
+  server_name ai.51easyai.com;
+
+  ssl_certificate /etc/nginx/tls/ai.51easyai.com/fullchain.pem;
+  ssl_certificate_key /etc/nginx/tls/ai.51easyai.com/privkey.pem;
+  ssl_session_cache shared:easyai_gateway_tls:10m;
+  ssl_session_timeout 1d;
+  ssl_protocols TLSv1.2 TLSv1.3;
+
+  client_max_body_size 200m;
+  client_body_timeout 300s;
+
+  include /etc/nginx/easyai-ai-gateway/legacy-static.inc;
+
+  location = /api/v1/metrics {
+    return 404;
+  }
+
+  location ^~ /api/v1/ {
+    proxy_http_version 1.1;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto https;
+    proxy_set_header Connection "";
+    proxy_connect_timeout 2s;
+    proxy_send_timeout 3600s;
+    proxy_read_timeout 3600s;
+    proxy_request_buffering off;
+    proxy_buffering off;
+    proxy_next_upstream error timeout http_502 http_503 http_504;
+    proxy_next_upstream_tries 2;
+    proxy_pass http://easyai_gateway_api;
+  }
+
+  location /gateway-api/ {
+    proxy_http_version 1.1;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto https;
+    proxy_set_header Connection "";
+    proxy_connect_timeout 2s;
+    proxy_send_timeout 3600s;
+    proxy_read_timeout 3600s;
+    proxy_request_buffering off;
+    proxy_buffering off;
+    proxy_next_upstream error timeout http_502 http_503 http_504;
+    proxy_next_upstream_tries 2;
+    proxy_pass http://easyai_gateway_api/;
+  }
+
+  location ^~ /static/ {
+    proxy_http_version 1.1;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto https;
+    proxy_set_header Connection "";
+    proxy_connect_timeout 2s;
+    proxy_send_timeout 300s;
+    proxy_read_timeout 300s;
+    proxy_buffering off;
+    proxy_next_upstream error timeout http_502 http_503 http_504;
+    proxy_next_upstream_tries 2;
+    proxy_pass http://easyai_gateway_api;
+  }
+
+  location / {
+    proxy_http_version 1.1;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto https;
+    proxy_set_header Connection "";
+    proxy_connect_timeout 2s;
+    proxy_send_timeout 300s;
+    proxy_read_timeout 300s;
+    proxy_next_upstream error timeout http_502 http_503 http_504;
+    proxy_next_upstream_tries 2;
+    proxy_pass http://easyai_gateway_web;
+  }
+}
@@ -0,0 +1,28 @@
+server {
+  listen 80;
+  listen [::]:80;
+  server_name ai.51easyai.com;
+  return 301 https://$host$request_uri;
+}
+
+server {
+  listen 443 ssl http2;
+  listen [::]:443 ssl http2;
+  server_name ai.51easyai.com;
+
+  ssl_certificate /etc/nginx/tls/ai.51easyai.com/fullchain.pem;
+  ssl_certificate_key /etc/nginx/tls/ai.51easyai.com/privkey.pem;
+  ssl_protocols TLSv1.2 TLSv1.3;
+
+  location ^~ /api/ {
+    default_type application/json;
+    add_header Retry-After 600 always;
+    return 503 '{"error":{"message":"Gateway database maintenance is in progress","type":"service_unavailable"}}';
+  }
+
+  location / {
+    default_type text/html;
+    add_header Retry-After 600 always;
+    return 503 '<!doctype html><html lang="zh-CN"><meta charset="utf-8"><title>系统维护</title><style>body{font-family:sans-serif;max-width:40rem;margin:15vh auto;padding:2rem;color:#1f2937}h1{font-size:1.8rem}</style><h1>系统正在进行高可用升级</h1><p>预计 5–10 分钟恢复，请稍后重试。</p></html>';
+  }
+}
@@ -0,0 +1,24 @@
+# Private-only deterministic path used by the cross-node acceptance test.
+server {
+  listen 10.77.0.1:18088;
+  server_name _;
+
+  allow 10.77.0.0/24;
+  deny all;
+  client_max_body_size 200m;
+
+  location / {
+    proxy_http_version 1.1;
+    proxy_set_header Host ai.51easyai.com;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto https;
+    proxy_set_header Connection "";
+    proxy_connect_timeout 2s;
+    proxy_send_timeout 3600s;
+    proxy_read_timeout 3600s;
+    proxy_request_buffering off;
+    proxy_buffering off;
+    proxy_pass http://10.77.0.1:30088;
+  }
+}
@@ -0,0 +1,20 @@
+# Hong Kong reads unexpired legacy files from the private Ningbo origin.
+location ^~ /static/generated/ {
+  proxy_http_version 1.1;
+  proxy_set_header Host ai.51easyai.com;
+  proxy_set_header Connection "";
+  proxy_connect_timeout 2s;
+  proxy_read_timeout 300s;
+  proxy_buffering off;
+  proxy_pass http://10.77.0.1:18080;
+}
+
+location ^~ /static/uploaded/ {
+  proxy_http_version 1.1;
+  proxy_set_header Host ai.51easyai.com;
+  proxy_set_header Connection "";
+  proxy_connect_timeout 2s;
+  proxy_read_timeout 300s;
+  proxy_buffering off;
+  proxy_pass http://10.77.0.1:18080;
+}
@@ -0,0 +1,13 @@
+# Temporary compatibility route. Remove after all files are strictly older
+# than 86,400 seconds and the cleanup acceptance check reports zero survivors.
+location ^~ /static/generated/ {
+  alias /var/lib/docker/volumes/easyai-ai-gateway_api_data/_data/static/generated/;
+  add_header Cache-Control "private, max-age=300";
+  try_files $request_filename =410;
+}
+
+location ^~ /static/uploaded/ {
+  alias /var/lib/docker/volumes/easyai-ai-gateway_api_data/_data/static/uploaded/;
+  add_header Cache-Control "private, max-age=300";
+  try_files $request_filename =410;
+}
@@ -0,0 +1,19 @@
+server {
+  listen 10.77.0.1:18080;
+  server_name _;
+
+  allow 10.77.0.0/24;
+  deny all;
+
+  location ^~ /static/generated/ {
+    alias /var/lib/docker/volumes/easyai-ai-gateway_api_data/_data/static/generated/;
+    add_header Cache-Control "private, max-age=300";
+    try_files $request_filename =410;
+  }
+
+  location ^~ /static/uploaded/ {
+    alias /var/lib/docker/volumes/easyai-ai-gateway_api_data/_data/static/uploaded/;
+    add_header Cache-Control "private, max-age=300";
+    try_files $request_filename =410;
+  }
+}
@@ -0,0 +1,16 @@
+[Unit]
+Description=Prune EasyAI Gateway legacy local files older than 24 hours
+After=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/easyai-prune-legacy-generated --apply
+User=root
+Group=root
+Nice=10
+IOSchedulingClass=idle
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectHome=true
+ProtectSystem=strict
+ReadWritePaths=/var/lib/docker/volumes/easyai-ai-gateway_api_data/_data/static
@@ -0,0 +1,11 @@
+[Unit]
+Description=Run EasyAI Gateway legacy local file cleanup hourly
+
+[Timer]
+OnCalendar=hourly
+Persistent=true
+RandomizedDelaySec=120
+Unit=easyai-legacy-generated-cleanup.service
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,129 @@
+# EasyAI Gateway 三节点 K3s 高可用运行手册
+
+## 范围与目标
+
+本手册覆盖宁波、香港、洛杉矶三节点迁移。宁波和香港承载 API、Web、River Worker 与
+CloudNativePG；洛杉矶只作为 K3s server/etcd 仲裁节点，并带
+`easyai.io/witness=true:NoSchedule` 污点。
+
+- 健康双库：同步复制，RPO=0。
+- 单库降级：`dataDurability: preferred` 保持写入，`archive_timeout=60s`，灾难 RPO 目标不超过 5 分钟。
+- 数据库故障 RTO：不超过 5 分钟。
+- 应用发布：香港验收后再滚动宁波，计划内零停机。
+- 公网入口：双 NGINX 加人工 AliDNS 切换。接入带健康检查的 GTM 前，不宣称公网入口自动高可用。
+
+所有命令从仓库根目录执行。生产凭据仅从 `0600` 的 `.env.local` 和
+`.local-secrets/cluster/` 读取，不进入 Git、发布 manifest 或验收输出。
+
+## 实施顺序
+
+### 1. 源码与镜像门禁
+
+```bash
+cd apps/api && env -u AI_GATEWAY_TEST_DATABASE_URL go test ./... -count=1
+cd apps/api && go vet ./...
+pnpm install --frozen-lockfile
+pnpm lint
+pnpm test
+pnpm build
+shellcheck scripts/cluster/*.sh deploy/kubernetes/easyai-ai-gateway-cluster-release
+kubectl kustomize deploy/kubernetes/production >/dev/null
+./tests/ci/migrations-test.sh
+node scripts/ci-validate-migrations.mjs <当前线上完整GitSHA>
+```
+
+代码提交并推送到 `origin/main` 后，发布 digest 固定镜像：
+
+```bash
+./scripts/publish-release-images.sh --components auto
+```
+
+### 2. WireGuard 与 K3s
+
+```bash
+./scripts/cluster/bootstrap-wireguard.sh
+./scripts/cluster/bootstrap-k3s.sh backup
+./scripts/cluster/bootstrap-k3s.sh install
+./scripts/cluster/harden-node-network.sh
+./scripts/cluster/verify-cluster.sh precutover
+```
+
+`backup` 会在卸载旧 K3s 前，将 SQLite、manifests 和 staging 资源归档并校验后上传
+OSS `easyai-ai-gateway/production/pre-ha/`。`install` 固定
+`v1.36.2+k3s1`，使用 embedded etcd、静态 Secret 加密、500ms heartbeat 和 5s election timeout。
+
+### 3. 平台和入口预置
+
+```bash
+./scripts/cluster/bootstrap-platform.sh prepare dist/releases/<SHA>.json
+./scripts/cluster/install-nginx-edges.sh prepare
+./scripts/cluster/install-certificate-sync.sh
+./scripts/cluster/install-legacy-cleaner.sh
+```
+
+`prepare` 将应用副本保持在 0，但启动 CNPG 双实例、WAL 归档和每日备份。SecretStore
+的 4 个值从旧 volume 直接导入 Kubernetes Secret，脚本只校验数量与总字节数，不打印值。
+
+### 4. 02:00–04:00 切换
+
+先执行只读预检：
+
+```bash
+./scripts/cluster/migrate-production.sh preflight dist/releases/<SHA>.json
+```
+
+进入北京时间窗口后执行：
+
+```bash
+./scripts/cluster/migrate-production.sh cutover dist/releases/<SHA>.json
+```
+
+脚本依次启用维护页、停止旧 API、完整备份旧 PostgreSQL、上传 OSS、恢复 CNPG、比对数据库
+版本/扩展/catalog/表数/序列/关键行数/抽样 ID、运行 digest 固定 migrator、冻结 Worker 启动
+两地应用、做 NodePort 验收，再切换双 NGINX。公开流量提交后，先启用香港 Worker，再启用宁波
+Worker。旧 Docker PostgreSQL 和数据卷停止但保留，不自动删除。
+
+在公共流量提交前，任一步失败或耗时超过 10 分钟都会恢复旧 Docker 与旧 NGINX。公共流量
+提交后禁止自动切回旧数据库，避免双写分叉。
+
+## 上线验收
+
+```bash
+./scripts/cluster/verify-cluster.sh postcutover
+./scripts/cluster/run-cross-node-file-e2e.sh
+./scripts/cluster/failure-drill.sh api --execute
+./scripts/cluster/failure-drill.sh database --execute
+./scripts/cluster/failure-drill.sh witness --execute
+./scripts/cluster/restore-backup-drill.sh --execute
+./scripts/cluster/verify-etcd-backup.sh
+```
+
+跨节点 E2E 通过宁波私有 NGINX 固定进入宁波 API，关闭宁波 River Worker，由香港 Worker
+执行真实 multipart 图片编辑任务；校验输入和输出 URL 在两节点下载的长度与 SHA-256
+一致，并检查 `river_job.attempted_by`、唯一成功 attempt、唯一结算/扣费及无重复回调。
+
+数据库故障演练会强制删除当前主 Pod，确认 5 分钟内晋升；在副本恢复前写入一条专用
+`system_settings` 探针，恢复同步后删除。备份恢复演练会从 OSS 创建一个独立 CNPG Cluster，
+验证数据库登录、schema、行数和抽样值后删除临时 Cluster 与 PVC。
+
+## 发布与回滚
+
+切换成功后，本地环境自动设置 `AI_GATEWAY_DEPLOY_MODE=kubernetes`。日常发布仍使用：
+
+```bash
+./scripts/deploy-production-release.sh dist/releases/<SHA>.json
+./scripts/deploy-production-release.sh --status
+./scripts/deploy-production-release.sh --rollback <历史完整SHA>
+```
+
+集群 helper 从 `easyai-gateway-release` ConfigMap 读取当前 release，数据库迁移前先等待一次
+CNPG 备份，然后香港滚动、验收，再滚动宁波。`--rollback` 只恢复应用 digest，绝不逆向执行迁移。
+
+## 旧文件与公网故障
+
+宁波每小时执行一次清理，以当前时间减文件 `mtime` 严格大于 86,400 秒为删除条件。未过期
+文件由宁波只读 NGINX 提供，香港通过 WireGuard 回源。清理结果持续为 0 个存量文件后，删除
+两个 legacy static include 和宁波 18080 origin；历史过期 URL 返回 410。
+
+公网入口故障时，先使用 `curl --resolve` 确认香港入口健康，再人工将 `ai.51easyai.com`
+AliDNS A 记录切到香港 IP。恢复后不自动回切；确认连接和任务稳定后再择时切回。
@@ -0,0 +1,249 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+
+action=${1:-all}
+[[ $action == backup || $action == install || $action == all ]] || {
+  echo 'usage: bootstrap-k3s.sh [backup|install|all]' >&2
+  exit 64
+}
+
+load_cluster_env
+require_commands node openssl ssh scp shasum
+
+secret_root="$cluster_root/.local-secrets/cluster"
+cluster_environment="$secret_root/cluster.env"
+kubeconfig="$secret_root/kubeconfig"
+k3s_version=v1.36.2+k3s1
+
+install -d -m 0700 "$secret_root"
+
+backup_legacy_k3s() {
+  local timestamp remote_archive local_directory local_archive checksum object_prefix
+  timestamp=$(date -u '+%Y%m%dT%H%M%SZ')
+  remote_archive=/root/easyai-k3s-sqlite-pre-ha-"$timestamp".tar.gz
+  local_directory=$(mktemp -d "$secret_root/backup.XXXXXX")
+  local_archive=$local_directory/$(basename "$remote_archive")
+  trap '[[ ${local_directory:-} == "$secret_root"/backup.* ]] && rm -rf -- "$local_directory"' RETURN
+
+  echo '[k3s] archiving legacy SQLite state and staging manifests'
+  cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$remote_archive" <<'REMOTE'
+set -euo pipefail
+archive=$1
+work_directory=$(mktemp -d /root/easyai-k3s-backup.XXXXXX)
+trap '[[ $work_directory == /root/easyai-k3s-backup.* ]] && rm -rf -- "$work_directory"' EXIT
+install -d -m 0700 "$work_directory/state"
+if [[ -d /var/lib/rancher/k3s/server/db ]]; then
+  cp -a /var/lib/rancher/k3s/server/db "$work_directory/state/"
+fi
+if [[ -d /var/lib/rancher/k3s/server/manifests ]]; then
+  cp -a /var/lib/rancher/k3s/server/manifests "$work_directory/state/"
+fi
+if command -v k3s >/dev/null 2>&1 && systemctl is-active --quiet k3s; then
+  k3s kubectl get namespace easyai-auth-center-staging -o yaml \
+    >"$work_directory/easyai-auth-center-staging-namespace.yaml" 2>/dev/null || true
+  k3s kubectl get all,configmap,secret,pvc,ingress \
+    -n easyai-auth-center-staging -o yaml \
+    >"$work_directory/easyai-auth-center-staging-resources.yaml" 2>/dev/null || true
+  k3s kubectl get nodes -o wide >"$work_directory/nodes.txt"
+  k3s --version >"$work_directory/k3s-version.txt"
+fi
+tar -C "$work_directory" -czf "$archive" .
+chmod 0600 "$archive"
+sha256sum "$archive" >"$archive.sha256"
+REMOTE
+
+  cluster_scp "$CLUSTER_NINGBO_HOST:$remote_archive" "$local_archive"
+  cluster_scp "$CLUSTER_NINGBO_HOST:$remote_archive.sha256" "$local_archive.remote.sha256"
+  checksum=$(shasum -a 256 "$local_archive" | awk '{print $1}')
+  grep -q "^$checksum " "$local_archive.remote.sha256" || {
+    echo 'legacy K3s archive checksum mismatch' >&2
+    return 1
+  }
+  printf '%s  %s\n' "$checksum" "$(basename "$local_archive")" >"$local_archive.sha256"
+  object_prefix="easyai-ai-gateway/production/pre-ha/${timestamp}"
+  node "$script_dir/oss-object.mjs" put "$object_prefix/$(basename "$local_archive")" "$local_archive"
+  node "$script_dir/oss-object.mjs" put "$object_prefix/$(basename "$local_archive").sha256" "$local_archive.sha256"
+  node "$script_dir/oss-object.mjs" head "$object_prefix/$(basename "$local_archive")"
+  cluster_ssh "$CLUSTER_NINGBO_HOST" \
+    "rm -f -- '$remote_archive' '$remote_archive.sha256'"
+  echo "legacy_k3s_backup=PASS object_prefix=$object_prefix sha256=$checksum"
+}
+
+write_node_config() {
+  local output=$1
+  local node_name=$2
+  local node_ip=$3
+  local mode=$4
+  local taint=${5:-}
+  umask 077
+  {
+    printf 'token-file: /etc/rancher/k3s/cluster-token\n'
+    printf 'node-name: %s\n' "$node_name"
+    printf 'node-ip: %s\n' "$node_ip"
+    printf 'advertise-address: %s\n' "$node_ip"
+    printf 'flannel-iface: wg0\n'
+    printf 'write-kubeconfig-mode: "0600"\n'
+    printf 'secrets-encryption: true\n'
+    printf 'disable:\n  - traefik\n  - servicelb\n'
+    printf 'tls-san:\n  - 10.77.0.1\n  - 10.77.0.2\n  - 10.77.0.3\n'
+    printf 'etcd-arg:\n  - heartbeat-interval=500\n  - election-timeout=5000\n'
+    printf 'etcd-snapshot-schedule-cron: "0 */6 * * *"\n'
+    printf 'etcd-snapshot-retention: 28\n'
+    printf 'etcd-snapshot-compress: true\n'
+    printf 'etcd-s3: true\n'
+    printf 'etcd-s3-config-secret: easyai-etcd-s3\n'
+    if [[ $mode == init ]]; then
+      printf 'cluster-init: true\n'
+    else
+      printf 'server: https://10.77.0.1:6443\n'
+    fi
+    if [[ -n $taint ]]; then
+      printf 'node-taint:\n  - %s\n' "$taint"
+    fi
+  } >"$output"
+  chmod 0600 "$output"
+}
+
+install_cluster() {
+  local configuration_directory
+  for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
+    cluster_ssh "$host" 'ping -c 2 -W 2 10.77.0.1 >/dev/null'
+  done
+
+  if [[ ! -f $cluster_environment ]]; then
+    umask 077
+    printf 'K3S_TOKEN=%s\n' "$(openssl rand -hex 32)" >"$cluster_environment"
+    chmod 0600 "$cluster_environment"
+  fi
+  assert_private_file "$cluster_environment"
+  # shellcheck source=/dev/null
+  source "$cluster_environment"
+  : "${K3S_TOKEN:?}"
+  [[ $K3S_TOKEN =~ ^[0-9a-f]{64}$ ]] || {
+    echo 'invalid local K3s token' >&2
+    return 1
+  }
+
+  configuration_directory=$(mktemp -d "$secret_root/config.XXXXXX")
+  trap '[[ ${configuration_directory:-} == "$secret_root"/config.* ]] && rm -rf -- "$configuration_directory"' RETURN
+  write_node_config "$configuration_directory/ningbo.yaml" easyai-ningbo 10.77.0.1 init
+  write_node_config "$configuration_directory/hongkong.yaml" easyai-hongkong 10.77.0.2 join
+  write_node_config "$configuration_directory/los-angeles.yaml" easyai-los-angeles 10.77.0.3 join \
+    'easyai.io/witness=true:NoSchedule'
+  umask 077
+  cat >"$configuration_directory/etcd-s3-secret.yaml" <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: easyai-etcd-s3
+  namespace: kube-system
+type: etcd.k3s.cattle.io/s3-config-secret
+stringData:
+  etcd-s3-endpoint: ${ALI_REGION}.aliyuncs.com
+  etcd-s3-access-key: ${ALI_KEY}
+  etcd-s3-secret-key: ${ALI_SECRET}
+  etcd-s3-bucket: ${ALI_BUCKET}
+  etcd-s3-folder: easyai-ai-gateway/production/k3s-etcd
+  etcd-s3-region: ${ALI_REGION}
+  etcd-s3-bucket-lookup-type: dns
+  etcd-s3-insecure: "false"
+  etcd-s3-skip-ssl-verify: "false"
+  etcd-s3-timeout: 5m
+EOF
+  chmod 0600 "$configuration_directory/etcd-s3-secret.yaml"
+
+  echo '[k3s] removing the archived legacy single-node installation'
+  cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+if command -v k3s >/dev/null 2>&1 && systemctl is-active --quiet k3s; then
+  k3s kubectl delete namespace easyai-auth-center-staging \
+    --ignore-not-found=true --wait=true --timeout=120s
+fi
+if [[ -x /usr/local/bin/k3s-uninstall.sh ]]; then
+  /usr/local/bin/k3s-uninstall.sh
+fi
+REMOTE
+  for host in "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
+    cluster_ssh "$host" \
+      '[[ ! -x /usr/local/bin/k3s-uninstall.sh ]] || /usr/local/bin/k3s-uninstall.sh'
+  done
+
+  node_names=(ningbo hongkong los-angeles)
+  node_hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
+  for index in 0 1 2; do
+    host=${node_hosts[$index]}
+    node_name=${node_names[$index]}
+    cluster_ssh "$host" 'install -d -m 0700 /etc/rancher/k3s'
+    printf '%s\n' "$K3S_TOKEN" | cluster_ssh "$host" \
+      'umask 077; cat >/etc/rancher/k3s/cluster-token; chmod 0600 /etc/rancher/k3s/cluster-token'
+    cluster_scp "$configuration_directory/$node_name.yaml" "$host:/etc/rancher/k3s/config.yaml"
+    cluster_ssh "$host" 'chmod 0600 /etc/rancher/k3s/config.yaml'
+  done
+
+  for index in 0 1 2; do
+    host=${node_hosts[$index]}
+    echo "[k3s] installing $k3s_version on $host"
+    cluster_ssh "$host" bash -s -- "$k3s_version" <<'REMOTE'
+set -euo pipefail
+version=$1
+install_script=$(mktemp)
+trap 'rm -f -- "$install_script"' EXIT
+curl -fsSL https://get.k3s.io -o "$install_script"
+chmod 0700 "$install_script"
+INSTALL_K3S_VERSION="$version" INSTALL_K3S_EXEC=server "$install_script"
+k3s --version | grep -F "$version"
+REMOTE
+    if [[ $index -eq 0 ]]; then
+      for _ in $(seq 1 60); do
+        if cluster_ssh "$host" 'k3s kubectl get --raw=/readyz >/dev/null 2>&1'; then
+          break
+        fi
+        sleep 2
+      done
+      cluster_scp "$configuration_directory/etcd-s3-secret.yaml" \
+        "$CLUSTER_NINGBO_HOST:/root/easyai-etcd-s3-secret.yaml"
+      cluster_ssh "$CLUSTER_NINGBO_HOST" \
+        'k3s kubectl apply -f /root/easyai-etcd-s3-secret.yaml >/dev/null && rm -f /root/easyai-etcd-s3-secret.yaml'
+    fi
+  done
+
+  for _ in $(seq 1 90); do
+    ready_count=$(cluster_ssh "$CLUSTER_NINGBO_HOST" \
+      "k3s kubectl get nodes --no-headers 2>/dev/null | awk '\$2 == \"Ready\" { count++ } END { print count + 0 }'")
+    [[ $ready_count == 3 ]] && break
+    sleep 2
+  done
+  [[ ${ready_count:-0} == 3 ]] || {
+    echo 'three K3s servers did not become Ready' >&2
+    return 1
+  }
+
+  cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+kubectl='k3s kubectl'
+$kubectl label node easyai-ningbo easyai.io/site=ningbo easyai.io/workload=true easyai.io/database=true --overwrite
+$kubectl label node easyai-hongkong easyai.io/site=hongkong easyai.io/workload=true easyai.io/database=true --overwrite
+$kubectl label node easyai-los-angeles easyai.io/site=los-angeles easyai.io/workload=false easyai.io/database=false --overwrite
+$kubectl taint node easyai-los-angeles easyai.io/witness=true:NoSchedule --overwrite
+k3s secrets-encrypt status
+k3s etcd-snapshot save --name post-bootstrap
+REMOTE
+
+  cluster_scp "$CLUSTER_NINGBO_HOST:/etc/rancher/k3s/k3s.yaml" "$kubeconfig"
+  chmod 0600 "$kubeconfig"
+  sed -i.bak 's#https://127.0.0.1:6443#https://10.77.0.1:6443#' "$kubeconfig"
+  rm -f -- "$kubeconfig.bak"
+  assert_private_file "$kubeconfig"
+  echo "k3s_bootstrap=PASS version=$k3s_version servers=3 kubeconfig=$kubeconfig"
+}
+
+if [[ $action == backup || $action == all ]]; then
+  backup_legacy_k3s
+fi
+if [[ $action == install || $action == all ]]; then
+  install_cluster
+fi
@@ -0,0 +1,225 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+
+mode=${1:-}
+release_manifest=${2:-}
+[[ $mode == prepare || $mode == activate ]] || {
+  echo 'usage: bootstrap-platform.sh {prepare|activate} dist/releases/<sha>.json' >&2
+  exit 64
+}
+[[ -f $release_manifest && ! -L $release_manifest ]] || {
+  echo 'release manifest must be a regular file' >&2
+  exit 1
+}
+
+load_cluster_env
+require_commands curl kubectl node openssl rg shasum
+
+node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null
+api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.api)
+web_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.web)
+source_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha)
+[[ $api_image =~ ^registry\.cn-shanghai\.aliyuncs\.com/easyaigc/ai-gateway@sha256:[0-9a-f]{64}$ ]]
+[[ $web_image =~ ^registry\.cn-shanghai\.aliyuncs\.com/easyaigc/ai-gateway-web@sha256:[0-9a-f]{64}$ ]]
+
+secret_root="$cluster_root/.local-secrets/cluster"
+postgres_environment="$secret_root/postgres.env"
+working_directory=$(mktemp -d "$secret_root/platform.XXXXXX")
+trap '[[ $working_directory == "$secret_root"/platform.* ]] && rm -rf -- "$working_directory"' EXIT HUP INT TERM
+install -d -m 0700 "$secret_root"
+
+if [[ ! -f $postgres_environment ]]; then
+  umask 077
+  printf 'POSTGRES_PASSWORD=%s\n' "$(openssl rand -hex 32)" >"$postgres_environment"
+  chmod 0600 "$postgres_environment"
+fi
+assert_private_file "$postgres_environment"
+# shellcheck source=/dev/null
+source "$postgres_environment"
+: "${POSTGRES_PASSWORD:?}"
+[[ $POSTGRES_PASSWORD =~ ^[0-9a-f]{64}$ ]] || {
+  echo 'invalid generated PostgreSQL password' >&2
+  exit 1
+}
+
+declare -A operator_urls=(
+  [cert-manager.yaml]=https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml
+  [cnpg.yaml]=https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.29/releases/cnpg-1.29.1.yaml
+  [barman.yaml]=https://github.com/cloudnative-pg/plugin-barman-cloud/releases/download/v0.13.0/manifest.yaml
+)
+declare -A operator_checksums=(
+  [cert-manager.yaml]=6e499c3f1ab356abe79a7853911f80cb09c213885bfdf81092fdff142ba63c4a
+  [cnpg.yaml]=e0c5ff41bf5b01c0775bf225929b8423d543cd23e9742e4274fe7de8499cf93a
+  [barman.yaml]=d2e71e7b06822448f1a421f05781846cfdb9cc621e7ef32eef5e20c5133213b0
+)
+for operator in cert-manager.yaml cnpg.yaml barman.yaml; do
+  curl -fsSL "${operator_urls[$operator]}" -o "$working_directory/$operator"
+  actual_checksum=$(shasum -a 256 "$working_directory/$operator" | awk '{print $1}')
+  [[ $actual_checksum == "${operator_checksums[$operator]}" ]] || {
+    echo "operator manifest checksum mismatch: $operator" >&2
+    exit 1
+  }
+done
+
+cp -R "$cluster_root/deploy/kubernetes/production" "$working_directory/production"
+api_digest=${api_image##*@}
+web_digest=${web_image##*@}
+awk -v api_digest="$api_digest" -v web_digest="$web_digest" '
+  $3 == "easyai-api" { image = "api" }
+  $3 == "easyai-web" { image = "web" }
+  $1 == "digest:" && image == "api" { $2 = api_digest; image = ""; print; next }
+  $1 == "digest:" && image == "web" { $2 = web_digest; image = ""; print; next }
+  { print }
+' "$working_directory/production/kustomization.yaml" \
+  >"$working_directory/production/kustomization.yaml.next"
+mv "$working_directory/production/kustomization.yaml.next" \
+  "$working_directory/production/kustomization.yaml"
+
+if [[ $mode == prepare ]]; then
+  sed -i.bak 's/^  replicas: 1$/  replicas: 0/' "$working_directory/production/application.yaml"
+  rm -f -- "$working_directory/production/application.yaml.bak"
+fi
+if [[ ${AI_GATEWAY_ACTIVATE_WORKERS:-true} == false ]]; then
+  sed -i.bak \
+    's/^  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED: "true"$/  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED: "false"/' \
+    "$working_directory/production/application-config.yaml"
+  rm -f -- "$working_directory/production/application-config.yaml.bak"
+fi
+kubectl kustomize "$working_directory/production" >"$working_directory/easyai-production.yaml"
+if rg -q 'sha256:0{64}|image: (easyai-api|easyai-web)$' "$working_directory/easyai-production.yaml"; then
+  echo 'application images were not digest-pinned during rendering' >&2
+  exit 1
+fi
+if rg -i -q '^[[:space:]]*(password|secret|token|credential|private.?key):[[:space:]]+[^<{[:space:]]' \
+    "$working_directory/easyai-production.yaml"; then
+  echo 'rendered production manifests appear to contain a literal secret' >&2
+  exit 1
+fi
+
+umask 077
+cat >"$working_directory/bootstrap-secrets.yaml" <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: easyai-postgres-app
+  namespace: easyai
+type: kubernetes.io/basic-auth
+stringData:
+  username: easyai
+  password: ${POSTGRES_PASSWORD}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: easyai-oss-backup
+  namespace: easyai
+type: Opaque
+stringData:
+  ACCESS_KEY_ID: ${ALI_KEY}
+  ACCESS_SECRET_KEY: ${ALI_SECRET}
+EOF
+chmod 0600 "$working_directory/bootstrap-secrets.yaml"
+
+for file in cert-manager.yaml cnpg.yaml barman.yaml easyai-production.yaml bootstrap-secrets.yaml; do
+  cluster_scp "$working_directory/$file" "$CLUSTER_NINGBO_HOST:/root/$file"
+done
+
+cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$source_sha" "$mode" <<'REMOTE'
+set -euo pipefail
+source_sha=$1
+mode=$2
+kubectl='k3s kubectl'
+
+$kubectl apply -f /root/cert-manager.yaml >/dev/null
+$kubectl rollout status deployment/cert-manager -n cert-manager --timeout=180s
+$kubectl rollout status deployment/cert-manager-webhook -n cert-manager --timeout=180s
+$kubectl rollout status deployment/cert-manager-cainjector -n cert-manager --timeout=180s
+$kubectl apply -f /root/cnpg.yaml >/dev/null
+$kubectl rollout status deployment/cnpg-controller-manager -n cnpg-system --timeout=180s
+$kubectl apply -f /root/barman.yaml >/dev/null
+$kubectl rollout status deployment/barman-cloud -n cnpg-system --timeout=180s
+
+$kubectl create namespace easyai --dry-run=client -o yaml | $kubectl apply -f - >/dev/null
+$kubectl apply -f /root/bootstrap-secrets.yaml >/dev/null
+postgres_password=$($kubectl get secret easyai-postgres-app -n easyai \
+  -o jsonpath='{.data.password}' | base64 -d)
+
+runtime_environment=$(mktemp)
+trap 'rm -f -- "$runtime_environment"' EXIT
+chmod 0600 "$runtime_environment"
+docker inspect easyai-ai-gateway-api-1 \
+  --format '{{range .Config.Env}}{{println .}}{{end}}' |
+  grep -E '^(CONFIG_JWT_SECRET|SERVER_MAIN_INTERNAL_TOKEN|SERVER_MAIN_INTERNAL_KEY|SERVER_MAIN_INTERNAL_SECRET)=' \
+  >"$runtime_environment"
+printf 'AI_GATEWAY_DATABASE_URL=postgresql://easyai:%s@easyai-postgres-rw:5432/easyai_ai_gateway?sslmode=require\n' \
+  "$postgres_password" >>"$runtime_environment"
+[[ $(grep -c '^[A-Z0-9_]*=' "$runtime_environment") -eq 5 ]] || {
+  echo 'runtime Secret allowlist is incomplete' >&2
+  exit 1
+}
+$kubectl create secret generic easyai-ai-gateway-runtime -n easyai \
+  --from-env-file="$runtime_environment" --dry-run=client -o yaml |
+  $kubectl apply -f - >/dev/null
+
+identity_directory=/var/lib/docker/volumes/easyai-ai-gateway_api_data/_data/identity/secrets
+[[ -d $identity_directory && ! -L $identity_directory ]] || {
+  echo 'legacy identity SecretStore directory is unavailable' >&2
+  exit 1
+}
+mapfile -d '' identity_files < <(find "$identity_directory" -maxdepth 1 -type f -print0)
+[[ ${#identity_files[@]} -eq 4 ]] || {
+  echo 'identity SecretStore file count changed from the accepted baseline' >&2
+  exit 1
+}
+identity_bytes=0
+identity_arguments=()
+for identity_file in "${identity_files[@]}"; do
+  identity_name=$(basename "$identity_file")
+  [[ $identity_name =~ ^[A-Za-z0-9._-]+$ ]] || {
+    echo 'identity SecretStore contains an invalid key name' >&2
+    exit 1
+  }
+  identity_bytes=$((identity_bytes + $(stat -c '%s' "$identity_file")))
+  identity_arguments+=("--from-file=$identity_name=$identity_file")
+done
+[[ $identity_bytes -eq 247 ]] || {
+  echo 'identity SecretStore byte count changed from the accepted baseline' >&2
+  exit 1
+}
+$kubectl create secret generic easyai-gateway-security-events -n easyai \
+  "${identity_arguments[@]}" --dry-run=client -o yaml |
+  $kubectl apply -f - >/dev/null
+
+registry_config=/root/.docker/config.json
+[[ -f $registry_config && ! -L $registry_config ]] || {
+  echo 'Docker registry config is unavailable on Ningbo' >&2
+  exit 1
+}
+$kubectl create secret generic easyai-registry -n easyai \
+  --type=kubernetes.io/dockerconfigjson \
+  --from-file=.dockerconfigjson="$registry_config" --dry-run=client -o yaml |
+  $kubectl apply -f - >/dev/null
+
+$kubectl apply -f /root/easyai-production.yaml >/dev/null
+$kubectl annotate namespace easyai easyai.io/release="$source_sha" --overwrite >/dev/null
+if [[ $mode == activate ]]; then
+  $kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s
+  $kubectl rollout status deployment/easyai-web-hongkong -n easyai --timeout=300s
+  $kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
+  $kubectl rollout status deployment/easyai-web-ningbo -n easyai --timeout=300s
+fi
+
+rm -f -- /root/cert-manager.yaml /root/cnpg.yaml /root/barman.yaml \
+  /root/easyai-production.yaml /root/bootstrap-secrets.yaml
+REMOTE
+
+if [[ $mode == prepare ]]; then
+  cluster_ssh "$CLUSTER_NINGBO_HOST" \
+    'k3s kubectl wait --for=condition=Ready cluster/easyai-postgres -n easyai --timeout=600s'
+fi
+
+echo "platform_bootstrap=PASS mode=$mode release=$source_sha"
@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+
+load_cluster_env
+require_commands ssh scp
+
+hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
+public_endpoints=("${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}" "${CLUSTER_LOS_ANGELES_HOST#*@}")
+wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
+public_keys=()
+
+for host in "${hosts[@]}"; do
+  echo "[wireguard] preparing $host"
+  public_key=$(cluster_ssh "$host" 'bash -s' <<'REMOTE'
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+if ! command -v wg >/dev/null 2>&1; then
+  apt-get update -qq
+  apt-get install -y -qq wireguard-tools
+fi
+install -d -m 0700 /etc/wireguard
+if [[ ! -s /etc/wireguard/privatekey ]]; then
+  umask 077
+  wg genkey >/etc/wireguard/privatekey
+fi
+chmod 0600 /etc/wireguard/privatekey
+wg pubkey </etc/wireguard/privatekey >/etc/wireguard/publickey
+cat /etc/wireguard/publickey
+REMOTE
+)
+  [[ $public_key =~ ^[A-Za-z0-9+/]{43}=$ ]] || {
+    echo "invalid WireGuard public key returned by $host" >&2
+    exit 1
+  }
+  public_keys+=("$public_key")
+done
+
+for local_index in 0 1 2; do
+  peer_indexes=()
+  for candidate in 0 1 2; do
+    [[ $candidate -eq $local_index ]] || peer_indexes+=("$candidate")
+  done
+  first=${peer_indexes[0]}
+  second=${peer_indexes[1]}
+  echo "[wireguard] configuring ${hosts[$local_index]} as ${wireguard_ips[$local_index]}"
+  cluster_ssh "${hosts[$local_index]}" bash -s -- \
+    "${wireguard_ips[$local_index]}" \
+    "${public_keys[$first]}" "${public_endpoints[$first]}" "${wireguard_ips[$first]}" \
+    "${public_keys[$second]}" "${public_endpoints[$second]}" "${wireguard_ips[$second]}" <<'REMOTE'
+set -euo pipefail
+local_ip=$1
+peer_one_key=$2
+peer_one_endpoint=$3
+peer_one_ip=$4
+peer_two_key=$5
+peer_two_endpoint=$6
+peer_two_ip=$7
+private_key=$(cat /etc/wireguard/privatekey)
+umask 077
+temporary_config=$(mktemp /etc/wireguard/wg0.conf.XXXXXX)
+cat >"$temporary_config" <<EOF
+[Interface]
+Address = ${local_ip}/24
+ListenPort = 51820
+PrivateKey = ${private_key}
+
+[Peer]
+PublicKey = ${peer_one_key}
+Endpoint = ${peer_one_endpoint}:51820
+AllowedIPs = ${peer_one_ip}/32
+PersistentKeepalive = 25
+
+[Peer]
+PublicKey = ${peer_two_key}
+Endpoint = ${peer_two_endpoint}:51820
+AllowedIPs = ${peer_two_ip}/32
+PersistentKeepalive = 25
+EOF
+chmod 0600 "$temporary_config"
+mv "$temporary_config" /etc/wireguard/wg0.conf
+systemctl enable wg-quick@wg0 >/dev/null
+systemctl restart wg-quick@wg0
+REMOTE
+done
+
+for source_index in 0 1 2; do
+  for target_index in 0 1 2; do
+    [[ $source_index -eq $target_index ]] && continue
+    cluster_ssh "${hosts[$source_index]}" \
+      "ping -c 3 -W 2 ${wireguard_ips[$target_index]} >/dev/null"
+  done
+done
+
+for host in "${hosts[@]}"; do
+  latest_handshake=$(cluster_ssh "$host" \
+    "wg show wg0 latest-handshakes | awk '\$2 > 0 { count++ } END { print count + 0 }'")
+  [[ $latest_handshake == 2 ]] || {
+    echo "WireGuard handshake verification failed on $host" >&2
+    exit 1
+  }
+done
+
+echo 'wireguard_bootstrap=PASS peers=3'
@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cluster_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+cluster_env_file=${AI_GATEWAY_CLUSTER_ENV_FILE:-"$cluster_root/.env.local"}
+cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-"$HOME/.ssh/id_ed25519_easyai_gateway_cluster"}
+
+load_cluster_env() {
+  [[ -f $cluster_env_file && ! -L $cluster_env_file ]] || {
+    echo "missing local cluster environment file: $cluster_env_file" >&2
+    return 1
+  }
+  set -a
+  # shellcheck source=/dev/null
+  source "$cluster_env_file"
+  set +a
+  cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-$cluster_ssh_key}
+
+  : "${AI_GATEWAY_DEPLOY_HOST:?}"
+  : "${AI_GATEWAY_DEPLOY_HOST_2:?}"
+  : "${AI_GATEWAY_DEPLOY_HOST_3:?}"
+  : "${ALI_KEY:?}"
+  : "${ALI_SECRET:?}"
+  : "${ALI_REGION:?}"
+  : "${ALI_BUCKET:?}"
+
+  CLUSTER_NINGBO_HOST=${AI_GATEWAY_NINGBO_HOST:-$AI_GATEWAY_DEPLOY_HOST}
+  CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-$AI_GATEWAY_DEPLOY_HOST_2}
+  CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-$AI_GATEWAY_DEPLOY_HOST_3}
+  export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST
+
+  for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do
+    [[ $host =~ ^root@[A-Za-z0-9.-]+$ ]] || {
+      echo "cluster hosts must use root@host syntax" >&2
+      return 1
+    }
+  done
+  [[ $ALI_REGION =~ ^[a-z0-9-]+$ && $ALI_BUCKET =~ ^[A-Za-z0-9.-]+$ ]] || {
+    echo 'invalid OSS region or bucket' >&2
+    return 1
+  }
+  [[ -f $cluster_ssh_key && ! -L $cluster_ssh_key ]] || {
+    echo "missing cluster SSH key: $cluster_ssh_key" >&2
+    return 1
+  }
+}
+
+cluster_ssh() {
+  local host=$1
+  shift
+  ssh \
+    -i "$cluster_ssh_key" \
+    -o BatchMode=yes \
+    -o ConnectTimeout=10 \
+    -o ServerAliveInterval=15 \
+    -o ServerAliveCountMax=3 \
+    "$host" "$@"
+}
+
+cluster_scp() {
+  scp \
+    -i "$cluster_ssh_key" \
+    -o BatchMode=yes \
+    -o ConnectTimeout=10 \
+    "$@"
+}
+
+require_commands() {
+  local command
+  for command in "$@"; do
+    command -v "$command" >/dev/null 2>&1 || {
+      echo "$command is required" >&2
+      return 1
+    }
+  done
+}
+
+assert_private_file() {
+  local path=$1
+  local mode
+  [[ -f $path && ! -L $path ]] || {
+    echo "missing private file: $path" >&2
+    return 1
+  }
+  if [[ $(uname -s) == Darwin ]]; then
+    mode=$(stat -f '%Lp' "$path")
+  else
+    mode=$(stat -c '%a' "$path")
+  fi
+  [[ $mode == 600 ]] || {
+    echo "private file must have mode 0600: $path" >&2
+    return 1
+  }
+}
@@ -0,0 +1,147 @@
+#!/usr/bin/env node
+
+import { createHash, randomUUID } from 'node:crypto'
+import { deflateSync } from 'node:zlib'
+
+function assert(condition, message) {
+  if (!condition) throw new Error(message)
+}
+
+function crc32(buffer) {
+  let value = 0xffffffff
+  for (const byte of buffer) {
+    value ^= byte
+    for (let bit = 0; bit < 8; bit += 1) {
+      value = (value >>> 1) ^ (value & 1 ? 0xedb88320 : 0)
+    }
+  }
+  return (value ^ 0xffffffff) >>> 0
+}
+
+function pngChunk(type, data) {
+  const typeBuffer = Buffer.from(type)
+  const length = Buffer.alloc(4)
+  length.writeUInt32BE(data.length)
+  const checksum = Buffer.alloc(4)
+  checksum.writeUInt32BE(crc32(Buffer.concat([typeBuffer, data])))
+  return Buffer.concat([length, typeBuffer, data, checksum])
+}
+
+function createFixturePNG(width = 256, height = 256) {
+  const header = Buffer.alloc(13)
+  header.writeUInt32BE(width, 0)
+  header.writeUInt32BE(height, 4)
+  header[8] = 8
+  header[9] = 2
+  const rows = []
+  for (let y = 0; y < height; y += 1) {
+    const row = Buffer.alloc(1 + width * 3)
+    for (let x = 0; x < width; x += 1) {
+      row[1 + x * 3] = Math.round((x / width) * 255)
+      row[2 + x * 3] = Math.round((y / height) * 255)
+      row[3 + x * 3] = 180
+    }
+    rows.push(row)
+  }
+  return Buffer.concat([
+    Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]),
+    pngChunk('IHDR', header),
+    pngChunk('IDAT', deflateSync(Buffer.concat(rows))),
+    pngChunk('IEND', Buffer.alloc(0)),
+  ])
+}
+
+const baseURL = process.env.AI_GATEWAY_E2E_BASE_URL || 'http://10.77.0.1:18088'
+const account = Buffer.from(process.env.E2E_ACCOUNT_B64 || '', 'base64').toString('utf8')
+const password = Buffer.from(process.env.E2E_PASSWORD_B64 || '', 'base64').toString('utf8')
+const model = process.env.AI_GATEWAY_E2E_IMAGE_MODEL || 'openai:gpt-image-1'
+assert(account && password, 'E2E login credentials are unavailable')
+
+async function request(path, { method = 'GET', token, body, form } = {}) {
+  const headers = { Host: 'ai.51easyai.com' }
+  if (token) headers.Authorization = `Bearer ${token}`
+  if (body !== undefined) headers['Content-Type'] = 'application/json'
+  if (form) headers['X-Async'] = 'true'
+  const response = await fetch(`${baseURL}${path}`, {
+    method,
+    headers,
+    body: form || (body === undefined ? undefined : JSON.stringify(body)),
+    signal: AbortSignal.timeout(3_600_000),
+  })
+  const text = await response.text()
+  let payload
+  try {
+    payload = text ? JSON.parse(text) : {}
+  } catch {
+    payload = { raw: text.slice(0, 500) }
+  }
+  if (!response.ok) {
+    throw new Error(`${method} ${path} returned HTTP ${response.status}: ${JSON.stringify(payload)}`)
+  }
+  return { response, payload }
+}
+
+function collectHTTPSURLs(value, output = []) {
+  if (typeof value === 'string' && value.startsWith('https://')) output.push(value)
+  else if (Array.isArray(value)) value.forEach((item) => collectHTTPSURLs(item, output))
+  else if (value && typeof value === 'object') Object.values(value).forEach((item) => collectHTTPSURLs(item, output))
+  return output
+}
+
+const login = await request('/api/v1/auth/login', {
+  method: 'POST',
+  body: { account, password },
+})
+const jwt = login.payload.accessToken
+assert(typeof jwt === 'string' && jwt.length > 20, 'login did not return an access token')
+
+const keyName = `cluster-cross-node-${Date.now()}`
+const keyResult = await request('/api/v1/api-keys', {
+  method: 'POST',
+  token: jwt,
+  body: { name: keyName },
+})
+const apiKey = keyResult.payload.secret
+const apiKeyId = keyResult.payload.id || keyResult.payload.item?.id
+assert(typeof apiKey === 'string' && apiKey.startsWith('sk-gw-'), 'API key creation failed')
+
+let result
+try {
+  const fixture = createFixturePNG()
+  const sourceSHA256 = createHash('sha256').update(fixture).digest('hex')
+  const form = new FormData()
+  form.append('model', model)
+  form.append('prompt', `Cross-node HA acceptance ${randomUUID()}: add a thin white border`)
+  form.append('size', '1024x1024')
+  form.append('image', new Blob([fixture], { type: 'image/png' }), 'cluster-e2e.png')
+  const accepted = await request('/api/v1/images/edits', {
+    method: 'POST',
+    token: apiKey,
+    form,
+  })
+  assert(accepted.response.status === 202, `image edit returned ${accepted.response.status}, expected 202`)
+  const taskId = accepted.payload.taskId || accepted.payload.task_id || accepted.payload.task?.id
+  assert(/^[0-9a-f-]{36}$/.test(taskId || ''), 'image edit response has no task ID')
+
+  let task
+  const deadline = Date.now() + 20 * 60_000
+  while (Date.now() < deadline) {
+    const taskResult = await request(`/api/v1/tasks/${taskId}`, { token: apiKey })
+    task = taskResult.payload.task || taskResult.payload
+    if (task.status === 'succeeded') break
+    if (['failed', 'cancelled'].includes(task.status)) {
+      throw new Error(`task ${taskId} ended as ${task.status}: ${task.errorMessage || task.error || ''}`)
+    }
+    await new Promise((resolve) => setTimeout(resolve, 3000))
+  }
+  assert(task?.status === 'succeeded', `task ${taskId} did not succeed before timeout`)
+  const resultURLs = [...new Set(collectHTTPSURLs(task.result))]
+  assert(resultURLs.length > 0, 'task result contains no shared HTTPS URL')
+  result = { taskId, sourceSHA256, resultURL: resultURLs[0], model }
+} finally {
+  if (apiKeyId) {
+    await request(`/api/v1/api-keys/${apiKeyId}`, { method: 'DELETE', token: jwt }).catch(() => {})
+  }
+}
+
+console.log(JSON.stringify(result))
@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+
+drill=${1:-}
+confirmation=${2:-}
+[[ $confirmation == --execute ]] || {
+  echo 'usage: failure-drill.sh {api|database|witness} --execute' >&2
+  exit 64
+}
+load_cluster_env
+
+case $drill in
+  api)
+    started=$(date +%s)
+    pod=$(cluster_ssh "$CLUSTER_NINGBO_HOST" \
+      "k3s kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo -o jsonpath='{.items[0].metadata.name}'")
+    cluster_ssh "$CLUSTER_NINGBO_HOST" \
+      "k3s kubectl delete pod '$pod' -n easyai --wait=false >/dev/null"
+    for _ in $(seq 1 20); do
+      if curl -fsS --max-time 3 --resolve "ai.51easyai.com:443:${CLUSTER_NINGBO_HOST#*@}" \
+        https://ai.51easyai.com/api/v1/readyz | grep -q '"ok":true'; then
+        elapsed=$(( $(date +%s) - started ))
+        break
+      fi
+      sleep 1
+    done
+    [[ ${elapsed:-99} -le 10 ]]
+    echo "api_failure_drill=PASS failover_seconds=$elapsed"
+    ;;
+  database)
+    old_primary=$(cluster_ssh "$CLUSTER_NINGBO_HOST" \
+      "k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}'")
+    started=$(date +%s)
+    cluster_ssh "$CLUSTER_NINGBO_HOST" \
+      "k3s kubectl delete pod '$old_primary' -n easyai --grace-period=0 --force >/dev/null"
+    for _ in $(seq 1 150); do
+      new_primary=$(cluster_ssh "$CLUSTER_NINGBO_HOST" \
+        "k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}'")
+      if [[ -n $new_primary && $new_primary != "$old_primary" ]]; then
+        elapsed=$(( $(date +%s) - started ))
+        break
+      fi
+      sleep 2
+    done
+    [[ ${elapsed:-999} -le 300 ]]
+    curl -fsS --max-time 10 https://ai.51easyai.com/api/v1/readyz | grep -q '"ok":true'
+    cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$new_primary" <<'REMOTE'
+set -euo pipefail
+primary=$1
+k3s kubectl exec -n easyai "$primary" -c postgres -- \
+  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -c \
+  "INSERT INTO system_settings(setting_key,value) VALUES('ha_acceptance_probe',jsonb_build_object('at',now())) ON CONFLICT(setting_key) DO UPDATE SET value=excluded.value,updated_at=now();" >/dev/null
+for _ in $(seq 1 180); do
+  if [[ $(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') == 2 ]]; then
+    replication=$(k3s kubectl exec -n easyai "$primary" -c postgres -- \
+      psql -X -U easyai -d easyai_ai_gateway -At -c \
+      "SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
+    [[ $replication == 1 ]] && break
+  fi
+  sleep 2
+done
+[[ ${replication:-0} == 1 ]]
+k3s kubectl exec -n easyai "$primary" -c postgres -- \
+  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -c \
+  "DELETE FROM system_settings WHERE setting_key='ha_acceptance_probe';" >/dev/null
+REMOTE
+    echo "database_failure_drill=PASS promotion_seconds=$elapsed old_primary=$old_primary new_primary=$new_primary"
+    ;;
+  witness)
+    cluster_ssh "$CLUSTER_LOS_ANGELES_HOST" 'systemctl stop k3s'
+    trap 'cluster_ssh "$CLUSTER_LOS_ANGELES_HOST" "systemctl start k3s" || true' EXIT HUP INT TERM
+    cluster_ssh "$CLUSTER_NINGBO_HOST" \
+      "k3s kubectl get --raw='/readyz?verbose' | grep -q '\\[+\\]etcd ok'"
+    curl -fsS --max-time 10 https://ai.51easyai.com/api/v1/readyz | grep -q '"ok":true'
+    cluster_ssh "$CLUSTER_LOS_ANGELES_HOST" 'systemctl start k3s'
+    cluster_ssh "$CLUSTER_NINGBO_HOST" \
+      'k3s kubectl wait --for=condition=Ready node/easyai-los-angeles --timeout=180s'
+    trap - EXIT HUP INT TERM
+    echo 'witness_failure_drill=PASS quorum_retained=true'
+    ;;
+  *)
+    echo 'drill must be api, database, or witness' >&2
+    exit 64
+    ;;
+esac
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+
+load_cluster_env
+hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
+
+for host in "${hosts[@]}"; do
+  echo "[firewall] hardening K3s ports on $host"
+  cluster_ssh "$host" 'bash -s' <<'REMOTE'
+set -euo pipefail
+[[ -d /sys/class/net/wg0 ]] || { echo 'wg0 is unavailable' >&2; exit 1; }
+public_interface=$(ip -o route get 1.1.1.1 | awk '{ for (i = 1; i <= NF; i++) if ($i == "dev") { print $(i + 1); exit } }')
+[[ $public_interface =~ ^[A-Za-z0-9._-]+$ && $public_interface != wg0 ]] || {
+  echo 'cannot determine public interface' >&2
+  exit 1
+}
+export DEBIAN_FRONTEND=noninteractive
+if ! command -v netfilter-persistent >/dev/null 2>&1; then
+  apt-get update -qq
+  apt-get install -y -qq iptables-persistent
+fi
+ensure_rule() {
+  if ! iptables -C INPUT "$@" 2>/dev/null; then
+    iptables -I INPUT 1 "$@"
+  fi
+}
+ensure_rule -i wg0 -s 10.77.0.0/24 -p tcp -m multiport \
+  --dports 2379,2380,6443,10250 -m comment --comment easyai-k3s-private -j ACCEPT
+ensure_rule -i wg0 -s 10.77.0.0/24 -p udp --dport 8472 \
+  -m comment --comment easyai-flannel-private -j ACCEPT
+ensure_rule -i wg0 -s 10.77.0.0/24 -p tcp --dport 30000:32767 \
+  -m comment --comment easyai-nodeport-private -j ACCEPT
+ensure_rule -i "$public_interface" -p udp --dport 51820 \
+  -m comment --comment easyai-wireguard-public -j ACCEPT
+ensure_rule -i "$public_interface" -p tcp -m multiport \
+  --dports 2379,2380,6443,10250 -m comment --comment easyai-k3s-public-drop -j DROP
+ensure_rule -i "$public_interface" -p udp --dport 8472 \
+  -m comment --comment easyai-flannel-public-drop -j DROP
+ensure_rule -i "$public_interface" -p tcp --dport 30000:32767 \
+  -m comment --comment easyai-nodeport-public-drop -j DROP
+netfilter-persistent save >/dev/null
+REMOTE
+done
+
+echo 'node_network_hardening=PASS nodes=3'
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+load_cluster_env
+
+public_key=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+install -d -m 0700 /root/.ssh
+if [[ ! -f /root/.ssh/id_ed25519_easyai_cert_sync ]]; then
+  ssh-keygen -q -t ed25519 -N '' -f /root/.ssh/id_ed25519_easyai_cert_sync
+fi
+cat /root/.ssh/id_ed25519_easyai_cert_sync.pub
+REMOTE
+)
+[[ $public_key == ssh-ed25519\ * ]] || {
+  echo 'invalid certificate sync public key' >&2
+  exit 1
+}
+cluster_ssh "$CLUSTER_HONGKONG_HOST" bash -s -- "$public_key" <<'REMOTE'
+set -euo pipefail
+public_key=$1
+install -d -m 0700 /root/.ssh
+touch /root/.ssh/authorized_keys
+chmod 0600 /root/.ssh/authorized_keys
+grep -Fqx "$public_key" /root/.ssh/authorized_keys || printf '%s\n' "$public_key" >>/root/.ssh/authorized_keys
+REMOTE
+
+cluster_scp "$script_dir/sync-nginx-certificate.sh" \
+  "$CLUSTER_NINGBO_HOST:/usr/local/sbin/easyai-sync-nginx-certificate"
+cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+chmod 0750 /usr/local/sbin/easyai-sync-nginx-certificate
+install -d -m 0755 /etc/letsencrypt/renewal-hooks/deploy
+ln -sfn /usr/local/sbin/easyai-sync-nginx-certificate \
+  /etc/letsencrypt/renewal-hooks/deploy/easyai-sync-nginx-certificate
+/usr/local/sbin/easyai-sync-nginx-certificate
+REMOTE
+echo 'certificate_sync_install=PASS'
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+load_cluster_env
+
+cluster_scp "$script_dir/prune-legacy-generated.sh" \
+  "$CLUSTER_NINGBO_HOST:/usr/local/sbin/easyai-prune-legacy-generated"
+cluster_scp "$cluster_root/deploy/systemd/easyai-legacy-generated-cleanup.service" \
+  "$CLUSTER_NINGBO_HOST:/etc/systemd/system/easyai-legacy-generated-cleanup.service"
+cluster_scp "$cluster_root/deploy/systemd/easyai-legacy-generated-cleanup.timer" \
+  "$CLUSTER_NINGBO_HOST:/etc/systemd/system/easyai-legacy-generated-cleanup.timer"
+cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+chmod 0750 /usr/local/sbin/easyai-prune-legacy-generated
+systemctl daemon-reload
+/usr/local/sbin/easyai-prune-legacy-generated --dry-run
+systemctl enable --now easyai-legacy-generated-cleanup.timer
+systemctl is-active --quiet easyai-legacy-generated-cleanup.timer
+REMOTE
+echo 'legacy_cleanup_timer_install=PASS interval=hourly strict_age_seconds=86400'
@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+
+mode=${1:-prepare}
+[[ $mode == prepare || $mode == activate ]] || {
+  echo 'usage: install-nginx-edges.sh [prepare|activate]' >&2
+  exit 64
+}
+load_cluster_env
+
+nginx_directory="$cluster_root/deploy/nginx"
+for file in ai.51easyai.com-cluster.conf ai.51easyai.com-maintenance.conf \
+  cluster-acceptance-origin-ningbo.conf legacy-static-ningbo.inc \
+  legacy-static-hongkong.inc legacy-static-origin-ningbo.conf; do
+  [[ -f $nginx_directory/$file ]] || {
+    echo "missing NGINX file: $file" >&2
+    exit 1
+  }
+done
+
+cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+install -d -m 0700 /etc/nginx/tls/ai.51easyai.com
+install -m 0644 /etc/letsencrypt/live/ai.51easyai.com/fullchain.pem \
+  /etc/nginx/tls/ai.51easyai.com/fullchain.pem
+install -m 0600 /etc/letsencrypt/live/ai.51easyai.com/privkey.pem \
+  /etc/nginx/tls/ai.51easyai.com/privkey.pem
+REMOTE
+cluster_scp \
+  "$CLUSTER_NINGBO_HOST:/etc/nginx/tls/ai.51easyai.com/fullchain.pem" \
+  "$cluster_root/.local-secrets/cluster/fullchain.pem"
+cluster_scp \
+  "$CLUSTER_NINGBO_HOST:/etc/nginx/tls/ai.51easyai.com/privkey.pem" \
+  "$cluster_root/.local-secrets/cluster/privkey.pem"
+chmod 0600 "$cluster_root/.local-secrets/cluster/privkey.pem"
+
+for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"; do
+  cluster_ssh "$host" 'bash -s' <<'REMOTE'
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+if ! command -v nginx >/dev/null 2>&1; then
+  apt-get update -qq
+  apt-get install -y -qq nginx
+fi
+install -d -m 0755 /etc/nginx/easyai-ai-gateway
+install -d -m 0700 /etc/nginx/tls/ai.51easyai.com
+REMOTE
+  cluster_scp "$cluster_root/.local-secrets/cluster/fullchain.pem" \
+    "$host:/etc/nginx/tls/ai.51easyai.com/fullchain.pem"
+  cluster_scp "$cluster_root/.local-secrets/cluster/privkey.pem" \
+    "$host:/etc/nginx/tls/ai.51easyai.com/privkey.pem"
+  cluster_scp "$nginx_directory/ai.51easyai.com-cluster.conf" \
+    "$host:/etc/nginx/sites-available/ai.51easyai.com-cluster"
+  cluster_scp "$nginx_directory/ai.51easyai.com-maintenance.conf" \
+    "$host:/etc/nginx/sites-available/ai.51easyai.com-maintenance"
+done
+
+cluster_scp "$nginx_directory/legacy-static-ningbo.inc" \
+  "$CLUSTER_NINGBO_HOST:/etc/nginx/easyai-ai-gateway/legacy-static.inc"
+cluster_scp "$nginx_directory/legacy-static-origin-ningbo.conf" \
+  "$CLUSTER_NINGBO_HOST:/etc/nginx/conf.d/easyai-legacy-static-origin.conf"
+cluster_scp "$nginx_directory/cluster-acceptance-origin-ningbo.conf" \
+  "$CLUSTER_NINGBO_HOST:/etc/nginx/conf.d/easyai-cluster-acceptance-origin.conf"
+cluster_scp "$nginx_directory/legacy-static-hongkong.inc" \
+  "$CLUSTER_HONGKONG_HOST:/etc/nginx/easyai-ai-gateway/legacy-static.inc"
+
+for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"; do
+  cluster_ssh "$host" 'bash -s' <<'REMOTE'
+set -euo pipefail
+chmod 0600 /etc/nginx/tls/ai.51easyai.com/privkey.pem
+chmod 0644 /etc/nginx/tls/ai.51easyai.com/fullchain.pem
+nginx -t
+REMOTE
+done
+
+if [[ $mode == activate ]]; then
+  for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"; do
+    cluster_ssh "$host" 'bash -s' <<'REMOTE'
+set -euo pipefail
+ln -sfn /etc/nginx/sites-available/ai.51easyai.com-cluster \
+  /etc/nginx/sites-enabled/ai.51easyai.com-cluster
+if [[ -L /etc/nginx/sites-enabled/ai.51easyai.com ]]; then
+  rm -f /etc/nginx/sites-enabled/ai.51easyai.com
+fi
+nginx -t
+systemctl reload nginx
+REMOTE
+  done
+fi
+
+echo "nginx_edges=PASS mode=$mode"
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+load_cluster_env
+
+cluster_ssh "$CLUSTER_NINGBO_HOST" \
+  'install -d -m 0755 /usr/local/lib/easyai-ai-gateway-release'
+cluster_scp "$cluster_root/deploy/kubernetes/easyai-ai-gateway-cluster-release" \
+  "$CLUSTER_NINGBO_HOST:/usr/local/sbin/easyai-ai-gateway-cluster-release"
+cluster_scp "$cluster_root/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example" \
+  "$CLUSTER_NINGBO_HOST:/etc/easyai-ai-gateway-cluster-release.conf"
+cluster_scp "$cluster_root/scripts/release-manifest.mjs" \
+  "$CLUSTER_NINGBO_HOST:/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs"
+cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+chmod 0750 /usr/local/sbin/easyai-ai-gateway-cluster-release
+chmod 0640 /etc/easyai-ai-gateway-cluster-release.conf
+chmod 0755 /usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs
+REMOTE
+echo 'cluster_release_helper_install=PASS'
@@ -0,0 +1,362 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+
+action=${1:-preflight}
+release_manifest=${2:-}
+[[ $action == preflight || $action == cutover ]] || {
+  echo 'usage: migrate-production.sh {preflight|cutover} dist/releases/<sha>.json' >&2
+  exit 64
+}
+[[ -f $release_manifest && ! -L $release_manifest ]] || {
+  echo 'release manifest must be a regular file' >&2
+  exit 1
+}
+
+load_cluster_env
+node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null
+source_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha)
+api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.api)
+
+if [[ $action == cutover && ${AI_GATEWAY_CUTOVER_WINDOW_OVERRIDE:-} != I_ACCEPT_PRODUCTION_RISK ]]; then
+  shanghai_hour=$(TZ=Asia/Shanghai date '+%H')
+  if (( 10#$shanghai_hour < 2 || 10#$shanghai_hour >= 4 )); then
+    echo 'cutover is restricted to 02:00-04:00 Asia/Shanghai' >&2
+    exit 1
+  fi
+fi
+
+echo '[cutover] checking Docker, K3s, CNPG, disk and backup prerequisites'
+cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+docker inspect easyai-ai-gateway-api-1 easyai-ai-gateway-postgres-1 >/dev/null
+docker exec easyai-ai-gateway-postgres-1 pg_isready >/dev/null
+k3s kubectl get --raw=/readyz >/dev/null
+[[ $(k3s kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq 3 ]]
+[[ $(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') -eq 2 ]]
+[[ $(df --output=avail -B1 /var/lib/docker | tail -1) -gt 2147483648 ]]
+[[ -f /etc/nginx/sites-available/ai.51easyai.com-maintenance ]]
+[[ -f /etc/nginx/sites-available/ai.51easyai.com-cluster ]]
+nginx -t
+REMOTE
+
+if [[ $action == preflight ]]; then
+  echo "production_migration_preflight=PASS release=$source_sha"
+  exit 0
+fi
+
+working_directory=$(mktemp -d "$cluster_root/.local-secrets/cluster/cutover.XXXXXX")
+trap '[[ $working_directory == "$cluster_root"/.local-secrets/cluster/cutover.* ]] && rm -rf -- "$working_directory"' EXIT HUP INT TERM
+cluster_scp "$script_dir/oss-object.mjs" "$CLUSTER_NINGBO_HOST:/root/easyai-oss-object.mjs"
+
+remote_cutover_script="$working_directory/remote-cutover.sh"
+umask 077
+cp /dev/null "$remote_cutover_script"
+chmod 0700 "$remote_cutover_script"
+cat >"$remote_cutover_script" <<'REMOTE'
+#!/usr/bin/env bash
+set -euo pipefail
+
+source_sha=$1
+api_image=$2
+active_site=${AI_GATEWAY_NGINX_ACTIVE_SITE:-/etc/nginx/sites-enabled/ai.51easyai.com}
+timestamp=$(date -u '+%Y%m%dT%H%M%SZ')
+backup_directory=/var/backups/easyai-ai-gateway/pre-cutover-"$timestamp"
+old_site_backup=$backup_directory/nginx-site.conf
+cutover_started=$(date +%s)
+traffic_committed=false
+
+rollback_before_commit() {
+  status=$?
+  trap - ERR
+  if [[ $traffic_committed == false ]]; then
+    echo '[cutover] rolling back to the frozen Docker database' >&2
+    k3s kubectl scale deployment -n easyai \
+      easyai-api-ningbo easyai-api-hongkong easyai-web-ningbo easyai-web-hongkong \
+      --replicas=0 >/dev/null 2>&1 || true
+    docker start easyai-ai-gateway-api-1 easyai-ai-gateway-web-1 >/dev/null 2>&1 || true
+    if [[ -f $old_site_backup ]]; then
+      install -m 0644 "$old_site_backup" /etc/nginx/sites-available/ai.51easyai.com-pre-ha
+      rm -f -- "$active_site"
+      ln -s /etc/nginx/sites-available/ai.51easyai.com-pre-ha "$active_site"
+      nginx -t && systemctl reload nginx
+    fi
+  else
+    echo '[cutover] traffic is already committed; automatic database rollback is unsafe' >&2
+  fi
+  exit "$status"
+}
+trap rollback_before_commit ERR
+
+install -d -m 0700 "$backup_directory"
+[[ -e $active_site || -L $active_site ]] || {
+  echo "active NGINX site is unavailable: $active_site" >&2
+  exit 1
+}
+cp -L "$active_site" "$old_site_backup"
+chmod 0600 "$old_site_backup"
+rm -f -- "$active_site"
+ln -s /etc/nginx/sites-available/ai.51easyai.com-maintenance "$active_site"
+nginx -t
+systemctl reload nginx
+curl -ksS --resolve ai.51easyai.com:443:127.0.0.1 \
+  -o /dev/null -w '%{http_code}' https://ai.51easyai.com/api/v1/healthz |
+  grep -qx 503
+
+docker stop --time 60 easyai-ai-gateway-api-1 >/dev/null
+postgres_user=$(docker inspect easyai-ai-gateway-postgres-1 \
+  --format '{{range .Config.Env}}{{println .}}{{end}}' |
+  awk -F= '$1 == "POSTGRES_USER" { print substr($0, index($0, "=") + 1) }')
+postgres_database=$(docker inspect easyai-ai-gateway-postgres-1 \
+  --format '{{range .Config.Env}}{{println .}}{{end}}' |
+  awk -F= '$1 == "POSTGRES_DB" { print substr($0, index($0, "=") + 1) }')
+[[ $postgres_user =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]
+[[ $postgres_database =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]
+
+snapshot_sql=$backup_directory/acceptance-snapshot.sql
+cat >"$snapshot_sql" <<'SQL'
+\pset tuples_only on
+\pset format unaligned
+SELECT 'server_version=' || current_setting('server_version');
+SELECT 'extensions=' || COALESCE(string_agg(extname || ':' || extversion, ',' ORDER BY extname), '')
+  FROM pg_extension WHERE extname <> 'plpgsql';
+SELECT 'tables=' || count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+  WHERE c.relkind = 'r' AND n.nspname = 'public';
+SELECT 'sequences=' || COALESCE(string_agg(schemaname || '.' || sequencename || ':' ||
+  COALESCE(last_value::text, 'null'), ',' ORDER BY schemaname, sequencename), '')
+  FROM pg_sequences WHERE schemaname = 'public';
+SELECT 'catalog=' || md5(COALESCE(string_agg(definition, E'\n' ORDER BY definition), ''))
+FROM (
+  SELECT 'column:' || table_name || ':' || column_name || ':' || data_type || ':' ||
+    is_nullable || ':' || COALESCE(column_default, '') AS definition
+  FROM information_schema.columns WHERE table_schema = 'public'
+  UNION ALL
+  SELECT 'constraint:' || conrelid::regclass::text || ':' || conname || ':' ||
+    pg_get_constraintdef(oid) FROM pg_constraint WHERE connamespace = 'public'::regnamespace
+  UNION ALL
+  SELECT 'index:' || indexrelid::regclass::text || ':' || pg_get_indexdef(indexrelid)
+    FROM pg_index WHERE indrelid IN (
+      SELECT oid FROM pg_class WHERE relnamespace = 'public'::regnamespace
+    )
+) definitions;
+SELECT 'gateway_tasks=' || count(*) FROM gateway_tasks;
+SELECT 'gateway_task_attempts=' || count(*) FROM gateway_task_attempts;
+SELECT 'gateway_wallet_transactions=' || count(*) FROM gateway_wallet_transactions;
+SELECT 'gateway_upload_assets=' || count(*) FROM gateway_upload_assets;
+SELECT 'gateway_users=' || count(*) FROM gateway_users;
+SELECT 'gateway_api_keys=' || count(*) FROM gateway_api_keys;
+SELECT 'gateway_providers=' || count(*) FROM gateway_providers;
+SELECT 'task_sample=' || md5(COALESCE(string_agg(id::text, ',' ORDER BY id), ''))
+  FROM (SELECT id FROM gateway_tasks ORDER BY id LIMIT 100) sample;
+SQL
+
+docker exec -i easyai-ai-gateway-postgres-1 \
+  psql -X -v ON_ERROR_STOP=1 -U "$postgres_user" -d "$postgres_database" \
+  <"$snapshot_sql" >"$backup_directory/source.snapshot"
+docker exec easyai-ai-gateway-postgres-1 \
+  pg_dumpall -U "$postgres_user" --globals-only \
+  >"$backup_directory/globals.sql"
+docker exec easyai-ai-gateway-postgres-1 \
+  pg_dump -U "$postgres_user" -d "$postgres_database" \
+  --format=custom --compress=6 \
+  >"$backup_directory/database.dump"
+[[ -s $backup_directory/database.dump && -s $backup_directory/globals.sql ]]
+docker exec -i easyai-ai-gateway-postgres-1 pg_restore -l \
+  <"$backup_directory/database.dump" >/dev/null
+sha256sum "$backup_directory/database.dump" "$backup_directory/globals.sql" \
+  "$backup_directory/source.snapshot" >"$backup_directory/SHA256SUMS"
+sha256sum -c "$backup_directory/SHA256SUMS"
+
+export ALI_KEY
+export ALI_SECRET
+ALI_KEY=$(k3s kubectl get secret easyai-oss-backup -n easyai \
+  -o jsonpath='{.data.ACCESS_KEY_ID}' | base64 -d)
+ALI_SECRET=$(k3s kubectl get secret easyai-oss-backup -n easyai \
+  -o jsonpath='{.data.ACCESS_SECRET_KEY}' | base64 -d)
+export ALI_REGION=oss-cn-shanghai
+export ALI_BUCKET=wangbo0808
+object_prefix=easyai-ai-gateway/production/pre-cutover/"$timestamp"
+for file in database.dump globals.sql source.snapshot SHA256SUMS; do
+  node /root/easyai-oss-object.mjs put "$object_prefix/$file" "$backup_directory/$file"
+done
+node /root/easyai-oss-object.mjs head "$object_prefix/database.dump"
+
+primary=$(k3s kubectl get cluster easyai-postgres -n easyai \
+  -o jsonpath='{.status.currentPrimary}')
+[[ $primary =~ ^easyai-postgres-[0-9]+$ ]]
+k3s kubectl exec -n easyai "$primary" -c postgres -- \
+  psql -X -v ON_ERROR_STOP=1 -U postgres -d postgres \
+  -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'easyai_ai_gateway' AND pid <> pg_backend_pid();" \
+  -c 'DROP DATABASE IF EXISTS easyai_ai_gateway;' \
+  -c 'CREATE DATABASE easyai_ai_gateway OWNER easyai;'
+k3s kubectl exec -i -n easyai "$primary" -c postgres -- \
+  pg_restore --exit-on-error --no-owner --role=easyai \
+  -U postgres -d easyai_ai_gateway <"$backup_directory/database.dump"
+k3s kubectl exec -i -n easyai "$primary" -c postgres -- \
+  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway \
+  <"$snapshot_sql" >"$backup_directory/restored.snapshot"
+diff -u "$backup_directory/source.snapshot" "$backup_directory/restored.snapshot"
+
+cat >"$backup_directory/migrator-job.yaml" <<EOF
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: easyai-migrator-${source_sha:0:12}
+  namespace: easyai
+  labels:
+    app.kubernetes.io/part-of: easyai-ai-gateway
+    easyai.io/release: ${source_sha}
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 86400
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: easyai-migrator
+    spec:
+      restartPolicy: Never
+      imagePullSecrets:
+        - name: easyai-registry
+      nodeSelector:
+        easyai.io/site: ningbo
+      containers:
+        - name: migrator
+          image: ${api_image}
+          command: ["/app/easyai-ai-gateway-migrate"]
+          envFrom:
+            - secretRef:
+                name: easyai-ai-gateway-runtime
+EOF
+k3s kubectl delete job "easyai-migrator-${source_sha:0:12}" -n easyai \
+  --ignore-not-found=true --wait=true >/dev/null
+k3s kubectl apply -f "$backup_directory/migrator-job.yaml" >/dev/null
+k3s kubectl wait --for=condition=complete \
+  "job/easyai-migrator-${source_sha:0:12}" -n easyai --timeout=300s
+k3s kubectl exec -n easyai "$primary" -c postgres -- \
+  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \
+  "SELECT count(*) FROM schema_migrations WHERE version='0089_file_storage_request_asset_scene';" |
+  grep -qx 1
+k3s kubectl exec -n easyai "$primary" -c postgres -- \
+  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \
+  "SELECT count(*) FROM file_storage_channels WHERE deleted_at IS NULL AND provider='server_main_openapi' AND config->'scenes' ? 'request_asset';" |
+  grep -Eq '^[1-9][0-9]*$'
+
+elapsed=$(( $(date +%s) - cutover_started ))
+if (( elapsed > 600 )); then
+  echo "cutover exceeded ten-minute rollback boundary: ${elapsed}s" >&2
+  exit 1
+fi
+printf '%s\n' "$backup_directory" >/root/.easyai-pre-cutover-backup
+printf 'database_restore=PASS seconds=%s backup=%s\n' "$elapsed" "$backup_directory"
+REMOTE
+
+cluster_scp "$remote_cutover_script" "$CLUSTER_NINGBO_HOST:/root/easyai-remote-cutover.sh"
+cluster_ssh "$CLUSTER_NINGBO_HOST" \
+  "chmod 0700 /root/easyai-remote-cutover.sh && /root/easyai-remote-cutover.sh '$source_sha' '$api_image'"
+
+traffic_committed=false
+rollback_local_before_commit() {
+  status=$?
+  trap - ERR
+  if [[ $traffic_committed == false ]]; then
+    echo '[cutover] local activation failed; restoring Docker before public traffic commit' >&2
+    cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' || true
+set -euo pipefail
+backup_directory=$(cat /root/.easyai-pre-cutover-backup)
+[[ $backup_directory == /var/backups/easyai-ai-gateway/pre-cutover-* ]]
+k3s kubectl scale deployment -n easyai \
+  easyai-api-ningbo easyai-api-hongkong easyai-web-ningbo easyai-web-hongkong \
+  --replicas=0 >/dev/null 2>&1 || true
+docker start easyai-ai-gateway-api-1 easyai-ai-gateway-web-1 >/dev/null
+rm -f -- /etc/nginx/sites-enabled/ai.51easyai.com
+install -m 0644 "$backup_directory/nginx-site.conf" \
+  /etc/nginx/sites-available/ai.51easyai.com-pre-ha
+ln -s /etc/nginx/sites-available/ai.51easyai.com-pre-ha \
+  /etc/nginx/sites-enabled/ai.51easyai.com
+nginx -t
+systemctl reload nginx
+REMOTE
+  fi
+  exit "$status"
+}
+trap rollback_local_before_commit ERR
+
+echo '[cutover] starting both application sites with River workers frozen'
+AI_GATEWAY_ACTIVATE_WORKERS=false \
+  "$script_dir/bootstrap-platform.sh" activate "$release_manifest"
+
+for site_ip in 10.77.0.1 10.77.0.2; do
+  cluster_ssh "$CLUSTER_NINGBO_HOST" \
+    "curl -fsS --max-time 10 http://$site_ip:30088/api/v1/healthz >/dev/null"
+  cluster_ssh "$CLUSTER_NINGBO_HOST" \
+    "curl -fsS --max-time 10 http://$site_ip:30088/api/v1/readyz | grep -q '\"ok\":true'"
+  cluster_ssh "$CLUSTER_NINGBO_HOST" \
+    "curl -fsS --max-time 10 http://$site_ip:30178/ | grep -q 'EasyAI AI Gateway'"
+done
+
+"$script_dir/install-nginx-edges.sh" activate
+for edge_host in "${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}"; do
+  curl -fsS --resolve "ai.51easyai.com:443:$edge_host" \
+    https://ai.51easyai.com/api/v1/healthz >/dev/null
+  curl -fsS --resolve "ai.51easyai.com:443:$edge_host" \
+    https://ai.51easyai.com/api/v1/readyz | grep -q '"ok":true'
+done
+
+traffic_committed=true
+echo '[cutover] traffic committed; enabling Hong Kong worker, then Ningbo worker'
+cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+kubectl='k3s kubectl'
+$kubectl set env deployment/easyai-api-hongkong -n easyai \
+  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true
+$kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s
+$kubectl set env deployment/easyai-api-ningbo -n easyai \
+  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true
+$kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
+cat <<'EOF' | $kubectl create -f - >/dev/null
+apiVersion: postgresql.cnpg.io/v1
+kind: Backup
+metadata:
+  generateName: easyai-post-cutover-
+  namespace: easyai
+spec:
+  cluster:
+    name: easyai-postgres
+  method: plugin
+  pluginConfiguration:
+    name: barman-cloud.cloudnative-pg.io
+EOF
+docker update --restart=no easyai-ai-gateway-api-1 \
+  easyai-ai-gateway-web-1 easyai-ai-gateway-postgres-1 >/dev/null
+docker stop --time 30 easyai-ai-gateway-web-1 easyai-ai-gateway-postgres-1 >/dev/null
+rm -f -- /root/easyai-remote-cutover.sh /root/easyai-oss-object.mjs
+REMOTE
+
+"$script_dir/install-release-helper.sh"
+remote_manifest=/tmp/easyai-cutover-release-"$source_sha".json
+cluster_scp "$release_manifest" "$CLUSTER_NINGBO_HOST:$remote_manifest"
+cluster_ssh "$CLUSTER_NINGBO_HOST" \
+  "/usr/local/sbin/easyai-ai-gateway-cluster-release adopt '$remote_manifest' && rm -f '$remote_manifest'"
+
+set_local_environment() {
+  local key=$1
+  local value=$2
+  local temporary
+  temporary=$(mktemp "$cluster_env_file.next.XXXXXX")
+  chmod 0600 "$temporary"
+  awk -v key="$key" -v value="$value" -F= '
+    $1 == key { if (!written) print key "=" value; written = 1; next }
+    { print }
+    END { if (!written) print key "=" value }
+  ' "$cluster_env_file" >"$temporary"
+  mv "$temporary" "$cluster_env_file"
+}
+set_local_environment AI_GATEWAY_DEPLOY_MODE kubernetes
+set_local_environment AI_GATEWAY_REMOTE_RELEASE_HELPER /usr/local/sbin/easyai-ai-gateway-cluster-release
+set_local_environment AI_GATEWAY_PRODUCTION_SSH_KEY "$cluster_ssh_key"
+
+trap - ERR
+echo "production_cutover=PASS release=$source_sha old_postgres_retained=true"
@@ -0,0 +1,103 @@
+#!/usr/bin/env node
+
+import { createHmac, createHash } from 'node:crypto'
+import { createReadStream, createWriteStream, statSync } from 'node:fs'
+import { request } from 'node:https'
+import { pipeline } from 'node:stream/promises'
+
+function fail(message) {
+  console.error(`oss_object=FAIL ${message}`)
+  process.exit(1)
+}
+
+function requiredEnv(name, pattern) {
+  const value = process.env[name] || ''
+  if (!value || (pattern && !pattern.test(value))) fail(`invalid or missing ${name}`)
+  return value
+}
+
+function encodeObjectKey(value) {
+  if (!value || value.startsWith('/') || value.includes('..')) fail('invalid object key')
+  return value.split('/').map(encodeURIComponent).join('/')
+}
+
+const [operation, objectKeyValue, filePath] = process.argv.slice(2)
+if (!['put', 'get', 'head', 'delete'].includes(operation || '')) {
+  fail('usage: oss-object.mjs {put|get|head|delete} <object-key> [file]')
+}
+if ((operation === 'put' || operation === 'get') && !filePath) fail(`${operation} requires a file`)
+
+const accessKey = requiredEnv('ALI_KEY')
+const secretKey = requiredEnv('ALI_SECRET')
+const region = requiredEnv('ALI_REGION', /^[a-z0-9-]+$/)
+const bucket = requiredEnv('ALI_BUCKET', /^[A-Za-z0-9.-]+$/)
+const objectKey = encodeObjectKey(objectKeyValue)
+const hostname = `${bucket}.${region}.aliyuncs.com`
+const canonicalResource = `/${bucket}/${objectKey}`
+const date = new Date().toUTCString()
+const method = operation.toUpperCase()
+const contentType = operation === 'put' ? 'application/octet-stream' : ''
+const stringToSign = `${method}\n\n${contentType}\n${date}\n${canonicalResource}`
+const signature = createHmac('sha1', secretKey).update(stringToSign).digest('base64')
+const headers = {
+  Authorization: `OSS ${accessKey}:${signature}`,
+  Date: date,
+}
+
+let expectedDigest = ''
+if (operation === 'put') {
+  let details
+  try {
+    details = statSync(filePath)
+  } catch (error) {
+    fail(`cannot stat upload file: ${error.message}`)
+  }
+  if (!details.isFile()) fail('upload source must be a regular file')
+  headers['Content-Type'] = contentType
+  headers['Content-Length'] = String(details.size)
+  expectedDigest = createHash('sha256')
+}
+
+const response = await new Promise((resolve, reject) => {
+  const outgoing = request({
+    method,
+    hostname,
+    path: `/${objectKey}`,
+    headers,
+    timeout: 120_000,
+  }, resolve)
+  outgoing.once('timeout', () => outgoing.destroy(new Error('request timed out')))
+  outgoing.once('error', reject)
+  if (operation === 'put') {
+    const source = createReadStream(filePath)
+    source.on('data', (chunk) => expectedDigest.update(chunk))
+    source.once('error', reject)
+    source.pipe(outgoing)
+  } else {
+    outgoing.end()
+  }
+}).catch((error) => fail(error.message))
+
+const successStatus = operation === 'get' || operation === 'head' ? 200 : [200, 204].includes(response.statusCode)
+if (!successStatus) {
+  let responseBody = ''
+  for await (const chunk of response) responseBody += chunk
+  fail(`HTTP ${response.statusCode} ${responseBody.slice(0, 240).replaceAll(/\s+/g, ' ')}`)
+}
+
+if (operation === 'get') {
+  const destination = createWriteStream(filePath, { flags: 'wx', mode: 0o600 })
+  await pipeline(response, destination).catch((error) => fail(`download failed: ${error.message}`))
+}
+
+if (operation === 'put') {
+  for await (const _chunk of response) {
+    // Drain the response so the request completes before reporting success.
+  }
+  console.log(`oss_object=PASS operation=put sha256=${expectedDigest.digest('hex')}`)
+} else {
+  for await (const _chunk of response) {
+    // GET is consumed by pipeline; other successful response bodies are empty.
+  }
+  console.log(`oss_object=PASS operation=${operation}`)
+}
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+mode=${1:---dry-run}
+storage_root=${AI_GATEWAY_LEGACY_STORAGE_ROOT:-/var/lib/docker/volumes/easyai-ai-gateway_api_data/_data/static}
+maximum_age_seconds=86400
+
+[[ $mode == --dry-run || $mode == --apply ]] || {
+  echo 'usage: prune-legacy-generated.sh [--dry-run|--apply]' >&2
+  exit 64
+}
+[[ $storage_root == /var/lib/docker/volumes/*/_data/static && -d $storage_root && ! -L $storage_root ]] || {
+  echo 'legacy storage root failed safety validation' >&2
+  exit 1
+}
+
+candidate_list=$(mktemp)
+trap 'rm -f -- "$candidate_list"' EXIT HUP INT TERM
+now_epoch=$(date +%s)
+while IFS= read -r -d '' file; do
+  modified_epoch=$(stat -c '%Y' "$file")
+  if (( now_epoch - modified_epoch > maximum_age_seconds )); then
+    printf '%s\0' "$file" >>"$candidate_list"
+  fi
+done < <(find "$storage_root" -xdev -type f -print0)
+
+file_count=$(tr -cd '\0' <"$candidate_list" | wc -c | tr -d ' ')
+byte_count=0
+while IFS= read -r -d '' file; do
+  [[ $file == "$storage_root/"* && ! -L $file ]] || {
+    echo "unsafe cleanup candidate: $file" >&2
+    exit 1
+  }
+  byte_count=$((byte_count + $(stat -c '%s' "$file")))
+done <"$candidate_list"
+
+if [[ $mode == --apply ]]; then
+  while IFS= read -r -d '' file; do
+    rm -f -- "$file"
+  done <"$candidate_list"
+  find "$storage_root" -xdev -depth -type d -empty -delete
+fi
+
+printf 'legacy_cleanup=%s files=%s bytes=%s age_seconds_strictly_greater_than=86400\n' \
+  "${mode#--}" "$file_count" "$byte_count"
@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+
+[[ ${1:-} == --execute ]] || {
+  echo 'usage: restore-backup-drill.sh --execute' >&2
+  exit 64
+}
+load_cluster_env
+
+cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+kubectl='k3s kubectl'
+suffix=$(date -u '+%m%d%H%M%S')
+restore_cluster=easyai-restore-drill-"$suffix"
+cleanup() {
+  $kubectl delete cluster "$restore_cluster" -n easyai \
+    --ignore-not-found=true --wait=true --timeout=300s || true
+  $kubectl delete pvc -n easyai -l "cnpg.io/cluster=$restore_cluster" \
+    --ignore-not-found=true --wait=true --timeout=300s || true
+}
+trap cleanup EXIT HUP INT TERM
+
+source_primary=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
+acceptance_sql=$(mktemp)
+trap 'rm -f -- "$acceptance_sql"; cleanup' EXIT HUP INT TERM
+cat >"$acceptance_sql" <<'SQL'
+\pset tuples_only on
+\pset format unaligned
+SELECT 'tables=' || count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
+ WHERE c.relkind='r' AND n.nspname='public';
+SELECT 'tasks=' || count(*) FROM gateway_tasks;
+SELECT 'attempts=' || count(*) FROM gateway_task_attempts;
+SELECT 'wallet=' || count(*) FROM gateway_wallet_transactions;
+SELECT 'users=' || count(*) FROM gateway_users;
+SELECT 'sample=' || md5(COALESCE(string_agg(id::text, ',' ORDER BY id),''))
+ FROM (SELECT id FROM gateway_tasks ORDER BY id LIMIT 100) rows;
+SQL
+source_snapshot=$($kubectl exec -i -n easyai "$source_primary" -c postgres -- \
+  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway <"$acceptance_sql")
+
+cat <<EOF | $kubectl apply -f - >/dev/null
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: $restore_cluster
+  namespace: easyai
+  labels:
+    easyai.io/acceptance-drill: backup-restore
+spec:
+  instances: 1
+  imageName: ghcr.io/cloudnative-pg/postgresql:18.4-standard-trixie@sha256:4587df73024408f5b2be9b4dd6ba2ccee8c9e5dc0c9a87c274c292291cc8a68c
+  bootstrap:
+    recovery:
+      source: origin
+      database: easyai_ai_gateway
+      owner: easyai
+      secret:
+        name: easyai-postgres-app
+  externalClusters:
+    - name: origin
+      plugin:
+        name: barman-cloud.cloudnative-pg.io
+        parameters:
+          barmanObjectName: easyai-postgres-oss
+          serverName: easyai-postgres
+  storage:
+    storageClass: local-path
+    size: 20Gi
+  walStorage:
+    storageClass: local-path
+    size: 5Gi
+  affinity:
+    nodeSelector:
+      easyai.io/site: ningbo
+  resources:
+    requests:
+      cpu: 250m
+      memory: 512Mi
+    limits:
+      cpu: "2"
+      memory: 3Gi
+EOF
+
+$kubectl wait --for=condition=Ready "cluster/$restore_cluster" -n easyai --timeout=1200s
+restore_primary=$($kubectl get cluster "$restore_cluster" -n easyai -o jsonpath='{.status.currentPrimary}')
+restored_snapshot=$($kubectl exec -i -n easyai "$restore_primary" -c postgres -- \
+  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway <"$acceptance_sql")
+[[ $restored_snapshot == "$source_snapshot" ]] || {
+  diff -u <(printf '%s\n' "$source_snapshot") <(printf '%s\n' "$restored_snapshot")
+  exit 1
+}
+$kubectl exec -n easyai "$restore_primary" -c postgres -- \
+  psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \
+  "SELECT current_user || ':' || current_database();" | grep -qx 'easyai:easyai_ai_gateway'
+cleanup
+trap - EXIT HUP INT TERM
+rm -f -- "$acceptance_sql"
+echo "backup_restore_drill=PASS cluster=$restore_cluster deleted=true"
+REMOTE
@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+load_cluster_env
+require_commands base64 jq node
+
+: "${AI_GATEWAY_ONLINE_ACCOUNT:?}"
+: "${AI_GATEWAY_ONLINE_PASSWORD:?}"
+
+restore_workers() {
+  cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' || true
+set -euo pipefail
+k3s kubectl set env deployment/easyai-api-ningbo -n easyai \
+  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true >/dev/null
+k3s kubectl set env deployment/easyai-api-hongkong -n easyai \
+  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true >/dev/null
+k3s kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
+k3s kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s
+REMOTE
+}
+trap restore_workers EXIT HUP INT TERM
+
+cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+k3s kubectl set env deployment/easyai-api-hongkong -n easyai \
+  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true >/dev/null
+k3s kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s
+k3s kubectl set env deployment/easyai-api-ningbo -n easyai \
+  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=false >/dev/null
+k3s kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
+curl -fsS http://10.77.0.1:30088/api/v1/readyz | grep -q '"ok":true'
+curl -fsS http://10.77.0.2:30088/api/v1/readyz | grep -q '"ok":true'
+REMOTE
+
+credentials_file=$(mktemp "$cluster_root/.local-secrets/cluster/e2e.XXXXXX")
+trap 'rm -f -- "$credentials_file"; restore_workers' EXIT HUP INT TERM
+chmod 0600 "$credentials_file"
+printf 'E2E_ACCOUNT_B64=%s\n' "$(printf '%s' "$AI_GATEWAY_ONLINE_ACCOUNT" | base64 | tr -d '\n')" \
+  >"$credentials_file"
+printf 'E2E_PASSWORD_B64=%s\n' "$(printf '%s' "$AI_GATEWAY_ONLINE_PASSWORD" | base64 | tr -d '\n')" \
+  >>"$credentials_file"
+printf 'AI_GATEWAY_E2E_IMAGE_MODEL=%s\n' "${AI_GATEWAY_E2E_IMAGE_MODEL:-openai:gpt-image-1}" \
+  >>"$credentials_file"
+cluster_scp "$credentials_file" "$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-e2e.env"
+cluster_scp "$script_dir/cross-node-file-e2e.mjs" \
+  "$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-file-e2e.mjs"
+
+result=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+chmod 0600 /root/easyai-cross-node-e2e.env
+set -a
+# shellcheck source=/dev/null
+source /root/easyai-cross-node-e2e.env
+set +a
+node /root/easyai-cross-node-file-e2e.mjs
+rm -f -- /root/easyai-cross-node-e2e.env /root/easyai-cross-node-file-e2e.mjs
+REMOTE
+)
+task_id=$(jq -r '.taskId' <<<"$result")
+source_sha=$(jq -r '.sourceSHA256' <<<"$result")
+result_url=$(jq -r '.resultURL' <<<"$result")
+[[ $task_id =~ ^[0-9a-f-]{36}$ && $source_sha =~ ^[0-9a-f]{64}$ && $result_url == https://* ]]
+
+database_evidence=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$task_id" "$source_sha" <<'REMOTE'
+set -euo pipefail
+task_id=$1
+source_sha=$2
+primary=$(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
+query() {
+  k3s kubectl exec -n easyai "$primary" -c postgres -- \
+    psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At "$@"
+}
+attempted_by=$(query -c \
+  "SELECT attempted_by::text FROM river_job WHERE id=(SELECT river_job_id FROM gateway_tasks WHERE id='$task_id'::uuid);")
+[[ $attempted_by == *easyai-api-hongkong* ]] || {
+  echo "River attempted_by does not identify Hong Kong: $attempted_by" >&2
+  exit 1
+}
+[[ $(query -c "SELECT count(*) FROM gateway_task_attempts WHERE task_id='$task_id'::uuid AND status='succeeded';") -eq 1 ]]
+[[ $(query -c "SELECT count(*) FROM gateway_task_attempts WHERE task_id='$task_id'::uuid;") -eq 1 ]]
+[[ $(query -c "SELECT count(*) FROM settlement_outbox WHERE task_id='$task_id'::uuid AND action='settle';") -eq 1 ]]
+[[ $(query -c "SELECT count(*) FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id='$task_id' AND transaction_type='task_billing';") -eq 1 ]]
+callback_counts=$(query -F ' ' -c \
+  "SELECT count(*), count(DISTINCT (seq, callback_url)) FROM gateway_task_callback_outbox WHERE task_id='$task_id'::uuid;")
+read -r callbacks unique_callbacks <<<"$callback_counts"
+[[ $callbacks -eq $unique_callbacks ]]
+source_url=$(query -c \
+  "SELECT url FROM gateway_request_assets WHERE sha256='$source_sha' ORDER BY created_at DESC LIMIT 1;")
+[[ $source_url == https://* ]]
+printf '%s\n' "$source_url"
+REMOTE
+)
+source_url=$(tail -1 <<<"$database_evidence")
+
+compare_url_from_nodes() {
+  local label=$1
+  local url=$2
+  local expected_hash=
+  local expected_size=
+  for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"; do
+    evidence=$(cluster_ssh "$host" bash -s -- "$url" <<'REMOTE'
+set -euo pipefail
+url=$1
+temporary=$(mktemp)
+trap 'rm -f -- "$temporary"' EXIT
+curl -fsSL --max-time 300 "$url" -o "$temporary"
+printf '%s %s\n' "$(sha256sum "$temporary" | awk '{print $1}')" "$(stat -c '%s' "$temporary")"
+REMOTE
+)
+    read -r hash size <<<"$evidence"
+    [[ $hash =~ ^[0-9a-f]{64}$ && $size -gt 0 ]]
+    if [[ -z $expected_hash ]]; then
+      expected_hash=$hash
+      expected_size=$size
+    else
+      [[ $hash == "$expected_hash" && $size == "$expected_size" ]]
+    fi
+  done
+  printf '%s_url=PASS sha256=%s bytes=%s\n' "$label" "$expected_hash" "$expected_size"
+}
+
+compare_url_from_nodes request_asset "$source_url"
+compare_url_from_nodes generated_result "$result_url"
+restore_workers
+trap - EXIT HUP INT TERM
+rm -f -- "$credentials_file"
+echo "cross_node_file_e2e=PASS task=$task_id worker=hongkong"
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+domain=ai.51easyai.com
+hongkong_host=${AI_GATEWAY_CERT_SYNC_HOST:-root@10.77.0.2}
+identity_file=${AI_GATEWAY_CERT_SYNC_KEY:-/root/.ssh/id_ed25519_easyai_cert_sync}
+
+[[ $(hostname -I) == *10.77.0.1* ]] || {
+  echo 'certificate sync must run on the Ningbo node' >&2
+  exit 1
+}
+[[ -f $identity_file && ! -L $identity_file ]] || {
+  echo 'certificate sync SSH identity is unavailable' >&2
+  exit 1
+}
+
+install -d -m 0700 "/etc/nginx/tls/$domain"
+install -m 0644 "/etc/letsencrypt/live/$domain/fullchain.pem" \
+  "/etc/nginx/tls/$domain/fullchain.pem"
+install -m 0600 "/etc/letsencrypt/live/$domain/privkey.pem" \
+  "/etc/nginx/tls/$domain/privkey.pem"
+ssh -i "$identity_file" -o BatchMode=yes "$hongkong_host" \
+  "install -d -m 0700 /etc/nginx/tls/$domain"
+scp -q -i "$identity_file" -o BatchMode=yes \
+  "/etc/nginx/tls/$domain/fullchain.pem" \
+  "$hongkong_host:/etc/nginx/tls/$domain/fullchain.pem"
+scp -q -i "$identity_file" -o BatchMode=yes \
+  "/etc/nginx/tls/$domain/privkey.pem" \
+  "$hongkong_host:/etc/nginx/tls/$domain/privkey.pem"
+ssh -i "$identity_file" -o BatchMode=yes "$hongkong_host" \
+  "chmod 0600 /etc/nginx/tls/$domain/privkey.pem && nginx -t && systemctl reload nginx"
+nginx -t
+systemctl reload nginx
+echo 'certificate_sync=PASS'
@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+
+mode=${1:-precutover}
+[[ $mode == precutover || $mode == postcutover ]] || {
+  echo 'usage: verify-cluster.sh [precutover|postcutover]' >&2
+  exit 64
+}
+load_cluster_env
+
+hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST")
+wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3)
+for source_index in 0 1 2; do
+  handshakes=$(cluster_ssh "${hosts[$source_index]}" \
+    "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime() - \$2 < 180 { count++ } END { print count + 0 }'")
+  [[ $handshakes -eq 2 ]] || {
+    echo "stale WireGuard handshake on ${hosts[$source_index]}" >&2
+    exit 1
+  }
+  for target_index in 0 1 2; do
+    [[ $source_index -eq $target_index ]] && continue
+    ping_output=$(cluster_ssh "${hosts[$source_index]}" \
+      "ping -c 10 -W 2 ${wireguard_ips[$target_index]}")
+    packet_loss=$(sed -nE 's/.* ([0-9.]+)% packet loss.*/\1/p' <<<"$ping_output")
+    average_rtt=$(sed -nE 's#.* = [0-9.]+/([0-9.]+)/.*#\1#p' <<<"$ping_output")
+    awk -v loss="$packet_loss" 'BEGIN { exit !(loss < 1) }'
+    if [[ $source_index -eq 2 || $target_index -eq 2 ]]; then
+      awk -v rtt="$average_rtt" 'BEGIN { exit !(rtt < 300) }'
+    else
+      awk -v rtt="$average_rtt" 'BEGIN { exit !(rtt < 80) }'
+    fi
+  done
+done
+echo 'wireguard_acceptance=PASS handshakes=6 packet_loss_lt_1=true'
+
+cluster_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+kubectl='k3s kubectl'
+[[ $($kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq 3 ]]
+$kubectl get --raw='/readyz?verbose' | grep -q '\[+\]etcd ok'
+[[ $($kubectl get node easyai-los-angeles -o json |
+  jq '[.spec.taints[]? | select(.key=="easyai.io/witness" and .effect=="NoSchedule")] | length') -eq 1 ]]
+[[ $($kubectl get pods -A -o json |
+  jq '[.items[] | select(.spec.nodeName=="easyai-los-angeles") |
+    select(.metadata.namespace != "kube-system" and .metadata.namespace != "cnpg-system" and .metadata.namespace != "cert-manager")] | length') -eq 0 ]]
+printf 'nodes=%s\n' "$($kubectl get nodes --no-headers | wc -l)"
+REMOTE
+)
+[[ $cluster_state == nodes=3 ]]
+echo 'k3s_control_plane=PASS servers=3 witness_business_pods=0'
+
+if [[ $mode == postcutover ]]; then
+  database_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+kubectl='k3s kubectl'
+[[ $($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') -eq 2 ]]
+primary=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}')
+[[ $($kubectl get pod "$primary" -n easyai -o jsonpath='{.spec.nodeName}') == easyai-ningbo ||
+   $($kubectl get pod "$primary" -n easyai -o jsonpath='{.spec.nodeName}') == easyai-hongkong ]]
+replication=$($kubectl exec -n easyai "$primary" -c postgres -- \
+  psql -X -U easyai -d easyai_ai_gateway -At -c \
+  "SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');")
+[[ $replication -eq 1 ]]
+[[ $($kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-api \
+  --field-selector=status.phase=Running --no-headers | wc -l) -eq 2 ]]
+[[ $($kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-web \
+  --field-selector=status.phase=Running --no-headers | wc -l) -eq 2 ]]
+images=$($kubectl get deployments -n easyai -o json |
+  jq -r '.items[] | select(.metadata.name | startswith("easyai-")) |
+    .spec.template.spec.containers[].image')
+if grep -Ev '@sha256:[0-9a-f]{64}$' <<<"$images" | grep -q .; then
+  echo 'one or more application images are not digest-pinned' >&2
+  exit 1
+fi
+[[ $($kubectl get secret easyai-gateway-security-events -n easyai \
+  -o json | jq '.data | length') -eq 4 ]]
+[[ $($kubectl auth can-i update secret/easyai-gateway-security-events -n easyai \
+  --as=system:serviceaccount:easyai:easyai-ai-gateway) == yes ]]
+[[ $($kubectl auth can-i update secret/easyai-ai-gateway-runtime -n easyai \
+  --as=system:serviceaccount:easyai:easyai-ai-gateway) == no ]]
+printf 'primary=%s\n' "$primary"
+REMOTE
+)
+  [[ $database_state == primary=easyai-postgres-* ]]
+  echo "postgres_acceptance=PASS $database_state synchronous_replicas=1"
+
+  for edge_ip in "${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}"; do
+    curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
+      https://ai.51easyai.com/api/v1/healthz | grep -q easyai-ai-gateway
+    curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
+      https://ai.51easyai.com/api/v1/readyz | grep -q '"ok":true'
+    curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
+      https://ai.51easyai.com/api/v1/openapi.json | grep -q '"/api/v1/chat/completions"'
+    curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
+      https://ai.51easyai.com/ | grep -q 'EasyAI AI Gateway'
+  done
+  echo 'nginx_edges=PASS ningbo=true hongkong=true dns_changed=false'
+fi
+
+echo "cluster_verification=PASS mode=$mode"
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=scripts/cluster/common.sh
+source "$script_dir/common.sh"
+load_cluster_env
+
+cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE'
+set -euo pipefail
+snapshot_name=acceptance-$(date -u '+%Y%m%dT%H%M%SZ')
+k3s etcd-snapshot save --name "$snapshot_name" >/dev/null
+snapshot=$(find /var/lib/rancher/k3s/server/db/snapshots -maxdepth 1 -type f \
+  -name "$snapshot_name-*" -printf '%T@ %p\n' | sort -rn | awk 'NR == 1 { sub(/^[^ ]+ /, ""); print }')
+[[ -n $snapshot && -s $snapshot ]]
+etcdutl=$(find /var/lib/rancher/k3s/data -type f -path '*/bin/etcdutl' -print -quit)
+[[ -x $etcdutl ]]
+working_directory=$(mktemp -d)
+trap 'rm -rf -- "$working_directory"' EXIT
+case $snapshot in
+  *.zip)
+    if command -v unzip >/dev/null 2>&1 && unzip -t "$snapshot" >/dev/null 2>&1; then
+      unzip -p "$snapshot" >"$working_directory/snapshot.db"
+    else
+      gzip -dc "$snapshot" >"$working_directory/snapshot.db"
+    fi
+    ;;
+  *) cp "$snapshot" "$working_directory/snapshot.db" ;;
+esac
+"$etcdutl" snapshot status "$working_directory/snapshot.db" -w json |
+  jq -e '.hash != null and .totalKey > 0 and .totalSize > 0' >/dev/null
+k3s etcd-snapshot ls --s3 | grep -F "$snapshot_name" >/dev/null
+echo "etcd_backup_verification=PASS snapshot=$snapshot_name local=true oss=true"
+REMOTE