From 971540a2a455e69a7bbfc4423ef4b644bb8c9ceb Mon Sep 17 00:00:00 2001 From: wangbo Date: Tue, 28 Jul 2026 04:37:16 +0800 Subject: [PATCH] =?UTF-8?q?feat(deploy):=20=E5=A2=9E=E5=8A=A0=E4=B8=89?= =?UTF-8?q?=E8=8A=82=E7=82=B9=20K3s=20=E9=AB=98=E5=8F=AF=E7=94=A8=E8=BF=81?= =?UTF-8?q?=E7=A7=BB=E8=83=BD=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 WireGuard 全互联、三 server embedded-etcd K3s、CloudNativePG 双实例、Barman OSS 备份、双 NGINX、Kubernetes Secret/RBAC 与本地旧文件按严格 24 小时清理。 新增维护窗口数据迁移、digest 固定滚动发布、应用回滚、跨节点文件 E2E、节点和数据库故障演练、CNPG 恢复与 etcd 快照验收脚本；洛杉矶仅作为带 NoSchedule 污点的仲裁节点。 所有生产 Secret 只在执行时从 0600 本地环境和旧生产容器导入，仓库不保存凭据；公网入口保持人工 DNS 故障切换边界。 验证：bash -n、ShellCheck、kubectl kustomize、Node 语法检查、Secret 扫描、OSS put/head/delete 实测。 --- README.md | 9 +- .../easyai-ai-gateway-cluster-release | 279 +++++++++++ ...ai-ai-gateway-cluster-release.conf.example | 4 + .../production/application-config.yaml | 39 ++ deploy/kubernetes/production/application.yaml | 439 ++++++++++++++++++ deploy/kubernetes/production/database.yaml | 113 +++++ .../kubernetes/production/kustomization.yaml | 19 + deploy/kubernetes/production/namespace.yaml | 9 + .../kubernetes/production/network-policy.yaml | 23 + .../production/service-account-rbac.yaml | 31 ++ deploy/nginx/ai.51easyai.com-cluster.conf | 106 +++++ deploy/nginx/ai.51easyai.com-maintenance.conf | 28 ++ .../cluster-acceptance-origin-ningbo.conf | 24 + deploy/nginx/legacy-static-hongkong.inc | 20 + deploy/nginx/legacy-static-ningbo.inc | 13 + deploy/nginx/legacy-static-origin-ningbo.conf | 19 + .../easyai-legacy-generated-cleanup.service | 16 + .../easyai-legacy-generated-cleanup.timer | 11 + docs/operations/k3s-ha-runbook.md | 129 +++++ scripts/cluster/bootstrap-k3s.sh | 249 ++++++++++ scripts/cluster/bootstrap-platform.sh | 225 +++++++++ scripts/cluster/bootstrap-wireguard.sh | 107 +++++ scripts/cluster/common.sh | 95 ++++ scripts/cluster/cross-node-file-e2e.mjs | 147 ++++++ scripts/cluster/failure-drill.sh | 89 ++++ scripts/cluster/harden-node-network.sh | 49 ++ scripts/cluster/install-certificate-sync.sh | 41 ++ scripts/cluster/install-legacy-cleaner.sh | 23 + scripts/cluster/install-nginx-edges.sh | 95 ++++ scripts/cluster/install-release-helper.sh | 23 + scripts/cluster/migrate-production.sh | 362 +++++++++++++++ scripts/cluster/oss-object.mjs | 103 ++++ scripts/cluster/prune-legacy-generated.sh | 45 ++ scripts/cluster/restore-backup-drill.sh | 103 ++++ scripts/cluster/run-cross-node-file-e2e.sh | 130 ++++++ scripts/cluster/sync-nginx-certificate.sh | 34 ++ scripts/cluster/verify-cluster.sh | 104 +++++ scripts/cluster/verify-etcd-backup.sh | 34 ++ 38 files changed, 3386 insertions(+), 3 deletions(-) create mode 100755 deploy/kubernetes/easyai-ai-gateway-cluster-release create mode 100644 deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example create mode 100644 deploy/kubernetes/production/application-config.yaml create mode 100644 deploy/kubernetes/production/application.yaml create mode 100644 deploy/kubernetes/production/database.yaml create mode 100644 deploy/kubernetes/production/kustomization.yaml create mode 100644 deploy/kubernetes/production/namespace.yaml create mode 100644 deploy/kubernetes/production/network-policy.yaml create mode 100644 deploy/kubernetes/production/service-account-rbac.yaml create mode 100644 deploy/nginx/ai.51easyai.com-cluster.conf create mode 100644 deploy/nginx/ai.51easyai.com-maintenance.conf create mode 100644 deploy/nginx/cluster-acceptance-origin-ningbo.conf create mode 100644 deploy/nginx/legacy-static-hongkong.inc create mode 100644 deploy/nginx/legacy-static-ningbo.inc create mode 100644 deploy/nginx/legacy-static-origin-ningbo.conf create mode 100644 deploy/systemd/easyai-legacy-generated-cleanup.service create mode 100644 deploy/systemd/easyai-legacy-generated-cleanup.timer create mode 100644 docs/operations/k3s-ha-runbook.md create mode 100755 scripts/cluster/bootstrap-k3s.sh create mode 100755 scripts/cluster/bootstrap-platform.sh create mode 100755 scripts/cluster/bootstrap-wireguard.sh create mode 100755 scripts/cluster/common.sh create mode 100755 scripts/cluster/cross-node-file-e2e.mjs create mode 100755 scripts/cluster/failure-drill.sh create mode 100755 scripts/cluster/harden-node-network.sh create mode 100755 scripts/cluster/install-certificate-sync.sh create mode 100755 scripts/cluster/install-legacy-cleaner.sh create mode 100755 scripts/cluster/install-nginx-edges.sh create mode 100755 scripts/cluster/install-release-helper.sh create mode 100755 scripts/cluster/migrate-production.sh create mode 100755 scripts/cluster/oss-object.mjs create mode 100755 scripts/cluster/prune-legacy-generated.sh create mode 100755 scripts/cluster/restore-backup-drill.sh create mode 100755 scripts/cluster/run-cross-node-file-e2e.sh create mode 100755 scripts/cluster/sync-nginx-certificate.sh create mode 100755 scripts/cluster/verify-cluster.sh create mode 100755 scripts/cluster/verify-etcd-backup.sh diff --git a/README.md b/README.md index 7b9b0c9..2ba3421 100644 --- a/README.md +++ b/README.md @@ -104,14 +104,14 @@ docker compose -f docker-compose.yml restart web 本仓库没有 Gitea Actions、Webhook、Tag、`main` Push、轮询或定时发布。`main` 不使用受保护分支限制；commit、push 和 Tag 都不会构建镜像或更新生产。 -生产发布由 Agent 在用户明确指令下分两次执行。第一步在本机构建 `linux/amd64` 镜像、运行临时 PostgreSQL + simulation API 冒烟、推送完整 SHA Tag，并生成带内容完整性校验的 digest-pinned manifest；该命令不会修改生产： +生产发布由 Agent 在用户明确指令下按两个阶段连续执行。第一步在本机构建 `linux/amd64` 镜像、运行临时 PostgreSQL + simulation API 冒烟、推送完整 SHA Tag，并生成带内容完整性校验的 digest-pinned manifest；该命令不会修改生产： ```bash docker login --username= registry.cn-shanghai.aliyuncs.com ./scripts/publish-release-images.sh --components auto ``` -Agent 必须报告 `dist/releases/.json` 后停止。用户再次明确确认上线后，才执行第二步： +用户要求“发布”“上线”或“部署到线上”时，Agent 核验并报告 `dist/releases/.json`、组件和 digest 后直接执行第二步，无需再次确认；用户仅要求构建或推送镜像时才在第一步后停止： ```bash ./scripts/deploy-production-release.sh dist/releases/.json @@ -124,7 +124,10 @@ Agent 必须报告 `dist/releases/.json` 后停止。用户再次明确确 ./scripts/deploy-production-release.sh --rollback <历史完整 Git SHA> ``` -完整安装、验证、失败处理和停用旧自动化步骤见[人工生产发布运行手册](docs/runbooks/production-ci-cd.md)，决策背景见 [ADR-003](docs/decisions/003-manual-agent-release.md)。 +完整安装、验证、失败处理和停用旧自动化步骤见[人工生产发布运行手册](docs/runbooks/production-ci-cd.md)； +三节点 K3s、CloudNativePG、双 NGINX、备份恢复和跨节点文件验收见 +[K3s 高可用运行手册](docs/operations/k3s-ha-runbook.md)。决策背景见 +[ADR-003](docs/decisions/003-manual-agent-release.md)。 Compose 默认使用独立容器数据库 `postgres:18-alpine`，数据卷会保留在 `postgres_data` 和 `api_data`。为避免本地开发 `.env` 中的 `localhost` 数据库地址污染容器部署，compose 使用 `AI_GATEWAY_COMPOSE_*` 变量作为容器部署专用覆盖，例如： diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release b/deploy/kubernetes/easyai-ai-gateway-cluster-release new file mode 100755 index 0000000..40029e9 --- /dev/null +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release @@ -0,0 +1,279 @@ +#!/usr/bin/env bash +set -euo pipefail + +[[ $(id -u) -eq 0 ]] || { echo 'run as root' >&2; exit 1; } + +config_file=${AI_GATEWAY_CLUSTER_RELEASE_CONFIG:-/etc/easyai-ai-gateway-cluster-release.conf} +manifest_tool=${AI_GATEWAY_RELEASE_MANIFEST_TOOL:-/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs} +[[ -f $config_file && ! -L $config_file ]] || { echo 'missing cluster release config' >&2; exit 1; } +[[ -f $manifest_tool && ! -L $manifest_tool ]] || { echo 'missing release manifest validator' >&2; exit 1; } +# shellcheck source=/dev/null +source "$config_file" + +: "${RELEASES_DIR:?}" +: "${NAMESPACE:=easyai}" +: "${PUBLIC_BASE_URL:=https://ai.51easyai.com}" +: "${K3S_BIN:=/usr/local/bin/k3s}" +[[ $RELEASES_DIR == /* && $NAMESPACE =~ ^[a-z0-9-]+$ ]] || { + echo 'invalid cluster release configuration' >&2 + exit 1 +} +[[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || { + echo 'invalid public base URL' >&2 + exit 1 +} +for command in "$K3S_BIN" curl flock install jq node; do + command -v "$command" >/dev/null 2>&1 || { echo "$command is required" >&2; exit 1; } +done + +kubectl=("$K3S_BIN" kubectl) +install -d -m 0700 "$RELEASES_DIR" +exec 9>"$RELEASES_DIR/.release.lock" +flock -n 9 || { echo 'another cluster release operation is running' >&2; exit 1; } + +manifest_get() { + node "$manifest_tool" get "$1" "$2" +} + +current_to_file() { + local output=$1 + "${kubectl[@]}" get configmap easyai-gateway-release -n "$NAMESPACE" \ + -o jsonpath='{.data.manifest\.json}' >"$output" + [[ -s $output ]] + node "$manifest_tool" validate "$output" >/dev/null +} + +wait_for_url() { + local label=$1 + local url=$2 + local expected=${3:-} + local body + for _ in $(seq 1 60); do + if body=$(curl -fsS --max-time 5 "$url" 2>/dev/null); then + if [[ -z $expected || $body == *"$expected"* ]]; then + echo "[cluster-release] verified $label" + return 0 + fi + fi + sleep 2 + done + echo "release probe failed: $label ($url)" >&2 + return 1 +} + +verify_site() { + local site=$1 + local address + case $site in + hongkong) address=10.77.0.2 ;; + ningbo) address=10.77.0.1 ;; + *) return 1 ;; + esac + wait_for_url "$site API health" "http://$address:30088/api/v1/healthz" easyai-ai-gateway + wait_for_url "$site API readiness" "http://$address:30088/api/v1/readyz" '"ok":true' + wait_for_url "$site Web" "http://$address:30178/" 'EasyAI AI Gateway' +} + +rollout_site() { + local site=$1 + local api_image=$2 + local web_image=$3 + local api_changed=$4 + local web_changed=$5 + if [[ $api_changed == true ]]; then + "${kubectl[@]}" set image "deployment/easyai-api-$site" -n "$NAMESPACE" \ + "api=$api_image" + "${kubectl[@]}" rollout status "deployment/easyai-api-$site" -n "$NAMESPACE" --timeout=300s + fi + if [[ $web_changed == true ]]; then + "${kubectl[@]}" set image "deployment/easyai-web-$site" -n "$NAMESPACE" \ + "web=$web_image" + "${kubectl[@]}" rollout status "deployment/easyai-web-$site" -n "$NAMESPACE" --timeout=300s + fi + verify_site "$site" +} + +run_backup() { + local source_sha=$1 + local backup=easyai-pre-release-${source_sha:0:12} + cat </dev/null +apiVersion: postgresql.cnpg.io/v1 +kind: Backup +metadata: + name: $backup + namespace: $NAMESPACE +spec: + cluster: + name: easyai-postgres + method: plugin + pluginConfiguration: + name: barman-cloud.cloudnative-pg.io +EOF + for _ in $(seq 1 120); do + phase=$("${kubectl[@]}" get backup "$backup" -n "$NAMESPACE" \ + -o jsonpath='{.status.phase}' 2>/dev/null || true) + [[ ${phase,,} == completed ]] && return 0 + [[ ${phase,,} == failed ]] && break + sleep 5 + done + echo "pre-migration backup failed: $backup phase=${phase:-unknown}" >&2 + return 1 +} + +run_migrator() { + local source_sha=$1 + local api_image=$2 + local job=easyai-migrator-${source_sha:0:12} + "${kubectl[@]}" delete job "$job" -n "$NAMESPACE" \ + --ignore-not-found=true --wait=true >/dev/null + cat </dev/null +apiVersion: batch/v1 +kind: Job +metadata: + name: $job + namespace: $NAMESPACE + labels: + easyai.io/release: $source_sha +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: Never + imagePullSecrets: + - name: easyai-registry + nodeSelector: + easyai.io/site: ningbo + containers: + - name: migrator + image: $api_image + command: ["/app/easyai-ai-gateway-migrate"] + envFrom: + - secretRef: + name: easyai-ai-gateway-runtime +EOF + "${kubectl[@]}" wait --for=condition=complete "job/$job" \ + -n "$NAMESPACE" --timeout=300s +} + +record_current() { + local manifest=$1 + local action=$2 + local started_at=$3 + local source_sha recorded temporary + source_sha=$(manifest_get "$manifest" sourceSha) + recorded=$RELEASES_DIR/$source_sha.json + temporary=$(mktemp "$RELEASES_DIR/.recorded.XXXXXX") + rm -f -- "$temporary" + RELEASE_DEPLOYED_AT=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \ + RELEASE_DEPLOY_DURATION_SECONDS=$(( $(date +%s) - started_at )) \ + RELEASE_DEPLOY_ACTION=$action \ + node "$manifest_tool" record-deployment "$manifest" "$temporary" >/dev/null + install -m 0600 "$temporary" "$recorded" + "${kubectl[@]}" create configmap easyai-gateway-release -n "$NAMESPACE" \ + --from-file=manifest.json="$recorded" --dry-run=client -o yaml | + "${kubectl[@]}" apply -f - >/dev/null + rm -f -- "$temporary" +} + +activate_manifest() { + local manifest=$1 + local action=${2:-deploy} + local source_sha base_sha api_image web_image migrations_changed components + local api_changed=false web_changed=false current_file previous_api previous_web started_at + [[ -f $manifest && ! -L $manifest ]] || { echo 'manifest must be a regular file' >&2; return 1; } + node "$manifest_tool" validate "$manifest" >/dev/null + source_sha=$(manifest_get "$manifest" sourceSha) + base_sha=$(manifest_get "$manifest" baseReleaseSha) + api_image=$(manifest_get "$manifest" images.api) + web_image=$(manifest_get "$manifest" images.web) + migrations_changed=$(manifest_get "$manifest" migrationsChanged) + components=$(manifest_get "$manifest" components) + [[ $api_image =~ @sha256:[0-9a-f]{64}$ && $web_image =~ @sha256:[0-9a-f]{64}$ ]] || { + echo 'release images must be digest-pinned' >&2 + return 1 + } + [[ $components == *'"api"'* ]] && api_changed=true + [[ $components == *'"web"'* ]] && web_changed=true + if [[ $action == rollback ]]; then + api_changed=true + web_changed=true + fi + + current_file=$(mktemp "$RELEASES_DIR/.current.XXXXXX") + if current_to_file "$current_file"; then + current_sha=$(manifest_get "$current_file" sourceSha) + previous_api=$(manifest_get "$current_file" images.api) + previous_web=$(manifest_get "$current_file" images.web) + if [[ $action == deploy && $current_sha != "$base_sha" ]]; then + echo "stale release manifest: production=$current_sha manifest_base=$base_sha" >&2 + rm -f -- "$current_file" + return 1 + fi + else + if [[ $action == deploy && -n $base_sha ]]; then + echo 'cluster release status is unavailable for a non-bootstrap manifest' >&2 + rm -f -- "$current_file" + return 1 + fi + previous_api= + previous_web= + fi + started_at=$(date +%s) + + if [[ $action == deploy && $migrations_changed == true ]]; then + run_backup "$source_sha" + run_migrator "$source_sha" "$api_image" + fi + + if ! rollout_site hongkong "$api_image" "$web_image" "$api_changed" "$web_changed" || + ! rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" || + ! wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then + if [[ -n $previous_api && -n $previous_web ]]; then + echo '[cluster-release] restoring previous application images' >&2 + rollout_site hongkong "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true + rollout_site ningbo "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true + fi + rm -f -- "$current_file" + return 1 + fi + + record_current "$manifest" "$action" "$started_at" + rm -f -- "$current_file" + echo "production_release=PASS action=$action release=$source_sha" +} + +case ${1:-} in + status) + [[ $# -eq 1 ]] || exit 64 + status_file=$(mktemp "$RELEASES_DIR/.status.XXXXXX") + current_to_file "$status_file" + cat "$status_file" + rm -f -- "$status_file" + ;; + adopt) + [[ $# -eq 2 ]] || exit 64 + node "$manifest_tool" validate "$2" >/dev/null + api_image=$(manifest_get "$2" images.api) + web_image=$(manifest_get "$2" images.web) + [[ $("${kubectl[@]}" get deployment easyai-api-ningbo -n "$NAMESPACE" \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="api")].image}') == "$api_image" ]] + [[ $("${kubectl[@]}" get deployment easyai-web-ningbo -n "$NAMESPACE" \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="web")].image}') == "$web_image" ]] + record_current "$2" deploy "$(date +%s)" + ;; + deploy) + [[ $# -eq 2 ]] || exit 64 + activate_manifest "$2" deploy + ;; + rollback) + [[ $# -eq 2 && $2 =~ ^[0-9a-f]{40}$ ]] || exit 64 + target=$RELEASES_DIR/$2.json + [[ -f $target && ! -L $target ]] || { echo 'unknown historical release' >&2; exit 1; } + activate_manifest "$target" rollback + ;; + *) + echo 'usage: easyai-ai-gateway-cluster-release {status|adopt |deploy |rollback }' >&2 + exit 64 + ;; +esac diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example b/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example new file mode 100644 index 0000000..d25fc35 --- /dev/null +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example @@ -0,0 +1,4 @@ +RELEASES_DIR=/var/lib/easyai-ai-gateway-cluster-release/releases +NAMESPACE=easyai +PUBLIC_BASE_URL=https://ai.51easyai.com +K3S_BIN=/usr/local/bin/k3s diff --git a/deploy/kubernetes/production/application-config.yaml b/deploy/kubernetes/production/application-config.yaml new file mode 100644 index 0000000..7c3aaa5 --- /dev/null +++ b/deploy/kubernetes/production/application-config.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: easyai-ai-gateway-config + labels: + app.kubernetes.io/part-of: easyai-ai-gateway +data: + APP_ENV: production + HTTP_ADDR: ":8088" + IDENTITY_MODE: hybrid + IDENTITY_SECRET_STORE: kubernetes + IDENTITY_KUBERNETES_NAMESPACE: easyai + IDENTITY_KUBERNETES_SECRET_NAME: easyai-gateway-security-events + IDENTITY_SECURITY_EVENTS_HEARTBEAT_INTERVAL_SECONDS: "60" + IDENTITY_SECURITY_EVENTS_STALE_AFTER_SECONDS: "180" + IDENTITY_SECURITY_EVENTS_CLOCK_SKEW_SECONDS: "60" + SERVER_MAIN_BASE_URL: http://10.77.0.1:3000 + TASK_PROGRESS_CALLBACK_ENABLED: "true" + TASK_PROGRESS_CALLBACK_URL: http://10.77.0.1:3000/internal/platform/task-progress-callbacks + TASK_PROGRESS_CALLBACK_TIMEOUT_MS: "5000" + TASK_PROGRESS_CALLBACK_MAX_ATTEMPTS: "10" + AI_GATEWAY_TASK_CLEANUP_ENABLED: "false" + AI_GATEWAY_TASK_RETENTION_DAYS: "30" + AI_GATEWAY_TASK_ANALYSIS_RETENTION_DAYS: "7" + AI_GATEWAY_TASK_CLEANUP_INTERVAL_SECONDS: "300" + AI_GATEWAY_TASK_CLEANUP_BATCH_SIZE: "1000" + CORS_ALLOWED_ORIGIN: https://ai.51easyai.com + AI_GATEWAY_PUBLIC_BASE_URL: https://ai.51easyai.com + AI_GATEWAY_WEB_BASE_URL: https://ai.51easyai.com + AI_GATEWAY_GENERATED_STORAGE_DIR: /app/data/static/generated + AI_GATEWAY_UPLOADED_STORAGE_DIR: /app/data/static/uploaded + AI_GATEWAY_LOCAL_TEMP_ASSET_TTL_HOURS: "24" + AI_GATEWAY_LOCAL_RESULT_TTL_HOURS: "24" + AI_GATEWAY_LOCAL_RESULT_MIN_FREE_BYTES: "10737418240" + AI_GATEWAY_LOCAL_RESULT_MAX_BYTES: "268435456" + AI_GATEWAY_LOCAL_RESULT_MAX_TASK_BYTES: "536870912" + AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED: "true" + AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT: "2048" + AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS: "5" diff --git a/deploy/kubernetes/production/application.yaml b/deploy/kubernetes/production/application.yaml new file mode 100644 index 0000000..f7481f7 --- /dev/null +++ b/deploy/kubernetes/production/application.yaml @@ -0,0 +1,439 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: easyai-api-ningbo + labels: + app.kubernetes.io/name: easyai-api + app.kubernetes.io/component: api + app.kubernetes.io/part-of: easyai-ai-gateway +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app.kubernetes.io/name: easyai-api + easyai.io/site: ningbo + template: + metadata: + labels: + app.kubernetes.io/name: easyai-api + app.kubernetes.io/component: api + app.kubernetes.io/part-of: easyai-ai-gateway + easyai.io/site: ningbo + spec: + serviceAccountName: easyai-ai-gateway + terminationGracePeriodSeconds: 90 + nodeSelector: + easyai.io/site: ningbo + easyai.io/workload: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: api + image: easyai-api + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: easyai-ai-gateway-config + - secretRef: + name: easyai-ai-gateway-runtime + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + ports: + - name: http + containerPort: 8088 + readinessProbe: + httpGet: + path: /api/v1/readyz + port: http + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /api/v1/healthz + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 6 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "sleep 10"] + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + volumeMounts: + - name: application-data + mountPath: /app/data + - name: temporary-files + mountPath: /tmp + volumes: + - name: application-data + emptyDir: + sizeLimit: 5Gi + - name: temporary-files + emptyDir: + sizeLimit: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: easyai-api-hongkong + labels: + app.kubernetes.io/name: easyai-api + app.kubernetes.io/component: api + app.kubernetes.io/part-of: easyai-ai-gateway +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app.kubernetes.io/name: easyai-api + easyai.io/site: hongkong + template: + metadata: + labels: + app.kubernetes.io/name: easyai-api + app.kubernetes.io/component: api + app.kubernetes.io/part-of: easyai-ai-gateway + easyai.io/site: hongkong + spec: + serviceAccountName: easyai-ai-gateway + terminationGracePeriodSeconds: 90 + nodeSelector: + easyai.io/site: hongkong + easyai.io/workload: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: api + image: easyai-api + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: easyai-ai-gateway-config + - secretRef: + name: easyai-ai-gateway-runtime + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + ports: + - name: http + containerPort: 8088 + readinessProbe: + httpGet: + path: /api/v1/readyz + port: http + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /api/v1/healthz + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 6 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "sleep 10"] + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + volumeMounts: + - name: application-data + mountPath: /app/data + - name: temporary-files + mountPath: /tmp + volumes: + - name: application-data + emptyDir: + sizeLimit: 5Gi + - name: temporary-files + emptyDir: + sizeLimit: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: easyai-web-ningbo + labels: + app.kubernetes.io/name: easyai-web + app.kubernetes.io/component: web + app.kubernetes.io/part-of: easyai-ai-gateway +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app.kubernetes.io/name: easyai-web + easyai.io/site: ningbo + template: + metadata: + labels: + app.kubernetes.io/name: easyai-web + app.kubernetes.io/component: web + app.kubernetes.io/part-of: easyai-ai-gateway + easyai.io/site: ningbo + spec: + automountServiceAccountToken: false + terminationGracePeriodSeconds: 30 + nodeSelector: + easyai.io/site: ningbo + easyai.io/workload: "true" + securityContext: + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + fsGroup: 101 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: web + image: easyai-web + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 80 + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 6 + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 6 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + add: ["NET_BIND_SERVICE"] + readOnlyRootFilesystem: true + volumeMounts: + - name: nginx-cache + mountPath: /var/cache/nginx + - name: nginx-run + mountPath: /var/run + volumes: + - name: nginx-cache + emptyDir: + sizeLimit: 128Mi + - name: nginx-run + emptyDir: + sizeLimit: 16Mi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: easyai-web-hongkong + labels: + app.kubernetes.io/name: easyai-web + app.kubernetes.io/component: web + app.kubernetes.io/part-of: easyai-ai-gateway +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app.kubernetes.io/name: easyai-web + easyai.io/site: hongkong + template: + metadata: + labels: + app.kubernetes.io/name: easyai-web + app.kubernetes.io/component: web + app.kubernetes.io/part-of: easyai-ai-gateway + easyai.io/site: hongkong + spec: + automountServiceAccountToken: false + terminationGracePeriodSeconds: 30 + nodeSelector: + easyai.io/site: hongkong + easyai.io/workload: "true" + securityContext: + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + fsGroup: 101 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: web + image: easyai-web + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 80 + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 6 + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 6 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + add: ["NET_BIND_SERVICE"] + readOnlyRootFilesystem: true + volumeMounts: + - name: nginx-cache + mountPath: /var/cache/nginx + - name: nginx-run + mountPath: /var/run + volumes: + - name: nginx-cache + emptyDir: + sizeLimit: 128Mi + - name: nginx-run + emptyDir: + sizeLimit: 16Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: api + labels: + app.kubernetes.io/name: easyai-api + app.kubernetes.io/component: api + app.kubernetes.io/part-of: easyai-ai-gateway +spec: + type: NodePort + externalTrafficPolicy: Local + selector: + app.kubernetes.io/name: easyai-api + ports: + - name: http + port: 8088 + targetPort: http + nodePort: 30088 +--- +apiVersion: v1 +kind: Service +metadata: + name: web + labels: + app.kubernetes.io/name: easyai-web + app.kubernetes.io/component: web + app.kubernetes.io/part-of: easyai-ai-gateway +spec: + type: NodePort + externalTrafficPolicy: Local + selector: + app.kubernetes.io/name: easyai-web + ports: + - name: http + port: 80 + targetPort: http + nodePort: 30178 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: easyai-api +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: easyai-api +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: easyai-web +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: easyai-web diff --git a/deploy/kubernetes/production/database.yaml b/deploy/kubernetes/production/database.yaml new file mode 100644 index 0000000..16db501 --- /dev/null +++ b/deploy/kubernetes/production/database.yaml @@ -0,0 +1,113 @@ +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: easyai-postgres-oss + labels: + app.kubernetes.io/part-of: easyai-ai-gateway +spec: + retentionPolicy: 14d + configuration: + destinationPath: s3://wangbo0808/easyai-ai-gateway/production/postgresql + endpointURL: https://oss-cn-shanghai.aliyuncs.com + s3Credentials: + accessKeyId: + name: easyai-oss-backup + key: ACCESS_KEY_ID + secretAccessKey: + name: easyai-oss-backup + key: ACCESS_SECRET_KEY + wal: + compression: gzip + maxParallel: 2 + data: + compression: gzip + jobs: 2 + instanceSidecarConfiguration: + env: + - name: AWS_DEFAULT_REGION + value: oss-cn-shanghai + - name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required + - name: AWS_RESPONSE_CHECKSUM_VALIDATION + value: when_required +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: easyai-postgres + labels: + app.kubernetes.io/name: easyai-postgres + app.kubernetes.io/part-of: easyai-ai-gateway +spec: + instances: 2 + imageName: ghcr.io/cloudnative-pg/postgresql:18.4-standard-trixie@sha256:4587df73024408f5b2be9b4dd6ba2ccee8c9e5dc0c9a87c274c292291cc8a68c + imagePullPolicy: IfNotPresent + primaryUpdateStrategy: unsupervised + primaryUpdateMethod: switchover + failoverDelay: 0 + switchoverDelay: 60 + stopDelay: 90 + smartShutdownTimeout: 30 + bootstrap: + initdb: + database: easyai_ai_gateway + owner: easyai + secret: + name: easyai-postgres-app + postInitSQL: + - CREATE EXTENSION IF NOT EXISTS pgcrypto; + storage: + storageClass: local-path + size: 20Gi + walStorage: + storageClass: local-path + size: 5Gi + affinity: + enablePodAntiAffinity: true + podAntiAffinityType: required + topologyKey: easyai.io/site + nodeSelector: + easyai.io/database: "true" + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "2" + memory: 3Gi + postgresql: + synchronous: + method: any + number: 1 + dataDurability: preferred + parameters: + archive_timeout: 60s + max_connections: "200" + shared_buffers: 512MB + wal_compression: "on" + wal_keep_size: 1GB + monitoring: + enablePodMonitor: false + plugins: + - name: barman-cloud.cloudnative-pg.io + isWALArchiver: true + parameters: + barmanObjectName: easyai-postgres-oss +--- +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: easyai-postgres-daily + labels: + app.kubernetes.io/part-of: easyai-ai-gateway +spec: + # Six-field cron, UTC: 18:30 UTC is 02:30 Asia/Shanghai. + schedule: "0 30 18 * * *" + backupOwnerReference: self + cluster: + name: easyai-postgres + method: plugin + pluginConfiguration: + name: barman-cloud.cloudnative-pg.io + immediate: false + suspend: false diff --git a/deploy/kubernetes/production/kustomization.yaml b/deploy/kubernetes/production/kustomization.yaml new file mode 100644 index 0000000..294c559 --- /dev/null +++ b/deploy/kubernetes/production/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: easyai +resources: + - namespace.yaml + - service-account-rbac.yaml + - application-config.yaml + - application.yaml + - database.yaml + - network-policy.yaml +images: + # Deployment tooling must replace both sentinel digests with release-manifest + # digests before server-side apply. + - name: easyai-api + newName: registry.cn-shanghai.aliyuncs.com/easyaigc/ai-gateway + digest: sha256:0000000000000000000000000000000000000000000000000000000000000000 + - name: easyai-web + newName: registry.cn-shanghai.aliyuncs.com/easyaigc/ai-gateway-web + digest: sha256:0000000000000000000000000000000000000000000000000000000000000000 diff --git a/deploy/kubernetes/production/namespace.yaml b/deploy/kubernetes/production/namespace.yaml new file mode 100644 index 0000000..91dc880 --- /dev/null +++ b/deploy/kubernetes/production/namespace.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: easyai + labels: + app.kubernetes.io/part-of: easyai-ai-gateway + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted diff --git a/deploy/kubernetes/production/network-policy.yaml b/deploy/kubernetes/production/network-policy.yaml new file mode 100644 index 0000000..d25e034 --- /dev/null +++ b/deploy/kubernetes/production/network-policy.yaml @@ -0,0 +1,23 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: easyai-application-ingress +spec: + podSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: [easyai-api, easyai-web] + policyTypes: [Ingress] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: easyai + - ipBlock: + cidr: 10.77.0.0/24 + ports: + - protocol: TCP + port: 8088 + - protocol: TCP + port: 80 diff --git a/deploy/kubernetes/production/service-account-rbac.yaml b/deploy/kubernetes/production/service-account-rbac.yaml new file mode 100644 index 0000000..4759ab3 --- /dev/null +++ b/deploy/kubernetes/production/service-account-rbac.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: easyai-ai-gateway + labels: + app.kubernetes.io/part-of: easyai-ai-gateway +automountServiceAccountToken: true +imagePullSecrets: + - name: easyai-registry +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: easyai-gateway-security-events +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["easyai-gateway-security-events"] + verbs: ["get", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: easyai-gateway-security-events +subjects: + - kind: ServiceAccount + name: easyai-ai-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: easyai-gateway-security-events diff --git a/deploy/nginx/ai.51easyai.com-cluster.conf b/deploy/nginx/ai.51easyai.com-cluster.conf new file mode 100644 index 0000000..468a200 --- /dev/null +++ b/deploy/nginx/ai.51easyai.com-cluster.conf @@ -0,0 +1,106 @@ +upstream easyai_gateway_api { + least_conn; + keepalive 64; + server 10.77.0.1:30088 max_fails=2 fail_timeout=5s; + server 10.77.0.2:30088 max_fails=2 fail_timeout=5s; +} + +upstream easyai_gateway_web { + least_conn; + keepalive 32; + server 10.77.0.1:30178 max_fails=2 fail_timeout=5s; + server 10.77.0.2:30178 max_fails=2 fail_timeout=5s; +} + +server { + listen 80; + listen [::]:80; + server_name ai.51easyai.com; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name ai.51easyai.com; + + ssl_certificate /etc/nginx/tls/ai.51easyai.com/fullchain.pem; + ssl_certificate_key /etc/nginx/tls/ai.51easyai.com/privkey.pem; + ssl_session_cache shared:easyai_gateway_tls:10m; + ssl_session_timeout 1d; + ssl_protocols TLSv1.2 TLSv1.3; + + client_max_body_size 200m; + client_body_timeout 300s; + + include /etc/nginx/easyai-ai-gateway/legacy-static.inc; + + location = /api/v1/metrics { + return 404; + } + + location ^~ /api/v1/ { + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Connection ""; + proxy_connect_timeout 2s; + proxy_send_timeout 3600s; + proxy_read_timeout 3600s; + proxy_request_buffering off; + proxy_buffering off; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_pass http://easyai_gateway_api; + } + + location /gateway-api/ { + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Connection ""; + proxy_connect_timeout 2s; + proxy_send_timeout 3600s; + proxy_read_timeout 3600s; + proxy_request_buffering off; + proxy_buffering off; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_pass http://easyai_gateway_api/; + } + + location ^~ /static/ { + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Connection ""; + proxy_connect_timeout 2s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + proxy_buffering off; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_pass http://easyai_gateway_api; + } + + location / { + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Connection ""; + proxy_connect_timeout 2s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_pass http://easyai_gateway_web; + } +} diff --git a/deploy/nginx/ai.51easyai.com-maintenance.conf b/deploy/nginx/ai.51easyai.com-maintenance.conf new file mode 100644 index 0000000..79682d3 --- /dev/null +++ b/deploy/nginx/ai.51easyai.com-maintenance.conf @@ -0,0 +1,28 @@ +server { + listen 80; + listen [::]:80; + server_name ai.51easyai.com; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name ai.51easyai.com; + + ssl_certificate /etc/nginx/tls/ai.51easyai.com/fullchain.pem; + ssl_certificate_key /etc/nginx/tls/ai.51easyai.com/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + location ^~ /api/ { + default_type application/json; + add_header Retry-After 600 always; + return 503 '{"error":{"message":"Gateway database maintenance is in progress","type":"service_unavailable"}}'; + } + + location / { + default_type text/html; + add_header Retry-After 600 always; + return 503 '系统维护

系统正在进行高可用升级

预计 5–10 分钟恢复，请稍后重试。

'; + } +} diff --git a/deploy/nginx/cluster-acceptance-origin-ningbo.conf b/deploy/nginx/cluster-acceptance-origin-ningbo.conf new file mode 100644 index 0000000..e14bf28 --- /dev/null +++ b/deploy/nginx/cluster-acceptance-origin-ningbo.conf @@ -0,0 +1,24 @@ +# Private-only deterministic path used by the cross-node acceptance test. +server { + listen 10.77.0.1:18088; + server_name _; + + allow 10.77.0.0/24; + deny all; + client_max_body_size 200m; + + location / { + proxy_http_version 1.1; + proxy_set_header Host ai.51easyai.com; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Connection ""; + proxy_connect_timeout 2s; + proxy_send_timeout 3600s; + proxy_read_timeout 3600s; + proxy_request_buffering off; + proxy_buffering off; + proxy_pass http://10.77.0.1:30088; + } +} diff --git a/deploy/nginx/legacy-static-hongkong.inc b/deploy/nginx/legacy-static-hongkong.inc new file mode 100644 index 0000000..77ad349 --- /dev/null +++ b/deploy/nginx/legacy-static-hongkong.inc @@ -0,0 +1,20 @@ +# Hong Kong reads unexpired legacy files from the private Ningbo origin. +location ^~ /static/generated/ { + proxy_http_version 1.1; + proxy_set_header Host ai.51easyai.com; + proxy_set_header Connection ""; + proxy_connect_timeout 2s; + proxy_read_timeout 300s; + proxy_buffering off; + proxy_pass http://10.77.0.1:18080; +} + +location ^~ /static/uploaded/ { + proxy_http_version 1.1; + proxy_set_header Host ai.51easyai.com; + proxy_set_header Connection ""; + proxy_connect_timeout 2s; + proxy_read_timeout 300s; + proxy_buffering off; + proxy_pass http://10.77.0.1:18080; +} diff --git a/deploy/nginx/legacy-static-ningbo.inc b/deploy/nginx/legacy-static-ningbo.inc new file mode 100644 index 0000000..2d99275 --- /dev/null +++ b/deploy/nginx/legacy-static-ningbo.inc @@ -0,0 +1,13 @@ +# Temporary compatibility route. Remove after all files are strictly older +# than 86,400 seconds and the cleanup acceptance check reports zero survivors. +location ^~ /static/generated/ { + alias /var/lib/docker/volumes/easyai-ai-gateway_api_data/_data/static/generated/; + add_header Cache-Control "private, max-age=300"; + try_files $request_filename =410; +} + +location ^~ /static/uploaded/ { + alias /var/lib/docker/volumes/easyai-ai-gateway_api_data/_data/static/uploaded/; + add_header Cache-Control "private, max-age=300"; + try_files $request_filename =410; +} diff --git a/deploy/nginx/legacy-static-origin-ningbo.conf b/deploy/nginx/legacy-static-origin-ningbo.conf new file mode 100644 index 0000000..b48a692 --- /dev/null +++ b/deploy/nginx/legacy-static-origin-ningbo.conf @@ -0,0 +1,19 @@ +server { + listen 10.77.0.1:18080; + server_name _; + + allow 10.77.0.0/24; + deny all; + + location ^~ /static/generated/ { + alias /var/lib/docker/volumes/easyai-ai-gateway_api_data/_data/static/generated/; + add_header Cache-Control "private, max-age=300"; + try_files $request_filename =410; + } + + location ^~ /static/uploaded/ { + alias /var/lib/docker/volumes/easyai-ai-gateway_api_data/_data/static/uploaded/; + add_header Cache-Control "private, max-age=300"; + try_files $request_filename =410; + } +} diff --git a/deploy/systemd/easyai-legacy-generated-cleanup.service b/deploy/systemd/easyai-legacy-generated-cleanup.service new file mode 100644 index 0000000..f35436e --- /dev/null +++ b/deploy/systemd/easyai-legacy-generated-cleanup.service @@ -0,0 +1,16 @@ +[Unit] +Description=Prune EasyAI Gateway legacy local files older than 24 hours +After=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/easyai-prune-legacy-generated --apply +User=root +Group=root +Nice=10 +IOSchedulingClass=idle +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +ReadWritePaths=/var/lib/docker/volumes/easyai-ai-gateway_api_data/_data/static diff --git a/deploy/systemd/easyai-legacy-generated-cleanup.timer b/deploy/systemd/easyai-legacy-generated-cleanup.timer new file mode 100644 index 0000000..ae32213 --- /dev/null +++ b/deploy/systemd/easyai-legacy-generated-cleanup.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run EasyAI Gateway legacy local file cleanup hourly + +[Timer] +OnCalendar=hourly +Persistent=true +RandomizedDelaySec=120 +Unit=easyai-legacy-generated-cleanup.service + +[Install] +WantedBy=timers.target diff --git a/docs/operations/k3s-ha-runbook.md b/docs/operations/k3s-ha-runbook.md new file mode 100644 index 0000000..b9257af --- /dev/null +++ b/docs/operations/k3s-ha-runbook.md @@ -0,0 +1,129 @@ +# EasyAI Gateway 三节点 K3s 高可用运行手册 + +## 范围与目标 + +本手册覆盖宁波、香港、洛杉矶三节点迁移。宁波和香港承载 API、Web、River Worker 与 +CloudNativePG；洛杉矶只作为 K3s server/etcd 仲裁节点，并带 +`easyai.io/witness=true:NoSchedule` 污点。 + +- 健康双库：同步复制，RPO=0。 +- 单库降级：`dataDurability: preferred` 保持写入，`archive_timeout=60s`，灾难 RPO 目标不超过 5 分钟。 +- 数据库故障 RTO：不超过 5 分钟。 +- 应用发布：香港验收后再滚动宁波，计划内零停机。 +- 公网入口：双 NGINX 加人工 AliDNS 切换。接入带健康检查的 GTM 前，不宣称公网入口自动高可用。 + +所有命令从仓库根目录执行。生产凭据仅从 `0600` 的 `.env.local` 和 +`.local-secrets/cluster/` 读取，不进入 Git、发布 manifest 或验收输出。 + +## 实施顺序 + +### 1. 源码与镜像门禁 + +```bash +cd apps/api && env -u AI_GATEWAY_TEST_DATABASE_URL go test ./... -count=1 +cd apps/api && go vet ./... +pnpm install --frozen-lockfile +pnpm lint +pnpm test +pnpm build +shellcheck scripts/cluster/*.sh deploy/kubernetes/easyai-ai-gateway-cluster-release +kubectl kustomize deploy/kubernetes/production >/dev/null +./tests/ci/migrations-test.sh +node scripts/ci-validate-migrations.mjs <当前线上完整GitSHA> +``` + +代码提交并推送到 `origin/main` 后，发布 digest 固定镜像： + +```bash +./scripts/publish-release-images.sh --components auto +``` + +### 2. WireGuard 与 K3s + +```bash +./scripts/cluster/bootstrap-wireguard.sh +./scripts/cluster/bootstrap-k3s.sh backup +./scripts/cluster/bootstrap-k3s.sh install +./scripts/cluster/harden-node-network.sh +./scripts/cluster/verify-cluster.sh precutover +``` + +`backup` 会在卸载旧 K3s 前，将 SQLite、manifests 和 staging 资源归档并校验后上传 +OSS `easyai-ai-gateway/production/pre-ha/`。`install` 固定 +`v1.36.2+k3s1`，使用 embedded etcd、静态 Secret 加密、500ms heartbeat 和 5s election timeout。 + +### 3. 平台和入口预置 + +```bash +./scripts/cluster/bootstrap-platform.sh prepare dist/releases/.json +./scripts/cluster/install-nginx-edges.sh prepare +./scripts/cluster/install-certificate-sync.sh +./scripts/cluster/install-legacy-cleaner.sh +``` + +`prepare` 将应用副本保持在 0，但启动 CNPG 双实例、WAL 归档和每日备份。SecretStore +的 4 个值从旧 volume 直接导入 Kubernetes Secret，脚本只校验数量与总字节数，不打印值。 + +### 4. 02:00–04:00 切换 + +先执行只读预检： + +```bash +./scripts/cluster/migrate-production.sh preflight dist/releases/.json +``` + +进入北京时间窗口后执行： + +```bash +./scripts/cluster/migrate-production.sh cutover dist/releases/.json +``` + +脚本依次启用维护页、停止旧 API、完整备份旧 PostgreSQL、上传 OSS、恢复 CNPG、比对数据库 +版本/扩展/catalog/表数/序列/关键行数/抽样 ID、运行 digest 固定 migrator、冻结 Worker 启动 +两地应用、做 NodePort 验收，再切换双 NGINX。公开流量提交后，先启用香港 Worker，再启用宁波 +Worker。旧 Docker PostgreSQL 和数据卷停止但保留，不自动删除。 + +在公共流量提交前，任一步失败或耗时超过 10 分钟都会恢复旧 Docker 与旧 NGINX。公共流量 +提交后禁止自动切回旧数据库，避免双写分叉。 + +## 上线验收 + +```bash +./scripts/cluster/verify-cluster.sh postcutover +./scripts/cluster/run-cross-node-file-e2e.sh +./scripts/cluster/failure-drill.sh api --execute +./scripts/cluster/failure-drill.sh database --execute +./scripts/cluster/failure-drill.sh witness --execute +./scripts/cluster/restore-backup-drill.sh --execute +./scripts/cluster/verify-etcd-backup.sh +``` + +跨节点 E2E 通过宁波私有 NGINX 固定进入宁波 API，关闭宁波 River Worker，由香港 Worker +执行真实 multipart 图片编辑任务；校验输入和输出 URL 在两节点下载的长度与 SHA-256 +一致，并检查 `river_job.attempted_by`、唯一成功 attempt、唯一结算/扣费及无重复回调。 + +数据库故障演练会强制删除当前主 Pod，确认 5 分钟内晋升；在副本恢复前写入一条专用 +`system_settings` 探针，恢复同步后删除。备份恢复演练会从 OSS 创建一个独立 CNPG Cluster， +验证数据库登录、schema、行数和抽样值后删除临时 Cluster 与 PVC。 + +## 发布与回滚 + +切换成功后，本地环境自动设置 `AI_GATEWAY_DEPLOY_MODE=kubernetes`。日常发布仍使用： + +```bash +./scripts/deploy-production-release.sh dist/releases/.json +./scripts/deploy-production-release.sh --status +./scripts/deploy-production-release.sh --rollback <历史完整SHA> +``` + +集群 helper 从 `easyai-gateway-release` ConfigMap 读取当前 release，数据库迁移前先等待一次 +CNPG 备份，然后香港滚动、验收，再滚动宁波。`--rollback` 只恢复应用 digest，绝不逆向执行迁移。 + +## 旧文件与公网故障 + +宁波每小时执行一次清理，以当前时间减文件 `mtime` 严格大于 86,400 秒为删除条件。未过期 +文件由宁波只读 NGINX 提供，香港通过 WireGuard 回源。清理结果持续为 0 个存量文件后，删除 +两个 legacy static include 和宁波 18080 origin；历史过期 URL 返回 410。 + +公网入口故障时，先使用 `curl --resolve` 确认香港入口健康，再人工将 `ai.51easyai.com` +AliDNS A 记录切到香港 IP。恢复后不自动回切；确认连接和任务稳定后再择时切回。 diff --git a/scripts/cluster/bootstrap-k3s.sh b/scripts/cluster/bootstrap-k3s.sh new file mode 100755 index 0000000..22ef470 --- /dev/null +++ b/scripts/cluster/bootstrap-k3s.sh @@ -0,0 +1,249 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" + +action=${1:-all} +[[ $action == backup || $action == install || $action == all ]] || { + echo 'usage: bootstrap-k3s.sh [backup|install|all]' >&2 + exit 64 +} + +load_cluster_env +require_commands node openssl ssh scp shasum + +secret_root="$cluster_root/.local-secrets/cluster" +cluster_environment="$secret_root/cluster.env" +kubeconfig="$secret_root/kubeconfig" +k3s_version=v1.36.2+k3s1 + +install -d -m 0700 "$secret_root" + +backup_legacy_k3s() { + local timestamp remote_archive local_directory local_archive checksum object_prefix + timestamp=$(date -u '+%Y%m%dT%H%M%SZ') + remote_archive=/root/easyai-k3s-sqlite-pre-ha-"$timestamp".tar.gz + local_directory=$(mktemp -d "$secret_root/backup.XXXXXX") + local_archive=$local_directory/$(basename "$remote_archive") + trap '[[ ${local_directory:-} == "$secret_root"/backup.* ]] && rm -rf -- "$local_directory"' RETURN + + echo '[k3s] archiving legacy SQLite state and staging manifests' + cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$remote_archive" <<'REMOTE' +set -euo pipefail +archive=$1 +work_directory=$(mktemp -d /root/easyai-k3s-backup.XXXXXX) +trap '[[ $work_directory == /root/easyai-k3s-backup.* ]] && rm -rf -- "$work_directory"' EXIT +install -d -m 0700 "$work_directory/state" +if [[ -d /var/lib/rancher/k3s/server/db ]]; then + cp -a /var/lib/rancher/k3s/server/db "$work_directory/state/" +fi +if [[ -d /var/lib/rancher/k3s/server/manifests ]]; then + cp -a /var/lib/rancher/k3s/server/manifests "$work_directory/state/" +fi +if command -v k3s >/dev/null 2>&1 && systemctl is-active --quiet k3s; then + k3s kubectl get namespace easyai-auth-center-staging -o yaml \ + >"$work_directory/easyai-auth-center-staging-namespace.yaml" 2>/dev/null || true + k3s kubectl get all,configmap,secret,pvc,ingress \ + -n easyai-auth-center-staging -o yaml \ + >"$work_directory/easyai-auth-center-staging-resources.yaml" 2>/dev/null || true + k3s kubectl get nodes -o wide >"$work_directory/nodes.txt" + k3s --version >"$work_directory/k3s-version.txt" +fi +tar -C "$work_directory" -czf "$archive" . +chmod 0600 "$archive" +sha256sum "$archive" >"$archive.sha256" +REMOTE + + cluster_scp "$CLUSTER_NINGBO_HOST:$remote_archive" "$local_archive" + cluster_scp "$CLUSTER_NINGBO_HOST:$remote_archive.sha256" "$local_archive.remote.sha256" + checksum=$(shasum -a 256 "$local_archive" | awk '{print $1}') + grep -q "^$checksum " "$local_archive.remote.sha256" || { + echo 'legacy K3s archive checksum mismatch' >&2 + return 1 + } + printf '%s %s\n' "$checksum" "$(basename "$local_archive")" >"$local_archive.sha256" + object_prefix="easyai-ai-gateway/production/pre-ha/${timestamp}" + node "$script_dir/oss-object.mjs" put "$object_prefix/$(basename "$local_archive")" "$local_archive" + node "$script_dir/oss-object.mjs" put "$object_prefix/$(basename "$local_archive").sha256" "$local_archive.sha256" + node "$script_dir/oss-object.mjs" head "$object_prefix/$(basename "$local_archive")" + cluster_ssh "$CLUSTER_NINGBO_HOST" \ + "rm -f -- '$remote_archive' '$remote_archive.sha256'" + echo "legacy_k3s_backup=PASS object_prefix=$object_prefix sha256=$checksum" +} + +write_node_config() { + local output=$1 + local node_name=$2 + local node_ip=$3 + local mode=$4 + local taint=${5:-} + umask 077 + { + printf 'token-file: /etc/rancher/k3s/cluster-token\n' + printf 'node-name: %s\n' "$node_name" + printf 'node-ip: %s\n' "$node_ip" + printf 'advertise-address: %s\n' "$node_ip" + printf 'flannel-iface: wg0\n' + printf 'write-kubeconfig-mode: "0600"\n' + printf 'secrets-encryption: true\n' + printf 'disable:\n - traefik\n - servicelb\n' + printf 'tls-san:\n - 10.77.0.1\n - 10.77.0.2\n - 10.77.0.3\n' + printf 'etcd-arg:\n - heartbeat-interval=500\n - election-timeout=5000\n' + printf 'etcd-snapshot-schedule-cron: "0 */6 * * *"\n' + printf 'etcd-snapshot-retention: 28\n' + printf 'etcd-snapshot-compress: true\n' + printf 'etcd-s3: true\n' + printf 'etcd-s3-config-secret: easyai-etcd-s3\n' + if [[ $mode == init ]]; then + printf 'cluster-init: true\n' + else + printf 'server: https://10.77.0.1:6443\n' + fi + if [[ -n $taint ]]; then + printf 'node-taint:\n - %s\n' "$taint" + fi + } >"$output" + chmod 0600 "$output" +} + +install_cluster() { + local configuration_directory + for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do + cluster_ssh "$host" 'ping -c 2 -W 2 10.77.0.1 >/dev/null' + done + + if [[ ! -f $cluster_environment ]]; then + umask 077 + printf 'K3S_TOKEN=%s\n' "$(openssl rand -hex 32)" >"$cluster_environment" + chmod 0600 "$cluster_environment" + fi + assert_private_file "$cluster_environment" + # shellcheck source=/dev/null + source "$cluster_environment" + : "${K3S_TOKEN:?}" + [[ $K3S_TOKEN =~ ^[0-9a-f]{64}$ ]] || { + echo 'invalid local K3s token' >&2 + return 1 + } + + configuration_directory=$(mktemp -d "$secret_root/config.XXXXXX") + trap '[[ ${configuration_directory:-} == "$secret_root"/config.* ]] && rm -rf -- "$configuration_directory"' RETURN + write_node_config "$configuration_directory/ningbo.yaml" easyai-ningbo 10.77.0.1 init + write_node_config "$configuration_directory/hongkong.yaml" easyai-hongkong 10.77.0.2 join + write_node_config "$configuration_directory/los-angeles.yaml" easyai-los-angeles 10.77.0.3 join \ + 'easyai.io/witness=true:NoSchedule' + umask 077 + cat >"$configuration_directory/etcd-s3-secret.yaml" </dev/null 2>&1 && systemctl is-active --quiet k3s; then + k3s kubectl delete namespace easyai-auth-center-staging \ + --ignore-not-found=true --wait=true --timeout=120s +fi +if [[ -x /usr/local/bin/k3s-uninstall.sh ]]; then + /usr/local/bin/k3s-uninstall.sh +fi +REMOTE + for host in "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do + cluster_ssh "$host" \ + '[[ ! -x /usr/local/bin/k3s-uninstall.sh ]] || /usr/local/bin/k3s-uninstall.sh' + done + + node_names=(ningbo hongkong los-angeles) + node_hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST") + for index in 0 1 2; do + host=${node_hosts[$index]} + node_name=${node_names[$index]} + cluster_ssh "$host" 'install -d -m 0700 /etc/rancher/k3s' + printf '%s\n' "$K3S_TOKEN" | cluster_ssh "$host" \ + 'umask 077; cat >/etc/rancher/k3s/cluster-token; chmod 0600 /etc/rancher/k3s/cluster-token' + cluster_scp "$configuration_directory/$node_name.yaml" "$host:/etc/rancher/k3s/config.yaml" + cluster_ssh "$host" 'chmod 0600 /etc/rancher/k3s/config.yaml' + done + + for index in 0 1 2; do + host=${node_hosts[$index]} + echo "[k3s] installing $k3s_version on $host" + cluster_ssh "$host" bash -s -- "$k3s_version" <<'REMOTE' +set -euo pipefail +version=$1 +install_script=$(mktemp) +trap 'rm -f -- "$install_script"' EXIT +curl -fsSL https://get.k3s.io -o "$install_script" +chmod 0700 "$install_script" +INSTALL_K3S_VERSION="$version" INSTALL_K3S_EXEC=server "$install_script" +k3s --version | grep -F "$version" +REMOTE + if [[ $index -eq 0 ]]; then + for _ in $(seq 1 60); do + if cluster_ssh "$host" 'k3s kubectl get --raw=/readyz >/dev/null 2>&1'; then + break + fi + sleep 2 + done + cluster_scp "$configuration_directory/etcd-s3-secret.yaml" \ + "$CLUSTER_NINGBO_HOST:/root/easyai-etcd-s3-secret.yaml" + cluster_ssh "$CLUSTER_NINGBO_HOST" \ + 'k3s kubectl apply -f /root/easyai-etcd-s3-secret.yaml >/dev/null && rm -f /root/easyai-etcd-s3-secret.yaml' + fi + done + + for _ in $(seq 1 90); do + ready_count=$(cluster_ssh "$CLUSTER_NINGBO_HOST" \ + "k3s kubectl get nodes --no-headers 2>/dev/null | awk '\$2 == \"Ready\" { count++ } END { print count + 0 }'") + [[ $ready_count == 3 ]] && break + sleep 2 + done + [[ ${ready_count:-0} == 3 ]] || { + echo 'three K3s servers did not become Ready' >&2 + return 1 + } + + cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +kubectl='k3s kubectl' +$kubectl label node easyai-ningbo easyai.io/site=ningbo easyai.io/workload=true easyai.io/database=true --overwrite +$kubectl label node easyai-hongkong easyai.io/site=hongkong easyai.io/workload=true easyai.io/database=true --overwrite +$kubectl label node easyai-los-angeles easyai.io/site=los-angeles easyai.io/workload=false easyai.io/database=false --overwrite +$kubectl taint node easyai-los-angeles easyai.io/witness=true:NoSchedule --overwrite +k3s secrets-encrypt status +k3s etcd-snapshot save --name post-bootstrap +REMOTE + + cluster_scp "$CLUSTER_NINGBO_HOST:/etc/rancher/k3s/k3s.yaml" "$kubeconfig" + chmod 0600 "$kubeconfig" + sed -i.bak 's#https://127.0.0.1:6443#https://10.77.0.1:6443#' "$kubeconfig" + rm -f -- "$kubeconfig.bak" + assert_private_file "$kubeconfig" + echo "k3s_bootstrap=PASS version=$k3s_version servers=3 kubeconfig=$kubeconfig" +} + +if [[ $action == backup || $action == all ]]; then + backup_legacy_k3s +fi +if [[ $action == install || $action == all ]]; then + install_cluster +fi diff --git a/scripts/cluster/bootstrap-platform.sh b/scripts/cluster/bootstrap-platform.sh new file mode 100755 index 0000000..d226660 --- /dev/null +++ b/scripts/cluster/bootstrap-platform.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" + +mode=${1:-} +release_manifest=${2:-} +[[ $mode == prepare || $mode == activate ]] || { + echo 'usage: bootstrap-platform.sh {prepare|activate} dist/releases/.json' >&2 + exit 64 +} +[[ -f $release_manifest && ! -L $release_manifest ]] || { + echo 'release manifest must be a regular file' >&2 + exit 1 +} + +load_cluster_env +require_commands curl kubectl node openssl rg shasum + +node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null +api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.api) +web_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.web) +source_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha) +[[ $api_image =~ ^registry\.cn-shanghai\.aliyuncs\.com/easyaigc/ai-gateway@sha256:[0-9a-f]{64}$ ]] +[[ $web_image =~ ^registry\.cn-shanghai\.aliyuncs\.com/easyaigc/ai-gateway-web@sha256:[0-9a-f]{64}$ ]] + +secret_root="$cluster_root/.local-secrets/cluster" +postgres_environment="$secret_root/postgres.env" +working_directory=$(mktemp -d "$secret_root/platform.XXXXXX") +trap '[[ $working_directory == "$secret_root"/platform.* ]] && rm -rf -- "$working_directory"' EXIT HUP INT TERM +install -d -m 0700 "$secret_root" + +if [[ ! -f $postgres_environment ]]; then + umask 077 + printf 'POSTGRES_PASSWORD=%s\n' "$(openssl rand -hex 32)" >"$postgres_environment" + chmod 0600 "$postgres_environment" +fi +assert_private_file "$postgres_environment" +# shellcheck source=/dev/null +source "$postgres_environment" +: "${POSTGRES_PASSWORD:?}" +[[ $POSTGRES_PASSWORD =~ ^[0-9a-f]{64}$ ]] || { + echo 'invalid generated PostgreSQL password' >&2 + exit 1 +} + +declare -A operator_urls=( + [cert-manager.yaml]=https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml + [cnpg.yaml]=https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.29/releases/cnpg-1.29.1.yaml + [barman.yaml]=https://github.com/cloudnative-pg/plugin-barman-cloud/releases/download/v0.13.0/manifest.yaml +) +declare -A operator_checksums=( + [cert-manager.yaml]=6e499c3f1ab356abe79a7853911f80cb09c213885bfdf81092fdff142ba63c4a + [cnpg.yaml]=e0c5ff41bf5b01c0775bf225929b8423d543cd23e9742e4274fe7de8499cf93a + [barman.yaml]=d2e71e7b06822448f1a421f05781846cfdb9cc621e7ef32eef5e20c5133213b0 +) +for operator in cert-manager.yaml cnpg.yaml barman.yaml; do + curl -fsSL "${operator_urls[$operator]}" -o "$working_directory/$operator" + actual_checksum=$(shasum -a 256 "$working_directory/$operator" | awk '{print $1}') + [[ $actual_checksum == "${operator_checksums[$operator]}" ]] || { + echo "operator manifest checksum mismatch: $operator" >&2 + exit 1 + } +done + +cp -R "$cluster_root/deploy/kubernetes/production" "$working_directory/production" +api_digest=${api_image##*@} +web_digest=${web_image##*@} +awk -v api_digest="$api_digest" -v web_digest="$web_digest" ' + $3 == "easyai-api" { image = "api" } + $3 == "easyai-web" { image = "web" } + $1 == "digest:" && image == "api" { $2 = api_digest; image = ""; print; next } + $1 == "digest:" && image == "web" { $2 = web_digest; image = ""; print; next } + { print } +' "$working_directory/production/kustomization.yaml" \ + >"$working_directory/production/kustomization.yaml.next" +mv "$working_directory/production/kustomization.yaml.next" \ + "$working_directory/production/kustomization.yaml" + +if [[ $mode == prepare ]]; then + sed -i.bak 's/^ replicas: 1$/ replicas: 0/' "$working_directory/production/application.yaml" + rm -f -- "$working_directory/production/application.yaml.bak" +fi +if [[ ${AI_GATEWAY_ACTIVATE_WORKERS:-true} == false ]]; then + sed -i.bak \ + 's/^ AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED: "true"$/ AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED: "false"/' \ + "$working_directory/production/application-config.yaml" + rm -f -- "$working_directory/production/application-config.yaml.bak" +fi +kubectl kustomize "$working_directory/production" >"$working_directory/easyai-production.yaml" +if rg -q 'sha256:0{64}|image: (easyai-api|easyai-web)$' "$working_directory/easyai-production.yaml"; then + echo 'application images were not digest-pinned during rendering' >&2 + exit 1 +fi +if rg -i -q '^[[:space:]]*(password|secret|token|credential|private.?key):[[:space:]]+[^<{[:space:]]' \ + "$working_directory/easyai-production.yaml"; then + echo 'rendered production manifests appear to contain a literal secret' >&2 + exit 1 +fi + +umask 077 +cat >"$working_directory/bootstrap-secrets.yaml" </dev/null +$kubectl rollout status deployment/cert-manager -n cert-manager --timeout=180s +$kubectl rollout status deployment/cert-manager-webhook -n cert-manager --timeout=180s +$kubectl rollout status deployment/cert-manager-cainjector -n cert-manager --timeout=180s +$kubectl apply -f /root/cnpg.yaml >/dev/null +$kubectl rollout status deployment/cnpg-controller-manager -n cnpg-system --timeout=180s +$kubectl apply -f /root/barman.yaml >/dev/null +$kubectl rollout status deployment/barman-cloud -n cnpg-system --timeout=180s + +$kubectl create namespace easyai --dry-run=client -o yaml | $kubectl apply -f - >/dev/null +$kubectl apply -f /root/bootstrap-secrets.yaml >/dev/null +postgres_password=$($kubectl get secret easyai-postgres-app -n easyai \ + -o jsonpath='{.data.password}' | base64 -d) + +runtime_environment=$(mktemp) +trap 'rm -f -- "$runtime_environment"' EXIT +chmod 0600 "$runtime_environment" +docker inspect easyai-ai-gateway-api-1 \ + --format '{{range .Config.Env}}{{println .}}{{end}}' | + grep -E '^(CONFIG_JWT_SECRET|SERVER_MAIN_INTERNAL_TOKEN|SERVER_MAIN_INTERNAL_KEY|SERVER_MAIN_INTERNAL_SECRET)=' \ + >"$runtime_environment" +printf 'AI_GATEWAY_DATABASE_URL=postgresql://easyai:%s@easyai-postgres-rw:5432/easyai_ai_gateway?sslmode=require\n' \ + "$postgres_password" >>"$runtime_environment" +[[ $(grep -c '^[A-Z0-9_]*=' "$runtime_environment") -eq 5 ]] || { + echo 'runtime Secret allowlist is incomplete' >&2 + exit 1 +} +$kubectl create secret generic easyai-ai-gateway-runtime -n easyai \ + --from-env-file="$runtime_environment" --dry-run=client -o yaml | + $kubectl apply -f - >/dev/null + +identity_directory=/var/lib/docker/volumes/easyai-ai-gateway_api_data/_data/identity/secrets +[[ -d $identity_directory && ! -L $identity_directory ]] || { + echo 'legacy identity SecretStore directory is unavailable' >&2 + exit 1 +} +mapfile -d '' identity_files < <(find "$identity_directory" -maxdepth 1 -type f -print0) +[[ ${#identity_files[@]} -eq 4 ]] || { + echo 'identity SecretStore file count changed from the accepted baseline' >&2 + exit 1 +} +identity_bytes=0 +identity_arguments=() +for identity_file in "${identity_files[@]}"; do + identity_name=$(basename "$identity_file") + [[ $identity_name =~ ^[A-Za-z0-9._-]+$ ]] || { + echo 'identity SecretStore contains an invalid key name' >&2 + exit 1 + } + identity_bytes=$((identity_bytes + $(stat -c '%s' "$identity_file"))) + identity_arguments+=("--from-file=$identity_name=$identity_file") +done +[[ $identity_bytes -eq 247 ]] || { + echo 'identity SecretStore byte count changed from the accepted baseline' >&2 + exit 1 +} +$kubectl create secret generic easyai-gateway-security-events -n easyai \ + "${identity_arguments[@]}" --dry-run=client -o yaml | + $kubectl apply -f - >/dev/null + +registry_config=/root/.docker/config.json +[[ -f $registry_config && ! -L $registry_config ]] || { + echo 'Docker registry config is unavailable on Ningbo' >&2 + exit 1 +} +$kubectl create secret generic easyai-registry -n easyai \ + --type=kubernetes.io/dockerconfigjson \ + --from-file=.dockerconfigjson="$registry_config" --dry-run=client -o yaml | + $kubectl apply -f - >/dev/null + +$kubectl apply -f /root/easyai-production.yaml >/dev/null +$kubectl annotate namespace easyai easyai.io/release="$source_sha" --overwrite >/dev/null +if [[ $mode == activate ]]; then + $kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s + $kubectl rollout status deployment/easyai-web-hongkong -n easyai --timeout=300s + $kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s + $kubectl rollout status deployment/easyai-web-ningbo -n easyai --timeout=300s +fi + +rm -f -- /root/cert-manager.yaml /root/cnpg.yaml /root/barman.yaml \ + /root/easyai-production.yaml /root/bootstrap-secrets.yaml +REMOTE + +if [[ $mode == prepare ]]; then + cluster_ssh "$CLUSTER_NINGBO_HOST" \ + 'k3s kubectl wait --for=condition=Ready cluster/easyai-postgres -n easyai --timeout=600s' +fi + +echo "platform_bootstrap=PASS mode=$mode release=$source_sha" diff --git a/scripts/cluster/bootstrap-wireguard.sh b/scripts/cluster/bootstrap-wireguard.sh new file mode 100755 index 0000000..e03c7d2 --- /dev/null +++ b/scripts/cluster/bootstrap-wireguard.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" + +load_cluster_env +require_commands ssh scp + +hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST") +public_endpoints=("${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}" "${CLUSTER_LOS_ANGELES_HOST#*@}") +wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3) +public_keys=() + +for host in "${hosts[@]}"; do + echo "[wireguard] preparing $host" + public_key=$(cluster_ssh "$host" 'bash -s' <<'REMOTE' +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive +if ! command -v wg >/dev/null 2>&1; then + apt-get update -qq + apt-get install -y -qq wireguard-tools +fi +install -d -m 0700 /etc/wireguard +if [[ ! -s /etc/wireguard/privatekey ]]; then + umask 077 + wg genkey >/etc/wireguard/privatekey +fi +chmod 0600 /etc/wireguard/privatekey +wg pubkey /etc/wireguard/publickey +cat /etc/wireguard/publickey +REMOTE +) + [[ $public_key =~ ^[A-Za-z0-9+/]{43}=$ ]] || { + echo "invalid WireGuard public key returned by $host" >&2 + exit 1 + } + public_keys+=("$public_key") +done + +for local_index in 0 1 2; do + peer_indexes=() + for candidate in 0 1 2; do + [[ $candidate -eq $local_index ]] || peer_indexes+=("$candidate") + done + first=${peer_indexes[0]} + second=${peer_indexes[1]} + echo "[wireguard] configuring ${hosts[$local_index]} as ${wireguard_ips[$local_index]}" + cluster_ssh "${hosts[$local_index]}" bash -s -- \ + "${wireguard_ips[$local_index]}" \ + "${public_keys[$first]}" "${public_endpoints[$first]}" "${wireguard_ips[$first]}" \ + "${public_keys[$second]}" "${public_endpoints[$second]}" "${wireguard_ips[$second]}" <<'REMOTE' +set -euo pipefail +local_ip=$1 +peer_one_key=$2 +peer_one_endpoint=$3 +peer_one_ip=$4 +peer_two_key=$5 +peer_two_endpoint=$6 +peer_two_ip=$7 +private_key=$(cat /etc/wireguard/privatekey) +umask 077 +temporary_config=$(mktemp /etc/wireguard/wg0.conf.XXXXXX) +cat >"$temporary_config" </dev/null +systemctl restart wg-quick@wg0 +REMOTE +done + +for source_index in 0 1 2; do + for target_index in 0 1 2; do + [[ $source_index -eq $target_index ]] && continue + cluster_ssh "${hosts[$source_index]}" \ + "ping -c 3 -W 2 ${wireguard_ips[$target_index]} >/dev/null" + done +done + +for host in "${hosts[@]}"; do + latest_handshake=$(cluster_ssh "$host" \ + "wg show wg0 latest-handshakes | awk '\$2 > 0 { count++ } END { print count + 0 }'") + [[ $latest_handshake == 2 ]] || { + echo "WireGuard handshake verification failed on $host" >&2 + exit 1 + } +done + +echo 'wireguard_bootstrap=PASS peers=3' diff --git a/scripts/cluster/common.sh b/scripts/cluster/common.sh new file mode 100755 index 0000000..9098619 --- /dev/null +++ b/scripts/cluster/common.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cluster_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +cluster_env_file=${AI_GATEWAY_CLUSTER_ENV_FILE:-"$cluster_root/.env.local"} +cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-"$HOME/.ssh/id_ed25519_easyai_gateway_cluster"} + +load_cluster_env() { + [[ -f $cluster_env_file && ! -L $cluster_env_file ]] || { + echo "missing local cluster environment file: $cluster_env_file" >&2 + return 1 + } + set -a + # shellcheck source=/dev/null + source "$cluster_env_file" + set +a + cluster_ssh_key=${AI_GATEWAY_CLUSTER_SSH_KEY:-$cluster_ssh_key} + + : "${AI_GATEWAY_DEPLOY_HOST:?}" + : "${AI_GATEWAY_DEPLOY_HOST_2:?}" + : "${AI_GATEWAY_DEPLOY_HOST_3:?}" + : "${ALI_KEY:?}" + : "${ALI_SECRET:?}" + : "${ALI_REGION:?}" + : "${ALI_BUCKET:?}" + + CLUSTER_NINGBO_HOST=${AI_GATEWAY_NINGBO_HOST:-$AI_GATEWAY_DEPLOY_HOST} + CLUSTER_LOS_ANGELES_HOST=${AI_GATEWAY_LOS_ANGELES_HOST:-$AI_GATEWAY_DEPLOY_HOST_2} + CLUSTER_HONGKONG_HOST=${AI_GATEWAY_HONGKONG_HOST:-$AI_GATEWAY_DEPLOY_HOST_3} + export CLUSTER_NINGBO_HOST CLUSTER_LOS_ANGELES_HOST CLUSTER_HONGKONG_HOST + + for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST"; do + [[ $host =~ ^root@[A-Za-z0-9.-]+$ ]] || { + echo "cluster hosts must use root@host syntax" >&2 + return 1 + } + done + [[ $ALI_REGION =~ ^[a-z0-9-]+$ && $ALI_BUCKET =~ ^[A-Za-z0-9.-]+$ ]] || { + echo 'invalid OSS region or bucket' >&2 + return 1 + } + [[ -f $cluster_ssh_key && ! -L $cluster_ssh_key ]] || { + echo "missing cluster SSH key: $cluster_ssh_key" >&2 + return 1 + } +} + +cluster_ssh() { + local host=$1 + shift + ssh \ + -i "$cluster_ssh_key" \ + -o BatchMode=yes \ + -o ConnectTimeout=10 \ + -o ServerAliveInterval=15 \ + -o ServerAliveCountMax=3 \ + "$host" "$@" +} + +cluster_scp() { + scp \ + -i "$cluster_ssh_key" \ + -o BatchMode=yes \ + -o ConnectTimeout=10 \ + "$@" +} + +require_commands() { + local command + for command in "$@"; do + command -v "$command" >/dev/null 2>&1 || { + echo "$command is required" >&2 + return 1 + } + done +} + +assert_private_file() { + local path=$1 + local mode + [[ -f $path && ! -L $path ]] || { + echo "missing private file: $path" >&2 + return 1 + } + if [[ $(uname -s) == Darwin ]]; then + mode=$(stat -f '%Lp' "$path") + else + mode=$(stat -c '%a' "$path") + fi + [[ $mode == 600 ]] || { + echo "private file must have mode 0600: $path" >&2 + return 1 + } +} diff --git a/scripts/cluster/cross-node-file-e2e.mjs b/scripts/cluster/cross-node-file-e2e.mjs new file mode 100755 index 0000000..00f4fbc --- /dev/null +++ b/scripts/cluster/cross-node-file-e2e.mjs @@ -0,0 +1,147 @@ +#!/usr/bin/env node + +import { createHash, randomUUID } from 'node:crypto' +import { deflateSync } from 'node:zlib' + +function assert(condition, message) { + if (!condition) throw new Error(message) +} + +function crc32(buffer) { + let value = 0xffffffff + for (const byte of buffer) { + value ^= byte + for (let bit = 0; bit < 8; bit += 1) { + value = (value >>> 1) ^ (value & 1 ? 0xedb88320 : 0) + } + } + return (value ^ 0xffffffff) >>> 0 +} + +function pngChunk(type, data) { + const typeBuffer = Buffer.from(type) + const length = Buffer.alloc(4) + length.writeUInt32BE(data.length) + const checksum = Buffer.alloc(4) + checksum.writeUInt32BE(crc32(Buffer.concat([typeBuffer, data]))) + return Buffer.concat([length, typeBuffer, data, checksum]) +} + +function createFixturePNG(width = 256, height = 256) { + const header = Buffer.alloc(13) + header.writeUInt32BE(width, 0) + header.writeUInt32BE(height, 4) + header[8] = 8 + header[9] = 2 + const rows = [] + for (let y = 0; y < height; y += 1) { + const row = Buffer.alloc(1 + width * 3) + for (let x = 0; x < width; x += 1) { + row[1 + x * 3] = Math.round((x / width) * 255) + row[2 + x * 3] = Math.round((y / height) * 255) + row[3 + x * 3] = 180 + } + rows.push(row) + } + return Buffer.concat([ + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), + pngChunk('IHDR', header), + pngChunk('IDAT', deflateSync(Buffer.concat(rows))), + pngChunk('IEND', Buffer.alloc(0)), + ]) +} + +const baseURL = process.env.AI_GATEWAY_E2E_BASE_URL || 'http://10.77.0.1:18088' +const account = Buffer.from(process.env.E2E_ACCOUNT_B64 || '', 'base64').toString('utf8') +const password = Buffer.from(process.env.E2E_PASSWORD_B64 || '', 'base64').toString('utf8') +const model = process.env.AI_GATEWAY_E2E_IMAGE_MODEL || 'openai:gpt-image-1' +assert(account && password, 'E2E login credentials are unavailable') + +async function request(path, { method = 'GET', token, body, form } = {}) { + const headers = { Host: 'ai.51easyai.com' } + if (token) headers.Authorization = `Bearer ${token}` + if (body !== undefined) headers['Content-Type'] = 'application/json' + if (form) headers['X-Async'] = 'true' + const response = await fetch(`${baseURL}${path}`, { + method, + headers, + body: form || (body === undefined ? undefined : JSON.stringify(body)), + signal: AbortSignal.timeout(3_600_000), + }) + const text = await response.text() + let payload + try { + payload = text ? JSON.parse(text) : {} + } catch { + payload = { raw: text.slice(0, 500) } + } + if (!response.ok) { + throw new Error(`${method} ${path} returned HTTP ${response.status}: ${JSON.stringify(payload)}`) + } + return { response, payload } +} + +function collectHTTPSURLs(value, output = []) { + if (typeof value === 'string' && value.startsWith('https://')) output.push(value) + else if (Array.isArray(value)) value.forEach((item) => collectHTTPSURLs(item, output)) + else if (value && typeof value === 'object') Object.values(value).forEach((item) => collectHTTPSURLs(item, output)) + return output +} + +const login = await request('/api/v1/auth/login', { + method: 'POST', + body: { account, password }, +}) +const jwt = login.payload.accessToken +assert(typeof jwt === 'string' && jwt.length > 20, 'login did not return an access token') + +const keyName = `cluster-cross-node-${Date.now()}` +const keyResult = await request('/api/v1/api-keys', { + method: 'POST', + token: jwt, + body: { name: keyName }, +}) +const apiKey = keyResult.payload.secret +const apiKeyId = keyResult.payload.id || keyResult.payload.item?.id +assert(typeof apiKey === 'string' && apiKey.startsWith('sk-gw-'), 'API key creation failed') + +let result +try { + const fixture = createFixturePNG() + const sourceSHA256 = createHash('sha256').update(fixture).digest('hex') + const form = new FormData() + form.append('model', model) + form.append('prompt', `Cross-node HA acceptance ${randomUUID()}: add a thin white border`) + form.append('size', '1024x1024') + form.append('image', new Blob([fixture], { type: 'image/png' }), 'cluster-e2e.png') + const accepted = await request('/api/v1/images/edits', { + method: 'POST', + token: apiKey, + form, + }) + assert(accepted.response.status === 202, `image edit returned ${accepted.response.status}, expected 202`) + const taskId = accepted.payload.taskId || accepted.payload.task_id || accepted.payload.task?.id + assert(/^[0-9a-f-]{36}$/.test(taskId || ''), 'image edit response has no task ID') + + let task + const deadline = Date.now() + 20 * 60_000 + while (Date.now() < deadline) { + const taskResult = await request(`/api/v1/tasks/${taskId}`, { token: apiKey }) + task = taskResult.payload.task || taskResult.payload + if (task.status === 'succeeded') break + if (['failed', 'cancelled'].includes(task.status)) { + throw new Error(`task ${taskId} ended as ${task.status}: ${task.errorMessage || task.error || ''}`) + } + await new Promise((resolve) => setTimeout(resolve, 3000)) + } + assert(task?.status === 'succeeded', `task ${taskId} did not succeed before timeout`) + const resultURLs = [...new Set(collectHTTPSURLs(task.result))] + assert(resultURLs.length > 0, 'task result contains no shared HTTPS URL') + result = { taskId, sourceSHA256, resultURL: resultURLs[0], model } +} finally { + if (apiKeyId) { + await request(`/api/v1/api-keys/${apiKeyId}`, { method: 'DELETE', token: jwt }).catch(() => {}) + } +} + +console.log(JSON.stringify(result)) diff --git a/scripts/cluster/failure-drill.sh b/scripts/cluster/failure-drill.sh new file mode 100755 index 0000000..93bd87c --- /dev/null +++ b/scripts/cluster/failure-drill.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" + +drill=${1:-} +confirmation=${2:-} +[[ $confirmation == --execute ]] || { + echo 'usage: failure-drill.sh {api|database|witness} --execute' >&2 + exit 64 +} +load_cluster_env + +case $drill in + api) + started=$(date +%s) + pod=$(cluster_ssh "$CLUSTER_NINGBO_HOST" \ + "k3s kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo -o jsonpath='{.items[0].metadata.name}'") + cluster_ssh "$CLUSTER_NINGBO_HOST" \ + "k3s kubectl delete pod '$pod' -n easyai --wait=false >/dev/null" + for _ in $(seq 1 20); do + if curl -fsS --max-time 3 --resolve "ai.51easyai.com:443:${CLUSTER_NINGBO_HOST#*@}" \ + https://ai.51easyai.com/api/v1/readyz | grep -q '"ok":true'; then + elapsed=$(( $(date +%s) - started )) + break + fi + sleep 1 + done + [[ ${elapsed:-99} -le 10 ]] + echo "api_failure_drill=PASS failover_seconds=$elapsed" + ;; + database) + old_primary=$(cluster_ssh "$CLUSTER_NINGBO_HOST" \ + "k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}'") + started=$(date +%s) + cluster_ssh "$CLUSTER_NINGBO_HOST" \ + "k3s kubectl delete pod '$old_primary' -n easyai --grace-period=0 --force >/dev/null" + for _ in $(seq 1 150); do + new_primary=$(cluster_ssh "$CLUSTER_NINGBO_HOST" \ + "k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}'") + if [[ -n $new_primary && $new_primary != "$old_primary" ]]; then + elapsed=$(( $(date +%s) - started )) + break + fi + sleep 2 + done + [[ ${elapsed:-999} -le 300 ]] + curl -fsS --max-time 10 https://ai.51easyai.com/api/v1/readyz | grep -q '"ok":true' + cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$new_primary" <<'REMOTE' +set -euo pipefail +primary=$1 +k3s kubectl exec -n easyai "$primary" -c postgres -- \ + psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -c \ + "INSERT INTO system_settings(setting_key,value) VALUES('ha_acceptance_probe',jsonb_build_object('at',now())) ON CONFLICT(setting_key) DO UPDATE SET value=excluded.value,updated_at=now();" >/dev/null +for _ in $(seq 1 180); do + if [[ $(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') == 2 ]]; then + replication=$(k3s kubectl exec -n easyai "$primary" -c postgres -- \ + psql -X -U easyai -d easyai_ai_gateway -At -c \ + "SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');") + [[ $replication == 1 ]] && break + fi + sleep 2 +done +[[ ${replication:-0} == 1 ]] +k3s kubectl exec -n easyai "$primary" -c postgres -- \ + psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -c \ + "DELETE FROM system_settings WHERE setting_key='ha_acceptance_probe';" >/dev/null +REMOTE + echo "database_failure_drill=PASS promotion_seconds=$elapsed old_primary=$old_primary new_primary=$new_primary" + ;; + witness) + cluster_ssh "$CLUSTER_LOS_ANGELES_HOST" 'systemctl stop k3s' + trap 'cluster_ssh "$CLUSTER_LOS_ANGELES_HOST" "systemctl start k3s" || true' EXIT HUP INT TERM + cluster_ssh "$CLUSTER_NINGBO_HOST" \ + "k3s kubectl get --raw='/readyz?verbose' | grep -q '\\[+\\]etcd ok'" + curl -fsS --max-time 10 https://ai.51easyai.com/api/v1/readyz | grep -q '"ok":true' + cluster_ssh "$CLUSTER_LOS_ANGELES_HOST" 'systemctl start k3s' + cluster_ssh "$CLUSTER_NINGBO_HOST" \ + 'k3s kubectl wait --for=condition=Ready node/easyai-los-angeles --timeout=180s' + trap - EXIT HUP INT TERM + echo 'witness_failure_drill=PASS quorum_retained=true' + ;; + *) + echo 'drill must be api, database, or witness' >&2 + exit 64 + ;; +esac diff --git a/scripts/cluster/harden-node-network.sh b/scripts/cluster/harden-node-network.sh new file mode 100755 index 0000000..bb7e34f --- /dev/null +++ b/scripts/cluster/harden-node-network.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" + +load_cluster_env +hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST") + +for host in "${hosts[@]}"; do + echo "[firewall] hardening K3s ports on $host" + cluster_ssh "$host" 'bash -s' <<'REMOTE' +set -euo pipefail +[[ -d /sys/class/net/wg0 ]] || { echo 'wg0 is unavailable' >&2; exit 1; } +public_interface=$(ip -o route get 1.1.1.1 | awk '{ for (i = 1; i <= NF; i++) if ($i == "dev") { print $(i + 1); exit } }') +[[ $public_interface =~ ^[A-Za-z0-9._-]+$ && $public_interface != wg0 ]] || { + echo 'cannot determine public interface' >&2 + exit 1 +} +export DEBIAN_FRONTEND=noninteractive +if ! command -v netfilter-persistent >/dev/null 2>&1; then + apt-get update -qq + apt-get install -y -qq iptables-persistent +fi +ensure_rule() { + if ! iptables -C INPUT "$@" 2>/dev/null; then + iptables -I INPUT 1 "$@" + fi +} +ensure_rule -i wg0 -s 10.77.0.0/24 -p tcp -m multiport \ + --dports 2379,2380,6443,10250 -m comment --comment easyai-k3s-private -j ACCEPT +ensure_rule -i wg0 -s 10.77.0.0/24 -p udp --dport 8472 \ + -m comment --comment easyai-flannel-private -j ACCEPT +ensure_rule -i wg0 -s 10.77.0.0/24 -p tcp --dport 30000:32767 \ + -m comment --comment easyai-nodeport-private -j ACCEPT +ensure_rule -i "$public_interface" -p udp --dport 51820 \ + -m comment --comment easyai-wireguard-public -j ACCEPT +ensure_rule -i "$public_interface" -p tcp -m multiport \ + --dports 2379,2380,6443,10250 -m comment --comment easyai-k3s-public-drop -j DROP +ensure_rule -i "$public_interface" -p udp --dport 8472 \ + -m comment --comment easyai-flannel-public-drop -j DROP +ensure_rule -i "$public_interface" -p tcp --dport 30000:32767 \ + -m comment --comment easyai-nodeport-public-drop -j DROP +netfilter-persistent save >/dev/null +REMOTE +done + +echo 'node_network_hardening=PASS nodes=3' diff --git a/scripts/cluster/install-certificate-sync.sh b/scripts/cluster/install-certificate-sync.sh new file mode 100755 index 0000000..3d959dd --- /dev/null +++ b/scripts/cluster/install-certificate-sync.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" +load_cluster_env + +public_key=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +install -d -m 0700 /root/.ssh +if [[ ! -f /root/.ssh/id_ed25519_easyai_cert_sync ]]; then + ssh-keygen -q -t ed25519 -N '' -f /root/.ssh/id_ed25519_easyai_cert_sync +fi +cat /root/.ssh/id_ed25519_easyai_cert_sync.pub +REMOTE +) +[[ $public_key == ssh-ed25519\ * ]] || { + echo 'invalid certificate sync public key' >&2 + exit 1 +} +cluster_ssh "$CLUSTER_HONGKONG_HOST" bash -s -- "$public_key" <<'REMOTE' +set -euo pipefail +public_key=$1 +install -d -m 0700 /root/.ssh +touch /root/.ssh/authorized_keys +chmod 0600 /root/.ssh/authorized_keys +grep -Fqx "$public_key" /root/.ssh/authorized_keys || printf '%s\n' "$public_key" >>/root/.ssh/authorized_keys +REMOTE + +cluster_scp "$script_dir/sync-nginx-certificate.sh" \ + "$CLUSTER_NINGBO_HOST:/usr/local/sbin/easyai-sync-nginx-certificate" +cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +chmod 0750 /usr/local/sbin/easyai-sync-nginx-certificate +install -d -m 0755 /etc/letsencrypt/renewal-hooks/deploy +ln -sfn /usr/local/sbin/easyai-sync-nginx-certificate \ + /etc/letsencrypt/renewal-hooks/deploy/easyai-sync-nginx-certificate +/usr/local/sbin/easyai-sync-nginx-certificate +REMOTE +echo 'certificate_sync_install=PASS' diff --git a/scripts/cluster/install-legacy-cleaner.sh b/scripts/cluster/install-legacy-cleaner.sh new file mode 100755 index 0000000..1a6ce2f --- /dev/null +++ b/scripts/cluster/install-legacy-cleaner.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" +load_cluster_env + +cluster_scp "$script_dir/prune-legacy-generated.sh" \ + "$CLUSTER_NINGBO_HOST:/usr/local/sbin/easyai-prune-legacy-generated" +cluster_scp "$cluster_root/deploy/systemd/easyai-legacy-generated-cleanup.service" \ + "$CLUSTER_NINGBO_HOST:/etc/systemd/system/easyai-legacy-generated-cleanup.service" +cluster_scp "$cluster_root/deploy/systemd/easyai-legacy-generated-cleanup.timer" \ + "$CLUSTER_NINGBO_HOST:/etc/systemd/system/easyai-legacy-generated-cleanup.timer" +cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +chmod 0750 /usr/local/sbin/easyai-prune-legacy-generated +systemctl daemon-reload +/usr/local/sbin/easyai-prune-legacy-generated --dry-run +systemctl enable --now easyai-legacy-generated-cleanup.timer +systemctl is-active --quiet easyai-legacy-generated-cleanup.timer +REMOTE +echo 'legacy_cleanup_timer_install=PASS interval=hourly strict_age_seconds=86400' diff --git a/scripts/cluster/install-nginx-edges.sh b/scripts/cluster/install-nginx-edges.sh new file mode 100755 index 0000000..16ba041 --- /dev/null +++ b/scripts/cluster/install-nginx-edges.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" + +mode=${1:-prepare} +[[ $mode == prepare || $mode == activate ]] || { + echo 'usage: install-nginx-edges.sh [prepare|activate]' >&2 + exit 64 +} +load_cluster_env + +nginx_directory="$cluster_root/deploy/nginx" +for file in ai.51easyai.com-cluster.conf ai.51easyai.com-maintenance.conf \ + cluster-acceptance-origin-ningbo.conf legacy-static-ningbo.inc \ + legacy-static-hongkong.inc legacy-static-origin-ningbo.conf; do + [[ -f $nginx_directory/$file ]] || { + echo "missing NGINX file: $file" >&2 + exit 1 + } +done + +cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +install -d -m 0700 /etc/nginx/tls/ai.51easyai.com +install -m 0644 /etc/letsencrypt/live/ai.51easyai.com/fullchain.pem \ + /etc/nginx/tls/ai.51easyai.com/fullchain.pem +install -m 0600 /etc/letsencrypt/live/ai.51easyai.com/privkey.pem \ + /etc/nginx/tls/ai.51easyai.com/privkey.pem +REMOTE +cluster_scp \ + "$CLUSTER_NINGBO_HOST:/etc/nginx/tls/ai.51easyai.com/fullchain.pem" \ + "$cluster_root/.local-secrets/cluster/fullchain.pem" +cluster_scp \ + "$CLUSTER_NINGBO_HOST:/etc/nginx/tls/ai.51easyai.com/privkey.pem" \ + "$cluster_root/.local-secrets/cluster/privkey.pem" +chmod 0600 "$cluster_root/.local-secrets/cluster/privkey.pem" + +for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"; do + cluster_ssh "$host" 'bash -s' <<'REMOTE' +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive +if ! command -v nginx >/dev/null 2>&1; then + apt-get update -qq + apt-get install -y -qq nginx +fi +install -d -m 0755 /etc/nginx/easyai-ai-gateway +install -d -m 0700 /etc/nginx/tls/ai.51easyai.com +REMOTE + cluster_scp "$cluster_root/.local-secrets/cluster/fullchain.pem" \ + "$host:/etc/nginx/tls/ai.51easyai.com/fullchain.pem" + cluster_scp "$cluster_root/.local-secrets/cluster/privkey.pem" \ + "$host:/etc/nginx/tls/ai.51easyai.com/privkey.pem" + cluster_scp "$nginx_directory/ai.51easyai.com-cluster.conf" \ + "$host:/etc/nginx/sites-available/ai.51easyai.com-cluster" + cluster_scp "$nginx_directory/ai.51easyai.com-maintenance.conf" \ + "$host:/etc/nginx/sites-available/ai.51easyai.com-maintenance" +done + +cluster_scp "$nginx_directory/legacy-static-ningbo.inc" \ + "$CLUSTER_NINGBO_HOST:/etc/nginx/easyai-ai-gateway/legacy-static.inc" +cluster_scp "$nginx_directory/legacy-static-origin-ningbo.conf" \ + "$CLUSTER_NINGBO_HOST:/etc/nginx/conf.d/easyai-legacy-static-origin.conf" +cluster_scp "$nginx_directory/cluster-acceptance-origin-ningbo.conf" \ + "$CLUSTER_NINGBO_HOST:/etc/nginx/conf.d/easyai-cluster-acceptance-origin.conf" +cluster_scp "$nginx_directory/legacy-static-hongkong.inc" \ + "$CLUSTER_HONGKONG_HOST:/etc/nginx/easyai-ai-gateway/legacy-static.inc" + +for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"; do + cluster_ssh "$host" 'bash -s' <<'REMOTE' +set -euo pipefail +chmod 0600 /etc/nginx/tls/ai.51easyai.com/privkey.pem +chmod 0644 /etc/nginx/tls/ai.51easyai.com/fullchain.pem +nginx -t +REMOTE +done + +if [[ $mode == activate ]]; then + for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"; do + cluster_ssh "$host" 'bash -s' <<'REMOTE' +set -euo pipefail +ln -sfn /etc/nginx/sites-available/ai.51easyai.com-cluster \ + /etc/nginx/sites-enabled/ai.51easyai.com-cluster +if [[ -L /etc/nginx/sites-enabled/ai.51easyai.com ]]; then + rm -f /etc/nginx/sites-enabled/ai.51easyai.com +fi +nginx -t +systemctl reload nginx +REMOTE + done +fi + +echo "nginx_edges=PASS mode=$mode" diff --git a/scripts/cluster/install-release-helper.sh b/scripts/cluster/install-release-helper.sh new file mode 100755 index 0000000..572e3c3 --- /dev/null +++ b/scripts/cluster/install-release-helper.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" +load_cluster_env + +cluster_ssh "$CLUSTER_NINGBO_HOST" \ + 'install -d -m 0755 /usr/local/lib/easyai-ai-gateway-release' +cluster_scp "$cluster_root/deploy/kubernetes/easyai-ai-gateway-cluster-release" \ + "$CLUSTER_NINGBO_HOST:/usr/local/sbin/easyai-ai-gateway-cluster-release" +cluster_scp "$cluster_root/deploy/kubernetes/easyai-ai-gateway-cluster-release.conf.example" \ + "$CLUSTER_NINGBO_HOST:/etc/easyai-ai-gateway-cluster-release.conf" +cluster_scp "$cluster_root/scripts/release-manifest.mjs" \ + "$CLUSTER_NINGBO_HOST:/usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs" +cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +chmod 0750 /usr/local/sbin/easyai-ai-gateway-cluster-release +chmod 0640 /etc/easyai-ai-gateway-cluster-release.conf +chmod 0755 /usr/local/lib/easyai-ai-gateway-release/release-manifest.mjs +REMOTE +echo 'cluster_release_helper_install=PASS' diff --git a/scripts/cluster/migrate-production.sh b/scripts/cluster/migrate-production.sh new file mode 100755 index 0000000..0bae364 --- /dev/null +++ b/scripts/cluster/migrate-production.sh @@ -0,0 +1,362 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" + +action=${1:-preflight} +release_manifest=${2:-} +[[ $action == preflight || $action == cutover ]] || { + echo 'usage: migrate-production.sh {preflight|cutover} dist/releases/.json' >&2 + exit 64 +} +[[ -f $release_manifest && ! -L $release_manifest ]] || { + echo 'release manifest must be a regular file' >&2 + exit 1 +} + +load_cluster_env +node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null +source_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha) +api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" images.api) + +if [[ $action == cutover && ${AI_GATEWAY_CUTOVER_WINDOW_OVERRIDE:-} != I_ACCEPT_PRODUCTION_RISK ]]; then + shanghai_hour=$(TZ=Asia/Shanghai date '+%H') + if (( 10#$shanghai_hour < 2 || 10#$shanghai_hour >= 4 )); then + echo 'cutover is restricted to 02:00-04:00 Asia/Shanghai' >&2 + exit 1 + fi +fi + +echo '[cutover] checking Docker, K3s, CNPG, disk and backup prerequisites' +cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +docker inspect easyai-ai-gateway-api-1 easyai-ai-gateway-postgres-1 >/dev/null +docker exec easyai-ai-gateway-postgres-1 pg_isready >/dev/null +k3s kubectl get --raw=/readyz >/dev/null +[[ $(k3s kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq 3 ]] +[[ $(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') -eq 2 ]] +[[ $(df --output=avail -B1 /var/lib/docker | tail -1) -gt 2147483648 ]] +[[ -f /etc/nginx/sites-available/ai.51easyai.com-maintenance ]] +[[ -f /etc/nginx/sites-available/ai.51easyai.com-cluster ]] +nginx -t +REMOTE + +if [[ $action == preflight ]]; then + echo "production_migration_preflight=PASS release=$source_sha" + exit 0 +fi + +working_directory=$(mktemp -d "$cluster_root/.local-secrets/cluster/cutover.XXXXXX") +trap '[[ $working_directory == "$cluster_root"/.local-secrets/cluster/cutover.* ]] && rm -rf -- "$working_directory"' EXIT HUP INT TERM +cluster_scp "$script_dir/oss-object.mjs" "$CLUSTER_NINGBO_HOST:/root/easyai-oss-object.mjs" + +remote_cutover_script="$working_directory/remote-cutover.sh" +umask 077 +cp /dev/null "$remote_cutover_script" +chmod 0700 "$remote_cutover_script" +cat >"$remote_cutover_script" <<'REMOTE' +#!/usr/bin/env bash +set -euo pipefail + +source_sha=$1 +api_image=$2 +active_site=${AI_GATEWAY_NGINX_ACTIVE_SITE:-/etc/nginx/sites-enabled/ai.51easyai.com} +timestamp=$(date -u '+%Y%m%dT%H%M%SZ') +backup_directory=/var/backups/easyai-ai-gateway/pre-cutover-"$timestamp" +old_site_backup=$backup_directory/nginx-site.conf +cutover_started=$(date +%s) +traffic_committed=false + +rollback_before_commit() { + status=$? + trap - ERR + if [[ $traffic_committed == false ]]; then + echo '[cutover] rolling back to the frozen Docker database' >&2 + k3s kubectl scale deployment -n easyai \ + easyai-api-ningbo easyai-api-hongkong easyai-web-ningbo easyai-web-hongkong \ + --replicas=0 >/dev/null 2>&1 || true + docker start easyai-ai-gateway-api-1 easyai-ai-gateway-web-1 >/dev/null 2>&1 || true + if [[ -f $old_site_backup ]]; then + install -m 0644 "$old_site_backup" /etc/nginx/sites-available/ai.51easyai.com-pre-ha + rm -f -- "$active_site" + ln -s /etc/nginx/sites-available/ai.51easyai.com-pre-ha "$active_site" + nginx -t && systemctl reload nginx + fi + else + echo '[cutover] traffic is already committed; automatic database rollback is unsafe' >&2 + fi + exit "$status" +} +trap rollback_before_commit ERR + +install -d -m 0700 "$backup_directory" +[[ -e $active_site || -L $active_site ]] || { + echo "active NGINX site is unavailable: $active_site" >&2 + exit 1 +} +cp -L "$active_site" "$old_site_backup" +chmod 0600 "$old_site_backup" +rm -f -- "$active_site" +ln -s /etc/nginx/sites-available/ai.51easyai.com-maintenance "$active_site" +nginx -t +systemctl reload nginx +curl -ksS --resolve ai.51easyai.com:443:127.0.0.1 \ + -o /dev/null -w '%{http_code}' https://ai.51easyai.com/api/v1/healthz | + grep -qx 503 + +docker stop --time 60 easyai-ai-gateway-api-1 >/dev/null +postgres_user=$(docker inspect easyai-ai-gateway-postgres-1 \ + --format '{{range .Config.Env}}{{println .}}{{end}}' | + awk -F= '$1 == "POSTGRES_USER" { print substr($0, index($0, "=") + 1) }') +postgres_database=$(docker inspect easyai-ai-gateway-postgres-1 \ + --format '{{range .Config.Env}}{{println .}}{{end}}' | + awk -F= '$1 == "POSTGRES_DB" { print substr($0, index($0, "=") + 1) }') +[[ $postgres_user =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] +[[ $postgres_database =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] + +snapshot_sql=$backup_directory/acceptance-snapshot.sql +cat >"$snapshot_sql" <<'SQL' +\pset tuples_only on +\pset format unaligned +SELECT 'server_version=' || current_setting('server_version'); +SELECT 'extensions=' || COALESCE(string_agg(extname || ':' || extversion, ',' ORDER BY extname), '') + FROM pg_extension WHERE extname <> 'plpgsql'; +SELECT 'tables=' || count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind = 'r' AND n.nspname = 'public'; +SELECT 'sequences=' || COALESCE(string_agg(schemaname || '.' || sequencename || ':' || + COALESCE(last_value::text, 'null'), ',' ORDER BY schemaname, sequencename), '') + FROM pg_sequences WHERE schemaname = 'public'; +SELECT 'catalog=' || md5(COALESCE(string_agg(definition, E'\n' ORDER BY definition), '')) +FROM ( + SELECT 'column:' || table_name || ':' || column_name || ':' || data_type || ':' || + is_nullable || ':' || COALESCE(column_default, '') AS definition + FROM information_schema.columns WHERE table_schema = 'public' + UNION ALL + SELECT 'constraint:' || conrelid::regclass::text || ':' || conname || ':' || + pg_get_constraintdef(oid) FROM pg_constraint WHERE connamespace = 'public'::regnamespace + UNION ALL + SELECT 'index:' || indexrelid::regclass::text || ':' || pg_get_indexdef(indexrelid) + FROM pg_index WHERE indrelid IN ( + SELECT oid FROM pg_class WHERE relnamespace = 'public'::regnamespace + ) +) definitions; +SELECT 'gateway_tasks=' || count(*) FROM gateway_tasks; +SELECT 'gateway_task_attempts=' || count(*) FROM gateway_task_attempts; +SELECT 'gateway_wallet_transactions=' || count(*) FROM gateway_wallet_transactions; +SELECT 'gateway_upload_assets=' || count(*) FROM gateway_upload_assets; +SELECT 'gateway_users=' || count(*) FROM gateway_users; +SELECT 'gateway_api_keys=' || count(*) FROM gateway_api_keys; +SELECT 'gateway_providers=' || count(*) FROM gateway_providers; +SELECT 'task_sample=' || md5(COALESCE(string_agg(id::text, ',' ORDER BY id), '')) + FROM (SELECT id FROM gateway_tasks ORDER BY id LIMIT 100) sample; +SQL + +docker exec -i easyai-ai-gateway-postgres-1 \ + psql -X -v ON_ERROR_STOP=1 -U "$postgres_user" -d "$postgres_database" \ + <"$snapshot_sql" >"$backup_directory/source.snapshot" +docker exec easyai-ai-gateway-postgres-1 \ + pg_dumpall -U "$postgres_user" --globals-only \ + >"$backup_directory/globals.sql" +docker exec easyai-ai-gateway-postgres-1 \ + pg_dump -U "$postgres_user" -d "$postgres_database" \ + --format=custom --compress=6 \ + >"$backup_directory/database.dump" +[[ -s $backup_directory/database.dump && -s $backup_directory/globals.sql ]] +docker exec -i easyai-ai-gateway-postgres-1 pg_restore -l \ + <"$backup_directory/database.dump" >/dev/null +sha256sum "$backup_directory/database.dump" "$backup_directory/globals.sql" \ + "$backup_directory/source.snapshot" >"$backup_directory/SHA256SUMS" +sha256sum -c "$backup_directory/SHA256SUMS" + +export ALI_KEY +export ALI_SECRET +ALI_KEY=$(k3s kubectl get secret easyai-oss-backup -n easyai \ + -o jsonpath='{.data.ACCESS_KEY_ID}' | base64 -d) +ALI_SECRET=$(k3s kubectl get secret easyai-oss-backup -n easyai \ + -o jsonpath='{.data.ACCESS_SECRET_KEY}' | base64 -d) +export ALI_REGION=oss-cn-shanghai +export ALI_BUCKET=wangbo0808 +object_prefix=easyai-ai-gateway/production/pre-cutover/"$timestamp" +for file in database.dump globals.sql source.snapshot SHA256SUMS; do + node /root/easyai-oss-object.mjs put "$object_prefix/$file" "$backup_directory/$file" +done +node /root/easyai-oss-object.mjs head "$object_prefix/database.dump" + +primary=$(k3s kubectl get cluster easyai-postgres -n easyai \ + -o jsonpath='{.status.currentPrimary}') +[[ $primary =~ ^easyai-postgres-[0-9]+$ ]] +k3s kubectl exec -n easyai "$primary" -c postgres -- \ + psql -X -v ON_ERROR_STOP=1 -U postgres -d postgres \ + -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'easyai_ai_gateway' AND pid <> pg_backend_pid();" \ + -c 'DROP DATABASE IF EXISTS easyai_ai_gateway;' \ + -c 'CREATE DATABASE easyai_ai_gateway OWNER easyai;' +k3s kubectl exec -i -n easyai "$primary" -c postgres -- \ + pg_restore --exit-on-error --no-owner --role=easyai \ + -U postgres -d easyai_ai_gateway <"$backup_directory/database.dump" +k3s kubectl exec -i -n easyai "$primary" -c postgres -- \ + psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway \ + <"$snapshot_sql" >"$backup_directory/restored.snapshot" +diff -u "$backup_directory/source.snapshot" "$backup_directory/restored.snapshot" + +cat >"$backup_directory/migrator-job.yaml" </dev/null +k3s kubectl apply -f "$backup_directory/migrator-job.yaml" >/dev/null +k3s kubectl wait --for=condition=complete \ + "job/easyai-migrator-${source_sha:0:12}" -n easyai --timeout=300s +k3s kubectl exec -n easyai "$primary" -c postgres -- \ + psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \ + "SELECT count(*) FROM schema_migrations WHERE version='0089_file_storage_request_asset_scene';" | + grep -qx 1 +k3s kubectl exec -n easyai "$primary" -c postgres -- \ + psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \ + "SELECT count(*) FROM file_storage_channels WHERE deleted_at IS NULL AND provider='server_main_openapi' AND config->'scenes' ? 'request_asset';" | + grep -Eq '^[1-9][0-9]*$' + +elapsed=$(( $(date +%s) - cutover_started )) +if (( elapsed > 600 )); then + echo "cutover exceeded ten-minute rollback boundary: ${elapsed}s" >&2 + exit 1 +fi +printf '%s\n' "$backup_directory" >/root/.easyai-pre-cutover-backup +printf 'database_restore=PASS seconds=%s backup=%s\n' "$elapsed" "$backup_directory" +REMOTE + +cluster_scp "$remote_cutover_script" "$CLUSTER_NINGBO_HOST:/root/easyai-remote-cutover.sh" +cluster_ssh "$CLUSTER_NINGBO_HOST" \ + "chmod 0700 /root/easyai-remote-cutover.sh && /root/easyai-remote-cutover.sh '$source_sha' '$api_image'" + +traffic_committed=false +rollback_local_before_commit() { + status=$? + trap - ERR + if [[ $traffic_committed == false ]]; then + echo '[cutover] local activation failed; restoring Docker before public traffic commit' >&2 + cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' || true +set -euo pipefail +backup_directory=$(cat /root/.easyai-pre-cutover-backup) +[[ $backup_directory == /var/backups/easyai-ai-gateway/pre-cutover-* ]] +k3s kubectl scale deployment -n easyai \ + easyai-api-ningbo easyai-api-hongkong easyai-web-ningbo easyai-web-hongkong \ + --replicas=0 >/dev/null 2>&1 || true +docker start easyai-ai-gateway-api-1 easyai-ai-gateway-web-1 >/dev/null +rm -f -- /etc/nginx/sites-enabled/ai.51easyai.com +install -m 0644 "$backup_directory/nginx-site.conf" \ + /etc/nginx/sites-available/ai.51easyai.com-pre-ha +ln -s /etc/nginx/sites-available/ai.51easyai.com-pre-ha \ + /etc/nginx/sites-enabled/ai.51easyai.com +nginx -t +systemctl reload nginx +REMOTE + fi + exit "$status" +} +trap rollback_local_before_commit ERR + +echo '[cutover] starting both application sites with River workers frozen' +AI_GATEWAY_ACTIVATE_WORKERS=false \ + "$script_dir/bootstrap-platform.sh" activate "$release_manifest" + +for site_ip in 10.77.0.1 10.77.0.2; do + cluster_ssh "$CLUSTER_NINGBO_HOST" \ + "curl -fsS --max-time 10 http://$site_ip:30088/api/v1/healthz >/dev/null" + cluster_ssh "$CLUSTER_NINGBO_HOST" \ + "curl -fsS --max-time 10 http://$site_ip:30088/api/v1/readyz | grep -q '\"ok\":true'" + cluster_ssh "$CLUSTER_NINGBO_HOST" \ + "curl -fsS --max-time 10 http://$site_ip:30178/ | grep -q 'EasyAI AI Gateway'" +done + +"$script_dir/install-nginx-edges.sh" activate +for edge_host in "${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}"; do + curl -fsS --resolve "ai.51easyai.com:443:$edge_host" \ + https://ai.51easyai.com/api/v1/healthz >/dev/null + curl -fsS --resolve "ai.51easyai.com:443:$edge_host" \ + https://ai.51easyai.com/api/v1/readyz | grep -q '"ok":true' +done + +traffic_committed=true +echo '[cutover] traffic committed; enabling Hong Kong worker, then Ningbo worker' +cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +kubectl='k3s kubectl' +$kubectl set env deployment/easyai-api-hongkong -n easyai \ + AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true +$kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s +$kubectl set env deployment/easyai-api-ningbo -n easyai \ + AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true +$kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s +cat <<'EOF' | $kubectl create -f - >/dev/null +apiVersion: postgresql.cnpg.io/v1 +kind: Backup +metadata: + generateName: easyai-post-cutover- + namespace: easyai +spec: + cluster: + name: easyai-postgres + method: plugin + pluginConfiguration: + name: barman-cloud.cloudnative-pg.io +EOF +docker update --restart=no easyai-ai-gateway-api-1 \ + easyai-ai-gateway-web-1 easyai-ai-gateway-postgres-1 >/dev/null +docker stop --time 30 easyai-ai-gateway-web-1 easyai-ai-gateway-postgres-1 >/dev/null +rm -f -- /root/easyai-remote-cutover.sh /root/easyai-oss-object.mjs +REMOTE + +"$script_dir/install-release-helper.sh" +remote_manifest=/tmp/easyai-cutover-release-"$source_sha".json +cluster_scp "$release_manifest" "$CLUSTER_NINGBO_HOST:$remote_manifest" +cluster_ssh "$CLUSTER_NINGBO_HOST" \ + "/usr/local/sbin/easyai-ai-gateway-cluster-release adopt '$remote_manifest' && rm -f '$remote_manifest'" + +set_local_environment() { + local key=$1 + local value=$2 + local temporary + temporary=$(mktemp "$cluster_env_file.next.XXXXXX") + chmod 0600 "$temporary" + awk -v key="$key" -v value="$value" -F= ' + $1 == key { if (!written) print key "=" value; written = 1; next } + { print } + END { if (!written) print key "=" value } + ' "$cluster_env_file" >"$temporary" + mv "$temporary" "$cluster_env_file" +} +set_local_environment AI_GATEWAY_DEPLOY_MODE kubernetes +set_local_environment AI_GATEWAY_REMOTE_RELEASE_HELPER /usr/local/sbin/easyai-ai-gateway-cluster-release +set_local_environment AI_GATEWAY_PRODUCTION_SSH_KEY "$cluster_ssh_key" + +trap - ERR +echo "production_cutover=PASS release=$source_sha old_postgres_retained=true" diff --git a/scripts/cluster/oss-object.mjs b/scripts/cluster/oss-object.mjs new file mode 100755 index 0000000..d34c4f4 --- /dev/null +++ b/scripts/cluster/oss-object.mjs @@ -0,0 +1,103 @@ +#!/usr/bin/env node + +import { createHmac, createHash } from 'node:crypto' +import { createReadStream, createWriteStream, statSync } from 'node:fs' +import { request } from 'node:https' +import { pipeline } from 'node:stream/promises' + +function fail(message) { + console.error(`oss_object=FAIL ${message}`) + process.exit(1) +} + +function requiredEnv(name, pattern) { + const value = process.env[name] || '' + if (!value || (pattern && !pattern.test(value))) fail(`invalid or missing ${name}`) + return value +} + +function encodeObjectKey(value) { + if (!value || value.startsWith('/') || value.includes('..')) fail('invalid object key') + return value.split('/').map(encodeURIComponent).join('/') +} + +const [operation, objectKeyValue, filePath] = process.argv.slice(2) +if (!['put', 'get', 'head', 'delete'].includes(operation || '')) { + fail('usage: oss-object.mjs {put|get|head|delete} [file]') +} +if ((operation === 'put' || operation === 'get') && !filePath) fail(`${operation} requires a file`) + +const accessKey = requiredEnv('ALI_KEY') +const secretKey = requiredEnv('ALI_SECRET') +const region = requiredEnv('ALI_REGION', /^[a-z0-9-]+$/) +const bucket = requiredEnv('ALI_BUCKET', /^[A-Za-z0-9.-]+$/) +const objectKey = encodeObjectKey(objectKeyValue) +const hostname = `${bucket}.${region}.aliyuncs.com` +const canonicalResource = `/${bucket}/${objectKey}` +const date = new Date().toUTCString() +const method = operation.toUpperCase() +const contentType = operation === 'put' ? 'application/octet-stream' : '' +const stringToSign = `${method}\n\n${contentType}\n${date}\n${canonicalResource}` +const signature = createHmac('sha1', secretKey).update(stringToSign).digest('base64') +const headers = { + Authorization: `OSS ${accessKey}:${signature}`, + Date: date, +} + +let expectedDigest = '' +if (operation === 'put') { + let details + try { + details = statSync(filePath) + } catch (error) { + fail(`cannot stat upload file: ${error.message}`) + } + if (!details.isFile()) fail('upload source must be a regular file') + headers['Content-Type'] = contentType + headers['Content-Length'] = String(details.size) + expectedDigest = createHash('sha256') +} + +const response = await new Promise((resolve, reject) => { + const outgoing = request({ + method, + hostname, + path: `/${objectKey}`, + headers, + timeout: 120_000, + }, resolve) + outgoing.once('timeout', () => outgoing.destroy(new Error('request timed out'))) + outgoing.once('error', reject) + if (operation === 'put') { + const source = createReadStream(filePath) + source.on('data', (chunk) => expectedDigest.update(chunk)) + source.once('error', reject) + source.pipe(outgoing) + } else { + outgoing.end() + } +}).catch((error) => fail(error.message)) + +const successStatus = operation === 'get' || operation === 'head' ? 200 : [200, 204].includes(response.statusCode) +if (!successStatus) { + let responseBody = '' + for await (const chunk of response) responseBody += chunk + fail(`HTTP ${response.statusCode} ${responseBody.slice(0, 240).replaceAll(/\s+/g, ' ')}`) +} + +if (operation === 'get') { + const destination = createWriteStream(filePath, { flags: 'wx', mode: 0o600 }) + await pipeline(response, destination).catch((error) => fail(`download failed: ${error.message}`)) +} + +if (operation === 'put') { + for await (const _chunk of response) { + // Drain the response so the request completes before reporting success. + } + console.log(`oss_object=PASS operation=put sha256=${expectedDigest.digest('hex')}`) +} else { + for await (const _chunk of response) { + // GET is consumed by pipeline; other successful response bodies are empty. + } + console.log(`oss_object=PASS operation=${operation}`) +} diff --git a/scripts/cluster/prune-legacy-generated.sh b/scripts/cluster/prune-legacy-generated.sh new file mode 100755 index 0000000..b83adee --- /dev/null +++ b/scripts/cluster/prune-legacy-generated.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +mode=${1:---dry-run} +storage_root=${AI_GATEWAY_LEGACY_STORAGE_ROOT:-/var/lib/docker/volumes/easyai-ai-gateway_api_data/_data/static} +maximum_age_seconds=86400 + +[[ $mode == --dry-run || $mode == --apply ]] || { + echo 'usage: prune-legacy-generated.sh [--dry-run|--apply]' >&2 + exit 64 +} +[[ $storage_root == /var/lib/docker/volumes/*/_data/static && -d $storage_root && ! -L $storage_root ]] || { + echo 'legacy storage root failed safety validation' >&2 + exit 1 +} + +candidate_list=$(mktemp) +trap 'rm -f -- "$candidate_list"' EXIT HUP INT TERM +now_epoch=$(date +%s) +while IFS= read -r -d '' file; do + modified_epoch=$(stat -c '%Y' "$file") + if (( now_epoch - modified_epoch > maximum_age_seconds )); then + printf '%s\0' "$file" >>"$candidate_list" + fi +done < <(find "$storage_root" -xdev -type f -print0) + +file_count=$(tr -cd '\0' <"$candidate_list" | wc -c | tr -d ' ') +byte_count=0 +while IFS= read -r -d '' file; do + [[ $file == "$storage_root/"* && ! -L $file ]] || { + echo "unsafe cleanup candidate: $file" >&2 + exit 1 + } + byte_count=$((byte_count + $(stat -c '%s' "$file"))) +done <"$candidate_list" + +if [[ $mode == --apply ]]; then + while IFS= read -r -d '' file; do + rm -f -- "$file" + done <"$candidate_list" + find "$storage_root" -xdev -depth -type d -empty -delete +fi + +printf 'legacy_cleanup=%s files=%s bytes=%s age_seconds_strictly_greater_than=86400\n' \ + "${mode#--}" "$file_count" "$byte_count" diff --git a/scripts/cluster/restore-backup-drill.sh b/scripts/cluster/restore-backup-drill.sh new file mode 100755 index 0000000..d76da8b --- /dev/null +++ b/scripts/cluster/restore-backup-drill.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" + +[[ ${1:-} == --execute ]] || { + echo 'usage: restore-backup-drill.sh --execute' >&2 + exit 64 +} +load_cluster_env + +cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +kubectl='k3s kubectl' +suffix=$(date -u '+%m%d%H%M%S') +restore_cluster=easyai-restore-drill-"$suffix" +cleanup() { + $kubectl delete cluster "$restore_cluster" -n easyai \ + --ignore-not-found=true --wait=true --timeout=300s || true + $kubectl delete pvc -n easyai -l "cnpg.io/cluster=$restore_cluster" \ + --ignore-not-found=true --wait=true --timeout=300s || true +} +trap cleanup EXIT HUP INT TERM + +source_primary=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}') +acceptance_sql=$(mktemp) +trap 'rm -f -- "$acceptance_sql"; cleanup' EXIT HUP INT TERM +cat >"$acceptance_sql" <<'SQL' +\pset tuples_only on +\pset format unaligned +SELECT 'tables=' || count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE c.relkind='r' AND n.nspname='public'; +SELECT 'tasks=' || count(*) FROM gateway_tasks; +SELECT 'attempts=' || count(*) FROM gateway_task_attempts; +SELECT 'wallet=' || count(*) FROM gateway_wallet_transactions; +SELECT 'users=' || count(*) FROM gateway_users; +SELECT 'sample=' || md5(COALESCE(string_agg(id::text, ',' ORDER BY id),'')) + FROM (SELECT id FROM gateway_tasks ORDER BY id LIMIT 100) rows; +SQL +source_snapshot=$($kubectl exec -i -n easyai "$source_primary" -c postgres -- \ + psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway <"$acceptance_sql") + +cat </dev/null +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: $restore_cluster + namespace: easyai + labels: + easyai.io/acceptance-drill: backup-restore +spec: + instances: 1 + imageName: ghcr.io/cloudnative-pg/postgresql:18.4-standard-trixie@sha256:4587df73024408f5b2be9b4dd6ba2ccee8c9e5dc0c9a87c274c292291cc8a68c + bootstrap: + recovery: + source: origin + database: easyai_ai_gateway + owner: easyai + secret: + name: easyai-postgres-app + externalClusters: + - name: origin + plugin: + name: barman-cloud.cloudnative-pg.io + parameters: + barmanObjectName: easyai-postgres-oss + serverName: easyai-postgres + storage: + storageClass: local-path + size: 20Gi + walStorage: + storageClass: local-path + size: 5Gi + affinity: + nodeSelector: + easyai.io/site: ningbo + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 3Gi +EOF + +$kubectl wait --for=condition=Ready "cluster/$restore_cluster" -n easyai --timeout=1200s +restore_primary=$($kubectl get cluster "$restore_cluster" -n easyai -o jsonpath='{.status.currentPrimary}') +restored_snapshot=$($kubectl exec -i -n easyai "$restore_primary" -c postgres -- \ + psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway <"$acceptance_sql") +[[ $restored_snapshot == "$source_snapshot" ]] || { + diff -u <(printf '%s\n' "$source_snapshot") <(printf '%s\n' "$restored_snapshot") + exit 1 +} +$kubectl exec -n easyai "$restore_primary" -c postgres -- \ + psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At -c \ + "SELECT current_user || ':' || current_database();" | grep -qx 'easyai:easyai_ai_gateway' +cleanup +trap - EXIT HUP INT TERM +rm -f -- "$acceptance_sql" +echo "backup_restore_drill=PASS cluster=$restore_cluster deleted=true" +REMOTE diff --git a/scripts/cluster/run-cross-node-file-e2e.sh b/scripts/cluster/run-cross-node-file-e2e.sh new file mode 100755 index 0000000..203957e --- /dev/null +++ b/scripts/cluster/run-cross-node-file-e2e.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" +load_cluster_env +require_commands base64 jq node + +: "${AI_GATEWAY_ONLINE_ACCOUNT:?}" +: "${AI_GATEWAY_ONLINE_PASSWORD:?}" + +restore_workers() { + cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' || true +set -euo pipefail +k3s kubectl set env deployment/easyai-api-ningbo -n easyai \ + AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true >/dev/null +k3s kubectl set env deployment/easyai-api-hongkong -n easyai \ + AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true >/dev/null +k3s kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s +k3s kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s +REMOTE +} +trap restore_workers EXIT HUP INT TERM + +cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +k3s kubectl set env deployment/easyai-api-hongkong -n easyai \ + AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=true >/dev/null +k3s kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=300s +k3s kubectl set env deployment/easyai-api-ningbo -n easyai \ + AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=false >/dev/null +k3s kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s +curl -fsS http://10.77.0.1:30088/api/v1/readyz | grep -q '"ok":true' +curl -fsS http://10.77.0.2:30088/api/v1/readyz | grep -q '"ok":true' +REMOTE + +credentials_file=$(mktemp "$cluster_root/.local-secrets/cluster/e2e.XXXXXX") +trap 'rm -f -- "$credentials_file"; restore_workers' EXIT HUP INT TERM +chmod 0600 "$credentials_file" +printf 'E2E_ACCOUNT_B64=%s\n' "$(printf '%s' "$AI_GATEWAY_ONLINE_ACCOUNT" | base64 | tr -d '\n')" \ + >"$credentials_file" +printf 'E2E_PASSWORD_B64=%s\n' "$(printf '%s' "$AI_GATEWAY_ONLINE_PASSWORD" | base64 | tr -d '\n')" \ + >>"$credentials_file" +printf 'AI_GATEWAY_E2E_IMAGE_MODEL=%s\n' "${AI_GATEWAY_E2E_IMAGE_MODEL:-openai:gpt-image-1}" \ + >>"$credentials_file" +cluster_scp "$credentials_file" "$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-e2e.env" +cluster_scp "$script_dir/cross-node-file-e2e.mjs" \ + "$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-file-e2e.mjs" + +result=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +chmod 0600 /root/easyai-cross-node-e2e.env +set -a +# shellcheck source=/dev/null +source /root/easyai-cross-node-e2e.env +set +a +node /root/easyai-cross-node-file-e2e.mjs +rm -f -- /root/easyai-cross-node-e2e.env /root/easyai-cross-node-file-e2e.mjs +REMOTE +) +task_id=$(jq -r '.taskId' <<<"$result") +source_sha=$(jq -r '.sourceSHA256' <<<"$result") +result_url=$(jq -r '.resultURL' <<<"$result") +[[ $task_id =~ ^[0-9a-f-]{36}$ && $source_sha =~ ^[0-9a-f]{64}$ && $result_url == https://* ]] + +database_evidence=$(cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$task_id" "$source_sha" <<'REMOTE' +set -euo pipefail +task_id=$1 +source_sha=$2 +primary=$(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}') +query() { + k3s kubectl exec -n easyai "$primary" -c postgres -- \ + psql -X -v ON_ERROR_STOP=1 -U easyai -d easyai_ai_gateway -At "$@" +} +attempted_by=$(query -c \ + "SELECT attempted_by::text FROM river_job WHERE id=(SELECT river_job_id FROM gateway_tasks WHERE id='$task_id'::uuid);") +[[ $attempted_by == *easyai-api-hongkong* ]] || { + echo "River attempted_by does not identify Hong Kong: $attempted_by" >&2 + exit 1 +} +[[ $(query -c "SELECT count(*) FROM gateway_task_attempts WHERE task_id='$task_id'::uuid AND status='succeeded';") -eq 1 ]] +[[ $(query -c "SELECT count(*) FROM gateway_task_attempts WHERE task_id='$task_id'::uuid;") -eq 1 ]] +[[ $(query -c "SELECT count(*) FROM settlement_outbox WHERE task_id='$task_id'::uuid AND action='settle';") -eq 1 ]] +[[ $(query -c "SELECT count(*) FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id='$task_id' AND transaction_type='task_billing';") -eq 1 ]] +callback_counts=$(query -F ' ' -c \ + "SELECT count(*), count(DISTINCT (seq, callback_url)) FROM gateway_task_callback_outbox WHERE task_id='$task_id'::uuid;") +read -r callbacks unique_callbacks <<<"$callback_counts" +[[ $callbacks -eq $unique_callbacks ]] +source_url=$(query -c \ + "SELECT url FROM gateway_request_assets WHERE sha256='$source_sha' ORDER BY created_at DESC LIMIT 1;") +[[ $source_url == https://* ]] +printf '%s\n' "$source_url" +REMOTE +) +source_url=$(tail -1 <<<"$database_evidence") + +compare_url_from_nodes() { + local label=$1 + local url=$2 + local expected_hash= + local expected_size= + for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"; do + evidence=$(cluster_ssh "$host" bash -s -- "$url" <<'REMOTE' +set -euo pipefail +url=$1 +temporary=$(mktemp) +trap 'rm -f -- "$temporary"' EXIT +curl -fsSL --max-time 300 "$url" -o "$temporary" +printf '%s %s\n' "$(sha256sum "$temporary" | awk '{print $1}')" "$(stat -c '%s' "$temporary")" +REMOTE +) + read -r hash size <<<"$evidence" + [[ $hash =~ ^[0-9a-f]{64}$ && $size -gt 0 ]] + if [[ -z $expected_hash ]]; then + expected_hash=$hash + expected_size=$size + else + [[ $hash == "$expected_hash" && $size == "$expected_size" ]] + fi + done + printf '%s_url=PASS sha256=%s bytes=%s\n' "$label" "$expected_hash" "$expected_size" +} + +compare_url_from_nodes request_asset "$source_url" +compare_url_from_nodes generated_result "$result_url" +restore_workers +trap - EXIT HUP INT TERM +rm -f -- "$credentials_file" +echo "cross_node_file_e2e=PASS task=$task_id worker=hongkong" diff --git a/scripts/cluster/sync-nginx-certificate.sh b/scripts/cluster/sync-nginx-certificate.sh new file mode 100755 index 0000000..de31d45 --- /dev/null +++ b/scripts/cluster/sync-nginx-certificate.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +domain=ai.51easyai.com +hongkong_host=${AI_GATEWAY_CERT_SYNC_HOST:-root@10.77.0.2} +identity_file=${AI_GATEWAY_CERT_SYNC_KEY:-/root/.ssh/id_ed25519_easyai_cert_sync} + +[[ $(hostname -I) == *10.77.0.1* ]] || { + echo 'certificate sync must run on the Ningbo node' >&2 + exit 1 +} +[[ -f $identity_file && ! -L $identity_file ]] || { + echo 'certificate sync SSH identity is unavailable' >&2 + exit 1 +} + +install -d -m 0700 "/etc/nginx/tls/$domain" +install -m 0644 "/etc/letsencrypt/live/$domain/fullchain.pem" \ + "/etc/nginx/tls/$domain/fullchain.pem" +install -m 0600 "/etc/letsencrypt/live/$domain/privkey.pem" \ + "/etc/nginx/tls/$domain/privkey.pem" +ssh -i "$identity_file" -o BatchMode=yes "$hongkong_host" \ + "install -d -m 0700 /etc/nginx/tls/$domain" +scp -q -i "$identity_file" -o BatchMode=yes \ + "/etc/nginx/tls/$domain/fullchain.pem" \ + "$hongkong_host:/etc/nginx/tls/$domain/fullchain.pem" +scp -q -i "$identity_file" -o BatchMode=yes \ + "/etc/nginx/tls/$domain/privkey.pem" \ + "$hongkong_host:/etc/nginx/tls/$domain/privkey.pem" +ssh -i "$identity_file" -o BatchMode=yes "$hongkong_host" \ + "chmod 0600 /etc/nginx/tls/$domain/privkey.pem && nginx -t && systemctl reload nginx" +nginx -t +systemctl reload nginx +echo 'certificate_sync=PASS' diff --git a/scripts/cluster/verify-cluster.sh b/scripts/cluster/verify-cluster.sh new file mode 100755 index 0000000..8186b7f --- /dev/null +++ b/scripts/cluster/verify-cluster.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" + +mode=${1:-precutover} +[[ $mode == precutover || $mode == postcutover ]] || { + echo 'usage: verify-cluster.sh [precutover|postcutover]' >&2 + exit 64 +} +load_cluster_env + +hosts=("$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST" "$CLUSTER_LOS_ANGELES_HOST") +wireguard_ips=(10.77.0.1 10.77.0.2 10.77.0.3) +for source_index in 0 1 2; do + handshakes=$(cluster_ssh "${hosts[$source_index]}" \ + "wg show wg0 latest-handshakes | awk '\$2 > 0 && systime() - \$2 < 180 { count++ } END { print count + 0 }'") + [[ $handshakes -eq 2 ]] || { + echo "stale WireGuard handshake on ${hosts[$source_index]}" >&2 + exit 1 + } + for target_index in 0 1 2; do + [[ $source_index -eq $target_index ]] && continue + ping_output=$(cluster_ssh "${hosts[$source_index]}" \ + "ping -c 10 -W 2 ${wireguard_ips[$target_index]}") + packet_loss=$(sed -nE 's/.* ([0-9.]+)% packet loss.*/\1/p' <<<"$ping_output") + average_rtt=$(sed -nE 's#.* = [0-9.]+/([0-9.]+)/.*#\1#p' <<<"$ping_output") + awk -v loss="$packet_loss" 'BEGIN { exit !(loss < 1) }' + if [[ $source_index -eq 2 || $target_index -eq 2 ]]; then + awk -v rtt="$average_rtt" 'BEGIN { exit !(rtt < 300) }' + else + awk -v rtt="$average_rtt" 'BEGIN { exit !(rtt < 80) }' + fi + done +done +echo 'wireguard_acceptance=PASS handshakes=6 packet_loss_lt_1=true' + +cluster_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +kubectl='k3s kubectl' +[[ $($kubectl get nodes --no-headers | awk '$2 == "Ready" { count++ } END { print count + 0 }') -eq 3 ]] +$kubectl get --raw='/readyz?verbose' | grep -q '\[+\]etcd ok' +[[ $($kubectl get node easyai-los-angeles -o json | + jq '[.spec.taints[]? | select(.key=="easyai.io/witness" and .effect=="NoSchedule")] | length') -eq 1 ]] +[[ $($kubectl get pods -A -o json | + jq '[.items[] | select(.spec.nodeName=="easyai-los-angeles") | + select(.metadata.namespace != "kube-system" and .metadata.namespace != "cnpg-system" and .metadata.namespace != "cert-manager")] | length') -eq 0 ]] +printf 'nodes=%s\n' "$($kubectl get nodes --no-headers | wc -l)" +REMOTE +) +[[ $cluster_state == nodes=3 ]] +echo 'k3s_control_plane=PASS servers=3 witness_business_pods=0' + +if [[ $mode == postcutover ]]; then + database_state=$(cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +kubectl='k3s kubectl' +[[ $($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') -eq 2 ]] +primary=$($kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.currentPrimary}') +[[ $($kubectl get pod "$primary" -n easyai -o jsonpath='{.spec.nodeName}') == easyai-ningbo || + $($kubectl get pod "$primary" -n easyai -o jsonpath='{.spec.nodeName}') == easyai-hongkong ]] +replication=$($kubectl exec -n easyai "$primary" -c postgres -- \ + psql -X -U easyai -d easyai_ai_gateway -At -c \ + "SELECT count(*) FROM pg_stat_replication WHERE sync_state IN ('sync','quorum');") +[[ $replication -eq 1 ]] +[[ $($kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-api \ + --field-selector=status.phase=Running --no-headers | wc -l) -eq 2 ]] +[[ $($kubectl get pods -n easyai -l app.kubernetes.io/name=easyai-web \ + --field-selector=status.phase=Running --no-headers | wc -l) -eq 2 ]] +images=$($kubectl get deployments -n easyai -o json | + jq -r '.items[] | select(.metadata.name | startswith("easyai-")) | + .spec.template.spec.containers[].image') +if grep -Ev '@sha256:[0-9a-f]{64}$' <<<"$images" | grep -q .; then + echo 'one or more application images are not digest-pinned' >&2 + exit 1 +fi +[[ $($kubectl get secret easyai-gateway-security-events -n easyai \ + -o json | jq '.data | length') -eq 4 ]] +[[ $($kubectl auth can-i update secret/easyai-gateway-security-events -n easyai \ + --as=system:serviceaccount:easyai:easyai-ai-gateway) == yes ]] +[[ $($kubectl auth can-i update secret/easyai-ai-gateway-runtime -n easyai \ + --as=system:serviceaccount:easyai:easyai-ai-gateway) == no ]] +printf 'primary=%s\n' "$primary" +REMOTE +) + [[ $database_state == primary=easyai-postgres-* ]] + echo "postgres_acceptance=PASS $database_state synchronous_replicas=1" + + for edge_ip in "${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}"; do + curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \ + https://ai.51easyai.com/api/v1/healthz | grep -q easyai-ai-gateway + curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \ + https://ai.51easyai.com/api/v1/readyz | grep -q '"ok":true' + curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \ + https://ai.51easyai.com/api/v1/openapi.json | grep -q '"/api/v1/chat/completions"' + curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \ + https://ai.51easyai.com/ | grep -q 'EasyAI AI Gateway' + done + echo 'nginx_edges=PASS ningbo=true hongkong=true dns_changed=false' +fi + +echo "cluster_verification=PASS mode=$mode" diff --git a/scripts/cluster/verify-etcd-backup.sh b/scripts/cluster/verify-etcd-backup.sh new file mode 100755 index 0000000..ae4768e --- /dev/null +++ b/scripts/cluster/verify-etcd-backup.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/cluster/common.sh +source "$script_dir/common.sh" +load_cluster_env + +cluster_ssh "$CLUSTER_NINGBO_HOST" 'bash -s' <<'REMOTE' +set -euo pipefail +snapshot_name=acceptance-$(date -u '+%Y%m%dT%H%M%SZ') +k3s etcd-snapshot save --name "$snapshot_name" >/dev/null +snapshot=$(find /var/lib/rancher/k3s/server/db/snapshots -maxdepth 1 -type f \ + -name "$snapshot_name-*" -printf '%T@ %p\n' | sort -rn | awk 'NR == 1 { sub(/^[^ ]+ /, ""); print }') +[[ -n $snapshot && -s $snapshot ]] +etcdutl=$(find /var/lib/rancher/k3s/data -type f -path '*/bin/etcdutl' -print -quit) +[[ -x $etcdutl ]] +working_directory=$(mktemp -d) +trap 'rm -rf -- "$working_directory"' EXIT +case $snapshot in + *.zip) + if command -v unzip >/dev/null 2>&1 && unzip -t "$snapshot" >/dev/null 2>&1; then + unzip -p "$snapshot" >"$working_directory/snapshot.db" + else + gzip -dc "$snapshot" >"$working_directory/snapshot.db" + fi + ;; + *) cp "$snapshot" "$working_directory/snapshot.db" ;; +esac +"$etcdutl" snapshot status "$working_directory/snapshot.db" -w json | + jq -e '.hash != null and .totalKey > 0 and .totalSize > 0' >/dev/null +k3s etcd-snapshot ls --s3 | grep -F "$snapshot_name" >/dev/null +echo "etcd_backup_verification=PASS snapshot=$snapshot_name local=true oss=true" +REMOTE