chore(git): 合并远端主分支最新变更

apps/api/internal/httpapi/async_worker_acceptance_integration_test.go

@@ -54,6 +54,7 @@ func TestAsyncWorkerDynamicConcurrencyAcceptance(t *testing.T) {
 		JWTSecret:                         "test-secret",
 		BillingEngineMode:                 "observe",
 		CORSAllowedOrigin:                 "*",
+		AsyncQueueWorkerEnabled:           true,
 		AsyncWorkerHardLimit:              256,
 		AsyncWorkerRefreshIntervalSeconds: 1,
 	}, db, slog.New(slog.NewTextHandler(io.Discard, nil))))

apps/api/internal/httpapi/server.go

@@ -122,8 +122,11 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
 	server.auth.LocalAPIKeyVerifier = db.VerifyLocalAPIKey
 	if cfg.AsyncQueueWorkerEnabled {
 		server.runner.StartAsyncQueueWorker(ctx)
-	} else if logger != nil {
+	} else {
-		logger.Info("asynchronous queue worker disabled for this process")
+		server.runner.StartAsyncQueueClient(ctx)
+		if logger != nil {
+			logger.Info("asynchronous queue worker disabled for this process")
+		}
 	}
 	server.runner.StartBillingSettlementWorker(ctx)
 	server.runner.StartTaskHistoryWorkers(ctx)

apps/api/internal/runner/queue_client_test.go

@@ -0,0 +1,74 @@
+package runner
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
+)
+
+func TestAsyncQueueClientEnqueuesWithoutExecutionWorker(t *testing.T) {
+	databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL"))
+	if databaseURL == "" {
+		t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run the queue client integration test")
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	db, err := store.Connect(ctx, databaseURL)
+	if err != nil {
+		t.Fatalf("connect store: %v", err)
+	}
+	t.Cleanup(db.Close)
+
+	suffix := strconv.FormatInt(time.Now().UnixNano(), 10)
+	task, err := db.CreateTask(ctx, store.CreateTaskInput{
+		Kind:    "images.edits",
+		Model:   "queue-client-" + suffix,
+		Request: map[string]any{"prompt": "enqueue without execution worker"},
+		Async:   true,
+		RunMode: "simulation",
+	}, &auth.User{ID: "queue-client-" + suffix, Source: "gateway"})
+	if err != nil {
+		t.Fatalf("create task: %v", err)
+	}
+	t.Cleanup(func() {
+		_, _ = db.Pool().Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id = $1::uuid`, task.ID)
+	})
+
+	service := New(config.Config{AppEnv: "test"}, db, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	service.StartAsyncQueueClient(ctx)
+	if err := service.EnqueueAsyncTask(ctx, task); err != nil {
+		t.Fatalf("enqueue task without execution worker: %v", err)
+	}
+
+	queued, err := db.GetTask(ctx, task.ID)
+	if err != nil {
+		t.Fatalf("load queued task: %v", err)
+	}
+	if queued.RiverJobID <= 0 {
+		t.Fatal("queue client did not persist a River job ID")
+	}
+	t.Cleanup(func() {
+		_, _ = db.Pool().Exec(context.Background(), `DELETE FROM river_job WHERE id = $1`, queued.RiverJobID)
+	})
+
+	var attemptedByCount int
+	if err := db.Pool().QueryRow(ctx, `
+SELECT cardinality(attempted_by)
+FROM river_job
+WHERE id = $1`, queued.RiverJobID).Scan(&attemptedByCount); err != nil {
+		t.Fatalf("load River job: %v", err)
+	}
+	if attemptedByCount != 0 {
+		t.Fatalf("control-only queue client executed the job: attempted_by=%d", attemptedByCount)
+	}
+}

apps/api/internal/runner/queue_worker.go

@@ -78,13 +78,20 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs
 }
 
 func (s *Service) StartAsyncQueueWorker(ctx context.Context) {
-	if err := s.startRiverQueue(ctx); err != nil {
+	if err := s.startRiverQueue(ctx, true); err != nil {
 		s.logger.Error("start river async queue failed", "error", err)
 		panic(err)
 	}
 }
 
-func (s *Service) startRiverQueue(ctx context.Context) error {
+func (s *Service) StartAsyncQueueClient(ctx context.Context) {
+	if err := s.startRiverQueue(ctx, false); err != nil {
+		s.logger.Error("start river async queue client failed", "error", err)
+		panic(err)
+	}
+}
+
+func (s *Service) startRiverQueue(ctx context.Context, workerEnabled bool) error {
 	driver := riverpgxv5.New(s.store.Pool())
 	migrator, err := rivermigrate.New(driver, nil)
 	if err != nil {
@@ -102,6 +109,23 @@ func (s *Service) startRiverQueue(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	if !workerEnabled {
+		s.riverMu.Lock()
+		s.riverControlClient = controlClient
+		s.riverExecutionClient = nil
+		s.riverWorkerCapacity = 0
+		s.riverDrainingClients = make(map[asyncExecutionClient]struct{})
+		s.riverMu.Unlock()
+		if err := s.recoverAsyncRiverJobs(ctx); err != nil {
+			s.riverMu.Lock()
+			s.riverControlClient = nil
+			s.riverMu.Unlock()
+			return err
+		}
+		go s.stopAsyncWorkersOnShutdown(ctx)
+		s.logger.Info("river async queue client initialized without execution workers")
+		return nil
+	}
 	snapshot, err := s.loadAsyncWorkerCapacity(ctx)
 	if err != nil {
 		return fmt.Errorf("calculate initial async worker capacity: %w", err)
@@ -271,6 +295,7 @@ func (s *Service) drainAsyncWorkerClient(client asyncExecutionClient) {
 func (s *Service) stopAsyncWorkersOnShutdown(ctx context.Context) {
 	<-ctx.Done()
 	s.riverMu.Lock()
+	s.riverControlClient = nil
 	clients := make([]asyncExecutionClient, 0, 1+len(s.riverDrainingClients))
 	if s.riverExecutionClient != nil {
 		clients = append(clients, s.riverExecutionClient)

deploy/kubernetes/easyai-ai-gateway-cluster-release

@@ -64,14 +64,24 @@ wait_for_url() {
 verify_site() {
   local site=$1
   local address
+  local api_port
+  local web_port
   case $site in
-    hongkong) address=10.77.0.2 ;;
+    hongkong)
-    ningbo) address=10.77.0.1 ;;
+      address=10.77.0.2
+      api_port=31089
+      web_port=31179
+      ;;
+    ningbo)
+      address=10.77.0.1
+      api_port=31088
+      web_port=31178
+      ;;
     *) return 1 ;;
   esac
-  wait_for_url "$site API health" "http://$address:30088/api/v1/healthz" easyai-ai-gateway
+  wait_for_url "$site API health" "http://$address:$api_port/api/v1/healthz" easyai-ai-gateway
-  wait_for_url "$site API readiness" "http://$address:30088/api/v1/readyz" '"ok":true'
+  wait_for_url "$site API readiness" "http://$address:$api_port/api/v1/readyz" '"ok":true'
-  wait_for_url "$site Web" "http://$address:30178/" 'EasyAI AI Gateway'
+  wait_for_url "$site Web" "http://$address:$web_port/" 'EasyAI AI Gateway'
 }
 
 rollout_site() {

deploy/kubernetes/production/application.yaml

@@ -32,6 +32,10 @@ spec:
         easyai.io/workload: "true"
       securityContext:
         runAsNonRoot: true
+        runAsUser: 10001
+        runAsGroup: 10001
+        fsGroup: 10001
+        fsGroupChangePolicy: OnRootMismatch
         seccompProfile:
           type: RuntimeDefault
       containers:
@@ -134,6 +138,10 @@ spec:
         easyai.io/workload: "true"
       securityContext:
         runAsNonRoot: true
+        runAsUser: 10001
+        runAsGroup: 10001
+        fsGroup: 10001
+        fsGroupChangePolicy: OnRootMismatch
         seccompProfile:
           type: RuntimeDefault
       containers:
@@ -418,6 +426,90 @@ spec:
       targetPort: http
       nodePort: 30178
 ---
+apiVersion: v1
+kind: Service
+metadata:
+  name: api-ningbo-edge
+  labels:
+    app.kubernetes.io/name: easyai-api
+    app.kubernetes.io/component: api
+    app.kubernetes.io/part-of: easyai-ai-gateway
+    easyai.io/site: ningbo
+spec:
+  type: NodePort
+  externalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: easyai-api
+    easyai.io/site: ningbo
+  ports:
+    - name: http
+      port: 8088
+      targetPort: http
+      nodePort: 31088
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: api-hongkong-edge
+  labels:
+    app.kubernetes.io/name: easyai-api
+    app.kubernetes.io/component: api
+    app.kubernetes.io/part-of: easyai-ai-gateway
+    easyai.io/site: hongkong
+spec:
+  type: NodePort
+  externalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: easyai-api
+    easyai.io/site: hongkong
+  ports:
+    - name: http
+      port: 8088
+      targetPort: http
+      nodePort: 31089
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: web-ningbo-edge
+  labels:
+    app.kubernetes.io/name: easyai-web
+    app.kubernetes.io/component: web
+    app.kubernetes.io/part-of: easyai-ai-gateway
+    easyai.io/site: ningbo
+spec:
+  type: NodePort
+  externalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: easyai-web
+    easyai.io/site: ningbo
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
+      nodePort: 31178
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: web-hongkong-edge
+  labels:
+    app.kubernetes.io/name: easyai-web
+    app.kubernetes.io/component: web
+    app.kubernetes.io/part-of: easyai-ai-gateway
+    easyai.io/site: hongkong
+spec:
+  type: NodePort
+  externalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: easyai-web
+    easyai.io/site: hongkong
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
+      nodePort: 31179
+---
 apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:

deploy/nginx/ai.51easyai.com-cluster.conf

@@ -1,15 +1,15 @@
 upstream easyai_gateway_api {
   least_conn;
   keepalive 64;
-  server 10.77.0.1:30088 max_fails=2 fail_timeout=5s;
+  server 10.77.0.1:31088 max_fails=2 fail_timeout=5s;
-  server 10.77.0.2:30088 max_fails=2 fail_timeout=5s;
+  server 10.77.0.2:31089 max_fails=2 fail_timeout=5s;
 }
 
 upstream easyai_gateway_web {
   least_conn;
   keepalive 32;
-  server 10.77.0.1:30178 max_fails=2 fail_timeout=5s;
+  server 10.77.0.1:31178 max_fails=2 fail_timeout=5s;
-  server 10.77.0.2:30178 max_fails=2 fail_timeout=5s;
+  server 10.77.0.2:31179 max_fails=2 fail_timeout=5s;
 }
 
 server {

deploy/nginx/cluster-acceptance-origin-ningbo.conf

@@ -1,6 +1,6 @@
 # Private-only deterministic path used by the cross-node acceptance test.
 server {
-  listen 10.77.0.1:18088;
+  listen 10.77.0.1:18089;
   server_name _;
 
   allow 10.77.0.0/24;
@@ -19,6 +19,6 @@ server {
     proxy_read_timeout 3600s;
     proxy_request_buffering off;
     proxy_buffering off;
-    proxy_pass http://10.77.0.1:30088;
+    proxy_pass http://10.77.0.1:31088;
   }
 }

scripts/cluster/bootstrap-platform.sh

@@ -57,15 +57,17 @@ operator_checksums=(
   e0c5ff41bf5b01c0775bf225929b8423d543cd23e9742e4274fe7de8499cf93a
   d2e71e7b06822448f1a421f05781846cfdb9cc621e7ef32eef5e20c5133213b0
 )
-for index in 0 1 2; do
+if [[ $mode == prepare ]]; then
-  operator=${operators[$index]}
+  for index in 0 1 2; do
-  curl -fsSL "${operator_urls[$index]}" -o "$working_directory/$operator"
+    operator=${operators[$index]}
-  actual_checksum=$(shasum -a 256 "$working_directory/$operator" | awk '{print $1}')
+    curl -fsSL "${operator_urls[$index]}" -o "$working_directory/$operator"
-  [[ $actual_checksum == "${operator_checksums[$index]}" ]] || {
+    actual_checksum=$(shasum -a 256 "$working_directory/$operator" | awk '{print $1}')
-    echo "operator manifest checksum mismatch: $operator" >&2
+    [[ $actual_checksum == "${operator_checksums[$index]}" ]] || {
-    exit 1
+      echo "operator manifest checksum mismatch: $operator" >&2
-  }
+      exit 1
-done
+    }
+  done
+fi
 
 cp -R "$cluster_root/deploy/kubernetes/production" "$working_directory/production"
 api_digest=${api_image##*@}
@@ -126,7 +128,11 @@ stringData:
 EOF
 chmod 0600 "$working_directory/bootstrap-secrets.yaml"
 
-for file in cert-manager.yaml cnpg.yaml barman.yaml easyai-production.yaml bootstrap-secrets.yaml; do
+platform_files=(easyai-production.yaml bootstrap-secrets.yaml)
+if [[ $mode == prepare ]]; then
+  platform_files=(cert-manager.yaml cnpg.yaml barman.yaml "${platform_files[@]}")
+fi
+for file in "${platform_files[@]}"; do
   cluster_scp "$working_directory/$file" "$CLUSTER_NINGBO_HOST:/root/$file"
 done
 
@@ -151,14 +157,16 @@ cleanup_bootstrap_files() {
 }
 trap cleanup_bootstrap_files EXIT HUP INT TERM
 
-$kubectl apply --server-side --force-conflicts -f /root/cert-manager.yaml >/dev/null
+if [[ $mode == prepare ]]; then
-$kubectl rollout status deployment/cert-manager -n cert-manager --timeout=180s
+  $kubectl apply --server-side --force-conflicts -f /root/cert-manager.yaml >/dev/null
-$kubectl rollout status deployment/cert-manager-webhook -n cert-manager --timeout=180s
+  $kubectl rollout status deployment/cert-manager -n cert-manager --timeout=180s
-$kubectl rollout status deployment/cert-manager-cainjector -n cert-manager --timeout=180s
+  $kubectl rollout status deployment/cert-manager-webhook -n cert-manager --timeout=180s
-$kubectl apply --server-side --force-conflicts -f /root/cnpg.yaml >/dev/null
+  $kubectl rollout status deployment/cert-manager-cainjector -n cert-manager --timeout=180s
-$kubectl rollout status deployment/cnpg-controller-manager -n cnpg-system --timeout=600s
+  $kubectl apply --server-side --force-conflicts -f /root/cnpg.yaml >/dev/null
-$kubectl apply --server-side --force-conflicts -f /root/barman.yaml >/dev/null
+  $kubectl rollout status deployment/cnpg-controller-manager -n cnpg-system --timeout=600s
-$kubectl rollout status deployment/barman-cloud -n cnpg-system --timeout=600s
+  $kubectl apply --server-side --force-conflicts -f /root/barman.yaml >/dev/null
+  $kubectl rollout status deployment/barman-cloud -n cnpg-system --timeout=600s
+fi
 
 $kubectl create namespace easyai --dry-run=client -o yaml | $kubectl apply -f - >/dev/null
 $kubectl apply -f /root/bootstrap-secrets.yaml >/dev/null

scripts/cluster/cross-node-file-e2e.mjs

@@ -51,11 +51,26 @@ function createFixturePNG(width = 256, height = 256) {
   ])
 }
 
-const baseURL = process.env.AI_GATEWAY_E2E_BASE_URL || 'http://10.77.0.1:18088'
+const baseURL = process.env.AI_GATEWAY_E2E_BASE_URL || 'http://10.77.0.1:18089'
 const account = Buffer.from(process.env.E2E_ACCOUNT_B64 || '', 'base64').toString('utf8')
 const password = Buffer.from(process.env.E2E_PASSWORD_B64 || '', 'base64').toString('utf8')
 const model = process.env.AI_GATEWAY_E2E_IMAGE_MODEL || 'openai:gpt-image-1'
+const imageSize = process.env.AI_GATEWAY_E2E_IMAGE_SIZE || '2048x2048'
 assert(account && password, 'E2E login credentials are unavailable')
+assert(/^\d+x\d+$/.test(imageSize), 'E2E image size is invalid')
+const [imageWidth, imageHeight] = imageSize.split('x').map(Number)
+const imageResolution =
+  process.env.AI_GATEWAY_E2E_IMAGE_RESOLUTION ||
+  (imageWidth === imageHeight && imageWidth % 1024 === 0 ? `${imageWidth / 1024}K` : '')
+const imageAspectRatio =
+  process.env.AI_GATEWAY_E2E_IMAGE_ASPECT_RATIO ||
+  `${imageWidth / greatestCommonDivisor(imageWidth, imageHeight)}:${imageHeight / greatestCommonDivisor(imageWidth, imageHeight)}`
+assert(imageResolution, 'E2E image resolution is unavailable')
+
+function greatestCommonDivisor(left, right) {
+  while (right !== 0) [left, right] = [right, left % right]
+  return left
+}
 
 async function request(path, { method = 'GET', token, body, form } = {}) {
   const headers = { Host: 'ai.51easyai.com' }
@@ -95,6 +110,13 @@ const login = await request('/api/v1/auth/login', {
 const jwt = login.payload.accessToken
 assert(typeof jwt === 'string' && jwt.length > 20, 'login did not return an access token')
 
+const existingKeys = await request('/api/v1/api-keys', { token: jwt })
+for (const item of existingKeys.payload.items || []) {
+  if (typeof item?.id === 'string' && item?.name?.startsWith('cluster-cross-node-')) {
+    await request(`/api/v1/api-keys/${item.id}`, { method: 'DELETE', token: jwt })
+  }
+}
+
 const keyName = `cluster-cross-node-${Date.now()}`
 const keyResult = await request('/api/v1/api-keys', {
   method: 'POST',
@@ -102,18 +124,61 @@ const keyResult = await request('/api/v1/api-keys', {
   body: { name: keyName },
 })
 const apiKey = keyResult.payload.secret
-const apiKeyId = keyResult.payload.id || keyResult.payload.item?.id
+const apiKeyId = keyResult.payload.apiKey?.id
 assert(typeof apiKey === 'string' && apiKey.startsWith('sk-gw-'), 'API key creation failed')
+assert(typeof apiKeyId === 'string' && apiKeyId.length > 0, 'API key ID is unavailable')
 
 let result
 try {
+  const assignableModels = await request('/api/v1/api-keys/assignable-models', { token: jwt })
+  const targetModels = (assignableModels.payload.items || []).filter(
+    (item) =>
+      [item?.modelName, item?.modelAlias].includes(model) &&
+      Array.isArray(item?.modelType) &&
+      item.modelType.includes('image_edit'),
+  )
+  assert(targetModels.length > 0, `no assignable image_edit platform model found for ${model}`)
+  const resourceMap = new Map()
+  for (const item of targetModels) {
+    for (const [resourceType, resourceId] of [
+      ['platform', item.platformId],
+      ['platform_model', item.id],
+      ['base_model', item.baseModelId],
+    ]) {
+      if (typeof resourceId === 'string' && resourceId) {
+        resourceMap.set(`${resourceType}:${resourceId}`, {
+          resourceType,
+          resourceId,
+          priority: 100,
+          minPermissionLevel: 0,
+          status: 'active',
+        })
+      }
+    }
+  }
+  await request('/api/v1/api-keys/access-rules/batch', {
+    method: 'POST',
+    token: jwt,
+    body: {
+      subjectType: 'api_key',
+      subjectId: apiKeyId,
+      effect: 'allow',
+      upsertResources: [...resourceMap.values()],
+      deleteResources: [],
+    },
+  })
+
   const fixture = createFixturePNG()
   const sourceSHA256 = createHash('sha256').update(fixture).digest('hex')
   const form = new FormData()
   form.append('model', model)
   form.append('prompt', `Cross-node HA acceptance ${randomUUID()}: add a thin white border`)
-  form.append('size', '1024x1024')
+  form.append('size', imageSize)
-  form.append('image', new Blob([fixture], { type: 'image/png' }), 'cluster-e2e.png')
+  form.append('width', String(imageWidth))
+  form.append('height', String(imageHeight))
+  form.append('resolution', imageResolution)
+  form.append('aspect_ratio', imageAspectRatio)
+  form.append('images', new Blob([fixture], { type: 'image/png' }), 'cluster-e2e.png')
   const accepted = await request('/api/v1/images/edits', {
     method: 'POST',
     token: apiKey,

scripts/cluster/failure-drill.sh

@@ -51,10 +51,16 @@ case $drill in
     cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$new_primary" <<'REMOTE'
 set -euo pipefail
 primary=$1
+cleanup_probe() {
+  k3s kubectl exec -n easyai "$primary" -c postgres -- \
+    psql -X -U postgres -d easyai_ai_gateway -c \
+    "DELETE FROM system_settings WHERE setting_key='ha_acceptance_probe';" >/dev/null 2>&1 || true
+}
+trap cleanup_probe EXIT HUP INT TERM
 k3s kubectl exec -n easyai "$primary" -c postgres -- \
   psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -c \
   "INSERT INTO system_settings(setting_key,value) VALUES('ha_acceptance_probe',jsonb_build_object('at',now())) ON CONFLICT(setting_key) DO UPDATE SET value=excluded.value,updated_at=now();" >/dev/null
-for _ in $(seq 1 180); do
+for _ in $(seq 1 900); do
   if [[ $(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') == 2 ]]; then
     replication=$(k3s kubectl exec -n easyai "$primary" -c postgres -- \
       psql -X -U postgres -d easyai_ai_gateway -At -c \
@@ -67,6 +73,7 @@ done
 k3s kubectl exec -n easyai "$primary" -c postgres -- \
   psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -c \
   "DELETE FROM system_settings WHERE setting_key='ha_acceptance_probe';" >/dev/null
+trap - EXIT HUP INT TERM
 REMOTE
     echo "database_failure_drill=PASS promotion_seconds=$elapsed old_primary=$old_primary new_primary=$new_primary"
     ;;

scripts/cluster/install-nginx-edges.sh

@@ -81,11 +81,10 @@ if [[ $mode == activate ]]; then
   for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"; do
     cluster_ssh "$host" 'bash -s' <<'REMOTE'
 set -euo pipefail
-ln -sfn /etc/nginx/sites-available/ai.51easyai.com-cluster \
+install -m 0644 /etc/nginx/sites-available/ai.51easyai.com-cluster \
+  /etc/nginx/conf.d/easyai-ai-gateway-ai.51easyai.com.conf
+rm -f /etc/nginx/sites-enabled/ai.51easyai.com \
   /etc/nginx/sites-enabled/ai.51easyai.com-cluster
-if [[ -L /etc/nginx/sites-enabled/ai.51easyai.com ]]; then
-  rm -f /etc/nginx/sites-enabled/ai.51easyai.com
-fi
 nginx -t
 systemctl reload nginx
 REMOTE

scripts/cluster/migrate-production.sh

@@ -101,7 +101,7 @@ set -euo pipefail
 
 source_sha=$1
 api_image=$2
-active_site=${AI_GATEWAY_NGINX_ACTIVE_SITE:-/etc/nginx/sites-enabled/ai.51easyai.com}
+active_site=${AI_GATEWAY_NGINX_ACTIVE_SITE:-/etc/nginx/conf.d/easyai-ai-gateway-ai.51easyai.com.conf}
 timestamp=$(date -u '+%Y%m%dT%H%M%SZ')
 backup_directory=/var/backups/easyai-ai-gateway/pre-cutover-"$timestamp"
 old_site_backup=$backup_directory/nginx-site.conf
@@ -118,9 +118,7 @@ rollback_before_commit() {
       --replicas=0 >/dev/null 2>&1 || true
     docker start easyai-ai-gateway-api-1 easyai-ai-gateway-web-1 >/dev/null 2>&1 || true
     if [[ -f $old_site_backup ]]; then
-      install -m 0644 "$old_site_backup" /etc/nginx/sites-available/ai.51easyai.com-pre-ha
+      install -m 0644 "$old_site_backup" "$active_site"
-      rm -f -- "$active_site"
-      ln -s /etc/nginx/sites-available/ai.51easyai.com-pre-ha "$active_site"
       nginx -t && systemctl reload nginx
     fi
   else
@@ -137,13 +135,20 @@ install -d -m 0700 "$backup_directory"
 }
 cp -L "$active_site" "$old_site_backup"
 chmod 0600 "$old_site_backup"
-rm -f -- "$active_site"
+install -m 0644 /etc/nginx/sites-available/ai.51easyai.com-maintenance "$active_site"
-ln -s /etc/nginx/sites-available/ai.51easyai.com-maintenance "$active_site"
 nginx -t
 systemctl reload nginx
-curl -ksS --resolve ai.51easyai.com:443:127.0.0.1 \
+maintenance_code=
-  -o /dev/null -w '%{http_code}' https://ai.51easyai.com/api/v1/healthz |
+for _ in $(seq 1 15); do
-  grep -qx 503
+  maintenance_code=$(curl -ksS --resolve ai.51easyai.com:443:127.0.0.1 \
+    -o /dev/null -w '%{http_code}' https://ai.51easyai.com/api/v1/healthz || true)
+  [[ $maintenance_code == 503 ]] && break
+  sleep 1
+done
+[[ $maintenance_code == 503 ]] || {
+  echo "maintenance NGINX did not become active: HTTP $maintenance_code" >&2
+  exit 1
+}
 
 docker stop --time 60 easyai-ai-gateway-api-1 >/dev/null
 postgres_user=$(docker inspect easyai-ai-gateway-postgres-1 \
@@ -159,7 +164,7 @@ snapshot_sql=$backup_directory/acceptance-snapshot.sql
 cat >"$snapshot_sql" <<'SQL'
 \pset tuples_only on
 \pset format unaligned
-SELECT 'server_version=' || current_setting('server_version');
+SELECT 'server_version_num=' || current_setting('server_version_num');
 SELECT 'extensions=' || COALESCE(string_agg(extname || ':' || extversion, ',' ORDER BY extname), '')
   FROM pg_extension WHERE extname <> 'plpgsql';
 SELECT 'tables=' || count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
@@ -187,7 +192,7 @@ SELECT 'gateway_wallet_transactions=' || count(*) FROM gateway_wallet_transactio
 SELECT 'gateway_upload_assets=' || count(*) FROM gateway_upload_assets;
 SELECT 'gateway_users=' || count(*) FROM gateway_users;
 SELECT 'gateway_api_keys=' || count(*) FROM gateway_api_keys;
-SELECT 'gateway_providers=' || count(*) FROM gateway_providers;
+SELECT 'model_catalog_providers=' || count(*) FROM model_catalog_providers;
 SELECT 'task_sample=' || md5(COALESCE(string_agg(id::text, ',' ORDER BY id), ''))
   FROM (SELECT id FROM gateway_tasks ORDER BY id LIMIT 100) sample;
 SQL
@@ -310,11 +315,8 @@ k3s kubectl scale deployment -n easyai \
   easyai-api-ningbo easyai-api-hongkong easyai-web-ningbo easyai-web-hongkong \
   --replicas=0 >/dev/null 2>&1 || true
 docker start easyai-ai-gateway-api-1 easyai-ai-gateway-web-1 >/dev/null
-rm -f -- /etc/nginx/sites-enabled/ai.51easyai.com
 install -m 0644 "$backup_directory/nginx-site.conf" \
-  /etc/nginx/sites-available/ai.51easyai.com-pre-ha
+  /etc/nginx/conf.d/easyai-ai-gateway-ai.51easyai.com.conf
-ln -s /etc/nginx/sites-available/ai.51easyai.com-pre-ha \
-  /etc/nginx/sites-enabled/ai.51easyai.com
 nginx -t
 systemctl reload nginx
 REMOTE
@@ -327,13 +329,14 @@ echo '[cutover] starting both application sites with River workers frozen'
 AI_GATEWAY_ACTIVATE_WORKERS=false \
   "$script_dir/bootstrap-platform.sh" activate "$release_manifest"
 
-for site_ip in 10.77.0.1 10.77.0.2; do
+for site_endpoint in 10.77.0.1:31088:31178 10.77.0.2:31089:31179; do
+  IFS=: read -r site_ip api_port web_port <<<"$site_endpoint"
   cluster_ssh "$CLUSTER_NINGBO_HOST" \
-    "curl -fsS --max-time 10 http://$site_ip:30088/api/v1/healthz >/dev/null"
+    "curl -fsS --max-time 10 http://$site_ip:$api_port/api/v1/healthz >/dev/null"
   cluster_ssh "$CLUSTER_NINGBO_HOST" \
-    "curl -fsS --max-time 10 http://$site_ip:30088/api/v1/readyz | grep -q '\"ok\":true'"
+    "curl -fsS --max-time 10 http://$site_ip:$api_port/api/v1/readyz | grep -q '\"ok\":true'"
   cluster_ssh "$CLUSTER_NINGBO_HOST" \
-    "curl -fsS --max-time 10 http://$site_ip:30178/ | grep -q 'EasyAI AI Gateway'"
+    "curl -fsS --max-time 10 http://$site_ip:$web_port/ | grep -q 'EasyAI AI Gateway'"
 done
 
 "$script_dir/install-nginx-edges.sh" activate

scripts/cluster/run-cross-node-file-e2e.sh

@@ -31,19 +31,42 @@ k3s kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=30
 k3s kubectl set env deployment/easyai-api-ningbo -n easyai \
   AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=false >/dev/null
 k3s kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
-curl -fsS http://10.77.0.1:30088/api/v1/readyz | grep -q '"ok":true'
+for _ in $(seq 1 60); do
-curl -fsS http://10.77.0.2:30088/api/v1/readyz | grep -q '"ok":true'
+  pods=$(k3s kubectl get pods -n easyai \
+    -l app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo \
+    -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
+  pod_count=$(wc -w <<<"$pods" | tr -d ' ')
+  if [[ $pod_count == 1 ]]; then
+    pod=$pods
+    worker_enabled=$(k3s kubectl get pod "$pod" -n easyai \
+      -o jsonpath='{.spec.containers[0].env[?(@.name=="AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED")].value}')
+    pod_ready=$(k3s kubectl get pod "$pod" -n easyai \
+      -o jsonpath='{.status.containerStatuses[0].ready}')
+    if [[ $worker_enabled == false && $pod_ready == true ]]; then
+      break
+    fi
+  fi
+  sleep 1
+done
+[[ ${pod_count:-0} == 1 && ${worker_enabled:-} == false && ${pod_ready:-} == true ]]
+k3s kubectl logs "$pod" -n easyai | grep -q 'asynchronous queue worker disabled for this process'
+curl -fsS http://10.77.0.1:31088/api/v1/readyz | grep -q '"ok":true'
+curl -fsS http://10.77.0.2:31089/api/v1/readyz | grep -q '"ok":true'
 REMOTE
 
 credentials_file=$(mktemp "$cluster_root/.local-secrets/cluster/e2e.XXXXXX")
 trap 'rm -f -- "$credentials_file"; restore_workers' EXIT HUP INT TERM
 chmod 0600 "$credentials_file"
-printf 'E2E_ACCOUNT_B64=%s\n' "$(printf '%s' "$AI_GATEWAY_ONLINE_ACCOUNT" | base64 | tr -d '\n')" \
+{
-  >"$credentials_file"
+  printf 'E2E_ACCOUNT_B64=%s\n' \
-printf 'E2E_PASSWORD_B64=%s\n' "$(printf '%s' "$AI_GATEWAY_ONLINE_PASSWORD" | base64 | tr -d '\n')" \
+    "$(printf '%s' "$AI_GATEWAY_ONLINE_ACCOUNT" | base64 | tr -d '\n')"
-  >>"$credentials_file"
+  printf 'E2E_PASSWORD_B64=%s\n' \
-printf 'AI_GATEWAY_E2E_IMAGE_MODEL=%s\n' "${AI_GATEWAY_E2E_IMAGE_MODEL:-openai:gpt-image-1}" \
+    "$(printf '%s' "$AI_GATEWAY_ONLINE_PASSWORD" | base64 | tr -d '\n')"
-  >>"$credentials_file"
+  printf 'AI_GATEWAY_E2E_IMAGE_MODEL=%s\n' \
+    "${AI_GATEWAY_E2E_IMAGE_MODEL:-openai:gpt-image-1}"
+  printf 'AI_GATEWAY_E2E_IMAGE_SIZE=%s\n' \
+    "${AI_GATEWAY_E2E_IMAGE_SIZE:-2048x2048}"
+} >"$credentials_file"
 cluster_scp "$credentials_file" "$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-e2e.env"
 cluster_scp "$script_dir/cross-node-file-e2e.mjs" \
   "$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-file-e2e.mjs"

scripts/cluster/verify-cluster.sh

@@ -115,14 +115,18 @@ REMOTE
   echo 'application_acceptance=PASS api=2 web=2 digests=pinned'
 
   for edge_ip in "${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}"; do
-    curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
+    health_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
-      https://ai.51easyai.com/api/v1/healthz | grep -q easyai-ai-gateway
+      https://ai.51easyai.com/api/v1/healthz)
-    curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
+    grep -q easyai-ai-gateway <<<"$health_body"
-      https://ai.51easyai.com/api/v1/readyz | grep -q '"ok":true'
+    ready_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
-    curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
+      https://ai.51easyai.com/api/v1/readyz)
-      https://ai.51easyai.com/api/v1/openapi.json | grep -q '"/api/v1/chat/completions"'
+    grep -q '"ok":true' <<<"$ready_body"
-    curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
+    openapi_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
-      https://ai.51easyai.com/ | grep -q 'EasyAI AI Gateway'
+      https://ai.51easyai.com/api/v1/openapi.json)
+    grep -q '"/api/v1/chat/completions"' <<<"$openapi_body"
+    web_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \
+      https://ai.51easyai.com/)
+    grep -q 'EasyAI AI Gateway' <<<"$web_body"
   done
   echo 'nginx_edges=PASS ningbo=true hongkong=true dns_changed=false'
 fi