diff --git a/apps/api/internal/httpapi/async_worker_acceptance_integration_test.go b/apps/api/internal/httpapi/async_worker_acceptance_integration_test.go index 3210ab7..38684b1 100644 --- a/apps/api/internal/httpapi/async_worker_acceptance_integration_test.go +++ b/apps/api/internal/httpapi/async_worker_acceptance_integration_test.go @@ -54,6 +54,7 @@ func TestAsyncWorkerDynamicConcurrencyAcceptance(t *testing.T) { JWTSecret: "test-secret", BillingEngineMode: "observe", CORSAllowedOrigin: "*", + AsyncQueueWorkerEnabled: true, AsyncWorkerHardLimit: 256, AsyncWorkerRefreshIntervalSeconds: 1, }, db, slog.New(slog.NewTextHandler(io.Discard, nil)))) diff --git a/apps/api/internal/httpapi/server.go b/apps/api/internal/httpapi/server.go index ab886c3..7835d0c 100644 --- a/apps/api/internal/httpapi/server.go +++ b/apps/api/internal/httpapi/server.go @@ -122,8 +122,11 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor server.auth.LocalAPIKeyVerifier = db.VerifyLocalAPIKey if cfg.AsyncQueueWorkerEnabled { server.runner.StartAsyncQueueWorker(ctx) - } else if logger != nil { - logger.Info("asynchronous queue worker disabled for this process") + } else { + server.runner.StartAsyncQueueClient(ctx) + if logger != nil { + logger.Info("asynchronous queue worker disabled for this process") + } } server.runner.StartBillingSettlementWorker(ctx) server.runner.StartTaskHistoryWorkers(ctx) diff --git a/apps/api/internal/runner/queue_client_test.go b/apps/api/internal/runner/queue_client_test.go new file mode 100644 index 0000000..9820f79 --- /dev/null +++ b/apps/api/internal/runner/queue_client_test.go @@ -0,0 +1,74 @@ +package runner + +import ( + "context" + "io" + "log/slog" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/config" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +func TestAsyncQueueClientEnqueuesWithoutExecutionWorker(t *testing.T) { + databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run the queue client integration test") + } + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + db, err := store.Connect(ctx, databaseURL) + if err != nil { + t.Fatalf("connect store: %v", err) + } + t.Cleanup(db.Close) + + suffix := strconv.FormatInt(time.Now().UnixNano(), 10) + task, err := db.CreateTask(ctx, store.CreateTaskInput{ + Kind: "images.edits", + Model: "queue-client-" + suffix, + Request: map[string]any{"prompt": "enqueue without execution worker"}, + Async: true, + RunMode: "simulation", + }, &auth.User{ID: "queue-client-" + suffix, Source: "gateway"}) + if err != nil { + t.Fatalf("create task: %v", err) + } + t.Cleanup(func() { + _, _ = db.Pool().Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id = $1::uuid`, task.ID) + }) + + service := New(config.Config{AppEnv: "test"}, db, slog.New(slog.NewTextHandler(io.Discard, nil))) + service.StartAsyncQueueClient(ctx) + if err := service.EnqueueAsyncTask(ctx, task); err != nil { + t.Fatalf("enqueue task without execution worker: %v", err) + } + + queued, err := db.GetTask(ctx, task.ID) + if err != nil { + t.Fatalf("load queued task: %v", err) + } + if queued.RiverJobID <= 0 { + t.Fatal("queue client did not persist a River job ID") + } + t.Cleanup(func() { + _, _ = db.Pool().Exec(context.Background(), `DELETE FROM river_job WHERE id = $1`, queued.RiverJobID) + }) + + var attemptedByCount int + if err := db.Pool().QueryRow(ctx, ` +SELECT cardinality(attempted_by) +FROM river_job +WHERE id = $1`, queued.RiverJobID).Scan(&attemptedByCount); err != nil { + t.Fatalf("load River job: %v", err) + } + if attemptedByCount != 0 { + t.Fatalf("control-only queue client executed the job: attempted_by=%d", attemptedByCount) + } +} diff --git a/apps/api/internal/runner/queue_worker.go b/apps/api/internal/runner/queue_worker.go index 72e0b9b..3de7bfa 100644 --- a/apps/api/internal/runner/queue_worker.go +++ b/apps/api/internal/runner/queue_worker.go @@ -78,13 +78,20 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs } func (s *Service) StartAsyncQueueWorker(ctx context.Context) { - if err := s.startRiverQueue(ctx); err != nil { + if err := s.startRiverQueue(ctx, true); err != nil { s.logger.Error("start river async queue failed", "error", err) panic(err) } } -func (s *Service) startRiverQueue(ctx context.Context) error { +func (s *Service) StartAsyncQueueClient(ctx context.Context) { + if err := s.startRiverQueue(ctx, false); err != nil { + s.logger.Error("start river async queue client failed", "error", err) + panic(err) + } +} + +func (s *Service) startRiverQueue(ctx context.Context, workerEnabled bool) error { driver := riverpgxv5.New(s.store.Pool()) migrator, err := rivermigrate.New(driver, nil) if err != nil { @@ -102,6 +109,23 @@ func (s *Service) startRiverQueue(ctx context.Context) error { if err != nil { return err } + if !workerEnabled { + s.riverMu.Lock() + s.riverControlClient = controlClient + s.riverExecutionClient = nil + s.riverWorkerCapacity = 0 + s.riverDrainingClients = make(map[asyncExecutionClient]struct{}) + s.riverMu.Unlock() + if err := s.recoverAsyncRiverJobs(ctx); err != nil { + s.riverMu.Lock() + s.riverControlClient = nil + s.riverMu.Unlock() + return err + } + go s.stopAsyncWorkersOnShutdown(ctx) + s.logger.Info("river async queue client initialized without execution workers") + return nil + } snapshot, err := s.loadAsyncWorkerCapacity(ctx) if err != nil { return fmt.Errorf("calculate initial async worker capacity: %w", err) @@ -271,6 +295,7 @@ func (s *Service) drainAsyncWorkerClient(client asyncExecutionClient) { func (s *Service) stopAsyncWorkersOnShutdown(ctx context.Context) { <-ctx.Done() s.riverMu.Lock() + s.riverControlClient = nil clients := make([]asyncExecutionClient, 0, 1+len(s.riverDrainingClients)) if s.riverExecutionClient != nil { clients = append(clients, s.riverExecutionClient) diff --git a/deploy/kubernetes/easyai-ai-gateway-cluster-release b/deploy/kubernetes/easyai-ai-gateway-cluster-release index 40029e9..d262e6e 100755 --- a/deploy/kubernetes/easyai-ai-gateway-cluster-release +++ b/deploy/kubernetes/easyai-ai-gateway-cluster-release @@ -64,14 +64,24 @@ wait_for_url() { verify_site() { local site=$1 local address + local api_port + local web_port case $site in - hongkong) address=10.77.0.2 ;; - ningbo) address=10.77.0.1 ;; + hongkong) + address=10.77.0.2 + api_port=31089 + web_port=31179 + ;; + ningbo) + address=10.77.0.1 + api_port=31088 + web_port=31178 + ;; *) return 1 ;; esac - wait_for_url "$site API health" "http://$address:30088/api/v1/healthz" easyai-ai-gateway - wait_for_url "$site API readiness" "http://$address:30088/api/v1/readyz" '"ok":true' - wait_for_url "$site Web" "http://$address:30178/" 'EasyAI AI Gateway' + wait_for_url "$site API health" "http://$address:$api_port/api/v1/healthz" easyai-ai-gateway + wait_for_url "$site API readiness" "http://$address:$api_port/api/v1/readyz" '"ok":true' + wait_for_url "$site Web" "http://$address:$web_port/" 'EasyAI AI Gateway' } rollout_site() { diff --git a/deploy/kubernetes/production/application.yaml b/deploy/kubernetes/production/application.yaml index f7481f7..6072027 100644 --- a/deploy/kubernetes/production/application.yaml +++ b/deploy/kubernetes/production/application.yaml @@ -32,6 +32,10 @@ spec: easyai.io/workload: "true" securityContext: runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + fsGroupChangePolicy: OnRootMismatch seccompProfile: type: RuntimeDefault containers: @@ -134,6 +138,10 @@ spec: easyai.io/workload: "true" securityContext: runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + fsGroupChangePolicy: OnRootMismatch seccompProfile: type: RuntimeDefault containers: @@ -418,6 +426,90 @@ spec: targetPort: http nodePort: 30178 --- +apiVersion: v1 +kind: Service +metadata: + name: api-ningbo-edge + labels: + app.kubernetes.io/name: easyai-api + app.kubernetes.io/component: api + app.kubernetes.io/part-of: easyai-ai-gateway + easyai.io/site: ningbo +spec: + type: NodePort + externalTrafficPolicy: Local + selector: + app.kubernetes.io/name: easyai-api + easyai.io/site: ningbo + ports: + - name: http + port: 8088 + targetPort: http + nodePort: 31088 +--- +apiVersion: v1 +kind: Service +metadata: + name: api-hongkong-edge + labels: + app.kubernetes.io/name: easyai-api + app.kubernetes.io/component: api + app.kubernetes.io/part-of: easyai-ai-gateway + easyai.io/site: hongkong +spec: + type: NodePort + externalTrafficPolicy: Local + selector: + app.kubernetes.io/name: easyai-api + easyai.io/site: hongkong + ports: + - name: http + port: 8088 + targetPort: http + nodePort: 31089 +--- +apiVersion: v1 +kind: Service +metadata: + name: web-ningbo-edge + labels: + app.kubernetes.io/name: easyai-web + app.kubernetes.io/component: web + app.kubernetes.io/part-of: easyai-ai-gateway + easyai.io/site: ningbo +spec: + type: NodePort + externalTrafficPolicy: Local + selector: + app.kubernetes.io/name: easyai-web + easyai.io/site: ningbo + ports: + - name: http + port: 80 + targetPort: http + nodePort: 31178 +--- +apiVersion: v1 +kind: Service +metadata: + name: web-hongkong-edge + labels: + app.kubernetes.io/name: easyai-web + app.kubernetes.io/component: web + app.kubernetes.io/part-of: easyai-ai-gateway + easyai.io/site: hongkong +spec: + type: NodePort + externalTrafficPolicy: Local + selector: + app.kubernetes.io/name: easyai-web + easyai.io/site: hongkong + ports: + - name: http + port: 80 + targetPort: http + nodePort: 31179 +--- apiVersion: policy/v1 kind: PodDisruptionBudget metadata: diff --git a/deploy/nginx/ai.51easyai.com-cluster.conf b/deploy/nginx/ai.51easyai.com-cluster.conf index 468a200..d3f31e1 100644 --- a/deploy/nginx/ai.51easyai.com-cluster.conf +++ b/deploy/nginx/ai.51easyai.com-cluster.conf @@ -1,15 +1,15 @@ upstream easyai_gateway_api { least_conn; keepalive 64; - server 10.77.0.1:30088 max_fails=2 fail_timeout=5s; - server 10.77.0.2:30088 max_fails=2 fail_timeout=5s; + server 10.77.0.1:31088 max_fails=2 fail_timeout=5s; + server 10.77.0.2:31089 max_fails=2 fail_timeout=5s; } upstream easyai_gateway_web { least_conn; keepalive 32; - server 10.77.0.1:30178 max_fails=2 fail_timeout=5s; - server 10.77.0.2:30178 max_fails=2 fail_timeout=5s; + server 10.77.0.1:31178 max_fails=2 fail_timeout=5s; + server 10.77.0.2:31179 max_fails=2 fail_timeout=5s; } server { diff --git a/deploy/nginx/cluster-acceptance-origin-ningbo.conf b/deploy/nginx/cluster-acceptance-origin-ningbo.conf index e14bf28..f3fc8b8 100644 --- a/deploy/nginx/cluster-acceptance-origin-ningbo.conf +++ b/deploy/nginx/cluster-acceptance-origin-ningbo.conf @@ -1,6 +1,6 @@ # Private-only deterministic path used by the cross-node acceptance test. server { - listen 10.77.0.1:18088; + listen 10.77.0.1:18089; server_name _; allow 10.77.0.0/24; @@ -19,6 +19,6 @@ server { proxy_read_timeout 3600s; proxy_request_buffering off; proxy_buffering off; - proxy_pass http://10.77.0.1:30088; + proxy_pass http://10.77.0.1:31088; } } diff --git a/scripts/cluster/bootstrap-platform.sh b/scripts/cluster/bootstrap-platform.sh index 74542b5..4e8d65f 100755 --- a/scripts/cluster/bootstrap-platform.sh +++ b/scripts/cluster/bootstrap-platform.sh @@ -57,15 +57,17 @@ operator_checksums=( e0c5ff41bf5b01c0775bf225929b8423d543cd23e9742e4274fe7de8499cf93a d2e71e7b06822448f1a421f05781846cfdb9cc621e7ef32eef5e20c5133213b0 ) -for index in 0 1 2; do - operator=${operators[$index]} - curl -fsSL "${operator_urls[$index]}" -o "$working_directory/$operator" - actual_checksum=$(shasum -a 256 "$working_directory/$operator" | awk '{print $1}') - [[ $actual_checksum == "${operator_checksums[$index]}" ]] || { - echo "operator manifest checksum mismatch: $operator" >&2 - exit 1 - } -done +if [[ $mode == prepare ]]; then + for index in 0 1 2; do + operator=${operators[$index]} + curl -fsSL "${operator_urls[$index]}" -o "$working_directory/$operator" + actual_checksum=$(shasum -a 256 "$working_directory/$operator" | awk '{print $1}') + [[ $actual_checksum == "${operator_checksums[$index]}" ]] || { + echo "operator manifest checksum mismatch: $operator" >&2 + exit 1 + } + done +fi cp -R "$cluster_root/deploy/kubernetes/production" "$working_directory/production" api_digest=${api_image##*@} @@ -126,7 +128,11 @@ stringData: EOF chmod 0600 "$working_directory/bootstrap-secrets.yaml" -for file in cert-manager.yaml cnpg.yaml barman.yaml easyai-production.yaml bootstrap-secrets.yaml; do +platform_files=(easyai-production.yaml bootstrap-secrets.yaml) +if [[ $mode == prepare ]]; then + platform_files=(cert-manager.yaml cnpg.yaml barman.yaml "${platform_files[@]}") +fi +for file in "${platform_files[@]}"; do cluster_scp "$working_directory/$file" "$CLUSTER_NINGBO_HOST:/root/$file" done @@ -151,14 +157,16 @@ cleanup_bootstrap_files() { } trap cleanup_bootstrap_files EXIT HUP INT TERM -$kubectl apply --server-side --force-conflicts -f /root/cert-manager.yaml >/dev/null -$kubectl rollout status deployment/cert-manager -n cert-manager --timeout=180s -$kubectl rollout status deployment/cert-manager-webhook -n cert-manager --timeout=180s -$kubectl rollout status deployment/cert-manager-cainjector -n cert-manager --timeout=180s -$kubectl apply --server-side --force-conflicts -f /root/cnpg.yaml >/dev/null -$kubectl rollout status deployment/cnpg-controller-manager -n cnpg-system --timeout=600s -$kubectl apply --server-side --force-conflicts -f /root/barman.yaml >/dev/null -$kubectl rollout status deployment/barman-cloud -n cnpg-system --timeout=600s +if [[ $mode == prepare ]]; then + $kubectl apply --server-side --force-conflicts -f /root/cert-manager.yaml >/dev/null + $kubectl rollout status deployment/cert-manager -n cert-manager --timeout=180s + $kubectl rollout status deployment/cert-manager-webhook -n cert-manager --timeout=180s + $kubectl rollout status deployment/cert-manager-cainjector -n cert-manager --timeout=180s + $kubectl apply --server-side --force-conflicts -f /root/cnpg.yaml >/dev/null + $kubectl rollout status deployment/cnpg-controller-manager -n cnpg-system --timeout=600s + $kubectl apply --server-side --force-conflicts -f /root/barman.yaml >/dev/null + $kubectl rollout status deployment/barman-cloud -n cnpg-system --timeout=600s +fi $kubectl create namespace easyai --dry-run=client -o yaml | $kubectl apply -f - >/dev/null $kubectl apply -f /root/bootstrap-secrets.yaml >/dev/null diff --git a/scripts/cluster/cross-node-file-e2e.mjs b/scripts/cluster/cross-node-file-e2e.mjs index 00f4fbc..19930c7 100755 --- a/scripts/cluster/cross-node-file-e2e.mjs +++ b/scripts/cluster/cross-node-file-e2e.mjs @@ -51,11 +51,26 @@ function createFixturePNG(width = 256, height = 256) { ]) } -const baseURL = process.env.AI_GATEWAY_E2E_BASE_URL || 'http://10.77.0.1:18088' +const baseURL = process.env.AI_GATEWAY_E2E_BASE_URL || 'http://10.77.0.1:18089' const account = Buffer.from(process.env.E2E_ACCOUNT_B64 || '', 'base64').toString('utf8') const password = Buffer.from(process.env.E2E_PASSWORD_B64 || '', 'base64').toString('utf8') const model = process.env.AI_GATEWAY_E2E_IMAGE_MODEL || 'openai:gpt-image-1' +const imageSize = process.env.AI_GATEWAY_E2E_IMAGE_SIZE || '2048x2048' assert(account && password, 'E2E login credentials are unavailable') +assert(/^\d+x\d+$/.test(imageSize), 'E2E image size is invalid') +const [imageWidth, imageHeight] = imageSize.split('x').map(Number) +const imageResolution = + process.env.AI_GATEWAY_E2E_IMAGE_RESOLUTION || + (imageWidth === imageHeight && imageWidth % 1024 === 0 ? `${imageWidth / 1024}K` : '') +const imageAspectRatio = + process.env.AI_GATEWAY_E2E_IMAGE_ASPECT_RATIO || + `${imageWidth / greatestCommonDivisor(imageWidth, imageHeight)}:${imageHeight / greatestCommonDivisor(imageWidth, imageHeight)}` +assert(imageResolution, 'E2E image resolution is unavailable') + +function greatestCommonDivisor(left, right) { + while (right !== 0) [left, right] = [right, left % right] + return left +} async function request(path, { method = 'GET', token, body, form } = {}) { const headers = { Host: 'ai.51easyai.com' } @@ -95,6 +110,13 @@ const login = await request('/api/v1/auth/login', { const jwt = login.payload.accessToken assert(typeof jwt === 'string' && jwt.length > 20, 'login did not return an access token') +const existingKeys = await request('/api/v1/api-keys', { token: jwt }) +for (const item of existingKeys.payload.items || []) { + if (typeof item?.id === 'string' && item?.name?.startsWith('cluster-cross-node-')) { + await request(`/api/v1/api-keys/${item.id}`, { method: 'DELETE', token: jwt }) + } +} + const keyName = `cluster-cross-node-${Date.now()}` const keyResult = await request('/api/v1/api-keys', { method: 'POST', @@ -102,18 +124,61 @@ const keyResult = await request('/api/v1/api-keys', { body: { name: keyName }, }) const apiKey = keyResult.payload.secret -const apiKeyId = keyResult.payload.id || keyResult.payload.item?.id +const apiKeyId = keyResult.payload.apiKey?.id assert(typeof apiKey === 'string' && apiKey.startsWith('sk-gw-'), 'API key creation failed') +assert(typeof apiKeyId === 'string' && apiKeyId.length > 0, 'API key ID is unavailable') let result try { + const assignableModels = await request('/api/v1/api-keys/assignable-models', { token: jwt }) + const targetModels = (assignableModels.payload.items || []).filter( + (item) => + [item?.modelName, item?.modelAlias].includes(model) && + Array.isArray(item?.modelType) && + item.modelType.includes('image_edit'), + ) + assert(targetModels.length > 0, `no assignable image_edit platform model found for ${model}`) + const resourceMap = new Map() + for (const item of targetModels) { + for (const [resourceType, resourceId] of [ + ['platform', item.platformId], + ['platform_model', item.id], + ['base_model', item.baseModelId], + ]) { + if (typeof resourceId === 'string' && resourceId) { + resourceMap.set(`${resourceType}:${resourceId}`, { + resourceType, + resourceId, + priority: 100, + minPermissionLevel: 0, + status: 'active', + }) + } + } + } + await request('/api/v1/api-keys/access-rules/batch', { + method: 'POST', + token: jwt, + body: { + subjectType: 'api_key', + subjectId: apiKeyId, + effect: 'allow', + upsertResources: [...resourceMap.values()], + deleteResources: [], + }, + }) + const fixture = createFixturePNG() const sourceSHA256 = createHash('sha256').update(fixture).digest('hex') const form = new FormData() form.append('model', model) form.append('prompt', `Cross-node HA acceptance ${randomUUID()}: add a thin white border`) - form.append('size', '1024x1024') - form.append('image', new Blob([fixture], { type: 'image/png' }), 'cluster-e2e.png') + form.append('size', imageSize) + form.append('width', String(imageWidth)) + form.append('height', String(imageHeight)) + form.append('resolution', imageResolution) + form.append('aspect_ratio', imageAspectRatio) + form.append('images', new Blob([fixture], { type: 'image/png' }), 'cluster-e2e.png') const accepted = await request('/api/v1/images/edits', { method: 'POST', token: apiKey, diff --git a/scripts/cluster/failure-drill.sh b/scripts/cluster/failure-drill.sh index 88bf7e5..edb2a27 100755 --- a/scripts/cluster/failure-drill.sh +++ b/scripts/cluster/failure-drill.sh @@ -51,10 +51,16 @@ case $drill in cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$new_primary" <<'REMOTE' set -euo pipefail primary=$1 +cleanup_probe() { + k3s kubectl exec -n easyai "$primary" -c postgres -- \ + psql -X -U postgres -d easyai_ai_gateway -c \ + "DELETE FROM system_settings WHERE setting_key='ha_acceptance_probe';" >/dev/null 2>&1 || true +} +trap cleanup_probe EXIT HUP INT TERM k3s kubectl exec -n easyai "$primary" -c postgres -- \ psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -c \ "INSERT INTO system_settings(setting_key,value) VALUES('ha_acceptance_probe',jsonb_build_object('at',now())) ON CONFLICT(setting_key) DO UPDATE SET value=excluded.value,updated_at=now();" >/dev/null -for _ in $(seq 1 180); do +for _ in $(seq 1 900); do if [[ $(k3s kubectl get cluster easyai-postgres -n easyai -o jsonpath='{.status.readyInstances}') == 2 ]]; then replication=$(k3s kubectl exec -n easyai "$primary" -c postgres -- \ psql -X -U postgres -d easyai_ai_gateway -At -c \ @@ -67,6 +73,7 @@ done k3s kubectl exec -n easyai "$primary" -c postgres -- \ psql -X -v ON_ERROR_STOP=1 -U postgres -d easyai_ai_gateway -c \ "DELETE FROM system_settings WHERE setting_key='ha_acceptance_probe';" >/dev/null +trap - EXIT HUP INT TERM REMOTE echo "database_failure_drill=PASS promotion_seconds=$elapsed old_primary=$old_primary new_primary=$new_primary" ;; diff --git a/scripts/cluster/install-nginx-edges.sh b/scripts/cluster/install-nginx-edges.sh index 16ba041..6b48dc7 100755 --- a/scripts/cluster/install-nginx-edges.sh +++ b/scripts/cluster/install-nginx-edges.sh @@ -81,11 +81,10 @@ if [[ $mode == activate ]]; then for host in "$CLUSTER_NINGBO_HOST" "$CLUSTER_HONGKONG_HOST"; do cluster_ssh "$host" 'bash -s' <<'REMOTE' set -euo pipefail -ln -sfn /etc/nginx/sites-available/ai.51easyai.com-cluster \ +install -m 0644 /etc/nginx/sites-available/ai.51easyai.com-cluster \ + /etc/nginx/conf.d/easyai-ai-gateway-ai.51easyai.com.conf +rm -f /etc/nginx/sites-enabled/ai.51easyai.com \ /etc/nginx/sites-enabled/ai.51easyai.com-cluster -if [[ -L /etc/nginx/sites-enabled/ai.51easyai.com ]]; then - rm -f /etc/nginx/sites-enabled/ai.51easyai.com -fi nginx -t systemctl reload nginx REMOTE diff --git a/scripts/cluster/migrate-production.sh b/scripts/cluster/migrate-production.sh index 604d985..75abab2 100755 --- a/scripts/cluster/migrate-production.sh +++ b/scripts/cluster/migrate-production.sh @@ -101,7 +101,7 @@ set -euo pipefail source_sha=$1 api_image=$2 -active_site=${AI_GATEWAY_NGINX_ACTIVE_SITE:-/etc/nginx/sites-enabled/ai.51easyai.com} +active_site=${AI_GATEWAY_NGINX_ACTIVE_SITE:-/etc/nginx/conf.d/easyai-ai-gateway-ai.51easyai.com.conf} timestamp=$(date -u '+%Y%m%dT%H%M%SZ') backup_directory=/var/backups/easyai-ai-gateway/pre-cutover-"$timestamp" old_site_backup=$backup_directory/nginx-site.conf @@ -118,9 +118,7 @@ rollback_before_commit() { --replicas=0 >/dev/null 2>&1 || true docker start easyai-ai-gateway-api-1 easyai-ai-gateway-web-1 >/dev/null 2>&1 || true if [[ -f $old_site_backup ]]; then - install -m 0644 "$old_site_backup" /etc/nginx/sites-available/ai.51easyai.com-pre-ha - rm -f -- "$active_site" - ln -s /etc/nginx/sites-available/ai.51easyai.com-pre-ha "$active_site" + install -m 0644 "$old_site_backup" "$active_site" nginx -t && systemctl reload nginx fi else @@ -137,13 +135,20 @@ install -d -m 0700 "$backup_directory" } cp -L "$active_site" "$old_site_backup" chmod 0600 "$old_site_backup" -rm -f -- "$active_site" -ln -s /etc/nginx/sites-available/ai.51easyai.com-maintenance "$active_site" +install -m 0644 /etc/nginx/sites-available/ai.51easyai.com-maintenance "$active_site" nginx -t systemctl reload nginx -curl -ksS --resolve ai.51easyai.com:443:127.0.0.1 \ - -o /dev/null -w '%{http_code}' https://ai.51easyai.com/api/v1/healthz | - grep -qx 503 +maintenance_code= +for _ in $(seq 1 15); do + maintenance_code=$(curl -ksS --resolve ai.51easyai.com:443:127.0.0.1 \ + -o /dev/null -w '%{http_code}' https://ai.51easyai.com/api/v1/healthz || true) + [[ $maintenance_code == 503 ]] && break + sleep 1 +done +[[ $maintenance_code == 503 ]] || { + echo "maintenance NGINX did not become active: HTTP $maintenance_code" >&2 + exit 1 +} docker stop --time 60 easyai-ai-gateway-api-1 >/dev/null postgres_user=$(docker inspect easyai-ai-gateway-postgres-1 \ @@ -159,7 +164,7 @@ snapshot_sql=$backup_directory/acceptance-snapshot.sql cat >"$snapshot_sql" <<'SQL' \pset tuples_only on \pset format unaligned -SELECT 'server_version=' || current_setting('server_version'); +SELECT 'server_version_num=' || current_setting('server_version_num'); SELECT 'extensions=' || COALESCE(string_agg(extname || ':' || extversion, ',' ORDER BY extname), '') FROM pg_extension WHERE extname <> 'plpgsql'; SELECT 'tables=' || count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace @@ -187,7 +192,7 @@ SELECT 'gateway_wallet_transactions=' || count(*) FROM gateway_wallet_transactio SELECT 'gateway_upload_assets=' || count(*) FROM gateway_upload_assets; SELECT 'gateway_users=' || count(*) FROM gateway_users; SELECT 'gateway_api_keys=' || count(*) FROM gateway_api_keys; -SELECT 'gateway_providers=' || count(*) FROM gateway_providers; +SELECT 'model_catalog_providers=' || count(*) FROM model_catalog_providers; SELECT 'task_sample=' || md5(COALESCE(string_agg(id::text, ',' ORDER BY id), '')) FROM (SELECT id FROM gateway_tasks ORDER BY id LIMIT 100) sample; SQL @@ -310,11 +315,8 @@ k3s kubectl scale deployment -n easyai \ easyai-api-ningbo easyai-api-hongkong easyai-web-ningbo easyai-web-hongkong \ --replicas=0 >/dev/null 2>&1 || true docker start easyai-ai-gateway-api-1 easyai-ai-gateway-web-1 >/dev/null -rm -f -- /etc/nginx/sites-enabled/ai.51easyai.com install -m 0644 "$backup_directory/nginx-site.conf" \ - /etc/nginx/sites-available/ai.51easyai.com-pre-ha -ln -s /etc/nginx/sites-available/ai.51easyai.com-pre-ha \ - /etc/nginx/sites-enabled/ai.51easyai.com + /etc/nginx/conf.d/easyai-ai-gateway-ai.51easyai.com.conf nginx -t systemctl reload nginx REMOTE @@ -327,13 +329,14 @@ echo '[cutover] starting both application sites with River workers frozen' AI_GATEWAY_ACTIVATE_WORKERS=false \ "$script_dir/bootstrap-platform.sh" activate "$release_manifest" -for site_ip in 10.77.0.1 10.77.0.2; do +for site_endpoint in 10.77.0.1:31088:31178 10.77.0.2:31089:31179; do + IFS=: read -r site_ip api_port web_port <<<"$site_endpoint" cluster_ssh "$CLUSTER_NINGBO_HOST" \ - "curl -fsS --max-time 10 http://$site_ip:30088/api/v1/healthz >/dev/null" + "curl -fsS --max-time 10 http://$site_ip:$api_port/api/v1/healthz >/dev/null" cluster_ssh "$CLUSTER_NINGBO_HOST" \ - "curl -fsS --max-time 10 http://$site_ip:30088/api/v1/readyz | grep -q '\"ok\":true'" + "curl -fsS --max-time 10 http://$site_ip:$api_port/api/v1/readyz | grep -q '\"ok\":true'" cluster_ssh "$CLUSTER_NINGBO_HOST" \ - "curl -fsS --max-time 10 http://$site_ip:30178/ | grep -q 'EasyAI AI Gateway'" + "curl -fsS --max-time 10 http://$site_ip:$web_port/ | grep -q 'EasyAI AI Gateway'" done "$script_dir/install-nginx-edges.sh" activate diff --git a/scripts/cluster/run-cross-node-file-e2e.sh b/scripts/cluster/run-cross-node-file-e2e.sh index 312be0c..0d7f91d 100755 --- a/scripts/cluster/run-cross-node-file-e2e.sh +++ b/scripts/cluster/run-cross-node-file-e2e.sh @@ -31,19 +31,42 @@ k3s kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=30 k3s kubectl set env deployment/easyai-api-ningbo -n easyai \ AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=false >/dev/null k3s kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s -curl -fsS http://10.77.0.1:30088/api/v1/readyz | grep -q '"ok":true' -curl -fsS http://10.77.0.2:30088/api/v1/readyz | grep -q '"ok":true' +for _ in $(seq 1 60); do + pods=$(k3s kubectl get pods -n easyai \ + -l app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + pod_count=$(wc -w <<<"$pods" | tr -d ' ') + if [[ $pod_count == 1 ]]; then + pod=$pods + worker_enabled=$(k3s kubectl get pod "$pod" -n easyai \ + -o jsonpath='{.spec.containers[0].env[?(@.name=="AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED")].value}') + pod_ready=$(k3s kubectl get pod "$pod" -n easyai \ + -o jsonpath='{.status.containerStatuses[0].ready}') + if [[ $worker_enabled == false && $pod_ready == true ]]; then + break + fi + fi + sleep 1 +done +[[ ${pod_count:-0} == 1 && ${worker_enabled:-} == false && ${pod_ready:-} == true ]] +k3s kubectl logs "$pod" -n easyai | grep -q 'asynchronous queue worker disabled for this process' +curl -fsS http://10.77.0.1:31088/api/v1/readyz | grep -q '"ok":true' +curl -fsS http://10.77.0.2:31089/api/v1/readyz | grep -q '"ok":true' REMOTE credentials_file=$(mktemp "$cluster_root/.local-secrets/cluster/e2e.XXXXXX") trap 'rm -f -- "$credentials_file"; restore_workers' EXIT HUP INT TERM chmod 0600 "$credentials_file" -printf 'E2E_ACCOUNT_B64=%s\n' "$(printf '%s' "$AI_GATEWAY_ONLINE_ACCOUNT" | base64 | tr -d '\n')" \ - >"$credentials_file" -printf 'E2E_PASSWORD_B64=%s\n' "$(printf '%s' "$AI_GATEWAY_ONLINE_PASSWORD" | base64 | tr -d '\n')" \ - >>"$credentials_file" -printf 'AI_GATEWAY_E2E_IMAGE_MODEL=%s\n' "${AI_GATEWAY_E2E_IMAGE_MODEL:-openai:gpt-image-1}" \ - >>"$credentials_file" +{ + printf 'E2E_ACCOUNT_B64=%s\n' \ + "$(printf '%s' "$AI_GATEWAY_ONLINE_ACCOUNT" | base64 | tr -d '\n')" + printf 'E2E_PASSWORD_B64=%s\n' \ + "$(printf '%s' "$AI_GATEWAY_ONLINE_PASSWORD" | base64 | tr -d '\n')" + printf 'AI_GATEWAY_E2E_IMAGE_MODEL=%s\n' \ + "${AI_GATEWAY_E2E_IMAGE_MODEL:-openai:gpt-image-1}" + printf 'AI_GATEWAY_E2E_IMAGE_SIZE=%s\n' \ + "${AI_GATEWAY_E2E_IMAGE_SIZE:-2048x2048}" +} >"$credentials_file" cluster_scp "$credentials_file" "$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-e2e.env" cluster_scp "$script_dir/cross-node-file-e2e.mjs" \ "$CLUSTER_NINGBO_HOST:/root/easyai-cross-node-file-e2e.mjs" diff --git a/scripts/cluster/verify-cluster.sh b/scripts/cluster/verify-cluster.sh index ef327da..3f75306 100755 --- a/scripts/cluster/verify-cluster.sh +++ b/scripts/cluster/verify-cluster.sh @@ -115,14 +115,18 @@ REMOTE echo 'application_acceptance=PASS api=2 web=2 digests=pinned' for edge_ip in "${CLUSTER_NINGBO_HOST#*@}" "${CLUSTER_HONGKONG_HOST#*@}"; do - curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \ - https://ai.51easyai.com/api/v1/healthz | grep -q easyai-ai-gateway - curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \ - https://ai.51easyai.com/api/v1/readyz | grep -q '"ok":true' - curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \ - https://ai.51easyai.com/api/v1/openapi.json | grep -q '"/api/v1/chat/completions"' - curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \ - https://ai.51easyai.com/ | grep -q 'EasyAI AI Gateway' + health_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \ + https://ai.51easyai.com/api/v1/healthz) + grep -q easyai-ai-gateway <<<"$health_body" + ready_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \ + https://ai.51easyai.com/api/v1/readyz) + grep -q '"ok":true' <<<"$ready_body" + openapi_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \ + https://ai.51easyai.com/api/v1/openapi.json) + grep -q '"/api/v1/chat/completions"' <<<"$openapi_body" + web_body=$(curl -fsS --max-time 10 --resolve "ai.51easyai.com:443:$edge_ip" \ + https://ai.51easyai.com/) + grep -q 'EasyAI AI Gateway' <<<"$web_body" done echo 'nginx_edges=PASS ningbo=true hongkong=true dns_changed=false' fi