feat(queue): 增加非文本模型分布式准入队列

使用 PostgreSQL 统一同步与异步非文本任务的并发准入、持久化等待和 Worker 容量分配，并将生产 API 与独立 Worker 角色拆分。补充策略管理、共享契约、OpenAPI、Kubernetes 双节点 Worker 清单及跨节点验收脚本；未默认启用任何生产 queue_size 策略。已在原基线完成 Go、前端、迁移、Shell、Kustomize 与长任务容量验收；合入最新主干后将重新执行发布门禁。
2026-07-29 16:15:43 +08:00
parent 2e5a90731b
commit 9e4fc7362d
55 changed files with 4565 additions and 237 deletions
@@ -84,6 +84,16 @@ verify_site() {
  wait_for_url "$site Web" "http://$address:$web_port/" 'EasyAI AI Gateway'
 }

+rollout_worker_site() {
+  local site=$1
+  local api_image=$2
+  "${kubectl[@]}" set image "deployment/easyai-worker-$site" -n "$NAMESPACE" \
+    "worker=$api_image"
+  "${kubectl[@]}" rollout status "deployment/easyai-worker-$site" -n "$NAMESPACE" --timeout=300s
+  [[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
+    -o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == worker ]]
+}
+
 rollout_site() {
  local site=$1
  local api_image=$2
@@ -236,11 +246,20 @@ activate_manifest() {
    run_migrator "$source_sha" "$api_image"
  fi

-  if ! rollout_site hongkong "$api_image" "$web_image" "$api_changed" "$web_changed" ||
-      ! rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" ||
-      ! wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then
+  if { [[ $api_changed != true ]] ||
+       { rollout_worker_site hongkong "$api_image" &&
+         rollout_worker_site ningbo "$api_image"; }; } &&
+     rollout_site hongkong "$api_image" "$web_image" "$api_changed" "$web_changed" &&
+     rollout_site ningbo "$api_image" "$web_image" "$api_changed" "$web_changed" &&
+     wait_for_url 'public readiness' "$PUBLIC_BASE_URL/api/v1/readyz" '"ok":true'; then
+    :
+  else
    if [[ -n $previous_api && -n $previous_web ]]; then
      echo '[cluster-release] restoring previous application images' >&2
+      if [[ $api_changed == true ]]; then
+        rollout_worker_site hongkong "$previous_api" || true
+        rollout_worker_site ningbo "$previous_api" || true
+      fi
      rollout_site hongkong "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true
      rollout_site ningbo "$previous_api" "$previous_web" "$api_changed" "$web_changed" || true
    fi
@@ -266,6 +285,10 @@ case ${1:-} in
    node "$manifest_tool" validate "$2" >/dev/null
    api_image=$(manifest_get "$2" images.api)
    web_image=$(manifest_get "$2" images.web)
+    for site in ningbo hongkong; do
+      [[ $("${kubectl[@]}" get deployment "easyai-worker-$site" -n "$NAMESPACE" \
+        -o jsonpath='{.spec.template.spec.containers[?(@.name=="worker")].image}') == "$api_image" ]]
+    done
    [[ $("${kubectl[@]}" get deployment easyai-api-ningbo -n "$NAMESPACE" \
      -o jsonpath='{.spec.template.spec.containers[?(@.name=="api")].image}') == "$api_image" ]]
    [[ $("${kubectl[@]}" get deployment easyai-web-ningbo -n "$NAMESPACE" \
@@ -48,6 +48,12 @@ spec:
            - secretRef:
                name: easyai-ai-gateway-runtime
          env:
+            - name: AI_GATEWAY_PROCESS_ROLE
+              value: api
+            - name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
+              value: "false"
+            - name: AI_GATEWAY_DATABASE_MAX_CONNS
+              value: "16"
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
@@ -56,6 +62,12 @@ spec:
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
+            - name: EASYAI_SITE
+              value: ningbo
          ports:
            - name: http
              containerPort: 8088
@@ -154,6 +166,12 @@ spec:
            - secretRef:
                name: easyai-ai-gateway-runtime
          env:
+            - name: AI_GATEWAY_PROCESS_ROLE
+              value: api
+            - name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
+              value: "false"
+            - name: AI_GATEWAY_DATABASE_MAX_CONNS
+              value: "16"
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
@@ -162,6 +180,12 @@ spec:
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
+            - name: EASYAI_SITE
+              value: hongkong
          ports:
            - name: http
              containerPort: 8088
@@ -212,6 +236,242 @@ spec:
 ---
 apiVersion: apps/v1
 kind: Deployment
+metadata:
+  name: easyai-worker-ningbo
+  labels:
+    app.kubernetes.io/name: easyai-worker
+    app.kubernetes.io/component: worker
+    app.kubernetes.io/part-of: easyai-ai-gateway
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: easyai-worker
+      easyai.io/site: ningbo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: easyai-worker
+        app.kubernetes.io/component: worker
+        app.kubernetes.io/part-of: easyai-ai-gateway
+        easyai.io/site: ningbo
+    spec:
+      serviceAccountName: easyai-ai-gateway
+      terminationGracePeriodSeconds: 90
+      nodeSelector:
+        easyai.io/site: ningbo
+        easyai.io/workload: "true"
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 10001
+        runAsGroup: 10001
+        fsGroup: 10001
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: worker
+          image: easyai-api
+          imagePullPolicy: IfNotPresent
+          envFrom:
+            - configMapRef:
+                name: easyai-ai-gateway-config
+            - secretRef:
+                name: easyai-ai-gateway-runtime
+          env:
+            - name: AI_GATEWAY_PROCESS_ROLE
+              value: worker
+            - name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
+              value: "true"
+            - name: AI_GATEWAY_DATABASE_MAX_CONNS
+              value: "24"
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
+            - name: EASYAI_SITE
+              value: ningbo
+          ports:
+            - name: health
+              containerPort: 8088
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: health
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: health
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 6
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "sleep 10"]
+          resources:
+            requests:
+              cpu: 250m
+              memory: 512Mi
+            limits:
+              cpu: "2"
+              memory: 2Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: application-data
+              mountPath: /app/data
+            - name: temporary-files
+              mountPath: /tmp
+      volumes:
+        - name: application-data
+          emptyDir:
+            sizeLimit: 5Gi
+        - name: temporary-files
+          emptyDir:
+            sizeLimit: 1Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: easyai-worker-hongkong
+  labels:
+    app.kubernetes.io/name: easyai-worker
+    app.kubernetes.io/component: worker
+    app.kubernetes.io/part-of: easyai-ai-gateway
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: easyai-worker
+      easyai.io/site: hongkong
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: easyai-worker
+        app.kubernetes.io/component: worker
+        app.kubernetes.io/part-of: easyai-ai-gateway
+        easyai.io/site: hongkong
+    spec:
+      serviceAccountName: easyai-ai-gateway
+      terminationGracePeriodSeconds: 90
+      nodeSelector:
+        easyai.io/site: hongkong
+        easyai.io/workload: "true"
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 10001
+        runAsGroup: 10001
+        fsGroup: 10001
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: worker
+          image: easyai-api
+          imagePullPolicy: IfNotPresent
+          envFrom:
+            - configMapRef:
+                name: easyai-ai-gateway-config
+            - secretRef:
+                name: easyai-ai-gateway-runtime
+          env:
+            - name: AI_GATEWAY_PROCESS_ROLE
+              value: worker
+            - name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
+              value: "true"
+            - name: AI_GATEWAY_DATABASE_MAX_CONNS
+              value: "24"
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
+            - name: EASYAI_SITE
+              value: hongkong
+          ports:
+            - name: health
+              containerPort: 8088
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: health
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: health
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 6
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "sleep 10"]
+          resources:
+            requests:
+              cpu: 250m
+              memory: 512Mi
+            limits:
+              cpu: "2"
+              memory: 2Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: application-data
+              mountPath: /app/data
+            - name: temporary-files
+              mountPath: /tmp
+      volumes:
+        - name: application-data
+          emptyDir:
+            sizeLimit: 5Gi
+        - name: temporary-files
+          emptyDir:
+            sizeLimit: 1Gi
+---
+apiVersion: apps/v1
+kind: Deployment
 metadata:
  name: easyai-web-ningbo
  labels:
@@ -529,3 +789,13 @@ spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: easyai-web
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: easyai-worker
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: easyai-worker
@@ -7,7 +7,7 @@ spec:
    matchExpressions:
      - key: app.kubernetes.io/name
        operator: In
-        values: [easyai-api, easyai-web]
+        values: [easyai-api, easyai-worker, easyai-web]
  policyTypes: [Ingress]
  ingress:
    - from: