feat(acceptance): 建立同构验收与弹性容量体系

实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护，并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
2026-07-31 18:02:24 +08:00
parent 93d36d0e55
commit e05922b0f4
94 changed files with 12379 additions and 826 deletions
@@ -43,4 +43,21 @@ data:
  AI_GATEWAY_MEDIA_OSS_OBJECT_PREFIX: easyai-ai-gateway/production/media
  AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED: "true"
  AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT: "48"
+  AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT: "48"
  AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS: "5"
+  AI_GATEWAY_WORKER_AUTOSCALING_ENABLED: "false"
+  AI_GATEWAY_WORKER_REPLICAS_NINGBO: "1"
+  AI_GATEWAY_WORKER_REPLICAS_HONGKONG: "1"
+  AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO: "1"
+  AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG: "1"
+  AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO: "1"
+  AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG: "1"
+  AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA: "48"
+  AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS: "20"
+  AI_GATEWAY_WORKER_SCALE_DOWN_STABILIZATION_SECONDS: "600"
+  AI_GATEWAY_WORKER_DRAIN_TIMEOUT_SECONDS: "600"
+  AI_GATEWAY_NODE_MEMORY_TARGET_PERCENT: "75"
+  AI_GATEWAY_NODE_MEMORY_HARD_PERCENT: "85"
+  AI_GATEWAY_NODE_CPU_TARGET_PERCENT: "70"
+  AI_GATEWAY_POSTGRES_CONNECTION_BUDGET: "150"
+  AI_GATEWAY_WORKER_DATABASE_MAX_CONNS: "32"
@@ -53,7 +53,11 @@ spec:
            - name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
              value: "false"
            - name: AI_GATEWAY_DATABASE_MAX_CONNS
-              value: "64"
+              value: "32"
+            - name: AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS
+              value: "4"
+            - name: AI_GATEWAY_DATABASE_RIVER_MAX_CONNS
+              value: "4"
            - name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
              value: "4"
            - name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
@@ -143,6 +147,128 @@ spec:
 ---
 apiVersion: apps/v1
 kind: Deployment
+metadata:
+  name: easyai-capacity-controller
+  labels:
+    app.kubernetes.io/name: easyai-capacity-controller
+    app.kubernetes.io/component: capacity-controller
+    app.kubernetes.io/part-of: easyai-ai-gateway
+spec:
+  replicas: 2
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: easyai-capacity-controller
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: easyai-capacity-controller
+        app.kubernetes.io/component: capacity-controller
+        app.kubernetes.io/part-of: easyai-ai-gateway
+    spec:
+      serviceAccountName: easyai-capacity-controller
+      terminationGracePeriodSeconds: 30
+      nodeSelector:
+        easyai.io/workload: "true"
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: easyai.io/site
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: easyai-capacity-controller
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 10001
+        runAsGroup: 10001
+        fsGroup: 10001
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: capacity-controller
+          image: easyai-api
+          imagePullPolicy: IfNotPresent
+          envFrom:
+            - configMapRef:
+                name: easyai-ai-gateway-config
+          env:
+            - name: AI_GATEWAY_PROCESS_ROLE
+              value: capacity-controller
+            - name: AI_GATEWAY_DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: easyai-ai-gateway-runtime
+                  key: AI_GATEWAY_DATABASE_URL
+            - name: AI_GATEWAY_DATABASE_MAX_CONNS
+              value: "4"
+            - name: AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS
+              value: "0"
+            - name: AI_GATEWAY_DATABASE_RIVER_MAX_CONNS
+              value: "0"
+            - name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
+              value: "1"
+            - name: AI_GATEWAY_WORKER_DATABASE_MAX_CONNS
+              value: "32"
+            - name: AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET
+              value: "72"
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          ports:
+            - name: health
+              containerPort: 8088
+          startupProbe:
+            httpGet:
+              path: /healthz
+              port: health
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 24
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: health
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: health
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 6
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 200m
+              memory: 256Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: temporary-files
+              mountPath: /tmp
+      volumes:
+        - name: temporary-files
+          emptyDir:
+            sizeLimit: 64Mi
+---
+apiVersion: apps/v1
+kind: Deployment
 metadata:
  name: easyai-api-hongkong
  labels:
@@ -196,7 +322,11 @@ spec:
            - name: AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED
              value: "false"
            - name: AI_GATEWAY_DATABASE_MAX_CONNS
-              value: "64"
+              value: "32"
+            - name: AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS
+              value: "4"
+            - name: AI_GATEWAY_DATABASE_RIVER_MAX_CONNS
+              value: "4"
            - name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
              value: "4"
            - name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
@@ -313,6 +443,14 @@ spec:
    spec:
      serviceAccountName: easyai-ai-gateway
      terminationGracePeriodSeconds: 90
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: easyai-worker
+              easyai.io/site: ningbo
      nodeSelector:
        easyai.io/site: ningbo
        easyai.io/workload: "true"
@@ -340,6 +478,10 @@ spec:
              value: "true"
            - name: AI_GATEWAY_DATABASE_MAX_CONNS
              value: "32"
+            - name: AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS
+              value: "4"
+            - name: AI_GATEWAY_DATABASE_RIVER_MAX_CONNS
+              value: "8"
            - name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
              value: "4"
            - name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
@@ -406,8 +548,8 @@ spec:
                command: ["/bin/sh", "-c", "sleep 10"]
          resources:
            requests:
-              cpu: 250m
-              memory: 512Mi
+              cpu: 500m
+              memory: 1536Mi
            limits:
              cpu: "2"
              memory: 2Gi
@@ -458,6 +600,14 @@ spec:
    spec:
      serviceAccountName: easyai-ai-gateway
      terminationGracePeriodSeconds: 90
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: easyai-worker
+              easyai.io/site: hongkong
      nodeSelector:
        easyai.io/site: hongkong
        easyai.io/workload: "true"
@@ -485,6 +635,10 @@ spec:
              value: "true"
            - name: AI_GATEWAY_DATABASE_MAX_CONNS
              value: "32"
+            - name: AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS
+              value: "4"
+            - name: AI_GATEWAY_DATABASE_RIVER_MAX_CONNS
+              value: "8"
            - name: AI_GATEWAY_DATABASE_MIN_IDLE_CONNS
              value: "4"
            - name: AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS
@@ -551,8 +705,8 @@ spec:
                command: ["/bin/sh", "-c", "sleep 10"]
          resources:
            requests:
-              cpu: 250m
-              memory: 512Mi
+              cpu: 500m
+              memory: 1536Mi
            limits:
              cpu: "2"
              memory: 2Gi
@@ -896,6 +1050,16 @@ spec:
 ---
 apiVersion: policy/v1
 kind: PodDisruptionBudget
+metadata:
+  name: easyai-capacity-controller
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: easyai-capacity-controller
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
 metadata:
  name: easyai-worker
 spec:
@@ -7,7 +7,7 @@ spec:
    matchExpressions:
      - key: app.kubernetes.io/name
        operator: In
-        values: [easyai-api, easyai-worker, easyai-web]
+        values: [easyai-api, easyai-worker, easyai-web, easyai-capacity-controller]
  policyTypes: [Ingress]
  ingress:
    - from:
@@ -29,3 +29,68 @@ roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: easyai-gateway-security-events
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: easyai-capacity-controller
+  labels:
+    app.kubernetes.io/name: easyai-capacity-controller
+    app.kubernetes.io/part-of: easyai-ai-gateway
+automountServiceAccountToken: true
+imagePullSecrets:
+  - name: easyai-registry
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: easyai-capacity-controller
+rules:
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"]
+    verbs: ["get"]
+  - apiGroups: ["apps"]
+    resources: ["deployments/scale"]
+    resourceNames: ["easyai-worker-ningbo", "easyai-worker-hongkong"]
+    verbs: ["get", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: easyai-capacity-controller
+subjects:
+  - kind: ServiceAccount
+    name: easyai-capacity-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: easyai-capacity-controller
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: easyai-capacity-controller
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list"]
+  - apiGroups: ["metrics.k8s.io"]
+    resources: ["nodes", "pods"]
+    verbs: ["get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: easyai-capacity-controller
+subjects:
+  - kind: ServiceAccount
+    name: easyai-capacity-controller
+    namespace: easyai
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: easyai-capacity-controller